tests: launch oss/azure containers from fixtures (#4178)

* tests: launch oss/azure containers from fixtures

Related to #4054

* ci: hdfs: remove unneeded fix

We are using pyarrow 0.17.0 already.

Author: efiop

scripts/ci/before_install.sh

@@ -32,8 +32,6 @@ if [[ "$TRAVIS_BUILD_STAGE_NAME" == "test" ]]; then
   fi
 
   if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-    bash "$scriptdir/install_azurite.sh"
-    bash "$scriptdir/install_oss.sh"
     bash "$scriptdir/install_hadoop.sh"
   fi
 

scripts/ci/install_azurite.sh

@@ -24,13 +24,6 @@ echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >>env.sh
 echo "export YARN_HOME=/usr/local/hadoop" >>env.sh
 echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >>env.sh
 
-# PyArrow==0.16.0 regression https://issues.apache.org/jira/browse/ARROW-7841
-# retrieves native library from $HADOOP_HOME directory instead of
-# `$HADOOP_HOME/lib/native`.
-# Fix: force search for `libhdfs.so` inside `$HADOOP_HOME/lib/native`.
-# Note: not needed for PyArrow==0.17.0.
-echo "export ARROW_LIBHDFS_DIR=/usr/local/hadoop/lib/native" >> env.sh
-
 echo "export JAVA_HOME=/usr/" >>env.sh
 echo "export PATH=\$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >>env.sh
 

scripts/ci/install_hadoop.sh

@@ -103,6 +103,7 @@ def run(self):
     "wheel>=0.31.1",
     # Test requirements:
     "pytest>=4.6.0",
+    "pytest-docker>=0.7.2",
     "pytest-timeout>=1.3.3",
     "pytest-cov>=2.6.1",
     "pytest-xdist>=1.26.1",

scripts/ci/install_oss.sh

@@ -0,0 +1,11 @@
+---
+version: '3.2'
+services:
+  azurite:
+    image: mcr.microsoft.com/azure-storage/azurite:3.3.0-preview
+    ports:
+      - "10000"
+  oss:
+    image: rkuprieiev/oss-emulator
+    ports:
+      - "8880"

setup.py

@@ -1,10 +1,12 @@
+import subprocess
+
 import pytest
 
-from .azure import Azure, azure  # noqa: F401
+from .azure import Azure, azure, azure_server  # noqa: F401
 from .hdfs import HDFS, hdfs  # noqa: F401
 from .http import HTTP, http, http_server  # noqa: F401
 from .local import Local, local_cloud, local_remote  # noqa: F401
-from .oss import OSS, TEST_OSS_REPO_BUCKET, oss  # noqa: F401
+from .oss import OSS, TEST_OSS_REPO_BUCKET, oss, oss_server  # noqa: F401
 from .s3 import S3, TEST_AWS_REPO_BUCKET, real_s3, s3  # noqa: F401
 
 TEST_REMOTE = "upstream"
@@ -35,6 +37,14 @@
 )
 
 
+@pytest.fixture(scope="session")
+def docker_compose():
+    try:
+        subprocess.check_output("docker-compose version", shell=True)
+    except (subprocess.CalledProcessError, OSError):
+        pytest.skip("no docker-compose installed")
+
+
 @pytest.fixture
 def remote(tmp_dir, dvc, request):
     cloud = request.param

tests/docker-compose.yml

@@ -1,36 +1,61 @@
 # pylint:disable=abstract-method
 
-import os
 import uuid
 
 import pytest
 
 from dvc.path_info import CloudURLInfo
-from dvc.utils import env2bool
 
 from .base import Base
 
+TEST_AZURE_CONTAINER = "tests"
+TEST_AZURE_CONNECTION_STRING = (
+    "DefaultEndpointsProtocol=http;"
+    "AccountName=devstoreaccount1;"
+    "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSR"
+    "Z6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;"
+    "BlobEndpoint=http://127.0.0.1:{port}/devstoreaccount1;"
+)
+
 
 class Azure(Base, CloudURLInfo):
-    @staticmethod
-    def should_test():
-        do_test = env2bool("DVC_TEST_AZURE", undefined=None)
-        if do_test is not None:
-            return do_test
+    pass
+
+
+@pytest.fixture(scope="session")
+def azure_server(docker_compose, docker_services):
+    from azure.storage.blob import (  # pylint: disable=no-name-in-module
+        BlockBlobService,
+    )
+    from azure.common import (  # pylint: disable=no-name-in-module
+        AzureException,
+    )
+
+    port = docker_services.port_for("azurite", 10000)
+    connection_string = TEST_AZURE_CONNECTION_STRING.format(port=port)
+
+    def _check():
+        try:
+            BlockBlobService(
+                connection_string=connection_string,
+            ).list_containers()
+            return True
+        except AzureException:
+            return False
 
-        return os.getenv("AZURE_STORAGE_CONTAINER_NAME") and os.getenv(
-            "AZURE_STORAGE_CONNECTION_STRING"
-        )
+    docker_services.wait_until_responsive(
+        timeout=60.0, pause=0.1, check=_check
+    )
 
-    @staticmethod
-    def get_url():
-        container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
-        assert container_name is not None
-        return "azure://{}/{}".format(container_name, str(uuid.uuid4()))
+    return connection_string
 
 
 @pytest.fixture
-def azure():
-    if not Azure.should_test():
-        pytest.skip("no azure running")
-    yield Azure(Azure.get_url())
+def azure(azure_server):
+    url = f"azure://{TEST_AZURE_CONTAINER}/{uuid.uuid4()}"
+    ret = Azure(url)
+    ret.config = {
+        "url": url,
+        "connection_string": azure_server,
+    }
+    return ret

tests/remotes/__init__.py

@@ -1,41 +1,54 @@
 # pylint:disable=abstract-method
-import os
 import uuid
 
 import pytest
 
 from dvc.path_info import CloudURLInfo
-from dvc.utils import env2bool
 
 from .base import Base
 
 TEST_OSS_REPO_BUCKET = "dvc-test"
+TEST_OSS_ENDPOINT = "127.0.0.1:{port}"
+TEST_OSS_ACCESS_KEY_ID = "AccessKeyID"
+TEST_OSS_ACCESS_KEY_SECRET = "AccessKeySecret"
 
 
 class OSS(Base, CloudURLInfo):
-    @staticmethod
-    def should_test():
-        do_test = env2bool("DVC_TEST_OSS", undefined=None)
-        if do_test is not None:
-            return do_test
+    pass
 
-        return (
-            os.getenv("OSS_ENDPOINT")
-            and os.getenv("OSS_ACCESS_KEY_ID")
-            and os.getenv("OSS_ACCESS_KEY_SECRET")
-        )
 
-    @staticmethod
-    def _get_storagepath():
-        return f"{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}"
+@pytest.fixture(scope="session")
+def oss_server(docker_compose, docker_services):
+    import oss2
 
-    @staticmethod
-    def get_url():
-        return f"oss://{OSS._get_storagepath()}"
+    port = docker_services.port_for("oss", 8880)
+    endpoint = TEST_OSS_ENDPOINT.format(port=port)
+
+    def _check():
+        try:
+            auth = oss2.Auth(
+                TEST_OSS_ACCESS_KEY_ID, TEST_OSS_ACCESS_KEY_SECRET
+            )
+            oss2.Bucket(auth, endpoint, "mybucket").get_bucket_info()
+            return True
+        except oss2.exceptions.NoSuchBucket:
+            return True
+        except oss2.exceptions.OssError:
+            return False
+
+    docker_services.wait_until_responsive(timeout=30.0, pause=5, check=_check)
+
+    return endpoint
 
 
 @pytest.fixture
-def oss():
-    if not OSS.should_test():
-        pytest.skip("no oss running")
-    yield OSS(OSS.get_url())
+def oss(oss_server):
+    url = f"oss://{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}"
+    ret = OSS(url)
+    ret.config = {
+        "url": url,
+        "oss_key_id": TEST_OSS_ACCESS_KEY_ID,
+        "oss_key_secret": TEST_OSS_ACCESS_KEY_SECRET,
+        "oss_endpoint": oss_server,
+    }
+    return ret

tests/remotes/azure.py

@@ -1,8 +1,5 @@
-import pytest
-
 from dvc.path_info import PathInfo
 from dvc.remote.azure import AzureRemoteTree
-from tests.remotes import Azure
 
 container_name = "container-name"
 connection_string = (
@@ -32,14 +29,11 @@ def test_init(dvc):
     assert tree._conn_kwargs["connection_string"] == connection_string
 
 
-def test_get_file_hash(tmp_dir):
-    if not Azure.should_test():
-        pytest.skip("no azurite running")
-
+def test_get_file_hash(tmp_dir, azure):
     tmp_dir.gen("foo", "foo")
 
-    tree = AzureRemoteTree(None, {})
-    to_info = tree.PATH_CLS(Azure.get_url())
+    tree = AzureRemoteTree(None, azure.config)
+    to_info = azure
     tree.upload(PathInfo("foo"), to_info)
     assert tree.exists(to_info)
     hash_ = tree.get_file_hash(to_info)