From 734f7cdc6c9602582b679494dd9a5e0a7ef1406f Mon Sep 17 00:00:00 2001 From: Nikita Kodenko Date: Sun, 26 Apr 2020 21:37:38 +0700 Subject: [PATCH] refactor: dvc/remotes name unification Partially fixes #2089 --- dvc/data_cloud.py | 6 +- dvc/dependency/http.py | 4 +- dvc/dependency/https.py | 4 +- dvc/output/__init__.py | 12 ++-- dvc/output/base.py | 4 +- dvc/output/gs.py | 4 +- dvc/output/hdfs.py | 4 +- dvc/output/local.py | 4 +- dvc/output/s3.py | 4 +- dvc/output/ssh.py | 4 +- dvc/remote/__init__.py | 42 +++++++------- dvc/remote/azure.py | 4 +- dvc/remote/base.py | 2 +- dvc/remote/gdrive.py | 12 ++-- dvc/remote/gs.py | 4 +- dvc/remote/hdfs.py | 4 +- dvc/remote/http.py | 4 +- dvc/remote/https.py | 4 +- dvc/remote/local.py | 4 +- dvc/remote/oss.py | 4 +- dvc/remote/s3.py | 4 +- dvc/remote/ssh/__init__.py | 6 +- dvc/stage/__init__.py | 8 +-- tests/func/remote/test_index.py | 16 +++--- tests/func/remote/test_local.py | 12 ++-- tests/func/test_add.py | 8 +-- tests/func/test_checkout.py | 12 ++-- tests/func/test_data_cloud.py | 84 ++++++++++++++-------------- tests/func/test_external_repo.py | 4 +- tests/func/test_gc.py | 6 +- tests/func/test_ignore.py | 6 +- tests/func/test_import.py | 2 +- tests/func/test_remote.py | 18 +++--- tests/func/test_repro.py | 10 ++-- tests/func/test_s3.py | 16 +++--- tests/func/test_stage.py | 6 +- tests/remotes.py | 12 ++-- tests/unit/output/test_local.py | 4 +- tests/unit/remote/ssh/test_ssh.py | 42 +++++++------- tests/unit/remote/test_azure.py | 6 +- tests/unit/remote/test_base.py | 18 +++--- tests/unit/remote/test_gdrive.py | 14 ++--- tests/unit/remote/test_gs.py | 8 +-- tests/unit/remote/test_http.py | 12 ++-- tests/unit/remote/test_local.py | 10 ++-- tests/unit/remote/test_oss.py | 4 +- tests/unit/remote/test_remote.py | 4 +- tests/unit/remote/test_remote_dir.py | 8 +-- tests/unit/remote/test_s3.py | 8 +-- 49 files changed, 251 insertions(+), 251 deletions(-) diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 6fe2683ee2..91c234397f 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -54,7 +54,7 @@ def push(self, cache, jobs=None, remote=None, show_checksums=False): Args: cache (NamedCache): named checksums to push to the cloud. jobs (int): number of jobs that can be running simultaneously. - remote (dvc.remote.base.RemoteBASE): optional remote to push to. + remote (dvc.remote.base.BaseRemote): optional remote to push to. By default remote from core.remote config option is used. show_checksums (bool): show checksums instead of file names in information messages. @@ -72,7 +72,7 @@ def pull(self, cache, jobs=None, remote=None, show_checksums=False): Args: cache (NamedCache): named checksums to pull from the cloud. jobs (int): number of jobs that can be running simultaneously. - remote (dvc.remote.base.RemoteBASE): optional remote to pull from. + remote (dvc.remote.base.BaseRemote): optional remote to pull from. By default remote from core.remote config option is used. show_checksums (bool): show checksums instead of file names in information messages. @@ -103,7 +103,7 @@ def status(self, cache, jobs=None, remote=None, show_checksums=False): Args: cache (NamedCache): named checksums to check status for. jobs (int): number of jobs that can be running simultaneously. - remote (dvc.remote.base.RemoteBASE): optional remote to compare + remote (dvc.remote.base.BaseRemote): optional remote to compare cache to. By default remote from core.remote config option is used. show_checksums (bool): show checksums instead of file names in diff --git a/dvc/dependency/http.py b/dvc/dependency/http.py index 9f5e26cc95..07a5207b0f 100644 --- a/dvc/dependency/http.py +++ b/dvc/dependency/http.py @@ -1,7 +1,7 @@ from dvc.dependency.base import DependencyBase from dvc.output.base import OutputBase -from dvc.remote.http import RemoteHTTP +from dvc.remote.http import HTTPRemote class DependencyHTTP(DependencyBase, OutputBase): - REMOTE = RemoteHTTP + REMOTE = HTTPRemote diff --git a/dvc/dependency/https.py b/dvc/dependency/https.py index e23967f39a..878df017ce 100644 --- a/dvc/dependency/https.py +++ b/dvc/dependency/https.py @@ -1,6 +1,6 @@ from .http import DependencyHTTP -from dvc.remote.https import RemoteHTTPS +from dvc.remote.https import HTTPSRemote class DependencyHTTPS(DependencyHTTP): - REMOTE = RemoteHTTPS + REMOTE = HTTPSRemote diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py index e86deeba56..fab6e5bdd8 100644 --- a/dvc/output/__init__.py +++ b/dvc/output/__init__.py @@ -8,9 +8,9 @@ from dvc.output.s3 import OutputS3 from dvc.output.ssh import OutputSSH from dvc.remote import Remote -from dvc.remote.hdfs import RemoteHDFS -from dvc.remote.local import RemoteLOCAL -from dvc.remote.s3 import RemoteS3 +from dvc.remote.hdfs import HDFSRemote +from dvc.remote.local import LocalRemote +from dvc.remote.s3 import S3Remote from dvc.scheme import Schemes OUTS = [ @@ -44,9 +44,9 @@ # so when a few types of outputs share the same name, we only need # specify it once. CHECKSUMS_SCHEMA = { - RemoteLOCAL.PARAM_CHECKSUM: CHECKSUM_SCHEMA, - RemoteS3.PARAM_CHECKSUM: CHECKSUM_SCHEMA, - RemoteHDFS.PARAM_CHECKSUM: CHECKSUM_SCHEMA, + LocalRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA, + S3Remote.PARAM_CHECKSUM: CHECKSUM_SCHEMA, + HDFSRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA, } TAGS_SCHEMA = {str: CHECKSUMS_SCHEMA} diff --git a/dvc/output/base.py b/dvc/output/base.py index a539e18e7f..8f144bae37 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -9,7 +9,7 @@ from dvc.cache import NamedCache from dvc.exceptions import CollectCacheError, RemoteCacheRequiredError from dvc.exceptions import DvcException -from dvc.remote.base import RemoteBASE +from dvc.remote.base import BaseRemote logger = logging.getLogger(__name__) @@ -41,7 +41,7 @@ def __init__(self, path): class OutputBase(object): IS_DEPENDENCY = False - REMOTE = RemoteBASE + REMOTE = BaseRemote PARAM_PATH = "path" PARAM_CACHE = "cache" diff --git a/dvc/output/gs.py b/dvc/output/gs.py index d2702bb199..924fdf5e7b 100644 --- a/dvc/output/gs.py +++ b/dvc/output/gs.py @@ -1,6 +1,6 @@ from dvc.output.s3 import OutputS3 -from dvc.remote.gs import RemoteGS +from dvc.remote.gs import GSRemote class OutputGS(OutputS3): - REMOTE = RemoteGS + REMOTE = GSRemote diff --git a/dvc/output/hdfs.py b/dvc/output/hdfs.py index c5b81f6a90..a8db0d8adc 100644 --- a/dvc/output/hdfs.py +++ b/dvc/output/hdfs.py @@ -1,6 +1,6 @@ from dvc.output.base import OutputBase -from dvc.remote.hdfs import RemoteHDFS +from dvc.remote.hdfs import HDFSRemote class OutputHDFS(OutputBase): - REMOTE = RemoteHDFS + REMOTE = HDFSRemote diff --git a/dvc/output/local.py b/dvc/output/local.py index 6e4123d1e3..1ba98f1ff7 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -5,7 +5,7 @@ from dvc.exceptions import DvcException from dvc.istextfile import istextfile from dvc.output.base import OutputBase -from dvc.remote.local import RemoteLOCAL +from dvc.remote.local import LocalRemote from dvc.utils import relpath from dvc.compat import fspath_py35 from dvc.utils.fs import path_isin @@ -15,7 +15,7 @@ class OutputLOCAL(OutputBase): - REMOTE = RemoteLOCAL + REMOTE = LocalRemote sep = os.sep def __init__(self, stage, path, *args, **kwargs): diff --git a/dvc/output/s3.py b/dvc/output/s3.py index fce3642881..9e9bf221b6 100644 --- a/dvc/output/s3.py +++ b/dvc/output/s3.py @@ -1,6 +1,6 @@ from dvc.output.base import OutputBase -from dvc.remote.s3 import RemoteS3 +from dvc.remote.s3 import S3Remote class OutputS3(OutputBase): - REMOTE = RemoteS3 + REMOTE = S3Remote diff --git a/dvc/output/ssh.py b/dvc/output/ssh.py index 8ae79e74a5..317eb1ace0 100644 --- a/dvc/output/ssh.py +++ b/dvc/output/ssh.py @@ -1,6 +1,6 @@ from dvc.output.base import OutputBase -from dvc.remote.ssh import RemoteSSH +from dvc.remote.ssh import SSHRemote class OutputSSH(OutputBase): - REMOTE = RemoteSSH + REMOTE = SSHRemote diff --git a/dvc/remote/__init__.py b/dvc/remote/__init__.py index 6c2de0a057..d356683c33 100644 --- a/dvc/remote/__init__.py +++ b/dvc/remote/__init__.py @@ -1,29 +1,29 @@ import posixpath from urllib.parse import urlparse -from dvc.remote.azure import RemoteAZURE -from dvc.remote.gdrive import RemoteGDrive -from dvc.remote.gs import RemoteGS -from dvc.remote.hdfs import RemoteHDFS -from dvc.remote.http import RemoteHTTP -from dvc.remote.https import RemoteHTTPS -from dvc.remote.local import RemoteLOCAL -from dvc.remote.oss import RemoteOSS -from dvc.remote.s3 import RemoteS3 -from dvc.remote.ssh import RemoteSSH +from dvc.remote.azure import AzureRemote +from dvc.remote.gdrive import GDriveRemote +from dvc.remote.gs import GSRemote +from dvc.remote.hdfs import HDFSRemote +from dvc.remote.http import HTTPRemote +from dvc.remote.https import HTTPSRemote +from dvc.remote.local import LocalRemote +from dvc.remote.oss import OSSRemote +from dvc.remote.s3 import S3Remote +from dvc.remote.ssh import SSHRemote REMOTES = [ - RemoteAZURE, - RemoteGDrive, - RemoteGS, - RemoteHDFS, - RemoteHTTP, - RemoteHTTPS, - RemoteS3, - RemoteSSH, - RemoteOSS, - # NOTE: RemoteLOCAL is the default + AzureRemote, + GDriveRemote, + GSRemote, + HDFSRemote, + HTTPRemote, + HTTPSRemote, + S3Remote, + SSHRemote, + OSSRemote, + # NOTE: LocalRemote is the default ] @@ -31,7 +31,7 @@ def _get(remote_conf): for remote in REMOTES: if remote.supported(remote_conf): return remote - return RemoteLOCAL + return LocalRemote def Remote(repo, **kwargs): diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index f126aaeb43..1c21a0b26d 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -10,14 +10,14 @@ from dvc.path_info import CloudURLInfo from dvc.progress import Tqdm -from dvc.remote.base import RemoteBASE +from dvc.remote.base import BaseRemote from dvc.scheme import Schemes logger = logging.getLogger(__name__) -class RemoteAZURE(RemoteBASE): +class AzureRemote(BaseRemote): scheme = Schemes.AZURE path_cls = CloudURLInfo REGEX = ( diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 811fd9fe52..c7da00e5dd 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -82,7 +82,7 @@ def wrapper(remote_obj, *args, **kwargs): return wrapper -class RemoteBASE(object): +class BaseRemote(object): scheme = "base" path_cls = URLInfo REQUIRES = {} diff --git a/dvc/remote/gdrive.py b/dvc/remote/gdrive.py index 760937412b..7472ed2fb8 100644 --- a/dvc/remote/gdrive.py +++ b/dvc/remote/gdrive.py @@ -12,7 +12,7 @@ from dvc.progress import Tqdm from dvc.scheme import Schemes from dvc.path_info import CloudURLInfo -from dvc.remote.base import RemoteBASE +from dvc.remote.base import BaseRemote from dvc.exceptions import DvcException from dvc.utils import tmp_fname, format_link @@ -94,7 +94,7 @@ def __init__(self, url): self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/")) -class RemoteGDrive(RemoteBASE): +class GDriveRemote(BaseRemote): scheme = Schemes.GDRIVE path_cls = GDriveURLInfo REQUIRES = {"pydrive2": "pydrive2"} @@ -139,7 +139,7 @@ def __init__(self, repo, config): self._validate_config() self._gdrive_user_credentials_path = ( tmp_fname(os.path.join(self.repo.tmp_dir, "")) - if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA) + if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA) else config.get( "gdrive_user_credentials_file", os.path.join( @@ -183,12 +183,12 @@ def _drive(self): from pydrive2.auth import GoogleAuth from pydrive2.drive import GoogleDrive - if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA): + if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA): with open( self._gdrive_user_credentials_path, "w" ) as credentials_file: credentials_file.write( - os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA) + os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA) ) GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings" @@ -237,7 +237,7 @@ def _drive(self): except Exception as exc: raise DvcException("Google Drive authentication failed") from exc finally: - if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA): + if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA): os.remove(self._gdrive_user_credentials_path) return GoogleDrive(gauth) diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index 78517cad96..86a062b3b2 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -11,7 +11,7 @@ from dvc.exceptions import DvcException from dvc.path_info import CloudURLInfo from dvc.progress import Tqdm -from dvc.remote.base import RemoteBASE +from dvc.remote.base import BaseRemote from dvc.scheme import Schemes logger = logging.getLogger(__name__) @@ -66,7 +66,7 @@ def _upload_to_bucket( blob.upload_from_file(wrapped) -class RemoteGS(RemoteBASE): +class GSRemote(BaseRemote): scheme = Schemes.GS path_cls = CloudURLInfo REQUIRES = {"google-cloud-storage": "google.cloud.storage"} diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index c874b20682..e1d18f67a7 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -8,7 +8,7 @@ from contextlib import closing, contextmanager from urllib.parse import urlparse -from .base import RemoteBASE, RemoteCmdError +from .base import BaseRemote, RemoteCmdError from .pool import get_connection from dvc.scheme import Schemes from dvc.utils import fix_env, tmp_fname @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) -class RemoteHDFS(RemoteBASE): +class HDFSRemote(BaseRemote): scheme = Schemes.HDFS REGEX = r"^hdfs://((?P.*)@)?.*$" PARAM_CHECKSUM = "checksum" diff --git a/dvc/remote/http.py b/dvc/remote/http.py index a40d9278d2..28f6715de6 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -8,7 +8,7 @@ import dvc.prompt as prompt from dvc.exceptions import DvcException, HTTPError from dvc.progress import Tqdm -from dvc.remote.base import RemoteBASE +from dvc.remote.base import BaseRemote from dvc.scheme import Schemes logger = logging.getLogger(__name__) @@ -23,7 +23,7 @@ def ask_password(host, user): ) -class RemoteHTTP(RemoteBASE): +class HTTPRemote(BaseRemote): scheme = Schemes.HTTP path_cls = HTTPURLInfo SESSION_RETRIES = 5 diff --git a/dvc/remote/https.py b/dvc/remote/https.py index d8602e589b..21b2513f5b 100644 --- a/dvc/remote/https.py +++ b/dvc/remote/https.py @@ -1,6 +1,6 @@ -from .http import RemoteHTTP +from .http import HTTPRemote from dvc.scheme import Schemes -class RemoteHTTPS(RemoteHTTP): +class HTTPSRemote(HTTPRemote): scheme = Schemes.HTTPS diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 9bebfe47a5..394fc084a0 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -15,7 +15,7 @@ from dvc.progress import Tqdm from dvc.remote.base import ( index_locked, - RemoteBASE, + BaseRemote, STATUS_MAP, STATUS_DELETED, STATUS_MISSING, @@ -31,7 +31,7 @@ logger = logging.getLogger(__name__) -class RemoteLOCAL(RemoteBASE): +class LocalRemote(BaseRemote): scheme = Schemes.LOCAL path_cls = PathInfo PARAM_CHECKSUM = "md5" diff --git a/dvc/remote/oss.py b/dvc/remote/oss.py index 9642fb511b..9ef6abc1b1 100644 --- a/dvc/remote/oss.py +++ b/dvc/remote/oss.py @@ -7,14 +7,14 @@ from dvc.path_info import CloudURLInfo from dvc.progress import Tqdm -from dvc.remote.base import RemoteBASE +from dvc.remote.base import BaseRemote from dvc.scheme import Schemes logger = logging.getLogger(__name__) -class RemoteOSS(RemoteBASE): +class OSSRemote(BaseRemote): """ oss2 document: https://www.alibabacloud.com/help/doc-detail/32026.htm diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index d75ca42546..e4eb3c355c 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -12,13 +12,13 @@ from dvc.exceptions import ETagMismatchError from dvc.path_info import CloudURLInfo from dvc.progress import Tqdm -from dvc.remote.base import RemoteBASE +from dvc.remote.base import BaseRemote from dvc.scheme import Schemes logger = logging.getLogger(__name__) -class RemoteS3(RemoteBASE): +class S3Remote(BaseRemote): scheme = Schemes.S3 path_cls = CloudURLInfo REQUIRES = {"boto3": "boto3"} diff --git a/dvc/remote/ssh/__init__.py b/dvc/remote/ssh/__init__.py index 7781669c4d..25887018f9 100644 --- a/dvc/remote/ssh/__init__.py +++ b/dvc/remote/ssh/__init__.py @@ -14,7 +14,7 @@ import dvc.prompt as prompt from dvc.progress import Tqdm -from dvc.remote.base import RemoteBASE +from dvc.remote.base import BaseRemote from dvc.remote.pool import get_connection from dvc.scheme import Schemes from dvc.utils import to_chunks @@ -33,7 +33,7 @@ def ask_password(host, user, port): ) -class RemoteSSH(RemoteBASE): +class SSHRemote(BaseRemote): scheme = Schemes.SSH REQUIRES = {"paramiko": "paramiko"} @@ -103,7 +103,7 @@ def ssh_config_filename(): def _load_user_ssh_config(hostname): import paramiko - user_config_file = RemoteSSH.ssh_config_filename() + user_config_file = SSHRemote.ssh_config_filename() user_ssh_config = {} if hostname and os.path.exists(user_config_file): ssh_config = paramiko.SSHConfig() diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index 1120772ed6..8a085c7c1b 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -355,8 +355,8 @@ def is_cached(self): """ Checks if this stage has been already ran and stored """ - from dvc.remote.local import RemoteLOCAL - from dvc.remote.s3 import RemoteS3 + from dvc.remote.local import LocalRemote + from dvc.remote.s3 import S3Remote old = self.reload() if old._changed_outs(): @@ -376,8 +376,8 @@ def is_cached(self): new_d.pop(self.PARAM_MD5, None) outs = old_d.get(self.PARAM_OUTS, []) for out in outs: - out.pop(RemoteLOCAL.PARAM_CHECKSUM, None) - out.pop(RemoteS3.PARAM_CHECKSUM, None) + out.pop(LocalRemote.PARAM_CHECKSUM, None) + out.pop(S3Remote.PARAM_CHECKSUM, None) if old_d != new_d: return False diff --git a/tests/func/remote/test_index.py b/tests/func/remote/test_index.py index 3158e61f2f..b331aa1591 100644 --- a/tests/func/remote/test_index.py +++ b/tests/func/remote/test_index.py @@ -2,9 +2,9 @@ from dvc.compat import fspath from dvc.exceptions import DownloadError, UploadError -from dvc.remote.base import RemoteBASE +from dvc.remote.base import BaseRemote from dvc.remote.index import RemoteIndex -from dvc.remote.local import RemoteLOCAL +from dvc.remote.local import LocalRemote from dvc.utils.fs import remove @@ -17,12 +17,12 @@ def remote(tmp_dir, dvc, tmp_path_factory, mocker): # patch cache_exists since the RemoteLOCAL normally overrides # RemoteBASE.cache_exists. def cache_exists(self, *args, **kwargs): - return RemoteBASE.cache_exists(self, *args, **kwargs) + return BaseRemote.cache_exists(self, *args, **kwargs) - mocker.patch.object(RemoteLOCAL, "cache_exists", cache_exists) + mocker.patch.object(LocalRemote, "cache_exists", cache_exists) # patch index class since RemoteLOCAL normally overrides index class - mocker.patch.object(RemoteLOCAL, "INDEX_CLS", RemoteIndex) + mocker.patch.object(LocalRemote, "INDEX_CLS", RemoteIndex) return dvc.cloud.get_remote("upstream") @@ -79,7 +79,7 @@ def test_clear_on_download_err(tmp_dir, dvc, tmp_path_factory, remote, mocker): remove(dvc.cache.local.cache_dir) mocked_clear = mocker.patch.object(remote.INDEX_CLS, "clear") - mocker.patch.object(RemoteLOCAL, "_download", side_effect=Exception) + mocker.patch.object(LocalRemote, "_download", side_effect=Exception) with pytest.raises(DownloadError): dvc.pull() mocked_clear.assert_called_once_with() @@ -89,14 +89,14 @@ def test_partial_upload(tmp_dir, dvc, tmp_path_factory, remote, mocker): tmp_dir.dvc_gen({"foo": "foo content"}) tmp_dir.dvc_gen({"bar": {"baz": "baz content"}}) - original = RemoteLOCAL._upload + original = LocalRemote._upload def unreliable_upload(self, from_file, to_info, name=None, **kwargs): if "baz" in name: raise Exception("stop baz") return original(self, from_file, to_info, name, **kwargs) - mocker.patch.object(RemoteLOCAL, "_upload", unreliable_upload) + mocker.patch.object(LocalRemote, "_upload", unreliable_upload) with pytest.raises(UploadError): dvc.push() with remote.index: diff --git a/tests/func/remote/test_local.py b/tests/func/remote/test_local.py index 48c89c354a..8fb109c309 100644 --- a/tests/func/remote/test_local.py +++ b/tests/func/remote/test_local.py @@ -3,7 +3,7 @@ import mock from dvc.exceptions import DvcException -from dvc.remote.local import RemoteLOCAL +from dvc.remote.local import LocalRemote from tests.utils import trees_equal @@ -11,7 +11,7 @@ def test_dont_fail_on_unpacked_create_fail(tmp_dir, dvc): (stage,) = tmp_dir.dvc_gen({"dir": {"file": "file_content"}}) with mock.patch.object( - RemoteLOCAL, "_create_unpacked_dir", side_effect=DvcException("msg") + LocalRemote, "_create_unpacked_dir", side_effect=DvcException("msg") ) as unpacked_create_spy, dvc.state: assert not dvc.cache.local.changed_cache(stage.outs[0].checksum) assert unpacked_create_spy.call_count == 1 @@ -19,14 +19,14 @@ def test_dont_fail_on_unpacked_create_fail(tmp_dir, dvc): def test_remove_unpacked_on_create_fail(tmp_dir, dvc): (stage,) = tmp_dir.dvc_gen({"dir": {"file": "file_content"}}) - unpacked_dir = stage.outs[0].cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX + unpacked_dir = stage.outs[0].cache_path + LocalRemote.UNPACKED_DIR_SUFFIX # artificial unpacked dir for test purpose os.makedirs(unpacked_dir) assert os.path.exists(unpacked_dir) with mock.patch.object( - RemoteLOCAL, "_create_unpacked_dir", side_effect=DvcException("msg") + LocalRemote, "_create_unpacked_dir", side_effect=DvcException("msg") ), dvc.state: assert not dvc.cache.local.changed_cache(stage.outs[0].checksum) @@ -35,7 +35,7 @@ def test_remove_unpacked_on_create_fail(tmp_dir, dvc): def test_create_unpacked_on_status(tmp_dir, dvc): (stage,) = tmp_dir.dvc_gen({"dir": {"file": "file_content"}}) - unpacked_dir = stage.outs[0].cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX + unpacked_dir = stage.outs[0].cache_path + LocalRemote.UNPACKED_DIR_SUFFIX assert not os.path.exists(unpacked_dir) with dvc.state: @@ -48,7 +48,7 @@ def test_dir_cache_changed_on_single_cache_file_modification(tmp_dir, dvc): (stage,) = tmp_dir.dvc_gen( {"dir": {"file1": "file1 content", "file2": "file2 content"}} ) - unpacked_dir = stage.outs[0].cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX + unpacked_dir = stage.outs[0].cache_path + LocalRemote.UNPACKED_DIR_SUFFIX assert not os.path.exists(unpacked_dir) file_md5 = stage.outs[0].dir_cache[0]["md5"] diff --git a/tests/func/test_add.py b/tests/func/test_add.py index f6d8a1c22e..7458156500 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -17,7 +17,7 @@ from dvc.exceptions import StageFileCorruptedError from dvc.main import main from dvc.output.base import OutputAlreadyTrackedError -from dvc.remote import RemoteLOCAL +from dvc.remote import LocalRemote from dvc.repo import Repo as DvcRepo from dvc.stage import Stage from dvc.system import System @@ -322,7 +322,7 @@ def test(self): def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc): tmp_dir.gen({"data/data": "foo"}) - get_dir_checksum_counter = mocker.spy(RemoteLOCAL, "get_dir_checksum") + get_dir_checksum_counter = mocker.spy(LocalRemote, "get_dir_checksum") ret = main(["add", "data"]) assert ret == 0 @@ -566,7 +566,7 @@ def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker): dvc.add("dir") tmp_dir.gen("dir/new_file", "new_file_content") - unprotect_spy = mocker.spy(RemoteLOCAL, "unprotect") + unprotect_spy = mocker.spy(LocalRemote, "unprotect") dvc.add("dir") assert not unprotect_spy.mock.called @@ -686,7 +686,7 @@ def test_add_empty_files(tmp_dir, dvc, link): def test_add_optimization_for_hardlink_on_empty_files(tmp_dir, dvc, mocker): dvc.cache.local.cache_types = ["hardlink"] tmp_dir.gen({"foo": "", "bar": "", "lorem": "lorem", "ipsum": "ipsum"}) - m = mocker.spy(RemoteLOCAL, "is_hardlink") + m = mocker.spy(LocalRemote, "is_hardlink") stages = dvc.add(["foo", "bar", "lorem", "ipsum"]) assert m.call_count == 1 diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index b6380c4f66..c1a8966a94 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -12,7 +12,7 @@ from dvc.exceptions import ConfirmRemoveError from dvc.exceptions import DvcException from dvc.main import main -from dvc.remote.local import RemoteLOCAL +from dvc.remote.local import LocalRemote from dvc.repo import Repo as DvcRepo from dvc.stage import Stage from dvc.dvcfile import Dvcfile, DVC_FILE_SUFFIX @@ -25,7 +25,7 @@ from dvc.utils.fs import walk_files from dvc.utils.stage import dump_stage_file from dvc.utils.stage import load_stage_file -from dvc.remote import RemoteS3 +from dvc.remote import S3Remote from tests.remotes import S3 from tests.basic_env import TestDvc from tests.basic_env import TestDvcGit @@ -306,8 +306,8 @@ def test(self): class TestCheckoutMissingMd5InStageFile(TestRepro): def test(self): d = load_stage_file(self.file1_stage) - del d[Stage.PARAM_OUTS][0][RemoteLOCAL.PARAM_CHECKSUM] - del d[Stage.PARAM_DEPS][0][RemoteLOCAL.PARAM_CHECKSUM] + del d[Stage.PARAM_OUTS][0][LocalRemote.PARAM_CHECKSUM] + del d[Stage.PARAM_DEPS][0][LocalRemote.PARAM_CHECKSUM] dump_stage_file(self.file1_stage, d) with pytest.raises(CheckoutError): @@ -738,9 +738,9 @@ def test_checkout_recursive(tmp_dir, dvc): not S3.should_test(), reason="Only run with S3 credentials" ) def test_checkout_for_external_outputs(tmp_dir, dvc): - dvc.cache.s3 = RemoteS3(dvc, {"url": S3.get_url()}) + dvc.cache.s3 = S3Remote(dvc, {"url": S3.get_url()}) - remote = RemoteS3(dvc, {"url": S3.get_url()}) + remote = S3Remote(dvc, {"url": S3.get_url()}) file_path = remote.path_info / "foo" remote.s3.put_object( Bucket=remote.path_info.bucket, Key=file_path.path, Body="foo" diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 1bf93ab7fb..4bec99b058 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -11,15 +11,15 @@ from dvc.cache import NamedCache from dvc.data_cloud import DataCloud from dvc.main import main -from dvc.remote import RemoteAZURE -from dvc.remote import RemoteGDrive -from dvc.remote import RemoteGS -from dvc.remote import RemoteHDFS -from dvc.remote import RemoteHTTP -from dvc.remote import RemoteLOCAL -from dvc.remote import RemoteOSS -from dvc.remote import RemoteS3 -from dvc.remote import RemoteSSH +from dvc.remote import AzureRemote +from dvc.remote import GDriveRemote +from dvc.remote import GSRemote +from dvc.remote import HDFSRemote +from dvc.remote import HTTPRemote +from dvc.remote import LocalRemote +from dvc.remote import OSSRemote +from dvc.remote import S3Remote +from dvc.remote import SSHRemote from dvc.remote.base import STATUS_DELETED, STATUS_NEW, STATUS_OK from dvc.utils import file_md5 from dvc.utils.fs import remove @@ -53,13 +53,13 @@ def test(self): config = copy.deepcopy(TEST_CONFIG) clist = [ - ("s3://mybucket/", RemoteS3), - ("gs://mybucket/", RemoteGS), - ("ssh://user@localhost:/", RemoteSSH), - ("http://localhost:8000/", RemoteHTTP), - ("azure://ContainerName=mybucket;conn_string;", RemoteAZURE), - ("oss://mybucket/", RemoteOSS), - (TestDvc.mkdtemp(), RemoteLOCAL), + ("s3://mybucket/", S3Remote), + ("gs://mybucket/", GSRemote), + ("ssh://user@localhost:/", SSHRemote), + ("http://localhost:8000/", HTTPRemote), + ("azure://ContainerName=mybucket;conn_string;", AzureRemote), + ("oss://mybucket/", OSSRemote), + (TestDvc.mkdtemp(), LocalRemote), ] for scheme, cl in clist: @@ -184,9 +184,9 @@ def test(self): self._test_cloud() -class TestRemoteS3(S3, TestDataCloudBase): +class TestS3Remote(S3, TestDataCloudBase): def _get_cloud_class(self): - return RemoteS3 + return S3Remote def setup_gdrive_cloud(remote_url, dvc): @@ -203,7 +203,7 @@ def setup_gdrive_cloud(remote_url, dvc): remote._gdrive_create_dir("root", remote.path_info.path) -class TestRemoteGDrive(GDrive, TestDataCloudBase): +class TestGDriveRemote(GDrive, TestDataCloudBase): def _setup_cloud(self): self._ensure_should_run() @@ -214,10 +214,10 @@ def _setup_cloud(self): self.assertIsInstance(remote, self._get_cloud_class()) def _get_cloud_class(self): - return RemoteGDrive + return GDriveRemote -class TestRemoteGS(GCP, TestDataCloudBase): +class TestGSRemote(GCP, TestDataCloudBase): def _setup_cloud(self): self._ensure_should_run() @@ -234,26 +234,26 @@ def _setup_cloud(self): self.assertIsInstance(self.cloud.get_remote(), self._get_cloud_class()) def _get_cloud_class(self): - return RemoteGS + return GSRemote -class TestRemoteAZURE(Azure, TestDataCloudBase): +class TestAzureRemote(Azure, TestDataCloudBase): def _get_cloud_class(self): - return RemoteAZURE + return AzureRemote -class TestRemoteOSS(OSS, TestDataCloudBase): +class TestOSSRemote(OSS, TestDataCloudBase): def _get_cloud_class(self): - return RemoteOSS + return OSSRemote -class TestRemoteLOCAL(Local, TestDataCloudBase): +class TestLocalRemote(Local, TestDataCloudBase): def _get_cloud_class(self): - return RemoteLOCAL + return LocalRemote @pytest.mark.usefixtures("ssh_server") -class TestRemoteSSHMocked(SSHMocked, TestDataCloudBase): +class TestSSHRemoteMocked(SSHMocked, TestDataCloudBase): @pytest.fixture(autouse=True) def setup_method_fixture(self, request, ssh_server): self.ssh_server = ssh_server @@ -284,16 +284,16 @@ def _get_keyfile(self): return self.ssh_server.test_creds["key_filename"] def _get_cloud_class(self): - return RemoteSSH + return SSHRemote -class TestRemoteHDFS(HDFS, TestDataCloudBase): +class TestHDFSRemote(HDFS, TestDataCloudBase): def _get_cloud_class(self): - return RemoteHDFS + return HDFSRemote @pytest.mark.usefixtures("http_server") -class TestRemoteHTTP(HTTP, TestDataCloudBase): +class TestHTTPRemote(HTTP, TestDataCloudBase): @pytest.fixture(autouse=True) def setup_method_fixture(self, request, http_server): self.http_server = http_server @@ -303,7 +303,7 @@ def get_url(self): return super().get_url(self.http_server.server_port) def _get_cloud_class(self): - return RemoteHTTP + return HTTPRemote class TestDataCloudCLIBase(TestDvc): @@ -395,7 +395,7 @@ def test(self): self._test() -class TestRemoteLOCALCLI(Local, TestDataCloudCLIBase): +class TestLocalRemoteCLI(Local, TestDataCloudCLIBase): def _test(self): url = self.get_url() @@ -413,7 +413,7 @@ def _test(self): self._test_cloud(TEST_REMOTE) -class TestRemoteS3CLI(S3, TestDataCloudCLIBase): +class TestS3RemoteCLI(S3, TestDataCloudCLIBase): def _test(self): url = self.get_url() @@ -422,7 +422,7 @@ def _test(self): self._test_cloud(TEST_REMOTE) -class TestRemoteGDriveCLI(GDrive, TestDataCloudCLIBase): +class TestGDriveRemoteCLI(GDrive, TestDataCloudCLIBase): def _setup_cloud(self): setup_gdrive_cloud(self.get_url(), self.dvc) @@ -461,7 +461,7 @@ def _test(self): self._test_cloud(TEST_REMOTE) -class TestRemoteGSCLI(GCP, TestDataCloudCLIBase): +class TestGSRemoteCLI(GCP, TestDataCloudCLIBase): def _test(self): url = self.get_url() @@ -479,7 +479,7 @@ def _test(self): self._test_cloud(TEST_REMOTE) -class TestRemoteAZURECLI(Azure, TestDataCloudCLIBase): +class TestAzureRemoteCLI(Azure, TestDataCloudCLIBase): def _test(self): url = self.get_url() @@ -488,7 +488,7 @@ def _test(self): self._test_cloud(TEST_REMOTE) -class TestRemoteOSSCLI(OSS, TestDataCloudCLIBase): +class TestOSSRemoteCLI(OSS, TestDataCloudCLIBase): def _test(self): url = self.get_url() @@ -548,7 +548,7 @@ def main(self, args): self.assertEqual(ret, 0) def _get_cloud_class(self): - return RemoteLOCAL + return LocalRemote def _prepare_repo(self): remote = self.cloud.get_remote() @@ -619,7 +619,7 @@ def test(self): def test_checksum_recalculation(mocker, dvc, tmp_dir): tmp_dir.gen({"foo": "foo"}) - test_get_file_checksum = mocker.spy(RemoteLOCAL, "get_file_checksum") + test_get_file_checksum = mocker.spy(LocalRemote, "get_file_checksum") url = Local.get_url() ret = main(["remote", "add", "-d", TEST_REMOTE, url]) assert ret == 0 diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index 43b8394ad2..2455a3352e 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -5,7 +5,7 @@ from dvc.external_repo import external_repo from dvc.scm.git import Git -from dvc.remote import RemoteLOCAL +from dvc.remote import LocalRemote from dvc.utils import relpath from dvc.utils.fs import remove @@ -47,7 +47,7 @@ def test_cache_reused(erepo_dir, mocker): with erepo_dir.chdir(): erepo_dir.dvc_gen("file", "text", commit="add file") - download_spy = mocker.spy(RemoteLOCAL, "download") + download_spy = mocker.spy(LocalRemote, "download") # Use URL to prevent any fishy optimizations url = "file://{}".format(erepo_dir) diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py index acbbd78e49..de4e61a787 100644 --- a/tests/func/test_gc.py +++ b/tests/func/test_gc.py @@ -9,7 +9,7 @@ from dvc.exceptions import CollectCacheError from dvc.main import main from dvc.repo import Repo as DvcRepo -from dvc.remote.local import RemoteLOCAL +from dvc.remote.local import LocalRemote from dvc.utils.fs import remove from tests.basic_env import TestDir, TestDvcGit @@ -216,7 +216,7 @@ def test_gc_no_unpacked_dir(tmp_dir, dvc): os.remove("dir.dvc") unpackeddir = ( - dir_stages[0].outs[0].cache_path + RemoteLOCAL.UNPACKED_DIR_SUFFIX + dir_stages[0].outs[0].cache_path + LocalRemote.UNPACKED_DIR_SUFFIX ) assert os.path.exists(unpackeddir) @@ -324,7 +324,7 @@ def test_gc_cloud_remove_order(tmp_dir, scm, dvc, tmp_path_factory, mocker): dvc.remove(dir2.relpath) dvc.gc(workspace=True) - mocked_remove = mocker.patch.object(RemoteLOCAL, "remove", autospec=True) + mocked_remove = mocker.patch.object(LocalRemote, "remove", autospec=True) dvc.gc(workspace=True, cloud=True) assert len(mocked_remove.mock_calls) == 8 # dir (and unpacked dir) should be first 4 checksums removed from diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index a4270750f4..9140234df7 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -14,7 +14,7 @@ from dvc.utils import relpath from dvc.compat import fspath_py35, fspath from dvc.utils.fs import get_mtime_and_size -from dvc.remote import RemoteLOCAL +from dvc.remote import LocalRemote from tests.dir_helpers import TmpDir from tests.utils import to_posixpath @@ -138,7 +138,7 @@ def test_match_nested(tmp_dir, dvc): } ) - remote = RemoteLOCAL(dvc, {}) + remote = LocalRemote(dvc, {}) result = {fspath(f) for f in remote.walk_files(".")} assert result == {".dvcignore", "foo"} @@ -148,6 +148,6 @@ def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory): ext_dir = TmpDir(fspath_py35(tmp_path_factory.mktemp("external_dir"))) ext_dir.gen({"y.backup": "y", "tmp": "ext tmp"}) - remote = RemoteLOCAL(dvc, {}) + remote = LocalRemote(dvc, {}) result = {relpath(f, ext_dir) for f in remote.walk_files(ext_dir)} assert result == {"y.backup", "tmp"} diff --git a/tests/func/test_import.py b/tests/func/test_import.py index c57a025829..4e3b911e34 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -239,7 +239,7 @@ def test_download_error_pulling_imported_stage(tmp_dir, dvc, erepo_dir): remove(dst_cache) with patch( - "dvc.remote.RemoteLOCAL._download", side_effect=Exception + "dvc.remote.LocalRemote._download", side_effect=Exception ), pytest.raises(DownloadError): dvc.pull(["foo_imported.dvc"]) diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index 0d3f9e9f54..6f56a4bd76 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -9,8 +9,8 @@ from dvc.exceptions import DownloadError, UploadError from dvc.main import main from dvc.path_info import PathInfo -from dvc.remote import RemoteLOCAL -from dvc.remote.base import RemoteBASE, RemoteCacheRequiredError +from dvc.remote import LocalRemote +from dvc.remote.base import BaseRemote, RemoteCacheRequiredError from dvc.compat import fspath from dvc.utils.fs import remove from tests.basic_env import TestDvc @@ -150,7 +150,7 @@ def test_dir_checksum_should_be_key_order_agnostic(tmp_dir, dvc): path_info = PathInfo("data") with dvc.state: with patch.object( - RemoteBASE, + BaseRemote, "_collect_dir", return_value=[ {"relpath": "1", "md5": "1"}, @@ -160,7 +160,7 @@ def test_dir_checksum_should_be_key_order_agnostic(tmp_dir, dvc): checksum1 = dvc.cache.local.get_dir_checksum(path_info) with patch.object( - RemoteBASE, + BaseRemote, "_collect_dir", return_value=[ {"md5": "1", "relpath": "1"}, @@ -182,14 +182,14 @@ def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory): baz = tmp_dir.dvc_gen({"baz": {"foo": "baz content"}})[0].outs[0] # Faulty upload version, failing on foo - original = RemoteLOCAL._upload + original = LocalRemote._upload def unreliable_upload(self, from_file, to_info, name=None, **kwargs): if "foo" in name: raise Exception("stop foo") return original(self, from_file, to_info, name, **kwargs) - with patch.object(RemoteLOCAL, "_upload", unreliable_upload): + with patch.object(LocalRemote, "_upload", unreliable_upload): with pytest.raises(UploadError) as upload_error_info: dvc.push() assert upload_error_info.value.amount == 3 @@ -203,7 +203,7 @@ def unreliable_upload(self, from_file, to_info, name=None, **kwargs): dvc.push() remove(dvc.cache.local.cache_dir) - with patch.object(RemoteLOCAL, "_download", side_effect=Exception): + with patch.object(LocalRemote, "_download", side_effect=Exception): with pytest.raises(DownloadError) as download_error_info: dvc.pull() # error count should be len(.dir + standalone file checksums) @@ -219,7 +219,7 @@ def test_raise_on_too_many_open_files(tmp_dir, dvc, tmp_path_factory, mocker): tmp_dir.dvc_gen({"file": "file content"}) mocker.patch.object( - RemoteLOCAL, + LocalRemote, "_upload", side_effect=OSError(errno.EMFILE, "Too many open files"), ) @@ -252,7 +252,7 @@ def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker): tmp_dir.dvc_gen({"foo": {"bar": "bar content"}}) tmp_dir.dvc_gen({"baz": "baz content"}) - mocked_upload = mocker.patch.object(RemoteLOCAL, "_upload", return_value=0) + mocked_upload = mocker.patch.object(LocalRemote, "_upload", return_value=0) dvc.push() # last uploaded file should be dir checksum assert mocked_upload.call_args[0][0].endswith(".dir") diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index 7c3e719486..5de6974803 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -25,7 +25,7 @@ from dvc.main import main from dvc.output.base import OutputBase from dvc.path_info import URLInfo -from dvc.remote.local import RemoteLOCAL +from dvc.remote.local import LocalRemote from dvc.repo import Repo as DvcRepo from dvc.stage import Stage from dvc.dvcfile import Dvcfile @@ -748,8 +748,8 @@ def test(self): class TestReproMissingMd5InStageFile(TestRepro): def test(self): d = load_stage_file(self.file1_stage) - del d[Stage.PARAM_OUTS][0][RemoteLOCAL.PARAM_CHECKSUM] - del d[Stage.PARAM_DEPS][0][RemoteLOCAL.PARAM_CHECKSUM] + del d[Stage.PARAM_OUTS][0][LocalRemote.PARAM_CHECKSUM] + del d[Stage.PARAM_DEPS][0][LocalRemote.PARAM_CHECKSUM] dump_stage_file(self.file1_stage, d) stages = self.dvc.reproduce(self.file1_stage) @@ -1329,9 +1329,9 @@ def test_force_import(self): self.assertEqual(ret, 0) patch_download = patch.object( - RemoteLOCAL, + LocalRemote, "download", - side_effect=RemoteLOCAL.download, + side_effect=LocalRemote.download, autospec=True, ) diff --git a/tests/func/test_s3.py b/tests/func/test_s3.py index 3e31765e5b..5a962f7100 100644 --- a/tests/func/test_s3.py +++ b/tests/func/test_s3.py @@ -5,7 +5,7 @@ import pytest from moto import mock_s3 -from dvc.remote.s3 import RemoteS3 +from dvc.remote.s3 import S3Remote from tests.remotes import S3 @@ -31,7 +31,7 @@ def wrapped(*args, **kwargs): def _get_src_dst(): - base_info = RemoteS3.path_cls(S3.get_url()) + base_info = S3Remote.path_cls(S3.get_url()) return base_info / "from", base_info / "to" @@ -43,16 +43,16 @@ def test_copy_singlepart_preserve_etag(): s3.create_bucket(Bucket=from_info.bucket) s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data") - RemoteS3._copy(s3, from_info, to_info, {}) + S3Remote._copy(s3, from_info, to_info, {}) @mock_s3 @pytest.mark.parametrize( "base_info", - [RemoteS3.path_cls("s3://bucket/"), RemoteS3.path_cls("s3://bucket/ns/")], + [S3Remote.path_cls("s3://bucket/"), S3Remote.path_cls("s3://bucket/ns/")], ) def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm): - remote = RemoteS3(dvc, {"url": str(base_info.parent)}) + remote = S3Remote(dvc, {"url": str(base_info.parent)}) remote.s3.create_bucket(Bucket=base_info.bucket) remote.s3.put_object( Bucket=base_info.bucket, Key=(base_info / "from").path, Body="data" @@ -65,8 +65,8 @@ def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm): @mock_s3 def test_makedirs_doesnot_try_on_top_level_paths(tmp_dir, dvc, scm): - base_info = RemoteS3.path_cls("s3://bucket/") - remote = RemoteS3(dvc, {"url": str(base_info)}) + base_info = S3Remote.path_cls("s3://bucket/") + remote = S3Remote(dvc, {"url": str(base_info)}) remote.makedirs(base_info) @@ -108,4 +108,4 @@ def test_copy_multipart_preserve_etag(): s3 = boto3.client("s3") s3.create_bucket(Bucket=from_info.bucket) _upload_multipart(s3, from_info.bucket, from_info.path) - RemoteS3._copy(s3, from_info, to_info, {}) + S3Remote._copy(s3, from_info, to_info, {}) diff --git a/tests/func/test_stage.py b/tests/func/test_stage.py index b0a751e5c7..9a827dbe97 100644 --- a/tests/func/test_stage.py +++ b/tests/func/test_stage.py @@ -4,7 +4,7 @@ from dvc.main import main from dvc.output.local import OutputLOCAL -from dvc.remote.local import RemoteLOCAL +from dvc.remote.local import LocalRemote from dvc.repo import Repo from dvc.stage import Stage from dvc.dvcfile import Dvcfile @@ -54,8 +54,8 @@ def test_empty_list(): def test_list(): lst = [ - {OutputLOCAL.PARAM_PATH: "foo", RemoteLOCAL.PARAM_CHECKSUM: "123"}, - {OutputLOCAL.PARAM_PATH: "bar", RemoteLOCAL.PARAM_CHECKSUM: None}, + {OutputLOCAL.PARAM_PATH: "foo", LocalRemote.PARAM_CHECKSUM: "123"}, + {OutputLOCAL.PARAM_PATH: "bar", LocalRemote.PARAM_CHECKSUM: None}, {OutputLOCAL.PARAM_PATH: "baz"}, ] d = {Stage.PARAM_DEPS: lst} diff --git a/tests/remotes.py b/tests/remotes.py index e2a6ed6790..c6444a3e56 100644 --- a/tests/remotes.py +++ b/tests/remotes.py @@ -7,9 +7,9 @@ from subprocess import CalledProcessError, check_output, Popen from dvc.utils import env2bool -from dvc.remote import RemoteGDrive -from dvc.remote.gs import RemoteGS -from dvc.remote.s3 import RemoteS3 +from dvc.remote import GDriveRemote +from dvc.remote.gs import GSRemote +from dvc.remote.s3 import S3Remote from tests.basic_env import TestDvc from moto.s3 import mock_s3 @@ -80,7 +80,7 @@ class S3Mocked(S3): @contextmanager def remote(cls, repo): with mock_s3(): - yield RemoteS3(repo, {"url": cls.get_url()}) + yield S3Remote(repo, {"url": cls.get_url()}) @staticmethod def put_objects(remote, objects): @@ -128,7 +128,7 @@ def get_url(): @classmethod @contextmanager def remote(cls, repo): - yield RemoteGS(repo, {"url": cls.get_url()}) + yield GSRemote(repo, {"url": cls.get_url()}) @staticmethod def put_objects(remote, objects): @@ -141,7 +141,7 @@ def put_objects(remote, objects): class GDrive: @staticmethod def should_test(): - return os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA) is not None + return os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA) is not None def get_url(self): if not getattr(self, "_remote_url", None): diff --git a/tests/unit/output/test_local.py b/tests/unit/output/test_local.py index ef029d2940..6b1bdefd8b 100644 --- a/tests/unit/output/test_local.py +++ b/tests/unit/output/test_local.py @@ -2,7 +2,7 @@ from mock import patch from dvc.output import OutputLOCAL -from dvc.remote.local import RemoteLOCAL +from dvc.remote.local import LocalRemote from dvc.stage import Stage from dvc.utils import relpath from tests.basic_env import TestDvc @@ -78,7 +78,7 @@ def test_return_0_on_no_cache(self): @patch.object(OutputLOCAL, "checksum", "12345678.dir") @patch.object( - RemoteLOCAL, + LocalRemote, "get_dir_cache", return_value=[{"md5": "asdf"}, {"md5": "qwe"}], ) diff --git a/tests/unit/remote/ssh/test_ssh.py b/tests/unit/remote/ssh/test_ssh.py index ac984af0fe..b556bfbb84 100644 --- a/tests/unit/remote/ssh/test_ssh.py +++ b/tests/unit/remote/ssh/test_ssh.py @@ -6,7 +6,7 @@ from mock import mock_open from mock import patch -from dvc.remote.ssh import RemoteSSH +from dvc.remote.ssh import SSHRemote from dvc.system import System from tests.remotes import SSHMocked @@ -21,20 +21,20 @@ def test_url(dvc): url = "ssh://{}@{}:{}{}".format(user, host, port, path) config = {"url": url} - remote = RemoteSSH(dvc, config) + remote = SSHRemote(dvc, config) assert remote.path_info == url # SCP-like URL ssh://[user@]host.xz:/absolute/path url = "ssh://{}@{}:{}".format(user, host, path) config = {"url": url} - remote = RemoteSSH(dvc, config) + remote = SSHRemote(dvc, config) assert remote.path_info == url def test_no_path(dvc): config = {"url": "ssh://127.0.0.1"} - remote = RemoteSSH(dvc, config) + remote = SSHRemote(dvc, config) assert remote.path_info.path == "" @@ -68,10 +68,10 @@ def test_no_path(dvc): def test_ssh_host_override_from_config( mock_file, mock_exists, dvc, config, expected_host ): - remote = RemoteSSH(dvc, config) + remote = SSHRemote(dvc, config) - mock_exists.assert_called_with(RemoteSSH.ssh_config_filename()) - mock_file.assert_called_with(RemoteSSH.ssh_config_filename()) + mock_exists.assert_called_with(SSHRemote.ssh_config_filename()) + mock_file.assert_called_with(SSHRemote.ssh_config_filename()) assert remote.path_info.host == expected_host @@ -96,10 +96,10 @@ def test_ssh_host_override_from_config( read_data=mock_ssh_config, ) def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user): - remote = RemoteSSH(dvc, config) + remote = SSHRemote(dvc, config) - mock_exists.assert_called_with(RemoteSSH.ssh_config_filename()) - mock_file.assert_called_with(RemoteSSH.ssh_config_filename()) + mock_exists.assert_called_with(SSHRemote.ssh_config_filename()) + mock_file.assert_called_with(SSHRemote.ssh_config_filename()) assert remote.path_info.user == expected_user @@ -109,7 +109,7 @@ def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user): ({"url": "ssh://example.com:2222"}, 2222), ({"url": "ssh://example.com"}, 1234), ({"url": "ssh://example.com", "port": 4321}, 4321), - ({"url": "ssh://not_in_ssh_config.com"}, RemoteSSH.DEFAULT_PORT), + ({"url": "ssh://not_in_ssh_config.com"}, SSHRemote.DEFAULT_PORT), ({"url": "ssh://not_in_ssh_config.com:2222"}, 2222), ({"url": "ssh://not_in_ssh_config.com:2222", "port": 4321}, 4321), ], @@ -121,10 +121,10 @@ def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user): read_data=mock_ssh_config, ) def test_ssh_port(mock_file, mock_exists, dvc, config, expected_port): - remote = RemoteSSH(dvc, config) + remote = SSHRemote(dvc, config) - mock_exists.assert_called_with(RemoteSSH.ssh_config_filename()) - mock_file.assert_called_with(RemoteSSH.ssh_config_filename()) + mock_exists.assert_called_with(SSHRemote.ssh_config_filename()) + mock_file.assert_called_with(SSHRemote.ssh_config_filename()) assert remote.path_info.port == expected_port @@ -156,10 +156,10 @@ def test_ssh_port(mock_file, mock_exists, dvc, config, expected_port): read_data=mock_ssh_config, ) def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile): - remote = RemoteSSH(dvc, config) + remote = SSHRemote(dvc, config) - mock_exists.assert_called_with(RemoteSSH.ssh_config_filename()) - mock_file.assert_called_with(RemoteSSH.ssh_config_filename()) + mock_exists.assert_called_with(SSHRemote.ssh_config_filename()) + mock_file.assert_called_with(SSHRemote.ssh_config_filename()) assert remote.keyfile == expected_keyfile @@ -178,10 +178,10 @@ def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile): read_data=mock_ssh_config, ) def test_ssh_gss_auth(mock_file, mock_exists, dvc, config, expected_gss_auth): - remote = RemoteSSH(dvc, config) + remote = SSHRemote(dvc, config) - mock_exists.assert_called_with(RemoteSSH.ssh_config_filename()) - mock_file.assert_called_with(RemoteSSH.ssh_config_filename()) + mock_exists.assert_called_with(SSHRemote.ssh_config_filename()) + mock_file.assert_called_with(SSHRemote.ssh_config_filename()) assert remote.gss_auth == expected_gss_auth @@ -195,7 +195,7 @@ def test_hardlink_optimization(dvc, tmp_dir, ssh_server): "user": user, "keyfile": ssh_server.test_creds["key_filename"], } - remote = RemoteSSH(dvc, config) + remote = SSHRemote(dvc, config) from_info = remote.path_info / "empty" to_info = remote.path_info / "link" diff --git a/tests/unit/remote/test_azure.py b/tests/unit/remote/test_azure.py index 41919e9396..cf509692c8 100644 --- a/tests/unit/remote/test_azure.py +++ b/tests/unit/remote/test_azure.py @@ -1,4 +1,4 @@ -from dvc.remote.azure import RemoteAZURE +from dvc.remote.azure import AzureRemote container_name = "container-name" @@ -17,7 +17,7 @@ def test_init_compat(dvc): container_name=container_name, connection_string=connection_string, ) config = {"url": url} - remote = RemoteAZURE(dvc, config) + remote = AzureRemote(dvc, config) assert remote.path_info == "azure://" + container_name assert remote.connection_string == connection_string @@ -26,6 +26,6 @@ def test_init(dvc): prefix = "some/prefix" url = "azure://{}/{}".format(container_name, prefix) config = {"url": url, "connection_string": connection_string} - remote = RemoteAZURE(dvc, config) + remote = AzureRemote(dvc, config) assert remote.path_info == url assert remote.connection_string == connection_string diff --git a/tests/unit/remote/test_base.py b/tests/unit/remote/test_base.py index 7334594627..28f906bf8a 100644 --- a/tests/unit/remote/test_base.py +++ b/tests/unit/remote/test_base.py @@ -3,7 +3,7 @@ import pytest from dvc.path_info import PathInfo -from dvc.remote.base import RemoteBASE +from dvc.remote.base import BaseRemote from dvc.remote.base import RemoteCmdError from dvc.remote.base import RemoteMissingDepsError @@ -16,7 +16,7 @@ def __eq__(self, other): CallableOrNone = _CallableOrNone() -REMOTE_CLS = RemoteBASE +REMOTE_CLS = BaseRemote def test_missing_deps(dvc): @@ -42,10 +42,10 @@ def test_cmd_error(dvc): REMOTE_CLS(dvc, config).remove("file") -@mock.patch.object(RemoteBASE, "_cache_checksums_traverse") -@mock.patch.object(RemoteBASE, "_cache_object_exists") +@mock.patch.object(BaseRemote, "_cache_checksums_traverse") +@mock.patch.object(BaseRemote, "_cache_object_exists") def test_cache_exists(object_exists, traverse, dvc): - remote = RemoteBASE(dvc, {}) + remote = BaseRemote(dvc, {}) # remote does not support traverse remote.CAN_TRAVERSE = False @@ -98,13 +98,13 @@ def test_cache_exists(object_exists, traverse, dvc): @mock.patch.object( - RemoteBASE, "cache_checksums", return_value=[], + BaseRemote, "cache_checksums", return_value=[], ) @mock.patch.object( - RemoteBASE, "path_to_checksum", side_effect=lambda x: x, + BaseRemote, "path_to_checksum", side_effect=lambda x: x, ) def test_cache_checksums_traverse(path_to_checksum, cache_checksums, dvc): - remote = RemoteBASE(dvc, {}) + remote = BaseRemote(dvc, {}) remote.path_info = PathInfo("foo") # parallel traverse @@ -129,7 +129,7 @@ def test_cache_checksums_traverse(path_to_checksum, cache_checksums, dvc): def test_cache_checksums(dvc): - remote = RemoteBASE(dvc, {}) + remote = BaseRemote(dvc, {}) remote.path_info = PathInfo("foo") with mock.patch.object( diff --git a/tests/unit/remote/test_gdrive.py b/tests/unit/remote/test_gdrive.py index 50aac91903..be0d37f12f 100644 --- a/tests/unit/remote/test_gdrive.py +++ b/tests/unit/remote/test_gdrive.py @@ -2,7 +2,7 @@ import os from dvc.remote.gdrive import ( - RemoteGDrive, + GDriveRemote, GDriveAccessTokenRefreshError, GDriveMissedCredentialKeyError, ) @@ -20,21 +20,21 @@ class TestRemoteGDrive(object): } def test_init(self, dvc): - remote = RemoteGDrive(dvc, self.CONFIG) + remote = GDriveRemote(dvc, self.CONFIG) assert str(remote.path_info) == self.CONFIG["url"] def test_drive(self, dvc): - remote = RemoteGDrive(dvc, self.CONFIG) + remote = GDriveRemote(dvc, self.CONFIG) os.environ[ - RemoteGDrive.GDRIVE_CREDENTIALS_DATA + GDriveRemote.GDRIVE_CREDENTIALS_DATA ] = USER_CREDS_TOKEN_REFRESH_ERROR with pytest.raises(GDriveAccessTokenRefreshError): remote._drive - os.environ[RemoteGDrive.GDRIVE_CREDENTIALS_DATA] = "" - remote = RemoteGDrive(dvc, self.CONFIG) + os.environ[GDriveRemote.GDRIVE_CREDENTIALS_DATA] = "" + remote = GDriveRemote(dvc, self.CONFIG) os.environ[ - RemoteGDrive.GDRIVE_CREDENTIALS_DATA + GDriveRemote.GDRIVE_CREDENTIALS_DATA ] = USER_CREDS_MISSED_KEY_ERROR with pytest.raises(GDriveMissedCredentialKeyError): remote._drive diff --git a/tests/unit/remote/test_gs.py b/tests/unit/remote/test_gs.py index 1485d0f1a7..43eeb00d78 100644 --- a/tests/unit/remote/test_gs.py +++ b/tests/unit/remote/test_gs.py @@ -3,7 +3,7 @@ import requests from dvc.remote.gs import dynamic_chunk_size -from dvc.remote.gs import RemoteGS +from dvc.remote.gs import GSRemote BUCKET = "bucket" @@ -19,7 +19,7 @@ def test_init(dvc): - remote = RemoteGS(dvc, CONFIG) + remote = GSRemote(dvc, CONFIG) assert remote.path_info == URL assert remote.projectname == PROJECT assert remote.credentialpath == CREDENTIALPATH @@ -27,7 +27,7 @@ def test_init(dvc): @mock.patch("google.cloud.storage.Client.from_service_account_json") def test_gs(mock_client, dvc): - remote = RemoteGS(dvc, CONFIG) + remote = GSRemote(dvc, CONFIG) assert remote.credentialpath remote.gs() mock_client.assert_called_once_with(CREDENTIALPATH) @@ -37,7 +37,7 @@ def test_gs(mock_client, dvc): def test_gs_no_credspath(mock_client, dvc): config = CONFIG.copy() del config["credentialpath"] - remote = RemoteGS(dvc, config) + remote = GSRemote(dvc, config) remote.gs() mock_client.assert_called_with(PROJECT) diff --git a/tests/unit/remote/test_http.py b/tests/unit/remote/test_http.py index f837e06178..48449f1a7f 100644 --- a/tests/unit/remote/test_http.py +++ b/tests/unit/remote/test_http.py @@ -2,7 +2,7 @@ from dvc.exceptions import HTTPError from dvc.path_info import URLInfo -from dvc.remote.http import RemoteHTTP +from dvc.remote.http import HTTPRemote from tests.utils.httpd import StaticFileServer @@ -11,7 +11,7 @@ def test_download_fails_on_error_code(dvc): url = "http://localhost:{}/".format(httpd.server_port) config = {"url": url} - remote = RemoteHTTP(dvc, config) + remote = HTTPRemote(dvc, config) with pytest.raises(HTTPError): remote._download(URLInfo(url) / "missing.txt", "missing.txt") @@ -25,7 +25,7 @@ def test_public_auth_method(dvc): "password": "", } - remote = RemoteHTTP(dvc, config) + remote = HTTPRemote(dvc, config) assert remote.auth_method() is None @@ -44,7 +44,7 @@ def test_basic_auth_method(dvc): "password": password, } - remote = RemoteHTTP(dvc, config) + remote = HTTPRemote(dvc, config) assert remote.auth_method() == auth assert isinstance(remote.auth_method(), HTTPBasicAuth) @@ -64,7 +64,7 @@ def test_digest_auth_method(dvc): "password": password, } - remote = RemoteHTTP(dvc, config) + remote = HTTPRemote(dvc, config) assert remote.auth_method() == auth assert isinstance(remote.auth_method(), HTTPDigestAuth) @@ -81,7 +81,7 @@ def test_custom_auth_method(dvc): "password": password, } - remote = RemoteHTTP(dvc, config) + remote = HTTPRemote(dvc, config) assert remote.auth_method() is None assert header in remote.headers diff --git a/tests/unit/remote/test_local.py b/tests/unit/remote/test_local.py index a5d653e971..1cc6b9f1dd 100644 --- a/tests/unit/remote/test_local.py +++ b/tests/unit/remote/test_local.py @@ -5,7 +5,7 @@ from dvc.cache import NamedCache from dvc.path_info import PathInfo -from dvc.remote.local import RemoteLOCAL +from dvc.remote.local import LocalRemote def test_status_download_optimization(mocker, dvc): @@ -13,7 +13,7 @@ def test_status_download_optimization(mocker, dvc): And the desired files to fetch are already on the local cache, Don't check the existence of the desired files on the remote cache """ - remote = RemoteLOCAL(dvc, {}) + remote = LocalRemote(dvc, {}) infos = NamedCache() infos.add("local", "acbd18db4cc2f85cedef654fccc4a4d8", "foo") @@ -33,7 +33,7 @@ def test_status_download_optimization(mocker, dvc): @pytest.mark.parametrize("link_name", ["hardlink", "symlink"]) def test_is_protected(tmp_dir, dvc, link_name): - remote = RemoteLOCAL(dvc, {}) + remote = LocalRemote(dvc, {}) link_method = getattr(remote, link_name) (tmp_dir / "foo").write_text("foo") @@ -66,7 +66,7 @@ def test_is_protected(tmp_dir, dvc, link_name): def test_protect_ignore_errors(tmp_dir, mocker, err): tmp_dir.gen("foo", "foo") foo = PathInfo("foo") - remote = RemoteLOCAL(None, {}) + remote = LocalRemote(None, {}) remote.protect(foo) @@ -80,7 +80,7 @@ def test_protect_ignore_errors(tmp_dir, mocker, err): def test_protect_ignore_erofs(tmp_dir, mocker): tmp_dir.gen("foo", "foo") foo = PathInfo("foo") - remote = RemoteLOCAL(None, {}) + remote = LocalRemote(None, {}) mock_chmod = mocker.patch( "os.chmod", side_effect=OSError(errno.EROFS, "read-only fs") diff --git a/tests/unit/remote/test_oss.py b/tests/unit/remote/test_oss.py index af1cb782d8..1c7df2a8eb 100644 --- a/tests/unit/remote/test_oss.py +++ b/tests/unit/remote/test_oss.py @@ -1,4 +1,4 @@ -from dvc.remote.oss import RemoteOSS +from dvc.remote.oss import OSSRemote bucket_name = "bucket-name" @@ -16,7 +16,7 @@ def test_init(dvc): "oss_key_secret": key_secret, "oss_endpoint": endpoint, } - remote = RemoteOSS(dvc, config) + remote = OSSRemote(dvc, config) assert remote.path_info == url assert remote.endpoint == endpoint assert remote.key_id == key_id diff --git a/tests/unit/remote/test_remote.py b/tests/unit/remote/test_remote.py index 59e91713ab..294ba57fbd 100644 --- a/tests/unit/remote/test_remote.py +++ b/tests/unit/remote/test_remote.py @@ -1,6 +1,6 @@ import pytest -from dvc.remote import Remote, RemoteS3, RemoteGS +from dvc.remote import Remote, S3Remote, GSRemote def test_remote_with_checksum_jobs(dvc): @@ -29,7 +29,7 @@ def test_remote_without_checksum_jobs_default(dvc): assert remote.checksum_jobs == remote.CHECKSUM_JOBS -@pytest.mark.parametrize("remote_cls", [RemoteGS, RemoteS3]) +@pytest.mark.parametrize("remote_cls", [GSRemote, S3Remote]) def test_makedirs_not_create_for_top_level_path(remote_cls, dvc, mocker): url = "{.scheme}://bucket/".format(remote_cls) remote = remote_cls(dvc, {"url": url}) diff --git a/tests/unit/remote/test_remote_dir.py b/tests/unit/remote/test_remote_dir.py index 342a5af557..614f100763 100644 --- a/tests/unit/remote/test_remote_dir.py +++ b/tests/unit/remote/test_remote_dir.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import pytest import os -from dvc.remote.s3 import RemoteS3 +from dvc.remote.s3 import S3Remote from dvc.utils.fs import walk_files from dvc.path_info import PathInfo from tests.remotes import GCP, S3Mocked @@ -90,15 +90,15 @@ def test_copy_preserve_etag_across_buckets(remote, dvc): s3 = remote.s3 s3.create_bucket(Bucket="another") - another = RemoteS3(dvc, {"url": "s3://another", "region": "us-east-1"}) + another = S3Remote(dvc, {"url": "s3://another", "region": "us-east-1"}) from_info = remote.path_info / "foo" to_info = another.path_info / "foo" remote.copy(from_info, to_info) - from_etag = RemoteS3.get_etag(s3, from_info.bucket, from_info.path) - to_etag = RemoteS3.get_etag(s3, "another", "foo") + from_etag = S3Remote.get_etag(s3, from_info.bucket, from_info.path) + to_etag = S3Remote.get_etag(s3, "another", "foo") assert from_etag == to_etag diff --git a/tests/unit/remote/test_s3.py b/tests/unit/remote/test_s3.py index 5cf2f9cab7..c7551abd49 100644 --- a/tests/unit/remote/test_s3.py +++ b/tests/unit/remote/test_s3.py @@ -1,7 +1,7 @@ import pytest from dvc.config import ConfigError -from dvc.remote.s3 import RemoteS3 +from dvc.remote.s3 import S3Remote bucket_name = "bucket-name" @@ -21,7 +21,7 @@ def grants(): def test_init(dvc): config = {"url": url} - remote = RemoteS3(dvc, config) + remote = S3Remote(dvc, config) assert remote.path_info == url @@ -34,7 +34,7 @@ def test_grants(dvc): "grant_write_acp": "id=write-acp-permission-id", "grant_full_control": "id=full-control-permission-id", } - remote = RemoteS3(dvc, config) + remote = S3Remote(dvc, config) assert ( remote.extra_args["GrantRead"] @@ -53,4 +53,4 @@ def test_grants_mutually_exclusive_acl_error(dvc, grants): config = {"url": url, "acl": "public-read", grant_option: grant_value} with pytest.raises(ConfigError): - RemoteS3(dvc, config) + S3Remote(dvc, config)