Skip to content

refactor: dvc/remotes class names unification #3684

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 26, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions dvc/data_cloud.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def push(self, cache, jobs=None, remote=None, show_checksums=False):
Args:
cache (NamedCache): named checksums to push to the cloud.
jobs (int): number of jobs that can be running simultaneously.
remote (dvc.remote.base.RemoteBASE): optional remote to push to.
remote (dvc.remote.base.BaseRemote): optional remote to push to.
By default remote from core.remote config option is used.
show_checksums (bool): show checksums instead of file names in
information messages.
Expand All @@ -72,7 +72,7 @@ def pull(self, cache, jobs=None, remote=None, show_checksums=False):
Args:
cache (NamedCache): named checksums to pull from the cloud.
jobs (int): number of jobs that can be running simultaneously.
remote (dvc.remote.base.RemoteBASE): optional remote to pull from.
remote (dvc.remote.base.BaseRemote): optional remote to pull from.
By default remote from core.remote config option is used.
show_checksums (bool): show checksums instead of file names in
information messages.
Expand Down Expand Up @@ -103,7 +103,7 @@ def status(self, cache, jobs=None, remote=None, show_checksums=False):
Args:
cache (NamedCache): named checksums to check status for.
jobs (int): number of jobs that can be running simultaneously.
remote (dvc.remote.base.RemoteBASE): optional remote to compare
remote (dvc.remote.base.BaseRemote): optional remote to compare
cache to. By default remote from core.remote config option
is used.
show_checksums (bool): show checksums instead of file names in
Expand Down
4 changes: 2 additions & 2 deletions dvc/dependency/http.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from dvc.dependency.base import DependencyBase
from dvc.output.base import OutputBase
from dvc.remote.http import RemoteHTTP
from dvc.remote.http import HTTPRemote


class DependencyHTTP(DependencyBase, OutputBase):
REMOTE = RemoteHTTP
REMOTE = HTTPRemote
4 changes: 2 additions & 2 deletions dvc/dependency/https.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from .http import DependencyHTTP
from dvc.remote.https import RemoteHTTPS
from dvc.remote.https import HTTPSRemote


class DependencyHTTPS(DependencyHTTP):
REMOTE = RemoteHTTPS
REMOTE = HTTPSRemote
12 changes: 6 additions & 6 deletions dvc/output/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
from dvc.output.s3 import OutputS3
from dvc.output.ssh import OutputSSH
from dvc.remote import Remote
from dvc.remote.hdfs import RemoteHDFS
from dvc.remote.local import RemoteLOCAL
from dvc.remote.s3 import RemoteS3
from dvc.remote.hdfs import HDFSRemote
from dvc.remote.local import LocalRemote
from dvc.remote.s3 import S3Remote
from dvc.scheme import Schemes

OUTS = [
Expand Down Expand Up @@ -44,9 +44,9 @@
# so when a few types of outputs share the same name, we only need
# specify it once.
CHECKSUMS_SCHEMA = {
RemoteLOCAL.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
RemoteS3.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
RemoteHDFS.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
LocalRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
S3Remote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
HDFSRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
}

TAGS_SCHEMA = {str: CHECKSUMS_SCHEMA}
Expand Down
4 changes: 2 additions & 2 deletions dvc/output/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from dvc.cache import NamedCache
from dvc.exceptions import CollectCacheError, RemoteCacheRequiredError
from dvc.exceptions import DvcException
from dvc.remote.base import RemoteBASE
from dvc.remote.base import BaseRemote


logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -41,7 +41,7 @@ def __init__(self, path):
class OutputBase(object):
IS_DEPENDENCY = False

REMOTE = RemoteBASE
REMOTE = BaseRemote

PARAM_PATH = "path"
PARAM_CACHE = "cache"
Expand Down
4 changes: 2 additions & 2 deletions dvc/output/gs.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from dvc.output.s3 import OutputS3
from dvc.remote.gs import RemoteGS
from dvc.remote.gs import GSRemote


class OutputGS(OutputS3):
REMOTE = RemoteGS
REMOTE = GSRemote
4 changes: 2 additions & 2 deletions dvc/output/hdfs.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from dvc.output.base import OutputBase
from dvc.remote.hdfs import RemoteHDFS
from dvc.remote.hdfs import HDFSRemote


class OutputHDFS(OutputBase):
REMOTE = RemoteHDFS
REMOTE = HDFSRemote
4 changes: 2 additions & 2 deletions dvc/output/local.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from dvc.exceptions import DvcException
from dvc.istextfile import istextfile
from dvc.output.base import OutputBase
from dvc.remote.local import RemoteLOCAL
from dvc.remote.local import LocalRemote
from dvc.utils import relpath
from dvc.compat import fspath_py35
from dvc.utils.fs import path_isin
Expand All @@ -15,7 +15,7 @@


class OutputLOCAL(OutputBase):
REMOTE = RemoteLOCAL
REMOTE = LocalRemote
sep = os.sep

def __init__(self, stage, path, *args, **kwargs):
Expand Down
4 changes: 2 additions & 2 deletions dvc/output/s3.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from dvc.output.base import OutputBase
from dvc.remote.s3 import RemoteS3
from dvc.remote.s3 import S3Remote


class OutputS3(OutputBase):
REMOTE = RemoteS3
REMOTE = S3Remote
4 changes: 2 additions & 2 deletions dvc/output/ssh.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from dvc.output.base import OutputBase
from dvc.remote.ssh import RemoteSSH
from dvc.remote.ssh import SSHRemote


class OutputSSH(OutputBase):
REMOTE = RemoteSSH
REMOTE = SSHRemote
42 changes: 21 additions & 21 deletions dvc/remote/__init__.py
Original file line number Diff line number Diff line change
@@ -1,37 +1,37 @@
import posixpath
from urllib.parse import urlparse

from dvc.remote.azure import RemoteAZURE
from dvc.remote.gdrive import RemoteGDrive
from dvc.remote.gs import RemoteGS
from dvc.remote.hdfs import RemoteHDFS
from dvc.remote.http import RemoteHTTP
from dvc.remote.https import RemoteHTTPS
from dvc.remote.local import RemoteLOCAL
from dvc.remote.oss import RemoteOSS
from dvc.remote.s3 import RemoteS3
from dvc.remote.ssh import RemoteSSH
from dvc.remote.azure import AzureRemote
from dvc.remote.gdrive import GDriveRemote
from dvc.remote.gs import GSRemote
from dvc.remote.hdfs import HDFSRemote
from dvc.remote.http import HTTPRemote
from dvc.remote.https import HTTPSRemote
from dvc.remote.local import LocalRemote
from dvc.remote.oss import OSSRemote
from dvc.remote.s3 import S3Remote
from dvc.remote.ssh import SSHRemote


REMOTES = [
RemoteAZURE,
RemoteGDrive,
RemoteGS,
RemoteHDFS,
RemoteHTTP,
RemoteHTTPS,
RemoteS3,
RemoteSSH,
RemoteOSS,
# NOTE: RemoteLOCAL is the default
AzureRemote,
GDriveRemote,
GSRemote,
HDFSRemote,
HTTPRemote,
HTTPSRemote,
S3Remote,
SSHRemote,
OSSRemote,
# NOTE: LocalRemote is the default
]


def _get(remote_conf):
for remote in REMOTES:
if remote.supported(remote_conf):
return remote
return RemoteLOCAL
return LocalRemote


def Remote(repo, **kwargs):
Expand Down
4 changes: 2 additions & 2 deletions dvc/remote/azure.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@

from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.remote.base import RemoteBASE
from dvc.remote.base import BaseRemote
from dvc.scheme import Schemes


logger = logging.getLogger(__name__)


class RemoteAZURE(RemoteBASE):
class AzureRemote(BaseRemote):
scheme = Schemes.AZURE
path_cls = CloudURLInfo
REGEX = (
Expand Down
2 changes: 1 addition & 1 deletion dvc/remote/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def wrapper(remote_obj, *args, **kwargs):
return wrapper


class RemoteBASE(object):
class BaseRemote(object):
scheme = "base"
path_cls = URLInfo
REQUIRES = {}
Expand Down
12 changes: 6 additions & 6 deletions dvc/remote/gdrive.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from dvc.progress import Tqdm
from dvc.scheme import Schemes
from dvc.path_info import CloudURLInfo
from dvc.remote.base import RemoteBASE
from dvc.remote.base import BaseRemote
from dvc.exceptions import DvcException
from dvc.utils import tmp_fname, format_link

Expand Down Expand Up @@ -94,7 +94,7 @@ def __init__(self, url):
self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))


class RemoteGDrive(RemoteBASE):
class GDriveRemote(BaseRemote):
scheme = Schemes.GDRIVE
path_cls = GDriveURLInfo
REQUIRES = {"pydrive2": "pydrive2"}
Expand Down Expand Up @@ -139,7 +139,7 @@ def __init__(self, repo, config):
self._validate_config()
self._gdrive_user_credentials_path = (
tmp_fname(os.path.join(self.repo.tmp_dir, ""))
if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA)
if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA)
else config.get(
"gdrive_user_credentials_file",
os.path.join(
Expand Down Expand Up @@ -183,12 +183,12 @@ def _drive(self):
from pydrive2.auth import GoogleAuth
from pydrive2.drive import GoogleDrive

if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA):
if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
with open(
self._gdrive_user_credentials_path, "w"
) as credentials_file:
credentials_file.write(
os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA)
os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA)
)

GoogleAuth.DEFAULT_SETTINGS["client_config_backend"] = "settings"
Expand Down Expand Up @@ -237,7 +237,7 @@ def _drive(self):
except Exception as exc:
raise DvcException("Google Drive authentication failed") from exc
finally:
if os.getenv(RemoteGDrive.GDRIVE_CREDENTIALS_DATA):
if os.getenv(GDriveRemote.GDRIVE_CREDENTIALS_DATA):
os.remove(self._gdrive_user_credentials_path)

return GoogleDrive(gauth)
Expand Down
4 changes: 2 additions & 2 deletions dvc/remote/gs.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from dvc.exceptions import DvcException
from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.remote.base import RemoteBASE
from dvc.remote.base import BaseRemote
from dvc.scheme import Schemes

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -66,7 +66,7 @@ def _upload_to_bucket(
blob.upload_from_file(wrapped)


class RemoteGS(RemoteBASE):
class GSRemote(BaseRemote):
scheme = Schemes.GS
path_cls = CloudURLInfo
REQUIRES = {"google-cloud-storage": "google.cloud.storage"}
Expand Down
4 changes: 2 additions & 2 deletions dvc/remote/hdfs.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@
from contextlib import closing, contextmanager
from urllib.parse import urlparse

from .base import RemoteBASE, RemoteCmdError
from .base import BaseRemote, RemoteCmdError
from .pool import get_connection
from dvc.scheme import Schemes
from dvc.utils import fix_env, tmp_fname

logger = logging.getLogger(__name__)


class RemoteHDFS(RemoteBASE):
class HDFSRemote(BaseRemote):
scheme = Schemes.HDFS
REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
PARAM_CHECKSUM = "checksum"
Expand Down
4 changes: 2 additions & 2 deletions dvc/remote/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import dvc.prompt as prompt
from dvc.exceptions import DvcException, HTTPError
from dvc.progress import Tqdm
from dvc.remote.base import RemoteBASE
from dvc.remote.base import BaseRemote
from dvc.scheme import Schemes

logger = logging.getLogger(__name__)
Expand All @@ -23,7 +23,7 @@ def ask_password(host, user):
)


class RemoteHTTP(RemoteBASE):
class HTTPRemote(BaseRemote):
scheme = Schemes.HTTP
path_cls = HTTPURLInfo
SESSION_RETRIES = 5
Expand Down
4 changes: 2 additions & 2 deletions dvc/remote/https.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from .http import RemoteHTTP
from .http import HTTPRemote
from dvc.scheme import Schemes


class RemoteHTTPS(RemoteHTTP):
class HTTPSRemote(HTTPRemote):
scheme = Schemes.HTTPS
4 changes: 2 additions & 2 deletions dvc/remote/local.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from dvc.progress import Tqdm
from dvc.remote.base import (
index_locked,
RemoteBASE,
BaseRemote,
STATUS_MAP,
STATUS_DELETED,
STATUS_MISSING,
Expand All @@ -31,7 +31,7 @@
logger = logging.getLogger(__name__)


class RemoteLOCAL(RemoteBASE):
class LocalRemote(BaseRemote):
scheme = Schemes.LOCAL
path_cls = PathInfo
PARAM_CHECKSUM = "md5"
Expand Down
4 changes: 2 additions & 2 deletions dvc/remote/oss.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@

from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.remote.base import RemoteBASE
from dvc.remote.base import BaseRemote
from dvc.scheme import Schemes


logger = logging.getLogger(__name__)


class RemoteOSS(RemoteBASE):
class OSSRemote(BaseRemote):
"""
oss2 document:
https://www.alibabacloud.com/help/doc-detail/32026.htm
Expand Down
4 changes: 2 additions & 2 deletions dvc/remote/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,13 @@
from dvc.exceptions import ETagMismatchError
from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.remote.base import RemoteBASE
from dvc.remote.base import BaseRemote
from dvc.scheme import Schemes

logger = logging.getLogger(__name__)


class RemoteS3(RemoteBASE):
class S3Remote(BaseRemote):
scheme = Schemes.S3
path_cls = CloudURLInfo
REQUIRES = {"boto3": "boto3"}
Expand Down
Loading