diff --git a/dvc/fs/base.py b/dvc/fs/base.py index 33a6d1fa76..5d992992f3 100644 --- a/dvc/fs/base.py +++ b/dvc/fs/base.py @@ -9,7 +9,6 @@ from dvc.progress import Tqdm from dvc.utils import tmp_fname from dvc.utils.fs import makedirs, move -from dvc.utils.http import open_url logger = logging.getLogger(__name__) @@ -126,12 +125,6 @@ def _check_requires(self, **kwargs): ) def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs): - if hasattr(self, "_generate_download_url"): - # pylint:disable=no-member - func = self._generate_download_url # type: ignore[attr-defined] - get_url = partial(func, path_info) - return open_url(get_url, mode=mode, encoding=encoding, **kwargs) - raise RemoteActionNotImplemented("open", self.scheme) def exists(self, path_info) -> bool: diff --git a/dvc/fs/http.py b/dvc/fs/http.py index 1805bbf16c..06579fcaef 100644 --- a/dvc/fs/http.py +++ b/dvc/fs/http.py @@ -65,9 +65,6 @@ def _auth_method(self, url): self.headers.update({self.custom_auth_header: self.password}) return None - def _generate_download_url(self, path_info): - return path_info.url - @wrap_prop(threading.Lock()) @cached_property def _session(self): @@ -190,6 +187,17 @@ def _upload( desc=name or to_info.url, ) + def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs): + from dvc.utils.http import open_url + + return open_url( + path_info.url, + mode=mode, + encoding=encoding, + auth=self._auth_method(path_info), + **kwargs, + ) + @staticmethod def _content_length(response) -> Optional[int]: res = response.headers.get("Content-Length") diff --git a/dvc/utils/http.py b/dvc/utils/http.py index c372db561d..4b03ca5620 100644 --- a/dvc/utils/http.py +++ b/dvc/utils/http.py @@ -24,13 +24,15 @@ def open_url(url, mode="r", encoding=None, **iter_opts): @contextmanager -def iter_url(url, chunk_size=io.DEFAULT_BUFFER_SIZE): +def iter_url(url, chunk_size=io.DEFAULT_BUFFER_SIZE, auth=None): """Iterate over chunks requested from url.""" import requests def request(headers=None): the_url = url() if callable(url) else url - response = requests.get(the_url, stream=True, headers=headers) + response = requests.get( + the_url, stream=True, headers=headers, auth=auth + ) if response.status_code == 404: raise FileNotFoundError(f"Can't open {the_url}") response.raise_for_status()