Skip to content

[6478] Fix 401 unauthorized for directories in basic auth protected http remote #6479

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions dvc/fs/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
from dvc.progress import Tqdm
from dvc.utils import tmp_fname
from dvc.utils.fs import makedirs, move
from dvc.utils.http import open_url

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -126,12 +125,6 @@ def _check_requires(self, **kwargs):
)

def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
if hasattr(self, "_generate_download_url"):
# pylint:disable=no-member
func = self._generate_download_url # type: ignore[attr-defined]
get_url = partial(func, path_info)
return open_url(get_url, mode=mode, encoding=encoding, **kwargs)

raise RemoteActionNotImplemented("open", self.scheme)

def exists(self, path_info) -> bool:
Expand Down
14 changes: 11 additions & 3 deletions dvc/fs/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,6 @@ def _auth_method(self, url):
self.headers.update({self.custom_auth_header: self.password})
return None

def _generate_download_url(self, path_info):
return path_info.url

@wrap_prop(threading.Lock())
@cached_property
def _session(self):
Expand Down Expand Up @@ -190,6 +187,17 @@ def _upload(
desc=name or to_info.url,
)

def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
from dvc.utils.http import open_url

return open_url(
path_info.url,
mode=mode,
encoding=encoding,
auth=self._auth_method(path_info),
**kwargs,
)

@staticmethod
def _content_length(response) -> Optional[int]:
res = response.headers.get("Content-Length")
Expand Down
6 changes: 4 additions & 2 deletions dvc/utils/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,15 @@ def open_url(url, mode="r", encoding=None, **iter_opts):


@contextmanager
def iter_url(url, chunk_size=io.DEFAULT_BUFFER_SIZE):
def iter_url(url, chunk_size=io.DEFAULT_BUFFER_SIZE, auth=None):
"""Iterate over chunks requested from url."""
import requests

def request(headers=None):
the_url = url() if callable(url) else url
response = requests.get(the_url, stream=True, headers=headers)
response = requests.get(
the_url, stream=True, headers=headers, auth=auth
)
if response.status_code == 404:
raise FileNotFoundError(f"Can't open {the_url}")
response.raise_for_status()
Expand Down