From 84fd957c7dbdc4a2cf4a9d2718d2cb5e3945cc87 Mon Sep 17 00:00:00 2001 From: Thiago Date: Wed, 24 Nov 2021 11:10:57 +0100 Subject: [PATCH 1/8] Pass the session's proxies property to request After waiting a long time for a fix on the side of the request that could fix pip install issues with proxy as already discussed in #9691. It seems that such changes as mentioned in psf/requests#5735 will take a while to happen, probably only on version 3 of the request. Therefore, I'm suggesting a change on the pip side as already proposed in [issuecomment-79051687]https://github.com/pypa/pip/issues/9568#issuecomment-790516875 and [issuecomment-939373868]https://github.com/psf/requests/pull/5735#issuecomment-939373868 I think it's important that pip address this issue asap since the newer versions of pip are not working properly behind a proxy. --- src/pip/_internal/network/session.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pip/_internal/network/session.py b/src/pip/_internal/network/session.py index cbe743ba6a1..e06ac2d3ee1 100644 --- a/src/pip/_internal/network/session.py +++ b/src/pip/_internal/network/session.py @@ -449,6 +449,8 @@ def is_secure_origin(self, location: Link) -> bool: def request(self, method: str, url: str, *args: Any, **kwargs: Any) -> Response: # Allow setting a default timeout on a session kwargs.setdefault("timeout", self.timeout) + # Allow setting a default proxies on a session + kwargs.setdefault("proxies", self.proxies) # Dispatch the actual request return super().request(method, url, *args, **kwargs) From 3c3f94c72caa38fdbf88a971b8a84471c3bcd28b Mon Sep 17 00:00:00 2001 From: Thiago Date: Mon, 7 Feb 2022 16:24:45 +0100 Subject: [PATCH 2/8] Update test_network_session.py Suggestion for testing proxy. A valid proxy must be provided so this test can work. --- tests/unit/test_network_session.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/unit/test_network_session.py b/tests/unit/test_network_session.py index f16843abfb1..b3fc582cab4 100644 --- a/tests/unit/test_network_session.py +++ b/tests/unit/test_network_session.py @@ -6,6 +6,9 @@ from pip import __version__ from pip._internal.models.link import Link from pip._internal.network.session import CI_ENVIRONMENT_VARIABLES, PipSession +from pip._vendor import requests +from urllib.request import getproxies +from urllib.parse import urlparse from tests.lib.path import Path @@ -242,3 +245,28 @@ def warning(self, *args: Any, **kwargs: Any) -> None: actual_level, actual_message = log_records[0] assert actual_level == "WARNING" assert "is not a trusted or secure host" in actual_message + + def test_proxy(self) -> None: + """ + Test proxy. + + """ + + session = PipSession(trusted_hosts=[]) + + # Set or get a valid proxy here. + http_proxy = getproxies().get("http", None) + assert http_proxy is not None, "Define a system proxy" + proxy = urlparse(http_proxy).netloc + + # set proxy scheme to session.proxies + session.proxies = {"http": f"{proxy}", "https": f"{proxy}", "ftp": f"{proxy}"} + + proxy_error = None + try: + session.request("GET", "https://pypi.org", timeout=1) + except requests.exceptions.ConnectionError as e: + proxy_error = e + + assert proxy_error is None, f"Invalid proxy {proxy} or session.proxies: {session.proxies} is not correctly " \ + f"passed to session.request." From 9bc20bf39c89f75226d091a79cb8019ff86c0059 Mon Sep 17 00:00:00 2001 From: Thiago Date: Tue, 8 Feb 2022 14:58:00 +0100 Subject: [PATCH 3/8] Update test_network_session.py Fixing indentation --- tests/unit/test_network_session.py | 35 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/tests/unit/test_network_session.py b/tests/unit/test_network_session.py index b3fc582cab4..2c1cd7e23fc 100644 --- a/tests/unit/test_network_session.py +++ b/tests/unit/test_network_session.py @@ -247,26 +247,25 @@ def warning(self, *args: Any, **kwargs: Any) -> None: assert "is not a trusted or secure host" in actual_message def test_proxy(self) -> None: - """ - Test proxy. - - """ + """ + Test proxy. + """ - session = PipSession(trusted_hosts=[]) + session = PipSession(trusted_hosts=[]) - # Set or get a valid proxy here. - http_proxy = getproxies().get("http", None) - assert http_proxy is not None, "Define a system proxy" - proxy = urlparse(http_proxy).netloc + # Set or get a valid proxy here. + http_proxy = getproxies().get("http", None) + assert http_proxy is not None, "Define a system proxy" + proxy = urlparse(http_proxy).netloc - # set proxy scheme to session.proxies - session.proxies = {"http": f"{proxy}", "https": f"{proxy}", "ftp": f"{proxy}"} + # set proxy scheme to session.proxies + session.proxies = {"http": f"{proxy}", "https": f"{proxy}", "ftp": f"{proxy}"} - proxy_error = None - try: - session.request("GET", "https://pypi.org", timeout=1) - except requests.exceptions.ConnectionError as e: - proxy_error = e + proxy_error = None + try: + session.request("GET", "https://pypi.org", timeout=1) + except requests.exceptions.ConnectionError as e: + proxy_error = e - assert proxy_error is None, f"Invalid proxy {proxy} or session.proxies: {session.proxies} is not correctly " \ - f"passed to session.request." + assert proxy_error is None, f"Invalid proxy {proxy} or session.proxies: {session.proxies} is not correctly " \ + f"passed to session.request." From 20387a6d19b742bf3e9165941addc21029fe81b1 Mon Sep 17 00:00:00 2001 From: junqfisica Date: Sun, 13 Feb 2022 14:35:57 +0100 Subject: [PATCH 4/8] Update test_network_session.py - Added --proxy parameter to pytest --- tests/conftest.py | 11 +++++++++++ tests/unit/test_network_session.py | 26 ++++++++++++++------------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f663b96ea46..e253b9fd9e6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -71,6 +71,12 @@ def pytest_addoption(parser: Parser) -> None: default=False, help="run 'pip search' tests", ) + parser.addoption( + "--proxy", + action="store", + default=None, + help="use given proxy in session network tests", + ) def pytest_collection_modifyitems(config: Config, items: List[pytest.Item]) -> None: @@ -624,3 +630,8 @@ def utc() -> Iterator[None]: tzset() yield tzset() + + +@pytest.fixture +def proxy(request: pytest.FixtureRequest): + return request.config.getoption("proxy") diff --git a/tests/unit/test_network_session.py b/tests/unit/test_network_session.py index 2c1cd7e23fc..c9a83a20e89 100644 --- a/tests/unit/test_network_session.py +++ b/tests/unit/test_network_session.py @@ -245,27 +245,29 @@ def warning(self, *args: Any, **kwargs: Any) -> None: actual_level, actual_message = log_records[0] assert actual_level == "WARNING" assert "is not a trusted or secure host" in actual_message - - def test_proxy(self) -> None: + + @pytest.mark.network + def test_proxy(self, proxy) -> None: """ Test proxy. """ session = PipSession(trusted_hosts=[]) - # Set or get a valid proxy here. - http_proxy = getproxies().get("http", None) - assert http_proxy is not None, "Define a system proxy" - proxy = urlparse(http_proxy).netloc + if not proxy: + # if user didn't pass --proxy then try to get it from the system. + env_proxy = getproxies().get("http", None) + proxy = urlparse(env_proxy).netloc if env_proxy else None - # set proxy scheme to session.proxies - session.proxies = {"http": f"{proxy}", "https": f"{proxy}", "ftp": f"{proxy}"} + if proxy: + # set proxy scheme to session.proxies + session.proxies = {"http": f"{proxy}", "https": f"{proxy}", "ftp": f"{proxy}"} - proxy_error = None + connection_error = None try: session.request("GET", "https://pypi.org", timeout=1) except requests.exceptions.ConnectionError as e: - proxy_error = e + connection_error = e - assert proxy_error is None, f"Invalid proxy {proxy} or session.proxies: {session.proxies} is not correctly " \ - f"passed to session.request." + assert connection_error is None, f"Invalid proxy {proxy} or session.proxies: {session.proxies} " \ + f"is not correctly passed to session.request." From ea941234bbc943a969b312e56e4e21eca7763bbd Mon Sep 17 00:00:00 2001 From: Thiago C Junqueira Date: Mon, 14 Feb 2022 09:37:50 +0100 Subject: [PATCH 5/8] Added news entry. - Reformatted code style in test --- news/9691.bugfix.rst | 1 + tests/conftest.py | 2 +- tests/unit/test_network_session.py | 18 +++++++++++------- 3 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 news/9691.bugfix.rst diff --git a/news/9691.bugfix.rst b/news/9691.bugfix.rst new file mode 100644 index 00000000000..addae89c54b --- /dev/null +++ b/news/9691.bugfix.rst @@ -0,0 +1 @@ +Fix pip install issues using a proxy due to an inconsistency in how Requests is currently handling variable precedence in session. \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index e253b9fd9e6..6860ecf5c95 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -633,5 +633,5 @@ def utc() -> Iterator[None]: @pytest.fixture -def proxy(request: pytest.FixtureRequest): +def proxy(request: pytest.FixtureRequest) -> str: return request.config.getoption("proxy") diff --git a/tests/unit/test_network_session.py b/tests/unit/test_network_session.py index c9a83a20e89..a9bd19ace91 100644 --- a/tests/unit/test_network_session.py +++ b/tests/unit/test_network_session.py @@ -1,14 +1,14 @@ import logging from typing import Any, List +from urllib.request import getproxies +from urllib.parse import urlparse import pytest +from pip._vendor import requests from pip import __version__ from pip._internal.models.link import Link from pip._internal.network.session import CI_ENVIRONMENT_VARIABLES, PipSession -from pip._vendor import requests -from urllib.request import getproxies -from urllib.parse import urlparse from tests.lib.path import Path @@ -247,7 +247,7 @@ def warning(self, *args: Any, **kwargs: Any) -> None: assert "is not a trusted or secure host" in actual_message @pytest.mark.network - def test_proxy(self, proxy) -> None: + def test_proxy(self, proxy: str) -> None: """ Test proxy. """ @@ -261,7 +261,11 @@ def test_proxy(self, proxy) -> None: if proxy: # set proxy scheme to session.proxies - session.proxies = {"http": f"{proxy}", "https": f"{proxy}", "ftp": f"{proxy}"} + session.proxies = { + "http": f"{proxy}", + "https": f"{proxy}", + "ftp": f"{proxy}" + } connection_error = None try: @@ -269,5 +273,5 @@ def test_proxy(self, proxy) -> None: except requests.exceptions.ConnectionError as e: connection_error = e - assert connection_error is None, f"Invalid proxy {proxy} or session.proxies: {session.proxies} " \ - f"is not correctly passed to session.request." + assert connection_error is None, ( + f"Invalid proxy {proxy} or session.proxies: {session.proxies} is not correctly passed to session.request.") From 1375aaac95ad55dedeebab9dc1f79411c3382761 Mon Sep 17 00:00:00 2001 From: Thiago C Junqueira Date: Tue, 15 Feb 2022 10:27:57 +0100 Subject: [PATCH 6/8] Trying to fix the pre-commit errors using the suggested code format. --- news/9691.bugfix.rst | 2 +- tests/unit/test_network_session.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/news/9691.bugfix.rst b/news/9691.bugfix.rst index addae89c54b..6a07d49c83a 100644 --- a/news/9691.bugfix.rst +++ b/news/9691.bugfix.rst @@ -1 +1 @@ -Fix pip install issues using a proxy due to an inconsistency in how Requests is currently handling variable precedence in session. \ No newline at end of file +Fix pip install issues using a proxy due to an inconsistency in how Requests is currently handling variable precedence in session. diff --git a/tests/unit/test_network_session.py b/tests/unit/test_network_session.py index a9bd19ace91..fbefbe8a465 100644 --- a/tests/unit/test_network_session.py +++ b/tests/unit/test_network_session.py @@ -1,7 +1,8 @@ import logging from typing import Any, List -from urllib.request import getproxies + from urllib.parse import urlparse +from urllib.request import getproxies import pytest from pip._vendor import requests @@ -264,7 +265,7 @@ def test_proxy(self, proxy: str) -> None: session.proxies = { "http": f"{proxy}", "https": f"{proxy}", - "ftp": f"{proxy}" + "ftp": f"{proxy}", } connection_error = None @@ -273,5 +274,6 @@ def test_proxy(self, proxy: str) -> None: except requests.exceptions.ConnectionError as e: connection_error = e - assert connection_error is None, ( - f"Invalid proxy {proxy} or session.proxies: {session.proxies} is not correctly passed to session.request.") + assert ( + connection_error is None + ), f"Invalid proxy {proxy} or session.proxies: {session.proxies} is not correctly passed to session.request." From 8ea5fb9bf478ed33a593f04474d48854b7adf032 Mon Sep 17 00:00:00 2001 From: Thiago C Junqueira Date: Thu, 17 Feb 2022 15:20:36 +0100 Subject: [PATCH 7/8] Trying to fix the pre-commit and docs errors using the suggested code format. --- tests/unit/test_network_session.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/unit/test_network_session.py b/tests/unit/test_network_session.py index fbefbe8a465..2cba1f9d063 100644 --- a/tests/unit/test_network_session.py +++ b/tests/unit/test_network_session.py @@ -249,10 +249,6 @@ def warning(self, *args: Any, **kwargs: Any) -> None: @pytest.mark.network def test_proxy(self, proxy: str) -> None: - """ - Test proxy. - """ - session = PipSession(trusted_hosts=[]) if not proxy: From a2302e971db5c80a40041bd6cc72a91727a2e397 Mon Sep 17 00:00:00 2001 From: junqfisica Date: Mon, 21 Feb 2022 18:47:05 +0100 Subject: [PATCH 8/8] Trying to fix the pre-commit errors. --- tests/unit/test_network_session.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_network_session.py b/tests/unit/test_network_session.py index 2cba1f9d063..18eb9539f7f 100644 --- a/tests/unit/test_network_session.py +++ b/tests/unit/test_network_session.py @@ -1,6 +1,5 @@ import logging -from typing import Any, List - +from typing import Any, List, Optional from urllib.parse import urlparse from urllib.request import getproxies @@ -248,7 +247,7 @@ def warning(self, *args: Any, **kwargs: Any) -> None: assert "is not a trusted or secure host" in actual_message @pytest.mark.network - def test_proxy(self, proxy: str) -> None: + def test_proxy(self, proxy: Optional[str]) -> None: session = PipSession(trusted_hosts=[]) if not proxy: @@ -270,6 +269,7 @@ def test_proxy(self, proxy: str) -> None: except requests.exceptions.ConnectionError as e: connection_error = e - assert ( - connection_error is None - ), f"Invalid proxy {proxy} or session.proxies: {session.proxies} is not correctly passed to session.request." + assert connection_error is None, ( + f"Invalid proxy {proxy} or session.proxies: " + f"{session.proxies} is not correctly passed to session.request." + )