From 92f5a5f0f582c3685924960a717e75ca966edf6e Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 21 Sep 2024 09:17:52 +0530 Subject: [PATCH 1/6] TLS intercept conditionally --- proxy/http/proxy/server.py | 37 ++++++++++---------- proxy/plugin/__init__.py | 38 +++++++++++---------- proxy/plugin/tls_intercept_conditionally.py | 21 ++++++++++++ 3 files changed, 59 insertions(+), 37 deletions(-) create mode 100644 proxy/plugin/tls_intercept_conditionally.py diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index 80af1686c0..11e1760948 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -281,10 +281,7 @@ async def read_from_descriptors(self, r: Readables) -> bool: # only for non-https requests and when # tls interception is enabled if raw is not None: - if ( - not self.request.is_https_tunnel - or self.tls_interception_enabled - ): + if not self.request.is_https_tunnel or self._tls_intercept_enabled: if self.response.is_complete: self.handle_pipeline_response(raw) else: @@ -429,8 +426,7 @@ def on_client_data(self, raw: memoryview) -> None: # We also handle pipeline scenario for https proxy # requests is TLS interception is enabled. if self.request.is_complete and ( - not self.request.is_https_tunnel or - self.tls_interception_enabled + not self.request.is_https_tunnel or self._tls_intercept_enabled ): if self.pipeline_request is not None and \ self.pipeline_request.is_connection_upgrade: @@ -474,6 +470,20 @@ def on_client_data(self, raw: memoryview) -> None: else: self.upstream.queue(raw) + @property + def _tls_intercept_enabled(self) -> bool: + do_intercept = self.tls_interception_enabled + if not do_intercept: + return do_intercept + # If enabled by flags, check if a plugin wants us to bypass + # interception for this particular request + for plugin in self.plugins.values(): + do_intercept = plugin.do_intercept(self.request) + # A plugin requested to not intercept the request + if do_intercept is False: + break + return do_intercept + def on_request_complete(self) -> Union[socket.socket, bool]: self.emit_request_complete() @@ -510,19 +520,8 @@ def on_request_complete(self) -> Union[socket.socket, bool]: if self.upstream: if self.request.is_https_tunnel: self.client.queue(PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT) - if self.tls_interception_enabled: - # Check if any plugin wants to - # disable interception even - # with flags available - do_intercept = True - for plugin in self.plugins.values(): - do_intercept = plugin.do_intercept(self.request) - # A plugin requested to not intercept - # the request - if do_intercept is False: - break - if do_intercept: - return self.intercept() + if self._tls_intercept_enabled: + return self.intercept() # If an upstream server connection was established for http request, # queue the request for upstream server. else: diff --git a/proxy/plugin/__init__.py b/proxy/plugin/__init__.py index 74c7e8d4ef..fa00a1a3ee 100644 --- a/proxy/plugin/__init__.py +++ b/proxy/plugin/__init__.py @@ -34,25 +34,27 @@ from .modify_chunk_response import ModifyChunkResponsePlugin from .modify_request_header import ModifyRequestHeaderPlugin from .redirect_to_custom_server import RedirectToCustomServerPlugin +from .tls_intercept_conditionally import TlsInterceptConditionallyPlugin __all__ = [ - 'CacheResponsesPlugin', - 'BaseCacheResponsesPlugin', - 'FilterByUpstreamHostPlugin', - 'ManInTheMiddlePlugin', - 'ProposedRestApiPlugin', - 'ModifyPostDataPlugin', - 'RedirectToCustomServerPlugin', - 'ShortLinkPlugin', - 'WebServerPlugin', - 'ReverseProxyPlugin', - 'ProxyPoolPlugin', - 'FilterByClientIpPlugin', - 'ModifyChunkResponsePlugin', - 'FilterByURLRegexPlugin', - 'CustomDnsResolverPlugin', - 'CloudflareDnsResolverPlugin', - 'ProgramNamePlugin', - 'ModifyRequestHeaderPlugin', + "CacheResponsesPlugin", + "BaseCacheResponsesPlugin", + "FilterByUpstreamHostPlugin", + "ManInTheMiddlePlugin", + "ProposedRestApiPlugin", + "ModifyPostDataPlugin", + "RedirectToCustomServerPlugin", + "ShortLinkPlugin", + "WebServerPlugin", + "ReverseProxyPlugin", + "ProxyPoolPlugin", + "FilterByClientIpPlugin", + "ModifyChunkResponsePlugin", + "FilterByURLRegexPlugin", + "CustomDnsResolverPlugin", + "CloudflareDnsResolverPlugin", + "ProgramNamePlugin", + "ModifyRequestHeaderPlugin", + "TlsInterceptConditionallyPlugin", ] diff --git a/proxy/plugin/tls_intercept_conditionally.py b/proxy/plugin/tls_intercept_conditionally.py new file mode 100644 index 0000000000..7fa2003097 --- /dev/null +++ b/proxy/plugin/tls_intercept_conditionally.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +from ..http.proxy import HttpProxyBasePlugin +from ..http.parser import HttpParser + + +class TlsInterceptConditionallyPlugin(HttpProxyBasePlugin): + """TLS intercept conditionally.""" + + def do_intercept(self, request: HttpParser) -> bool: + if request.host == b"httpbin.org": + return False + return super().do_intercept(request) From 2c6808a15dedf8f3cfa73e2e185ffed8fa533a76 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 21 Sep 2024 09:18:14 +0530 Subject: [PATCH 2/6] Lint fixes --- proxy/plugin/__init__.py | 38 ++++++++++----------- proxy/plugin/tls_intercept_conditionally.py | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/proxy/plugin/__init__.py b/proxy/plugin/__init__.py index fa00a1a3ee..48b7996d56 100644 --- a/proxy/plugin/__init__.py +++ b/proxy/plugin/__init__.py @@ -38,23 +38,23 @@ __all__ = [ - "CacheResponsesPlugin", - "BaseCacheResponsesPlugin", - "FilterByUpstreamHostPlugin", - "ManInTheMiddlePlugin", - "ProposedRestApiPlugin", - "ModifyPostDataPlugin", - "RedirectToCustomServerPlugin", - "ShortLinkPlugin", - "WebServerPlugin", - "ReverseProxyPlugin", - "ProxyPoolPlugin", - "FilterByClientIpPlugin", - "ModifyChunkResponsePlugin", - "FilterByURLRegexPlugin", - "CustomDnsResolverPlugin", - "CloudflareDnsResolverPlugin", - "ProgramNamePlugin", - "ModifyRequestHeaderPlugin", - "TlsInterceptConditionallyPlugin", + 'CacheResponsesPlugin', + 'BaseCacheResponsesPlugin', + 'FilterByUpstreamHostPlugin', + 'ManInTheMiddlePlugin', + 'ProposedRestApiPlugin', + 'ModifyPostDataPlugin', + 'RedirectToCustomServerPlugin', + 'ShortLinkPlugin', + 'WebServerPlugin', + 'ReverseProxyPlugin', + 'ProxyPoolPlugin', + 'FilterByClientIpPlugin', + 'ModifyChunkResponsePlugin', + 'FilterByURLRegexPlugin', + 'CustomDnsResolverPlugin', + 'CloudflareDnsResolverPlugin', + 'ProgramNamePlugin', + 'ModifyRequestHeaderPlugin', + 'TlsInterceptConditionallyPlugin', ] diff --git a/proxy/plugin/tls_intercept_conditionally.py b/proxy/plugin/tls_intercept_conditionally.py index 7fa2003097..58f992e4c9 100644 --- a/proxy/plugin/tls_intercept_conditionally.py +++ b/proxy/plugin/tls_intercept_conditionally.py @@ -16,6 +16,6 @@ class TlsInterceptConditionallyPlugin(HttpProxyBasePlugin): """TLS intercept conditionally.""" def do_intercept(self, request: HttpParser) -> bool: - if request.host == b"httpbin.org": + if request.host == b'httpbin.org': return False return super().do_intercept(request) From d6e220fcd3e9cfebad1a0615e46eb821b2d75602 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 21 Sep 2024 09:34:12 +0530 Subject: [PATCH 3/6] Fix test assertions --- tests/http/proxy/test_http_proxy_tls_interception.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/http/proxy/test_http_proxy_tls_interception.py b/tests/http/proxy/test_http_proxy_tls_interception.py index fad6eff189..497d80af90 100644 --- a/tests/http/proxy/test_http_proxy_tls_interception.py +++ b/tests/http/proxy/test_http_proxy_tls_interception.py @@ -204,7 +204,7 @@ async def asyncReturn(val: T) -> T: ) self.proxy_plugin.return_value.before_upstream_connection.assert_called() self.proxy_plugin.return_value.handle_client_request.assert_called_once() - self.proxy_plugin.return_value.do_intercept.assert_called_once() + self.assertEqual(self.proxy_plugin.return_value.do_intercept.call_count, 2) # All the invoked lifecycle callbacks will receive the CONNECT request # packet with / as the path callback_request: HttpParser = \ @@ -297,7 +297,7 @@ async def asyncReturn(val: T) -> T: self.proxy_plugin.return_value.handle_client_request.call_count, 2, ) - self.proxy_plugin.return_value.do_intercept.assert_called_once() + self.assertEqual(self.proxy_plugin.return_value.do_intercept.call_count, 2) callback_request = \ self.proxy_plugin.return_value.handle_client_request.call_args_list[1][0][0] From 38a877634617049b7501d23f99ffd4766e953c5f Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 21 Sep 2024 10:08:07 +0530 Subject: [PATCH 4/6] Fix tests --- tests/http/proxy/test_http_proxy_tls_interception.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/http/proxy/test_http_proxy_tls_interception.py b/tests/http/proxy/test_http_proxy_tls_interception.py index 497d80af90..4ee8246d1e 100644 --- a/tests/http/proxy/test_http_proxy_tls_interception.py +++ b/tests/http/proxy/test_http_proxy_tls_interception.py @@ -204,7 +204,7 @@ async def asyncReturn(val: T) -> T: ) self.proxy_plugin.return_value.before_upstream_connection.assert_called() self.proxy_plugin.return_value.handle_client_request.assert_called_once() - self.assertEqual(self.proxy_plugin.return_value.do_intercept.call_count, 2) + self.proxy_plugin.return_value.do_intercept.assert_called_once() # All the invoked lifecycle callbacks will receive the CONNECT request # packet with / as the path callback_request: HttpParser = \ From 2dbcf33830d54bb4e8aca0b36d19ddfdd1fd3ce5 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 21 Sep 2024 10:36:14 +0530 Subject: [PATCH 5/6] Bump `sphinxcontrib-spelling == 8.0.0` due to deprecated pypi XML APIs, ref https://github.com/pypi/warehouse/issues/16642 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 80a5e4615a..f660260f2c 100644 --- a/tox.ini +++ b/tox.ini @@ -142,7 +142,7 @@ commands = . "{toxworkdir}/docs_out" changedir = {[testenv:build-docs]changedir} deps = - sphinxcontrib-spelling == 7.7.0 + sphinxcontrib-spelling == 8.0.0 -r{toxinidir}/docs/requirements.in description = Spellcheck The Docs isolated_build = {[testenv:build-docs]isolated_build} From 9c97cca436e9d733acd19e705fc233fe7eb67822 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Sat, 21 Sep 2024 10:44:53 +0530 Subject: [PATCH 6/6] Disable spellcheck-docs for now until we have a clear path forward --- .github/workflows/test-library.yml | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 0857fd1111..38375b4004 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -340,7 +340,7 @@ jobs: - build-docs - doctest-docs - linkcheck-docs - - spellcheck-docs + # - spellcheck-docs fail-fast: false env: diff --git a/tox.ini b/tox.ini index f660260f2c..80a5e4615a 100644 --- a/tox.ini +++ b/tox.ini @@ -142,7 +142,7 @@ commands = . "{toxworkdir}/docs_out" changedir = {[testenv:build-docs]changedir} deps = - sphinxcontrib-spelling == 8.0.0 + sphinxcontrib-spelling == 7.7.0 -r{toxinidir}/docs/requirements.in description = Spellcheck The Docs isolated_build = {[testenv:build-docs]isolated_build}