From 381c4de9ac41740470e7df3af8510d8a996a7e27 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Mon, 8 Nov 2021 19:35:00 +0530 Subject: [PATCH 01/13] Expose within __all__ --- proxy/__init__.py | 2 +- proxy/common/constants.py | 4 ++-- proxy/core/ssh/__init__.py | 7 +++++++ proxy/dashboard/__init__.py | 9 +++++++++ proxy/http/handler.py | 5 ++++- proxy/testing/__init__.py | 5 +++++ tests/test_main.py | 18 ++++++++++-------- 7 files changed, 38 insertions(+), 12 deletions(-) diff --git a/proxy/__init__.py b/proxy/__init__.py index 16cacc417d..a2e0fa77ad 100755 --- a/proxy/__init__.py +++ b/proxy/__init__.py @@ -9,7 +9,7 @@ :license: BSD, see LICENSE for more details. """ from .proxy import entry_point, main, Proxy -from .testing.test_case import TestCase +from .testing import TestCase __all__ = [ # PyPi package entry_point. See diff --git a/proxy/common/constants.py b/proxy/common/constants.py index 43c449be0f..d1e5b6b2d4 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -104,8 +104,8 @@ PLUGIN_WEB_SERVER = 'proxy.http.server.HttpWebServerPlugin' PLUGIN_PAC_FILE = 'proxy.http.server.HttpWebServerPacFilePlugin' PLUGIN_DEVTOOLS_PROTOCOL = 'proxy.http.inspector.DevtoolsProtocolPlugin' -PLUGIN_DASHBOARD = 'proxy.dashboard.dashboard.ProxyDashboard' -PLUGIN_INSPECT_TRAFFIC = 'proxy.dashboard.inspect_traffic.InspectTrafficPlugin' +PLUGIN_DASHBOARD = 'proxy.dashboard.ProxyDashboard' +PLUGIN_INSPECT_TRAFFIC = 'proxy.dashboard.InspectTrafficPlugin' PLUGIN_PROXY_AUTH = 'proxy.http.proxy.AuthPlugin' PY2_DEPRECATION_MESSAGE = '''DEPRECATION: proxy.py no longer supports Python 2.7. Kindly upgrade to Python 3+. ' diff --git a/proxy/core/ssh/__init__.py b/proxy/core/ssh/__init__.py index 232621f0b5..1a1f602379 100644 --- a/proxy/core/ssh/__init__.py +++ b/proxy/core/ssh/__init__.py @@ -8,3 +8,10 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +from .client import SshClient +from .tunnel import Tunnel + +__all__ = [ + 'SshClient', + 'Tunnel' +] diff --git a/proxy/dashboard/__init__.py b/proxy/dashboard/__init__.py index 232621f0b5..baa18675fd 100644 --- a/proxy/dashboard/__init__.py +++ b/proxy/dashboard/__init__.py @@ -8,3 +8,12 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +from .dashboard import ProxyDashboard +from .inspect_traffic import InspectTrafficPlugin +from .plugin import ProxyDashboardWebsocketPlugin + +__all__ = [ + 'ProxyDashboard', + 'InspectTrafficPlugin', + 'ProxyDashboardWebsocketPlugin' +] diff --git a/proxy/http/handler.py b/proxy/http/handler.py index 9154f9f59d..446dde3a34 100644 --- a/proxy/http/handler.py +++ b/proxy/http/handler.py @@ -286,7 +286,10 @@ def handle_readables(self, readables: Readables) -> bool: return False except socket.error as e: if e.errno == errno.ECONNRESET: - logger.warning('%r' % e) + # Most requests for mobile devices will end up + # with client closed connection. Using `debug` + # here to avoid flooding the logs. + logger.debug('%r' % e) else: logger.exception( 'Exception while receiving from %s connection %r with reason %r' % diff --git a/proxy/testing/__init__.py b/proxy/testing/__init__.py index 232621f0b5..e841545b30 100644 --- a/proxy/testing/__init__.py +++ b/proxy/testing/__init__.py @@ -8,3 +8,8 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +from .test_case import TestCase + +__all__ = [ + 'TestCase', +] diff --git a/tests/test_main.py b/tests/test_main.py index eb951fbcf6..d0357255fc 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -27,6 +27,8 @@ from proxy.common.constants import DEFAULT_PAC_FILE, DEFAULT_PLUGINS, DEFAULT_PID_FILE, DEFAULT_PORT, DEFAULT_BASIC_AUTH from proxy.common.constants import DEFAULT_NUM_WORKERS, DEFAULT_OPEN_FILE_LIMIT, DEFAULT_IPV6_HOSTNAME from proxy.common.constants import DEFAULT_SERVER_RECVBUF_SIZE, DEFAULT_CLIENT_RECVBUF_SIZE, PY2_DEPRECATION_MESSAGE +from proxy.common.constants import PLUGIN_INSPECT_TRAFFIC, PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL, PLUGIN_WEB_SERVER +from proxy.common.constants import PLUGIN_HTTP_PROXY from proxy.common.version import __version__ @@ -163,11 +165,11 @@ def test_enable_dashboard( mock_load_plugins.assert_called() self.assertEqual( mock_load_plugins.call_args_list[0][0][0], [ - b'proxy.http.server.HttpWebServerPlugin', - b'proxy.dashboard.dashboard.ProxyDashboard', - b'proxy.dashboard.inspect_traffic.InspectTrafficPlugin', - b'proxy.http.inspector.DevtoolsProtocolPlugin', - b'proxy.http.proxy.HttpProxyPlugin', + bytes_(PLUGIN_WEB_SERVER), + bytes_(PLUGIN_DASHBOARD), + bytes_(PLUGIN_INSPECT_TRAFFIC), + bytes_(PLUGIN_DEVTOOLS_PROTOCOL), + bytes_(PLUGIN_HTTP_PROXY), ], ) mock_parse_args.assert_called_once() @@ -199,9 +201,9 @@ def test_enable_devtools( mock_load_plugins.assert_called() self.assertEqual( mock_load_plugins.call_args_list[0][0][0], [ - b'proxy.http.inspector.DevtoolsProtocolPlugin', - b'proxy.http.server.HttpWebServerPlugin', - b'proxy.http.proxy.HttpProxyPlugin', + bytes_(PLUGIN_DEVTOOLS_PROTOCOL), + bytes_(PLUGIN_WEB_SERVER), + bytes_(PLUGIN_HTTP_PROXY), ], ) mock_parse_args.assert_called_once() From e04934343c8d412feb63923bbf69e4e91f28ba69 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Mon, 8 Nov 2021 19:41:04 +0530 Subject: [PATCH 02/13] Enable `--numprocesses=auto` for `pytest.ini` --- pytest.ini | 2 +- tests/http/test_web_server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index 650a369303..4db289ec2b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,7 +2,7 @@ addopts = # FIXME: Enable this once the test suite has no race conditions # `pytest-xdist`: - # --numprocesses=auto + --numprocesses=auto # Show 10 slowest invocations: --durations=10 diff --git a/tests/http/test_web_server.py b/tests/http/test_web_server.py index 6333481f34..2f276d3a0e 100644 --- a/tests/http/test_web_server.py +++ b/tests/http/test_web_server.py @@ -137,7 +137,7 @@ def test_default_web_server_returns_404( ) @unittest.skipIf( - os.environ.get('GITHUB_ACTIONS', True), + os.environ.get('GITHUB_ACTIONS', "false") == "true", 'Disabled on GitHub actions because this test is flaky on GitHub infrastructure.', ) @mock.patch('selectors.DefaultSelector') From 4198852b2355aa592f8ac5381bc6a2f3dd1390a7 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Mon, 8 Nov 2021 19:42:26 +0530 Subject: [PATCH 03/13] make lib-lint --- proxy/core/ssh/__init__.py | 2 +- proxy/dashboard/__init__.py | 2 +- tests/http/test_web_server.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/proxy/core/ssh/__init__.py b/proxy/core/ssh/__init__.py index 1a1f602379..f793bc839a 100644 --- a/proxy/core/ssh/__init__.py +++ b/proxy/core/ssh/__init__.py @@ -13,5 +13,5 @@ __all__ = [ 'SshClient', - 'Tunnel' + 'Tunnel', ] diff --git a/proxy/dashboard/__init__.py b/proxy/dashboard/__init__.py index baa18675fd..0f5d329522 100644 --- a/proxy/dashboard/__init__.py +++ b/proxy/dashboard/__init__.py @@ -15,5 +15,5 @@ __all__ = [ 'ProxyDashboard', 'InspectTrafficPlugin', - 'ProxyDashboardWebsocketPlugin' + 'ProxyDashboardWebsocketPlugin', ] diff --git a/tests/http/test_web_server.py b/tests/http/test_web_server.py index 2f276d3a0e..ebd563d8c7 100644 --- a/tests/http/test_web_server.py +++ b/tests/http/test_web_server.py @@ -137,7 +137,7 @@ def test_default_web_server_returns_404( ) @unittest.skipIf( - os.environ.get('GITHUB_ACTIONS', "false") == "true", + os.environ.get('GITHUB_ACTIONS', 'false') == 'true', 'Disabled on GitHub actions because this test is flaky on GitHub infrastructure.', ) @mock.patch('selectors.DefaultSelector') From d9f4a4d3d5205cb9a249e9b960d351026acc8675 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Mon, 8 Nov 2021 19:53:22 +0530 Subject: [PATCH 04/13] Also consider `--plugins` flag when bootstrapping plugins --- proxy/common/flag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/common/flag.py b/proxy/common/flag.py index 33f4021db3..2b7cb2d41c 100644 --- a/proxy/common/flag.py +++ b/proxy/common/flag.py @@ -96,7 +96,7 @@ def initialize( # Discover flags from requested plugin. # This also surface external plugin flags under --help for i, f in enumerate(input_args): - if f == '--plugin': + if f in ('--plugin', '--plugins'): import_plugin(bytes_(input_args[i + 1])) # Parse flags From 8ed712fecb7c977ebe99332c3f5279c25adb6403 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Mon, 8 Nov 2021 20:07:16 +0530 Subject: [PATCH 05/13] Add `from .dashboard import ProxyDashboard` in top-level `__init__.py` to make `ProxyDashboard` flags auto discoverable --- proxy/__init__.py | 9 +++++++++ proxy/common/constants.py | 6 ++++++ proxy/common/utils.py | 18 +++++++++--------- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/proxy/__init__.py b/proxy/__init__.py index a2e0fa77ad..ba07c44a0b 100755 --- a/proxy/__init__.py +++ b/proxy/__init__.py @@ -10,6 +10,7 @@ """ from .proxy import entry_point, main, Proxy from .testing import TestCase +from .dashboard import ProxyDashboard __all__ = [ # PyPi package entry_point. See @@ -22,4 +23,12 @@ # https://github.com/abhinavsingh/proxy.py#unit-testing-with-proxypy 'TestCase', 'Proxy', + # This is here to make sure --enable-dashboard + # flag is discoverable by automagically. + # + # Because, ProxyDashboard is not imported anywhere, + # without this patch, users will have to explicitly + # enable proxy.dashboard.ProxyDashboard plugin + # to use --enable-dashboard flag. + 'ProxyDashboard', ] diff --git a/proxy/common/constants.py b/proxy/common/constants.py index d1e5b6b2d4..738594036c 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -100,6 +100,12 @@ DEFAULT_DATA_DIRECTORY_PATH = os.path.join(str(pathlib.Path.home()), '.proxy') # Cor plugins enabled by default or via flags +DEFAULT_ABC_PLUGINS = [ + 'HttpProtocolHandlerPlugin', + 'HttpProxyBasePlugin', + 'HttpWebServerBasePlugin', + 'ProxyDashboardWebsocketPlugin', +] PLUGIN_HTTP_PROXY = 'proxy.http.proxy.HttpProxyPlugin' PLUGIN_WEB_SERVER = 'proxy.http.server.HttpWebServerPlugin' PLUGIN_PAC_FILE = 'proxy.http.server.HttpWebServerPacFilePlugin' diff --git a/proxy/common/utils.py b/proxy/common/utils.py index bcbc872a33..aea42954aa 100644 --- a/proxy/common/utils.py +++ b/proxy/common/utils.py @@ -25,7 +25,7 @@ from .constants import HTTP_1_1, COLON, WHITESPACE, CRLF, DEFAULT_TIMEOUT from .constants import DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL -from .constants import DOT +from .constants import DOT, DEFAULT_ABC_PLUGINS if os.name != 'nt': import resource @@ -291,15 +291,15 @@ def setup_logger( def load_plugins( plugins: List[Union[bytes, type]], + abc_plugins: List[bytes] = DEFAULT_ABC_PLUGINS, ) -> Dict[bytes, List[type]]: - """Accepts a comma separated list of Python modules and returns - a list of respective Python classes.""" - p: Dict[bytes, List[type]] = { - b'HttpProtocolHandlerPlugin': [], - b'HttpProxyBasePlugin': [], - b'HttpWebServerBasePlugin': [], - b'ProxyDashboardWebsocketPlugin': [], - } + """Accepts a list Python modules, scans them to identify + if they are an implementation of abstract plugin classes and + returns a dictionary of matching plugins for each abstract class. + """ + p: Dict[bytes, List[type]] = {} + for abc_plugin in abc_plugins: + p[bytes_(abc_plugin)] = [] for plugin_ in plugins: klass, module_name = import_plugin(plugin_) assert klass and module_name From a5bb7ac3a397dba2ec0dd2dc2c07b008fd3f2c75 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Mon, 8 Nov 2021 20:16:42 +0530 Subject: [PATCH 06/13] Move `--enable-dashboard` to top-level --- proxy/__init__.py | 9 --------- proxy/dashboard/dashboard.py | 10 ---------- proxy/proxy.py | 15 +++++++++++++++ 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/proxy/__init__.py b/proxy/__init__.py index ba07c44a0b..a2e0fa77ad 100755 --- a/proxy/__init__.py +++ b/proxy/__init__.py @@ -10,7 +10,6 @@ """ from .proxy import entry_point, main, Proxy from .testing import TestCase -from .dashboard import ProxyDashboard __all__ = [ # PyPi package entry_point. See @@ -23,12 +22,4 @@ # https://github.com/abhinavsingh/proxy.py#unit-testing-with-proxypy 'TestCase', 'Proxy', - # This is here to make sure --enable-dashboard - # flag is discoverable by automagically. - # - # Because, ProxyDashboard is not imported anywhere, - # without this patch, users will have to explicitly - # enable proxy.dashboard.ProxyDashboard plugin - # to use --enable-dashboard flag. - 'ProxyDashboard', ] diff --git a/proxy/dashboard/dashboard.py b/proxy/dashboard/dashboard.py index 49ae0f2cfd..57fb5aa81c 100644 --- a/proxy/dashboard/dashboard.py +++ b/proxy/dashboard/dashboard.py @@ -14,9 +14,7 @@ from .plugin import ProxyDashboardWebsocketPlugin -from ..common.flag import flags from ..common.utils import build_http_response, bytes_ -from ..common.constants import DEFAULT_ENABLE_DASHBOARD from ..http.server import HttpWebServerPlugin, HttpWebServerBasePlugin, httpProtocolTypes from ..http.parser import HttpParser from ..http.websocket import WebsocketFrame @@ -25,14 +23,6 @@ logger = logging.getLogger(__name__) -flags.add_argument( - '--enable-dashboard', - action='store_true', - default=DEFAULT_ENABLE_DASHBOARD, - help='Default: False. Enables proxy.py dashboard.', -) - - class ProxyDashboard(HttpWebServerBasePlugin): """Proxy Dashboard.""" diff --git a/proxy/proxy.py b/proxy/proxy.py index 99956d45ae..7059fd2e78 100644 --- a/proxy/proxy.py +++ b/proxy/proxy.py @@ -25,6 +25,7 @@ from .common.flag import FlagParser, flags from .common.constants import DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL from .common.constants import DEFAULT_OPEN_FILE_LIMIT, DEFAULT_PLUGINS, DEFAULT_VERSION +from .common.constants import DEFAULT_ENABLE_DASHBOARD logger = logging.getLogger(__name__) @@ -76,6 +77,20 @@ help='Comma separated plugins', ) +# TODO: Ideally all `--enable-*` flags must be at the top-level. +# --enable-dashboard is specially needed here because +# ProxyDashboard class is not imported by anyone. +# +# If we move this flag definition within dashboard.py, +# users will also have to explicitly enable dashboard plugin +# to also use flags provided by it. +flags.add_argument( + '--enable-dashboard', + action='store_true', + default=DEFAULT_ENABLE_DASHBOARD, + help='Default: False. Enables proxy.py dashboard.', +) + class Proxy: """Context manager to control core AcceptorPool server lifecycle. From dd36d79beeb11fc89f4cd47960e17ab629b86cf9 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Mon, 8 Nov 2021 20:33:30 +0530 Subject: [PATCH 07/13] Move logging utility within `Logger` class --- proxy/common/flag.py | 11 +++++--- proxy/common/logger.py | 44 +++++++++++++++++++++++++++++++ proxy/common/utils.py | 27 ------------------- proxy/core/acceptor/acceptor.py | 4 +-- proxy/core/acceptor/threadless.py | 4 +-- 5 files changed, 56 insertions(+), 34 deletions(-) create mode 100644 proxy/common/logger.py diff --git a/proxy/common/flag.py b/proxy/common/flag.py index 2b7cb2d41c..564e0e15e9 100644 --- a/proxy/common/flag.py +++ b/proxy/common/flag.py @@ -20,13 +20,14 @@ from typing import Optional, List, Any, cast from .types import IpAddress -from .utils import text_, bytes_, setup_logger, is_py2, set_open_file_limit +from .utils import text_, bytes_, is_py2, set_open_file_limit from .utils import import_plugin, load_plugins from .constants import COMMA, DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_NUM_WORKERS from .constants import DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DISABLE_HEADERS, PY2_DEPRECATION_MESSAGE from .constants import PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL from .constants import PLUGIN_HTTP_PROXY, PLUGIN_INSPECT_TRAFFIC, PLUGIN_PAC_FILE from .constants import PLUGIN_WEB_SERVER, PLUGIN_PROXY_AUTH +from .logger import Logger from .version import __version__ @@ -94,7 +95,11 @@ def initialize( sys.exit(1) # Discover flags from requested plugin. - # This also surface external plugin flags under --help + # This will also surface external plugin flags + # under --help. + # + # TODO: --plugin(s) can either be a comma separated + # list of plugins or could be a list of plugins. for i, f in enumerate(input_args): if f in ('--plugin', '--plugins'): import_plugin(bytes_(input_args[i + 1])) @@ -108,7 +113,7 @@ def initialize( sys.exit(0) # Setup logging module - setup_logger(args.log_file, args.log_level, args.log_format) + Logger.setup_logger(args.log_file, args.log_level, args.log_format) # Setup limits set_open_file_limit(args.open_file_limit) diff --git a/proxy/common/logger.py b/proxy/common/logger.py new file mode 100644 index 0000000000..e872ef18c9 --- /dev/null +++ b/proxy/common/logger.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import logging + +from typing import Optional + +from .constants import DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL + +SINGLE_CHAR_TO_LEVEL = { + 'D': 'DEBUG', + 'I': 'INFO', + 'W': 'WARNING', + 'E': 'ERROR', + 'C': 'CRITICAL', +} + + +class Logger: + """Common logging utilities and setup.""" + + @staticmethod + def setup_logger( + log_file: Optional[str] = DEFAULT_LOG_FILE, + log_level: str = DEFAULT_LOG_LEVEL, + log_format: str = DEFAULT_LOG_FORMAT, + ) -> None: + ll = getattr(logging, SINGLE_CHAR_TO_LEVEL[log_level.upper()[0]]) + if log_file: + logging.basicConfig( + filename=log_file, + filemode='a', + level=ll, + format=log_format, + ) + else: + logging.basicConfig(level=ll, format=log_format) diff --git a/proxy/common/utils.py b/proxy/common/utils.py index aea42954aa..e60bc580be 100644 --- a/proxy/common/utils.py +++ b/proxy/common/utils.py @@ -24,7 +24,6 @@ from typing import Optional, Dict, Any, List, Tuple, Type, Callable, Union from .constants import HTTP_1_1, COLON, WHITESPACE, CRLF, DEFAULT_TIMEOUT -from .constants import DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL from .constants import DOT, DEFAULT_ABC_PLUGINS if os.name != 'nt': @@ -263,32 +262,6 @@ def get_available_port() -> int: return int(port) -def setup_logger( - log_file: Optional[str] = DEFAULT_LOG_FILE, - log_level: str = DEFAULT_LOG_LEVEL, - log_format: str = DEFAULT_LOG_FORMAT, -) -> None: - ll = getattr( - logging, - { - 'D': 'DEBUG', - 'I': 'INFO', - 'W': 'WARNING', - 'E': 'ERROR', - 'C': 'CRITICAL', - }[log_level.upper()[0]], - ) - if log_file: - logging.basicConfig( - filename=log_file, - filemode='a', - level=ll, - format=log_format, - ) - else: - logging.basicConfig(level=ll, format=log_format) - - def load_plugins( plugins: List[Union[bytes, type]], abc_plugins: List[bytes] = DEFAULT_ABC_PLUGINS, diff --git a/proxy/core/acceptor/acceptor.py b/proxy/core/acceptor/acceptor.py index 02ae41473f..cfa8045fc3 100644 --- a/proxy/core/acceptor/acceptor.py +++ b/proxy/core/acceptor/acceptor.py @@ -27,7 +27,7 @@ from ..event import EventQueue, eventNames from ...common.constants import DEFAULT_THREADLESS from ...common.flag import flags -from ...common.utils import setup_logger +from ...common.logger import Logger logger = logging.getLogger(__name__) @@ -159,7 +159,7 @@ def run_once(self) -> None: self._start_threaded_work(conn, addr) def run(self) -> None: - setup_logger( + Logger.setup_logger( self.flags.log_file, self.flags.log_level, self.flags.log_format, ) diff --git a/proxy/core/acceptor/threadless.py b/proxy/core/acceptor/threadless.py index 3cdd323cd7..7a957f4180 100644 --- a/proxy/core/acceptor/threadless.py +++ b/proxy/core/acceptor/threadless.py @@ -26,7 +26,7 @@ from ..connection import TcpClientConnection from ..event import EventQueue, eventNames -from ...common.utils import setup_logger +from ...common.logger import Logger from ...common.types import Readables, Writables from ...common.constants import DEFAULT_TIMEOUT @@ -204,7 +204,7 @@ def run_once(self) -> None: self.cleanup_inactive() def run(self) -> None: - setup_logger( + Logger.setup_logger( self.flags.log_file, self.flags.log_level, self.flags.log_format, ) From 01f569a89a812b6c215e46daadae1f7ef0919d9e Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Mon, 8 Nov 2021 20:40:54 +0530 Subject: [PATCH 08/13] Consider comma separated --plugin and --plugins during discover_plugins --- proxy/common/flag.py | 9 ++------- proxy/common/utils.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/proxy/common/flag.py b/proxy/common/flag.py index 564e0e15e9..bb359a7cc1 100644 --- a/proxy/common/flag.py +++ b/proxy/common/flag.py @@ -21,7 +21,7 @@ from .types import IpAddress from .utils import text_, bytes_, is_py2, set_open_file_limit -from .utils import import_plugin, load_plugins +from .utils import discover_plugins, load_plugins from .constants import COMMA, DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_NUM_WORKERS from .constants import DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DISABLE_HEADERS, PY2_DEPRECATION_MESSAGE from .constants import PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL @@ -97,12 +97,7 @@ def initialize( # Discover flags from requested plugin. # This will also surface external plugin flags # under --help. - # - # TODO: --plugin(s) can either be a comma separated - # list of plugins or could be a list of plugins. - for i, f in enumerate(input_args): - if f in ('--plugin', '--plugins'): - import_plugin(bytes_(input_args[i + 1])) + discover_plugins(input_args) # Parse flags args = flags.parse_args(input_args) diff --git a/proxy/common/utils.py b/proxy/common/utils.py index e60bc580be..f6e3d55507 100644 --- a/proxy/common/utils.py +++ b/proxy/common/utils.py @@ -262,6 +262,18 @@ def get_available_port() -> int: return int(port) +def discover_plugins(input_args: Optional[List[str]]) -> None: + for i, f in enumerate(input_args): + if f in ('--plugin', '--plugins'): + v = input_args[i + 1] + if type(v) == str: + parts = v.split(',') + for part in parts: + import_plugin(bytes_(part)) + else: + import_plugin(bytes_(v)) + + def load_plugins( plugins: List[Union[bytes, type]], abc_plugins: List[bytes] = DEFAULT_ABC_PLUGINS, From 4e8e01cd60ea3278fc2f3d1e88e0d8f7baa5f7dd Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Mon, 8 Nov 2021 20:55:17 +0530 Subject: [PATCH 09/13] Refactor plugin related utilities in Plugins module --- proxy/common/flag.py | 6 +-- proxy/common/plugins.py | 81 +++++++++++++++++++++++++++++ proxy/common/utils.py | 61 ---------------------- tests/http/test_protocol_handler.py | 11 ++-- tests/http/test_web_server.py | 13 ++--- tests/test_main.py | 4 +- 6 files changed, 99 insertions(+), 77 deletions(-) create mode 100644 proxy/common/plugins.py diff --git a/proxy/common/flag.py b/proxy/common/flag.py index bb359a7cc1..aace87c70e 100644 --- a/proxy/common/flag.py +++ b/proxy/common/flag.py @@ -19,9 +19,9 @@ from typing import Optional, List, Any, cast +from .plugins import Plugins from .types import IpAddress from .utils import text_, bytes_, is_py2, set_open_file_limit -from .utils import discover_plugins, load_plugins from .constants import COMMA, DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_NUM_WORKERS from .constants import DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DISABLE_HEADERS, PY2_DEPRECATION_MESSAGE from .constants import PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL @@ -97,7 +97,7 @@ def initialize( # Discover flags from requested plugin. # This will also surface external plugin flags # under --help. - discover_plugins(input_args) + Plugins.discover(input_args) # Parse flags args = flags.parse_args(input_args) @@ -125,7 +125,7 @@ def initialize( ] # Load default plugins along with user provided --plugins - plugins = load_plugins(default_plugins + extra_plugins) + plugins = Plugins.load(default_plugins + extra_plugins) # proxy.py currently cannot serve over HTTPS and also perform TLS interception # at the same time. Check if user is trying to enable both feature diff --git a/proxy/common/plugins.py b/proxy/common/plugins.py new file mode 100644 index 0000000000..901a991482 --- /dev/null +++ b/proxy/common/plugins.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import os +import abc +import logging +import inspect +import importlib + +from typing import Any, List, Optional, Dict, Union + +from .utils import bytes_, text_ +from .constants import DOT, DEFAULT_ABC_PLUGINS + +logger = logging.getLogger(__name__) + + +class Plugins: + + @staticmethod + def discover(input_args: Optional[List[str]]) -> None: + """Search for plugin and plugins flag in command line arguments, + then iterates over each value and discovers the plugin. + """ + for i, f in enumerate(input_args): + if f in ('--plugin', '--plugins'): + v = input_args[i + 1] + parts = v.split(',') + for part in parts: + Plugins.importer(bytes_(part)) + + @staticmethod + def load( + plugins: List[Union[bytes, type]], + abc_plugins: List[bytes] = DEFAULT_ABC_PLUGINS, + ) -> Dict[bytes, List[type]]: + """Accepts a list Python modules, scans them to identify + if they are an implementation of abstract plugin classes and + returns a dictionary of matching plugins for each abstract class. + """ + p: Dict[bytes, List[type]] = {} + for abc_plugin in abc_plugins: + p[bytes_(abc_plugin)] = [] + for plugin_ in plugins: + klass, module_name = Plugins.importer(plugin_) + assert klass and module_name + mro = list(inspect.getmro(klass)) + mro.reverse() + iterator = iter(mro) + while next(iterator) is not abc.ABC: + pass + base_klass = next(iterator) + if klass not in p[bytes_(base_klass.__name__)]: + p[bytes_(base_klass.__name__)].append(klass) + logger.info('Loaded plugin %s.%s', module_name, klass.__name__) + return p + + @staticmethod + def importer(plugin: Union[bytes, type]) -> Any: + """Import and returns the plugin.""" + if isinstance(plugin, type): + return (plugin, '__main__') + plugin_ = text_(plugin.strip()) + assert plugin_ != '' + module_name, klass_name = plugin_.rsplit(text_(DOT), 1) + klass = getattr( + importlib.import_module( + module_name.replace( + os.path.sep, text_(DOT), + ), + ), + klass_name, + ) + return (klass, module_name) diff --git a/proxy/common/utils.py b/proxy/common/utils.py index f6e3d55507..49df1eeafc 100644 --- a/proxy/common/utils.py +++ b/proxy/common/utils.py @@ -9,13 +9,10 @@ :license: BSD, see LICENSE for more details. """ import os -import abc import sys import ssl import socket import logging -import inspect -import importlib import functools import ipaddress import contextlib @@ -24,7 +21,6 @@ from typing import Optional, Dict, Any, List, Tuple, Type, Callable, Union from .constants import HTTP_1_1, COLON, WHITESPACE, CRLF, DEFAULT_TIMEOUT -from .constants import DOT, DEFAULT_ABC_PLUGINS if os.name != 'nt': import resource @@ -262,63 +258,6 @@ def get_available_port() -> int: return int(port) -def discover_plugins(input_args: Optional[List[str]]) -> None: - for i, f in enumerate(input_args): - if f in ('--plugin', '--plugins'): - v = input_args[i + 1] - if type(v) == str: - parts = v.split(',') - for part in parts: - import_plugin(bytes_(part)) - else: - import_plugin(bytes_(v)) - - -def load_plugins( - plugins: List[Union[bytes, type]], - abc_plugins: List[bytes] = DEFAULT_ABC_PLUGINS, -) -> Dict[bytes, List[type]]: - """Accepts a list Python modules, scans them to identify - if they are an implementation of abstract plugin classes and - returns a dictionary of matching plugins for each abstract class. - """ - p: Dict[bytes, List[type]] = {} - for abc_plugin in abc_plugins: - p[bytes_(abc_plugin)] = [] - for plugin_ in plugins: - klass, module_name = import_plugin(plugin_) - assert klass and module_name - mro = list(inspect.getmro(klass)) - mro.reverse() - iterator = iter(mro) - while next(iterator) is not abc.ABC: - pass - base_klass = next(iterator) - if klass not in p[bytes_(base_klass.__name__)]: - p[bytes_(base_klass.__name__)].append(klass) - logger.info('Loaded plugin %s.%s', module_name, klass.__name__) - return p - - -def import_plugin(plugin: Union[bytes, type]) -> Any: - if isinstance(plugin, type): - module_name = '__main__' - klass = plugin - else: - plugin_ = text_(plugin.strip()) - assert plugin_ != '' - module_name, klass_name = plugin_.rsplit(text_(DOT), 1) - klass = getattr( - importlib.import_module( - module_name.replace( - os.path.sep, text_(DOT), - ), - ), - klass_name, - ) - return (klass, module_name) - - def set_open_file_limit(soft_limit: int) -> None: """Configure open file description soft limit on supported OS.""" if os.name != 'nt': # resource module not available on Windows OS diff --git a/tests/http/test_protocol_handler.py b/tests/http/test_protocol_handler.py index c5af7f3cff..93d821fdd6 100644 --- a/tests/http/test_protocol_handler.py +++ b/tests/http/test_protocol_handler.py @@ -15,9 +15,10 @@ from typing import cast from unittest import mock +from proxy.common.plugins import Plugins from proxy.common.flag import FlagParser from proxy.common.version import __version__ -from proxy.common.utils import bytes_, load_plugins +from proxy.common.utils import bytes_ from proxy.common.constants import CRLF, PLUGIN_HTTP_PROXY, PLUGIN_PROXY_AUTH, PLUGIN_WEB_SERVER from proxy.core.connection import TcpClientConnection from proxy.http.parser import HttpParser @@ -42,7 +43,7 @@ def setUp( self.http_server_port = 65535 self.flags = FlagParser.initialize() - self.flags.plugins = load_plugins([ + self.flags.plugins = Plugins.load([ bytes_(PLUGIN_HTTP_PROXY), bytes_(PLUGIN_WEB_SERVER), ]) @@ -215,7 +216,7 @@ def test_proxy_authentication_failed( flags = FlagParser.initialize( auth_code=base64.b64encode(b'user:pass'), ) - flags.plugins = load_plugins([ + flags.plugins = Plugins.load([ bytes_(PLUGIN_HTTP_PROXY), bytes_(PLUGIN_WEB_SERVER), bytes_(PLUGIN_PROXY_AUTH), @@ -253,7 +254,7 @@ def test_authenticated_proxy_http_get( flags = FlagParser.initialize( auth_code=base64.b64encode(b'user:pass'), ) - flags.plugins = load_plugins([ + flags.plugins = Plugins.load([ bytes_(PLUGIN_HTTP_PROXY), bytes_(PLUGIN_WEB_SERVER), ]) @@ -308,7 +309,7 @@ def test_authenticated_proxy_http_tunnel( flags = FlagParser.initialize( auth_code=base64.b64encode(b'user:pass'), ) - flags.plugins = load_plugins([ + flags.plugins = Plugins.load([ bytes_(PLUGIN_HTTP_PROXY), bytes_(PLUGIN_WEB_SERVER), ]) diff --git a/tests/http/test_web_server.py b/tests/http/test_web_server.py index ebd563d8c7..f7e48e5fb1 100644 --- a/tests/http/test_web_server.py +++ b/tests/http/test_web_server.py @@ -15,11 +15,12 @@ import selectors from unittest import mock +from proxy.common.plugins import Plugins from proxy.common.flag import FlagParser from proxy.core.connection import TcpClientConnection from proxy.http.handler import HttpProtocolHandler from proxy.http.parser import httpParserStates -from proxy.common.utils import build_http_response, build_http_request, bytes_, text_, load_plugins +from proxy.common.utils import build_http_response, build_http_request, bytes_, text_ from proxy.common.constants import CRLF, PLUGIN_HTTP_PROXY, PLUGIN_PAC_FILE, PLUGIN_WEB_SERVER, PROXY_PY_DIR from proxy.http.server import HttpWebServerPlugin @@ -34,7 +35,7 @@ def setUp(self, mock_fromfd: mock.Mock, mock_selector: mock.Mock) -> None: self._conn = mock_fromfd.return_value self.mock_selector = mock_selector self.flags = FlagParser.initialize() - self.flags.plugins = load_plugins([ + self.flags.plugins = Plugins.load([ bytes_(PLUGIN_HTTP_PROXY), bytes_(PLUGIN_WEB_SERVER), ]) @@ -113,7 +114,7 @@ def test_default_web_server_returns_404( ), ] flags = FlagParser.initialize() - flags.plugins = load_plugins([ + flags.plugins = Plugins.load([ bytes_(PLUGIN_HTTP_PROXY), bytes_(PLUGIN_WEB_SERVER), ]) @@ -183,7 +184,7 @@ def test_static_web_server_serves( enable_static_server=True, static_server_dir=static_server_dir, ) - flags.plugins = load_plugins([ + flags.plugins = Plugins.load([ bytes_(PLUGIN_HTTP_PROXY), bytes_(PLUGIN_WEB_SERVER), ]) @@ -247,7 +248,7 @@ def test_static_web_server_serves_404( ] flags = FlagParser.initialize(enable_static_server=True) - flags.plugins = load_plugins([ + flags.plugins = Plugins.load([ bytes_(PLUGIN_HTTP_PROXY), bytes_(PLUGIN_WEB_SERVER), ]) @@ -290,7 +291,7 @@ def test_on_client_connection_called_on_teardown( def init_and_make_pac_file_request(self, pac_file: str) -> None: flags = FlagParser.initialize(pac_file=pac_file) - flags.plugins = load_plugins([ + flags.plugins = Plugins.load([ bytes_(PLUGIN_HTTP_PROXY), bytes_(PLUGIN_WEB_SERVER), bytes_(PLUGIN_PAC_FILE), diff --git a/tests/test_main.py b/tests/test_main.py index d0357255fc..94994b1c26 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -145,7 +145,7 @@ def test_enable_events( mock_sleep.assert_called() @mock.patch('time.sleep') - @mock.patch('proxy.common.flag.load_plugins') + @mock.patch('proxy.common.plugins.Plugins.load') @mock.patch('proxy.common.flag.FlagParser.parse_args') @mock.patch('proxy.proxy.EventManager') @mock.patch('proxy.proxy.AcceptorPool') @@ -181,7 +181,7 @@ def test_enable_dashboard( mock_event_manager.return_value.stop_event_dispatcher.assert_called_once() @mock.patch('time.sleep') - @mock.patch('proxy.common.flag.load_plugins') + @mock.patch('proxy.common.plugins.Plugins.load') @mock.patch('proxy.common.flag.FlagParser.parse_args') @mock.patch('proxy.proxy.EventManager') @mock.patch('proxy.proxy.AcceptorPool') From 6c01ca32e0d16fcdc0007f203d4c500ac9119d3f Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Mon, 8 Nov 2021 20:57:35 +0530 Subject: [PATCH 10/13] mypy and lint --- proxy/common/plugins.py | 4 ++-- proxy/common/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/proxy/common/plugins.py b/proxy/common/plugins.py index 901a991482..abcc138f19 100644 --- a/proxy/common/plugins.py +++ b/proxy/common/plugins.py @@ -25,7 +25,7 @@ class Plugins: @staticmethod - def discover(input_args: Optional[List[str]]) -> None: + def discover(input_args: List[str]) -> None: """Search for plugin and plugins flag in command line arguments, then iterates over each value and discovers the plugin. """ @@ -39,7 +39,7 @@ def discover(input_args: Optional[List[str]]) -> None: @staticmethod def load( plugins: List[Union[bytes, type]], - abc_plugins: List[bytes] = DEFAULT_ABC_PLUGINS, + abc_plugins: List[str] = DEFAULT_ABC_PLUGINS, ) -> Dict[bytes, List[type]]: """Accepts a list Python modules, scans them to identify if they are an implementation of abstract plugin classes and diff --git a/proxy/common/utils.py b/proxy/common/utils.py index 49df1eeafc..1abb2bdece 100644 --- a/proxy/common/utils.py +++ b/proxy/common/utils.py @@ -18,7 +18,7 @@ import contextlib from types import TracebackType -from typing import Optional, Dict, Any, List, Tuple, Type, Callable, Union +from typing import Optional, Dict, Any, List, Tuple, Type, Callable from .constants import HTTP_1_1, COLON, WHITESPACE, CRLF, DEFAULT_TIMEOUT From e329c5689735b8b5f81c99687e128c217a3fe3d2 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Mon, 8 Nov 2021 21:01:48 +0530 Subject: [PATCH 11/13] Fix unused import --- proxy/common/plugins.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proxy/common/plugins.py b/proxy/common/plugins.py index abcc138f19..3234d9468d 100644 --- a/proxy/common/plugins.py +++ b/proxy/common/plugins.py @@ -14,7 +14,7 @@ import inspect import importlib -from typing import Any, List, Optional, Dict, Union +from typing import Any, List, Dict, Union from .utils import bytes_, text_ from .constants import DOT, DEFAULT_ABC_PLUGINS @@ -23,6 +23,7 @@ class Plugins: + """Common utilities for plugin discovery.""" @staticmethod def discover(input_args: List[str]) -> None: From d6e51ccf30f48731c5a06c7f13e22985aa2bc5e0 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Mon, 8 Nov 2021 21:23:21 +0530 Subject: [PATCH 12/13] Safe to use tempdir on Github actions to avoid race conditions??? --- proxy/common/plugins.py | 6 +++--- tests/common/test_pki.py | 12 ++++++++---- tests/test_main.py | 6 +----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/proxy/common/plugins.py b/proxy/common/plugins.py index 3234d9468d..2c04530b20 100644 --- a/proxy/common/plugins.py +++ b/proxy/common/plugins.py @@ -14,7 +14,7 @@ import inspect import importlib -from typing import Any, List, Dict, Union +from typing import Any, List, Dict, Optional, Union from .utils import bytes_, text_ from .constants import DOT, DEFAULT_ABC_PLUGINS @@ -40,14 +40,14 @@ def discover(input_args: List[str]) -> None: @staticmethod def load( plugins: List[Union[bytes, type]], - abc_plugins: List[str] = DEFAULT_ABC_PLUGINS, + abc_plugins: Optional[List[str]] = None, ) -> Dict[bytes, List[type]]: """Accepts a list Python modules, scans them to identify if they are an implementation of abstract plugin classes and returns a dictionary of matching plugins for each abstract class. """ p: Dict[bytes, List[type]] = {} - for abc_plugin in abc_plugins: + for abc_plugin in (abc_plugins or DEFAULT_ABC_PLUGINS): p[bytes_(abc_plugin)] = [] for plugin_ in plugins: klass, module_name = Plugins.importer(plugin_) diff --git a/tests/common/test_pki.py b/tests/common/test_pki.py index 76dd723837..1abcb1c625 100644 --- a/tests/common/test_pki.py +++ b/tests/common/test_pki.py @@ -20,6 +20,10 @@ class TestPki(unittest.TestCase): + def setUp(self) -> None: + self._tempdir = tempfile.gettempdir() + return super().setUp() + @mock.patch('subprocess.Popen') def test_run_openssl_command(self, mock_popen: mock.Mock) -> None: command = ['my', 'custom', 'command'] @@ -103,7 +107,7 @@ def test_gen_public_key(self) -> None: def test_gen_csr(self) -> None: key_path, nopass_key_path, crt_path = self._gen_public_private_key() - csr_path = os.path.join(tempfile.gettempdir(), 'test_gen_public.csr') + csr_path = os.path.join(self._tempdir, 'test_gen_public.csr') pki.gen_csr(csr_path, key_path, 'password', crt_path) self.assertTrue(os.path.exists(csr_path)) # TODO: Assert CSR is valid for provided crt and key @@ -117,14 +121,14 @@ def test_sign_csr(self) -> None: def _gen_public_private_key(self) -> Tuple[str, str, str]: key_path, nopass_key_path = self._gen_private_key() - crt_path = os.path.join(tempfile.gettempdir(), 'test_gen_public.crt') + crt_path = os.path.join(self._tempdir, 'test_gen_public.crt') pki.gen_public_key(crt_path, key_path, 'password', '/CN=example.com') return (key_path, nopass_key_path, crt_path) def _gen_private_key(self) -> Tuple[str, str]: - key_path = os.path.join(tempfile.gettempdir(), 'test_gen_private.key') + key_path = os.path.join(self._tempdir, 'test_gen_private.key') nopass_key_path = os.path.join( - tempfile.gettempdir(), + self._tempdir, 'test_gen_private_nopass.key', ) pki.gen_private_key(key_path, 'password') diff --git a/tests/test_main.py b/tests/test_main.py index 94994b1c26..b9115cff4e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -32,10 +32,6 @@ from proxy.common.version import __version__ -def get_temp_file(name: str) -> str: - return os.path.join(tempfile.gettempdir(), name) - - class TestMain(unittest.TestCase): @staticmethod @@ -229,7 +225,7 @@ def test_pid_file_is_written_and_removed( mock_remove: mock.Mock, mock_sleep: mock.Mock, ) -> None: - pid_file = get_temp_file('pid') + pid_file = os.path.join(tempfile.gettempdir(), 'pid') mock_sleep.side_effect = KeyboardInterrupt() mock_args = mock_parse_args.return_value self.mock_default_args(mock_args) From 7ef05e6fd4584b0ec86ed4be62850a13fa726bd9 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Mon, 8 Nov 2021 21:30:59 +0530 Subject: [PATCH 13/13] pki (generically disk based file) based tests are flaky on macOS under parallel execution --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 4db289ec2b..650a369303 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,7 +2,7 @@ addopts = # FIXME: Enable this once the test suite has no race conditions # `pytest-xdist`: - --numprocesses=auto + # --numprocesses=auto # Show 10 slowest invocations: --durations=10