Fix --enable-dashboard flags (#707)

* Expose within __all__

* Enable `--numprocesses=auto` for `pytest.ini`

* make lib-lint

* Also consider `--plugins` flag when bootstrapping plugins

* Add `from .dashboard import ProxyDashboard` in top-level `__init__.py` to make `ProxyDashboard` flags auto discoverable

* Move `--enable-dashboard` to top-level

* Move logging utility within `Logger` class

* Consider comma separated --plugin and --plugins during discover_plugins

* Refactor plugin related utilities in Plugins module

* mypy and lint

* Fix unused import

* Safe to use tempdir on Github actions to avoid race conditions???

* pki (generically disk based file) based tests are flaky on macOS under parallel execution

Author: abhinavsingh

proxy/__init__.py

@@ -9,7 +9,7 @@
     :license: BSD, see LICENSE for more details.
 """
 from .proxy import entry_point, main, Proxy
-from .testing.test_case import TestCase
+from .testing import TestCase
 
 __all__ = [
     # PyPi package entry_point. See

proxy/common/constants.py

@@ -100,12 +100,18 @@
 DEFAULT_DATA_DIRECTORY_PATH = os.path.join(str(pathlib.Path.home()), '.proxy')
 
 # Cor plugins enabled by default or via flags
+DEFAULT_ABC_PLUGINS = [
+    'HttpProtocolHandlerPlugin',
+    'HttpProxyBasePlugin',
+    'HttpWebServerBasePlugin',
+    'ProxyDashboardWebsocketPlugin',
+]
 PLUGIN_HTTP_PROXY = 'proxy.http.proxy.HttpProxyPlugin'
 PLUGIN_WEB_SERVER = 'proxy.http.server.HttpWebServerPlugin'
 PLUGIN_PAC_FILE = 'proxy.http.server.HttpWebServerPacFilePlugin'
 PLUGIN_DEVTOOLS_PROTOCOL = 'proxy.http.inspector.DevtoolsProtocolPlugin'
-PLUGIN_DASHBOARD = 'proxy.dashboard.dashboard.ProxyDashboard'
-PLUGIN_INSPECT_TRAFFIC = 'proxy.dashboard.inspect_traffic.InspectTrafficPlugin'
+PLUGIN_DASHBOARD = 'proxy.dashboard.ProxyDashboard'
+PLUGIN_INSPECT_TRAFFIC = 'proxy.dashboard.InspectTrafficPlugin'
 PLUGIN_PROXY_AUTH = 'proxy.http.proxy.AuthPlugin'
 
 PY2_DEPRECATION_MESSAGE = '''DEPRECATION: proxy.py no longer supports Python 2.7.  Kindly upgrade to Python 3+. '

proxy/common/flag.py

@@ -19,14 +19,15 @@
 
 from typing import Optional, List, Any, cast
 
+from .plugins import Plugins
 from .types import IpAddress
-from .utils import text_, bytes_, setup_logger, is_py2, set_open_file_limit
-from .utils import import_plugin, load_plugins
+from .utils import text_, bytes_, is_py2, set_open_file_limit
 from .constants import COMMA, DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_NUM_WORKERS
 from .constants import DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DISABLE_HEADERS, PY2_DEPRECATION_MESSAGE
 from .constants import PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL
 from .constants import PLUGIN_HTTP_PROXY, PLUGIN_INSPECT_TRAFFIC, PLUGIN_PAC_FILE
 from .constants import PLUGIN_WEB_SERVER, PLUGIN_PROXY_AUTH
+from .logger import Logger
 
 from .version import __version__
 
@@ -94,10 +95,9 @@ def initialize(
             sys.exit(1)
 
         # Discover flags from requested plugin.
-        # This also surface external plugin flags under --help
-        for i, f in enumerate(input_args):
-            if f == '--plugin':
-                import_plugin(bytes_(input_args[i + 1]))
+        # This will also surface external plugin flags
+        # under --help.
+        Plugins.discover(input_args)
 
         # Parse flags
         args = flags.parse_args(input_args)
@@ -108,7 +108,7 @@ def initialize(
             sys.exit(0)
 
         # Setup logging module
-        setup_logger(args.log_file, args.log_level, args.log_format)
+        Logger.setup_logger(args.log_file, args.log_level, args.log_format)
 
         # Setup limits
         set_open_file_limit(args.open_file_limit)
@@ -125,7 +125,7 @@ def initialize(
         ]
 
         # Load default plugins along with user provided --plugins
-        plugins = load_plugins(default_plugins + extra_plugins)
+        plugins = Plugins.load(default_plugins + extra_plugins)
 
         # proxy.py currently cannot serve over HTTPS and also perform TLS interception
         # at the same time.  Check if user is trying to enable both feature

proxy/common/logger.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+import logging
+
+from typing import Optional
+
+from .constants import DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL
+
+SINGLE_CHAR_TO_LEVEL = {
+    'D': 'DEBUG',
+    'I': 'INFO',
+    'W': 'WARNING',
+    'E': 'ERROR',
+    'C': 'CRITICAL',
+}
+
+
+class Logger:
+    """Common logging utilities and setup."""
+
+    @staticmethod
+    def setup_logger(
+            log_file: Optional[str] = DEFAULT_LOG_FILE,
+            log_level: str = DEFAULT_LOG_LEVEL,
+            log_format: str = DEFAULT_LOG_FORMAT,
+    ) -> None:
+        ll = getattr(logging, SINGLE_CHAR_TO_LEVEL[log_level.upper()[0]])
+        if log_file:
+            logging.basicConfig(
+                filename=log_file,
+                filemode='a',
+                level=ll,
+                format=log_format,
+            )
+        else:
+            logging.basicConfig(level=ll, format=log_format)

proxy/common/plugins.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+"""
+    proxy.py
+    ~~~~~~~~
+    ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on
+    Network monitoring, controls & Application development, testing, debugging.
+
+    :copyright: (c) 2013-present by Abhinav Singh and contributors.
+    :license: BSD, see LICENSE for more details.
+"""
+import os
+import abc
+import logging
+import inspect
+import importlib
+
+from typing import Any, List, Dict, Optional, Union
+
+from .utils import bytes_, text_
+from .constants import DOT, DEFAULT_ABC_PLUGINS
+
+logger = logging.getLogger(__name__)
+
+
+class Plugins:
+    """Common utilities for plugin discovery."""
+
+    @staticmethod
+    def discover(input_args: List[str]) -> None:
+        """Search for plugin and plugins flag in command line arguments,
+        then iterates over each value and discovers the plugin.
+        """
+        for i, f in enumerate(input_args):
+            if f in ('--plugin', '--plugins'):
+                v = input_args[i + 1]
+                parts = v.split(',')
+                for part in parts:
+                    Plugins.importer(bytes_(part))
+
+    @staticmethod
+    def load(
+        plugins: List[Union[bytes, type]],
+        abc_plugins: Optional[List[str]] = None,
+    ) -> Dict[bytes, List[type]]:
+        """Accepts a list Python modules, scans them to identify
+        if they are an implementation of abstract plugin classes and
+        returns a dictionary of matching plugins for each abstract class.
+        """
+        p: Dict[bytes, List[type]] = {}
+        for abc_plugin in (abc_plugins or DEFAULT_ABC_PLUGINS):
+            p[bytes_(abc_plugin)] = []
+        for plugin_ in plugins:
+            klass, module_name = Plugins.importer(plugin_)
+            assert klass and module_name
+            mro = list(inspect.getmro(klass))
+            mro.reverse()
+            iterator = iter(mro)
+            while next(iterator) is not abc.ABC:
+                pass
+            base_klass = next(iterator)
+            if klass not in p[bytes_(base_klass.__name__)]:
+                p[bytes_(base_klass.__name__)].append(klass)
+            logger.info('Loaded plugin %s.%s', module_name, klass.__name__)
+        return p
+
+    @staticmethod
+    def importer(plugin: Union[bytes, type]) -> Any:
+        """Import and returns the plugin."""
+        if isinstance(plugin, type):
+            return (plugin, '__main__')
+        plugin_ = text_(plugin.strip())
+        assert plugin_ != ''
+        module_name, klass_name = plugin_.rsplit(text_(DOT), 1)
+        klass = getattr(
+            importlib.import_module(
+                module_name.replace(
+                    os.path.sep, text_(DOT),
+                ),
+            ),
+            klass_name,
+        )
+        return (klass, module_name)

proxy/common/utils.py

@@ -9,23 +9,18 @@
     :license: BSD, see LICENSE for more details.
 """
 import os
-import abc
 import sys
 import ssl
 import socket
 import logging
-import inspect
-import importlib
 import functools
 import ipaddress
 import contextlib
 
 from types import TracebackType
-from typing import Optional, Dict, Any, List, Tuple, Type, Callable, Union
+from typing import Optional, Dict, Any, List, Tuple, Type, Callable
 
 from .constants import HTTP_1_1, COLON, WHITESPACE, CRLF, DEFAULT_TIMEOUT
-from .constants import DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL
-from .constants import DOT
 
 if os.name != 'nt':
     import resource
@@ -263,77 +258,6 @@ def get_available_port() -> int:
     return int(port)
 
 
-def setup_logger(
-        log_file: Optional[str] = DEFAULT_LOG_FILE,
-        log_level: str = DEFAULT_LOG_LEVEL,
-        log_format: str = DEFAULT_LOG_FORMAT,
-) -> None:
-    ll = getattr(
-        logging,
-        {
-            'D': 'DEBUG',
-            'I': 'INFO',
-            'W': 'WARNING',
-            'E': 'ERROR',
-            'C': 'CRITICAL',
-        }[log_level.upper()[0]],
-    )
-    if log_file:
-        logging.basicConfig(
-            filename=log_file,
-            filemode='a',
-            level=ll,
-            format=log_format,
-        )
-    else:
-        logging.basicConfig(level=ll, format=log_format)
-
-
-def load_plugins(
-    plugins: List[Union[bytes, type]],
-) -> Dict[bytes, List[type]]:
-    """Accepts a comma separated list of Python modules and returns
-    a list of respective Python classes."""
-    p: Dict[bytes, List[type]] = {
-        b'HttpProtocolHandlerPlugin': [],
-        b'HttpProxyBasePlugin': [],
-        b'HttpWebServerBasePlugin': [],
-        b'ProxyDashboardWebsocketPlugin': [],
-    }
-    for plugin_ in plugins:
-        klass, module_name = import_plugin(plugin_)
-        assert klass and module_name
-        mro = list(inspect.getmro(klass))
-        mro.reverse()
-        iterator = iter(mro)
-        while next(iterator) is not abc.ABC:
-            pass
-        base_klass = next(iterator)
-        if klass not in p[bytes_(base_klass.__name__)]:
-            p[bytes_(base_klass.__name__)].append(klass)
-        logger.info('Loaded plugin %s.%s', module_name, klass.__name__)
-    return p
-
-
-def import_plugin(plugin: Union[bytes, type]) -> Any:
-    if isinstance(plugin, type):
-        module_name = '__main__'
-        klass = plugin
-    else:
-        plugin_ = text_(plugin.strip())
-        assert plugin_ != ''
-        module_name, klass_name = plugin_.rsplit(text_(DOT), 1)
-        klass = getattr(
-            importlib.import_module(
-                module_name.replace(
-                    os.path.sep, text_(DOT),
-                ),
-            ),
-            klass_name,
-        )
-    return (klass, module_name)
-
-
 def set_open_file_limit(soft_limit: int) -> None:
     """Configure open file description soft limit on supported OS."""
     if os.name != 'nt':  # resource module not available on Windows OS

proxy/core/acceptor/acceptor.py

@@ -27,7 +27,7 @@
 from ..event import EventQueue, eventNames
 from ...common.constants import DEFAULT_THREADLESS
 from ...common.flag import flags
-from ...common.utils import setup_logger
+from ...common.logger import Logger
 
 logger = logging.getLogger(__name__)
 
@@ -159,7 +159,7 @@ def run_once(self) -> None:
             self._start_threaded_work(conn, addr)
 
     def run(self) -> None:
-        setup_logger(
+        Logger.setup_logger(
             self.flags.log_file, self.flags.log_level,
             self.flags.log_format,
         )

proxy/core/acceptor/threadless.py

@@ -26,7 +26,7 @@
 from ..connection import TcpClientConnection
 from ..event import EventQueue, eventNames
 
-from ...common.utils import setup_logger
+from ...common.logger import Logger
 from ...common.types import Readables, Writables
 from ...common.constants import DEFAULT_TIMEOUT
 
@@ -204,7 +204,7 @@ def run_once(self) -> None:
         self.cleanup_inactive()
 
     def run(self) -> None:
-        setup_logger(
+        Logger.setup_logger(
             self.flags.log_file, self.flags.log_level,
             self.flags.log_format,
         )

proxy/core/ssh/__init__.py

@@ -8,3 +8,10 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
+from .client import SshClient
+from .tunnel import Tunnel
+
+__all__ = [
+    'SshClient',
+    'Tunnel',
+]

proxy/dashboard/__init__.py

@@ -8,3 +8,12 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
+from .dashboard import ProxyDashboard
+from .inspect_traffic import InspectTrafficPlugin
+from .plugin import ProxyDashboardWebsocketPlugin
+
+__all__ = [
+    'ProxyDashboard',
+    'InspectTrafficPlugin',
+    'ProxyDashboardWebsocketPlugin',
+]

proxy/dashboard/dashboard.py

@@ -14,9 +14,7 @@
 
 from .plugin import ProxyDashboardWebsocketPlugin
 
-from ..common.flag import flags
 from ..common.utils import build_http_response, bytes_
-from ..common.constants import DEFAULT_ENABLE_DASHBOARD
 from ..http.server import HttpWebServerPlugin, HttpWebServerBasePlugin, httpProtocolTypes
 from ..http.parser import HttpParser
 from ..http.websocket import WebsocketFrame
@@ -25,14 +23,6 @@
 logger = logging.getLogger(__name__)
 
 
-flags.add_argument(
-    '--enable-dashboard',
-    action='store_true',
-    default=DEFAULT_ENABLE_DASHBOARD,
-    help='Default: False.  Enables proxy.py dashboard.',
-)
-
-
 class ProxyDashboard(HttpWebServerBasePlugin):
     """Proxy Dashboard."""