Put core flags where they belong (#702)

* Move flags to where they belong

* Move `get_default_plugins` within FlagParser as it depends upon args

TODO: We need plugin dependency system

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: abhinavsingh

README.md

@@ -1870,70 +1870,125 @@ for list of tests.
 
 ```console
 ❯ proxy -h
-usage: proxy [-h] [--threadless] [--backlog BACKLOG] [--enable-events] [--hostname HOSTNAME] [--port PORT] [--num-workers NUM_WORKERS] [--client-recvbuf-size CLIENT_RECVBUF_SIZE] [--key-file KEY_FILE]
-          [--timeout TIMEOUT] [--pid-file PID_FILE] [--version] [--disable-http-proxy] [--enable-dashboard] [--enable-devtools] [--enable-static-server] [--enable-web-server] [--log-level LOG_LEVEL]
-          [--log-file LOG_FILE] [--log-format LOG_FORMAT] [--open-file-limit OPEN_FILE_LIMIT] [--plugins PLUGINS] [--ca-key-file CA_KEY_FILE] [--ca-cert-dir CA_CERT_DIR] [--ca-cert-file CA_CERT_FILE]
-          [--ca-file CA_FILE] [--ca-signing-key-file CA_SIGNING_KEY_FILE] [--cert-file CERT_FILE] [--disable-headers DISABLE_HEADERS] [--server-recvbuf-size SERVER_RECVBUF_SIZE] [--basic-auth BASIC_AUTH]
-          [--cache-dir CACHE_DIR] [--static-server-dir STATIC_SERVER_DIR] [--pac-file PAC_FILE] [--pac-file-url-path PAC_FILE_URL_PATH] [--filtered-client-ips FILTERED_CLIENT_IPS]
+usage: proxy [-h] [--enable-events] [--enable-conn-pool] [--threadless] [--pid-file PID_FILE]
+          [--backlog BACKLOG] [--hostname HOSTNAME] [--port PORT] [--num-workers NUM_WORKERS]
+          [--unix-socket-path UNIX_SOCKET_PATH] [--client-recvbuf-size CLIENT_RECVBUF_SIZE]
+          [--key-file KEY_FILE] [--timeout TIMEOUT] [--version] [--log-level LOG_LEVEL]
+          [--log-file LOG_FILE] [--log-format LOG_FORMAT] [--open-file-limit OPEN_FILE_LIMIT]
+          [--plugins PLUGINS] [--disable-http-proxy] [--ca-key-file CA_KEY_FILE]
+          [--ca-cert-dir CA_CERT_DIR] [--ca-cert-file CA_CERT_FILE] [--ca-file CA_FILE]
+          [--ca-signing-key-file CA_SIGNING_KEY_FILE] [--cert-file CERT_FILE]
+          [--disable-headers DISABLE_HEADERS] [--server-recvbuf-size SERVER_RECVBUF_SIZE]
+          [--basic-auth BASIC_AUTH] [--cache-dir CACHE_DIR]
+          [--filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS] [--enable-web-server]
+          [--enable-static-server] [--static-server-dir STATIC_SERVER_DIR]
+          [--pac-file PAC_FILE] [--pac-file-url-path PAC_FILE_URL_PATH]
+          [--filtered-client-ips FILTERED_CLIENT_IPS]
+          [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
+          [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE]
 
 proxy.py v2.4.0
 
 options:
   -h, --help            show this help message and exit
-  --threadless          Default: False. When disabled a new thread is spawned to handle each client connection.
+  --enable-events       Default: False. Enables core to dispatch lifecycle events. Plugins
+                        can be used to subscribe for core events.
+  --enable-conn-pool    Default: False. (WIP) Enable upstream connection pooling.
+  --threadless          Default: False. When disabled a new thread is spawned to handle each
+                        client connection.
+  --pid-file PID_FILE   Default: None. Save parent process ID to a file.
   --backlog BACKLOG     Default: 100. Maximum number of pending connections to proxy server
-  --enable-events       Default: False. Enables core to dispatch lifecycle events. Plugins can be used to subscribe for core events.
   --hostname HOSTNAME   Default: ::1. Server IP address.
   --port PORT           Default: 8899. Server port.
   --num-workers NUM_WORKERS
                         Defaults to number of CPU cores.
+  --unix-socket-path UNIX_SOCKET_PATH
+                        Default: None. Unix socket path to use. When provided --host and
+                        --port flags are ignored
   --client-recvbuf-size CLIENT_RECVBUF_SIZE
-                        Default: 1 MB. Maximum amount of data received from the client in a single recv() operation. Bump this value for faster uploads at the expense of increased RAM.
-  --key-file KEY_FILE   Default: None. Server key file to enable end-to-end TLS encryption with clients. If used, must also pass --cert-file.
-  --timeout TIMEOUT     Default: 10. Number of seconds after which an inactive connection must be dropped. Inactivity is defined by no data sent or received by the client.
-  --pid-file PID_FILE   Default: None. Save parent process ID to a file.
+                        Default: 1 MB. Maximum amount of data received from the client in a
+                        single recv() operation. Bump this value for faster uploads at the
+                        expense of increased RAM.
+  --key-file KEY_FILE   Default: None. Server key file to enable end-to-end TLS encryption
+                        with clients. If used, must also pass --cert-file.
+  --timeout TIMEOUT     Default: 10.0. Number of seconds after which an inactive connection
+                        must be dropped. Inactivity is defined by no data sent or received by
+                        the client.
   --version, -v         Prints proxy.py version.
-  --disable-http-proxy  Default: False. Whether to disable proxy.HttpProxyPlugin.
-  --enable-dashboard    Default: False. Enables proxy.py dashboard.
-  --enable-devtools     Default: False. Enables integration with Chrome Devtool Frontend. Also see --devtools-ws-path.
-  --enable-static-server
-                        Default: False. Enable inbuilt static file server. Optionally, also use --static-server-dir to serve static content from custom directory. By default, static file server serves out of
-                        installed proxy.py python module folder.
-  --enable-web-server   Default: False. Whether to enable proxy.HttpWebServerPlugin.
   --log-level LOG_LEVEL
-                        Valid options: DEBUG, INFO (default), WARNING, ERROR, CRITICAL. Both upper and lowercase values are allowed. You may also simply use the leading character e.g. --log-level d
+                        Valid options: DEBUG, INFO (default), WARNING, ERROR, CRITICAL. Both
+                        upper and lowercase values are allowed. You may also simply use the
+                        leading character e.g. --log-level d
   --log-file LOG_FILE   Default: sys.stdout. Log file destination.
   --log-format LOG_FORMAT
                         Log format for Python logger.
   --open-file-limit OPEN_FILE_LIMIT
-                        Default: 1024. Maximum number of files (TCP connections) that proxy.py can open concurrently.
+                        Default: 1024. Maximum number of files (TCP connections) that
+                        proxy.py can open concurrently.
   --plugins PLUGINS     Comma separated plugins
+  --disable-http-proxy  Default: False. Whether to disable proxy.HttpProxyPlugin.
   --ca-key-file CA_KEY_FILE
-                        Default: None. CA key to use for signing dynamically generated HTTPS certificates. If used, must also pass --ca-cert-file and --ca-signing-key-file
+                        Default: None. CA key to use for signing dynamically generated HTTPS
+                        certificates. If used, must also pass --ca-cert-file and --ca-
+                        signing-key-file
   --ca-cert-dir CA_CERT_DIR
-                        Default: ~/.proxy.py. Directory to store dynamically generated certificates. Also see --ca-key-file, --ca-cert-file and --ca-signing-key-file
+                        Default: ~/.proxy.py. Directory to store dynamically generated
+                        certificates. Also see --ca-key-file, --ca-cert-file and --ca-
+                        signing-key-file
   --ca-cert-file CA_CERT_FILE
-                        Default: None. Signing certificate to use for signing dynamically generated HTTPS certificates. If used, must also pass --ca-key-file and --ca-signing-key-file
-  --ca-file CA_FILE     Default: None. Provide path to custom CA file for peer certificate validation. Specially useful on MacOS.
+                        Default: None. Signing certificate to use for signing dynamically
+                        generated HTTPS certificates. If used, must also pass --ca-key-file
+                        and --ca-signing-key-file
+  --ca-file CA_FILE     Default:
+                        /Users/abhinavsingh/Dev/proxy.py/venv310/lib/python3.10/site-
+                        packages/certifi/cacert.pem. Provide path to custom CA bundle for
+                        peer certificate verification
   --ca-signing-key-file CA_SIGNING_KEY_FILE
-                        Default: None. CA signing key to use for dynamic generation of HTTPS certificates. If used, must also pass --ca-key-file and --ca-cert-file
+                        Default: None. CA signing key to use for dynamic generation of HTTPS
+                        certificates. If used, must also pass --ca-key-file and --ca-cert-
+                        file
   --cert-file CERT_FILE
-                        Default: None. Server certificate to enable end-to-end TLS encryption with clients. If used, must also pass --key-file.
+                        Default: None. Server certificate to enable end-to-end TLS encryption
+                        with clients. If used, must also pass --key-file.
   --disable-headers DISABLE_HEADERS
-                        Default: None. Comma separated list of headers to remove before dispatching client request to upstream server.
+                        Default: None. Comma separated list of headers to remove before
+                        dispatching client request to upstream server.
   --server-recvbuf-size SERVER_RECVBUF_SIZE
-                        Default: 1 MB. Maximum amount of data received from the server in a single recv() operation. Bump this value for faster downloads at the expense of increased RAM.
+                        Default: 1 MB. Maximum amount of data received from the server in a
+                        single recv() operation. Bump this value for faster downloads at the
+                        expense of increased RAM.
   --basic-auth BASIC_AUTH
-                        Default: No authentication. Specify colon separated user:password to enable basic authentication.
+                        Default: No authentication. Specify colon separated user:password to
+                        enable basic authentication.
   --cache-dir CACHE_DIR
-                        Default: A temporary directory. Flag only applicable when cache plugin is used with on-disk storage.
+                        Default: A temporary directory. Flag only applicable when cache
+                        plugin is used with on-disk storage.
+  --filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS
+                        Default: Blocks Facebook. Comma separated list of IPv4 and IPv6
+                        addresses.
+  --enable-web-server   Default: False. Whether to enable proxy.HttpWebServerPlugin.
+  --enable-static-server
+                        Default: False. Enable inbuilt static file server. Optionally, also
+                        use --static-server-dir to serve static content from custom
+                        directory. By default, static file server serves out of installed
+                        proxy.py python module folder.
   --static-server-dir STATIC_SERVER_DIR
-                        Default: "public" folder in directory where proxy.py is placed. This option is only applicable when static server is also enabled. See --enable-static-server.
-  --pac-file PAC_FILE   A file (Proxy Auto Configuration) or string to serve when the server receives a direct file request. Using this option enables proxy.HttpWebServerPlugin.
+                        Default: "public" folder in directory where proxy.py is placed. This
+                        option is only applicable when static server is also enabled. See
+                        --enable-static-server.
+  --pac-file PAC_FILE   A file (Proxy Auto Configuration) or string to serve when the server
+                        receives a direct file request. Using this option enables
+                        proxy.HttpWebServerPlugin.
   --pac-file-url-path PAC_FILE_URL_PATH
                         Default: /. Web server path to serve the PAC file.
   --filtered-client-ips FILTERED_CLIENT_IPS
-                        Default: 127.0.0.1,::1. Comma separated list of IPv4 and IPv6 addresses.
+                        Default: 127.0.0.1,::1. Comma separated list of IPv4 and IPv6
+                        addresses.
+  --filtered-url-regex-config FILTERED_URL_REGEX_CONFIG
+                        Default: No config. Comma separated list of IPv4 and IPv6 addresses.
+  --cloudflare-dns-mode CLOUDFLARE_DNS_MODE
+                        Default: security. Either "security" (for malware protection) or
+                        "family" (for malware and adult content protection)
 
 Proxy.py not working? Report at: https://github.com/abhinavsingh/proxy.py/issues/new
 ```

examples/pubsub_eventing.py

@@ -14,8 +14,7 @@
 
 from typing import Dict, Any
 
-from proxy.core.event import EventQueue, EventSubscriber, eventNames
-from proxy.core.event.manager import EventManager
+from proxy.core.event import EventManager, EventQueue, EventSubscriber, eventNames
 
 # Enable debug logging to view core event logs
 logging.basicConfig(level=logging.DEBUG)

proxy/common/flag.py

@@ -14,15 +14,19 @@
 import socket
 import argparse
 import ipaddress
+import collections
 import multiprocessing
 
 from typing import Optional, List, Any, cast
 
 from .types import IpAddress
 from .utils import text_, bytes_, setup_logger, is_py2, set_open_file_limit
-from .utils import get_default_plugins, import_plugin, load_plugins
+from .utils import import_plugin, load_plugins
 from .constants import COMMA, DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_NUM_WORKERS
 from .constants import DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DISABLE_HEADERS, PY2_DEPRECATION_MESSAGE
+from .constants import PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL
+from .constants import PLUGIN_HTTP_PROXY, PLUGIN_INSPECT_TRAFFIC, PLUGIN_PAC_FILE
+from .constants import PLUGIN_WEB_SERVER, PLUGIN_PROXY_AUTH
 
 from .version import __version__
 
@@ -110,7 +114,10 @@ def initialize(
         set_open_file_limit(args.open_file_limit)
 
         # Load plugins
-        default_plugins = [bytes_(p) for p in get_default_plugins(args)]
+        default_plugins = [
+            bytes_(p)
+            for p in FlagParser.get_default_plugins(args)
+        ]
         extra_plugins = [
             p if isinstance(p, type) else bytes_(p)
             for p in opts.get('plugins', args.plugins.split(text_(COMMA)))
@@ -289,5 +296,35 @@ def initialize(
 
         return args
 
+    @staticmethod
+    def get_default_plugins(
+            args: argparse.Namespace,
+    ) -> List[str]:
+        """Prepare list of plugins to load based upon
+        --enable-*, --disable-* and --basic-auth flags.
+        """
+        default_plugins: List[str] = []
+        if args.basic_auth is not None:
+            default_plugins.append(PLUGIN_PROXY_AUTH)
+        if hasattr(args, 'enable_dashboard') and args.enable_dashboard:
+            default_plugins.append(PLUGIN_WEB_SERVER)
+            args.enable_static_server = True
+            default_plugins.append(PLUGIN_DASHBOARD)
+            default_plugins.append(PLUGIN_INSPECT_TRAFFIC)
+            args.enable_events = True
+            args.enable_devtools = True
+        if hasattr(args, 'enable_devtools') and args.enable_devtools:
+            default_plugins.append(PLUGIN_DEVTOOLS_PROTOCOL)
+            default_plugins.append(PLUGIN_WEB_SERVER)
+        if not args.disable_http_proxy:
+            default_plugins.append(PLUGIN_HTTP_PROXY)
+        if args.enable_web_server or \
+                args.pac_file is not None or \
+                args.enable_static_server:
+            default_plugins.append(PLUGIN_WEB_SERVER)
+        if args.pac_file is not None:
+            default_plugins.append(PLUGIN_PAC_FILE)
+        return list(collections.OrderedDict.fromkeys(default_plugins).keys())
+
 
 flags = FlagParser()

proxy/common/utils.py

@@ -15,21 +15,17 @@
 import socket
 import logging
 import inspect
-import argparse
 import importlib
 import functools
 import ipaddress
 import contextlib
-import collections
 
 from types import TracebackType
 from typing import Optional, Dict, Any, List, Tuple, Type, Callable, Union
 
 from .constants import HTTP_1_1, COLON, WHITESPACE, CRLF, DEFAULT_TIMEOUT
 from .constants import DEFAULT_LOG_FILE, DEFAULT_LOG_FORMAT, DEFAULT_LOG_LEVEL
-from .constants import DOT, PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL
-from .constants import PLUGIN_HTTP_PROXY, PLUGIN_INSPECT_TRAFFIC, PLUGIN_PAC_FILE
-from .constants import PLUGIN_WEB_SERVER, PLUGIN_PROXY_AUTH
+from .constants import DOT
 
 if os.name != 'nt':
     import resource
@@ -338,36 +334,6 @@ def import_plugin(plugin: Union[bytes, type]) -> Any:
     return (klass, module_name)
 
 
-def get_default_plugins(
-        args: argparse.Namespace,
-) -> List[str]:
-    """Prepare list of plugins to load based upon
-    --enable-*, --disable-* and --basic-auth flags.
-    """
-    default_plugins: List[str] = []
-    if args.basic_auth is not None:
-        default_plugins.append(PLUGIN_PROXY_AUTH)
-    if args.enable_dashboard:
-        default_plugins.append(PLUGIN_WEB_SERVER)
-        args.enable_static_server = True
-        default_plugins.append(PLUGIN_DASHBOARD)
-        default_plugins.append(PLUGIN_INSPECT_TRAFFIC)
-        args.enable_events = True
-        args.enable_devtools = True
-    if args.enable_devtools:
-        default_plugins.append(PLUGIN_DEVTOOLS_PROTOCOL)
-        default_plugins.append(PLUGIN_WEB_SERVER)
-    if not args.disable_http_proxy:
-        default_plugins.append(PLUGIN_HTTP_PROXY)
-    if args.enable_web_server or \
-            args.pac_file is not None or \
-            args.enable_static_server:
-        default_plugins.append(PLUGIN_WEB_SERVER)
-    if args.pac_file is not None:
-        default_plugins.append(PLUGIN_PAC_FILE)
-    return list(collections.OrderedDict.fromkeys(default_plugins).keys())
-
-
 def set_open_file_limit(soft_limit: int) -> None:
     """Configure open file description soft limit on supported OS."""
     if os.name != 'nt':  # resource module not available on Windows OS

proxy/core/acceptor/pool.py

@@ -28,6 +28,7 @@
 from ...common.flag import flags
 from ...common.constants import DEFAULT_BACKLOG, DEFAULT_IPV6_HOSTNAME
 from ...common.constants import DEFAULT_NUM_WORKERS, DEFAULT_PORT
+from ...common.constants import DEFAULT_PID_FILE
 
 logger = logging.getLogger(__name__)
 
@@ -36,6 +37,13 @@
 LOCK = multiprocessing.Lock()
 
 
+flags.add_argument(
+    '--pid-file',
+    type=str,
+    default=DEFAULT_PID_FILE,
+    help='Default: None. Save parent process ID to a file.',
+)
+
 flags.add_argument(
     '--backlog',
     type=int,

proxy/core/event/manager.py

@@ -17,9 +17,21 @@
 from .queue import EventQueue
 from .dispatcher import EventDispatcher
 
+from ...common.flag import flags
+from ...common.constants import DEFAULT_ENABLE_EVENTS
+
 logger = logging.getLogger(__name__)
 
 
+flags.add_argument(
+    '--enable-events',
+    action='store_true',
+    default=DEFAULT_ENABLE_EVENTS,
+    help='Default: False.  Enables core to dispatch lifecycle events. '
+    'Plugins can be used to subscribe for core events.',
+)
+
+
 class EventManager:
     """Event manager is an encapsulation around various initialization, dispatcher
     start / stop API required for end-to-end eventing.

proxy/dashboard/dashboard.py

@@ -14,7 +14,9 @@
 
 from .plugin import ProxyDashboardWebsocketPlugin
 
+from ..common.flag import flags
 from ..common.utils import build_http_response, bytes_
+from ..common.constants import DEFAULT_ENABLE_DASHBOARD
 from ..http.server import HttpWebServerPlugin, HttpWebServerBasePlugin, httpProtocolTypes
 from ..http.parser import HttpParser
 from ..http.websocket import WebsocketFrame
@@ -23,6 +25,14 @@
 logger = logging.getLogger(__name__)
 
 
+flags.add_argument(
+    '--enable-dashboard',
+    action='store_true',
+    default=DEFAULT_ENABLE_DASHBOARD,
+    help='Default: False.  Enables proxy.py dashboard.',
+)
+
+
 class ProxyDashboard(HttpWebServerBasePlugin):
     """Proxy Dashboard."""