Allow --plugins flag to be used multiple times (#725)

* deprecate server_file_or_404

* Optionally compress static content.  Currently only if content length higher than 300

* trailing comma

* Allow `--plugins` flag to be used multiple times

Following are valid invocation:
1) `--plugins A`
2) `--plugins A,B`
3) `--plugins A --plugins B`
4) `--plugins A,B --plugins C`

* mypy

* Flake8

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* correct type

* Add `HttpParser.is_https_tunnel()` utility method

* mypy

* lint checks

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: abhinavsingh

README.md

@@ -1941,24 +1941,24 @@ for list of tests.
 
 ```console
 ❯ proxy -h
-usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless] [--threaded]
-          [--num-workers NUM_WORKERS] [--pid-file PID_FILE] [--backlog BACKLOG]
-          [--hostname HOSTNAME] [--port PORT] [--num-acceptors NUM_ACCEPTORS]
-          [--unix-socket-path UNIX_SOCKET_PATH]
-          [--client-recvbuf-size CLIENT_RECVBUF_SIZE] [--key-file KEY_FILE]
-          [--timeout TIMEOUT] [--version] [--log-level LOG_LEVEL]
+usage: proxy [-h] [--enable-events] [--enable-conn-pool] [--threadless] [--threaded]
+          [--num-workers NUM_WORKERS] [--backlog BACKLOG] [--hostname HOSTNAME]
+          [--port PORT] [--unix-socket-path UNIX_SOCKET_PATH]
+          [--num-acceptors NUM_ACCEPTORS] [--version] [--log-level LOG_LEVEL]
           [--log-file LOG_FILE] [--log-format LOG_FORMAT]
-          [--open-file-limit OPEN_FILE_LIMIT] [--plugins PLUGINS]
-          [--enable-dashboard] [--work-klass WORK_KLASS] [--disable-http-proxy]
-          [--ca-key-file CA_KEY_FILE] [--ca-cert-dir CA_CERT_DIR]
-          [--ca-cert-file CA_CERT_FILE] [--ca-file CA_FILE]
-          [--ca-signing-key-file CA_SIGNING_KEY_FILE] [--cert-file CERT_FILE]
-          [--disable-headers DISABLE_HEADERS]
+          [--open-file-limit OPEN_FILE_LIMIT] [--plugins PLUGINS [PLUGINS ...]]
+          [--enable-dashboard] [--work-klass WORK_KLASS] [--pid-file PID_FILE]
+          [--client-recvbuf-size CLIENT_RECVBUF_SIZE] [--key-file KEY_FILE]
+          [--timeout TIMEOUT] [--disable-http-proxy] [--ca-key-file CA_KEY_FILE]
+          [--ca-cert-dir CA_CERT_DIR] [--ca-cert-file CA_CERT_FILE]
+          [--ca-file CA_FILE] [--ca-signing-key-file CA_SIGNING_KEY_FILE]
+          [--cert-file CERT_FILE] [--disable-headers DISABLE_HEADERS]
           [--server-recvbuf-size SERVER_RECVBUF_SIZE] [--basic-auth BASIC_AUTH]
           [--cache-dir CACHE_DIR]
           [--filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS] [--enable-web-server]
           [--enable-static-server] [--static-server-dir STATIC_SERVER_DIR]
-          [--pac-file PAC_FILE] [--pac-file-url-path PAC_FILE_URL_PATH]
+          [--min-compression-length MIN_COMPRESSION_LENGTH] [--pac-file PAC_FILE]
+          [--pac-file-url-path PAC_FILE_URL_PATH]
           [--filtered-client-ips FILTERED_CLIENT_IPS]
           [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
           [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE]
@@ -1971,50 +1971,51 @@ options:
                         Plugins can be used to subscribe for core events.
   --enable-conn-pool    Default: False. (WIP) Enable upstream connection pooling.
   --threadless          Default: True. Enabled by default on Python 3.8+ (mac,
-                        linux). When disabled a new thread is spawned to handle
-                        each client connection.
+                        linux). When disabled a new thread is spawned to handle each
+                        client connection.
   --threaded            Default: False. Disabled by default on Python < 3.8 and
-                        windows. When enabled a new thread is spawned to handle
-                        each client connection.
+                        windows. When enabled a new thread is spawned to handle each
+                        client connection.
   --num-workers NUM_WORKERS
                         Defaults to number of CPU cores.
-  --pid-file PID_FILE   Default: None. Save parent process ID to a file.
-  --backlog BACKLOG     Default: 100. Maximum number of pending connections to
-                        proxy server
+  --backlog BACKLOG     Default: 100. Maximum number of pending connections to proxy
+                        server
   --hostname HOSTNAME   Default: ::1. Server IP address.
   --port PORT           Default: 8899. Server port.
+  --unix-socket-path UNIX_SOCKET_PATH
+                        Default: None. Unix socket path to use. When provided --host
+                        and --port flags are ignored
   --num-acceptors NUM_ACCEPTORS
                         Defaults to number of CPU cores.
-  --unix-socket-path UNIX_SOCKET_PATH
-                        Default: None. Unix socket path to use. When provided
-                        --host and --port flags are ignored
-  --client-recvbuf-size CLIENT_RECVBUF_SIZE
-                        Default: 1 MB. Maximum amount of data received from the
-                        client in a single recv() operation. Bump this value for
-                        faster uploads at the expense of increased RAM.
-  --key-file KEY_FILE   Default: None. Server key file to enable end-to-end TLS
-                        encryption with clients. If used, must also pass --cert-
-                        file.
-  --timeout TIMEOUT     Default: 10.0. Number of seconds after which an inactive
-                        connection must be dropped. Inactivity is defined by no
-                        data sent or received by the client.
   --version, -v         Prints proxy.py version.
   --log-level LOG_LEVEL
                         Valid options: DEBUG, INFO (default), WARNING, ERROR,
                         CRITICAL. Both upper and lowercase values are allowed. You
-                        may also simply use the leading character e.g. --log-level
-                        d
+                        may also simply use the leading character e.g. --log-level d
   --log-file LOG_FILE   Default: sys.stdout. Log file destination.
   --log-format LOG_FORMAT
                         Log format for Python logger.
   --open-file-limit OPEN_FILE_LIMIT
                         Default: 1024. Maximum number of files (TCP connections)
                         that proxy.py can open concurrently.
-  --plugins PLUGINS     Comma separated plugins
+  --plugins PLUGINS [PLUGINS ...]
+                        Comma separated plugins. You may use --plugins flag multiple
+                        times.
   --enable-dashboard    Default: False. Enables proxy.py dashboard.
   --work-klass WORK_KLASS
-                        Default: proxy.http.handler.HttpProtocolHandler. Work klass
-                        to use for work execution.
+                        Default: proxy.http.HttpProtocolHandler. Work klass to use
+                        for work execution.
+  --pid-file PID_FILE   Default: None. Save "parent" process ID to a file.
+  --client-recvbuf-size CLIENT_RECVBUF_SIZE
+                        Default: 1 MB. Maximum amount of data received from the
+                        client in a single recv() operation. Bump this value for
+                        faster uploads at the expense of increased RAM.
+  --key-file KEY_FILE   Default: None. Server key file to enable end-to-end TLS
+                        encryption with clients. If used, must also pass --cert-
+                        file.
+  --timeout TIMEOUT     Default: 10.0. Number of seconds after which an inactive
+                        connection must be dropped. Inactivity is defined by no data
+                        sent or received by the client.
   --disable-http-proxy  Default: False. Whether to disable proxy.HttpProxyPlugin.
   --ca-key-file CA_KEY_FILE
                         Default: None. CA key to use for signing dynamically
@@ -2026,19 +2027,18 @@ options:
                         file and --ca-signing-key-file
   --ca-cert-file CA_CERT_FILE
                         Default: None. Signing certificate to use for signing
-                        dynamically generated HTTPS certificates. If used, must
-                        also pass --ca-key-file and --ca-signing-key-file
-  --ca-file CA_FILE     Default: /Users/abhinavsingh/Dev/proxy.py/venv310/lib/pytho
-                        n3.10/site-packages/certifi/cacert.pem. Provide path to
+                        dynamically generated HTTPS certificates. If used, must also
+                        pass --ca-key-file and --ca-signing-key-file
+  --ca-file CA_FILE     Default: /Users/abhinavsingh/Dev/proxy.py/venv310/lib/python
+                        3.10/site-packages/certifi/cacert.pem. Provide path to
                         custom CA bundle for peer certificate verification
   --ca-signing-key-file CA_SIGNING_KEY_FILE
                         Default: None. CA signing key to use for dynamic generation
-                        of HTTPS certificates. If used, must also pass --ca-key-
-                        file and --ca-cert-file
+                        of HTTPS certificates. If used, must also pass --ca-key-file
+                        and --ca-cert-file
   --cert-file CERT_FILE
                         Default: None. Server certificate to enable end-to-end TLS
-                        encryption with clients. If used, must also pass --key-
-                        file.
+                        encryption with clients. If used, must also pass --key-file.
   --disable-headers DISABLE_HEADERS
                         Default: None. Comma separated list of headers to remove
                         before dispatching client request to upstream server.
@@ -2055,8 +2055,7 @@ options:
   --filtered-upstream-hosts FILTERED_UPSTREAM_HOSTS
                         Default: Blocks Facebook. Comma separated list of IPv4 and
                         IPv6 addresses.
-  --enable-web-server   Default: False. Whether to enable
-                        proxy.HttpWebServerPlugin.
+  --enable-web-server   Default: False. Whether to enable proxy.HttpWebServerPlugin.
   --enable-static-server
                         Default: False. Enable inbuilt static file server.
                         Optionally, also use --static-server-dir to serve static
@@ -2065,11 +2064,14 @@ options:
                         folder.
   --static-server-dir STATIC_SERVER_DIR
                         Default: "public" folder in directory where proxy.py is
-                        placed. This option is only applicable when static server
-                        is also enabled. See --enable-static-server.
+                        placed. This option is only applicable when static server is
+                        also enabled. See --enable-static-server.
+  --min-compression-length MIN_COMPRESSION_LENGTH
+                        Default: 20 bytes. Sets the minimum length of a response
+                        that will be compressed (gzipped).
   --pac-file PAC_FILE   A file (Proxy Auto Configuration) or string to serve when
-                        the server receives a direct file request. Using this
-                        option enables proxy.HttpWebServerPlugin.
+                        the server receives a direct file request. Using this option
+                        enables proxy.HttpWebServerPlugin.
   --pac-file-url-path PAC_FILE_URL_PATH
                         Default: /. Web server path to serve the PAC file.
   --filtered-client-ips FILTERED_CLIENT_IPS
@@ -2083,8 +2085,7 @@ options:
                         protection) or "family" (for malware and adult content
                         protection)
 
-Proxy.py not working? Report at:
-https://github.com/abhinavsingh/proxy.py/issues/new
+Proxy.py not working? Report at: https://github.com/abhinavsingh/proxy.py/issues/new
 ```
 
 # Changelog

examples/https_connect_tunnel.py

@@ -16,7 +16,6 @@
 from proxy.common.utils import build_http_response
 from proxy.http.codes import httpStatusCodes
 from proxy.http.parser import httpParserStates
-from proxy.http.methods import httpMethods
 from proxy.core.base import BaseTcpTunnelHandler
 
 
@@ -51,7 +50,7 @@ def handle_data(self, data: memoryview) -> Optional[bool]:
         self.request.parse(data)
 
         # Drop the request if not a CONNECT request
-        if self.request.method != httpMethods.CONNECT:
+        if not self.request.is_https_tunnel():
             self.work.queue(
                 HttpsConnectTunnelHandler.PROXY_TUNNEL_UNSUPPORTED_SCHEME,
             )

proxy/common/constants.py

@@ -16,7 +16,7 @@
 import sysconfig
 import ipaddress
 
-from typing import List
+from typing import Any, List
 
 from .version import __version__
 
@@ -93,10 +93,11 @@ def _env_threadless_compliant() -> bool:
 DEFAULT_PAC_FILE = None
 DEFAULT_PAC_FILE_URL_PATH = b'/'
 DEFAULT_PID_FILE = None
-DEFAULT_PLUGINS = ''
+DEFAULT_PLUGINS: List[Any] = []
 DEFAULT_PORT = 8899
 DEFAULT_SERVER_RECVBUF_SIZE = DEFAULT_BUFFER_SIZE
 DEFAULT_STATIC_SERVER_DIR = os.path.join(PROXY_PY_DIR, "public")
+DEFAULT_MIN_COMPRESSION_LIMIT = 20  # In bytes
 DEFAULT_THREADLESS = _env_threadless_compliant()
 DEFAULT_TIMEOUT = 10.0
 DEFAULT_VERSION = False

proxy/common/flag.py

@@ -21,10 +21,10 @@
 
 from .plugins import Plugins
 from .types import IpAddress
-from .utils import text_, bytes_, is_py2, set_open_file_limit
+from .utils import bytes_, is_py2, set_open_file_limit
 from .constants import COMMA, DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_NUM_ACCEPTORS, DEFAULT_NUM_WORKERS
 from .constants import DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DISABLE_HEADERS, PY2_DEPRECATION_MESSAGE
-from .constants import PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL
+from .constants import PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL, DEFAULT_MIN_COMPRESSION_LIMIT
 from .constants import PLUGIN_HTTP_PROXY, PLUGIN_INSPECT_TRAFFIC, PLUGIN_PAC_FILE
 from .constants import PLUGIN_WEB_SERVER, PLUGIN_PROXY_AUTH
 from .logger import Logger
@@ -110,6 +110,9 @@ def initialize(
         # proxy.py currently cannot serve over HTTPS and also perform TLS interception
         # at the same time.  Check if user is trying to enable both feature
         # at the same time.
+        #
+        # TODO: Use parser.add_mutually_exclusive_group()
+        # and remove this logic from here.
         if (args.cert_file and args.key_file) and \
                 (args.ca_key_file and args.ca_cert_file and args.ca_signing_key_file):
             print(
@@ -140,18 +143,16 @@ def initialize(
             bytes_(p)
             for p in FlagParser.get_default_plugins(args)
         ]
-        extra_plugins = [
-            p if isinstance(p, type) else bytes_(p)
-            for p in opts.get('plugins', args.plugins.split(text_(COMMA)))
-            if not (isinstance(p, str) and len(p) == 0)
-        ]
-        plugins = Plugins.load(default_plugins + extra_plugins)
+        plugins = Plugins.load(
+            default_plugins + Plugins.resolve_plugin_flag(
+                args.plugins, opts.get('plugins', None),
+            ),
+        )
 
         # https://github.com/python/mypy/issues/5865
         #
         # def option(t: object, key: str, default: Any) -> Any:
         #     return cast(t, opts.get(key, default))
-
         args.work_klass = work_klass
         args.plugins = plugins
         args.auth_code = cast(
@@ -284,6 +285,16 @@ def initialize(
                 args.enable_static_server,
             ),
         )
+        args.min_compression_limit = cast(
+            bool,
+            opts.get(
+                'min_compression_limit',
+                getattr(
+                    args, 'min_compression_limit',
+                    DEFAULT_MIN_COMPRESSION_LIMIT,
+                ),
+            ),
+        )
         args.devtools_ws_path = cast(
             bytes,
             opts.get(

proxy/common/plugins.py

@@ -12,19 +12,38 @@
 import abc
 import logging
 import inspect
+import itertools
 import importlib
 
 from typing import Any, List, Dict, Optional, Union
 
 from .utils import bytes_, text_
-from .constants import DOT, DEFAULT_ABC_PLUGINS
+from .constants import DOT, DEFAULT_ABC_PLUGINS, COMMA
 
 logger = logging.getLogger(__name__)
 
 
 class Plugins:
     """Common utilities for plugin discovery."""
 
+    @staticmethod
+    def resolve_plugin_flag(flag_plugins: Any, opt_plugins: Optional[Any] = None) -> List[Union[bytes, type]]:
+        if isinstance(flag_plugins, list):
+            requested_plugins = list(
+                itertools.chain.from_iterable([
+                    p.split(text_(COMMA)) for p in list(
+                        itertools.chain.from_iterable(flag_plugins),
+                    )
+                ]),
+            )
+        else:
+            requested_plugins = flag_plugins.split(text_(COMMA))
+        return [
+            p if isinstance(p, type) else bytes_(p)
+            for p in (opt_plugins if opt_plugins is not None else requested_plugins)
+            if not (isinstance(p, str) and len(p) == 0)
+        ]
+
     @staticmethod
     def discover(input_args: List[str]) -> None:
         """Search for plugin and plugins flag in command line arguments,

proxy/core/connection/connection.py

@@ -56,6 +56,7 @@ def connection(self) -> Union[ssl.SSLSocket, socket.socket]:
 
     def send(self, data: bytes) -> int:
         """Users must handle BrokenPipeError exceptions"""
+        # logger.info(data)
         return self.connection.send(data)
 
     def recv(

proxy/dashboard/dashboard.py

@@ -67,6 +67,7 @@ def handle_request(self, request: HttpParser) -> None:
                         self.flags.static_server_dir,
                         'dashboard', 'proxy.html',
                     ),
+                    self.flags.min_compression_limit,
                 ),
             )
         elif request.path in (

proxy/http/parser.py

@@ -138,11 +138,14 @@ def set_url(self, url: bytes) -> None:
         # For CONNECT requests, request line contains
         # upstream_host:upstream_port which is not complaint
         # with urlsplit, which expects a fully qualified url.
-        if self.method == httpMethods.CONNECT:
+        if self.is_https_tunnel():
             url = b'https://' + url
         self._url = urlparse.urlsplit(url)
         self._set_line_attributes()
 
+    def is_https_tunnel(self) -> bool:
+        return self.method == httpMethods.CONNECT
+
     def is_chunked_encoded(self) -> bool:
         return b'transfer-encoding' in self.headers and \
                self.headers[b'transfer-encoding'][1].lower() == b'chunked'
@@ -184,7 +187,7 @@ def build(self, disable_headers: Optional[List[bytes]] = None, for_proxy: bool =
                 COLON +
                 str(self.port).encode() +
                 self.path
-            ) if self.method != httpMethods.CONNECT else (self.host + COLON + str(self.port).encode())
+            ) if not self.is_https_tunnel() else (self.host + COLON + str(self.port).encode())
         return build_http_request(
             self.method, path, self.version,
             headers={} if not self.headers else {
@@ -305,7 +308,7 @@ def _get_body_or_chunks(self) -> Optional[bytes]:
 
     def _set_line_attributes(self) -> None:
         if self.type == httpParserTypes.REQUEST_PARSER:
-            if self.method == httpMethods.CONNECT and self._url:
+            if self.is_https_tunnel() and self._url:
                 self.host = self._url.hostname
                 self.port = 443 if self._url.port is None else self._url.port
             elif self._url: