v2.4.0rc4

.github/workflows/test-library.yml

@@ -67,6 +67,7 @@ jobs:
       sdist-artifact-name: ${{ steps.artifact-name.outputs.sdist }}
       wheel-artifact-name: ${{ steps.artifact-name.outputs.wheel }}
       container-version: v${{ steps.container.outputs.version }}
+      container-platforms: ${{ steps.container.outputs.platforms }}
     steps:
     - name: Switch to using Python 3.9 by default
       uses: actions/setup-python@v2
@@ -204,7 +205,9 @@ jobs:
             && github.event.inputs.release-version
             || steps.scm-version.outputs.dist-version
         }}' | tr + .);
-        echo "::set-output name=version::$VER"
+        PLATFORMS="linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/s390x";
+        echo "::set-output name=version::$VER";
+        echo "::set-output name=platforms::$PLATFORMS"
 
   build:
     name: 👷 dists ${{ needs.pre-setup.outputs.git-tag }}
@@ -727,19 +730,36 @@ jobs:
         --local-executor && ./tests/integration/test_integration.sh 8899
     - name: Push to GHCR
       run: >-
-        PLATFORMS=linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/s390x;
         REGISTRY_URL="ghcr.io/abhinavsingh/proxy.py";
         CONTAINER_TAG=$REGISTRY_URL:${{
           needs.pre-setup.outputs.container-version
         }};
+        docker buildx build
+        --push
+        --platform ${{
+          needs.pre-setup.outputs.container-platforms
+        }}
+        --build-arg PROXYPY_PKG_PATH='dist/${{
+          needs.pre-setup.outputs.wheel-artifact-name
+        }}'
+        -t $CONTAINER_TAG .
+    - name: Tag latest on GHCR
+      if: >-
+        github.event_name == 'push' &&
+        github.ref == format(
+          'refs/heads/{0}', github.event.repository.default_branch
+        )
+      run: >-
+        REGISTRY_URL="ghcr.io/abhinavsingh/proxy.py";
         LATEST_TAG=$REGISTRY_URL:latest;
         docker buildx build
         --push
-        --platform $PLATFORMS
+        --platform ${{
+          needs.pre-setup.outputs.container-platforms
+        }}
         --build-arg PROXYPY_PKG_PATH='dist/${{
           needs.pre-setup.outputs.wheel-artifact-name
         }}'
-        -t $CONTAINER_TAG
         -t $LATEST_TAG .
     - name: Login to DockerHub
       uses: docker/login-action@v1
@@ -748,14 +768,15 @@ jobs:
         password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
     - name: Push to DockerHub
       run: >-
-        PLATFORMS=linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/s390x;
         REGISTRY_URL="abhinavsingh/proxy.py";
         CONTAINER_TAG=$REGISTRY_URL:${{
           needs.pre-setup.outputs.container-version
         }};
         docker buildx build
         --push
-        --platform $PLATFORMS
+        --platform ${{
+          needs.pre-setup.outputs.container-platforms
+        }}
         --build-arg PROXYPY_PKG_PATH='dist/${{
           needs.pre-setup.outputs.wheel-artifact-name
         }}'
@@ -839,35 +860,6 @@ jobs:
         password: ${{ secrets.TESTPYPI_API_TOKEN }}
         repository_url: https://test.pypi.org/legacy/
 
-  # publish-docker:
-  #   name: Publish 🐳 📦 ${{ needs.pre-setup.outputs.git-tag }} to Docker Hub
-  #   needs:
-  #   - check
-  #   - pre-setup  # transitive, for accessing settings
-  #   if: >-
-  #     fromJSON(needs.pre-setup.outputs.release-requested)
-  #   runs-on: Ubuntu-latest
-
-  #   environment:
-  #     name: release-docker
-  #     url: >-
-  #       https://test.pypi.org/project/proxy.py/${{
-  #         needs.pre-setup.outputs.dist-version
-  #       }}
-
-  #   steps:
-  #   - name: Download all the dists
-  #     uses: actions/download-artifact@v2
-  #     with:
-  #       name: python-package-distributions
-  #       path: dist/
-  #   - name: >-
-  #       Publish 🐳 📦 ${{ needs.pre-setup.outputs.git-tag }} to Docker Hub
-  #     uses: pypa/gh-action-pypi-publish@release/v1
-  #     with:
-  #       password: ${{ secrets.TESTPYPI_API_TOKEN }}
-  #       repository_url: https://test.pypi.org/legacy/
-
   post-release-repo-update:
     name: >-
       Publish post-release Git tag

benchmark/requirements.txt

@@ -1,5 +1,5 @@
 aiohttp==3.8.1
-blacksheep==1.2.1
+blacksheep==1.2.2
 starlette==0.17.1
 tornado==6.1
 uvicorn==0.16.0

dashboard/package.json

@@ -30,7 +30,7 @@
     "@types/js-cookie": "^3.0.1",
     "@typescript-eslint/eslint-plugin": "^2.34.0",
     "@typescript-eslint/parser": "^2.34.0",
-    "chrome-devtools-frontend": "^1.0.949424",
+    "chrome-devtools-frontend": "^1.0.952865",
     "eslint": "^6.8.0",
     "eslint-config-standard": "^14.1.1",
     "eslint-plugin-import": "^2.25.3",
@@ -40,7 +40,7 @@
     "http-server": "^14.0.0",
     "jasmine": "^3.10.0",
     "jasmine-ts": "^0.3.0",
-    "jquery": "^3.5.1",
+    "jquery": "^3.6.0",
     "js-cookie": "^3.0.1",
     "jsdom": "^15.2.1",
     "ncp": "^2.0.0",

helper/benchmark.sh

@@ -64,8 +64,6 @@ run_benchmark
 
 POST_RUN_OPEN_FILES=$(./helper/monitor_open_files.sh)
 
-echo $output
-
 echo "Open files diff:"
 diff <( echo "$PRE_RUN_OPEN_FILES" ) <( echo "$POST_RUN_OPEN_FILES" )
 

proxy/common/backports.py

@@ -55,6 +55,7 @@ def randint(self):
        backports
        getter
        Arndt
+       del
     """
 
     def __init__(self, ttl: float = 0):

proxy/common/flag.py

@@ -22,7 +22,7 @@
 from ._compat import IS_WINDOWS  # noqa: WPS436
 from .plugins import Plugins
 from .types import IpAddress
-from .utils import bytes_, is_py2, set_open_file_limit
+from .utils import bytes_, is_py2, is_threadless, set_open_file_limit
 from .constants import COMMA, DEFAULT_DATA_DIRECTORY_PATH, DEFAULT_NUM_ACCEPTORS, DEFAULT_NUM_WORKERS
 from .constants import DEFAULT_DEVTOOLS_WS_PATH, DEFAULT_DISABLE_HEADERS, PY2_DEPRECATION_MESSAGE
 from .constants import PLUGIN_DASHBOARD, PLUGIN_DEVTOOLS_PROTOCOL, DEFAULT_MIN_COMPRESSION_LIMIT
@@ -186,7 +186,7 @@ def initialize(
             for p in FlagParser.get_default_plugins(args)
         ]
         requested_plugins = Plugins.resolve_plugin_flag(
-            args.plugins, opts.get('plugins', None),
+            args.plugins, opts.get('plugins'),
         )
         plugins = Plugins.load(
             default_plugins + auth_plugins + requested_plugins,
@@ -340,21 +340,29 @@ def initialize(
             ),
         )
         args.timeout = cast(int, opts.get('timeout', args.timeout))
-        args.threadless = cast(bool, opts.get('threadless', args.threadless))
-        args.threaded = cast(bool, opts.get('threaded', args.threaded))
-        args.pid_file = cast(
-            Optional[str], opts.get(
-                'pid_file',
-                args.pid_file,
-            ),
-        )
         args.local_executor = cast(
             bool,
             opts.get(
                 'local_executor',
                 args.local_executor,
             ),
         )
+        args.threaded = cast(bool, opts.get('threaded', args.threaded))
+        # Pre-evaluate threadless values based upon environment and config
+        #
+        # --threadless is now default mode of execution
+        # but we still have exceptions based upon OS config.
+        # Make sure executors are not started if is_threadless
+        # evaluates to False.
+        args.threadless = cast(bool, opts.get('threadless', args.threadless))
+        args.threadless = is_threadless(args.threadless, args.threaded)
+
+        args.pid_file = cast(
+            Optional[str], opts.get(
+                'pid_file',
+                args.pid_file,
+            ),
+        )
 
         args.proxy_py_data_dir = DEFAULT_DATA_DIRECTORY_PATH
         os.makedirs(args.proxy_py_data_dir, exist_ok=True)

proxy/common/pki.py

@@ -298,16 +298,16 @@ def run_openssl_command(command: List[str], timeout: int) -> bool:
             ', '.join(available_actions),
         )
         sys.exit(1)
-    if args.action in ('gen_private_key', 'gen_public_key'):
-        if args.private_key_path is None:
-            logger.error('--private-key-path is required for ' + args.action)
-            sys.exit(1)
-    if args.action == 'gen_public_key':
-        if args.public_key_path is None:
-            logger.error(
-                '--public-key-file is required for private key generation',
-            )
-            sys.exit(1)
+    if args.action in ('gen_private_key', 'gen_public_key') and \
+            args.private_key_path is None:
+        logger.error('--private-key-path is required for ' + args.action)
+        sys.exit(1)
+    if args.action == 'gen_public_key' and \
+            args.public_key_path is None:
+        logger.error(
+            '--public-key-file is required for private key generation',
+        )
+        sys.exit(1)
 
     # Execute
     if args.action == 'gen_private_key':

proxy/common/plugins.py

@@ -74,12 +74,18 @@ def load(
             mro = list(inspect.getmro(klass))
             mro.reverse()
             iterator = iter(mro)
-            while next(iterator) is not abc.ABC:
-                pass
-            base_klass = next(iterator)
-            if klass not in p[bytes_(base_klass.__name__)]:
-                p[bytes_(base_klass.__name__)].append(klass)
-            logger.info('Loaded plugin %s.%s', module_name, klass.__name__)
+            try:
+                while next(iterator) is not abc.ABC:
+                    pass
+                base_klass = next(iterator)
+                if klass not in p[bytes_(base_klass.__name__)]:
+                    p[bytes_(base_klass.__name__)].append(klass)
+                logger.info('Loaded plugin %s.%s', module_name, klass.__name__)
+            except StopIteration:
+                logger.warn(
+                    '%s is NOT a valid plugin',
+                    text_(plugin_),
+                )
         return p
 
     @staticmethod

proxy/common/utils.py

@@ -99,7 +99,7 @@ def build_http_response(
         headers = {}
     has_content_length = False
     has_transfer_encoding = False
-    for k in headers:
+    for k, _ in headers.items():
         if k.lower() == b'content-length':
             has_content_length = True
         if k.lower() == b'transfer-encoding':
@@ -124,8 +124,8 @@ def build_http_pkt(
     """Build and returns a HTTP request or response packet."""
     pkt = WHITESPACE.join(line) + CRLF
     if headers is not None:
-        for k in headers:
-            pkt += build_http_header(k, headers[k]) + CRLF
+        for k, v in headers.items():
+            pkt += build_http_header(k, v) + CRLF
     pkt += CRLF
     if body:
         pkt += body

proxy/core/acceptor/acceptor.py

@@ -26,7 +26,6 @@
 from typing import List, Optional, Tuple
 
 from ...common.flag import flags
-from ...common.utils import is_threadless
 from ...common.logger import Logger
 from ...common.backports import NonBlockingQueue
 from ...common.constants import DEFAULT_LOCAL_EXECUTOR
@@ -113,17 +112,17 @@ def accept(
     ) -> List[Tuple[socket.socket, Optional[Tuple[str, int]]]]:
         works = []
         for _, mask in events:
-            if mask & selectors.EVENT_READ:
-                if self.sock is not None:
-                    try:
-                        conn, addr = self.sock.accept()
-                        logging.debug(
-                            'Accepting new work#{0}'.format(conn.fileno()),
-                        )
-                        works.append((conn, addr or None))
-                    except BlockingIOError:
-                        # logger.info('blocking io error')
-                        pass
+            if mask & selectors.EVENT_READ and \
+                    self.sock is not None:
+                try:
+                    conn, addr = self.sock.accept()
+                    logging.debug(
+                        'Accepting new work#{0}'.format(conn.fileno()),
+                    )
+                    works.append((conn, addr or None))
+                except BlockingIOError:
+                    # logger.info('blocking io error')
+                    pass
         return works
 
     def run_once(self) -> None:
@@ -207,7 +206,7 @@ def _stop_local(self) -> None:
 
     def _work(self, conn: socket.socket, addr: Optional[Tuple[str, int]]) -> None:
         self._total = self._total or 0
-        if is_threadless(self.flags.threadless, self.flags.threaded):
+        if self.flags.threadless:
             # Index of worker to which this work should be dispatched
             # Use round-robin strategy by default.
             #