[TlsInterception] GHA integration tests

tests/integration/test_integration.py

@@ -11,16 +11,70 @@
     Test the simplest proxy use scenario for smoke.
 """
 import time
+import pytest
 import tempfile
-from typing import Any, Generator
+
 from pathlib import Path
+from typing import Any, Generator
 from subprocess import Popen, check_output
 
-import pytest
-
 from proxy.common.constants import IS_WINDOWS
 
 
+TLS_INTERCEPTION_FLAGS = ' '.join((
+    '--ca-cert-file', 'ca-cert.pem',
+    '--ca-key-file', 'ca-key.pem',
+    '--ca-signing-key', 'ca-signing-key.pem',
+))
+
+PROXY_PY_FLAGS_INTEGRATION = (
+    ('--threadless'),
+    ('--threadless --local-executor 0'),
+    ('--threaded'),
+)
+
+PROXY_PY_FLAGS_TLS_INTERCEPTION = (
+    ('--threadless ' + TLS_INTERCEPTION_FLAGS),
+    ('--threadless --local-executor 0 ' + TLS_INTERCEPTION_FLAGS),
+    ('--threaded ' + TLS_INTERCEPTION_FLAGS),
+)
+
+PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN = (
+    (
+        '--threadless --plugin proxy.plugin.ModifyChunkResponsePlugin ' +
+        TLS_INTERCEPTION_FLAGS
+    ),
+    (
+        '--threadless --local-executor 0 --plugin proxy.plugin.ModifyChunkResponsePlugin ' +
+        TLS_INTERCEPTION_FLAGS
+    ),
+    (
+        '--threaded --plugin proxy.plugin.ModifyChunkResponsePlugin ' +
+        TLS_INTERCEPTION_FLAGS
+    ),
+)
+
+PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN = (
+    (
+        '--threadless --plugin proxy.plugin.ModifyPostDataPlugin ' +
+        TLS_INTERCEPTION_FLAGS
+    ),
+    (
+        '--threadless --local-executor 0 --plugin proxy.plugin.ModifyPostDataPlugin ' +
+        TLS_INTERCEPTION_FLAGS
+    ),
+    (
+        '--threaded --plugin proxy.plugin.ModifyPostDataPlugin ' +
+        TLS_INTERCEPTION_FLAGS
+    ),
+)
+
+
+@pytest.fixture(scope='session', autouse=True)  # type: ignore[misc]
+def _gen_ca_certificates() -> None:
+    check_output(['make', 'ca-certificates'])
+
+
 # FIXME: Ignore is necessary for as long as pytest hasn't figured out
 # FIXME: typing for their fixtures.
 # Refs:
@@ -42,6 +96,8 @@ def proxy_py_subprocess(request: Any) -> Generator[int, None, None]:
         '--port', '0',
         '--port-file', str(port_file),
         '--enable-web-server',
+        '--num-acceptors', '3',
+        '--num-workers', '3',
     ) + tuple(request.param.split())
     proxy_proc = Popen(proxy_cmd)
     # Needed because port file might not be available immediately
@@ -62,11 +118,7 @@ def proxy_py_subprocess(request: Any) -> Generator[int, None, None]:
 @pytest.mark.smoke  # type: ignore[misc]
 @pytest.mark.parametrize(
     'proxy_py_subprocess',
-    (
-        ('--threadless'),
-        ('--threadless --local-executor 0'),
-        ('--threaded'),
-    ),
+    PROXY_PY_FLAGS_INTEGRATION,
     indirect=True,
 )   # type: ignore[misc]
 @pytest.mark.skipif(
@@ -78,3 +130,53 @@ def test_integration(proxy_py_subprocess: int) -> None:
     this_test_module = Path(__file__)
     shell_script_test = this_test_module.with_suffix('.sh')
     check_output([str(shell_script_test), str(proxy_py_subprocess)])
+
+
+@pytest.mark.smoke  # type: ignore[misc]
+@pytest.mark.parametrize(
+    'proxy_py_subprocess',
+    PROXY_PY_FLAGS_TLS_INTERCEPTION,
+    indirect=True,
+)   # type: ignore[misc]
+@pytest.mark.skipif(
+    IS_WINDOWS,
+    reason='OSError: [WinError 193] %1 is not a valid Win32 application',
+)  # type: ignore[misc]
+def test_integration_with_interception_flags(proxy_py_subprocess: int) -> None:
+    """An acceptance test for TLS interception using ``curl`` through proxy.py."""
+    shell_script_test = Path(__file__).parent / 'test_interception.sh'
+    check_output([str(shell_script_test), str(proxy_py_subprocess)])
+
+
+# @pytest.mark.smoke  # type: ignore[misc]
+# @pytest.mark.parametrize(
+#     'proxy_py_subprocess',
+#     PROXY_PY_FLAGS_MODIFY_CHUNK_RESPONSE_PLUGIN,
+#     indirect=True,
+# )   # type: ignore[misc]
+# @pytest.mark.skipif(
+#     IS_WINDOWS,
+#     reason='OSError: [WinError 193] %1 is not a valid Win32 application',
+# )  # type: ignore[misc]
+# def test_modify_chunk_response_integration(proxy_py_subprocess: int) -> None:
+#     """An acceptance test for :py:class:`~proxy.plugin.ModifyChunkResponsePlugin`
+#     interception using ``curl`` through proxy.py."""
+#     shell_script_test = Path(__file__).parent / 'test_modify_chunk_response.sh'
+#     check_output([str(shell_script_test), str(proxy_py_subprocess)])
+
+
+@pytest.mark.smoke  # type: ignore[misc]
+@pytest.mark.parametrize(
+    'proxy_py_subprocess',
+    PROXY_PY_FLAGS_MODIFY_POST_DATA_PLUGIN,
+    indirect=True,
+)   # type: ignore[misc]
+@pytest.mark.skipif(
+    IS_WINDOWS,
+    reason='OSError: [WinError 193] %1 is not a valid Win32 application',
+)  # type: ignore[misc]
+def test_modify_post_response_integration(proxy_py_subprocess: int) -> None:
+    """An acceptance test for :py:class:`~proxy.plugin.ModifyPostDataPlugin`
+    interception using ``curl`` through proxy.py."""
+    shell_script_test = Path(__file__).parent / 'test_modify_post_data.sh'
+    check_output([str(shell_script_test), str(proxy_py_subprocess)])

tests/integration/test_integration.sh

@@ -1,5 +1,13 @@
 #!/bin/bash
-
+#
+# proxy.py
+# ~~~~~~~~
+# ⚡⚡⚡ Fast, Lightweight, Programmable, TLS interception capable
+#     proxy server for Application debugging, testing and development.
+#
+# :copyright: (c) 2013-present by Abhinav Singh and contributors.
+# :license: BSD, see LICENSE for more details.
+#
 # TODO: Option to also shutdown proxy.py after
 # integration testing is done.  At least on
 # macOS and ubuntu, pkill and kill commands

tests/integration/test_interception.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+#
+# proxy.py
+# ~~~~~~~~
+# ⚡⚡⚡ Fast, Lightweight, Programmable, TLS interception capable
+#     proxy server for Application debugging, testing and development.
+#
+# :copyright: (c) 2013-present by Abhinav Singh and contributors.
+# :license: BSD, see LICENSE for more details.
+#
+# TODO: Option to also shutdown proxy.py after
+# integration testing is done.  At least on
+# macOS and ubuntu, pkill and kill commands
+# will do the job.
+#
+# For github action, we simply bank upon GitHub
+# to clean up any background process including
+# proxy.py
+
+PROXY_PY_PORT=$1
+if [[ -z "$PROXY_PY_PORT" ]]; then
+  echo "PROXY_PY_PORT required as argument."
+  exit 1
+fi
+
+PROXY_URL="127.0.0.1:$PROXY_PY_PORT"
+
+# Wait for server to come up
+WAIT_FOR_PROXY="lsof -i TCP:$PROXY_PY_PORT | wc -l | tr -d ' '"
+while true; do
+    if [[ $WAIT_FOR_PORT == 0 ]]; then
+        echo "Waiting for proxy..."
+        sleep 1
+    else
+        break
+    fi
+done
+
+# Wait for http proxy and web server to start
+while true; do
+    curl -v \
+        --max-time 1 \
+        --connect-timeout 1 \
+        -x $PROXY_URL \
+        --cacert ca-cert.pem \
+        http://$PROXY_URL/ 2>/dev/null
+    if [[ $? == 0 ]]; then
+        break
+    fi
+    echo "Waiting for web server to start accepting requests..."
+    sleep 1
+done
+
+verify_response() {
+    if [ "$1" == "" ];
+    then
+        echo "Empty response";
+        return 1;
+    else
+        if [ "$1" == "$2" ];
+        then
+            echo "Ok";
+            return 0;
+        else
+            echo "Invalid response: '$1', expected: '$2'";
+            return 1;
+        fi
+    fi;
+}
+
+# Check if proxy was started with integration
+# testing web server plugin.  If detected, use
+# internal web server for integration testing.
+
+# If integration testing plugin is not found,
+# detect if we have internet access.  If we do,
+# then use httpbin.org for integration testing.
+read -r -d '' ROBOTS_RESPONSE << EOM
+User-agent: *
+Disallow: /deny
+EOM
+
+echo "[Test HTTP Request via Proxy]"
+CMD="curl -v -x $PROXY_URL --cacert ca-cert.pem http://httpbin.org/robots.txt"
+RESPONSE=$($CMD 2> /dev/null)
+verify_response "$RESPONSE" "$ROBOTS_RESPONSE"
+VERIFIED1=$?
+
+echo "[Test HTTPS Request via Proxy]"
+CMD="curl -v -x $PROXY_URL --cacert ca-cert.pem https://httpbin.org/robots.txt"
+RESPONSE=$($CMD 2> /dev/null)
+verify_response "$RESPONSE" "$ROBOTS_RESPONSE"
+VERIFIED2=$?
+
+echo "[Test Internal Web Server via Proxy]"
+curl -v \
+    -x $PROXY_URL \
+    --cacert ca-cert.pem \
+    http://$PROXY_URL/
+VERIFIED3=$?
+
+SHASUM=sha256sum
+if [ "$(uname)" = "Darwin" ];
+then
+    SHASUM="shasum -a 256"
+fi
+
+echo "[Test Download File Hash Verifies 1]"
+touch downloaded.hash
+echo "3d1921aab49d3464a712c1c1397b6babf8b461a9873268480aa8064da99441bc  -" > downloaded.hash
+curl -vL \
+    -o downloaded.whl \
+    -x $PROXY_URL \
+    --cacert ca-cert.pem \
+    https://files.pythonhosted.org/packages/88/78/e642316313b1cd6396e4b85471a316e003eff968f29773e95ea191ea1d08/proxy.py-2.4.0rc4-py3-none-any.whl#sha256=3d1921aab49d3464a712c1c1397b6babf8b461a9873268480aa8064da99441bc
+cat downloaded.whl | $SHASUM -c downloaded.hash
+VERIFIED4=$?
+rm downloaded.whl downloaded.hash
+
+echo "[Test Download File Hash Verifies 2]"
+touch downloaded.hash
+echo "077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8  -" > downloaded.hash
+curl -vL \
+    -o downloaded.whl \
+    -x $PROXY_URL \
+    --cacert ca-cert.pem \
+    https://files.pythonhosted.org/packages/20/9a/e5d9ec41927401e41aea8af6d16e78b5e612bca4699d417f646a9610a076/Jinja2-3.0.3-py3-none-any.whl#sha256=077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8
+cat downloaded.whl | $SHASUM -c downloaded.hash
+VERIFIED5=$?
+rm downloaded.whl downloaded.hash
+
+EXIT_CODE=$(( $VERIFIED1 || $VERIFIED2 || $VERIFIED3 || $VERIFIED4 || $VERIFIED5 ))
+exit $EXIT_CODE

tests/integration/test_modify_chunk_response.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+#
+# proxy.py
+# ~~~~~~~~
+# ⚡⚡⚡ Fast, Lightweight, Programmable, TLS interception capable
+#     proxy server for Application debugging, testing and development.
+#
+# :copyright: (c) 2013-present by Abhinav Singh and contributors.
+# :license: BSD, see LICENSE for more details.
+#
+# TODO: Option to also shutdown proxy.py after
+# integration testing is done.  At least on
+# macOS and ubuntu, pkill and kill commands
+# will do the job.
+#
+# For github action, we simply bank upon GitHub
+# to clean up any background process including
+# proxy.py
+
+PROXY_PY_PORT=$1
+if [[ -z "$PROXY_PY_PORT" ]]; then
+  echo "PROXY_PY_PORT required as argument."
+  exit 1
+fi
+
+PROXY_URL="127.0.0.1:$PROXY_PY_PORT"
+
+# Wait for server to come up
+WAIT_FOR_PROXY="lsof -i TCP:$PROXY_PY_PORT | wc -l | tr -d ' '"
+while true; do
+    if [[ $WAIT_FOR_PORT == 0 ]]; then
+        echo "Waiting for proxy..."
+        sleep 1
+    else
+        break
+    fi
+done
+
+# Wait for http proxy and web server to start
+while true; do
+    curl -v \
+        --max-time 1 \
+        --connect-timeout 1 \
+        -x $PROXY_URL \
+        --cacert ca-cert.pem \
+        http://$PROXY_URL/ 2>/dev/null
+    if [[ $? == 0 ]]; then
+        break
+    fi
+    echo "Waiting for web server to start accepting requests..."
+    sleep 1
+done
+
+verify_response() {
+    if [ "$1" == "" ];
+    then
+        echo "Empty response";
+        return 1;
+    else
+        if [ "$1" == "$2" ];
+        then
+            echo "Ok";
+            return 0;
+        else
+            echo "Invalid response: '$1', expected: '$2'";
+            return 1;
+        fi
+    fi;
+}
+
+read -r -d '' MODIFIED_CHUNK_RESPONSE << EOM
+modify
+chunk
+response
+plugin
+EOM
+
+echo "[Test ModifyChunkResponsePlugin]"
+CMD="curl -v -x $PROXY_URL --cacert ca-cert.pem https://httpbin.org/stream/5"
+RESPONSE=$($CMD 2> /dev/null)
+verify_response "$RESPONSE" "$MODIFIED_CHUNK_RESPONSE"
+VERIFIED1=$?
+
+EXIT_CODE=$(( $VERIFIED1 ))
+exit $EXIT_CODE