GH-86275: Implementation of hypothesis stubs for property-based tests, with zoneinfo tests

.github/workflows/build.yml

@@ -36,6 +36,7 @@ jobs:
     timeout-minutes: 10
     outputs:
       run_tests: ${{ steps.check.outputs.run_tests }}
+      run_hypothesis: ${{ steps.check.outputs.run_hypothesis }}
     steps:
       - uses: actions/checkout@v3
       - name: Check for source changes
@@ -61,6 +62,17 @@ jobs:
             git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true
           fi
 
+          # Check if we should run hypothesis tests
+          GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}
+          echo $GIT_BRANCH
+          if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then
+            echo "Branch too old for hypothesis tests"
+            echo "run_hypothesis=false" >> $GITHUB_OUTPUT
+          else
+            echo "Run hypothesis tests"
+            echo "run_hypothesis=true" >> $GITHUB_OUTPUT
+          fi
+
   check_generated_files:
     name: 'Check if generated files are up to date'
     runs-on: ubuntu-latest
@@ -291,6 +303,90 @@ jobs:
     - name: SSL tests
       run: ./python Lib/test/ssltests.py
 
+  test_hypothesis:
+    name: "Hypothesis Tests on Ubuntu"
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
+    env:
+      OPENSSL_VER: 1.1.1t
+      PYTHONSTRICTEXTENSIONBUILD: 1
+    steps:
+    - uses: actions/checkout@v3
+    - name: Register gcc problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/gcc.json"
+    - name: Install Dependencies
+      run: sudo ./.github/workflows/posix-deps-apt.sh
+    - name: Configure OpenSSL env vars
+      run: |
+        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
+        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
+    - name: 'Restore OpenSSL build'
+      id: cache-openssl
+      uses: actions/cache@v3
+      with:
+        path: ./multissl/openssl/${{ env.OPENSSL_VER }}
+        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+    - name: Install OpenSSL
+      if: steps.cache-openssl.outputs.cache-hit != 'true'
+      run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
+    - name: Add ccache to PATH
+      run: |
+        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+    - name: Configure ccache action
+      uses: hendrikmuhs/[email protected]
+    - name: Setup directory envs for out-of-tree builds
+      run: |
+        echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV
+        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+    - name: Create directories for read-only out-of-tree builds
+      run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
+    - name: Bind mount sources read-only
+      run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Configure CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR
+    - name: Build CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: make -j4
+    - name: Display build info
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: make pythoninfo
+    - name: Remount sources writable for tests
+      # some tests write to srcdir, lack of pyc files slows down testing
+      run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
+    - name: Setup directory envs for out-of-tree builds
+      run: |
+        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+    - name: "Create hypothesis venv"
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: |
+        VENV_LOC=$(realpath -m .)/hypovenv
+        VENV_PYTHON=$VENV_LOC/bin/python
+        echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV
+        echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV
+        ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -U hypothesis
+    - name: "Run tests"
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: |
+        # Most of the excluded tests are slow test suites with no property tests
+        #
+        # (GH-104097) test_sysconfig is skipped because it has tests that are
+        # failing when executed from inside a virtual environment.
+        ${{ env.VENV_PYTHON }} -m test \
+          -W \
+          -x test_asyncio \
+          -x test_multiprocessing_fork \
+          -x test_multiprocessing_forkserver \
+          -x test_multiprocessing_spawn \
+          -x test_concurrent_futures \
+          -x test_socket \
+          -x test_subprocess \
+          -x test_signal \
+          -x test_sysconfig
+
 
   build_asan:
     name: 'Address sanitizer'

Lib/test/libregrtest/save_env.py

@@ -257,8 +257,10 @@ def restore_sysconfig__INSTALL_SCHEMES(self, saved):
         sysconfig._INSTALL_SCHEMES.update(saved[2])
 
     def get_files(self):
+        # XXX: Maybe add an allow-list here?
         return sorted(fn + ('/' if os.path.isdir(fn) else '')
-                      for fn in os.listdir())
+                      for fn in os.listdir()
+                      if not fn.startswith(".hypothesis"))
     def restore_files(self, saved_value):
         fn = os_helper.TESTFN
         if fn not in saved_value and (fn + '/') not in saved_value:

Lib/test/support/_hypothesis_stubs/__init__.py

@@ -0,0 +1,111 @@
+from enum import Enum
+import functools
+import unittest
+
+__all__ = [
+    "given",
+    "example",
+    "assume",
+    "reject",
+    "register_random",
+    "strategies",
+    "HealthCheck",
+    "settings",
+    "Verbosity",
+]
+
+from . import strategies
+
+
+def given(*_args, **_kwargs):
+    def decorator(f):
+        if examples := getattr(f, "_examples", []):
+
+            @functools.wraps(f)
+            def test_function(self):
+                for example_args, example_kwargs in examples:
+                    with self.subTest(*example_args, **example_kwargs):
+                        f(self, *example_args, **example_kwargs)
+
+        else:
+            # If we have found no examples, we must skip the test. If @example
+            # is applied after @given, it will re-wrap the test to remove the
+            # skip decorator.
+            test_function = unittest.skip(
+                "Hypothesis required for property test with no " +
+                "specified examples"
+            )(f)
+
+        test_function._given = True
+        return test_function
+
+    return decorator
+
+
+def example(*args, **kwargs):
+    if bool(args) == bool(kwargs):
+        raise ValueError("Must specify exactly one of *args or **kwargs")
+
+    def decorator(f):
+        base_func = getattr(f, "__wrapped__", f)
+        if not hasattr(base_func, "_examples"):
+            base_func._examples = []
+
+        base_func._examples.append((args, kwargs))
+
+        if getattr(f, "_given", False):
+            # If the given decorator is below all the example decorators,
+            # it would be erroneously skipped, so we need to re-wrap the new
+            # base function.
+            f = given()(base_func)
+
+        return f
+
+    return decorator
+
+
+def assume(condition):
+    if not condition:
+        raise unittest.SkipTest("Unsatisfied assumption")
+    return True
+
+
+def reject():
+    assume(False)
+
+
+def register_random(*args, **kwargs):
+    pass  # pragma: no cover
+
+
+def settings(*args, **kwargs):
+    return lambda f: f  # pragma: nocover
+
+
+class HealthCheck(Enum):
+    data_too_large = 1
+    filter_too_much = 2
+    too_slow = 3
+    return_value = 5
+    large_base_example = 7
+    not_a_test_method = 8
+
+    @classmethod
+    def all(cls):
+        return list(cls)
+
+
+class Verbosity(Enum):
+    quiet = 0
+    normal = 1
+    verbose = 2
+    debug = 3
+
+
+class Phase(Enum):
+    explicit = 0
+    reuse = 1
+    generate = 2
+    target = 3
+    shrink = 4
+    explain = 5

Lib/test/support/_hypothesis_stubs/_helpers.py

@@ -0,0 +1,43 @@
+# Stub out only the subset of the interface that we actually use in our tests.
+class StubClass:
+    def __init__(self, *args, **kwargs):
+        self.__stub_args = args
+        self.__stub_kwargs = kwargs
+        self.__repr = None
+
+    def _with_repr(self, new_repr):
+        new_obj = self.__class__(*self.__stub_args, **self.__stub_kwargs)
+        new_obj.__repr = new_repr
+        return new_obj
+
+    def __repr__(self):
+        if self.__repr is not None:
+            return self.__repr
+
+        argstr = ", ".join(self.__stub_args)
+        kwargstr = ", ".join(f"{kw}={val}" for kw, val in self.__stub_kwargs.items())
+
+        in_parens = argstr
+        if kwargstr:
+            in_parens += ", " + kwargstr
+
+        return f"{self.__class__.__qualname__}({in_parens})"
+
+
+def stub_factory(klass, name, *, with_repr=None, _seen={}):
+    if (klass, name) not in _seen:
+
+        class Stub(klass):
+            def __init__(self, *args, **kwargs):
+                super().__init__()
+                self.__stub_args = args
+                self.__stub_kwargs = kwargs
+
+        Stub.__name__ = name
+        Stub.__qualname__ = name
+        if with_repr is not None:
+            Stub._repr = None
+
+        _seen.setdefault((klass, name, with_repr), Stub)
+
+    return _seen[(klass, name, with_repr)]

Lib/test/support/_hypothesis_stubs/strategies.py

@@ -0,0 +1,91 @@
+import functools
+
+from ._helpers import StubClass, stub_factory
+
+
+class StubStrategy(StubClass):
+    def __make_trailing_repr(self, transformation_name, func):
+        func_name = func.__name__ or repr(func)
+        return f"{self!r}.{transformation_name}({func_name})"
+
+    def map(self, pack):
+        return self._with_repr(self.__make_trailing_repr("map", pack))
+
+    def flatmap(self, expand):
+        return self._with_repr(self.__make_trailing_repr("flatmap", expand))
+
+    def filter(self, condition):
+        return self._with_repr(self.__make_trailing_repr("filter", condition))
+
+    def __or__(self, other):
+        new_repr = f"one_of({self!r}, {other!r})"
+        return self._with_repr(new_repr)
+
+
+_STRATEGIES = {
+    "binary",
+    "booleans",
+    "builds",
+    "characters",
+    "complex_numbers",
+    "composite",
+    "data",
+    "dates",
+    "datetimes",
+    "decimals",
+    "deferred",
+    "dictionaries",
+    "emails",
+    "fixed_dictionaries",
+    "floats",
+    "fractions",
+    "from_regex",
+    "from_type",
+    "frozensets",
+    "functions",
+    "integers",
+    "iterables",
+    "just",
+    "lists",
+    "none",
+    "nothing",
+    "one_of",
+    "permutations",
+    "random_module",
+    "randoms",
+    "recursive",
+    "register_type_strategy",
+    "runner",
+    "sampled_from",
+    "sets",
+    "shared",
+    "slices",
+    "timedeltas",
+    "times",
+    "text",
+    "tuples",
+    "uuids",
+}
+
+__all__ = sorted(_STRATEGIES)
+
+
+def composite(f):
+    strategy = stub_factory(StubStrategy, f.__name__)
+
+    @functools.wraps(f)
+    def inner(*args, **kwargs):
+        return strategy(*args, **kwargs)
+
+    return inner
+
+
+def __getattr__(name):
+    if name not in _STRATEGIES:
+        raise AttributeError(f"Unknown attribute {name}")
+
+    return stub_factory(StubStrategy, f"hypothesis.strategies.{name}")
+
+
+def __dir__():
+    return __all__

Lib/test/support/hypothesis_helper.py

@@ -0,0 +1,4 @@
+try:
+    import hypothesis
+except ImportError:
+    from . import _hypothesis_stubs as hypothesis

Lib/test/test_zoneinfo/__init__.py

@@ -1 +1,2 @@
 from .test_zoneinfo import *
+from .test_zoneinfo_property import *