From 80c665c5cea8e7d032b1ef000dcd079813446e53 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Fri, 22 Jul 2022 16:11:25 -0700 Subject: [PATCH 01/10] PYTHON-3366 Add "all" extra for installing all optional dependencies --- README.rst | 2 +- doc/installation.rst | 2 +- setup.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index f60b8da680..f2d15def08 100644 --- a/README.rst +++ b/README.rst @@ -136,7 +136,7 @@ Client-Side Field Level Encryption requires `pymongocrypt You can install all dependencies automatically with the following command:: - $ python -m pip install "pymongo[gssapi,aws,ocsp,snappy,srv,tls,zstd,encryption]" + $ python -m pip install "pymongo[all]" Additional dependencies are: diff --git a/doc/installation.rst b/doc/installation.rst index 788faf46cc..d0dc923187 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -74,7 +74,7 @@ Wire protocol compression with zstandard requires `zstandard You can install all dependencies automatically with the following command:: - $ python3 -m pip install "pymongo[gssapi,aws,ocsp,snappy,srv,zstd,encryption]" + $ python3 -m pip install "pymongo[all]" Installing from source ---------------------- diff --git a/setup.py b/setup.py index 0e983e4642..3a65ad2b63 100755 --- a/setup.py +++ b/setup.py @@ -290,6 +290,8 @@ def build_extension(self, ext): else: extras_require["gssapi"] = ["pykerberos"] +extras_require["all"] = [dep for deps in extras_require.values() for dep in deps] + extra_opts = {} if "--no_ext" in sys.argv: From 03a72b93f2ca19a21aa15a5591e59ab93f3e0a09 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Fri, 22 Jul 2022 16:16:32 -0700 Subject: [PATCH 02/10] PYTHON-3366 Support mypy 0.971 and test with latest version --- .github/workflows/test-python.yml | 4 ++-- bson/__init__.py | 12 ++++++------ pymongo/pyopenssl_context.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 89d9830e82..5ceb05fd7c 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -59,8 +59,8 @@ jobs: cache-dependency-path: 'setup.py' - name: Install dependencies run: | - python -m pip install -U pip mypy==0.942 - pip install -e ".[zstd, srv]" + python -m pip install -U pip mypy + pip install -e ".[all]" - name: Run mypy run: | mypy --install-types --non-interactive bson gridfs tools pymongo diff --git a/bson/__init__.py b/bson/__init__.py index cc0850709e..3a7ca61d50 100644 --- a/bson/__init__.py +++ b/bson/__init__.py @@ -61,8 +61,8 @@ import struct import sys import uuid -from codecs import utf_8_decode as _utf_8_decode # type: ignore[attr-defined] -from codecs import utf_8_encode as _utf_8_encode # type: ignore[attr-defined] +from codecs import utf_8_decode as _utf_8_decode +from codecs import utf_8_encode as _utf_8_encode from collections import abc as _abc from typing import ( IO, @@ -621,7 +621,7 @@ def _make_c_string_check(string: Union[str, bytes]) -> bytes: else: if "\x00" in string: raise InvalidDocument("BSON keys / regex patterns must not contain a NUL character") - return cast(bytes, _utf_8_encode(string)[0]) + b"\x00" + return _utf_8_encode(string)[0] + b"\x00" def _make_c_string(string: Union[str, bytes]) -> bytes: @@ -633,7 +633,7 @@ def _make_c_string(string: Union[str, bytes]) -> bytes: except UnicodeError: raise InvalidStringData("strings in documents must be valid UTF-8: %r" % string) else: - return cast(bytes, _utf_8_encode(string)[0]) + b"\x00" + return _utf_8_encode(string)[0] + b"\x00" def _make_name(string: str) -> bytes: @@ -641,7 +641,7 @@ def _make_name(string: str) -> bytes: # Keys can only be text in python 3. if "\x00" in string: raise InvalidDocument("BSON keys / regex patterns must not contain a NUL character") - return cast(bytes, _utf_8_encode(string)[0]) + b"\x00" + return _utf_8_encode(string)[0] + b"\x00" def _encode_float(name: bytes, value: float, dummy0: Any, dummy1: Any) -> bytes: @@ -1308,7 +1308,7 @@ def encode( """ return cls(encode(document, check_keys, codec_options)) - def decode(self, codec_options: CodecOptions = DEFAULT_CODEC_OPTIONS) -> _DocumentType: # type: ignore[override] + def decode(self, codec_options: CodecOptions[_DocumentType] = DEFAULT_CODEC_OPTIONS) -> _DocumentType: # type: ignore[override,assignment] """Decode this BSON data. By default, returns a BSON document represented as a Python diff --git a/pymongo/pyopenssl_context.py b/pymongo/pyopenssl_context.py index 758a741b6f..9f45d31967 100644 --- a/pymongo/pyopenssl_context.py +++ b/pymongo/pyopenssl_context.py @@ -192,7 +192,7 @@ def __init__(self, protocol): # side configuration and wrap_socket tries to support both client and # server side sockets. self._callback_data.check_ocsp_endpoint = True - self._ctx.set_ocsp_client_callback(callback=_ocsp_callback, data=self._callback_data) + self._ctx.set_ocsp_client_callback(callback=_ocsp_callback, data=self._callback_data) # type: ignore[arg-type] @property def protocol(self): From 2a24de94f220b7b90321f0e5ab90a4a91dc60f94 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Fri, 22 Jul 2022 16:20:44 -0700 Subject: [PATCH 03/10] PYTHON-3366 Fix BSON.decode type annotation --- bson/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bson/__init__.py b/bson/__init__.py index 3a7ca61d50..2db1fb5d0b 100644 --- a/bson/__init__.py +++ b/bson/__init__.py @@ -1308,7 +1308,7 @@ def encode( """ return cls(encode(document, check_keys, codec_options)) - def decode(self, codec_options: CodecOptions[_DocumentType] = DEFAULT_CODEC_OPTIONS) -> _DocumentType: # type: ignore[override,assignment] + def decode(self, codec_options: "CodecOptions[_DocumentType]" = DEFAULT_CODEC_OPTIONS) -> _DocumentType: # type: ignore[override,assignment] """Decode this BSON data. By default, returns a BSON document represented as a Python From 7242c0b4934735833129fed405568a088f2daf29 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Fri, 22 Jul 2022 16:24:16 -0700 Subject: [PATCH 04/10] Revert "PYTHON-3366 Add "all" extra for installing all optional dependencies" This reverts commit 80c665c5cea8e7d032b1ef000dcd079813446e53. --- README.rst | 2 +- doc/installation.rst | 2 +- setup.py | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index f2d15def08..f60b8da680 100644 --- a/README.rst +++ b/README.rst @@ -136,7 +136,7 @@ Client-Side Field Level Encryption requires `pymongocrypt You can install all dependencies automatically with the following command:: - $ python -m pip install "pymongo[all]" + $ python -m pip install "pymongo[gssapi,aws,ocsp,snappy,srv,tls,zstd,encryption]" Additional dependencies are: diff --git a/doc/installation.rst b/doc/installation.rst index d0dc923187..788faf46cc 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -74,7 +74,7 @@ Wire protocol compression with zstandard requires `zstandard You can install all dependencies automatically with the following command:: - $ python3 -m pip install "pymongo[all]" + $ python3 -m pip install "pymongo[gssapi,aws,ocsp,snappy,srv,zstd,encryption]" Installing from source ---------------------- diff --git a/setup.py b/setup.py index 3a65ad2b63..0e983e4642 100755 --- a/setup.py +++ b/setup.py @@ -290,8 +290,6 @@ def build_extension(self, ext): else: extras_require["gssapi"] = ["pykerberos"] -extras_require["all"] = [dep for deps in extras_require.values() for dep in deps] - extra_opts = {} if "--no_ext" in sys.argv: From 8e3abf36accd7173b3de05301db86cbc462b03f9 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Fri, 22 Jul 2022 16:24:59 -0700 Subject: [PATCH 05/10] PYTHON-3366 remove "all" extra --- .github/workflows/test-python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 5ceb05fd7c..90b02c6ac7 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -60,7 +60,7 @@ jobs: - name: Install dependencies run: | python -m pip install -U pip mypy - pip install -e ".[all]" + pip install -e ".[zstd, srv, encryption]" - name: Run mypy run: | mypy --install-types --non-interactive bson gridfs tools pymongo From 3c445f1060f18df6b29687cd661f12afa2fedb80 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Fri, 22 Jul 2022 16:31:14 -0700 Subject: [PATCH 06/10] PYTHON-3366 add ocsp to mypy check --- .github/workflows/test-python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 90b02c6ac7..6d5f26c503 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -60,7 +60,7 @@ jobs: - name: Install dependencies run: | python -m pip install -U pip mypy - pip install -e ".[zstd, srv, encryption]" + pip install -e ".[zstd, srv, encryption, ocsp]" - name: Run mypy run: | mypy --install-types --non-interactive bson gridfs tools pymongo From 43d1064662fc92c2c29772fd8abff244aa7d9ae5 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Fri, 22 Jul 2022 16:44:56 -0700 Subject: [PATCH 07/10] PYTHON-3366 Fix one test type issue --- test/test_collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_collection.py b/test/test_collection.py index bea2ed6ca6..37f1b1eae2 100644 --- a/test/test_collection.py +++ b/test/test_collection.py @@ -19,7 +19,7 @@ import contextlib import re import sys -from codecs import utf_8_decode # type: ignore +from codecs import utf_8_decode from collections import defaultdict from typing import Iterable, no_type_check From 50c45f085e2d4cda1902042ca2177d84b7735fb3 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Mon, 25 Jul 2022 11:12:50 -0700 Subject: [PATCH 08/10] PYTHON-3366 cleanup pyopenssl sendall() --- pymongo/pyopenssl_context.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pymongo/pyopenssl_context.py b/pymongo/pyopenssl_context.py index 9f45d31967..2d9c904bb3 100644 --- a/pymongo/pyopenssl_context.py +++ b/pymongo/pyopenssl_context.py @@ -135,7 +135,7 @@ def recv(self, *args, **kwargs): def recv_into(self, *args, **kwargs): try: - return self._call(super(_sslConn, self).recv_into, *args, **kwargs) # type: ignore + return self._call(super(_sslConn, self).recv_into, *args, **kwargs) except _SSL.SysCallError as exc: # Suppress ragged EOFs to match the stdlib. if self.suppress_ragged_eofs and _ragged_eof(exc): @@ -146,12 +146,9 @@ def sendall(self, buf, flags=0): view = memoryview(buf) total_length = len(buf) total_sent = 0 - sent = 0 while total_sent < total_length: try: - sent = self._call( - super(_sslConn, self).send, view[total_sent:], flags # type: ignore - ) + sent = self._call(super(_sslConn, self).send, view[total_sent:], flags) # XXX: It's not clear if this can actually happen. PyOpenSSL # doesn't appear to have any interrupt handling, nor any interrupt # errors for OpenSSL connections. @@ -162,7 +159,7 @@ def sendall(self, buf, flags=0): # https://github.com/pyca/pyopenssl/blob/19.1.0/src/OpenSSL/SSL.py#L1756 # https://www.openssl.org/docs/man1.0.2/man3/SSL_write.html if sent <= 0: - raise Exception("Connection closed") + raise OSError("connection closed") total_sent += sent @@ -192,7 +189,7 @@ def __init__(self, protocol): # side configuration and wrap_socket tries to support both client and # server side sockets. self._callback_data.check_ocsp_endpoint = True - self._ctx.set_ocsp_client_callback(callback=_ocsp_callback, data=self._callback_data) # type: ignore[arg-type] + self._ctx.set_ocsp_client_callback(callback=_ocsp_callback, data=self._callback_data) @property def protocol(self): From 3c2d3f0d928b0c7225b13ac59be95a732af966e5 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Mon, 25 Jul 2022 15:12:13 -0700 Subject: [PATCH 09/10] PYTHON-3369 https://www.gevent.org --- doc/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index 1e18eb29bf..f66de3868a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -192,6 +192,6 @@ intersphinx_mapping = { - "gevent": ("http://www.gevent.org/", None), + "gevent": ("https://www.gevent.org/", None), "py": ("https://docs.python.org/3/", None), } From c11389f2b0f457ccc296a9df579e8de314e92138 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Mon, 25 Jul 2022 15:21:04 -0700 Subject: [PATCH 10/10] PYTHON-3366 Workaround mypy bug --- test/test_auth.py | 4 ++-- test/test_bson.py | 21 ++++++++++++++++----- test/test_change_stream.py | 3 ++- test/test_database.py | 5 +++-- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/test/test_auth.py b/test/test_auth.py index 69ed27bda0..20d53ef24b 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -329,8 +329,8 @@ def auth_string(user, password): bad_user = MongoClient(auth_string("not-user", SASL_PASS)) bad_pwd = MongoClient(auth_string(SASL_USER, "not-pwd")) # OperationFailure raised upon connecting. - self.assertRaises(OperationFailure, bad_user.admin.command, "ping") - self.assertRaises(OperationFailure, bad_pwd.admin.command, "ping") + self.assertRaises(OperationFailure, bad_user.admin.command, "ping") # type: ignore[arg-type] + self.assertRaises(OperationFailure, bad_pwd.admin.command, "ping") # type: ignore[arg-type] class TestSCRAMSHA1(IntegrationTest): diff --git a/test/test_bson.py b/test/test_bson.py index 8ad65f3412..aa77954fa2 100644 --- a/test/test_bson.py +++ b/test/test_bson.py @@ -117,7 +117,8 @@ def tzname(self, dt): class TestBSON(unittest.TestCase): def assertInvalid(self, data): - self.assertRaises(InvalidBSON, decode, data) + # Remove type ignore after: https://github.com/python/mypy/issues/13220 + self.assertRaises(InvalidBSON, decode, data) # type: ignore[arg-type] def check_encode_then_decode(self, doc_class=dict, decoder=decode, encoder=encode): @@ -1025,11 +1026,17 @@ def test_unicode_decode_error_handler(self): # Ensure that strict mode raises an error. for invalid in [invalid_key, invalid_val, invalid_both]: + # Remove type ignore after: https://github.com/python/mypy/issues/13220 self.assertRaises( - InvalidBSON, decode, invalid, CodecOptions(unicode_decode_error_handler="strict") + InvalidBSON, + decode, # type: ignore[arg-type] + invalid, + CodecOptions(unicode_decode_error_handler="strict"), ) - self.assertRaises(InvalidBSON, decode, invalid, CodecOptions()) - self.assertRaises(InvalidBSON, decode, invalid) + self.assertRaises( + InvalidBSON, decode, invalid, CodecOptions() # type: ignore[arg-type] + ) + self.assertRaises(InvalidBSON, decode, invalid) # type: ignore[arg-type] # Test all other error handlers. for handler in ["replace", "backslashreplace", "surrogateescape", "ignore"]: @@ -1046,8 +1053,12 @@ def test_unicode_decode_error_handler(self): dec = decode(enc, CodecOptions(unicode_decode_error_handler="junk")) self.assertEqual(dec, {"keystr": "foobar"}) + # Remove type ignore after: https://github.com/python/mypy/issues/13220 self.assertRaises( - InvalidBSON, decode, invalid_both, CodecOptions(unicode_decode_error_handler="junk") + InvalidBSON, + decode, # type: ignore[arg-type] + invalid_both, + CodecOptions(unicode_decode_error_handler="junk"), ) def round_trip_pickle(self, obj, pickled_with_older): diff --git a/test/test_change_stream.py b/test/test_change_stream.py index 11ed2895ac..b5b260086d 100644 --- a/test/test_change_stream.py +++ b/test/test_change_stream.py @@ -1084,8 +1084,9 @@ def setFailPoint(self, scenario_dict): fail_cmd = SON([("configureFailPoint", "failCommand")]) fail_cmd.update(fail_point) client_context.client.admin.command(fail_cmd) + # Remove type ignore after: https://github.com/python/mypy/issues/13220 self.addCleanup( - client_context.client.admin.command, + client_context.client.admin.command, # type: ignore[arg-type] "configureFailPoint", fail_cmd["configureFailPoint"], mode="off", diff --git a/test/test_database.py b/test/test_database.py index d49ac8324f..a1c0439089 100644 --- a/test/test_database.py +++ b/test/test_database.py @@ -604,13 +604,14 @@ def test_command_max_time_ms(self): try: db = self.client.pymongo_test db.command("count", "test") - self.assertRaises(ExecutionTimeout, db.command, "count", "test", maxTimeMS=1) + # Remove type ignore after: https://github.com/python/mypy/issues/13220 + self.assertRaises(ExecutionTimeout, db.command, "count", "test", maxTimeMS=1) # type: ignore[arg-type] pipeline = [{"$project": {"name": 1, "count": 1}}] # Database command helper. db.command("aggregate", "test", pipeline=pipeline, cursor={}) self.assertRaises( ExecutionTimeout, - db.command, + db.command, # type: ignore[arg-type] "aggregate", "test", pipeline=pipeline,