General maintenance and tidying to ensure tests pass. (#136)

* Add M1 support and update dependencies.

* Restrict to pubsub client < 2 because of breaking changes.

* Add platform option.

* Improve error messages.

* Clean up database tests.

* Add missing skipif reason.

* Various changes to make tests pass.

* Fix linting errors.

* Fail fast given long test cycle.

* Only use pymssql for arm.

* Only use azure sql edge image for arm.

* Use pymssql instead of pyodbc.

* Add ConnectionError to list of transient errors.

* Add BrokenPipeError to transient errors.

* Mark kafka tests as failing.

* Simplify Kafka configuration.

* Use pymssql in container.

* Resolve dependency conflict with flake8.

* Partially revert `kafka` changes.

* Fix `rabbitmq` readiness probe.

* Add ValueError to transient kafka errors.

Author: tillahoffmann

.github/workflows/main.yml

@@ -25,13 +25,6 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pip-
             ${{ runner.os }}-
-      - name: Install system requirements
-        run: |
-          sudo apt-get install -y --no-install-recommends unixodbc-dev # required for pyodbc
-          curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
-          curl https://packages.microsoft.com/config/ubuntu/18.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list
-          sudo apt-get -qq update
-          sudo ACCEPT_EULA=Y apt-get -y install msodbcsql17
       - name: Install Python dependencies
         run: |
           python -m pip install --upgrade pip
@@ -51,5 +44,5 @@ jobs:
         run: |
           flake8
           sphinx-build -nW docs docs/_build/html
-          py.test -sv --cov-config .coveragerc --cov-report html:skip-covered --cov-report term:skip-covered --cov=testcontainers --tb=short tests/
+          py.test -svx --cov-config .coveragerc --cov-report html:skip-covered --cov-report term:skip-covered --cov=testcontainers --tb=short tests/
           codecov

Dockerfile

@@ -1,6 +1,12 @@
 ARG version=3.8
 FROM python:${version}
 
+WORKDIR /workspace
+RUN pip install --upgrade pip \
+    && apt-get update \
+    && apt-get install -y \
+        freetds-dev \
+    && rm -rf /var/lib/apt/lists/*
 WORKDIR /workspace
 ARG version=3.8
 COPY requirements/${version}.txt requirements.txt

Makefile

@@ -1,8 +1,9 @@
-PYTHON_VERSIONS = 3.6 3.7 3.8
+PYTHON_VERSIONS = 3.6 3.7 3.8 3.9
 REQUIREMENTS = $(addprefix requirements/,${PYTHON_VERSIONS:=.txt})
 TESTS = $(addprefix tests/,${PYTHON_VERSIONS})
 IMAGES = $(addprefix image/,${PYTHON_VERSIONS})
 RUN = docker run --rm -it
+
 .PHONY : docs
 
 # Default target
@@ -17,7 +18,7 @@ requirements : ${REQUIREMENTS}
 ${REQUIREMENTS} : requirements/%.txt : requirements.in setup.py
 	mkdir -p $(dir $@)
 	${RUN} -w /workspace -v `pwd`:/workspace python:$* bash -c \
-		"pip install pip-tools && pip-compile -v -o $@ $<"
+		"pip install pip-tools && pip-compile -v --upgrade -o $@ $<"
 
 
 # Targets to build docker images
@@ -33,8 +34,8 @@ ${IMAGES} : image/% : requirements/%.txt
 tests : ${TESTS}
 
 ${TESTS} : tests/% : image/%
-	${RUN} -v /var/run/docker.sock:/var/run/docker.sock testcontainers-python:$* bash -c \
-		"flake8 && pytest -v ${ARGS}"
+	${RUN} -v /var/run/docker.sock:/var/run/docker.sock testcontainers-python:$* \
+		bash -c "flake8 && pytest -v ${ARGS}"
 
 # Target to build the documentation
 

requirements.in

@@ -1,7 +1,6 @@
 -e file:.[docker-compose,mysql,oracle,postgresql,selenium,google-cloud-pubsub,mongo,redis,mssqlserver,neo4j,kafka,rabbitmq]
 codecov>=2.1.0
-flake8
+flake8<3.8.0  # 3.8.0 adds a dependency on importlib-metadata which conflicts with other packages.
 pytest
 pytest-cov
 sphinx
-docker<4.3.0

requirements/3.6.txt

@@ -8,221 +8,235 @@
     # via -r requirements.in
 alabaster==0.7.12
     # via sphinx
-attrs==19.3.0
+async-timeout==4.0.2
+    # via redis
+attrs==21.4.0
     # via
     #   jsonschema
     #   pytest
-babel==2.8.0
+babel==2.9.1
     # via sphinx
 bcrypt==3.2.0
     # via paramiko
-cached-property==1.5.1
+cached-property==1.5.2
     # via docker-compose
-cachetools==4.1.1
+cachetools==4.2.4
     # via google-auth
-certifi==2020.6.20
+certifi==2021.10.8
     # via requests
-cffi==1.14.2
+cffi==1.15.0
     # via
     #   bcrypt
     #   cryptography
     #   pynacl
-chardet==3.0.4
+charset-normalizer==2.0.12
     # via requests
-codecov==2.1.8
+codecov==2.1.12
     # via -r requirements.in
-coverage==5.2.1
+coverage[toml]==6.2
     # via
     #   codecov
     #   pytest-cov
-cryptography==3.0
+cryptography==36.0.2
     # via paramiko
-cx-oracle==8.0.0
+cx-oracle==8.3.0
     # via testcontainers
+deprecated==1.2.13
+    # via redis
 deprecation==2.1.0
     # via testcontainers
-distro==1.5.0
+distro==1.7.0
     # via docker-compose
-docker[ssh]==4.2.2
+docker[ssh]==5.0.3
     # via
-    #   -r requirements.in
     #   docker-compose
     #   testcontainers
-docker-compose==1.26.2
+docker-compose==1.29.2
     # via testcontainers
 dockerpty==0.4.1
     # via docker-compose
 docopt==0.6.2
     # via docker-compose
-docutils==0.16
+docutils==0.17.1
     # via sphinx
-flake8==3.8.3
+entrypoints==0.3
+    # via flake8
+flake8==3.7.9
     # via -r requirements.in
-google-api-core[grpc]==1.22.1
+google-api-core[grpc]==2.7.1
     # via google-cloud-pubsub
-google-auth==1.20.1
+google-auth==2.6.2
     # via google-api-core
 google-cloud-pubsub==1.7.0
     # via testcontainers
-googleapis-common-protos[grpc]==1.52.0
+googleapis-common-protos[grpc]==1.56.0
     # via
     #   google-api-core
     #   grpc-google-iam-v1
+    #   grpcio-status
+greenlet==1.1.2
+    # via sqlalchemy
 grpc-google-iam-v1==0.12.3
     # via google-cloud-pubsub
-grpcio==1.31.0
+grpcio==1.45.0
     # via
     #   google-api-core
     #   googleapis-common-protos
     #   grpc-google-iam-v1
-idna==2.10
+    #   grpcio-status
+grpcio-status==1.45.0
+    # via google-api-core
+idna==3.3
     # via requests
-imagesize==1.2.0
+imagesize==1.3.0
     # via sphinx
-importlib-metadata==1.7.0
+importlib-metadata==4.8.3
     # via
-    #   flake8
     #   jsonschema
     #   pluggy
     #   pytest
-iniconfig==1.0.1
+    #   redis
+    #   sphinx
+    #   sqlalchemy
+iniconfig==1.1.1
     # via pytest
-jinja2==2.11.2
+jinja2==3.0.3
     # via sphinx
 jsonschema==3.2.0
     # via docker-compose
 kafka-python==2.0.2
     # via testcontainers
-markupsafe==1.1.1
+markupsafe==2.0.1
     # via jinja2
 mccabe==0.6.1
     # via flake8
-more-itertools==8.4.0
-    # via pytest
-neo4j==4.1.0
+neo4j==4.4.2
     # via testcontainers
-packaging==20.4
+packaging==21.3
     # via
     #   deprecation
     #   pytest
+    #   redis
     #   sphinx
-paramiko==2.7.1
+paramiko==2.10.3
     # via docker
 pika==1.2.0
     # via testcontainers
-pluggy==0.13.1
+pluggy==1.0.0
     # via pytest
-protobuf==3.13.0
+protobuf==3.19.4
     # via
     #   google-api-core
     #   googleapis-common-protos
-psycopg2-binary==2.8.5
+    #   grpcio-status
+psycopg2-binary==2.9.3
     # via testcontainers
-py==1.9.0
+py==1.11.0
     # via pytest
 pyasn1==0.4.8
     # via
     #   pyasn1-modules
     #   rsa
 pyasn1-modules==0.2.8
     # via google-auth
-pycodestyle==2.6.0
+pycodestyle==2.5.0
     # via flake8
-pycparser==2.20
+pycparser==2.21
     # via cffi
-pyflakes==2.2.0
+pyflakes==2.1.1
     # via flake8
-pygments==2.6.1
+pygments==2.11.2
     # via sphinx
-pymongo==3.11.0
+pymongo==4.0.2
     # via testcontainers
-pymysql==0.10.0
+pymssql==2.2.4
     # via testcontainers
-pynacl==1.4.0
-    # via paramiko
-pyodbc==4.0.30
+pymysql==1.0.2
     # via testcontainers
-pyparsing==2.4.7
+pynacl==1.5.0
+    # via paramiko
+pyparsing==3.0.7
     # via packaging
-pyrsistent==0.16.0
+pyrsistent==0.18.0
     # via jsonschema
-pytest==6.0.1
+pytest==7.0.1
     # via
     #   -r requirements.in
     #   pytest-cov
-pytest-cov==2.10.1
+pytest-cov==3.0.0
     # via -r requirements.in
-python-dotenv==0.14.0
+python-dotenv==0.20.0
     # via docker-compose
-pytz==2020.1
+pytz==2022.1
     # via
     #   babel
-    #   google-api-core
     #   neo4j
-pyyaml==5.3.1
+pyyaml==5.4.1
     # via docker-compose
-redis==3.5.3
+redis==4.2.0
     # via testcontainers
-requests==2.24.0
+requests==2.27.1
     # via
     #   codecov
     #   docker
     #   docker-compose
     #   google-api-core
     #   sphinx
-rsa==4.6
+rsa==4.8
     # via google-auth
 selenium==3.141.0
     # via testcontainers
-six==1.15.0
+six==1.16.0
     # via
     #   bcrypt
-    #   cryptography
-    #   docker
-    #   docker-compose
     #   dockerpty
-    #   google-api-core
     #   google-auth
     #   grpcio
     #   jsonschema
-    #   packaging
-    #   protobuf
-    #   pynacl
-    #   pyrsistent
+    #   paramiko
     #   websocket-client
-snowballstemmer==2.0.0
+snowballstemmer==2.2.0
     # via sphinx
-sphinx==3.2.1
+sphinx==4.4.0
     # via -r requirements.in
 sphinxcontrib-applehelp==1.0.2
     # via sphinx
 sphinxcontrib-devhelp==1.0.2
     # via sphinx
-sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-htmlhelp==2.0.0
     # via sphinx
 sphinxcontrib-jsmath==1.0.1
     # via sphinx
 sphinxcontrib-qthelp==1.0.3
     # via sphinx
-sphinxcontrib-serializinghtml==1.1.4
+sphinxcontrib-serializinghtml==1.1.5
     # via sphinx
-sqlalchemy==1.3.18
+sqlalchemy==1.4.32
     # via testcontainers
-texttable==1.6.2
+texttable==1.6.4
     # via docker-compose
-toml==0.10.1
-    # via pytest
-urllib3==1.25.10
+tomli==1.2.3
+    # via
+    #   coverage
+    #   pytest
+typing-extensions==4.1.1
+    # via
+    #   async-timeout
+    #   importlib-metadata
+    #   redis
+urllib3==1.26.9
     # via
     #   requests
     #   selenium
-websocket-client==0.57.0
+websocket-client==0.59.0
     # via
     #   docker
     #   docker-compose
-wrapt==1.12.1
-    # via testcontainers
-zipp==3.1.0
+wrapt==1.14.0
+    # via
+    #   deprecated
+    #   testcontainers
+zipp==3.6.0
     # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file: