From cb83cb446054717b45a40a08d35ff677232848d7 Mon Sep 17 00:00:00 2001 From: Terry Smith Date: Tue, 6 May 2025 17:38:43 -0300 Subject: [PATCH 1/5] chore: enable docker in docker tests to pass on arm machines --- Dockerfile | 15 +++++++++++++-- core/__init__.py | 0 core/tests/__init__.py | 0 core/tests/conftest.py | 10 +++++++++- core/tests/list_arm_extras.py | 18 ++++++++++++++++++ 5 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 core/__init__.py create mode 100644 core/tests/__init__.py create mode 100644 core/tests/list_arm_extras.py diff --git a/Dockerfile b/Dockerfile index 865771fa1..001dbefc6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,8 @@ ARG PYTHON_VERSION=3.10 FROM python:${PYTHON_VERSION}-slim-bookworm +ARG POETRY_EXTRAS + +ENV PYTHONPATH=/workspace ENV POETRY_NO_INTERACTION=1 \ POETRY_VIRTUALENVS_IN_PROJECT=1 \ @@ -20,10 +23,18 @@ RUN bash -c 'python -m venv /opt/poetry-venv && source $_/bin/activate && pip in # install dependencies with poetry COPY pyproject.toml . COPY poetry.lock . -RUN poetry install --all-extras --with dev --no-root +RUN if [ "$POETRY_EXTRAS" = "" ]; then \ + poetry install --all-extras --with dev --no-root; \ + else \ + poetry install --extras "$POETRY_EXTRAS" --with dev --no-root; \ + fi # copy project source COPY . . # install project with poetry -RUN poetry install --all-extras --with dev +RUN if [ "$POETRY_EXTRAS" = "" ]; then \ + poetry install --all-extras --with dev; \ + else \ + poetry install --extras "$POETRY_EXTRAS" --with dev; \ + fi diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/core/tests/__init__.py b/core/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/core/tests/conftest.py b/core/tests/conftest.py index cbacddc92..017882754 100644 --- a/core/tests/conftest.py +++ b/core/tests/conftest.py @@ -4,8 +4,11 @@ from typing import Callable from testcontainers.core.container import DockerClient from pprint import pprint +from testcontainers.core.utils import is_arm import sys +from .list_arm_extras import get_arm_extras + PROJECT_DIR = Path(__file__).parent.parent.parent.resolve() @@ -25,11 +28,16 @@ def python_testcontainer_image() -> str: py_version = ".".join(map(str, sys.version_info[:2])) image_name = f"testcontainers-python:{py_version}" client = DockerClient() + build_args = {"PYTHON_VERSION": py_version} + + if is_arm(): + build_args["POETRY_EXTRAS"] = get_arm_extras() + client.build( path=str(PROJECT_DIR), tag=image_name, rm=False, - buildargs={"PYTHON_VERSION": py_version}, + buildargs=build_args, ) return image_name diff --git a/core/tests/list_arm_extras.py b/core/tests/list_arm_extras.py new file mode 100644 index 000000000..573505d5f --- /dev/null +++ b/core/tests/list_arm_extras.py @@ -0,0 +1,18 @@ +from pathlib import Path +from testcontainers.core.utils import is_arm + +try: + import tomllib # Python 3.11+ +except ImportError: + import tomli as tomllib + +SKIPPED_EXTRAS = {"db2"} # skip incompatible extras + + +def get_arm_extras(): + with Path("pyproject.toml").open("rb") as f: + data = tomllib.load(f) + + extras = data["tool"]["poetry"]["extras"] + skip = SKIPPED_EXTRAS + return " ".join(k for k in extras if k not in skip) From dedb059139aaed6c3be9063f3655afb9e146e51d Mon Sep 17 00:00:00 2001 From: Terry Smith Date: Tue, 6 May 2025 18:13:25 -0300 Subject: [PATCH 2/5] chore: add mac integration skips --- Makefile | 3 +++ core/tests/test_core_registry.py | 9 +++++++++ core/tests/test_docker_in_docker.py | 8 ++++++++ core/tests/test_ryuk.py | 8 ++++++++ 4 files changed, 28 insertions(+) diff --git a/Makefile b/Makefile index 9c820ffa5..9a9dde8c6 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,9 @@ DOCTESTS = $(addsuffix /doctests,$(filter-out modules/README.md,${PACKAGES})) install: ## Set up the project for development +ifeq ($(IS_ARM),$(ARCH)) + poetry install $(foreach extra,$(EXTRAS_LIST),--extras $(extra)) +else poetry install --all-extras poetry run pre-commit install diff --git a/core/tests/test_core_registry.py b/core/tests/test_core_registry.py index 384b06693..36e4730f9 100644 --- a/core/tests/test_core_registry.py +++ b/core/tests/test_core_registry.py @@ -18,8 +18,13 @@ from testcontainers.core.waiting_utils import wait_container_is_ready from testcontainers.registry import DockerRegistryContainer +from testcontainers.core.utils import is_mac +@pytest.mark.skipif( + is_mac(), + reason="Docker Desktop on macOS does not support insecure private registries without daemon reconfiguration", +) def test_missing_on_private_registry(monkeypatch): username = "user" password = "pass" @@ -41,6 +46,10 @@ def test_missing_on_private_registry(monkeypatch): wait_container_is_ready(test_container) +@pytest.mark.skipif( + is_mac(), + reason="Docker Desktop on macOS does not support local insecure registries over HTTP without modifying daemon settings", +) @pytest.mark.parametrize( "image,tag,username,password", [ diff --git a/core/tests/test_docker_in_docker.py b/core/tests/test_docker_in_docker.py index b07f80e9a..02b8e1fc4 100644 --- a/core/tests/test_docker_in_docker.py +++ b/core/tests/test_docker_in_docker.py @@ -15,6 +15,7 @@ from testcontainers.core.container import DockerContainer from testcontainers.core.docker_client import DockerClient, LOGGER from testcontainers.core.utils import inside_container +from testcontainers.core.utils import is_mac from testcontainers.core.waiting_utils import wait_for_logs @@ -36,6 +37,7 @@ def _wait_for_dind_return_ip(client, dind): return docker_host_ip +@pytest.mark.skipif(is_mac(), reason="Docker socket forwarding (socat) is unsupported on Docker Desktop for macOS") def test_wait_for_logs_docker_in_docker(): # real dind isn't possible (AFAIK) in CI # forwarding the socket to a container port is at least somewhat the same @@ -64,6 +66,9 @@ def test_wait_for_logs_docker_in_docker(): not_really_dind.remove() +@pytest.mark.skipif( + is_mac(), reason="Bridge networking and Docker socket forwarding are not supported on Docker Desktop for macOS" +) def test_dind_inherits_network(): client = DockerClient() try: @@ -158,6 +163,9 @@ def test_find_host_network_in_dood() -> None: assert DockerClient().find_host_network() == os.environ[EXPECTED_NETWORK_VAR] +@pytest.mark.skipif( + is_mac(), reason="Docker socket mounting and container networking do not work reliably on Docker Desktop for macOS" +) @pytest.mark.skipif(not Path(tcc.ryuk_docker_socket).exists(), reason="No docker socket available") def test_dood(python_testcontainer_image: str) -> None: """ diff --git a/core/tests/test_ryuk.py b/core/tests/test_ryuk.py index 5d6b208af..76556d4f4 100644 --- a/core/tests/test_ryuk.py +++ b/core/tests/test_ryuk.py @@ -8,9 +8,14 @@ from testcontainers.core.config import testcontainers_config from testcontainers.core.container import Reaper from testcontainers.core.container import DockerContainer +from testcontainers.core.utils import is_mac from testcontainers.core.waiting_utils import wait_for_logs +@pytest.mark.skipif( + is_mac(), + reason="Ryuk container reaping is unreliable on Docker Desktop for macOS due to VM-based container lifecycle handling", +) @pytest.mark.inside_docker_check def test_wait_for_reaper(monkeypatch: MonkeyPatch): Reaper.delete_instance() @@ -41,6 +46,9 @@ def test_wait_for_reaper(monkeypatch: MonkeyPatch): Reaper.delete_instance() +@pytest.mark.skipif( + is_mac(), reason="Ryuk disabling behavior is unreliable on Docker Desktop for macOS due to Docker socket emulation" +) @pytest.mark.inside_docker_check def test_container_without_ryuk(monkeypatch: MonkeyPatch): Reaper.delete_instance() From 80495630ae12cc731449cda25a9b56de1dc827c1 Mon Sep 17 00:00:00 2001 From: Terry Smith Date: Wed, 7 May 2025 14:05:15 -0300 Subject: [PATCH 3/5] refactor: experiment with simplifying build using unreliable markers which seems to work with ibm specifically --- Dockerfile | 15 ++------------- Makefile | 4 ---- core/__init__.py | 0 core/tests/__init__.py | 0 core/tests/conftest.py | 10 +--------- core/tests/list_arm_extras.py | 18 ------------------ poetry.lock | 8 ++++---- pyproject.toml | 2 +- 8 files changed, 8 insertions(+), 49 deletions(-) delete mode 100644 core/__init__.py delete mode 100644 core/tests/__init__.py delete mode 100644 core/tests/list_arm_extras.py diff --git a/Dockerfile b/Dockerfile index 001dbefc6..865771fa1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,5 @@ ARG PYTHON_VERSION=3.10 FROM python:${PYTHON_VERSION}-slim-bookworm -ARG POETRY_EXTRAS - -ENV PYTHONPATH=/workspace ENV POETRY_NO_INTERACTION=1 \ POETRY_VIRTUALENVS_IN_PROJECT=1 \ @@ -23,18 +20,10 @@ RUN bash -c 'python -m venv /opt/poetry-venv && source $_/bin/activate && pip in # install dependencies with poetry COPY pyproject.toml . COPY poetry.lock . -RUN if [ "$POETRY_EXTRAS" = "" ]; then \ - poetry install --all-extras --with dev --no-root; \ - else \ - poetry install --extras "$POETRY_EXTRAS" --with dev --no-root; \ - fi +RUN poetry install --all-extras --with dev --no-root # copy project source COPY . . # install project with poetry -RUN if [ "$POETRY_EXTRAS" = "" ]; then \ - poetry install --all-extras --with dev; \ - else \ - poetry install --extras "$POETRY_EXTRAS" --with dev; \ - fi +RUN poetry install --all-extras --with dev diff --git a/Makefile b/Makefile index 9a9dde8c6..5ff4f99b6 100644 --- a/Makefile +++ b/Makefile @@ -10,11 +10,7 @@ TESTS = $(addsuffix /tests,$(filter-out meta,${PACKAGES})) TESTS_DIND = $(addsuffix -dind,${TESTS}) DOCTESTS = $(addsuffix /doctests,$(filter-out modules/README.md,${PACKAGES})) - install: ## Set up the project for development -ifeq ($(IS_ARM),$(ARCH)) - poetry install $(foreach extra,$(EXTRAS_LIST),--extras $(extra)) -else poetry install --all-extras poetry run pre-commit install diff --git a/core/__init__.py b/core/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/core/tests/__init__.py b/core/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/core/tests/conftest.py b/core/tests/conftest.py index 017882754..cbacddc92 100644 --- a/core/tests/conftest.py +++ b/core/tests/conftest.py @@ -4,11 +4,8 @@ from typing import Callable from testcontainers.core.container import DockerClient from pprint import pprint -from testcontainers.core.utils import is_arm import sys -from .list_arm_extras import get_arm_extras - PROJECT_DIR = Path(__file__).parent.parent.parent.resolve() @@ -28,16 +25,11 @@ def python_testcontainer_image() -> str: py_version = ".".join(map(str, sys.version_info[:2])) image_name = f"testcontainers-python:{py_version}" client = DockerClient() - build_args = {"PYTHON_VERSION": py_version} - - if is_arm(): - build_args["POETRY_EXTRAS"] = get_arm_extras() - client.build( path=str(PROJECT_DIR), tag=image_name, rm=False, - buildargs=build_args, + buildargs={"PYTHON_VERSION": py_version}, ) return image_name diff --git a/core/tests/list_arm_extras.py b/core/tests/list_arm_extras.py deleted file mode 100644 index 573505d5f..000000000 --- a/core/tests/list_arm_extras.py +++ /dev/null @@ -1,18 +0,0 @@ -from pathlib import Path -from testcontainers.core.utils import is_arm - -try: - import tomllib # Python 3.11+ -except ImportError: - import tomli as tomllib - -SKIPPED_EXTRAS = {"db2"} # skip incompatible extras - - -def get_arm_extras(): - with Path("pyproject.toml").open("rb") as f: - data = tomllib.load(f) - - extras = data["tool"]["poetry"]["extras"] - skip = SKIPPED_EXTRAS - return " ".join(k for k in extras if k not in skip) diff --git a/poetry.lock b/poetry.lock index 4930fa8c1..7101e1e45 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. [[package]] name = "alabaster" @@ -1908,7 +1908,7 @@ description = "Python DBI driver for DB2 (LUW, zOS, i5) and IDS" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"db2\"" +markers = "platform_machine != \"aarch64\" and platform_machine != \"arm64\" and extra == \"db2\"" files = [ {file = "ibm_db-3.2.3-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:3399466141c29704f4e8ba709a67ba27ab413239c0244c3c4510126e946ff603"}, {file = "ibm_db-3.2.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e12ff6426d4f718e1ff6615e64a2880bd570826f19a031c82dbf296714cafd7d"}, @@ -1955,7 +1955,7 @@ description = "SQLAlchemy support for IBM Data Servers" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"db2\"" +markers = "platform_machine != \"aarch64\" and platform_machine != \"arm64\" and extra == \"db2\"" files = [ {file = "ibm_db_sa-0.4.1-py3-none-any.whl", hash = "sha256:49926ba9799e6ebd9ddd847141537c83d179ecf32fe24b7e997ac4614d3f616a"}, {file = "ibm_db_sa-0.4.1.tar.gz", hash = "sha256:a46df130a3681646490925cf4e1bca12b46283f71eea39b70b4f9a56e95341ac"}, @@ -6157,4 +6157,4 @@ weaviate = ["weaviate-client"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "ffdbe7b233214e09e1b586e0bf6145e00621fa83d7680c083aa02cf2a8609550" +content-hash = "91d17ef0905329e552e8181a3b12f03e4c5244a7f23b9278168588e76c2a5802" diff --git a/pyproject.toml b/pyproject.toml index 70df36b73..bc55e7f72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,7 +118,7 @@ httpx = { version = "*", optional = true } azure-cosmos = { version = "*", optional = true } cryptography = { version = "*", optional = true } trino = { version = "*", optional = true } -ibm_db_sa = { version = "*", optional = true } +ibm_db_sa = { version = "*", optional = true, markers = "platform_machine != 'aarch64' and platform_machine != 'arm64'" } [tool.poetry.extras] arangodb = ["python-arango"] From fd43936be04190ac4e438d2a5cbc616d670e5cfc Mon Sep 17 00:00:00 2001 From: Terry Smith Date: Thu, 8 May 2025 12:55:09 -0300 Subject: [PATCH 4/5] fix: resolve failed checks --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 5ff4f99b6..9c820ffa5 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ TESTS = $(addsuffix /tests,$(filter-out meta,${PACKAGES})) TESTS_DIND = $(addsuffix -dind,${TESTS}) DOCTESTS = $(addsuffix /doctests,$(filter-out modules/README.md,${PACKAGES})) + install: ## Set up the project for development poetry install --all-extras poetry run pre-commit install From 119497392024cf2bffaf81952f13d8dbf819391b Mon Sep 17 00:00:00 2001 From: David Ankin Date: Fri, 13 Jun 2025 11:19:31 -0400 Subject: [PATCH 5/5] last minute fixes --- core/testcontainers/core/docker_client.py | 10 +++++----- poetry.lock | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/testcontainers/core/docker_client.py b/core/testcontainers/core/docker_client.py index 527852532..07c7ef53a 100644 --- a/core/testcontainers/core/docker_client.py +++ b/core/testcontainers/core/docker_client.py @@ -151,7 +151,7 @@ def find_host_network(self) -> Optional[str]: except ipaddress.AddressValueError: continue if docker_host in subnet: - return cast(str, network.name) + return cast("str", network.name) except (ipaddress.AddressValueError, OSError): pass return None @@ -163,7 +163,7 @@ def port(self, container_id: str, port: int) -> str: port_mappings = self.client.api.port(container_id, port) if not port_mappings: raise ConnectionError(f"Port mapping for container {container_id} and port {port} is not available") - return cast(str, port_mappings[0]["HostPort"]) + return cast("str", port_mappings[0]["HostPort"]) def get_container(self, container_id: str) -> dict[str, Any]: """ @@ -172,7 +172,7 @@ def get_container(self, container_id: str) -> dict[str, Any]: containers = self.client.api.containers(filters={"id": container_id}) if not containers: raise RuntimeError(f"Could not get container with id {container_id}") - return cast(dict[str, Any], containers[0]) + return cast("dict[str, Any]", containers[0]) def bridge_ip(self, container_id: str) -> str: """ @@ -241,7 +241,7 @@ def host(self) -> str: hostname = url.hostname if not hostname or (hostname == "localnpipe" and utils.is_windows()): return "localhost" - return cast(str, url.hostname) + return cast("str", url.hostname) if utils.inside_container() and ("unix" in url.scheme or "npipe" in url.scheme): ip_address = utils.default_gateway_ip() if ip_address: @@ -257,7 +257,7 @@ def login(self, auth_config: DockerAuthInfo) -> None: def client_networks_create(self, name: str, param: dict[str, Any]) -> dict[str, Any]: labels = create_labels("", param.get("labels")) - return cast(dict[str, Any], self.client.networks.create(name, **{**param, "labels": labels})) + return cast("dict[str, Any]", self.client.networks.create(name, **{**param, "labels": labels})) def get_docker_host() -> Optional[str]: diff --git a/poetry.lock b/poetry.lock index 7101e1e45..563589b6c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -6157,4 +6157,4 @@ weaviate = ["weaviate-client"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "91d17ef0905329e552e8181a3b12f03e4c5244a7f23b9278168588e76c2a5802" +content-hash = "e17b2d64a82b0929e19aa488550d2159c713979a3145fdfe103c62cd486f79fc"