From 94abb682b835d47a41d7c1064234db6bce67dac5 Mon Sep 17 00:00:00 2001 From: Max Pfeiffer Date: Sun, 29 Oct 2023 12:19:46 +0100 Subject: [PATCH 1/5] Added a new Docker registry test container and tests --- registry/README.rst | 2 + registry/setup.py | 18 +++++ registry/testcontainers/registry/__init__.py | 74 ++++++++++++++++++++ registry/tests/test_registry.py | 26 +++++++ 4 files changed, 120 insertions(+) create mode 100644 registry/README.rst create mode 100644 registry/setup.py create mode 100644 registry/testcontainers/registry/__init__.py create mode 100644 registry/tests/test_registry.py diff --git a/registry/README.rst b/registry/README.rst new file mode 100644 index 000000000..398e5df9c --- /dev/null +++ b/registry/README.rst @@ -0,0 +1,2 @@ +.. autoclass:: testcontainers.registry.DockerRegistryContainer + \ No newline at end of file diff --git a/registry/setup.py b/registry/setup.py new file mode 100644 index 000000000..ee78d6831 --- /dev/null +++ b/registry/setup.py @@ -0,0 +1,18 @@ +from setuptools import setup, find_namespace_packages + +description = "Docker registry component of testcontainers-python." + +setup( + name="testcontainers-registry", + version="0.0.1rc1", + packages=find_namespace_packages(), + description=description, + long_description=description, + long_description_content_type="text/x-rst", + url="https://github.com/testcontainers/testcontainers-python", + install_requires=[ + "testcontainers-core", + "bcrypt", + ], + python_requires=">=3.7", +) diff --git a/registry/testcontainers/registry/__init__.py b/registry/testcontainers/registry/__init__.py new file mode 100644 index 000000000..281efa9a2 --- /dev/null +++ b/registry/testcontainers/registry/__init__.py @@ -0,0 +1,74 @@ +import time +from io import BytesIO +from tarfile import TarFile, TarInfo +from typing import Optional + +import bcrypt +from requests import Response, get +from requests.auth import HTTPBasicAuth +from requests.exceptions import ConnectionError, ReadTimeout +from testcontainers.core.container import DockerContainer +from testcontainers.core.waiting_utils import wait_container_is_ready + +class DockerRegistryContainer(DockerContainer): + # https://docs.docker.com/registry/ + credentials_path: str = "/htpasswd/credentials.txt" + + def __init__( + self, + image: str = "registry:2", + port: int = 5000, + username: str = None, + password: str = None, + **kwargs, + ) -> None: + super().__init__(image=image, **kwargs) + self.port: int = port + self.username: Optional[str] = username + self.password: Optional[str] = password + self.with_exposed_ports(self.port) + + def _copy_credentials(self) -> None: + # Create credentials and write them to the container + hashed_password: str = bcrypt.hashpw( + self.password.encode("utf-8"), + bcrypt.gensalt(rounds=12, prefix=b"2a"), + ).decode("utf-8") + content = f"{self.username}:{hashed_password}".encode("utf-8") + + with BytesIO() as tar_archive_object, TarFile( + fileobj=tar_archive_object, mode="w" + ) as tmp_tarfile: + tarinfo: TarInfo = TarInfo(name=self.credentials_path) + tarinfo.size = len(content) + tarinfo.mtime = time.time() + + tmp_tarfile.addfile(tarinfo, BytesIO(content)) + tar_archive_object.seek(0) + self.get_wrapped_container().put_archive("/", tar_archive_object) + + @wait_container_is_ready(ConnectionError, ReadTimeout) + def _readiness_probe(self) -> None: + url: str = f"http://{self.get_registry()}/v2" + if self.username and self.password: + response: Response = get(url, auth=HTTPBasicAuth(self.username, self.password), timeout=1) + else: + response: Response = get(url, timeout=1) + response.raise_for_status() + + def start(self): + if self.username and self.password: + self.with_env("REGISTRY_AUTH_HTPASSWD_REALM", "local-registry") + self.with_env("REGISTRY_AUTH_HTPASSWD_PATH", self.credentials_path) + super().start() + self._copy_credentials() + else: + super().start() + + self._readiness_probe() + return self + + def get_registry(self) -> str: + host: str = self.get_container_host_ip() + port: str = self.get_exposed_port(self.port) + return f"{host}:{port}" diff --git a/registry/tests/test_registry.py b/registry/tests/test_registry.py new file mode 100644 index 000000000..82502683d --- /dev/null +++ b/registry/tests/test_registry.py @@ -0,0 +1,26 @@ +from requests import Response, get +from requests.auth import HTTPBasicAuth +from testcontainers.registry import DockerRegistryContainer + + +REGISTRY_USERNAME: str = "foo" +REGISTRY_PASSWORD: str ="bar" + +def test_registry(): + with DockerRegistryContainer().with_bind_ports(5000, 5000) as registry_container: + url: str = f"http://{registry_container.get_registry()}/v2/_catalog" + + response: Response = get(url) + + assert response.status_code == 200 + + +def test_registry_with_authentication(): + with DockerRegistryContainer( + username=REGISTRY_USERNAME, password=REGISTRY_PASSWORD + ).with_bind_ports(5000, 5000) as registry_container: + url: str = f"http://{registry_container.get_registry()}/v2/_catalog" + + response: Response = get(url, auth=HTTPBasicAuth(REGISTRY_USERNAME, REGISTRY_PASSWORD)) + + assert response.status_code == 200 From 9de42f4491b7f7b5f63ce2916bd568ba89e8d73e Mon Sep 17 00:00:00 2001 From: Max Pfeiffer Date: Sat, 2 Mar 2024 13:47:10 +0100 Subject: [PATCH 2/5] Refactored registry module to match the new project setup --- {registry => modules/registry}/README.rst | 0 .../testcontainers/registry/__init__.py | 0 .../registry}/tests/test_registry.py | 0 poetry.lock | 43 ++++++++++++++++++- pyproject.toml | 3 ++ registry/setup.py | 18 -------- 6 files changed, 45 insertions(+), 19 deletions(-) rename {registry => modules/registry}/README.rst (100%) rename {registry => modules/registry}/testcontainers/registry/__init__.py (100%) rename {registry => modules/registry}/tests/test_registry.py (100%) delete mode 100644 registry/setup.py diff --git a/registry/README.rst b/modules/registry/README.rst similarity index 100% rename from registry/README.rst rename to modules/registry/README.rst diff --git a/registry/testcontainers/registry/__init__.py b/modules/registry/testcontainers/registry/__init__.py similarity index 100% rename from registry/testcontainers/registry/__init__.py rename to modules/registry/testcontainers/registry/__init__.py diff --git a/registry/tests/test_registry.py b/modules/registry/tests/test_registry.py similarity index 100% rename from registry/tests/test_registry.py rename to modules/registry/tests/test_registry.py diff --git a/poetry.lock b/poetry.lock index 7297f586f..5e126249d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -220,6 +220,46 @@ files = [ {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, ] +[[package]] +name = "bcrypt" +version = "4.1.2" +description = "Modern password hashing for your software and your servers" +optional = true +python-versions = ">=3.7" +files = [ + {file = "bcrypt-4.1.2-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:ac621c093edb28200728a9cca214d7e838529e557027ef0581685909acd28b5e"}, + {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea505c97a5c465ab8c3ba75c0805a102ce526695cd6818c6de3b1a38f6f60da1"}, + {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:57fa9442758da926ed33a91644649d3e340a71e2d0a5a8de064fb621fd5a3326"}, + {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb3bd3321517916696233b5e0c67fd7d6281f0ef48e66812db35fc963a422a1c"}, + {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6cad43d8c63f34b26aef462b6f5e44fdcf9860b723d2453b5d391258c4c8e966"}, + {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:44290ccc827d3a24604f2c8bcd00d0da349e336e6503656cb8192133e27335e2"}, + {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:732b3920a08eacf12f93e6b04ea276c489f1c8fb49344f564cca2adb663b3e4c"}, + {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1c28973decf4e0e69cee78c68e30a523be441972c826703bb93099868a8ff5b5"}, + {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b8df79979c5bae07f1db22dcc49cc5bccf08a0380ca5c6f391cbb5790355c0b0"}, + {file = "bcrypt-4.1.2-cp37-abi3-win32.whl", hash = "sha256:fbe188b878313d01b7718390f31528be4010fed1faa798c5a1d0469c9c48c369"}, + {file = "bcrypt-4.1.2-cp37-abi3-win_amd64.whl", hash = "sha256:9800ae5bd5077b13725e2e3934aa3c9c37e49d3ea3d06318010aa40f54c63551"}, + {file = "bcrypt-4.1.2-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:71b8be82bc46cedd61a9f4ccb6c1a493211d031415a34adde3669ee1b0afbb63"}, + {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e3c6642077b0c8092580c819c1684161262b2e30c4f45deb000c38947bf483"}, + {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:387e7e1af9a4dd636b9505a465032f2f5cb8e61ba1120e79a0e1cd0b512f3dfc"}, + {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f70d9c61f9c4ca7d57f3bfe88a5ccf62546ffbadf3681bb1e268d9d2e41c91a7"}, + {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2a298db2a8ab20056120b45e86c00a0a5eb50ec4075b6142db35f593b97cb3fb"}, + {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:ba55e40de38a24e2d78d34c2d36d6e864f93e0d79d0b6ce915e4335aa81d01b1"}, + {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:3566a88234e8de2ccae31968127b0ecccbb4cddb629da744165db72b58d88ca4"}, + {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b90e216dc36864ae7132cb151ffe95155a37a14e0de3a8f64b49655dd959ff9c"}, + {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:69057b9fc5093ea1ab00dd24ede891f3e5e65bee040395fb1e66ee196f9c9b4a"}, + {file = "bcrypt-4.1.2-cp39-abi3-win32.whl", hash = "sha256:02d9ef8915f72dd6daaef40e0baeef8a017ce624369f09754baf32bb32dba25f"}, + {file = "bcrypt-4.1.2-cp39-abi3-win_amd64.whl", hash = "sha256:be3ab1071662f6065899fe08428e45c16aa36e28bc42921c4901a191fda6ee42"}, + {file = "bcrypt-4.1.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d75fc8cd0ba23f97bae88a6ec04e9e5351ff3c6ad06f38fe32ba50cbd0d11946"}, + {file = "bcrypt-4.1.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:a97e07e83e3262599434816f631cc4c7ca2aa8e9c072c1b1a7fec2ae809a1d2d"}, + {file = "bcrypt-4.1.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e51c42750b7585cee7892c2614be0d14107fad9581d1738d954a262556dd1aab"}, + {file = "bcrypt-4.1.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba4e4cc26610581a6329b3937e02d319f5ad4b85b074846bf4fef8a8cf51e7bb"}, + {file = "bcrypt-4.1.2.tar.gz", hash = "sha256:33313a1200a3ae90b75587ceac502b048b840fc69e7f7a0905b5f87fac7a1258"}, +] + +[package.extras] +tests = ["pytest (>=3.2.1,!=3.3.0)"] +typecheck = ["mypy"] + [[package]] name = "boto3" version = "1.34.59" @@ -4128,10 +4168,11 @@ postgres = [] qdrant = ["qdrant-client"] rabbitmq = ["pika"] redis = ["redis"] +registry = ["bcrypt"] selenium = ["selenium"] weaviate = ["weaviate-client"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "b1683eae41b087e97eca3bfa0c2105824827f2911756f6dad5d69424eb1e976a" +content-hash = "1f8acb3c00fa87c82b3e283406826f36b138a304428696454a9d108de7445120" diff --git a/pyproject.toml b/pyproject.toml index 9281283a7..9f70b7904 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,7 @@ packages = [ { include = "testcontainers", from = "modules/qdrant" }, { include = "testcontainers", from = "modules/rabbitmq" }, { include = "testcontainers", from = "modules/redis" }, + { include = "testcontainers", from = "modules/registry" }, { include = "testcontainers", from = "modules/selenium" }, { include = "testcontainers", from = "modules/weaviate" } ] @@ -94,6 +95,7 @@ selenium = { version = "*", optional = true } weaviate-client = { version = "^4.5.4", optional = true } chromadb-client = { version = "*", optional = true } qdrant-client = { version = "*", optional = true } +bcrypt = { version = "*", optional = true } [tool.poetry.extras] arangodb = ["python-arango"] @@ -120,6 +122,7 @@ postgres = [] qdrant = ["qdrant-client"] rabbitmq = ["pika"] redis = ["redis"] +registry = ["bcrypt"] selenium = ["selenium"] weaviate = ["weaviate-client"] chroma = ["chromadb-client"] diff --git a/registry/setup.py b/registry/setup.py deleted file mode 100644 index ee78d6831..000000000 --- a/registry/setup.py +++ /dev/null @@ -1,18 +0,0 @@ -from setuptools import setup, find_namespace_packages - -description = "Docker registry component of testcontainers-python." - -setup( - name="testcontainers-registry", - version="0.0.1rc1", - packages=find_namespace_packages(), - description=description, - long_description=description, - long_description_content_type="text/x-rst", - url="https://github.com/testcontainers/testcontainers-python", - install_requires=[ - "testcontainers-core", - "bcrypt", - ], - python_requires=">=3.7", -) From f07c71f82c3ddb027b337afd8120b232d03268ee Mon Sep 17 00:00:00 2001 From: Max Pfeiffer Date: Sat, 2 Mar 2024 13:51:54 +0100 Subject: [PATCH 3/5] Updated registry module docs --- modules/registry/README.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/registry/README.rst b/modules/registry/README.rst index 398e5df9c..7fbe081c0 100644 --- a/modules/registry/README.rst +++ b/modules/registry/README.rst @@ -1,2 +1,8 @@ .. autoclass:: testcontainers.registry.DockerRegistryContainer - \ No newline at end of file + +When building Docker containers with Docker Buildx there is currently no option to test your containers locally without +a local registry. Otherwise Buildx pushes your image to Docker Hub, which is not what you want in a test case. More +and more you need to use Buildx for effienciently building images and especially multi arch images. + +When you use Docker Python libraries like docker-py or python-on-whales to build and test Docker images, what a lot of +persons and DevOps engineers like me nowadays do, a test container comes in very handy. From eea3b1662ae44fac032e7d9e940a6738e036dbaa Mon Sep 17 00:00:00 2001 From: Max Pfeiffer Date: Sat, 2 Mar 2024 13:52:08 +0100 Subject: [PATCH 4/5] Updated registry module docs --- modules/registry/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/registry/README.rst b/modules/registry/README.rst index 7fbe081c0..f503f338e 100644 --- a/modules/registry/README.rst +++ b/modules/registry/README.rst @@ -2,7 +2,7 @@ When building Docker containers with Docker Buildx there is currently no option to test your containers locally without a local registry. Otherwise Buildx pushes your image to Docker Hub, which is not what you want in a test case. More -and more you need to use Buildx for effienciently building images and especially multi arch images. +and more you need to use Buildx for efficiently building images and especially multi arch images. When you use Docker Python libraries like docker-py or python-on-whales to build and test Docker images, what a lot of persons and DevOps engineers like me nowadays do, a test container comes in very handy. From 13ac88211aaa3dd6ebb463acaf1fc1d72bbbb6bc Mon Sep 17 00:00:00 2001 From: Max Pfeiffer Date: Sun, 3 Mar 2024 09:59:36 +0100 Subject: [PATCH 5/5] Fixed linter findings, formatted code --- index.rst | 1 + .../testcontainers/registry/__init__.py | 21 +++++++++++-------- modules/registry/tests/test_registry.py | 9 ++++---- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/index.rst b/index.rst index 71af37256..4459624cd 100644 --- a/index.rst +++ b/index.rst @@ -40,6 +40,7 @@ testcontainers-python facilitates the use of Docker containers for functional an modules/qdrant/README modules/rabbitmq/README modules/redis/README + modules/registry/README modules/selenium/README modules/weaviate/README diff --git a/modules/registry/testcontainers/registry/__init__.py b/modules/registry/testcontainers/registry/__init__.py index 281efa9a2..7b846ad5c 100644 --- a/modules/registry/testcontainers/registry/__init__.py +++ b/modules/registry/testcontainers/registry/__init__.py @@ -1,15 +1,20 @@ import time from io import BytesIO from tarfile import TarFile, TarInfo -from typing import Optional +from typing import TYPE_CHECKING, Optional import bcrypt -from requests import Response, get +from requests import get from requests.auth import HTTPBasicAuth from requests.exceptions import ConnectionError, ReadTimeout + from testcontainers.core.container import DockerContainer from testcontainers.core.waiting_utils import wait_container_is_ready +if TYPE_CHECKING: + from requests import Response + + class DockerRegistryContainer(DockerContainer): # https://docs.docker.com/registry/ credentials_path: str = "/htpasswd/credentials.txt" @@ -18,8 +23,8 @@ def __init__( self, image: str = "registry:2", port: int = 5000, - username: str = None, - password: str = None, + username: Optional[str] = None, + password: Optional[str] = None, **kwargs, ) -> None: super().__init__(image=image, **kwargs) @@ -34,11 +39,9 @@ def _copy_credentials(self) -> None: self.password.encode("utf-8"), bcrypt.gensalt(rounds=12, prefix=b"2a"), ).decode("utf-8") - content = f"{self.username}:{hashed_password}".encode("utf-8") + content: bytes = f"{self.username}:{hashed_password}".encode("utf-8") # noqa: UP012 - with BytesIO() as tar_archive_object, TarFile( - fileobj=tar_archive_object, mode="w" - ) as tmp_tarfile: + with BytesIO() as tar_archive_object, TarFile(fileobj=tar_archive_object, mode="w") as tmp_tarfile: tarinfo: TarInfo = TarInfo(name=self.credentials_path) tarinfo.size = len(content) tarinfo.mtime = time.time() @@ -65,7 +68,7 @@ def start(self): else: super().start() - self._readiness_probe() + self._readiness_probe() return self def get_registry(self) -> str: diff --git a/modules/registry/tests/test_registry.py b/modules/registry/tests/test_registry.py index 82502683d..0aa568ee5 100644 --- a/modules/registry/tests/test_registry.py +++ b/modules/registry/tests/test_registry.py @@ -4,7 +4,8 @@ REGISTRY_USERNAME: str = "foo" -REGISTRY_PASSWORD: str ="bar" +REGISTRY_PASSWORD: str = "bar" + def test_registry(): with DockerRegistryContainer().with_bind_ports(5000, 5000) as registry_container: @@ -16,9 +17,9 @@ def test_registry(): def test_registry_with_authentication(): - with DockerRegistryContainer( - username=REGISTRY_USERNAME, password=REGISTRY_PASSWORD - ).with_bind_ports(5000, 5000) as registry_container: + with DockerRegistryContainer(username=REGISTRY_USERNAME, password=REGISTRY_PASSWORD).with_bind_ports( + 5000, 5000 + ) as registry_container: url: str = f"http://{registry_container.get_registry()}/v2/_catalog" response: Response = get(url, auth=HTTPBasicAuth(REGISTRY_USERNAME, REGISTRY_PASSWORD))