From 6bad02a5c3dd6570c884633e430febaf91f0e7e3 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Wed, 14 Feb 2024 06:41:59 -0500 Subject: [PATCH 01/14] compose wip 1 --- core/testcontainers/compose/__init__.py | 7 + core/testcontainers/compose/compose.py | 309 ++++++++++++++++++ .../basic/docker-compose.yaml | 10 + core/tests/test_compose.py | 75 +++++ 4 files changed, 401 insertions(+) create mode 100644 core/testcontainers/compose/__init__.py create mode 100644 core/testcontainers/compose/compose.py create mode 100644 core/tests/compose_fixtures/basic/docker-compose.yaml create mode 100644 core/tests/test_compose.py diff --git a/core/testcontainers/compose/__init__.py b/core/testcontainers/compose/__init__.py new file mode 100644 index 000000000..ee4bff35c --- /dev/null +++ b/core/testcontainers/compose/__init__.py @@ -0,0 +1,7 @@ +from testcontainers.compose.compose import ( + ContainerIsNotRunning, + PortIsNotExposed, + PublishedPort, + ComposeContainer, + DockerCompose +) diff --git a/core/testcontainers/compose/compose.py b/core/testcontainers/compose/compose.py new file mode 100644 index 000000000..7666e3553 --- /dev/null +++ b/core/testcontainers/compose/compose.py @@ -0,0 +1,309 @@ +import subprocess +from dataclasses import dataclass, field, fields +from functools import cached_property +from json import loads +from os import PathLike +from typing import List, Optional, Tuple, Union + +from typing import TypeVar, Type + +_IPT = TypeVar('_IPT') + + +def _ignore_properties(cls: Type[_IPT], dict_: any) -> _IPT: + """omits extra fields like @JsonIgnoreProperties(ignoreUnknown = true) + + https://gist.github.com/alexanderankin/2a4549ac03554a31bef6eaaf2eaf7fd5""" + if isinstance(dict_, cls): return dict_ # noqa + class_fields = {f.name for f in fields(cls)} + filtered = {k: v for k, v in dict_.items() if k in class_fields} + return cls(**filtered) + + +class ContainerIsNotRunning(RuntimeError): + pass + + +class PortIsNotExposed(RuntimeError): + pass + + +@dataclass +class PublishedPort: + """ + Class that represents the response we get from compose when inquiring status + via `DockerCompose.get_running_containers()`. + """ + URL: Optional[str] = None + TargetPort: Optional[str] = None + PublishedPort: Optional[str] = None + Protocol: Optional[str] = None + + +@dataclass +class ComposeContainer: + """ + A container class that represents a container managed by compose. + It is not a true testcontainers.core.container.DockerContainer, + but you can use the id with DockerClient to get that one too. + """ + ID: Optional[str] = None + Name: Optional[str] = None + Command: Optional[str] = None + Project: Optional[str] = None + Service: Optional[str] = None + State: Optional[str] = None + Health: Optional[str] = None + ExitCode: Optional[str] = None + Publishers: List[PublishedPort] = field(default_factory=list) + + def __post_init__(self): + if self.Publishers: + self.Publishers = [ + _ignore_properties(PublishedPort, p) for p in self.Publishers + ] + + # TODO: you can ask testcontainers.core.docker_client.DockerClient.get_container(self, id: str) + # to get you a testcontainer instance which then can stop/restart the instance individually + + def get_publisher( + self, + by_port: Optional[int] = None, + by_host: Optional[str] = None + ) -> PublishedPort: + remaining_publishers = self.Publishers + + if by_port: + remaining_publishers = [ + item for item in remaining_publishers + if item.TargetPort == by_port + ] + if by_host: + remaining_publishers = [ + item for item in remaining_publishers + if item.URL == by_host + ] + if len(remaining_publishers) == 0: + raise PortIsNotExposed( + f"Could not find publisher for for service {self.Service}") + return remaining_publishers[0] + + +@dataclass +class DockerCompose: + """ + Manage docker compose environments. + + Args: + context: + The docker context. It corresponds to the directory containing + the docker compose configuration file. + compose_file_name: + Optional. File name of the docker compose configuration file. + If specified, you need to also specify the overrides if any. + pull: + Pull images before launching environment. + build: + Run `docker compose build` before running the environment. + wait: + Wait for the services to be healthy + (as per healthcheck definitions in the docker compose configuration) + env_file: + Path to an '.env' file containing environment variables + to pass to docker compose. + services: + The list of services to use from this DockerCompose. + + Example: + + This example spins up chrome and firefox containers using docker compose. + + .. doctest:: + + >>> from testcontainers.compose import DockerCompose + + >>> compose = DockerCompose("compose/tests", compose_file_name="docker-compose-4.yml", + ... pull=True) + >>> with compose: + ... stdout, stderr = compose.get_logs() + >>> b"Hello from Docker!" in stdout + True + + .. code-block:: yaml + + services: + hello-world: + image: "hello-world" + """ + + context: Union[str, PathLike] + compose_file_name: Optional[Union[str, List[str]]] = None + pull: bool = False + build: bool = False + wait: bool = True + env_file: Optional[str] = None + services: Optional[List[str]] = None + + def __post_init__(self): + if isinstance(self.compose_file_name, str): + self.compose_file_name = [self.compose_file_name] + + def __enter__(self) -> "DockerCompose": + self.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + self.stop() + + @cached_property + def docker_compose_command(self) -> List[str]: + """ + Returns command parts used for the docker compose commands + + Returns: + cmd: Docker compose command parts. + """ + docker_compose_cmd = ['docker', 'compose'] + if self.compose_file_name: + for file in self.compose_file_name: + docker_compose_cmd += ['-f', file] + if self.env_file: + docker_compose_cmd += ['--env-file', self.env_file] + return docker_compose_cmd + + def start(self) -> None: + """ + Starts the docker compose environment. + """ + base_cmd = self.docker_compose_command or [] + + # pull means running a separate command before starting + if self.pull: + pull_cmd = base_cmd + ['pull'] + self._call_command(cmd=pull_cmd) + + up_cmd = base_cmd + ['up'] + + # build means modifying the up command + if self.build: + up_cmd.append('--build') + + if self.wait: + up_cmd.append('--wait') + else: + # we run in detached mode instead of blocking + up_cmd.append('--detach') + + if self.services: + up_cmd.extend(self.services) + + self._call_command(cmd=up_cmd) + + def stop(self, down=True) -> None: + """ + Stops the docker compose environment. + """ + down_cmd = self.docker_compose_command[:] + if down: + down_cmd += ['down', '--volumes'] + else: + down_cmd += ['stop'] + self._call_command(cmd=down_cmd) + + def get_logs(self, *services: str) -> Tuple[str, str]: + """ + Returns all log output from stdout and stderr of a specific container. + + :param services: which services to get the logs for (or omit, for all) + + Returns: + stdout: Standard output stream. + stderr: Standard error stream. + """ + logs_cmd = self.docker_compose_command + ["logs", *services] + + result = subprocess.run( + logs_cmd, + cwd=self.context, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return result.stdout.decode("utf-8"), result.stderr.decode("utf-8") + + def get_containers(self, include_all=False) -> List[ComposeContainer]: + """ + Fetch information about running containers via `docker compose ps --format json`. + Available only in V2 of compose. + + Returns: + The list of running containers. + + """ + + cmd = self.docker_compose_command + ["ps", "--format", "json"] + if include_all: + cmd += ["-a"] + result = subprocess.run(cmd, cwd=self.context, check=True, stdout=subprocess.PIPE) + stdout = result.stdout.decode("utf-8") + if not stdout: + return [] + json_object = loads(stdout) + + if not isinstance(json_object, list): + return [_ignore_properties(ComposeContainer, json_object)] + + return [ + _ignore_properties(ComposeContainer, item) for item in json_object + ] + + def get_container(self, service_name: str, include_all=False) -> ComposeContainer: + matching_containers = [ + item for item in self.get_containers(include_all=include_all) + if item.Service == service_name + ] + + if not matching_containers: + raise ContainerIsNotRunning( + f"{service_name} is not running in the compose context") + + return matching_containers[0] + + def exec_in_container( + self, + service_name: str, + command: List[str] + ) -> Tuple[str, str, int]: + """ + Executes a command in the container of one of the services. + + Args: + service_name: Name of the docker compose service to run the command in. + command: Command to execute. + + :param service_name: specify the service name + :param command: the command to run in the container + + Returns: + stdout: Standard output stream. + stderr: Standard error stream. + exit_code: The command's exit code. + """ + exec_cmd = self.docker_compose_command + ['exec', '-T', service_name] + command + result = subprocess.run( + exec_cmd, + cwd=self.context, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + return ( + result.stdout.decode("utf-8"), + result.stderr.decode("utf-8"), + result.returncode + ) + + def _call_command(self, + cmd: Union[str, List[str]], + context: Optional[str] = None) -> None: + context = context or self.context + subprocess.call(cmd, cwd=context) diff --git a/core/tests/compose_fixtures/basic/docker-compose.yaml b/core/tests/compose_fixtures/basic/docker-compose.yaml new file mode 100644 index 000000000..ff3f74220 --- /dev/null +++ b/core/tests/compose_fixtures/basic/docker-compose.yaml @@ -0,0 +1,10 @@ +version: '3.0' + +services: + alpine: + image: alpine:latest + init: true + command: + - sh + - -c + - 'while true; do sleep 0.1 ; date -Ins; done' diff --git a/core/tests/test_compose.py b/core/tests/test_compose.py new file mode 100644 index 000000000..7b313c416 --- /dev/null +++ b/core/tests/test_compose.py @@ -0,0 +1,75 @@ +from pathlib import Path +from time import sleep + +import pytest + +from testcontainers.compose import DockerCompose, ContainerIsNotRunning + +FIXTURES = Path(__file__).parent.joinpath('compose_fixtures') + + +def test_compose_no_file_name(): + basic = DockerCompose(context=FIXTURES / 'basic') + assert basic.compose_file_name is None + + +def test_compose_str_file_name(): + basic = DockerCompose(context=FIXTURES / 'basic', + compose_file_name='docker-compose.yaml') + assert basic.compose_file_name == ['docker-compose.yaml'] + + +def test_compose_list_file_name(): + basic = DockerCompose(context=FIXTURES / 'basic', + compose_file_name=['docker-compose.yaml']) + assert basic.compose_file_name == ['docker-compose.yaml'] + + +def test_compose_stop(): + basic = DockerCompose(context=FIXTURES / 'basic') + basic.stop() + + +def test_compose_start_stop(): + basic = DockerCompose(context=FIXTURES / 'basic') + basic.start() + basic.stop() + + +def test_compose(): + basic = DockerCompose(context=FIXTURES / 'basic') + try: + basic.start() + containers = basic.get_containers(include_all=True) + assert len(containers) == 1 + containers = basic.get_containers() + assert len(containers) == 1 + sleep(1) # container produces some logs + + from_all = containers[0] + assert from_all.State == "running" + assert from_all.Service == "alpine" + + by_name = basic.get_container("alpine") + + assert by_name.Name == from_all.Name + assert by_name.Service == from_all.Service + assert by_name.State == from_all.State + assert by_name.ID == from_all.ID + + assert by_name.ExitCode == 0 + + basic.stop(down=False) + + with pytest.raises(ContainerIsNotRunning): + assert basic.get_container('alpine') is None + + stopped = basic.get_container('alpine', include_all=True) + assert stopped.State == "exited" + finally: + basic.stop() + + +def test_compose_ports(): + single = DockerCompose(context=FIXTURES / 'port_single') + # todo continue From b8dc0cfd86e74badaf3d7d2ea1c7f93677bd3262 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Mon, 19 Feb 2024 20:53:54 -0500 Subject: [PATCH 02/14] propose compose implementation re: #306, #358 --- core/testcontainers/compose/__init__.py | 2 +- core/testcontainers/compose/compose.py | 136 ++++++++++++---- core/testcontainers/core/exceptions.py | 4 + .../port_multiple/compose.yaml | 28 ++++ .../compose_fixtures/port_single/compose.yaml | 14 ++ core/tests/test_compose.py | 150 +++++++++++++++++- 6 files changed, 298 insertions(+), 36 deletions(-) create mode 100644 core/tests/compose_fixtures/port_multiple/compose.yaml create mode 100644 core/tests/compose_fixtures/port_single/compose.yaml diff --git a/core/testcontainers/compose/__init__.py b/core/testcontainers/compose/__init__.py index ee4bff35c..3e5ff578c 100644 --- a/core/testcontainers/compose/__init__.py +++ b/core/testcontainers/compose/__init__.py @@ -1,6 +1,6 @@ from testcontainers.compose.compose import ( ContainerIsNotRunning, - PortIsNotExposed, + NoSuchPortExposed, PublishedPort, ComposeContainer, DockerCompose diff --git a/core/testcontainers/compose/compose.py b/core/testcontainers/compose/compose.py index 7666e3553..c4e8c568f 100644 --- a/core/testcontainers/compose/compose.py +++ b/core/testcontainers/compose/compose.py @@ -3,9 +3,10 @@ from functools import cached_property from json import loads from os import PathLike -from typing import List, Optional, Tuple, Union +from re import split +from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union -from typing import TypeVar, Type +from testcontainers.core.exceptions import NoSuchPortExposed, ContainerIsNotRunning _IPT = TypeVar('_IPT') @@ -20,14 +21,6 @@ def _ignore_properties(cls: Type[_IPT], dict_: any) -> _IPT: return cls(**filtered) -class ContainerIsNotRunning(RuntimeError): - pass - - -class PortIsNotExposed(RuntimeError): - pass - - @dataclass class PublishedPort: """ @@ -40,6 +33,16 @@ class PublishedPort: Protocol: Optional[str] = None +OT = TypeVar('OT') + + +def one(array: List[OT], exception: Callable[[], Exception]) -> OT: + if len(array) != 1: + e = exception() + raise e + return array[0] + + @dataclass class ComposeContainer: """ @@ -63,9 +66,6 @@ def __post_init__(self): _ignore_properties(PublishedPort, p) for p in self.Publishers ] - # TODO: you can ask testcontainers.core.docker_client.DockerClient.get_container(self, id: str) - # to get you a testcontainer instance which then can stop/restart the instance individually - def get_publisher( self, by_port: Optional[int] = None, @@ -84,9 +84,17 @@ def get_publisher( if item.URL == by_host ] if len(remaining_publishers) == 0: - raise PortIsNotExposed( + raise NoSuchPortExposed( f"Could not find publisher for for service {self.Service}") - return remaining_publishers[0] + return one( + remaining_publishers, + lambda: NoSuchPortExposed( + 'get_publisher failed because there is ' + f'not exactly 1 publisher for service {self.Service}' + f' when filtering by_port={by_port}, by_host={by_host}' + f' (but {len(remaining_publishers)})' + ) + ) @dataclass @@ -113,6 +121,8 @@ class DockerCompose: to pass to docker compose. services: The list of services to use from this DockerCompose. + client_args: + arguments to pass to docker.from_env() Example: @@ -155,7 +165,6 @@ def __enter__(self) -> "DockerCompose": def __exit__(self, exc_type, exc_val, exc_tb) -> None: self.stop() - @cached_property def docker_compose_command(self) -> List[str]: """ Returns command parts used for the docker compose commands @@ -163,6 +172,10 @@ def docker_compose_command(self) -> List[str]: Returns: cmd: Docker compose command parts. """ + return self.compose_command_property + + @cached_property + def compose_command_property(self) -> List[str]: docker_compose_cmd = ['docker', 'compose'] if self.compose_file_name: for file in self.compose_file_name: @@ -175,7 +188,7 @@ def start(self) -> None: """ Starts the docker compose environment. """ - base_cmd = self.docker_compose_command or [] + base_cmd = self.compose_command_property or [] # pull means running a separate command before starting if self.pull: @@ -203,7 +216,7 @@ def stop(self, down=True) -> None: """ Stops the docker compose environment. """ - down_cmd = self.docker_compose_command[:] + down_cmd = self.compose_command_property[:] if down: down_cmd += ['down', '--volumes'] else: @@ -220,7 +233,7 @@ def get_logs(self, *services: str) -> Tuple[str, str]: stdout: Standard output stream. stderr: Standard error stream. """ - logs_cmd = self.docker_compose_command + ["logs", *services] + logs_cmd = self.compose_command_property + ["logs", *services] result = subprocess.run( logs_cmd, @@ -240,23 +253,28 @@ def get_containers(self, include_all=False) -> List[ComposeContainer]: """ - cmd = self.docker_compose_command + ["ps", "--format", "json"] + cmd = self.compose_command_property + ["ps", "--format", "json"] if include_all: cmd += ["-a"] result = subprocess.run(cmd, cwd=self.context, check=True, stdout=subprocess.PIPE) - stdout = result.stdout.decode("utf-8") - if not stdout: - return [] - json_object = loads(stdout) - - if not isinstance(json_object, list): - return [_ignore_properties(ComposeContainer, json_object)] + stdout = split(r'\r?\n', result.stdout.decode("utf-8")) return [ - _ignore_properties(ComposeContainer, item) for item in json_object + _ignore_properties(ComposeContainer, loads(line)) for line in stdout if line ] - def get_container(self, service_name: str, include_all=False) -> ComposeContainer: + def get_container( + self, + service_name: Optional[str] = None, + include_all: bool = False + ) -> ComposeContainer: + if not service_name: + c = self.get_containers(include_all=include_all) + return one(c, lambda: ContainerIsNotRunning( + 'get_container failed because no service_name given ' + f'and there is not exactly 1 container (but {len(c)})' + )) + matching_containers = [ item for item in self.get_containers(include_all=include_all) if item.Service == service_name @@ -270,8 +288,8 @@ def get_container(self, service_name: str, include_all=False) -> ComposeContaine def exec_in_container( self, - service_name: str, - command: List[str] + command: List[str], + service_name: Optional[str] = None, ) -> Tuple[str, str, int]: """ Executes a command in the container of one of the services. @@ -288,7 +306,9 @@ def exec_in_container( stderr: Standard error stream. exit_code: The command's exit code. """ - exec_cmd = self.docker_compose_command + ['exec', '-T', service_name] + command + if not service_name: + service_name = self.get_container().Service + exec_cmd = self.compose_command_property + ['exec', '-T', service_name] + command result = subprocess.run( exec_cmd, cwd=self.context, @@ -307,3 +327,55 @@ def _call_command(self, context: Optional[str] = None) -> None: context = context or self.context subprocess.call(cmd, cwd=context) + + def get_service_port( + self, + service_name: Optional[str] = None, + port: Optional[int] = None + ): + """ + Returns the mapped port for one of the services. + + Parameters + ---------- + service_name: str + Name of the docker compose service + port: int + The internal port to get the mapping for + + Returns + ------- + str: + The mapped port on the host + """ + return self.get_container(service_name).get_publisher(by_port=port).PublishedPort + + def get_service_host( + self, + service_name: Optional[str] = None, + port: Optional[int] = None + ): + """ + Returns the host for one of the services. + + Parameters + ---------- + service_name: str + Name of the docker compose service + port: int + The internal port to get the host for + + Returns + ------- + str: + The hostname for the service + """ + return self.get_container(service_name).get_publisher(by_port=port).URL + + def get_service_host_and_port( + self, + service_name: Optional[str] = None, + port: Optional[int] = None + ): + publisher = self.get_container(service_name).get_publisher(by_port=port) + return publisher.URL, publisher.PublishedPort diff --git a/core/testcontainers/core/exceptions.py b/core/testcontainers/core/exceptions.py index 8bf027630..6694e598b 100644 --- a/core/testcontainers/core/exceptions.py +++ b/core/testcontainers/core/exceptions.py @@ -16,5 +16,9 @@ class ContainerStartException(RuntimeError): pass +class ContainerIsNotRunning(RuntimeError): + pass + + class NoSuchPortExposed(RuntimeError): pass diff --git a/core/tests/compose_fixtures/port_multiple/compose.yaml b/core/tests/compose_fixtures/port_multiple/compose.yaml new file mode 100644 index 000000000..65717fc4a --- /dev/null +++ b/core/tests/compose_fixtures/port_multiple/compose.yaml @@ -0,0 +1,28 @@ +version: '3.0' + +services: + alpine: + image: nginx:alpine-slim + init: true + ports: + - '81' + - '82' + - target: 80 + host_ip: 127.0.0.1 + protocol: tcp + command: + - sh + - -c + - 'd=/etc/nginx/conf.d; echo "server { listen 81; location / { return 202; } }" > $$d/81.conf && echo "server { listen 82; location / { return 204; } }" > $$d/82.conf && nginx -g "daemon off;"' + + alpine2: + image: nginx:alpine-slim + init: true + ports: + - target: 80 + host_ip: 127.0.0.1 + protocol: tcp + command: + - sh + - -c + - 'd=/etc/nginx/conf.d; echo "server { listen 81; location / { return 202; } }" > $$d/81.conf && echo "server { listen 82; location / { return 204; } }" > $$d/82.conf && nginx -g "daemon off;"' diff --git a/core/tests/compose_fixtures/port_single/compose.yaml b/core/tests/compose_fixtures/port_single/compose.yaml new file mode 100644 index 000000000..d1bf9eb45 --- /dev/null +++ b/core/tests/compose_fixtures/port_single/compose.yaml @@ -0,0 +1,14 @@ +version: '3.0' + +services: + alpine: + image: nginx:alpine-slim + init: true + ports: + - target: 80 + host_ip: 127.0.0.1 + protocol: tcp + command: + - sh + - -c + - 'nginx -g "daemon off;"' diff --git a/core/tests/test_compose.py b/core/tests/test_compose.py index 7b313c416..efb3bd054 100644 --- a/core/tests/test_compose.py +++ b/core/tests/test_compose.py @@ -1,9 +1,12 @@ from pathlib import Path +from re import split from time import sleep +from typing import Union +from urllib.request import urlopen, Request import pytest -from testcontainers.compose import DockerCompose, ContainerIsNotRunning +from testcontainers.compose import DockerCompose, ContainerIsNotRunning, NoSuchPortExposed FIXTURES = Path(__file__).parent.joinpath('compose_fixtures') @@ -37,15 +40,21 @@ def test_compose_start_stop(): def test_compose(): + """stream-of-consciousness e2e test""" basic = DockerCompose(context=FIXTURES / 'basic') try: + # first it does not exist + containers = basic.get_containers(include_all=True) + assert len(containers) == 0 + + # then we create it and it exists basic.start() containers = basic.get_containers(include_all=True) assert len(containers) == 1 containers = basic.get_containers() assert len(containers) == 1 - sleep(1) # container produces some logs + # test that get_container returns the same object, value assertions, etc from_all = containers[0] assert from_all.State == "running" assert from_all.Service == "alpine" @@ -59,17 +68,152 @@ def test_compose(): assert by_name.ExitCode == 0 + # what if you want to get logs after it crashes: basic.stop(down=False) with pytest.raises(ContainerIsNotRunning): assert basic.get_container('alpine') is None + # what it looks like after it exits stopped = basic.get_container('alpine', include_all=True) assert stopped.State == "exited" finally: basic.stop() +def test_compose_logs(): + basic = DockerCompose(context=FIXTURES / 'basic') + with basic: + sleep(1) # generate some logs every 200ms + stdout, stderr = basic.get_logs() + container = basic.get_container() + + assert not stderr + assert stdout + lines = split(r'\r?\n', stdout) + + assert len(lines) > 5 # actually 10 + for line in lines[1:]: + assert not line or line.startswith(container.Service) + + +# noinspection HttpUrlsUsage def test_compose_ports(): + # fairly straight forward - can we get the right port to request it + single = DockerCompose(context=FIXTURES / 'port_single') + with single: + host, port = single.get_service_host_and_port() + endpoint = f'http://{host}:{port}' + code, response = fetch(Request(method='GET', url=endpoint)) + assert code == 200 + assert '

' in response + + +# noinspection HttpUrlsUsage +def test_compose_multiple_containers_and_ports(): + """test for the logic encapsulated in 'one' function + + assert correctness of multiple logic + """ + multiple = DockerCompose(context=FIXTURES / 'port_multiple') + with multiple: + with pytest.raises(ContainerIsNotRunning) as e: + multiple.get_container() + e.match('get_container failed') + e.match('not exactly 1 container') + + assert multiple.get_container('alpine') + assert multiple.get_container('alpine2') + + a2p = multiple.get_service_port('alpine2') + assert a2p > 0 # > 1024 + + with pytest.raises(NoSuchPortExposed) as e: + multiple.get_service_port('alpine') + e.match('not exactly 1') + with pytest.raises(NoSuchPortExposed) as e: + multiple.get_container('alpine').get_publisher(by_host='example.com') + e.match('not exactly 1') + with pytest.raises(NoSuchPortExposed) as e: + multiple.get_container('alpine').get_publisher(by_host='localhost') + e.match('not exactly 1') + ports = [ + (80, + multiple.get_service_host(service_name='alpine', port=80), + multiple.get_service_port(service_name='alpine', port=80)), + (81, + multiple.get_service_host(service_name='alpine', port=81), + multiple.get_service_port(service_name='alpine', port=81)), + (82, + multiple.get_service_host(service_name='alpine', port=82), + multiple.get_service_port(service_name='alpine', port=82)), + ] + + # test correctness of port lookup + for target, host, mapped in ports: + assert mapped, f'we have a mapped port for target port {target}' + code, body = fetch(Request(method='GET', url=f'http://{host}:{mapped}')) + if target == 81: + assert code == 202 + if target == 82: + assert code == 204 + + +# noinspection HttpUrlsUsage +def test_exec_in_container(): + """we test that we can manipulate a container via exec""" single = DockerCompose(context=FIXTURES / 'port_single') - # todo continue + with single: + url = f'http://{single.get_service_host()}:{single.get_service_port()}' + + # unchanged + code, body = fetch(url) + assert code == 200 + assert 'test_exec_in_container' not in body + + # change it + single.exec_in_container(command=[ + 'sh', '-c', + 'echo "test_exec_in_container" > /usr/share/nginx/html/index.html' + ]) + + # and it is changed + code, body = fetch(url) + assert code == 200 + assert 'test_exec_in_container' in body + + +# noinspection HttpUrlsUsage +def test_exec_in_container_multiple(): + """same as above, except we exec into a particular service""" + multiple = DockerCompose(context=FIXTURES / 'port_multiple') + with multiple: + sn = 'alpine2' # service name + host, port = multiple.get_service_host_and_port(service_name=sn) + url = f'http://{host}:{port}' + + # unchanged + code, body = fetch(url) + assert code == 200 + assert 'test_exec_in_container' not in body + + # change it + multiple.exec_in_container(command=[ + 'sh', '-c', + 'echo "test_exec_in_container" > /usr/share/nginx/html/index.html' + ], service_name=sn) + + # and it is changed + code, body = fetch(url) + assert code == 200 + assert 'test_exec_in_container' in body + + +def fetch(req: Union[Request, str]): + if isinstance(req, str): + req = Request(method='GET', url=req) + with urlopen(req) as res: + body = res.read().decode('utf-8') + if 200 < res.getcode() >= 400: + raise Exception(f"HTTP Error: {res.getcode()} - {res.reason}: {body}") + return res.getcode(), body From 4e398996b68560a32642a11402de35066bc2dcb8 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Mon, 19 Feb 2024 20:58:57 -0500 Subject: [PATCH 03/14] flake8 --- core/testcontainers/compose/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/testcontainers/compose/__init__.py b/core/testcontainers/compose/__init__.py index 3e5ff578c..47c458012 100644 --- a/core/testcontainers/compose/__init__.py +++ b/core/testcontainers/compose/__init__.py @@ -1,3 +1,4 @@ +# flake8: noqa from testcontainers.compose.compose import ( ContainerIsNotRunning, NoSuchPortExposed, From 1e4fa4edfde97a9fe8d1c399c66f14179d1f236a Mon Sep 17 00:00:00 2001 From: Dave Ankin Date: Mon, 19 Feb 2024 22:29:31 -0500 Subject: [PATCH 04/14] foiled by ipv6, almost --- core/testcontainers/compose/compose.py | 6 +++++- core/tests/test_compose.py | 8 ++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/core/testcontainers/compose/compose.py b/core/testcontainers/compose/compose.py index c4e8c568f..9c4209e31 100644 --- a/core/testcontainers/compose/compose.py +++ b/core/testcontainers/compose/compose.py @@ -69,10 +69,14 @@ def __post_init__(self): def get_publisher( self, by_port: Optional[int] = None, - by_host: Optional[str] = None + by_host: Optional[str] = None, + ipv4_only: bool = True, ) -> PublishedPort: remaining_publishers = self.Publishers + if ipv4_only: + remaining_publishers = [r for r in remaining_publishers if ':' not in r.URL] + if by_port: remaining_publishers = [ item for item in remaining_publishers diff --git a/core/tests/test_compose.py b/core/tests/test_compose.py index efb3bd054..90c93e4b7 100644 --- a/core/tests/test_compose.py +++ b/core/tests/test_compose.py @@ -137,6 +137,14 @@ def test_compose_multiple_containers_and_ports(): with pytest.raises(NoSuchPortExposed) as e: multiple.get_container('alpine').get_publisher(by_host='localhost') e.match('not exactly 1') + + try: + # this fails when ipv6 is enabled and docker is forwarding for both 4 + 6 + multiple.get_container(service_name='alpine') \ + .get_publisher(by_port=81, ipv4_only=False) + except: # noqa + pass + ports = [ (80, multiple.get_service_host(service_name='alpine', port=80), From b4e6887076fd0e0197c3394aa816cee7ae3965cb Mon Sep 17 00:00:00 2001 From: Dave Ankin Date: Mon, 19 Feb 2024 22:43:35 -0500 Subject: [PATCH 05/14] restore waiting --- core/testcontainers/compose/compose.py | 16 ++++++++++++++++ core/tests/test_compose.py | 3 +++ 2 files changed, 19 insertions(+) diff --git a/core/testcontainers/compose/compose.py b/core/testcontainers/compose/compose.py index 9c4209e31..5cb65fcbf 100644 --- a/core/testcontainers/compose/compose.py +++ b/core/testcontainers/compose/compose.py @@ -5,8 +5,11 @@ from os import PathLike from re import split from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union +from urllib.request import urlopen +from urllib.error import URLError, HTTPError from testcontainers.core.exceptions import NoSuchPortExposed, ContainerIsNotRunning +from testcontainers.core.waiting_utils import wait_container_is_ready _IPT = TypeVar('_IPT') @@ -383,3 +386,16 @@ def get_service_host_and_port( ): publisher = self.get_container(service_name).get_publisher(by_port=port) return publisher.URL, publisher.PublishedPort + + @wait_container_is_ready(HTTPError, URLError) + def wait_for(self, url: str) -> 'DockerCompose': + """ + Waits for a response from a given URL. This is typically used to block until a service in + the environment has started and is responding. Note that it does not assert any sort of + return code, only check that the connection was successful. + + Args: + url: URL from one of the services in the environment to use to wait on. + """ + with urlopen(url) as r: r.read() # noqa + return self diff --git a/core/tests/test_compose.py b/core/tests/test_compose.py index 90c93e4b7..97cb8e026 100644 --- a/core/tests/test_compose.py +++ b/core/tests/test_compose.py @@ -104,6 +104,7 @@ def test_compose_ports(): with single: host, port = single.get_service_host_and_port() endpoint = f'http://{host}:{port}' + single.wait_for(endpoint) code, response = fetch(Request(method='GET', url=endpoint)) assert code == 200 assert '

' in response @@ -173,6 +174,7 @@ def test_exec_in_container(): single = DockerCompose(context=FIXTURES / 'port_single') with single: url = f'http://{single.get_service_host()}:{single.get_service_port()}' + single.wait_for(url) # unchanged code, body = fetch(url) @@ -199,6 +201,7 @@ def test_exec_in_container_multiple(): sn = 'alpine2' # service name host, port = multiple.get_service_host_and_port(service_name=sn) url = f'http://{host}:{port}' + multiple.wait_for(url) # unchanged code, body = fetch(url) From 13321c2ed9f2ada9355aab7bc673141dfb51b3fb Mon Sep 17 00:00:00 2001 From: David Ankin Date: Mon, 26 Feb 2024 22:35:51 -0700 Subject: [PATCH 06/14] address feedback, not sure if adequately? --- core/testcontainers/compose/compose.py | 10 ++++++---- core/tests/test_compose.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/core/testcontainers/compose/compose.py b/core/testcontainers/compose/compose.py index 5cb65fcbf..1bc61b628 100644 --- a/core/testcontainers/compose/compose.py +++ b/core/testcontainers/compose/compose.py @@ -4,7 +4,7 @@ from json import loads from os import PathLike from re import split -from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union +from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union, Literal from urllib.request import urlopen from urllib.error import URLError, HTTPError @@ -73,12 +73,14 @@ def get_publisher( self, by_port: Optional[int] = None, by_host: Optional[str] = None, - ipv4_only: bool = True, + prefer_ip_version: Literal["IPV4", "IPv6"] = "IPv4", ) -> PublishedPort: remaining_publishers = self.Publishers - if ipv4_only: - remaining_publishers = [r for r in remaining_publishers if ':' not in r.URL] + remaining_publishers = [ + r for r in remaining_publishers + if (':' in r.URL) is (prefer_ip_version == "IPv6") + ] if by_port: remaining_publishers = [ diff --git a/core/tests/test_compose.py b/core/tests/test_compose.py index 97cb8e026..f3fd1296c 100644 --- a/core/tests/test_compose.py +++ b/core/tests/test_compose.py @@ -142,7 +142,7 @@ def test_compose_multiple_containers_and_ports(): try: # this fails when ipv6 is enabled and docker is forwarding for both 4 + 6 multiple.get_container(service_name='alpine') \ - .get_publisher(by_port=81, ipv4_only=False) + .get_publisher(by_port=81, prefer_ip_version="IPv6") except: # noqa pass From fda3709b2e62ff8ca657570bfef2f3b36b140de4 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Tue, 27 Feb 2024 07:56:21 -0700 Subject: [PATCH 07/14] hate is not a strong enough word for l*nters --- core/testcontainers/compose/__init__.py | 2 +- core/testcontainers/compose/compose.py | 152 +++++++++++++----------- core/tests/test_compose.py | 127 ++++++++++---------- 3 files changed, 147 insertions(+), 134 deletions(-) diff --git a/core/testcontainers/compose/__init__.py b/core/testcontainers/compose/__init__.py index 47c458012..9af994f30 100644 --- a/core/testcontainers/compose/__init__.py +++ b/core/testcontainers/compose/__init__.py @@ -4,5 +4,5 @@ NoSuchPortExposed, PublishedPort, ComposeContainer, - DockerCompose + DockerCompose, ) diff --git a/core/testcontainers/compose/compose.py b/core/testcontainers/compose/compose.py index 1bc61b628..7e69ca860 100644 --- a/core/testcontainers/compose/compose.py +++ b/core/testcontainers/compose/compose.py @@ -4,21 +4,22 @@ from json import loads from os import PathLike from re import split -from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union, Literal +from typing import Callable, Literal, Optional, TypeVar, Union +from urllib.error import HTTPError, URLError from urllib.request import urlopen -from urllib.error import URLError, HTTPError -from testcontainers.core.exceptions import NoSuchPortExposed, ContainerIsNotRunning +from testcontainers.core.exceptions import ContainerIsNotRunning, NoSuchPortExposed from testcontainers.core.waiting_utils import wait_container_is_ready -_IPT = TypeVar('_IPT') +_IPT = TypeVar("_IPT") -def _ignore_properties(cls: Type[_IPT], dict_: any) -> _IPT: +def _ignore_properties(cls: type[_IPT], dict_: any) -> _IPT: """omits extra fields like @JsonIgnoreProperties(ignoreUnknown = true) https://gist.github.com/alexanderankin/2a4549ac03554a31bef6eaaf2eaf7fd5""" - if isinstance(dict_, cls): return dict_ # noqa + if isinstance(dict_, cls): + return dict_ class_fields = {f.name for f in fields(cls)} filtered = {k: v for k, v in dict_.items() if k in class_fields} return cls(**filtered) @@ -30,16 +31,17 @@ class PublishedPort: Class that represents the response we get from compose when inquiring status via `DockerCompose.get_running_containers()`. """ + URL: Optional[str] = None TargetPort: Optional[str] = None PublishedPort: Optional[str] = None Protocol: Optional[str] = None -OT = TypeVar('OT') +OT = TypeVar("OT") -def one(array: List[OT], exception: Callable[[], Exception]) -> OT: +def one(array: list[OT], exception: Callable[[], Exception]) -> OT: if len(array) != 1: e = exception() raise e @@ -53,6 +55,7 @@ class ComposeContainer: It is not a true testcontainers.core.container.DockerContainer, but you can use the id with DockerClient to get that one too. """ + ID: Optional[str] = None Name: Optional[str] = None Command: Optional[str] = None @@ -61,36 +64,35 @@ class ComposeContainer: State: Optional[str] = None Health: Optional[str] = None ExitCode: Optional[str] = None - Publishers: List[PublishedPort] = field(default_factory=list) + Publishers: list[PublishedPort] = field(default_factory=list) def __post_init__(self): if self.Publishers: - self.Publishers = [ - _ignore_properties(PublishedPort, p) for p in self.Publishers - ] + self.Publishers = [_ignore_properties(PublishedPort, p) for p in self.Publishers] + # fmt:off def get_publisher( - self, - by_port: Optional[int] = None, - by_host: Optional[str] = None, - prefer_ip_version: Literal["IPV4", "IPv6"] = "IPv4", + self, + by_port: Optional[int] = None, + by_host: Optional[str] = None, + prefer_ip_version: Literal["IPV4", "IPv6"] = "IPv4", ) -> PublishedPort: remaining_publishers = self.Publishers remaining_publishers = [ r for r in remaining_publishers - if (':' in r.URL) is (prefer_ip_version == "IPv6") + if (":" in r.URL) is (prefer_ip_version == "IPv6") ] if by_port: remaining_publishers = [ item for item in remaining_publishers - if item.TargetPort == by_port + if by_port == item.TargetPort ] if by_host: remaining_publishers = [ item for item in remaining_publishers - if item.URL == by_host + if by_host == item.URL ] if len(remaining_publishers) == 0: raise NoSuchPortExposed( @@ -98,12 +100,13 @@ def get_publisher( return one( remaining_publishers, lambda: NoSuchPortExposed( - 'get_publisher failed because there is ' - f'not exactly 1 publisher for service {self.Service}' - f' when filtering by_port={by_port}, by_host={by_host}' - f' (but {len(remaining_publishers)})' - ) + "get_publisher failed because there is " + f"not exactly 1 publisher for service {self.Service}" + f" when filtering by_port={by_port}, by_host={by_host}" + f" (but {len(remaining_publishers)})" + ), ) + # fmt:on @dataclass @@ -156,12 +159,12 @@ class DockerCompose: """ context: Union[str, PathLike] - compose_file_name: Optional[Union[str, List[str]]] = None + compose_file_name: Optional[Union[str, list[str]]] = None pull: bool = False build: bool = False wait: bool = True env_file: Optional[str] = None - services: Optional[List[str]] = None + services: Optional[list[str]] = None def __post_init__(self): if isinstance(self.compose_file_name, str): @@ -174,7 +177,7 @@ def __enter__(self) -> "DockerCompose": def __exit__(self, exc_type, exc_val, exc_tb) -> None: self.stop() - def docker_compose_command(self) -> List[str]: + def docker_compose_command(self) -> list[str]: """ Returns command parts used for the docker compose commands @@ -184,13 +187,13 @@ def docker_compose_command(self) -> List[str]: return self.compose_command_property @cached_property - def compose_command_property(self) -> List[str]: - docker_compose_cmd = ['docker', 'compose'] + def compose_command_property(self) -> list[str]: + docker_compose_cmd = ["docker", "compose"] if self.compose_file_name: for file in self.compose_file_name: - docker_compose_cmd += ['-f', file] + docker_compose_cmd += ["-f", file] if self.env_file: - docker_compose_cmd += ['--env-file', self.env_file] + docker_compose_cmd += ["--env-file", self.env_file] return docker_compose_cmd def start(self) -> None: @@ -201,20 +204,20 @@ def start(self) -> None: # pull means running a separate command before starting if self.pull: - pull_cmd = base_cmd + ['pull'] + pull_cmd = [*base_cmd, "pull"] self._call_command(cmd=pull_cmd) - up_cmd = base_cmd + ['up'] + up_cmd = [*base_cmd, "up"] # build means modifying the up command if self.build: - up_cmd.append('--build') + up_cmd.append("--build") if self.wait: - up_cmd.append('--wait') + up_cmd.append("--wait") else: # we run in detached mode instead of blocking - up_cmd.append('--detach') + up_cmd.append("--detach") if self.services: up_cmd.extend(self.services) @@ -227,12 +230,12 @@ def stop(self, down=True) -> None: """ down_cmd = self.compose_command_property[:] if down: - down_cmd += ['down', '--volumes'] + down_cmd += ["down", "--volumes"] else: - down_cmd += ['stop'] + down_cmd += ["stop"] self._call_command(cmd=down_cmd) - def get_logs(self, *services: str) -> Tuple[str, str]: + def get_logs(self, *services: str) -> tuple[str, str]: """ Returns all log output from stdout and stderr of a specific container. @@ -242,17 +245,16 @@ def get_logs(self, *services: str) -> Tuple[str, str]: stdout: Standard output stream. stderr: Standard error stream. """ - logs_cmd = self.compose_command_property + ["logs", *services] + logs_cmd = [*self.compose_command_property, "logs", *services] result = subprocess.run( logs_cmd, cwd=self.context, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, ) return result.stdout.decode("utf-8"), result.stderr.decode("utf-8") - def get_containers(self, include_all=False) -> List[ComposeContainer]: + def get_containers(self, include_all=False) -> list[ComposeContainer]: """ Fetch information about running containers via `docker compose ps --format json`. Available only in V2 of compose. @@ -262,20 +264,23 @@ def get_containers(self, include_all=False) -> List[ComposeContainer]: """ - cmd = self.compose_command_property + ["ps", "--format", "json"] + cmd = [*self.compose_command_property, "ps", "--format", "json"] if include_all: - cmd += ["-a"] + cmd = [*cmd, "-a"] result = subprocess.run(cmd, cwd=self.context, check=True, stdout=subprocess.PIPE) - stdout = split(r'\r?\n', result.stdout.decode("utf-8")) + stdout = split(r"\r?\n", result.stdout.decode("utf-8")) + # fmt:off return [ _ignore_properties(ComposeContainer, loads(line)) for line in stdout if line ] + # fmt:on + # fmt:off def get_container( - self, - service_name: Optional[str] = None, - include_all: bool = False + self, + service_name: Optional[str] = None, + include_all: bool = False, ) -> ComposeContainer: if not service_name: c = self.get_containers(include_all=include_all) @@ -294,12 +299,13 @@ def get_container( f"{service_name} is not running in the compose context") return matching_containers[0] + # fmt:on def exec_in_container( - self, - command: List[str], - service_name: Optional[str] = None, - ) -> Tuple[str, str, int]: + self, + command: list[str], + service_name: Optional[str] = None, + ) -> tuple[str, str, int]: """ Executes a command in the container of one of the services. @@ -317,30 +323,34 @@ def exec_in_container( """ if not service_name: service_name = self.get_container().Service - exec_cmd = self.compose_command_property + ['exec', '-T', service_name] + command + exec_cmd = [*self.compose_command_property, "exec", "-T", service_name, *command] result = subprocess.run( exec_cmd, cwd=self.context, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + capture_output=True, check=True, ) + + # fmt:off return ( result.stdout.decode("utf-8"), result.stderr.decode("utf-8"), result.returncode ) + # fmt:on - def _call_command(self, - cmd: Union[str, List[str]], - context: Optional[str] = None) -> None: + def _call_command( + self, + cmd: Union[str, list[str]], + context: Optional[str] = None, + ) -> None: context = context or self.context subprocess.call(cmd, cwd=context) def get_service_port( - self, - service_name: Optional[str] = None, - port: Optional[int] = None + self, + service_name: Optional[str] = None, + port: Optional[int] = None, ): """ Returns the mapped port for one of the services. @@ -360,9 +370,9 @@ def get_service_port( return self.get_container(service_name).get_publisher(by_port=port).PublishedPort def get_service_host( - self, - service_name: Optional[str] = None, - port: Optional[int] = None + self, + service_name: Optional[str] = None, + port: Optional[int] = None, ): """ Returns the host for one of the services. @@ -382,15 +392,15 @@ def get_service_host( return self.get_container(service_name).get_publisher(by_port=port).URL def get_service_host_and_port( - self, - service_name: Optional[str] = None, - port: Optional[int] = None + self, + service_name: Optional[str] = None, + port: Optional[int] = None, ): publisher = self.get_container(service_name).get_publisher(by_port=port) return publisher.URL, publisher.PublishedPort @wait_container_is_ready(HTTPError, URLError) - def wait_for(self, url: str) -> 'DockerCompose': + def wait_for(self, url: str) -> "DockerCompose": """ Waits for a response from a given URL. This is typically used to block until a service in the environment has started and is responding. Note that it does not assert any sort of @@ -399,5 +409,7 @@ def wait_for(self, url: str) -> 'DockerCompose': Args: url: URL from one of the services in the environment to use to wait on. """ - with urlopen(url) as r: r.read() # noqa + # fmt:off + with urlopen(url) as r: r.read() # noqa: E701 + # fmt:on return self diff --git a/core/tests/test_compose.py b/core/tests/test_compose.py index f3fd1296c..1e157c429 100644 --- a/core/tests/test_compose.py +++ b/core/tests/test_compose.py @@ -8,40 +8,38 @@ from testcontainers.compose import DockerCompose, ContainerIsNotRunning, NoSuchPortExposed -FIXTURES = Path(__file__).parent.joinpath('compose_fixtures') +FIXTURES = Path(__file__).parent.joinpath("compose_fixtures") def test_compose_no_file_name(): - basic = DockerCompose(context=FIXTURES / 'basic') + basic = DockerCompose(context=FIXTURES / "basic") assert basic.compose_file_name is None def test_compose_str_file_name(): - basic = DockerCompose(context=FIXTURES / 'basic', - compose_file_name='docker-compose.yaml') - assert basic.compose_file_name == ['docker-compose.yaml'] + basic = DockerCompose(context=FIXTURES / "basic", compose_file_name="docker-compose.yaml") + assert basic.compose_file_name == ["docker-compose.yaml"] def test_compose_list_file_name(): - basic = DockerCompose(context=FIXTURES / 'basic', - compose_file_name=['docker-compose.yaml']) - assert basic.compose_file_name == ['docker-compose.yaml'] + basic = DockerCompose(context=FIXTURES / "basic", compose_file_name=["docker-compose.yaml"]) + assert basic.compose_file_name == ["docker-compose.yaml"] def test_compose_stop(): - basic = DockerCompose(context=FIXTURES / 'basic') + basic = DockerCompose(context=FIXTURES / "basic") basic.stop() def test_compose_start_stop(): - basic = DockerCompose(context=FIXTURES / 'basic') + basic = DockerCompose(context=FIXTURES / "basic") basic.start() basic.stop() def test_compose(): """stream-of-consciousness e2e test""" - basic = DockerCompose(context=FIXTURES / 'basic') + basic = DockerCompose(context=FIXTURES / "basic") try: # first it does not exist containers = basic.get_containers(include_all=True) @@ -72,17 +70,17 @@ def test_compose(): basic.stop(down=False) with pytest.raises(ContainerIsNotRunning): - assert basic.get_container('alpine') is None + assert basic.get_container("alpine") is None # what it looks like after it exits - stopped = basic.get_container('alpine', include_all=True) + stopped = basic.get_container("alpine", include_all=True) assert stopped.State == "exited" finally: basic.stop() def test_compose_logs(): - basic = DockerCompose(context=FIXTURES / 'basic') + basic = DockerCompose(context=FIXTURES / "basic") with basic: sleep(1) # generate some logs every 200ms stdout, stderr = basic.get_logs() @@ -90,7 +88,7 @@ def test_compose_logs(): assert not stderr assert stdout - lines = split(r'\r?\n', stdout) + lines = split(r"\r?\n", stdout) assert len(lines) > 5 # actually 10 for line in lines[1:]: @@ -100,14 +98,14 @@ def test_compose_logs(): # noinspection HttpUrlsUsage def test_compose_ports(): # fairly straight forward - can we get the right port to request it - single = DockerCompose(context=FIXTURES / 'port_single') + single = DockerCompose(context=FIXTURES / "port_single") with single: host, port = single.get_service_host_and_port() - endpoint = f'http://{host}:{port}' + endpoint = f"http://{host}:{port}" single.wait_for(endpoint) - code, response = fetch(Request(method='GET', url=endpoint)) + code, response = fetch(Request(method="GET", url=endpoint)) assert code == 200 - assert '

' in response + assert "

" in response # noinspection HttpUrlsUsage @@ -116,52 +114,57 @@ def test_compose_multiple_containers_and_ports(): assert correctness of multiple logic """ - multiple = DockerCompose(context=FIXTURES / 'port_multiple') + multiple = DockerCompose(context=FIXTURES / "port_multiple") with multiple: with pytest.raises(ContainerIsNotRunning) as e: multiple.get_container() - e.match('get_container failed') - e.match('not exactly 1 container') + e.match("get_container failed") + e.match("not exactly 1 container") - assert multiple.get_container('alpine') - assert multiple.get_container('alpine2') + assert multiple.get_container("alpine") + assert multiple.get_container("alpine2") - a2p = multiple.get_service_port('alpine2') + a2p = multiple.get_service_port("alpine2") assert a2p > 0 # > 1024 with pytest.raises(NoSuchPortExposed) as e: - multiple.get_service_port('alpine') - e.match('not exactly 1') + multiple.get_service_port("alpine") + e.match("not exactly 1") with pytest.raises(NoSuchPortExposed) as e: - multiple.get_container('alpine').get_publisher(by_host='example.com') - e.match('not exactly 1') + multiple.get_container("alpine").get_publisher(by_host="example.com") + e.match("not exactly 1") with pytest.raises(NoSuchPortExposed) as e: - multiple.get_container('alpine').get_publisher(by_host='localhost') - e.match('not exactly 1') + multiple.get_container("alpine").get_publisher(by_host="localhost") + e.match("not exactly 1") try: # this fails when ipv6 is enabled and docker is forwarding for both 4 + 6 - multiple.get_container(service_name='alpine') \ - .get_publisher(by_port=81, prefer_ip_version="IPv6") + multiple.get_container(service_name="alpine").get_publisher(by_port=81, prefer_ip_version="IPv6") except: # noqa pass ports = [ - (80, - multiple.get_service_host(service_name='alpine', port=80), - multiple.get_service_port(service_name='alpine', port=80)), - (81, - multiple.get_service_host(service_name='alpine', port=81), - multiple.get_service_port(service_name='alpine', port=81)), - (82, - multiple.get_service_host(service_name='alpine', port=82), - multiple.get_service_port(service_name='alpine', port=82)), + ( + 80, + multiple.get_service_host(service_name="alpine", port=80), + multiple.get_service_port(service_name="alpine", port=80), + ), + ( + 81, + multiple.get_service_host(service_name="alpine", port=81), + multiple.get_service_port(service_name="alpine", port=81), + ), + ( + 82, + multiple.get_service_host(service_name="alpine", port=82), + multiple.get_service_port(service_name="alpine", port=82), + ), ] # test correctness of port lookup for target, host, mapped in ports: - assert mapped, f'we have a mapped port for target port {target}' - code, body = fetch(Request(method='GET', url=f'http://{host}:{mapped}')) + assert mapped, f"we have a mapped port for target port {target}" + code, body = fetch(Request(method="GET", url=f"http://{host}:{mapped}")) if target == 81: assert code == 202 if target == 82: @@ -171,60 +174,58 @@ def test_compose_multiple_containers_and_ports(): # noinspection HttpUrlsUsage def test_exec_in_container(): """we test that we can manipulate a container via exec""" - single = DockerCompose(context=FIXTURES / 'port_single') + single = DockerCompose(context=FIXTURES / "port_single") with single: - url = f'http://{single.get_service_host()}:{single.get_service_port()}' + url = f"http://{single.get_service_host()}:{single.get_service_port()}" single.wait_for(url) # unchanged code, body = fetch(url) assert code == 200 - assert 'test_exec_in_container' not in body + assert "test_exec_in_container" not in body # change it - single.exec_in_container(command=[ - 'sh', '-c', - 'echo "test_exec_in_container" > /usr/share/nginx/html/index.html' - ]) + single.exec_in_container( + command=["sh", "-c", 'echo "test_exec_in_container" > /usr/share/nginx/html/index.html'] + ) # and it is changed code, body = fetch(url) assert code == 200 - assert 'test_exec_in_container' in body + assert "test_exec_in_container" in body # noinspection HttpUrlsUsage def test_exec_in_container_multiple(): """same as above, except we exec into a particular service""" - multiple = DockerCompose(context=FIXTURES / 'port_multiple') + multiple = DockerCompose(context=FIXTURES / "port_multiple") with multiple: - sn = 'alpine2' # service name + sn = "alpine2" # service name host, port = multiple.get_service_host_and_port(service_name=sn) - url = f'http://{host}:{port}' + url = f"http://{host}:{port}" multiple.wait_for(url) # unchanged code, body = fetch(url) assert code == 200 - assert 'test_exec_in_container' not in body + assert "test_exec_in_container" not in body # change it - multiple.exec_in_container(command=[ - 'sh', '-c', - 'echo "test_exec_in_container" > /usr/share/nginx/html/index.html' - ], service_name=sn) + multiple.exec_in_container( + command=["sh", "-c", 'echo "test_exec_in_container" > /usr/share/nginx/html/index.html'], service_name=sn + ) # and it is changed code, body = fetch(url) assert code == 200 - assert 'test_exec_in_container' in body + assert "test_exec_in_container" in body def fetch(req: Union[Request, str]): if isinstance(req, str): - req = Request(method='GET', url=req) + req = Request(method="GET", url=req) with urlopen(req) as res: - body = res.read().decode('utf-8') + body = res.read().decode("utf-8") if 200 < res.getcode() >= 400: raise Exception(f"HTTP Error: {res.getcode()} - {res.reason}: {body}") return res.getcode(), body From ef9df2816f80b55b51e714e7a4264c538dbc8051 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Tue, 27 Feb 2024 08:12:48 -0700 Subject: [PATCH 08/14] name the _matches_protocol function --- core/testcontainers/compose/compose.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/testcontainers/compose/compose.py b/core/testcontainers/compose/compose.py index 7e69ca860..3dcfe0f06 100644 --- a/core/testcontainers/compose/compose.py +++ b/core/testcontainers/compose/compose.py @@ -81,7 +81,7 @@ def get_publisher( remaining_publishers = [ r for r in remaining_publishers - if (":" in r.URL) is (prefer_ip_version == "IPv6") + if self._matches_protocol(prefer_ip_version, r) ] if by_port: @@ -106,6 +106,10 @@ def get_publisher( f" (but {len(remaining_publishers)})" ), ) + + @staticmethod + def _matches_protocol(prefer_ip_version, r): + return (":" in r.URL) is (prefer_ip_version == "IPv6") # fmt:on From a23445a2232d2ed6aad89952ecdfaa2538bd6358 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Mon, 4 Mar 2024 10:44:57 -0500 Subject: [PATCH 09/14] tried dind 24, found stuff to fix --- core/testcontainers/compose/compose.py | 15 ++++++++++++--- core/tests/test_compose.py | 6 +++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/core/testcontainers/compose/compose.py b/core/testcontainers/compose/compose.py index 3dcfe0f06..37da80757 100644 --- a/core/testcontainers/compose/compose.py +++ b/core/testcontainers/compose/compose.py @@ -275,9 +275,18 @@ def get_containers(self, include_all=False) -> list[ComposeContainer]: stdout = split(r"\r?\n", result.stdout.decode("utf-8")) # fmt:off - return [ - _ignore_properties(ComposeContainer, loads(line)) for line in stdout if line - ] + containers = [] + # one line per service in docker 25, single array for docker 24.0.2 + for line in stdout: + if not line: + continue + data = loads(line) + if isinstance(data, list): + containers += [_ignore_properties(ComposeContainer, d) for d in data] + else: + containers.append(_ignore_properties(ComposeContainer, data)) + + return containers # fmt:on # fmt:off diff --git a/core/tests/test_compose.py b/core/tests/test_compose.py index 1e157c429..548e22b51 100644 --- a/core/tests/test_compose.py +++ b/core/tests/test_compose.py @@ -92,7 +92,10 @@ def test_compose_logs(): assert len(lines) > 5 # actually 10 for line in lines[1:]: - assert not line or line.startswith(container.Service) + # either the line is blank or the first column (|-separated) contains the service name + # this is a safe way to split the string + # docker changes the prefix between versions 24 and 25 + assert not line or container.Service in next(iter(line.split("|")), None) # noinspection HttpUrlsUsage @@ -143,6 +146,7 @@ def test_compose_multiple_containers_and_ports(): except: # noqa pass + containers = multiple.get_containers(include_all=True) ports = [ ( 80, From 90a157310c08cf7941a823341a498b55d9ea9481 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Mon, 4 Mar 2024 10:48:12 -0500 Subject: [PATCH 10/14] rename function per feedback about name --- core/testcontainers/compose/compose.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/testcontainers/compose/compose.py b/core/testcontainers/compose/compose.py index 37da80757..a138dd492 100644 --- a/core/testcontainers/compose/compose.py +++ b/core/testcontainers/compose/compose.py @@ -41,7 +41,7 @@ class PublishedPort: OT = TypeVar("OT") -def one(array: list[OT], exception: Callable[[], Exception]) -> OT: +def get_only_element_or_raise(array: list[OT], exception: Callable[[], Exception]) -> OT: if len(array) != 1: e = exception() raise e @@ -97,7 +97,7 @@ def get_publisher( if len(remaining_publishers) == 0: raise NoSuchPortExposed( f"Could not find publisher for for service {self.Service}") - return one( + return get_only_element_or_raise( remaining_publishers, lambda: NoSuchPortExposed( "get_publisher failed because there is " @@ -297,7 +297,7 @@ def get_container( ) -> ComposeContainer: if not service_name: c = self.get_containers(include_all=include_all) - return one(c, lambda: ContainerIsNotRunning( + return get_only_element_or_raise(c, lambda: ContainerIsNotRunning( 'get_container failed because no service_name given ' f'and there is not exactly 1 container (but {len(c)})' )) From c640ec9a9db98c9f7787f65dfd5e137190893ded Mon Sep 17 00:00:00 2001 From: David Ankin Date: Mon, 4 Mar 2024 11:01:27 -0500 Subject: [PATCH 11/14] keep exploring multiple ports issue --- core/tests/test_compose.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/core/tests/test_compose.py b/core/tests/test_compose.py index 548e22b51..0e2241e79 100644 --- a/core/tests/test_compose.py +++ b/core/tests/test_compose.py @@ -168,11 +168,19 @@ def test_compose_multiple_containers_and_ports(): # test correctness of port lookup for target, host, mapped in ports: assert mapped, f"we have a mapped port for target port {target}" - code, body = fetch(Request(method="GET", url=f"http://{host}:{mapped}")) - if target == 81: - assert code == 202 - if target == 82: - assert code == 204 + url = f"http://{host}:{mapped}" + code, body = fetch(Request(method="GET", url=url)) + + expected_code = { + 81: 202, + 82: 204, + }.get(code, None) + + if not expected_code: + continue + + message = f"response '{body}' ({code}) from url {url} should have code {expected_code}" + assert code == expected_code, message # noinspection HttpUrlsUsage From 533319455df3e44cb937fa12e342f600ace4c20a Mon Sep 17 00:00:00 2001 From: David Ankin Date: Mon, 4 Mar 2024 11:05:23 -0500 Subject: [PATCH 12/14] keep exploring multiple ports issue.2 --- core/tests/test_compose.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/tests/test_compose.py b/core/tests/test_compose.py index 0e2241e79..4e4bdd489 100644 --- a/core/tests/test_compose.py +++ b/core/tests/test_compose.py @@ -146,7 +146,6 @@ def test_compose_multiple_containers_and_ports(): except: # noqa pass - containers = multiple.get_containers(include_all=True) ports = [ ( 80, From 14121d9da58b5d17a59e09735c7eb270fbd2c708 Mon Sep 17 00:00:00 2001 From: David Ankin Date: Mon, 4 Mar 2024 12:04:36 -0500 Subject: [PATCH 13/14] keep exploring multiple ports issue.3 --- core/tests/test_compose.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/tests/test_compose.py b/core/tests/test_compose.py index 4e4bdd489..0a244220b 100644 --- a/core/tests/test_compose.py +++ b/core/tests/test_compose.py @@ -171,6 +171,7 @@ def test_compose_multiple_containers_and_ports(): code, body = fetch(Request(method="GET", url=url)) expected_code = { + 80: 200, 81: 202, 82: 204, }.get(code, None) From cdbce5cd15edc8ba68ad1d9db59d97cc5e0dd4ac Mon Sep 17 00:00:00 2001 From: Dave Ankin Date: Tue, 5 Mar 2024 17:15:37 -0500 Subject: [PATCH 14/14] butcher formatting, rip pretty code --- core/testcontainers/compose/compose.py | 56 ++++++++------------------ 1 file changed, 17 insertions(+), 39 deletions(-) diff --git a/core/testcontainers/compose/compose.py b/core/testcontainers/compose/compose.py index a138dd492..e72824bd1 100644 --- a/core/testcontainers/compose/compose.py +++ b/core/testcontainers/compose/compose.py @@ -70,7 +70,6 @@ def __post_init__(self): if self.Publishers: self.Publishers = [_ignore_properties(PublishedPort, p) for p in self.Publishers] - # fmt:off def get_publisher( self, by_port: Optional[int] = None, @@ -79,24 +78,14 @@ def get_publisher( ) -> PublishedPort: remaining_publishers = self.Publishers - remaining_publishers = [ - r for r in remaining_publishers - if self._matches_protocol(prefer_ip_version, r) - ] + remaining_publishers = [r for r in remaining_publishers if self._matches_protocol(prefer_ip_version, r)] if by_port: - remaining_publishers = [ - item for item in remaining_publishers - if by_port == item.TargetPort - ] + remaining_publishers = [item for item in remaining_publishers if by_port == item.TargetPort] if by_host: - remaining_publishers = [ - item for item in remaining_publishers - if by_host == item.URL - ] + remaining_publishers = [item for item in remaining_publishers if by_host == item.URL] if len(remaining_publishers) == 0: - raise NoSuchPortExposed( - f"Could not find publisher for for service {self.Service}") + raise NoSuchPortExposed(f"Could not find publisher for for service {self.Service}") return get_only_element_or_raise( remaining_publishers, lambda: NoSuchPortExposed( @@ -110,7 +99,6 @@ def get_publisher( @staticmethod def _matches_protocol(prefer_ip_version, r): return (":" in r.URL) is (prefer_ip_version == "IPv6") - # fmt:on @dataclass @@ -274,7 +262,6 @@ def get_containers(self, include_all=False) -> list[ComposeContainer]: result = subprocess.run(cmd, cwd=self.context, check=True, stdout=subprocess.PIPE) stdout = split(r"\r?\n", result.stdout.decode("utf-8")) - # fmt:off containers = [] # one line per service in docker 25, single array for docker 24.0.2 for line in stdout: @@ -287,32 +274,30 @@ def get_containers(self, include_all=False) -> list[ComposeContainer]: containers.append(_ignore_properties(ComposeContainer, data)) return containers - # fmt:on - # fmt:off def get_container( self, service_name: Optional[str] = None, include_all: bool = False, ) -> ComposeContainer: if not service_name: - c = self.get_containers(include_all=include_all) - return get_only_element_or_raise(c, lambda: ContainerIsNotRunning( - 'get_container failed because no service_name given ' - f'and there is not exactly 1 container (but {len(c)})' - )) + containers = self.get_containers(include_all=include_all) + return get_only_element_or_raise( + containers, + lambda: ContainerIsNotRunning( + "get_container failed because no service_name given " + f"and there is not exactly 1 container (but {len(containers)})" + ), + ) matching_containers = [ - item for item in self.get_containers(include_all=include_all) - if item.Service == service_name + item for item in self.get_containers(include_all=include_all) if item.Service == service_name ] if not matching_containers: - raise ContainerIsNotRunning( - f"{service_name} is not running in the compose context") + raise ContainerIsNotRunning(f"{service_name} is not running in the compose context") return matching_containers[0] - # fmt:on def exec_in_container( self, @@ -344,13 +329,7 @@ def exec_in_container( check=True, ) - # fmt:off - return ( - result.stdout.decode("utf-8"), - result.stderr.decode("utf-8"), - result.returncode - ) - # fmt:on + return (result.stdout.decode("utf-8"), result.stderr.decode("utf-8"), result.returncode) def _call_command( self, @@ -422,7 +401,6 @@ def wait_for(self, url: str) -> "DockerCompose": Args: url: URL from one of the services in the environment to use to wait on. """ - # fmt:off - with urlopen(url) as r: r.read() # noqa: E701 - # fmt:on + with urlopen(url) as response: + response.read() return self