diff --git a/.gitignore b/.gitignore index 2dc9b69..3addb4e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,4 @@ *~ -.build.server -.build.opensuse-server -.build.ad-server -.build.opensuse-ad-server -.build.client -.build.opensuse-client -.build.toolbox -.build.opensuse-toolbox -.build.nightly.server -.build.nightly.ad-server +.build.* .common .bin diff --git a/hack/build-image b/hack/build-image index 34e8530..3f3145e 100755 --- a/hack/build-image +++ b/hack/build-image @@ -36,6 +36,8 @@ Usage: """ import argparse +import functools +import json import logging import os import pathlib @@ -102,9 +104,9 @@ DEVBUILDS = "devbuilds" PACKAGE_SOURCES = [DEFAULT, NIGHTLY, DEVBUILDS] PACKAGES_FROM = { - DEFAULT: '', - NIGHTLY: 'samba-nightly', - DEVBUILDS: 'devbuilds', + DEFAULT: "", + NIGHTLY: "samba-nightly", + DEVBUILDS: "devbuilds", } # SOURCE_DIRS - image source paths @@ -121,7 +123,13 @@ DEFAULT_DISTRO_BASES = [FEDORA] LATEST = "latest" QUAL_NONE = "unqualified" QUAL_DISTRO = "distro-qualified" -QUAL_FQIN = 'fqin' +QUAL_FQIN = "fqin" + +# conditions +EXISTS = "exists" +AUTO = "auto" +AUTO_INDEX = "auto-index" +REBUILD = "rebuild" _DISCOVERED_CONTAINER_ENGINES = [] @@ -207,45 +215,122 @@ def container_build(cli, target): tasks = [] # For docker cross-builds we need to use buildx - if "docker" in eng and target.arch != host_arch(): + if "docker" in eng and target.arch != host_arch(): args = [eng, "buildx"] # Docker's default builder only supports the host architecture. # Therefore, we need to create a new builder to support other # architectures, and we must ensure we start with a fresh builder # that does not contain any images from previous builds. - tasks.append(lambda : run(cli, args + ["rm", target.flat_name()], check=False)) - tasks.append(lambda : run(cli, args + ["create", f"--name={target.flat_name()}"], check=True)) + tasks.append( + {"cmd": args + ["rm", target.flat_name()], "check": False} + ) + tasks.append( + { + "cmd": args + ["create", f"--name={target.flat_name()}"], + "check": True, + } + ) - tasks.append(lambda : run(cli, args + [ - "build", - f"--builder={target.flat_name()}", - f"--platform=linux/{target.arch}", - "--load"] + create_common_container_engine_args(cli, target), check=True)) + tasks.append( + { + "cmd": args + + [ + "build", + f"--builder={target.flat_name()}", + f"--platform=linux/{target.arch}", + "--load", + ] + + create_common_container_engine_args(cli, target), + "check": True, + } + ) - tasks.append(lambda : run(cli, args + ["rm", target.flat_name()], check=True)) + tasks.append( + {"cmd": args + ["rm", target.flat_name()], "check": True} + ) else: args = [eng, "build"] if target.arch != host_arch() or FORCE_ARCH_FLAG: - # We've noticed a few small quirks when using podman with the --arch - # option. The main issue is that building the client image works - # but then the toolbox image fails because it somehow doesn't see - # the image we just built as usable. This doesn't happen when + # We've noticed a few small quirks when using podman with the + # --arch option. The main issue is that building the client image + # works but then the toolbox image fails because it somehow doesn't + # see the image we just built as usable. This doesn't happen when # --arch is not provided. So if the target arch and the host_arch # are the same, skip passing the extra argument. args += [f"--arch={target.arch}"] - tasks.append(lambda : run(cli, args + create_common_container_engine_args(cli, target), check=True)) + tasks.append( + { + "cmd": args + + create_common_container_engine_args(cli, target), + "check": True, + } + ) for task in tasks: - task() + run(cli, **task) + + +def index_build(cli, target): + """Construct a new index or manifest.""" + logger.debug("Building index: %s", target) + eng = container_engine(cli) + args = [eng, "manifest", "create", target.image_name()] + run(cli, args, check=True) + # add images to index + for img in target.contents: + add_args = [eng, "manifest", "add"] + # Currently annotations can't be used for much, but perhaps in the + # future podman/docker/etc will let you filter on annotations. Then + # you could choose base distro, etc using a single big index with + # annotations indicating various features. For now, it's mostly + # acedemic and for practice. + add_args.append( + "--annotation=org.samba.samba-container.pkg-source=" + f"{img.pkg_source}" + ) + add_args.append( + "--annotation=org.samba.samba-container.distro=" f"{img.distro}" + ) + add_args += [target.image_name(), img.image_name()] + run(cli, add_args, check=True) + # apply additional tag names + for tname in target.all_names(baseless=cli.without_repo_bases): + tag_args = [eng, "tag", target.image_name(), tname] + if target.image_name() != tname: + run(cli, tag_args, check=True) + # verfication step + inspect_args = [eng, "manifest", "inspect", target.image_name()] + res = run(cli, inspect_args, check=True, capture_output=True) + idx_info = json.loads(res.stdout) + if len(idx_info["manifests"]) != len(target.contents): + logger.error("unexpected index info: %r", idx_info) + logger.error( + "saw %d entries, expected %d (%r)", + len(idx_info["manifests"]), + len(target.contents), + target.contents, + ) + raise ValueError("unexpected number of manifest entries") + return target.image_name() + + +def image_build(cli, target): + if isinstance(target, TargetIndex): + logger.debug("target is an index (or manifest)") + return index_build(cli, target) + else: + logger.debug("target is a container image") + return container_build(cli, target) + def create_common_container_engine_args(cli, target): args = [] pkgs_from = PACKAGES_FROM[target.pkg_source] if pkgs_from: args.append(f"--build-arg=INSTALL_PACKAGES_FROM={pkgs_from}") - + if cli.extra_build_arg: args.extend(cli.extra_build_arg) @@ -258,9 +343,19 @@ def create_common_container_engine_args(cli, target): args.append(kind_source_dir(target.name)) return [str(a) for a in args] -def container_push(cli, push_name): + +def container_push(cli, push_name, manifest=False): """Construct and execute a command to push a container image.""" - args = [container_engine(cli), "push", push_name] + eng = container_engine(cli) + if manifest: + args = [eng, "manifest", "push", "--all"] + else: + args = [eng, "push"] + if "podman" in eng: + # very strange bug? sometimes it fails to push without the format + # option. but I swear it also does work without it some times + args.append("--format=oci") + args.append(push_name) run(cli, args, check=True) @@ -268,6 +363,15 @@ def container_id(cli, target): """Construct and run a command to fetch a hexidecimal id for a container image. """ + if isinstance(target, TargetIndex): + args = [ + container_engine(cli), + "manifest", + "exists", + target.image_name(), + ] + run(cli, args, check=True) + return target.image_name() args = [ container_engine(cli), "inspect", @@ -279,6 +383,16 @@ def container_id(cli, target): return res.stdout.decode("utf8").strip() +def maybe_container_id(cli, target): + """Same as container_id but returns None if no match for the target was + found. + """ + try: + return container_id(cli, target) + except subprocess.CalledProcessError: + return None + + def container_tag(cli, target, tag, *tags): """Add additional tags to the existing target image.""" if isinstance(target, str): @@ -306,7 +420,9 @@ def kind_source_dir(kind): def target_containerfile(target): """Return the path to a containerfile given an image target.""" - return str(kind_source_dir(target.name) / f"Containerfile.{target.distro}") + return str( + kind_source_dir(target.name) / f"Containerfile.{target.distro}" + ) def host_arch(): @@ -328,24 +444,47 @@ class RepoConfig: return self.distro_map.get(distro, self.default) -class TargetImage: +def _split_img(image_name, max_tag_split=3): + if "/" in image_name: + base, rest = image_name.rsplit("/", 1) + else: + base = "" + rest = image_name + iname, tag = rest.split(":", 1) + tparts = tag.split("-", max_tag_split) + return base, iname, tparts + + +class Target: def __init__( - self, name, pkg_source, distro, arch, extra_tag="", *, repo_base="" + self, name, *, pkg_source, distro, extra_tag="", repo_base="" ): self.name = name self.pkg_source = pkg_source self.distro = distro - self.arch = arch self.extra_tag = extra_tag self.repo_base = repo_base self.additional_tags = [] + def all_names(self, baseless=False): + yield self.image_name() + for tag, _ in self.additional_tags: + yield self.image_name(tag=tag) + if self.repo_base and baseless: + yield self.image_name(repo_base="") + for tag, qual in self.additional_tags: + if qual == QUAL_NONE: + continue + yield self.image_name(tag=tag, repo_base="") + + def supports_arch(self, arch): + return False + + def flat_name(self): + return f"{self.name}.{self.tag_name()}" + def tag_name(self): - tag_parts = [self.pkg_source, self.distro, self.arch] - if self.extra_tag: - tag_parts.append(self.extra_tag) - tag = "-".join(tag_parts) - return tag + raise NotImplementedError() def image_name(self, *, tag=None, repo_base=None): if not tag: @@ -357,32 +496,36 @@ class TargetImage: image_name = f"{repo_base}/{image_name}" return image_name - def flat_name(self): - return f"{self.name}.{self.tag_name()}" - def __str__(self): return self.image_name() - def all_names(self, baseless=False): - yield self.image_name() - for tag, _ in self.additional_tags: - yield self.image_name(tag=tag) - if self.repo_base and baseless: - yield self.image_name(repo_base="") - for tag, qual in self.additional_tags: - if qual == QUAL_NONE: - continue - yield self.image_name(tag=tag, repo_base="") + +class TargetImage(Target): + def __init__( + self, name, pkg_source, distro, arch, extra_tag="", *, repo_base="" + ): + super().__init__( + name, + pkg_source=pkg_source, + distro=distro, + extra_tag=extra_tag, + repo_base=repo_base, + ) + self.arch = arch + + def tag_name(self): + tag_parts = [self.pkg_source, self.distro, self.arch] + if self.extra_tag: + tag_parts.append(self.extra_tag) + tag = "-".join(tag_parts) + return tag + + def supports_arch(self, arch): + return arch == self.arch @classmethod def parse(cls, image_name): - if "/" in image_name: - base, rest = image_name.rsplit("/", 1) - else: - base = "" - rest = image_name - iname, tag = rest.split(":", 1) - tparts = tag.split("-", 3) + base, iname, tparts = _split_img(image_name) if len(tparts) < 3: raise ValueError(f"too few tag components: {tag!r}") return cls( @@ -395,9 +538,108 @@ class TargetImage: ) -def generate_images(cli): - """Given full image names or a matrix of kind/pkg_source/distro_base/arch - values generate a list of target images to build/process. +class TargetIndex(Target): + def __init__( + self, + name, + *, + pkg_source, + distro, + contents=None, + extra_tag="", + repo_base="", + ): + super().__init__( + name, + pkg_source=pkg_source, + distro=distro, + extra_tag=extra_tag, + repo_base=repo_base, + ) + self.contents = contents or [] + + def key(self): + return (self.name, self.pkg_source, self.distro) + + def tag_name(self): + tag_parts = [self.pkg_source, self.distro] + if self.extra_tag: + tag_parts.append(self.extra_tag) + tag = "-".join(tag_parts) + return tag + + def merge(self, other): + assert self.name == other.name + assert self.pkg_source == other.pkg_source + assert self.distro == other.distro + self.contents.extend(other.contents) + + def supports_arch(self, arch): + return True + + @classmethod + def from_image(cls, img): + return cls( + img.name, + pkg_source=img.pkg_source, + distro=img.distro, + contents=[img], + repo_base=img.repo_base or "", + ) + + +class BuildRequest: + def __init__(self, images=None, indexes=None): + self.images = list(images or []) + self.indexes = list(indexes or []) + + def __bool__(self): + return bool(self.images or self.indexes) + + def expanded( + self, indexes=False, distro_qualified=True, unqualified=True + ): + new_req = self.__class__(self.images, self.indexes) + if indexes: + new_req._build_indexes() + new_req._expand_special_tags( + distro_qualified=distro_qualified, unqualified=unqualified + ) + return new_req + + def _expand_special_tags(self, distro_qualified=True, unqualified=True): + if self.indexes: + for image in self.indexes: + # distro qualified is redundant with the default tag of an + # index/manifest as well as mainly needed for backwards + # compatibility something we don't want for indexes. + add_special_tags( + image, distro_qualified=False, unqualified=unqualified + ) + else: + for image in self.images: + add_special_tags( + image, + distro_qualified=distro_qualified, + unqualified=unqualified, + ) + + def _build_indexes(self): + _indexes = {} + for image in self.images: + image_index = TargetIndex.from_image(image) + key = image_index.key() + if key in _indexes: + _indexes[key].merge(image_index) + else: + _indexes[key] = image_index + self.indexes = list(_indexes.values()) + + +def generate_request(cli): + """Given command line parameters with full image names or a matrix of + kind/pkg_source/distro_base/arch values generate request object containing + the target images or indexes to build and/or otherwise process. """ images = {} for img in cli.image or []: @@ -416,10 +658,14 @@ def generate_images(cli): repo_base=rc.find_base(distro_base), ) images[str(timg)] = timg - return list(images.values()) + return BuildRequest(images=images.values()).expanded( + indexes=cli.combined, + distro_qualified=cli.distro_qualified, + unqualified=cli.unqualified, + ) -def add_special_tags(img, distro_qualified=True): +def add_special_tags(img, distro_qualified=True, unqualified=True): """Certain images have special tags. Given an image, add general (non-FQIN) tags to that image. """ @@ -427,32 +673,41 @@ def add_special_tags(img, distro_qualified=True): # that certain images deserve some extra special tags. Mostly this serves # to keep us compatible with older tagging schemes from earlier versions of # the project. - if img.distro in [FEDORA, OPENSUSE]: - if img.arch == host_arch() and img.pkg_source == DEFAULT: + _host_arch = host_arch() + arch_ok = img.supports_arch(_host_arch) + if unqualified and img.distro in [FEDORA, OPENSUSE]: + if arch_ok and img.pkg_source == DEFAULT: img.additional_tags.append((LATEST, QUAL_NONE)) - if img.arch == host_arch() and img.pkg_source == NIGHTLY: + if arch_ok and img.pkg_source == NIGHTLY: img.additional_tags.append((NIGHTLY, QUAL_NONE)) if not distro_qualified: return # skip creating "distro qualified" tags - if img.arch == host_arch() and img.pkg_source == "default": + if arch_ok and img.pkg_source == DEFAULT: img.additional_tags.append((f"{img.distro}-{LATEST}", QUAL_DISTRO)) - if img.arch == host_arch() and img.pkg_source == "nightly": + if arch_ok and img.pkg_source == NIGHTLY: img.additional_tags.append((f"{img.distro}-{NIGHTLY}", QUAL_DISTRO)) -def build(cli, target): +def build(cli, target, rebuild=False): """Command to build images.""" build_file = pathlib.Path(f"{cli.buildfile_prefix}{target.flat_name()}") common_src = "./images/common" common_dst = str(kind_source_dir(target.name) / ".common") - logger.debug("Copying common tree: %r -> %r", common_src, common_dst) - shutil.copytree(common_src, common_dst, dirs_exist_ok=True) - container_build(cli, target) - cid = container_id(cli, target) + cid = maybe_container_id(cli, target) + logger.debug("target: %s, cid=%s, rebuild=%s", target, cid, rebuild) + if not cid or rebuild: + logger.debug("Copying common tree: %r -> %r", common_src, common_dst) + shutil.copytree(common_src, common_dst, dirs_exist_ok=True) + image_build(cli, target) + cid = container_id(cli, target) with open(build_file, "w") as fh: fh.write(f"{cid} {target.image_name()}\n") +def rebuild(cli, target): + return build(cli, target, rebuild=True) + + class QMatcher: """Push only tags that meet the specified criteria: all - all tags; @@ -500,15 +755,25 @@ class QMatcher: return False +def _autobuild(cli, target, is_index=None): + if is_index is None: + is_index = isinstance(target, TargetIndex) + if cli.condition == REBUILD: + rebuild(cli, target) + elif maybe_container_id(cli, target): + logger.debug("target item exists: %s", target) + elif cli.condition == AUTO or (cli.condition == AUTO_INDEX and is_index): + logger.debug("target item auto build: %s", target) + build(cli, target) + else: + log.error("no existing image or index: %s", target) + raise ValueError("not present", target) + + def push(cli, target): """Command to push images.""" - if cli.push_state == "rebuild": - build(cli, target) - if cli.push_state == "exists": - try: - container_id(cli, target) - except subprocess.CalledProcessError: - build(cli, target) + is_index = isinstance(target, TargetIndex) # is it a manifest push? + _autobuild(cli, target, is_index=is_index) to_push = [] push_name = target.image_name() @@ -521,7 +786,32 @@ def push(cli, target): qmatcher = cli.push_selected_tags or QMatcher("") for push_name, tag_qual in to_push: if qmatcher(tag_qual): - container_push(cli, push_name) + logger.debug( + "pushing named object: %s (manifest=%s)", push_name, is_index + ) + container_push(cli, push_name, manifest=is_index) + + +def archive(cli, target, location): + """Write tarballs to archive location.""" + if isinstance(target, TargetIndex): + return # ignore indexes/manifests + _autobuild(cli, target) + + eng = container_engine(cli) + fname = pathlib.Path(location) / f"{target.flat_name()}.tar" + args = [eng, "save", f"-o{fname}", target.image_name()] + run(cli, args, check=True) + + +def load_archived(cli, target, location): + """Load tarballs from archive location.""" + fname = pathlib.Path(location) / f"{target.flat_name()}.tar" + logger.info("Loading from: %s", fname) + + eng = container_engine(cli) + args = [eng, "load", f"-i{fname}"] + run(cli, args, check=True) def retag(cli, target): @@ -562,6 +852,16 @@ def print_tags(cli, target): print(f"{prefix}{name}") +def _kwbind(func, key, conversion=None): + """Attach an argument value to a command-linked argument.""" + + def _capture(arg_value): + value = conversion(arg_value) if conversion else arg_value + return functools.partial(func, **{key: value}) + + return _capture + + def main(): parser = argparse.ArgumentParser() parser.add_argument( @@ -649,17 +949,22 @@ def main(): ) parser.add_argument( "--push-state", - choices=("exists", "rebuild"), - default="exists", + "--image-condition", + dest="condition", + choices=(EXISTS, AUTO, AUTO_INDEX, REBUILD), + default=EXISTS, help=( - "Only push if a state is met:" - "exists - image exists; rebuild - image must be rebuilt." + "Image state must be met before continuing:" + " exists - item already exists;" + " auto - automatically build missing image;" + " auto-index - automatically build missing indexes only;" + " rebuild - image must be rebuilt." ), ) parser.add_argument( "--push-selected-tags", type=QMatcher, - help=QMatcher.__doc__ + help=QMatcher.__doc__, ) parser.add_argument( "--buildfile-prefix", @@ -686,10 +991,25 @@ def main(): action=argparse.BooleanOptionalAction, default=True, help=( - "Specify if image tags like fedora-nightly or centos-latest" + "Specify if image tags like 'fedora-nightly' or 'centos-latest'" " will be created." ), ) + parser.add_argument( + "--unqualified", + action=argparse.BooleanOptionalAction, + default=True, + help=( + "Specify if image tags like 'nightly' or 'latest' will be created." + ), + ) + parser.add_argument( + "--combined", + action=argparse.BooleanOptionalAction, + default=False, + help=("Specify if manifests/image indexes should be created."), + ) + behaviors = parser.add_mutually_exclusive_group() behaviors.add_argument( "--push", @@ -724,8 +1044,31 @@ def main(): action="store_const", dest="main_action", const=retag, - help=("Regenerate any short (unqualified) tags expected to exist" - " for a given FQIN. Requires FQIN to already exist locally."), + help=( + "Regenerate any short (unqualified) tags expected to exist" + " for a given FQIN. Requires FQIN to already exist locally." + ), + ) + behaviors.add_argument( + "--rebuild", + action="store_const", + dest="main_action", + const=rebuild, + help="Always rebuild images even if matching images exist locally.", + ) + behaviors.add_argument( + "--archive", + metavar="LOCATION", + dest="main_action", + type=_kwbind(archive, "location"), + help="Write archive files (tarballs) to a specified location", + ) + behaviors.add_argument( + "--load", + metavar="LOCATION", + dest="main_action", + type=_kwbind(load_archived, "location"), + help="Read in archive files (tarballs) from a specified location", ) cli = parser.parse_args() @@ -734,17 +1077,23 @@ def main(): logging.basicConfig(level=cli.log_level) _action = cli.main_action if cli.main_action else build - imgs = [] + req = None try: - imgs = generate_images(cli) - for img in imgs: - add_special_tags(img, cli.distro_qualified) + req = generate_request(cli) + for img in req.images: logger.info("Image %s, extra tags: %s", img, img.additional_tags) _action(cli, img) + for index in req.indexes: + logger.info( + "Index (Manifest) %s, extra tags: %s", + index, + index.additional_tags, + ) + _action(cli, index) except subprocess.CalledProcessError as err: logger.error("Failed command: %s", _cmd_to_str(err.cmd)) sys.exit(err.returncode) - if not imgs: + if not req: logger.error("No images or image kinds supplied") sys.exit(2)