diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 515d699f9..eaf404754 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ { - "image": "mcr.microsoft.com/dotnet/sdk:9.0-noble", + "image": "mcr.microsoft.com/dotnet/sdk:10.0-noble", "features": { "ghcr.io/devcontainers/features/common-utils": { "username": "app", diff --git a/.gitignore b/.gitignore index 2dfc1016a..0b71563a0 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ # Test files *.trx + +# BenchmarkDotNet Artifacts +BenchmarkDotNet.Artifacts/ diff --git a/Microsoft.DotNet.DockerTools.slnx b/Microsoft.DotNet.DockerTools.slnx index 13ca555a4..753147abe 100644 --- a/Microsoft.DotNet.DockerTools.slnx +++ b/Microsoft.DotNet.DockerTools.slnx @@ -1,11 +1,20 @@ + + + + + + + + + diff --git a/eng/common/Install-DotNetSdk.ps1 b/eng/common/Install-DotNetSdk.ps1 index 114faa260..ce2696f81 100644 --- a/eng/common/Install-DotNetSdk.ps1 +++ b/eng/common/Install-DotNetSdk.ps1 @@ -20,7 +20,7 @@ param( [string] $InstallPath, [string] - $Channel = "9.0" + $Channel = "10.0" ) Set-StrictMode -Version Latest diff --git a/eng/common/templates/jobs/cg-build-projects.yml b/eng/common/templates/jobs/cg-build-projects.yml index de372c4c4..712804989 100644 --- a/eng/common/templates/jobs/cg-build-projects.yml +++ b/eng/common/templates/jobs/cg-build-projects.yml @@ -10,7 +10,7 @@ parameters: # See https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script#options for possible Channel values - name: dotnetVersionChannel type: string - default: '9.0' + default: '10.0' displayName: .NET Version jobs: diff --git a/src/Dockerfile.linux b/src/Dockerfile.linux index 85135e072..1f2f08485 100644 --- a/src/Dockerfile.linux +++ b/src/Dockerfile.linux @@ -3,7 +3,7 @@ # docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v :/repo -w /repo image-builder # build Microsoft.DotNet.ImageBuilder -FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:9.0-azurelinux3.0 AS build-env +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:10.0-azurelinux3.0 AS build-env ARG TARGETARCH # download oras package tarball @@ -19,6 +19,7 @@ WORKDIR /image-builder # restore packages before copying entire source - provides optimizations when rebuilding COPY NuGet.config ./ COPY ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj ./ImageBuilder/ +COPY ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj ./ImageBuilder.Models/ RUN dotnet restore -r linux-$TARGETARCH ./ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj # copy everything else and publish @@ -27,7 +28,7 @@ RUN dotnet publish -r linux-$TARGETARCH ./ImageBuilder/Microsoft.DotNet.ImageBui # build runtime image -FROM mcr.microsoft.com/dotnet/runtime-deps:9.0-azurelinux3.0 +FROM mcr.microsoft.com/dotnet/runtime-deps:10.0-azurelinux3.0 # install tooling RUN tdnf install -y \ diff --git a/src/Dockerfile.windows b/src/Dockerfile.windows index 4ef231a5f..d3b8f0746 100644 --- a/src/Dockerfile.windows +++ b/src/Dockerfile.windows @@ -4,12 +4,13 @@ ARG WINDOWS_BASE ARG WINDOWS_SDK # build Microsoft.DotNet.ImageBuilder -FROM mcr.microsoft.com/dotnet/sdk:9.0-$WINDOWS_SDK AS build-env +FROM mcr.microsoft.com/dotnet/sdk:10.0-$WINDOWS_SDK AS build-env WORKDIR /image-builder # restore packages before copying entire source - provides optimizations when rebuilding COPY NuGet.config ./ COPY ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj ./ImageBuilder/ +COPY ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj ./ImageBuilder.Models/ RUN dotnet restore -r win-x64 ./ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj # copy everything else and publish diff --git a/src/ImageBuilder.Models/Manifest/Architecture.cs b/src/ImageBuilder.Models/Manifest/Architecture.cs new file mode 100644 index 000000000..b39ed37f4 --- /dev/null +++ b/src/ImageBuilder.Models/Manifest/Architecture.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +// Enum values must align with the $GOARCH values specified at https://golang.org/doc/install/source#environment +public enum Architecture +{ + ARM, + ARM64, + AMD64, +} diff --git a/src/ImageBuilder.Models/Manifest/CustomBuildLegDependencyType.cs b/src/ImageBuilder.Models/Manifest/CustomBuildLegDependencyType.cs new file mode 100644 index 000000000..b1027eab0 --- /dev/null +++ b/src/ImageBuilder.Models/Manifest/CustomBuildLegDependencyType.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.ComponentModel; + +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +[Description( + "The type of dependency an image has for a specific scenario." + )] +public enum CustomBuildLegDependencyType +{ + [Description( + "Indicates the dependency is considered to be integral to the depending image." + + "This means the dependent image will not have its own dependency graph considered for build leg " + + "generation. An example of this is when a custom build leg dependency is defined from sdk to " + + "aspnet; in that case, aspnet and sdk will be included in a leg together but the sdk will not " + + "have its own leg generated." + )] + Integral, + + [Description( + "Indicates the dependency is considered to be a supplemental companion to the depending image." + + "This means the dependent image will have its own dependency graph considered for build leg " + + "generation. An example of this is when a custom build leg dependency is defined to " + + "include an SDK image supported on a particular architecture in order to test a runtime OS " + + "that doesn't its own SDK on that architecture (Trixie ARM SDK to test Alpine ARM runtime); " + + "in that case, the SDK will be included in a leg together with the runtime and the SDK will " + + "still have have its own leg." + )] + Supplemental +} diff --git a/src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs b/src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs new file mode 100644 index 000000000..c0aaf3abb --- /dev/null +++ b/src/ImageBuilder.Models/Manifest/CustomBuildLegGroup.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.ComponentModel; +using Newtonsoft.Json; + +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +[Description( + "This object describes the tag dependencies of the image for a specific named scenario. This is " + + "for advanced cases only. It allows tooling to modify the build matrix that would normally be " + + "generated for the image by including the customizations described in this metadata. An example " + + "usage of this is in PR builds where it is necessary to build and test in the same job. In such " + + "a scenario, some images are part of a test matrix that require images to be available on the " + + "build machine that aren't part of that images dependency graph in normal scenarios. By " + + "specifying a customBuildLegGroup for this scenario, those additional image dependencies can " + + "be specified and the build pipeline can make use of them when constructing its build graph when " + + "specified to do so." + )] +public class CustomBuildLegGroup +{ + [Description( + "Name of the group describing the scenario in which it's relevant. This is just a " + + " custom label that can then be used by tooling to lookup the group when necessary." + )] + [JsonProperty(Required = Required.Always)] + public required string Name { get; set; } + + [Description("The type of the dependency which impacts how it's used during the build.")] + [JsonProperty(Required = Required.Always)] + public required CustomBuildLegDependencyType Type { get; set; } + + [Description("The set of dependencies the image has for this scenario.")] + [JsonProperty(Required = Required.Always)] + public required string[] Dependencies { get; set; } +} diff --git a/src/ImageBuilder.Models/Manifest/Image.cs b/src/ImageBuilder.Models/Manifest/Image.cs new file mode 100644 index 000000000..1ed3eee3c --- /dev/null +++ b/src/ImageBuilder.Models/Manifest/Image.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.ComponentModel; +using Newtonsoft.Json; + +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +[Description("An image object contains metadata about a specific Docker image.")] +public class Image +{ + [Description( + "The set of platforms that describe the platform-specific variations of the Docker image.")] + [JsonProperty(Required = Required.Always)] + public Platform[] Platforms { get; set; } = []; + + [Description( + "The set of tags that are shared amongst all platform-specific versions of the image. An " + + "example of a shared tag, including its repo name, is dotnet/core/runtime:2.2; running " + + "`docker pull mcr.microsoft.com/dotnet/core/runtime:2.2` on Windows will get the " + + "default Windows-based tag whereas running it on Linux will get the default " + + "Linux-based tag.")] + public IDictionary? SharedTags { get; set; } + + [Description("The full version of the product that the Docker image contains.")] + public string? ProductVersion { get; set; } + + public Image() + { + } +} diff --git a/src/ImageBuilder.Models/Manifest/Manifest.cs b/src/ImageBuilder.Models/Manifest/Manifest.cs new file mode 100644 index 000000000..2262b8c65 --- /dev/null +++ b/src/ImageBuilder.Models/Manifest/Manifest.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.ComponentModel; + +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +[Description( + "The manifest file is the primary source of metadata that drives the production " + + "of all .NET Docker images. It describes various attributes of the Docker images " + + "that are to be produced by a given GitHub repo. .NET Docker's engineering system " + + "consumes this file in various ways as part of the automated build pipelines and " + + "other tools. It's intended to be product-agnostic meaning that it could be used " + + "to describe metadata for Docker image production of any product, not just .NET.")] +public class Manifest +{ + [Description( + "Additional json files to be loaded with this manifest. This is a convienent" + + "way to split the manifest apart into logical parts." + )] + public string[] Includes { get; set; } = []; + + [Description( + "Info about the readme that documents the product family." + )] + public Readme? Readme { get; set; } + + [Description( + "The location of the Docker registry where the images are to be published." + )] + public string? Registry { get; set; } + + [Description( + "The set of Docker repositories described by this manifest." + )] + public Repo[] Repos { get; set; } = []; + + [Description( + "A set of custom variables that can be referenced in various parts of the " + + "manifest. This provides a few benefits: 1) allows a commmonly used value " + + "to be defined only once and referenced by its variable name many times" + + "2) allows tools that consume the manifest file to provide a mechanism to " + + "dynamically override the value of these variables. Variables may be " + + "referenced in other parts of the manifest by using the following syntax: " + + "$(_VariableName_).")] + public IDictionary Variables { get; set; } = new Dictionary(); + + public Manifest() + { + } +} diff --git a/src/ImageBuilder/Models/Manifest/OS.cs b/src/ImageBuilder.Models/Manifest/OS.cs similarity index 63% rename from src/ImageBuilder/Models/Manifest/OS.cs rename to src/ImageBuilder.Models/Manifest/OS.cs index ca0984a66..8690bfaf8 100644 --- a/src/ImageBuilder/Models/Manifest/OS.cs +++ b/src/ImageBuilder.Models/Manifest/OS.cs @@ -2,11 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +public enum OS { - public enum OS - { - Linux, - Windows, - } + Linux, + Windows, } diff --git a/src/ImageBuilder.Models/Manifest/Platform.cs b/src/ImageBuilder.Models/Manifest/Platform.cs new file mode 100644 index 000000000..14fd9cc9d --- /dev/null +++ b/src/ImageBuilder.Models/Manifest/Platform.cs @@ -0,0 +1,76 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.ComponentModel; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +[Description( + "A platform object contains metadata about a platform-specific version of an " + + "image and refers to the actual Dockerfile used to build the image.")] +public class Platform +{ + [Description( + "The processor architecture associated with the image." + )] + [DefaultValue(Architecture.AMD64)] + [JsonConverter(typeof(StringEnumConverter))] + [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] + public Architecture Architecture { get; set; } = Architecture.AMD64; + + [Description( + "A set of values that will passed to the `docker build` command " + + "to override variables defined in the Dockerfile.")] + public IDictionary BuildArgs { get; set; } = new Dictionary(); + + [Description( + "Relative path to the associated Dockerfile. This can be a file or a " + + "directory. If it is a directory, the file name defaults to Dockerfile." + )] + [JsonProperty(Required = Required.Always)] + public string Dockerfile { get; set; } = string.Empty; + + [Description( + "Relative path to the template the Dockerfile is generated from." + )] + public string? DockerfileTemplate { get; set; } + + [Description( + "The generic name of the operating system associated with the image." + )] + [JsonConverter(typeof(StringEnumConverter))] + [JsonProperty(Required = Required.Always)] + public OS OS { get; set; } + + [Description( + "The specific version of the operating system associated with the image. " + + "Examples: alpine3.9, bionic, nanoserver-1903." + )] + [JsonProperty(Required = Required.Always)] + public string OsVersion { get; set; } = string.Empty; + + [Description( + "The set of platform-specific tags associated with the image." + )] + [JsonProperty(Required = Required.Always)] + public IDictionary Tags { get; set; } = new Dictionary(); + + [Description( + "The custom build leg groups associated with the platform." + )] + public CustomBuildLegGroup[] CustomBuildLegGroups { get; set; } = Array.Empty(); + + [Description( + "A label which further distinguishes the architecture when it " + + "contains variants. For example, the ARM architecture has variants " + + "named v6, v7, etc." + )] + public string? Variant { get; set; } + + public Platform() + { + } +} diff --git a/src/ImageBuilder/Models/Manifest/Readme.cs b/src/ImageBuilder.Models/Manifest/Readme.cs similarity index 96% rename from src/ImageBuilder/Models/Manifest/Readme.cs rename to src/ImageBuilder.Models/Manifest/Readme.cs index e3e4f1b4b..5b12a3251 100644 --- a/src/ImageBuilder/Models/Manifest/Readme.cs +++ b/src/ImageBuilder.Models/Manifest/Readme.cs @@ -6,7 +6,6 @@ namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; -#nullable enable public class Readme { [Description( @@ -32,4 +31,3 @@ public Readme(string path, string? templatePath) TemplatePath = templatePath; } } -#nullable disable diff --git a/src/ImageBuilder.Models/Manifest/Repo.cs b/src/ImageBuilder.Models/Manifest/Repo.cs new file mode 100644 index 000000000..879650473 --- /dev/null +++ b/src/ImageBuilder.Models/Manifest/Repo.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.ComponentModel; +using Newtonsoft.Json; + +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +[Description( + "A repository object contains metadata about a target Docker repository " + + "and the images to be contained in it." + )] +public class Repo +{ + [Description( + "A unique identifier of the repo. This is purely within the context " + + "of the manifest and not exposed to Docker in any way." + )] + public string? Id { get; set; } + + [Description( + "The set of images contained in this repository." + )] + [JsonProperty(Required = Required.Always)] + public Image[] Images { get; set; } = []; + + [Description( + "Relative path to the MCR tags template YAML file that is used by " + + "tooling to generate the tags section of the readme file." + )] + public string? McrTagsMetadataTemplate { get; set; } + + [Description( + "The name of the Docker repository where the described images are to " + + "be published (example: dotnet/core/runtime)." + )] + [JsonProperty(Required = Required.Always)] + public required string Name { get; set; } + + [Description( + "Info about the readme that documents the repo." + )] + public Readme[] Readmes { get; set; } = []; + + public Repo() + { + } +} diff --git a/src/ImageBuilder.Models/Manifest/Tag.cs b/src/ImageBuilder.Models/Manifest/Tag.cs new file mode 100644 index 000000000..a241b3b05 --- /dev/null +++ b/src/ImageBuilder.Models/Manifest/Tag.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.ComponentModel; + +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +[Description( + "A tag object contains metadata about a Docker tag. It is a JSON object " + + "with its tag name used as the attribute name." + )] +public class Tag +{ + [Description( + "An identifier used to conceptually group related tags in the readme " + + "documentation." + )] + public string? DocumentationGroup { get; set; } + + [Description( + "Indicates how this tag should not be documented in the readme file. Regardless of the " + + "setting, the image will still be tagged with this tag and will still be published. " + + "This is useful when deprecating a tag that still needs to be kept up-to-date " + + "but not wanting it documented." + )] + [DefaultValue(TagDocumentationType.Documented)] + public TagDocumentationType DocType { get; set; } = TagDocumentationType.Documented; + + [Description( + "Description of where the tag should be syndicated to.")] + public TagSyndication? Syndication { get; set; } + + public Tag() + { + } +} diff --git a/src/ImageBuilder.Models/Manifest/TagDocumentationType.cs b/src/ImageBuilder.Models/Manifest/TagDocumentationType.cs new file mode 100644 index 000000000..e15f19466 --- /dev/null +++ b/src/ImageBuilder.Models/Manifest/TagDocumentationType.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +public enum TagDocumentationType +{ + /// + /// The tag is always documented. + /// + Documented, + + /// + /// The tag is never documented. + /// + Undocumented, + + /// + /// The tag is only documented if there are corresponding platform tags that are documented. + /// + PlatformDocumented +} diff --git a/src/ImageBuilder.Models/Manifest/TagSyndication.cs b/src/ImageBuilder.Models/Manifest/TagSyndication.cs new file mode 100644 index 000000000..e2c4b4a68 --- /dev/null +++ b/src/ImageBuilder.Models/Manifest/TagSyndication.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// + +using System.ComponentModel; + +namespace Microsoft.DotNet.ImageBuilder.Models.Manifest; + +[Description( + "A description of where a tag should be syndicated to." + )] +public class TagSyndication +{ + [Description( + "Name of the repo to syndicate the tag to." + )] + public required string Repo { get; set; } + + [Description( + "List of destination tag names to syndicate the tag to." + )] + public string[] DestinationTags { get; set; } = []; +} diff --git a/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj b/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj new file mode 100644 index 000000000..c19d9f995 --- /dev/null +++ b/src/ImageBuilder.Models/Microsoft.DotNet.ImageBuilder.Models.csproj @@ -0,0 +1,15 @@ + + + + net10.0 + enable + enable + Microsoft.DotNet.ImageBuilder + true + + + + + + + diff --git a/src/ImageBuilder.Models/ReadModel/EmptyVariableStore.cs b/src/ImageBuilder.Models/ReadModel/EmptyVariableStore.cs new file mode 100644 index 000000000..c8932683b --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/EmptyVariableStore.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +internal sealed class EmptyVariableStore : IVariableStore +{ + public string ResolveInnerVariables(string expression) => expression; +} diff --git a/src/ImageBuilder.Models/ReadModel/IVariableStore.cs b/src/ImageBuilder.Models/ReadModel/IVariableStore.cs new file mode 100644 index 000000000..307c8b509 --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/IVariableStore.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +internal interface IVariableStore +{ + string ResolveInnerVariables(string expression); +} diff --git a/src/ImageBuilder.Models/ReadModel/JsonNodeExtensions.cs b/src/ImageBuilder.Models/ReadModel/JsonNodeExtensions.cs new file mode 100644 index 000000000..1d2fe388f --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/JsonNodeExtensions.cs @@ -0,0 +1,73 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Nodes; + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +internal static class JsonNodeExtensions +{ + extension(JsonNode baseNode) + { + // Based on https://gist.github.com/cajuncoding/bf78bdcf790782090d231590cbc2438f + public JsonNode Merge(JsonNode incomingNode) + { + switch (baseNode) + { + case JsonObject baseObject when incomingNode is JsonObject incomingObject: + MergeObjects(baseObject, incomingObject); + break; + case JsonArray baseArray when incomingNode is JsonArray incomingArray: + MergeArrays(baseArray, incomingArray); + break; + default: + throw new ArgumentException( + $"The JsonNode type [{baseNode.GetType().Name}] is incompatible for" + + $" merging with the target/base type {incomingNode.GetType().Name}." + + " Merging requires the types to be the same." + ); + } + + return baseNode; + } + } + + private static void MergeObjects(JsonObject baseObject, JsonObject incomingObject) + { + // Clear object so that the child elements no longer have a parent + var incomingObjectSnapshot = incomingObject.ToArray(); + incomingObject.Clear(); + + foreach (KeyValuePair incomingProperty in incomingObjectSnapshot) + { + var baseObjectValue = baseObject[incomingProperty.Key]; + baseObject[incomingProperty.Key] = baseObjectValue switch + { + // If both are JsonObjects, merge them recursively + JsonObject baseChildObject when incomingProperty.Value is JsonObject incomingChildObject => + baseChildObject.Merge(incomingChildObject), + + // If both are JsonArrays, merge them recursively + JsonArray baseChildArray when incomingProperty.Value is JsonArray incomingChildArray => + baseChildArray.Merge(incomingChildArray), + + // If the base property and incoming property are of different + // types, or if the base property does not exist, overwrite + // with the incoming property. + _ => incomingProperty.Value + }; + } + } + + private static void MergeArrays(JsonArray baseArray, JsonArray incomingArray) + { + // Clear array so that the child elements no longer have a parent + var incomingArraySnapshot = incomingArray.ToArray(); + incomingArray.Clear(); + + foreach (JsonNode? incomingElement in incomingArraySnapshot) + { + baseArray.Add(incomingElement); + } + } +} diff --git a/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs b/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs new file mode 100644 index 000000000..4e299343d --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/ManifestInfo.cs @@ -0,0 +1,138 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Microsoft.DotNet.ImageBuilder.Models.Manifest; + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +public sealed record ManifestInfo( + Manifest Model, + string FilePath, + ManifestReadmeInfo? Readme, + ImmutableList Repos) +{ + private readonly ImmutableDictionary _reposById = + Repos.Where(repo => repo.Model.Id is not null) + .ToImmutableDictionary(repo => repo.Model.Id!); + + private readonly ImmutableDictionary _reposByName = + Repos.ToImmutableDictionary(repo => repo.Model.Name); + + public RepoInfo? GetRepoById(string id) => _reposById.GetValueOrDefault(id); + public RepoInfo? GetRepoByName(string name) => _reposByName.GetValueOrDefault(name); + + internal static ManifestInfo Create(Manifest model, string manifestFilePath) + { + var manifestDir = Path.GetDirectoryName(manifestFilePath) ?? ""; + var repoInfos = model.Repos + .Select(repo => RepoInfo.Create(repo, model, manifestDir)) + .ToImmutableList(); + + var readmeInfo = model.Readme is not null + ? ManifestReadmeInfo.Create(model.Readme, model, manifestDir) + : null; + + return new ManifestInfo(model, manifestFilePath, readmeInfo, repoInfos); + } +} + +public sealed record ManifestReadmeInfo(Readme Model, Manifest Manifest, string FilePath, string? TemplatePath) +{ + internal static ManifestReadmeInfo Create(Readme model, Manifest manifest, string manifestDir) + { + string path = Path.Combine(manifestDir, model.Path); + string? templatePath = PathHelper.MaybeCombine(manifestDir, model.TemplatePath); + + return new ManifestReadmeInfo(model, manifest, path, templatePath); + } +} + +public sealed record RepoReadmeInfo(Readme Model, Repo Repo, string FilePath, string? TemplatePath) +{ + internal static RepoReadmeInfo Create(Readme model, Repo repo, string manifestDir) + { + string path = Path.Combine(manifestDir, model.Path); + string? templatePath = PathHelper.MaybeCombine(manifestDir, model.TemplatePath); + + return new RepoReadmeInfo(model, repo, path, templatePath); + } +} + +public sealed record RepoInfo( + Repo Model, + Manifest Manifest, + string FullName, + ImmutableList Images, + ImmutableList Readmes) +{ + internal static RepoInfo Create(Repo model, Manifest manifest, string manifestDir) + { + var imageInfos = model.Images + .Select(image => ImageInfo.Create(image, model, manifestDir)) + .ToImmutableList(); + + var readmeInfos = model.Readmes + .Select(readme => RepoReadmeInfo.Create(readme, model, manifestDir)) + .ToImmutableList(); + + var fullName = manifest.Registry + "/" + model.Name; + + return new RepoInfo(model, manifest, fullName, imageInfos, readmeInfos); + } +} + +public sealed record ImageInfo(Image Model, Repo repo, ImmutableList Platforms) +{ + internal static ImageInfo Create(Image model, Repo repo, string manifestDir) + { + var platformInfos = model.Platforms + .Select(platform => PlatformInfo.Create(platform, model, manifestDir)) + .ToImmutableList(); + + return new ImageInfo(model, repo, platformInfos); + } +} + +public sealed record PlatformInfo( + Platform Model, + Image Image, + ImmutableList Tags, + string DockerfilePath, + string RelativeDockerfilePath, + string? DockerfileTemplatePath = null) +{ + internal static PlatformInfo Create(Platform model, Image image, string manifestDir) + { + var relativeDockerfilePath = Path.Combine(model.Dockerfile, "Dockerfile"); + var fullDockerfilePath = Path.Combine(manifestDir, relativeDockerfilePath); + var dockerfileTemplatePath = model.DockerfileTemplate is not null + ? Path.Combine(manifestDir, model.DockerfileTemplate) : null; + + var tagInfos = model.Tags + .Select(tag => TagInfo.Create(tag.Value, model, tag.Key)) + .ToImmutableList(); + + return new PlatformInfo( + model, + image, + tagInfos, + fullDockerfilePath, + relativeDockerfilePath, + dockerfileTemplatePath); + } +} + +public sealed record TagInfo(Tag Model, Platform Platform, string Tag, bool IsDocumented) +{ + internal static TagInfo Create(Tag model, Platform platform, string tag) + { + bool isDocumented = model.DocType switch + { + TagDocumentationType.Undocumented => false, + _ => true + }; + + return new TagInfo(model, platform, tag, isDocumented); + } +} diff --git a/src/ImageBuilder.Models/ReadModel/ManifestInfoExtensions.cs b/src/ImageBuilder.Models/ReadModel/ManifestInfoExtensions.cs new file mode 100644 index 000000000..67db1a6de --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/ManifestInfoExtensions.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +public static class ManifestInfoExtensions +{ + extension(ManifestInfo manifest) + { + public IEnumerable AllImages => manifest.Repos.SelectMany(repo => repo.Images); + public IEnumerable AllPlatforms => manifest.AllImages.SelectMany(image => image.Platforms); + } +} diff --git a/src/ImageBuilder.Models/ReadModel/ManifestPreprocessor.cs b/src/ImageBuilder.Models/ReadModel/ManifestPreprocessor.cs new file mode 100644 index 000000000..150372e4d --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/ManifestPreprocessor.cs @@ -0,0 +1,106 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.DotNet.ImageBuilder.Models.Manifest; +using Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +internal sealed class ManifestPreprocessor +{ + private IVariableStore _variableStore = new EmptyVariableStore(); + + public JsonNode Process(JsonObject root, IEnumerable includesNodes) + { + // Process includes first so variables in included files can be processed + ProcessIncludes(root, includesNodes); + + var rawManifest = JsonHelper.Deserialize(root, ManifestSerializationContext.Default.Manifest); + var variables = rawManifest.Variables ?? new Dictionary(); + + // Add variables for each repo name (e.g. "Repo:dotnet" -> "mcr.microsoft.com/dotnet") + foreach (var kvp in rawManifest.RepoVariables) + { + variables.Add(kvp); + } + + _variableStore = new VariableStore(variables); + + ProcessVariables(root); + return root; + } + + /// + /// Replace keys and string values in JSON that reference variables defined + /// in the variable store. + /// + private void ProcessVariables(JsonNode? node) + { + switch (node) + { + case JsonObject jsonObject: + var jsonObjectSnapshot = jsonObject.ToList(); + foreach ((string oldKey, JsonNode? value) in jsonObjectSnapshot) + { + var newKey = _variableStore.ResolveInnerVariables(oldKey); + if (newKey != oldKey) + { + jsonObject.Remove(oldKey); + jsonObject[newKey] = value; + } + + ProcessVariables(value); + } + break; + + case JsonArray jsonArray: + for (int i = 0; i < jsonArray.Count; i++) + { + ProcessVariables(jsonArray[i]); + } + break; + + case JsonValue jsonValue: + if (jsonValue.TryGetValue(out string? stringValue)) + { + var newValue = _variableStore.ResolveInnerVariables(stringValue); + + #pragma warning disable IL2026 + #pragma warning disable IL3050 + // JsonValue.ReplaceWith is annotated with 'RequiresDynamicCodeAttribute', but it doesn't use + // dynamic code in all cases. Since we're giving it a JsonNode, it'll short-circuit before + // resorting to dynamic code for serialization. + jsonValue.ReplaceWith( + JsonSerializer.SerializeToNode(newValue, ManifestSerializationContext.Default.String) + ); + #pragma warning restore IL2026 + #pragma warning restore IL3050 + } + break; + + case null: + break; + } + } + + private static void ProcessIncludes(JsonObject jsonObject, IEnumerable includes) + { + foreach (JsonObject includeObject in includes) + { + jsonObject.Merge(includeObject); + } + } +} + +internal static class ManifestRepoVariableExtensions +{ + extension(Manifest manifest) + { + public IEnumerable> RepoVariables => + manifest.Repos + .Where(repo => !string.IsNullOrWhiteSpace(repo.Id)) + .Select(repo => new KeyValuePair($"Repo:{repo.Id}", repo.Name)); + } +} diff --git a/src/ImageBuilder.Models/ReadModel/PathHelper.cs b/src/ImageBuilder.Models/ReadModel/PathHelper.cs new file mode 100644 index 000000000..542a29eca --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/PathHelper.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +internal static class PathHelper +{ + [return: NotNullIfNotNull(nameof(optionalFilePath))] + public static string? MaybeCombine(string basePath, string? optionalFilePath) => + (basePath, optionalFilePath) switch + { + // If the optionalFilePath is null, then we want to maintain that null value. + (_, null) => null, + _ => Path.Combine(basePath, optionalFilePath) + }; +} diff --git a/src/ImageBuilder.Models/ReadModel/Serialization/JsonHelper.cs b/src/ImageBuilder.Models/ReadModel/Serialization/JsonHelper.cs new file mode 100644 index 000000000..d9ff07094 --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/Serialization/JsonHelper.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization.Metadata; + +namespace Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; + +internal static class JsonHelper +{ + public static T Deserialize(JsonNode jsonNode, JsonTypeInfo typeInfo) => + JsonSerializer.Deserialize(jsonNode, typeInfo) + ?? throw new Exception($"Failed to deserialize JSON object to {typeof(T)}."); + + public static string Serialize(T model, JsonTypeInfo typeInfo) => + JsonSerializer.Serialize(model, typeInfo); +} diff --git a/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs new file mode 100644 index 000000000..549ef4a90 --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestInfoSerializationExtensions.cs @@ -0,0 +1,94 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Nodes; +using static Microsoft.DotNet.ImageBuilder.ReadModel.Serialization.JsonHelper; + +namespace Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; + +public static class ManifestInfoSerializationExtensions +{ + extension(ManifestInfo manifestInfo) + { + public static async Task LoadAsync(string manifestJsonPath) + { + var manifestJsonObject = await LoadModelFromFileAsync(manifestJsonPath); + var manifestDir = Path.GetDirectoryName(manifestJsonPath) ?? ""; + + // Load and deserialize included files + IEnumerable includesJsonNodes = []; + var includesNode = manifestJsonObject["includes"]; + if (includesNode is not null) + { + var includesFiles = Deserialize(includesNode, ManifestSerializationContext.Default.StringArray); + + includesJsonNodes = await Task.WhenAll( + includesFiles + // Make includes paths relative to the manifest file + .Select(includesFile => Path.Combine(manifestDir, includesFile)) + .Select(LoadModelFromFileAsync)); + } + + var preprocessor = new ManifestPreprocessor(); + var processedRootJsonNode = preprocessor.Process(manifestJsonObject, includesJsonNodes); + var processedModel = Deserialize(processedRootJsonNode, ManifestSerializationContext.Default.Manifest); + + return ManifestInfo.Create(processedModel, manifestJsonPath); + } + + public static ManifestInfo Load(string manifestJsonPath) + { + var manifestJsonObject = LoadModelFromFile(manifestJsonPath); + var manifestDir = Path.GetDirectoryName(manifestJsonPath) ?? ""; + + // Load and deserialize included files + IEnumerable includesJsonNodes = []; + var includesNode = manifestJsonObject["includes"]; + if (includesNode is not null) + { + var includesFiles = Deserialize(includesNode, ManifestSerializationContext.Default.StringArray); + includesJsonNodes = includesFiles + .Select(includesFile => Path.Combine(manifestDir, includesFile)) + .Select(LoadModelFromFile); + } + + var preprocessor = new ManifestPreprocessor(); + var processedRootJsonNode = preprocessor.Process(manifestJsonObject, includesJsonNodes); + var processedModel = Deserialize(processedRootJsonNode, ManifestSerializationContext.Default.Manifest); + + return ManifestInfo.Create(processedModel, manifestJsonPath); + } + + public string ToJsonString() => Serialize(manifestInfo.Model, ManifestSerializationContext.Default.Manifest); + } + + private static async Task LoadModelFromFileAsync(string manifestJsonPath) + { + var jsonStream = File.OpenRead(manifestJsonPath); + var rootJsonNode = await JsonNode.ParseAsync(jsonStream) + ?? throw new Exception( + $"Failed to parse manifest JSON from file: {manifestJsonPath}"); + + if (rootJsonNode is not JsonObject rootJsonObject) + { + throw new InvalidDataException($"Manifest root must be a JSON object."); + } + + return rootJsonObject; + } + + private static JsonObject LoadModelFromFile(string manifestJsonPath) + { + var jsonStream = File.OpenRead(manifestJsonPath); + var rootJsonNode = JsonNode.Parse(jsonStream) + ?? throw new Exception( + $"Failed to parse manifest JSON from file: {manifestJsonPath}"); + + if (rootJsonNode is not JsonObject rootJsonObject) + { + throw new InvalidDataException($"Manifest root must be a JSON object."); + } + + return rootJsonObject; + } +} diff --git a/src/ImageBuilder.Models/ReadModel/Serialization/ManifestSerializationContext.cs b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestSerializationContext.cs new file mode 100644 index 000000000..a23e93c41 --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/Serialization/ManifestSerializationContext.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Serialization; +using Microsoft.DotNet.ImageBuilder.Models.Manifest; + +namespace Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; + +[JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + WriteIndented = true, + UseStringEnumConverter = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault +)] +[JsonSerializable(typeof(Manifest))] +[JsonSerializable(typeof(string[]))] +[JsonSerializable(typeof(string))] +public partial class ManifestSerializationContext : JsonSerializerContext +{ +} diff --git a/src/ImageBuilder.Models/ReadModel/VariableStore.cs b/src/ImageBuilder.Models/ReadModel/VariableStore.cs new file mode 100644 index 000000000..20b2ea46e --- /dev/null +++ b/src/ImageBuilder.Models/ReadModel/VariableStore.cs @@ -0,0 +1,67 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.RegularExpressions; + +namespace Microsoft.DotNet.ImageBuilder.ReadModel; + +internal sealed partial class VariableStore : IVariableStore +{ + private const string VariableGroupName = "variable"; + + private readonly Dictionary _resolvedVariables; + + /// + /// Creates a new . + /// + /// + /// The order of variable definitions determines which order they are + /// resolved in. Variable references must come after their definition. + /// + public VariableStore(IDictionary variables) + { + _resolvedVariables = new Dictionary(); + foreach (var (variable, unresolvedValue) in variables) + { + string resolvedValue = ResolveInnerVariables(unresolvedValue); + _resolvedVariables.Add(variable, resolvedValue); + } + } + + /// + /// Evaluates an expression and replaces any variables inside with their + /// fully-resolved values. + /// + /// + /// Variable references inside this expression will be replaced. + /// + public string ResolveInnerVariables(string expression) + { + var subVariableMatches = TagVariableRegex.Matches(expression); + foreach (Match match in subVariableMatches) + { + string variableName = match.Groups[VariableGroupName].Value; + string? variableValue = GetResolvedValue(variableName) + ?? throw new InvalidOperationException($"A value was not found for the variable '{match.Value}'"); + expression = expression.Replace(match.Value, variableValue); + } + + return expression; + } + + /// + /// Get the resolved value for a variable name. Returns null if the + /// variable is not found. + /// + /// + /// The variable value, or null if the variable is not found + /// + private string? GetResolvedValue(string variableName) + { + _resolvedVariables.TryGetValue(variableName, out string? variableValue); + return variableValue; + } + + [GeneratedRegex($"\\$\\((?<{VariableGroupName}>[\\w:\\-.| ]+)\\)")] + private static partial Regex TagVariableRegex { get; } +} diff --git a/src/ImageBuilder.Tests/Microsoft.DotNet.ImageBuilder.Tests.csproj b/src/ImageBuilder.Tests/Microsoft.DotNet.ImageBuilder.Tests.csproj index c629747c6..81e783557 100644 --- a/src/ImageBuilder.Tests/Microsoft.DotNet.ImageBuilder.Tests.csproj +++ b/src/ImageBuilder.Tests/Microsoft.DotNet.ImageBuilder.Tests.csproj @@ -1,7 +1,7 @@ - net9.0 + net10.0 false true true diff --git a/src/ImageBuilder/Commands/CopyBaseImagesCommand.cs b/src/ImageBuilder/Commands/CopyBaseImagesCommand.cs index afae10335..9b6ad76a5 100644 --- a/src/ImageBuilder/Commands/CopyBaseImagesCommand.cs +++ b/src/ImageBuilder/Commands/CopyBaseImagesCommand.cs @@ -77,7 +77,8 @@ public override async Task ExecuteAsync() private IEnumerable GetFromImages(ManifestInfo manifest) => manifest.GetExternalFromImages() .Select(fromImage => Options.BaseImageOverrideOptions.ApplyBaseImageOverride(fromImage)) - .Where(fromImage => !fromImage.StartsWith(manifest.Model.Registry)); + .Where(fromImage => string.IsNullOrEmpty(manifest.Model.Registry) + || !fromImage.StartsWith(manifest.Model.Registry)); private Task CopyImageAsync(string fromImage, string destinationRegistryName) { diff --git a/src/ImageBuilder/ImageNameResolver.cs b/src/ImageBuilder/ImageNameResolver.cs index 27104a984..4e0467ec2 100644 --- a/src/ImageBuilder/ImageNameResolver.cs +++ b/src/ImageBuilder/ImageNameResolver.cs @@ -79,8 +79,8 @@ private string GetFromImageTag(string fromImage, string? registry) { fromImage = _baseImageOverrideOptions.ApplyBaseImageOverride(fromImage); - if ((registry is not null && DockerHelper.IsInRegistry(fromImage, registry)) || - DockerHelper.IsInRegistry(fromImage, Manifest.Model.Registry) + if ((registry is not null && DockerHelper.IsInRegistry(fromImage, registry)) + || (Manifest.Model.Registry is not null && DockerHelper.IsInRegistry(fromImage, Manifest.Model.Registry)) || _sourceRepoPrefix is null) { return fromImage; @@ -97,7 +97,7 @@ protected string TrimInternallyOwnedRegistryAndRepoPrefix(string imageTag) => private bool IsInInternallyOwnedRegistry(string imageTag) => DockerHelper.IsInRegistry(imageTag, Manifest.Registry) || - DockerHelper.IsInRegistry(imageTag, Manifest.Model.Registry); + (Manifest.Model.Registry is not null && DockerHelper.IsInRegistry(imageTag, Manifest.Model.Registry)); } public class ImageNameResolverForBuild : ImageNameResolver diff --git a/src/ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj b/src/ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj index a23aa0427..55841cc2d 100644 --- a/src/ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj +++ b/src/ImageBuilder/Microsoft.DotNet.ImageBuilder.csproj @@ -3,12 +3,16 @@ Exe False - net9.0 + net10.0 Microsoft.DotNet.ImageBuilder true true + + + + @@ -26,12 +30,9 @@ - - - diff --git a/src/ImageBuilder/Models/Manifest/Architecture.cs b/src/ImageBuilder/Models/Manifest/Architecture.cs deleted file mode 100644 index cdcbb328e..000000000 --- a/src/ImageBuilder/Models/Manifest/Architecture.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest -{ - // Enum values must align with the $GOARCH values specified at https://golang.org/doc/install/source#environment - public enum Architecture - { - ARM, - ARM64, - AMD64, - } -} diff --git a/src/ImageBuilder/Models/Manifest/CustomBuildLegDependencyType.cs b/src/ImageBuilder/Models/Manifest/CustomBuildLegDependencyType.cs deleted file mode 100644 index 67fb43082..000000000 --- a/src/ImageBuilder/Models/Manifest/CustomBuildLegDependencyType.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.ComponentModel; - -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest -{ - [Description( - "The type of dependency an image has for a specific scenario." - )] - public enum CustomBuildLegDependencyType - { - [Description( - "Indicates the dependency is considered to be integral to the depending image." + - "This means the dependent image will not have its own dependency graph considered for build leg " + - "generation. An example of this is when a custom build leg dependency is defined from sdk to " + - "aspnet; in that case, aspnet and sdk will be included in a leg together but the sdk will not " + - "have its own leg generated." - )] - Integral, - - [Description( - "Indicates the dependency is considered to be a supplemental companion to the depending image." + - "This means the dependent image will have its own dependency graph considered for build leg " + - "generation. An example of this is when a custom build leg dependency is defined to " + - "include an SDK image supported on a particular architecture in order to test a runtime OS " + - "that doesn't its own SDK on that architecture (Trixie ARM SDK to test Alpine ARM runtime); " + - "in that case, the SDK will be included in a leg together with the runtime and the SDK will " + - "still have have its own leg." - )] - Supplemental - } -} diff --git a/src/ImageBuilder/Models/Manifest/CustomBuildLegGroup.cs b/src/ImageBuilder/Models/Manifest/CustomBuildLegGroup.cs deleted file mode 100644 index 5c3a856e1..000000000 --- a/src/ImageBuilder/Models/Manifest/CustomBuildLegGroup.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.ComponentModel; -using Newtonsoft.Json; - -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest -{ - [Description( - "This object describes the tag dependencies of the image for a specific named scenario. This is " + - "for advanced cases only. It allows tooling to modify the build matrix that would normally be " + - "generated for the image by including the customizations described in this metadata. An example " + - "usage of this is in PR builds where it is necessary to build and test in the same job. In such " + - "a scenario, some images are part of a test matrix that require images to be available on the " + - "build machine that aren't part of that images dependency graph in normal scenarios. By " + - "specifying a customBuildLegGroup for this scenario, those additional image dependencies can " + - "be specified and the build pipeline can make use of them when constructing its build graph when " + - "specified to do so." - )] - public class CustomBuildLegGroup - { - [Description( - "Name of the group describing the scenario in which it's relevant. This is just a " + - " custom label that can then be used by tooling to lookup the group when necessary." - )] - [JsonProperty(Required = Required.Always)] - public string Name { get; set; } - - [Description("The type of the dependency which impacts how it's used during the build.")] - [JsonProperty(Required = Required.Always)] - public CustomBuildLegDependencyType Type { get; set; } - - [Description("The set of dependencies the image has for this scenario.")] - [JsonProperty(Required = Required.Always)] - public string[] Dependencies { get; set; } = Array.Empty(); - } -} diff --git a/src/ImageBuilder/Models/Manifest/Image.cs b/src/ImageBuilder/Models/Manifest/Image.cs deleted file mode 100644 index 68124f0c0..000000000 --- a/src/ImageBuilder/Models/Manifest/Image.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Collections.Generic; -using System.ComponentModel; -using Newtonsoft.Json; - -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest -{ - [Description("An image object contains metadata about a specific Docker image.")] - public class Image - { - [Description( - "The set of platforms that describe the platform-specific variations of the Docker image.")] - [JsonProperty(Required = Required.Always)] - public Platform[] Platforms { get; set; } - - [Description( - "The set of tags that are shared amongst all platform-specific versions of the image. An " + - "example of a shared tag, including its repo name, is dotnet/core/runtime:2.2; running " + - "`docker pull mcr.microsoft.com/dotnet/core/runtime:2.2` on Windows will get the " + - "default Windows-based tag whereas running it on Linux will get the default " + - "Linux-based tag.")] - public IDictionary SharedTags { get; set; } - - [Description("The full version of the product that the Docker image contains.")] - public string ProductVersion { get; set; } - - public Image() - { - } - } -} diff --git a/src/ImageBuilder/Models/Manifest/Manifest.cs b/src/ImageBuilder/Models/Manifest/Manifest.cs deleted file mode 100644 index d3e5cb090..000000000 --- a/src/ImageBuilder/Models/Manifest/Manifest.cs +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.ComponentModel; - -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest -{ - [Description( - "The manifest file is the primary source of metadata that drives the production " + - "of all .NET Docker images. It describes various attributes of the Docker images " + - "that are to be produced by a given GitHub repo. .NET Docker's engineering system " + - "consumes this file in various ways as part of the automated build pipelines and " + - "other tools. It's intended to be product-agnostic meaning that it could be used " + - "to describe metadata for Docker image production of any product, not just .NET.")] - public class Manifest - { - [Description( - "Additional json files to be loaded with this manifest. This is a convienent" + - "way to split the manifest apart into logical parts." - )] - public string[] Includes { get; set; } - - [Description( - "Info about the readme that documents the product family." - )] - public Readme Readme { get; set; } - - [Description( - "The location of the Docker registry where the images are to be published." - )] - public string Registry { get; set; } - - [Description( - "The set of Docker repositories described by this manifest." - )] - public Repo[] Repos { get; set; } = Array.Empty(); - - [Description( - "A set of custom variables that can be referenced in various parts of the " + - "manifest. This provides a few benefits: 1) allows a commmonly used value " + - "to be defined only once and referenced by its variable name many times" + - "2) allows tools that consume the manifest file to provide a mechanism to " + - "dynamically override the value of these variables. Variables may be " + - "referenced in other parts of the manifest by using the following syntax: " + - "$(_VariableName_).")] - public IDictionary Variables { get; set; } = new Dictionary(); - - public Manifest() - { - } - } -} diff --git a/src/ImageBuilder/Models/Manifest/PackageQueryInfo.cs b/src/ImageBuilder/Models/Manifest/PackageQueryInfo.cs deleted file mode 100644 index 0d04f92ce..000000000 --- a/src/ImageBuilder/Models/Manifest/PackageQueryInfo.cs +++ /dev/null @@ -1,25 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -#nullable enable -using System.ComponentModel; - -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest -{ - [Description( - "Relative path to the template the Dockerfile is generated from." - )] - public class PackageQueryInfo - { - [Description( - "Relative path from the manifest file to the script which queries the packages installed for a platform Dockerfile." - )] - public string? GetInstalledPackagesPath { get; set; } - - [Description( - "Relative path from the manifest file to the script which queries the packages available for upgrade for a platform Dockerfile." - )] - public string? GetUpgradablePackagesPath { get; set; } - } -} -#nullable disable diff --git a/src/ImageBuilder/Models/Manifest/Platform.cs b/src/ImageBuilder/Models/Manifest/Platform.cs deleted file mode 100644 index ccac058a4..000000000 --- a/src/ImageBuilder/Models/Manifest/Platform.cs +++ /dev/null @@ -1,81 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.ComponentModel; -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; - -#nullable enable -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest -{ - [Description( - "A platform object contains metadata about a platform-specific version of an " + - "image and refers to the actual Dockerfile used to build the image.")] - public class Platform - { - [Description( - "The processor architecture associated with the image." - )] - [DefaultValue(Architecture.AMD64)] - [JsonConverter(typeof(StringEnumConverter))] - [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)] - public Architecture Architecture { get; set; } = Architecture.AMD64; - - [Description( - "A set of values that will passed to the `docker build` command " + - "to override variables defined in the Dockerfile.")] - public IDictionary BuildArgs { get; set; } = new Dictionary(); - - [Description( - "Relative path to the associated Dockerfile. This can be a file or a " + - "directory. If it is a directory, the file name defaults to Dockerfile." - )] - [JsonProperty(Required = Required.Always)] - public string Dockerfile { get; set; } = string.Empty; - - [Description( - "Relative path to the template the Dockerfile is generated from." - )] - public string? DockerfileTemplate { get; set; } - - [Description( - "The generic name of the operating system associated with the image." - )] - [JsonConverter(typeof(StringEnumConverter))] - [JsonProperty(Required = Required.Always)] - public OS OS { get; set; } - - [Description( - "The specific version of the operating system associated with the image. " + - "Examples: alpine3.9, bionic, nanoserver-1903." - )] - [JsonProperty(Required = Required.Always)] - public string OsVersion { get; set; } = string.Empty; - - [Description( - "The set of platform-specific tags associated with the image." - )] - [JsonProperty(Required = Required.Always)] - public IDictionary Tags { get; set; } = new Dictionary(); - - [Description( - "The custom build leg groups associated with the platform." - )] - public CustomBuildLegGroup[] CustomBuildLegGroups { get; set; } = Array.Empty(); - - [Description( - "A label which further distinguishes the architecture when it " + - "contains variants. For example, the ARM architecture has variants " + - "named v6, v7, etc." - )] - public string? Variant { get; set; } - - public Platform() - { - } - } -} -#nullable disable diff --git a/src/ImageBuilder/Models/Manifest/Repo.cs b/src/ImageBuilder/Models/Manifest/Repo.cs deleted file mode 100644 index 3cbd0a57d..000000000 --- a/src/ImageBuilder/Models/Manifest/Repo.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.ComponentModel; -using Newtonsoft.Json; - -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest -{ - [Description( - "A repository object contains metadata about a target Docker repository " + - "and the images to be contained in it." - )] - public class Repo - { - [Description( - "A unique identifier of the repo. This is purely within the context " + - "of the manifest and not exposed to Docker in any way." - )] - public string Id { get; set; } - - [Description( - "The set of images contained in this repository." - )] - [JsonProperty(Required = Required.Always)] - public Image[] Images { get; set; } - - [Description( - "Relative path to the MCR tags template YAML file that is used by " + - "tooling to generate the tags section of the readme file." - )] - public string McrTagsMetadataTemplate { get; set; } - - [Description( - "The name of the Docker repository where the described images are to " + - "be published (example: dotnet/core/runtime)." - )] - [JsonProperty(Required = Required.Always)] - public string Name { get; set; } - - [Description( - "Info about the readme that documents the repo." - )] - public Readme[] Readmes { get; set; } = Array.Empty(); - - public Repo() - { - } - } -} diff --git a/src/ImageBuilder/Models/Manifest/Tag.cs b/src/ImageBuilder/Models/Manifest/Tag.cs deleted file mode 100644 index 61567e3d3..000000000 --- a/src/ImageBuilder/Models/Manifest/Tag.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.ComponentModel; - -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest -{ - [Description( - "A tag object contains metadata about a Docker tag. It is a JSON object " + - "with its tag name used as the attribute name." - )] - public class Tag - { - [Description( - "An identifier used to conceptually group related tags in the readme " + - "documentation." - )] - public string DocumentationGroup { get; set; } - - [Description( - "Indicates how this tag should not be documented in the readme file. Regardless of the " + - "setting, the image will still be tagged with this tag and will still be published. " + - "This is useful when deprecating a tag that still needs to be kept up-to-date " + - "but not wanting it documented." - )] - [DefaultValue(TagDocumentationType.Documented)] - public TagDocumentationType DocType { get; set; } - - [Description( - "Description of where the tag should be syndicated to.")] - public TagSyndication Syndication { get; set; } - - public Tag() - { - } - } -} diff --git a/src/ImageBuilder/Models/Manifest/TagDocumentationType.cs b/src/ImageBuilder/Models/Manifest/TagDocumentationType.cs deleted file mode 100644 index 8946dbe4e..000000000 --- a/src/ImageBuilder/Models/Manifest/TagDocumentationType.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest -{ - public enum TagDocumentationType - { - /// - /// The tag is always documented. - /// - Documented, - - /// - /// The tag is never documented. - /// - Undocumented, - - /// - /// The tag is only documented if there are corresponding platform tags that are documented. - /// - PlatformDocumented - } -} diff --git a/src/ImageBuilder/Models/Manifest/TagSyndication.cs b/src/ImageBuilder/Models/Manifest/TagSyndication.cs deleted file mode 100644 index 835fd58d1..000000000 --- a/src/ImageBuilder/Models/Manifest/TagSyndication.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// - -using System.ComponentModel; - -namespace Microsoft.DotNet.ImageBuilder.Models.Manifest -{ - [Description( - "A description of where a tag should be syndicated to." - )] - public class TagSyndication - { - [Description( - "Name of the repo to syndicate the tag to." - )] - public string Repo { get; set; } - - [Description( - "List of destination tag names to syndicate the tag to." - )] - public string[] DestinationTags { get; set; } - } -} diff --git a/src/ImageBuilder/ViewModel/ImageInfo.cs b/src/ImageBuilder/ViewModel/ImageInfo.cs index 219bd7abe..46e533aee 100644 --- a/src/ImageBuilder/ViewModel/ImageInfo.cs +++ b/src/ImageBuilder/ViewModel/ImageInfo.cs @@ -54,7 +54,11 @@ public static ImageInfo Create( .Select(platform => PlatformInfo.Create(platform, fullRepoModelName, repoName, variableHelper, baseDirectory)) .ToArray(); - string? productVersion = variableHelper.SubstituteValues(model.ProductVersion); + string? productVersion = model.ProductVersion; + if (productVersion is not null) + { + productVersion = variableHelper.SubstituteValues(productVersion); + } IEnumerable filteredPlatformModels = manifestFilter.FilterPlatforms(model.Platforms, productVersion); IEnumerable filteredPlatforms = allPlatforms diff --git a/src/ImageBuilder/ViewModel/PlatformInfo.cs b/src/ImageBuilder/ViewModel/PlatformInfo.cs index cdfe84173..ee3922cbc 100644 --- a/src/ImageBuilder/ViewModel/PlatformInfo.cs +++ b/src/ImageBuilder/ViewModel/PlatformInfo.cs @@ -28,7 +28,7 @@ public class PlatformInfo private IEnumerable _internalRepos = Enumerable.Empty(); public string BaseOsVersion { get; private set; } - public IDictionary BuildArgs { get; private set; } = ImmutableDictionary.Empty; + public IDictionary BuildArgs { get; private set; } = ImmutableDictionary.Empty; public string BuildContextPath { get; private set; } public string DockerfilePath { get; private set; } public string DockerfilePathRelativeToManifest { get; private set; } @@ -101,7 +101,7 @@ public void Initialize(IEnumerable internalRepos, string registry) Name = group.Name, Type = group.Type, Dependencies = group.Dependencies - .Select(dependency => VariableHelper.SubstituteValues(dependency)) + .Select(dependency => VariableHelper.SubstituteValues(dependency)!) .ToArray() }) .ToDictionary(info => info.Name) diff --git a/src/ImageBuilder/ViewModel/VariableHelper.cs b/src/ImageBuilder/ViewModel/VariableHelper.cs index de58a8178..8ccdcba56 100644 --- a/src/ImageBuilder/ViewModel/VariableHelper.cs +++ b/src/ImageBuilder/ViewModel/VariableHelper.cs @@ -34,7 +34,7 @@ public VariableHelper(Manifest manifest, IManifestOptionsInfo options, Func kvp in Manifest.Variables) + foreach (KeyValuePair kvp in Manifest.Variables) { string? variableValue; if (Options.Variables is not null && Options.Variables.TryGetValue(kvp.Key, out string? overridenValue)) @@ -56,22 +56,17 @@ public VariableHelper(Manifest manifest, IManifestOptionsInfo options, Func kvp in Options.Variables) { - if (!ResolvedVariables.ContainsKey(kvp.Key)) + if (!ResolvedVariables.ContainsKey(kvp.Key) && kvp.Value is not null) { - string? value = SubstituteValues(kvp.Value); + string value = SubstituteValues(kvp.Value); ResolvedVariables.Add(kvp.Key, value); } } } } - public string? SubstituteValues(string? expression, Func? getContextBasedSystemValue = null) + public string SubstituteValues(string expression, Func? getContextBasedSystemValue = null) { - if (expression == null) - { - return null; - } - foreach (Match match in Regex.Matches(expression, s_tagVariablePattern)) { string? variableValue; diff --git a/src/TemplateGenerator.Benchmarks/GenerateDockerfilesBenchmarks.cs b/src/TemplateGenerator.Benchmarks/GenerateDockerfilesBenchmarks.cs new file mode 100644 index 000000000..127961b1c --- /dev/null +++ b/src/TemplateGenerator.Benchmarks/GenerateDockerfilesBenchmarks.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using BenchmarkDotNet.Attributes; + +namespace Microsoft.DotNet.DockerTools.TemplateGenerator.Benchmarks; + +[MemoryDiagnoser] +public class GenerateDockerfilesBenchmarks +{ + private static readonly string s_manifestPath = + Environment.GetEnvironmentVariable("MANIFEST_PATH") ?? "manifest.json"; + + [Benchmark] + public void GenerateDockerfiles() + { + var generator = new TemplateGeneratorCli(); + generator.GenerateDockerfiles(s_manifestPath); + } + + [Benchmark] + public void GenerateReadmes() + { + var generator = new TemplateGeneratorCli(); + generator.GenerateReadmes(s_manifestPath); + } + + [Benchmark] + public void GenerateAll() + { + var generator = new TemplateGeneratorCli(); + generator.GenerateAll(s_manifestPath); + } +} diff --git a/src/TemplateGenerator.Benchmarks/ManifestBenchmarks.cs b/src/TemplateGenerator.Benchmarks/ManifestBenchmarks.cs new file mode 100644 index 000000000..1d91a3ffa --- /dev/null +++ b/src/TemplateGenerator.Benchmarks/ManifestBenchmarks.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using BenchmarkDotNet.Attributes; +using Microsoft.DotNet.ImageBuilder.ReadModel; +using Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; + +namespace Microsoft.DotNet.DockerTools.TemplateGenerator.Benchmarks; + +[MemoryDiagnoser] +public class ManifestBenchmarks +{ + private static readonly string s_manifestPath = + Environment.GetEnvironmentVariable("MANIFEST_PATH") ?? "manifest.json"; + + [Benchmark] + public async Task RoundTripManifestSerialization() + { + ManifestInfo manifest = await ManifestInfo.LoadAsync(s_manifestPath); + return manifest.ToJsonString(); + } +} diff --git a/src/TemplateGenerator.Benchmarks/Program.cs b/src/TemplateGenerator.Benchmarks/Program.cs new file mode 100644 index 000000000..f4223ff64 --- /dev/null +++ b/src/TemplateGenerator.Benchmarks/Program.cs @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using BenchmarkDotNet.Running; +using Microsoft.DotNet.DockerTools.TemplateGenerator.Benchmarks; + +BenchmarkRunner.Run(); +BenchmarkRunner.Run(); diff --git a/src/TemplateGenerator.Benchmarks/TemplateGenerator.Benchmarks.csproj b/src/TemplateGenerator.Benchmarks/TemplateGenerator.Benchmarks.csproj new file mode 100644 index 000000000..d168def05 --- /dev/null +++ b/src/TemplateGenerator.Benchmarks/TemplateGenerator.Benchmarks.csproj @@ -0,0 +1,19 @@ + + + + Exe + net10.0 + enable + enable + Microsoft.DotNet.DockerTools.TemplateGenerator.Benchmarks + + + + + + + + + + + diff --git a/src/TemplateGenerator/LoggingExtensions.cs b/src/TemplateGenerator/LoggingExtensions.cs new file mode 100644 index 000000000..55719c559 --- /dev/null +++ b/src/TemplateGenerator/LoggingExtensions.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.DockerTools.Templating.Cottle; +using Microsoft.DotNet.DockerTools.Templating; + +namespace Microsoft.DotNet.DockerTools.TemplateGenerator; + +internal static class LoggingExtensions +{ + extension(FileSystem fs) + { + public void LogStatistics() => Console.WriteLine( + $""" + Read {fs.FilesRead} files ({fs.BytesRead} bytes) + Wrote {fs.FilesWritten} files ({fs.BytesWritten} bytes) + """ + ); + } + + extension(FileSystemCache fsCache) + { + public void LogStatistics() => Console.WriteLine( + $"File system cache hits: {fsCache.CacheHits}, misses: {fsCache.CacheMisses}" + ); + } + + extension(CottleTemplateEngine engine) + { + public void LogStatistics() => Console.WriteLine( + $"Compiled template cache hits: {engine.CompiledTemplateCacheHits}," + + $" misses: {engine.CompiledTemplateCacheMisses}" + ); + } +} diff --git a/src/TemplateGenerator/Program.cs b/src/TemplateGenerator/Program.cs new file mode 100644 index 000000000..6b96b3653 --- /dev/null +++ b/src/TemplateGenerator/Program.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.DockerTools.TemplateGenerator; +using ConsoleAppFramework; + +var app = ConsoleApp.Create(); +app.Add(); +app.Run(args); diff --git a/src/TemplateGenerator/TemplateGenerator.csproj b/src/TemplateGenerator/TemplateGenerator.csproj new file mode 100644 index 000000000..dac186433 --- /dev/null +++ b/src/TemplateGenerator/TemplateGenerator.csproj @@ -0,0 +1,20 @@ + + + + Exe + net10.0 + enable + enable + Microsoft.DotNet.DockerTools.TemplateGenerator + true + + + + + + + + + + + diff --git a/src/TemplateGenerator/TemplateGeneratorCli.cs b/src/TemplateGenerator/TemplateGeneratorCli.cs new file mode 100644 index 000000000..621929103 --- /dev/null +++ b/src/TemplateGenerator/TemplateGeneratorCli.cs @@ -0,0 +1,136 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using ConsoleAppFramework; +using Microsoft.DotNet.ImageBuilder.ReadModel; +using Microsoft.DotNet.ImageBuilder.ReadModel.Serialization; +using Microsoft.DotNet.DockerTools.Templating.Cottle; +using Microsoft.DotNet.DockerTools.Templating; +using System.Diagnostics; + +namespace Microsoft.DotNet.DockerTools.TemplateGenerator; + +public sealed class TemplateGeneratorCli +{ + /// + /// Generates Dockerfiles from a manifest file. + /// + /// Path to manifest JSON file + public void GenerateDockerfiles([Argument] string manifestPath) + { + var manifest = ManifestInfo.Load(manifestPath); + + var fileSystem = new FileSystem(); + var fileSystemCache = new FileSystemCache(fileSystem); + var engine = new CottleTemplateEngine(fileSystemCache); + engine.AddGlobalVariables(manifest.Model.Variables); + + var platformsWithTemplates = manifest.AllPlatforms + .Where(platform => platform.DockerfileTemplatePath is not null); + + var compiledTemplates = platformsWithTemplates + .Select(platform => platform.DockerfileTemplatePath!) + .Select(engine.ReadAndCompile); + + var compiledTemplateInfos = platformsWithTemplates + .Zip(compiledTemplates); + + foreach (var (platform, compiledTemplate) in compiledTemplateInfos) + { + var platformContext = engine.CreateContext( + variables: platform.PlatformSpecificTemplateVariables, + // Null-forgiving operator is safe here because we filtered out + // platforms without templates above. + templatePath: platform.DockerfileTemplatePath!); + + var output = compiledTemplate.Render(platformContext); + fileSystem.WriteAllText(platform.DockerfilePath, output); + } + + fileSystem.LogStatistics(); + fileSystemCache.LogStatistics(); + engine.LogStatistics(); + } + + /// + /// Generates README.md files from a manifest file. + /// + /// Path to manifest JSON file + public void GenerateReadmes([Argument] string manifestPath) + { + var manifest = ManifestInfo.Load(manifestPath); + + var fileSystem = new FileSystem(); + var fileSystemCache = new FileSystemCache(fileSystem); + var engine = new CottleTemplateEngine(fileSystemCache); + engine.AddGlobalVariables(manifest.Model.Variables); + + var templatedRepoReadmes = + manifest.Repos + .SelectMany(repo => repo.Readmes + .Where(readme => readme.TemplatePath is not null) + .Select(readme => (Repo: repo, Readme: readme))); + + foreach (var (repo, readme) in templatedRepoReadmes) + { + // Null-forgiving operator is safe here because we filtered out + // readmes without templates above. + var readmeTemplatePath = readme.TemplatePath!; + var compiledTemplate = engine.ReadAndCompile(readmeTemplatePath); + var repoContext = engine.CreateContext(repo.TemplateVariables, readmeTemplatePath); + var output = compiledTemplate.Render(repoContext); + fileSystem.WriteAllText(readme.FilePath, output); + } + + var manifestReadme = manifest.Readme; + if (manifestReadme is not null && manifestReadme.TemplatePath is not null) + { + var compiledTemplate = engine.ReadAndCompile(manifestReadme.TemplatePath); + var manifestContext = engine.CreateContext(manifest.TemplateVariables, manifestReadme.TemplatePath); + var output = compiledTemplate.Render(manifestContext); + fileSystem.WriteAllText(manifestReadme.FilePath, output); + } + + fileSystem.LogStatistics(); + fileSystemCache.LogStatistics(); + engine.LogStatistics(); + } + + /// + /// Generates both Dockerfiles and READMEs from a manifest file. + /// + /// Path to manifest JSON file + public void GenerateAll([Argument] string manifestPath) + { + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + Console.WriteLine( + """ + + --- Generating Dockerfiles --- + """); + GenerateDockerfiles(manifestPath); + + stopwatch.Stop(); + + Console.WriteLine( + $""" + ({stopwatch.ElapsedMilliseconds} ms) + + --- Generating READMEs --- + """); + + stopwatch.Reset(); + stopwatch.Start(); + + GenerateReadmes(manifestPath); + + Console.WriteLine( + $""" + ({stopwatch.ElapsedMilliseconds} ms) + + """ + ); + } +} diff --git a/src/Templating/Abstractions/ICompiledTemplate.cs b/src/Templating/Abstractions/ICompiledTemplate.cs new file mode 100644 index 000000000..18251be5c --- /dev/null +++ b/src/Templating/Abstractions/ICompiledTemplate.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.DockerTools.Templating.Abstractions; + +public interface ICompiledTemplate +{ + string Render(TContext context, bool trim = false, string indent = ""); +} diff --git a/src/Templating/Abstractions/IFileSystem.cs b/src/Templating/Abstractions/IFileSystem.cs new file mode 100644 index 000000000..f3f6332ad --- /dev/null +++ b/src/Templating/Abstractions/IFileSystem.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.DockerTools.Templating.Abstractions; + +public interface IFileSystem +{ + string ReadAllText(string path); + void WriteAllText(string path, string content); +} diff --git a/src/Templating/Abstractions/ITableBuilder.cs b/src/Templating/Abstractions/ITableBuilder.cs new file mode 100644 index 000000000..9bb5cc02b --- /dev/null +++ b/src/Templating/Abstractions/ITableBuilder.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.DockerTools.Templating.Abstractions; + +internal interface ITableBuilder +{ + ITableBuilder WithColumnHeadings(params IEnumerable headings); + void AddRow(params IEnumerable row); + string ToString(); +} diff --git a/src/Templating/Abstractions/ITemplateEngine.cs b/src/Templating/Abstractions/ITemplateEngine.cs new file mode 100644 index 000000000..def63a418 --- /dev/null +++ b/src/Templating/Abstractions/ITemplateEngine.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.DockerTools.Templating.Abstractions; + +public interface ITemplateEngine +{ + /// + /// Add global variables that will be available in all newly created + /// template contexts. + /// + void AddGlobalVariables(IDictionary variables); + + /// + /// Create a new context for rendering a template. + /// + /// + /// Dictionary of variables to add to context. These variables will take + /// precedence over any global variables already set in the engine. + /// + /// + /// The path to the current template is needed to correctly resolve paths + /// to sub-templates + /// + TContext CreateContext(IDictionary variables, string templatePath); + + /// + /// Read a template from a file and compile it. + /// + ICompiledTemplate ReadAndCompile(string path); +} diff --git a/src/Templating/Cottle/CottleContextExtensions.cs b/src/Templating/Cottle/CottleContextExtensions.cs new file mode 100644 index 000000000..efec5662f --- /dev/null +++ b/src/Templating/Cottle/CottleContextExtensions.cs @@ -0,0 +1,194 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Cottle; +using Microsoft.DotNet.DockerTools.Templating.Shared; +using Microsoft.DotNet.ImageBuilder.Models.Manifest; +using Microsoft.DotNet.ImageBuilder.ReadModel; +using System.Text.RegularExpressions; + +namespace Microsoft.DotNet.DockerTools.Templating.Cottle; + +public static class CottleContextExtensions +{ + extension(IContext context) + { + public IContext Add(Value key, Value value) + { + var newContext = Context.CreateCustom(new Dictionary { { key, value } }); + return Context.CreateCascade(primary: newContext, fallback: context); + } + + public IContext Add(Dictionary symbols) + { + var newContext = Context.CreateCustom(symbols); + return Context.CreateCascade(primary: newContext, fallback: context); + } + + public IContext Add(IDictionary variables) + { + var variablesDictionary = variables.ToCottleDictionary(); + var newContext = Context.CreateCustom(variablesDictionary); + return Context.CreateCascade(newContext, context); + } + } + + extension(IDictionary stringDictionary) + { + public Dictionary ToCottleDictionary() + { + return stringDictionary.ToDictionary( + kv => (Value)kv.Key, + kv => (Value)kv.Value + ); + } + } +} + +public static class PlatformInfoVariableExtensions +{ + extension(PlatformInfo platform) + { + public Dictionary PlatformSpecificTemplateVariables => new() + { + { "ARCH_SHORT", platform.Model.Architecture.ShortName }, + { "ARCH_NUPKG", platform.Model.Architecture.NupkgName }, + { "ARCH_VERSIONED", platform.ArchWithVariant }, + { "ARCH_TAG_SUFFIX", $"-{platform.ArchWithVariant}" }, + { "PRODUCT_VERSION", platform.Image.ProductVersion ?? "" }, + { "OS_VERSION", platform.Model.OsVersion }, + { "OS_VERSION_BASE", platform.BaseOsVersion }, + { "OS_VERSION_NUMBER", platform.GetOsVersionNumber() }, + { "OS_ARCH_HYPHENATED", platform.GetOsArchHyphenatedName() }, + }; + } +} + +public static class ManifestInfoVariableExtensions +{ + extension(ManifestInfo manifest) + { + public Dictionary TemplateVariables => new() + { + { "IS_PRODUCT_FAMILY", true.ToString() }, + }; + } +} + +public static class RepoInfoVariableExtensions +{ + extension(RepoInfo repo) + { + public Dictionary TemplateVariables => new() + { + { "REPO", repo.Model.Name }, + { "FULL_REPO", repo.FullName }, + { "PARENT_REPO", repo.GetParentRepoName() }, + { "SHORT_REPO", repo.ShortName }, + }; + + private string ShortName => + // LastIndexOf returns -1 when not found, so in the case the repo + // name doesn't have any slashes, (-1 + 1) becomes 0 which selects + // the whole string. + repo.Model.Name[(repo.Model.Name.LastIndexOf('/') + 1)..]; + + private string GetParentRepoName() + { + // Avoid using string.Split(...) to prevent array allocation. + var name = repo.Model.Name; + int last = name.LastIndexOf('/'); + if (last <= 0) + { + return string.Empty; + } + + int prev = name.LastIndexOf('/', last - 1); + return name[(prev + 1)..last]; + } + } +} + +internal static partial class PlatformInfoExtensions +{ + extension(PlatformInfo platform) + { + public string ArchWithVariant => platform.Model.Architecture.LongName + platform.ArchVariant; + public string ArchVariant => platform.Model.Variant?.ToLowerInvariant() ?? ""; + public string BaseOsVersion => platform.Model.OsVersion.TrimEndString("-slim"); + + public string GetOsVersionNumber() + { + const string PrefixGroup = "Prefix"; + const string VersionGroup = "Version"; + const string LtscPrefix = "ltsc"; + Match match = OsVersionRegex.Match(platform.Model.OsVersion); + + string versionNumber = string.Empty; + if (match.Groups[PrefixGroup].Success && match.Groups[PrefixGroup].Value == LtscPrefix) + { + versionNumber = LtscPrefix; + } + + versionNumber += match.Groups[VersionGroup].Value; + return versionNumber; + } + + public string GetOsArchHyphenatedName() + { + string osName; + if (platform.BaseOsVersion.Contains("nanoserver")) + { + string version = platform.BaseOsVersion.Split('-')[1]; + osName = $"NanoServer-{version}"; + } + else if (platform.BaseOsVersion.Contains("windowsservercore")) + { + string version = platform.BaseOsVersion.Split('-')[1]; + osName = $"WindowsServerCore-{version}"; + } + else + { + osName = platform.OSDisplayName.Replace(' ', '-'); + } + + string archName = platform.Model.Architecture != Architecture.AMD64 + ? $"-{platform.Model.Architecture.GetDisplayName()}" + : string.Empty; + + return osName + archName; + } + } + + extension(Architecture architecture) + { + public string ShortName => architecture switch + { + Architecture.AMD64 => "x64", + _ => architecture.ToString().ToLowerInvariant(), + }; + + public string NupkgName => architecture switch + { + Architecture.AMD64 => "x64", + Architecture.ARM => "arm32", + _ => architecture.ToString().ToLowerInvariant(), + }; + + public string LongName => architecture switch + { + Architecture.ARM => "arm32", + _ => architecture.ToString().ToLowerInvariant(), + }; + + public string DockerName => architecture.ToString().ToLowerInvariant(); + } + + extension(OS os) + { + public string DockerName => os.ToString().ToLowerInvariant(); + } + + [GeneratedRegex(@"(-(?[a-zA-Z_]*))?(?\d+.\d+)")] + private static partial Regex OsVersionRegex { get; } +} diff --git a/src/Templating/Cottle/CottleTemplate.cs b/src/Templating/Cottle/CottleTemplate.cs new file mode 100644 index 000000000..109c355d6 --- /dev/null +++ b/src/Templating/Cottle/CottleTemplate.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Cottle; +using Microsoft.DotNet.DockerTools.Templating.Abstractions; + +namespace Microsoft.DotNet.DockerTools.Templating.Cottle; + +public sealed class CottleTemplate(IDocument document) : ICompiledTemplate +{ + private readonly IDocument _document = document; + + public string Render(IContext context, bool trim = false, string indent = "") + { + var content = _document.Render(context); + + if (trim) + { + content = content.Trim(); + } + + if (!string.IsNullOrEmpty(indent)) + { + content = content.Replace("\n", "\n" + indent); + } + + return content; + } +} diff --git a/src/Templating/Cottle/CottleTemplateEngine.cs b/src/Templating/Cottle/CottleTemplateEngine.cs new file mode 100644 index 000000000..395cc2f5b --- /dev/null +++ b/src/Templating/Cottle/CottleTemplateEngine.cs @@ -0,0 +1,120 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Cottle; +using Microsoft.DotNet.DockerTools.Templating.Abstractions; + +namespace Microsoft.DotNet.DockerTools.Templating.Cottle; + +public sealed class CottleTemplateEngine : ITemplateEngine +{ + private static readonly DocumentConfiguration s_config = new() + { + BlockBegin = "{{", + BlockContinue = "^", + BlockEnd = "}}", + Escape = '@', + Trimmer = DocumentConfiguration.TrimNothing + }; + + private readonly IFileSystem _fileSystem; + private readonly ForeverCache _templateCache; + + private IContext _globalContext = Context.CreateBuiltin( + new Dictionary() + { + { "replace", ReplaceFunction } + } + ); + + public CottleTemplateEngine(IFileSystem fileSystem) + { + _fileSystem = fileSystem; + _templateCache = new ForeverCache(valueFactory: ReadAndCompileWithNoCache); + } + + public int CompiledTemplateCacheHits => _templateCache.Hits; + public int CompiledTemplateCacheMisses => _templateCache.Misses; + + public ICompiledTemplate ReadAndCompile(string path) + { + return _templateCache.GetOrAdd(path); + } + + public void AddGlobalVariables(IDictionary variables) + { + var variableSymbols = new Dictionary + { + { "VARIABLES", variables.ToCottleDictionary() } + }; + + _globalContext = _globalContext.Add(variableSymbols); + } + + /// + public IContext CreateContext(IDictionary variables, string templatePath) + { + var variableSymbols = variables.ToCottleDictionary(); + var variableContext = Context.CreateCustom(variableSymbols); + var newContext = Context.CreateCascade(primary: variableContext, fallback: _globalContext); + + // It's OK for the insert template function not to have a reference to itself. Any sub-templates will have + // their own InsertTemplate function created for them when they are rendered. + var insertTemplateFunction = CreateInsertTemplateFunction(newContext, templatePath); + newContext = newContext.Add("InsertTemplate", insertTemplateFunction); + + return newContext; + } + + private Value CreateInsertTemplateFunction(IContext platformContext, string currentTemplatePath) + { + var function = Function.CreatePure( + (state, args) => + { + // Resolve arguments to InsertTemplate + var templateRelativePath = args[0].AsString; + var templateArgs = args.Count > 1 ? args[1] : Value.EmptyMap; + var indent = args.Count > 2 ? args[2].AsString : ""; + + // Resolve the path of the sub-template to be inserted, relative to the current template + var parentTemplateDir = Path.GetDirectoryName(currentTemplatePath) ?? string.Empty; + var newTemplatePath = Path.Combine(parentTemplateDir, templateRelativePath); + var compiledTemplate = ReadAndCompile(newTemplatePath); + + var newSymbols = new Dictionary + { + { "InsertTemplate", CreateInsertTemplateFunction(platformContext, newTemplatePath) }, + { "ARGS", new Dictionary(templateArgs.Fields) }, + }; + + var newContext = platformContext.Add(newSymbols); + return compiledTemplate.Render(newContext, trim: true, indent: indent); + } + ); + + return Value.FromFunction(function); + } + + private CottleTemplate ReadAndCompileWithNoCache(string path) + { + string content = _fileSystem.ReadAllText(path); + var documentResult = Document.CreateDefault(content, s_config); + var document = documentResult.DocumentOrThrow; + var compiledTemplate = new CottleTemplate(document); + return compiledTemplate; + } + + private static Value ReplaceFunction = Value.FromFunction( + Function.CreatePure( + (state, args) => + { + string source = args[0].AsString; + string oldValue = args[1].AsString; + string newValue = args[2].AsString; + return Value.FromString(source.Replace(oldValue, newValue)); + }, + min: 3, + max: 3 + ) + ); +} diff --git a/src/Templating/FileSystem.cs b/src/Templating/FileSystem.cs new file mode 100644 index 000000000..1559e9a4c --- /dev/null +++ b/src/Templating/FileSystem.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using Microsoft.DotNet.DockerTools.Templating.Abstractions; + +namespace Microsoft.DotNet.DockerTools.Templating; + +/// +/// General purpose synchronous file system implementation that tracks simple +/// metrics about reads and writes. +/// +public sealed class FileSystem : IFileSystem +{ + private int _reads = 0; + private int _bytesRead = 0; + private int _writes = 0; + private int _bytesWritten = 0; + + public int FilesRead => _reads; + public int BytesRead => _bytesRead; + public int FilesWritten => _writes; + public int BytesWritten => _bytesWritten; + + public string ReadAllText(string path) + { + string content = File.ReadAllText(path); + + _bytesRead += GetBytes(content); + _reads += 1; + + return content; + } + + public void WriteAllText(string path, string content) + { + File.WriteAllText(path, content); + + _bytesWritten += GetBytes(content); + _writes += 1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int GetBytes(string content) => System.Text.Encoding.UTF8.GetByteCount(content); +} diff --git a/src/Templating/FileSystemCache.cs b/src/Templating/FileSystemCache.cs new file mode 100644 index 000000000..fd11b3600 --- /dev/null +++ b/src/Templating/FileSystemCache.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.DockerTools.Templating.Abstractions; + +namespace Microsoft.DotNet.DockerTools.Templating; + +public sealed class FileSystemCache : IFileSystem +{ + private readonly IFileSystem _fileSystem; + private readonly ICache _cache; + + public int CacheHits => _cache.Hits; + public int CacheMisses => _cache.Misses; + + public FileSystemCache(IFileSystem fileSystem) + { + _fileSystem = fileSystem; + _cache = new ForeverCache(key => _fileSystem.ReadAllText(key)); + } + + public string ReadAllText(string path) + { + var content = _cache.GetOrAdd(path); + return content; + } + + public void WriteAllText(string path, string content) + { + _fileSystem.WriteAllText(path, content); + } +} diff --git a/src/Templating/ForeverCache.cs b/src/Templating/ForeverCache.cs new file mode 100644 index 000000000..4bb9d604b --- /dev/null +++ b/src/Templating/ForeverCache.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.DockerTools.Templating; + +/// +/// A cache that retains values for the lifetime of the object. +/// +public sealed class ForeverCache(Func valueFactory) : ICache +{ + private readonly Func _valueFactory = valueFactory; + private readonly Dictionary _cache = []; + + /// + /// Number of times a cached value was returned. + /// + public int Hits { get; private set; } = 0; + + /// + /// Number of times a new value was created and added to the cache. + /// + public int Misses { get; private set; } = 0; + + public T GetOrAdd(string key) + { + if (!_cache.TryGetValue(key, out T? value)) + { + value = _valueFactory(key); + _cache[key] = value; + Misses += 1; + } + else + { + Hits += 1; + } + + return value; + } +} diff --git a/src/Templating/ICache.cs b/src/Templating/ICache.cs new file mode 100644 index 000000000..e487fd0a6 --- /dev/null +++ b/src/Templating/ICache.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.DockerTools.Templating; + +public interface ICache +{ + int Hits { get; } + int Misses { get; } + + T GetOrAdd(string key); +} diff --git a/src/Templating/Readmes/MarkdownTableBuilder.cs b/src/Templating/Readmes/MarkdownTableBuilder.cs new file mode 100644 index 000000000..ef6ca1ca2 --- /dev/null +++ b/src/Templating/Readmes/MarkdownTableBuilder.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text; +using Microsoft.DotNet.DockerTools.Templating.Abstractions; +using Microsoft.DotNet.DockerTools.Templating.Shared; +using Microsoft.DotNet.ImageBuilder.ReadModel; + +namespace Microsoft.DotNet.DockerTools.Templating.Readmes; + +internal sealed class MarkdownTableBuilder : ITableBuilder +{ + private readonly List _headings = []; + private readonly List> _rows = []; + + public ITableBuilder WithColumnHeadings(params IEnumerable headings) + { + _headings.Clear(); + _headings.AddRange(headings); + return this; + } + + public void AddRow(params IEnumerable row) => _rows.Add(row); + + public override string ToString() + { + var table = new StringBuilder(); + + if (_headings.Count > 0) + { + AppendRow(table, _headings); + table.AppendLine(); + + AppendRow(table, _headings.Select(_ => "---")); + table.AppendLine(); + } + + foreach (var row in _rows.WithIndex()) + { + AppendRow(table, row.Item); + + // Put lines between rows but not after the last row + if (row.Index != _rows.Count - 1) table.AppendLine(); + } + + return table.ToString(); + } + + private static StringBuilder AppendRow(StringBuilder stringBuilder, IEnumerable cells) => + stringBuilder.Append("| ").AppendJoin(" | ", cells).Append(" |"); +} diff --git a/src/Templating/Readmes/TagExtensions.cs b/src/Templating/Readmes/TagExtensions.cs new file mode 100644 index 000000000..1445f75ee --- /dev/null +++ b/src/Templating/Readmes/TagExtensions.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.ImageBuilder.Models.Manifest; + +namespace Microsoft.DotNet.DockerTools.Templating.Readmes; + +internal static class TagExtensions +{ + extension(Tag tag) + { + public bool IsDocumented => tag.DocType switch + { + TagDocumentationType.Undocumented => false, + _ => true + }; + } +} diff --git a/src/Templating/Readmes/TagsTableGenerator.cs b/src/Templating/Readmes/TagsTableGenerator.cs new file mode 100644 index 000000000..5d8938b5b --- /dev/null +++ b/src/Templating/Readmes/TagsTableGenerator.cs @@ -0,0 +1,60 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text; +using Microsoft.DotNet.DockerTools.Templating.Shared; +using Microsoft.DotNet.ImageBuilder.ReadModel; + +namespace Microsoft.DotNet.DockerTools.Templating.Readmes; + +public static class TagsTableGenerator +{ + public static string GenerateTagsTables(RepoInfo repo) + { + var output = new StringBuilder(); + + var documentedPlatforms = repo.Images + .SelectMany(image => image.Platforms) + .Where(platform => platform.Tags + .Any(tag => tag.IsDocumented)); + + var platformsByOsArch = documentedPlatforms + .GroupBy(platform => (platform.Model.OS, platform.Model.Architecture)); + + foreach (var archGroup in platformsByOsArch) + { + var os = archGroup.Key.OS.ToString(); + var arch = archGroup.Key.Architecture.GetDisplayName(); + + output.AppendLine($""" + + ### {os} {arch} Tags + + """); + + output.AppendLine(GeneratePlatformsTable(archGroup)); + } + + return output.ToString(); + } + + private static string GeneratePlatformsTable(IEnumerable platforms) + { + var table = new MarkdownTableBuilder() + .WithColumnHeadings("Tags", "Dockerfile", "OS Version"); + + foreach (var platform in platforms) + { + var tags = platform.Tags + .Where(tag => tag.IsDocumented) + .Select(tag => tag.Tag); + + table.AddRow( + string.Join(", ", tags), + platform.RelativeDockerfilePath, + platform.OSDisplayName); + } + + return table.ToString(); + } +} diff --git a/src/Templating/Shared/ArchDisplayExtensions.cs b/src/Templating/Shared/ArchDisplayExtensions.cs new file mode 100644 index 000000000..436fa4d48 --- /dev/null +++ b/src/Templating/Shared/ArchDisplayExtensions.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.ImageBuilder.Models.Manifest; + +namespace Microsoft.DotNet.DockerTools.Templating.Shared; + +internal static class ArchDisplayExtensions +{ + extension(Architecture architecture) + { + public string GetDisplayName(string? variant = null) + { + string displayName = architecture switch + { + Architecture.ARM => "arm32", + _ => architecture.ToString().ToLowerInvariant(), + }; + + if (variant != null) + { + displayName += variant.ToLowerInvariant(); + } + + return displayName; + } + + } +} diff --git a/src/Templating/Shared/OsHelper.cs b/src/Templating/Shared/OsHelper.cs new file mode 100644 index 000000000..7fa07c221 --- /dev/null +++ b/src/Templating/Shared/OsHelper.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.RegularExpressions; + +namespace Microsoft.DotNet.DockerTools.Templating.Shared; + +internal static class OsHelper +{ + public static string GetWindowsOSDisplayName(string osName) + { + string version = osName.Split('-')[1]; + return osName switch + { + var s when s.StartsWith("nanoserver") => + GetWindowsVersionDisplayName("Nano Server", version), + var s when s.StartsWith("windowsservercore") => + GetWindowsVersionDisplayName("Windows Server Core", version), + _ => throw new NotSupportedException($"The OS version '{osName}' is not supported.") + }; + } + + public static string GetLinuxOSDisplayName(string osName) => osName switch + { + string s when s.Contains("debian") => "Debian", + string s when s.Contains("bookworm") => "Debian 12", + string s when s.Contains("trixie") => "Debian 13", + string s when s.Contains("forky") => "Debian 14", + string s when s.Contains("duke") => "Debian 15", + string s when s.Contains("jammy") => "Ubuntu 22.04", + string s when s.Contains("noble") => "Ubuntu 24.04", + string s when s.Contains("azurelinux") => FormatVersionableOsName(osName, name => "Azure Linux"), + string s when s.Contains("cbl-mariner") => FormatVersionableOsName(osName, name => "CBL-Mariner"), + string s when s.Contains("leap") => FormatVersionableOsName(osName, name => "openSUSE Leap"), + string s when s.Contains("ubuntu") => FormatVersionableOsName(osName, name => "Ubuntu"), + string s when s.Contains("alpine") + || s.Contains("centos") + || s.Contains("fedora") => FormatVersionableOsName(osName, name => name.FirstCharToUpper()), + _ => throw new NotSupportedException($"The OS version '{osName}' is not supported.") + }; + + private static string GetWindowsVersionDisplayName(string windowsName, string version) => + version.StartsWith("ltsc") switch + { + true => $"{windowsName} {version.TrimStartString("ltsc")}", + false => $"{windowsName}, version {version}" + }; + + private static string FormatVersionableOsName(string os, Func formatName) + { + (string osName, string osVersion) = GetOsVersionInfo(os); + if (string.IsNullOrEmpty(osVersion)) + { + return formatName(osName); + } + else + { + return $"{formatName(osName)} {osVersion}"; + } + } + + private static (string Name, string Version) GetOsVersionInfo(string os) + { + // Regex matches an os name ending in a non-numeric or decimal character and up to + // a 3 part version number. Any additional characters are dropped (e.g. -distroless). + Regex versionRegex = new Regex(@"(?.+[^0-9\.])(?\d+(\.\d*){0,2})"); + Match match = versionRegex.Match(os); + + if (match.Success) + { + return (match.Groups["name"].Value, match.Groups["version"].Value); + } + else + { + return (os, string.Empty); + } + } +} diff --git a/src/Templating/Shared/PlatformInfoSharedExtensions.cs b/src/Templating/Shared/PlatformInfoSharedExtensions.cs new file mode 100644 index 000000000..6bd3ddbfa --- /dev/null +++ b/src/Templating/Shared/PlatformInfoSharedExtensions.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.ImageBuilder.ReadModel; +using Microsoft.DotNet.ImageBuilder.Models.Manifest; +using Microsoft.DotNet.DockerTools.Templating.Cottle; + +namespace Microsoft.DotNet.DockerTools.Templating.Shared; + +internal static class PlatformInfoSharedExtensions +{ + extension(PlatformInfo platform) + { + public string OSDisplayName => platform.Model.OS switch + { + OS.Windows => OsHelper.GetWindowsOSDisplayName(platform.BaseOsVersion), + _ => OsHelper.GetLinuxOSDisplayName(platform.BaseOsVersion) + }; + } +} diff --git a/src/Templating/Shared/StringExtensions.cs b/src/Templating/Shared/StringExtensions.cs new file mode 100644 index 000000000..1a84bdd09 --- /dev/null +++ b/src/Templating/Shared/StringExtensions.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.DotNet.DockerTools.Templating.Shared; + +internal static class StringExtensions +{ + public static string FirstCharToUpper(this string source) => char.ToUpper(source[0]) + source.Substring(1); + + [return: NotNullIfNotNull(nameof(source))] + public static string? TrimStartString(this string? source, string trim) => source switch + { + string s when s.StartsWith(trim) => s.Substring(trim.Length).TrimStartString(trim), + _ => source, + }; + + [return: NotNullIfNotNull(nameof(source))] + public static string? TrimEndString(this string? source, string trim) => source switch + { + string s when s.EndsWith(trim) => s.Substring(0, s.Length - trim.Length).TrimEndString(trim), + _ => source, + }; +} + +internal static class EnumerableExtensions +{ + extension(IEnumerable source) + { + public IEnumerable<(T Item, int Index)> WithIndex() => + source.Select((item, index) => (item, index)); + } +} diff --git a/src/Templating/Templating.csproj b/src/Templating/Templating.csproj new file mode 100644 index 000000000..8171c2951 --- /dev/null +++ b/src/Templating/Templating.csproj @@ -0,0 +1,19 @@ + + + + net10.0 + enable + enable + Microsoft.DotNet.DockerTools.Templating + + + + + + + + + + + +