diff --git a/src/spec-common/injectHeadless.ts b/src/spec-common/injectHeadless.ts index d81947162..344914537 100644 --- a/src/spec-common/injectHeadless.ts +++ b/src/spec-common/injectHeadless.ts @@ -60,6 +60,7 @@ export interface ResolverParameters { buildxPush: boolean; skipFeatureAutoMapping: boolean; skipPostAttach: boolean; + experimentalImageMetadata: boolean; } export interface PostCreate { @@ -125,6 +126,37 @@ export interface CommonDevContainerConfig { userEnvProbe?: UserEnvProbe; } +export interface CommonContainerMetadata { + onCreateCommand?: string | string[]; + updateContentCommand?: string | string[]; + postCreateCommand?: string | string[]; + postStartCommand?: string | string[]; + postAttachCommand?: string | string[]; + waitFor?: DevContainerConfigCommand; + remoteEnv?: Record; + userEnvProbe?: UserEnvProbe; +} + +export type CommonMergedDevContainerConfig = MergedConfig; + +type MergedConfig = Omit & UpdatedConfigProperties; + +const replaceProperties = [ + 'onCreateCommand', + 'updateContentCommand', + 'postCreateCommand', + 'postStartCommand', + 'postAttachCommand', +] as const; + +interface UpdatedConfigProperties { + onCreateCommands?: (string | string[])[]; + updateContentCommands?: (string | string[])[]; + postCreateCommands?: (string | string[])[]; + postStartCommands?: (string | string[])[]; + postAttachCommands?: (string | string[])[]; +} + export interface OSRelease { hardware: string; id: string; @@ -262,7 +294,7 @@ export function getSystemVarFolder(params: ResolverParameters): string { return params.containerSystemDataFolder || '/var/devcontainer'; } -export async function setupInContainer(params: ResolverParameters, containerProperties: ContainerProperties, config: CommonDevContainerConfig) { +export async function setupInContainer(params: ResolverParameters, containerProperties: ContainerProperties, config: CommonMergedDevContainerConfig) { await patchEtcEnvironment(params, containerProperties); await patchEtcProfile(params, containerProperties); const computeRemoteEnv = params.computeExtensionHostEnv || params.postCreate.enabled; @@ -276,7 +308,7 @@ export async function setupInContainer(params: ResolverParameters, containerProp }; } -export function probeRemoteEnv(params: ResolverParameters, containerProperties: ContainerProperties, config: CommonDevContainerConfig) { +export function probeRemoteEnv(params: ResolverParameters, containerProperties: ContainerProperties, config: CommonMergedDevContainerConfig) { return probeUserEnv(params, containerProperties, config) .then>(shellEnv => ({ ...shellEnv, @@ -285,7 +317,7 @@ export function probeRemoteEnv(params: ResolverParameters, containerProperties: } as Record)); } -export async function runPostCreateCommands(params: ResolverParameters, containerProperties: ContainerProperties, config: CommonDevContainerConfig, remoteEnv: Promise>, stopForPersonalization: boolean): Promise<'skipNonBlocking' | 'prebuild' | 'stopForPersonalization' | 'done'> { +export async function runPostCreateCommands(params: ResolverParameters, containerProperties: ContainerProperties, config: CommonMergedDevContainerConfig, remoteEnv: Promise>, stopForPersonalization: boolean): Promise<'skipNonBlocking' | 'prebuild' | 'stopForPersonalization' | 'done'> { const skipNonBlocking = params.postCreate.skipNonBlocking; const waitFor = config.waitFor || defaultWaitFor; if (skipNonBlocking && waitFor === 'initializeCommand') { @@ -342,16 +374,16 @@ export async function getOSRelease(shellServer: ShellServer) { return { hardware, id, version }; } -async function runPostCreateCommand(params: ResolverParameters, containerProperties: ContainerProperties, config: CommonDevContainerConfig, postCommandName: 'onCreateCommand' | 'updateContentCommand' | 'postCreateCommand', remoteEnv: Promise>, rerun: boolean) { +async function runPostCreateCommand(params: ResolverParameters, containerProperties: ContainerProperties, config: CommonMergedDevContainerConfig, postCommandName: 'onCreateCommand' | 'updateContentCommand' | 'postCreateCommand', remoteEnv: Promise>, rerun: boolean) { const markerFile = path.posix.join(containerProperties.userDataFolder, `.${postCommandName}Marker`); const doRun = !!containerProperties.createdAt && await updateMarkerFile(containerProperties.shellServer, markerFile, containerProperties.createdAt) || rerun; - await runPostCommand(params, containerProperties, config, postCommandName, remoteEnv, doRun); + await runPostCommands(params, containerProperties, config, postCommandName, remoteEnv, doRun); } -async function runPostStartCommand(params: ResolverParameters, containerProperties: ContainerProperties, config: CommonDevContainerConfig, remoteEnv: Promise>) { +async function runPostStartCommand(params: ResolverParameters, containerProperties: ContainerProperties, config: CommonMergedDevContainerConfig, remoteEnv: Promise>) { const markerFile = path.posix.join(containerProperties.userDataFolder, '.postStartCommandMarker'); const doRun = !!containerProperties.startedAt && await updateMarkerFile(containerProperties.shellServer, markerFile, containerProperties.startedAt); - await runPostCommand(params, containerProperties, config, 'postStartCommand', remoteEnv, doRun); + await runPostCommands(params, containerProperties, config, 'postStartCommand', remoteEnv, doRun); } async function updateMarkerFile(shellServer: ShellServer, location: string, content: string) { @@ -363,12 +395,15 @@ async function updateMarkerFile(shellServer: ShellServer, location: string, cont } } -async function runPostAttachCommand(params: ResolverParameters, containerProperties: ContainerProperties, config: CommonDevContainerConfig, remoteEnv: Promise>) { - await runPostCommand(params, containerProperties, config, 'postAttachCommand', remoteEnv, true); +async function runPostAttachCommand(params: ResolverParameters, containerProperties: ContainerProperties, config: CommonMergedDevContainerConfig, remoteEnv: Promise>) { + await runPostCommands(params, containerProperties, config, 'postAttachCommand', remoteEnv, true); +} + +async function runPostCommands(params: ResolverParameters, containerProperties: ContainerProperties, config: CommonMergedDevContainerConfig, postCommandName: 'onCreateCommand' | 'updateContentCommand' | 'postCreateCommand' | 'postStartCommand' | 'postAttachCommand', remoteEnv: Promise>, doRun: boolean) { + return Promise.all((config[`${postCommandName}s`] || []).map(config => runPostCommand(params, containerProperties, config, postCommandName, remoteEnv, doRun))); } -async function runPostCommand({ postCreate }: ResolverParameters, containerProperties: ContainerProperties, config: CommonDevContainerConfig, postCommandName: 'onCreateCommand' | 'updateContentCommand' | 'postCreateCommand' | 'postStartCommand' | 'postAttachCommand', remoteEnv: Promise>, doRun: boolean) { - const postCommand = config[postCommandName]; +async function runPostCommand({ postCreate }: ResolverParameters, containerProperties: ContainerProperties, postCommand: string | string[], postCommandName: 'onCreateCommand' | 'updateContentCommand' | 'postCreateCommand' | 'postStartCommand' | 'postAttachCommand', remoteEnv: Promise>, doRun: boolean) { if (doRun && postCommand && (typeof postCommand === 'string' ? postCommand.trim() : postCommand.length)) { const progressName = `Running ${postCommandName}...`; const progressDetail = typeof postCommand === 'string' ? postCommand : postCommand.join(' '); @@ -585,7 +620,7 @@ async function patchEtcProfile(params: ResolverParameters, containerProperties: } } -async function probeUserEnv(params: { defaultUserEnvProbe: UserEnvProbe; allowSystemConfigChange: boolean; output: Log }, containerProperties: { shell: string; remoteExec: ExecFunction; installFolder?: string; env?: NodeJS.ProcessEnv; shellServer?: ShellServer; launchRootShellServer?: (() => Promise); user?: string }, config?: { userEnvProbe?: UserEnvProbe }) { +async function probeUserEnv(params: { defaultUserEnvProbe: UserEnvProbe; allowSystemConfigChange: boolean; output: Log }, containerProperties: { shell: string; remoteExec: ExecFunction; installFolder?: string; env?: NodeJS.ProcessEnv; shellServer?: ShellServer; launchRootShellServer?: (() => Promise); user?: string }, config?: CommonMergedDevContainerConfig) { const env = await runUserEnvProbe(params, containerProperties, config, 'cat /proc/self/environ', '\0'); if (env) { return env; @@ -595,8 +630,8 @@ async function probeUserEnv(params: { defaultUserEnvProbe: UserEnvProbe; allowSy return env2 || {}; } -async function runUserEnvProbe(params: { defaultUserEnvProbe: UserEnvProbe; allowSystemConfigChange: boolean; output: Log }, containerProperties: { shell: string; remoteExec: ExecFunction; installFolder?: string; env?: NodeJS.ProcessEnv; shellServer?: ShellServer; launchRootShellServer?: (() => Promise); user?: string }, config: { userEnvProbe?: UserEnvProbe } | undefined, cmd: string, sep: string) { - let { userEnvProbe } = config || {}; +async function runUserEnvProbe(params: { defaultUserEnvProbe: UserEnvProbe; allowSystemConfigChange: boolean; output: Log }, containerProperties: { shell: string; remoteExec: ExecFunction; installFolder?: string; env?: NodeJS.ProcessEnv; shellServer?: ShellServer; launchRootShellServer?: (() => Promise); user?: string }, config: CommonMergedDevContainerConfig | undefined, cmd: string, sep: string) { + let userEnvProbe = config?.userEnvProbe; params.output.write(`userEnvProbe: ${userEnvProbe || params.defaultUserEnvProbe}${userEnvProbe ? '' : ' (default)'}`); if (!userEnvProbe) { userEnvProbe = params.defaultUserEnvProbe; diff --git a/src/spec-configuration/configuration.ts b/src/spec-configuration/configuration.ts index 4d4c4fc6d..18ae74eea 100644 --- a/src/spec-configuration/configuration.ts +++ b/src/spec-configuration/configuration.ts @@ -6,6 +6,7 @@ import * as path from 'path'; import { URI } from 'vscode-uri'; import { FileHost, parentURI, uriToFsPath } from './configurationCommonUtils'; +import { Mount } from './containerFeaturesConfiguration'; import { RemoteDocuments } from './editableFiles'; export type DevContainerConfig = DevContainerFromImageConfig | DevContainerFromDockerfileConfig | DevContainerFromDockerComposeConfig; @@ -52,16 +53,21 @@ export interface DevContainerFromImageConfig { /** remote path to folder or workspace */ workspaceFolder?: string; workspaceMount?: string; - mounts?: string[]; + mounts?: (Mount | string)[]; containerEnv?: Record; - remoteEnv?: Record; containerUser?: string; + init?: boolean; + privileged?: boolean; + capAdd?: string[]; + securityOpt?: string[]; + remoteEnv?: Record; remoteUser?: string; updateRemoteUserUID?: boolean; userEnvProbe?: UserEnvProbe; features?: Record>; overrideFeatureInstallOrder?: string[]; hostRequirements?: HostRequirements; + customizations?: Record; } export type DevContainerFromDockerfileConfig = { @@ -84,16 +90,21 @@ export type DevContainerFromDockerfileConfig = { /** remote path to folder or workspace */ workspaceFolder?: string; workspaceMount?: string; - mounts?: string[]; + mounts?: (Mount | string)[]; containerEnv?: Record; - remoteEnv?: Record; containerUser?: string; + init?: boolean; + privileged?: boolean; + capAdd?: string[]; + securityOpt?: string[]; + remoteEnv?: Record; remoteUser?: string; updateRemoteUserUID?: boolean; userEnvProbe?: UserEnvProbe; features?: Record>; overrideFeatureInstallOrder?: string[]; hostRequirements?: HostRequirements; + customizations?: Record; } & ( { dockerFile: string; @@ -135,6 +146,13 @@ export interface DevContainerFromDockerComposeConfig { postAttachCommand?: string | string[]; waitFor?: DevContainerConfigCommand; runServices?: string[]; + mounts?: (Mount | string)[]; + containerEnv?: Record; + containerUser?: string; + init?: boolean; + privileged?: boolean; + capAdd?: string[]; + securityOpt?: string[]; remoteEnv?: Record; remoteUser?: string; updateRemoteUserUID?: boolean; @@ -142,6 +160,7 @@ export interface DevContainerFromDockerComposeConfig { features?: Record>; overrideFeatureInstallOrder?: string[]; hostRequirements?: HostRequirements; + customizations?: Record; } interface DevContainerVSCodeConfig { diff --git a/src/spec-configuration/containerFeaturesConfiguration.ts b/src/spec-configuration/containerFeaturesConfiguration.ts index c76887680..84a9b89fa 100644 --- a/src/spec-configuration/containerFeaturesConfiguration.ts +++ b/src/spec-configuration/containerFeaturesConfiguration.ts @@ -70,6 +70,12 @@ export interface Mount { external?: boolean; } +export function parseMount(str: string): Mount { + return str.split(',') + .map(s => s.split('=')) + .reduce((acc, [key, value]) => ({ ...acc, [key]: value }), {}) as Mount; +} + export type SourceInformation = LocalCacheSourceInformation | GithubSourceInformation | DirectTarballSourceInformation | FilePathSourceInformation | OCISourceInformation; interface BaseSourceInformation { @@ -221,6 +227,8 @@ COPY --from=dev_containers_feature_content_source {contentSourceRootPath} /tmp/b ARG _DEV_CONTAINERS_IMAGE_USER=root USER $_DEV_CONTAINERS_IMAGE_USER + +#{devcontainerMetadata} `; } diff --git a/src/spec-node/configContainer.ts b/src/spec-node/configContainer.ts index 0f3084c2a..593f7bb9b 100644 --- a/src/spec-node/configContainer.ts +++ b/src/spec-node/configContainer.ts @@ -9,7 +9,7 @@ import * as jsonc from 'jsonc-parser'; import { openDockerfileDevContainer } from './singleContainer'; import { openDockerComposeDevContainer } from './dockerCompose'; -import { ResolverResult, DockerResolverParameters, isDockerFileConfig, runUserCommand, createDocuments, getWorkspaceConfiguration, BindMountConsistency, uriToFsPath, DevContainerAuthority, isDevContainerAuthority } from './utils'; +import { ResolverResult, DockerResolverParameters, isDockerFileConfig, runUserCommand, createDocuments, getWorkspaceConfiguration, BindMountConsistency, uriToFsPath, DevContainerAuthority, isDevContainerAuthority, SubstituteConfig, SubstitutedConfig } from './utils'; import { substitute } from '../spec-common/variableSubstitution'; import { ContainerError } from '../spec-common/errors'; import { Workspace, workspaceFromPath, isWorkspacePath } from '../spec-utils/workspaces'; @@ -17,7 +17,7 @@ import { URI } from 'vscode-uri'; import { CLIHost } from '../spec-common/commonUtils'; import { Log } from '../spec-utils/log'; import { getDefaultDevContainerConfigPath, getDevContainerConfigPathIn } from '../spec-configuration/configurationCommonUtils'; -import { DevContainerConfig, updateFromOldProperties } from '../spec-configuration/configuration'; +import { DevContainerConfig, DevContainerFromDockerComposeConfig, DevContainerFromDockerfileConfig, DevContainerFromImageConfig, updateFromOldProperties } from '../spec-configuration/configuration'; export { getWellKnownDevContainerPaths as getPossibleDevContainerPaths } from '../spec-configuration/configurationCommonUtils'; @@ -52,18 +52,19 @@ async function resolveWithLocalFolder(params: DockerResolverParameters, parsedAu throw new ContainerError({ description: `No dev container config and no workspace found.` }); } } - const config = configs.config; + const configWithRaw = configs.config; + const { config } = configWithRaw; await runUserCommand({ ...params, common: { ...common, output: common.postCreate.output } }, config.initializeCommand, common.postCreate.onDidInput); let result: ResolverResult; if (isDockerFileConfig(config) || 'image' in config) { - result = await openDockerfileDevContainer(params, config, configs.workspaceConfig, idLabels); + result = await openDockerfileDevContainer(params, configWithRaw as SubstitutedConfig, configs.workspaceConfig, idLabels); } else if ('dockerComposeFile' in config) { if (!workspace) { throw new ContainerError({ description: `A Dev Container using Docker Compose requires a workspace folder.` }); } - result = await openDockerComposeDevContainer(params, workspace, config, idLabels); + result = await openDockerComposeDevContainer(params, workspace, configWithRaw as SubstitutedConfig, idLabels); } else { throw new ContainerError({ description: `Dev container config (${(config as DevContainerConfig).configFilePath}) is missing one of "image", "dockerFile" or "dockerComposeFile" properties.` }); } @@ -82,13 +83,14 @@ export async function readDevContainerConfigFile(cliHost: CLIHost, workspace: Wo throw new ContainerError({ description: `Dev container config (${uriToFsPath(configFile, cliHost.platform)}) must contain a JSON object literal.` }); } const workspaceConfig = await getWorkspaceConfiguration(cliHost, workspace, updated, mountWorkspaceGitRoot, output, consistency); - const config: DevContainerConfig = substitute({ + const substitute0: SubstituteConfig = value => substitute({ platform: cliHost.platform, localWorkspaceFolder: workspace?.rootFolderPath, containerWorkspaceFolder: workspaceConfig.workspaceFolder, configFile, env: cliHost.env, - }, updated); + }, value); + const config: DevContainerConfig = substitute0(updated); if (typeof config.workspaceFolder === 'string') { workspaceConfig.workspaceFolder = config.workspaceFolder; } @@ -97,7 +99,11 @@ export async function readDevContainerConfigFile(cliHost: CLIHost, workspace: Wo } config.configFilePath = configFile; return { - config, + config: { + config, + raw: updated, + substitute: substitute0, + }, workspaceConfig, }; } diff --git a/src/spec-node/containerFeatures.ts b/src/spec-node/containerFeatures.ts index d1f83e0a7..143e18199 100644 --- a/src/spec-node/containerFeatures.ts +++ b/src/spec-node/containerFeatures.ts @@ -13,8 +13,9 @@ import { LogLevel, makeLog, toErrorText } from '../spec-utils/log'; import { FeaturesConfig, getContainerFeaturesFolder, getContainerFeaturesBaseDockerFile, getFeatureInstallWrapperScript, getFeatureLayers, getFeatureMainValue, getFeatureValueObject, generateFeaturesConfig, getSourceInfoString, collapseFeaturesConfig, Feature, multiStageBuildExploration, V1_DEVCONTAINER_FEATURES_FILE_NAME } from '../spec-configuration/containerFeaturesConfiguration'; import { readLocalFile } from '../spec-utils/pfs'; import { includeAllConfiguredFeatures } from '../spec-utils/product'; -import { createFeaturesTempFolder, DockerResolverParameters, getCacheFolder, getFolderImageName, inspectDockerImage, getEmptyContextFolder } from './utils'; +import { createFeaturesTempFolder, DockerResolverParameters, getCacheFolder, getFolderImageName, getEmptyContextFolder, SubstitutedConfig } from './utils'; import { isEarlierVersion, parseVersion } from '../spec-common/commonUtils'; +import { getDevcontainerMetadata, getDevcontainerMetadataLabel, getImageBuildInfoFromImage, ImageBuildInfo, MergedDevContainerConfig } from './imageMetadata'; // Escapes environment variable keys. // @@ -27,20 +28,18 @@ export const getSafeId = (str: string) => str .replace(/^[\d_]+/g, '_') .toUpperCase(); -export async function extendImage(params: DockerResolverParameters, config: DevContainerConfig, imageName: string, pullImageOnError: boolean) { - let cache: Promise | undefined; +export async function extendImage(params: DockerResolverParameters, config: SubstitutedConfig, imageName: string) { const { common } = params; const { cliHost, output } = common; - const imageDetails = () => cache || (cache = inspectDockerImage(params, imageName, pullImageOnError)); - const imageUser = async () => (await imageDetails()).Config.User || 'root'; - const extendImageDetails = await getExtendImageBuildInfo(params, config, imageName, imageUser); + const imageBuildInfo = await getImageBuildInfoFromImage(params, imageName, config.substitute, common.experimentalImageMetadata); + const extendImageDetails = await getExtendImageBuildInfo(params, config, imageName, imageBuildInfo); if (!extendImageDetails || !extendImageDetails.featureBuildInfo) { // no feature extensions - return return { updatedImageName: [imageName], - collapsedFeaturesConfig: undefined, - imageDetails + imageMetadata: imageBuildInfo.metadata, + imageDetails: async () => imageBuildInfo.imageDetails, }; } const { featureBuildInfo, collapsedFeaturesConfig } = extendImageDetails; @@ -87,28 +86,32 @@ export async function extendImage(params: DockerResolverParameters, config: DevC const infoParams = { ...toExecParameters(params), output: makeLog(output, LogLevel.Info), print: 'continuous' as 'continuous' }; await dockerCLI(infoParams, ...args); } - return { updatedImageName:[updatedImageName], collapsedFeaturesConfig, imageDetails }; + return { + updatedImageName: [ updatedImageName ], + imageMetadata: getDevcontainerMetadata(imageBuildInfo.metadata, config, collapsedFeaturesConfig?.allFeatures || []), + imageDetails: async () => imageBuildInfo.imageDetails, + }; } -export async function getExtendImageBuildInfo(params: DockerResolverParameters, config: DevContainerConfig, baseName: string, imageUser: () => Promise) { +export async function getExtendImageBuildInfo(params: DockerResolverParameters, config: SubstitutedConfig, baseName: string, imageBuildInfo: ImageBuildInfo) { // Creates the folder where the working files will be setup. - const tempFolder = await createFeaturesTempFolder(params.common); + const dstFolder = await createFeaturesTempFolder(params.common); // Extracts the local cache of features. - await createLocalFeatures(params, tempFolder); + await createLocalFeatures(params, dstFolder); // Processes the user's configuration. - const featuresConfig = await generateFeaturesConfig(params.common, tempFolder, config, getContainerFeaturesFolder); + const featuresConfig = await generateFeaturesConfig(params.common, dstFolder, config.config, getContainerFeaturesFolder); if (!featuresConfig) { - return null; + return { featureBuildInfo: getImageBuildOptions(params, config, dstFolder, baseName, imageBuildInfo) }; } // Generates the end configuration. const collapsedFeaturesConfig = collapseFeaturesConfig(featuresConfig); - const featureBuildInfo = await getContainerFeaturesBuildInfo(params, featuresConfig, baseName, imageUser); + const featureBuildInfo = await getFeaturesBuildOptions(params, config, featuresConfig, baseName, imageBuildInfo); if (!featureBuildInfo) { - return null; + return undefined; } return { featureBuildInfo, collapsedFeaturesConfig }; @@ -174,14 +177,41 @@ async function createLocalFeatures(params: DockerResolverParameters, dstFolder: await createExit; // Allow errors to surface. } -async function getContainerFeaturesBuildInfo(params: DockerResolverParameters, featuresConfig: FeaturesConfig, baseName: string, imageUser: () => Promise): Promise<{ dstFolder: string; dockerfileContent: string; overrideTarget: string; dockerfilePrefixContent: string; buildArgs: Record; buildKitContexts: Record } | null> { +export interface ImageBuildOptions { + dstFolder: string; + dockerfileContent: string; + overrideTarget: string; + dockerfilePrefixContent: string; + buildArgs: Record; + buildKitContexts: Record; +} + +function getImageBuildOptions(params: DockerResolverParameters, config: SubstitutedConfig, dstFolder: string, baseName: string, imageBuildInfo: ImageBuildInfo) { + return { + dstFolder, + dockerfileContent: ` +FROM $_DEV_CONTAINERS_BASE_IMAGE AS dev_containers_target_stage +${getDevcontainerMetadataLabel(imageBuildInfo.metadata, config, [], params.common.experimentalImageMetadata)} +`, + overrideTarget: 'dev_containers_target_stage', + dockerfilePrefixContent: ` +ARG _DEV_CONTAINERS_BASE_IMAGE=placeholder +`, + buildArgs: { + _DEV_CONTAINERS_BASE_IMAGE: baseName, + } as Record, + buildKitContexts: {} as Record, + }; +} + +async function getFeaturesBuildOptions(params: DockerResolverParameters, devContainerConfig: SubstitutedConfig, featuresConfig: FeaturesConfig, baseName: string, imageBuildInfo: ImageBuildInfo): Promise { const { common } = params; const { cliHost, output } = common; const { dstFolder } = featuresConfig; if (!dstFolder || dstFolder === '') { output.write('dstFolder is undefined or empty in addContainerFeatures', LogLevel.Error); - return null; + return undefined; } const buildStageScripts = await Promise.all(featuresConfig.featureSets @@ -219,6 +249,7 @@ async function getContainerFeaturesBuildInfo(params: DockerResolverParameters, f .replace('#{featureLayer}', getFeatureLayers(featuresConfig)) .replace('#{containerEnv}', generateContainerEnvs(featuresConfig)) .replace('#{copyFeatureBuildStages}', getCopyFeatureBuildStages(featuresConfig, buildStageScripts)) + .replace('#{devcontainerMetadata}', getDevcontainerMetadataLabel(imageBuildInfo.metadata, devContainerConfig, featuresConfig, common.experimentalImageMetadata)) ; const dockerfilePrefixContent = `${useBuildKitBuildContexts ? '# syntax=docker/dockerfile:1.4' : ''} ARG _DEV_CONTAINERS_BASE_IMAGE=placeholder @@ -297,7 +328,7 @@ ARG _DEV_CONTAINERS_BASE_IMAGE=placeholder dockerfilePrefixContent, buildArgs: { _DEV_CONTAINERS_BASE_IMAGE: baseName, - _DEV_CONTAINERS_IMAGE_USER: await imageUser(), + _DEV_CONTAINERS_IMAGE_USER: imageBuildInfo.user, _DEV_CONTAINERS_FEATURE_CONTENT_SOURCE: buildContentImageName, }, buildKitContexts: useBuildKitBuildContexts ? { dev_containers_feature_content_source: dstFolder } : {}, @@ -358,14 +389,16 @@ function getFeatureEnvVariables(f: Feature) { } } -export async function getRemoteUserUIDUpdateDetails(params: DockerResolverParameters, config: DevContainerConfig, imageName: string, imageDetails: () => Promise, runArgsUser: string | undefined) { +export async function getRemoteUserUIDUpdateDetails(params: DockerResolverParameters, mergedConfig: MergedDevContainerConfig, imageName: string, imageDetails: () => Promise, runArgsUser: string | undefined) { const { common } = params; const { cliHost } = common; - if (params.updateRemoteUserUIDDefault === 'never' || !(typeof config.updateRemoteUserUID === 'boolean' ? config.updateRemoteUserUID : params.updateRemoteUserUIDDefault === 'on') || !(cliHost.platform === 'linux' || params.updateRemoteUserUIDOnMacOS && cliHost.platform === 'darwin')) { + const { updateRemoteUserUID } = mergedConfig; + if (params.updateRemoteUserUIDDefault === 'never' || !(typeof updateRemoteUserUID === 'boolean' ? updateRemoteUserUID : params.updateRemoteUserUIDDefault === 'on') || !(cliHost.platform === 'linux' || params.updateRemoteUserUIDOnMacOS && cliHost.platform === 'darwin')) { return null; } - const imageUser = (await imageDetails()).Config.User || 'root'; - const remoteUser = config.remoteUser || runArgsUser || imageUser; + const details = await imageDetails(); + const imageUser = details.Config.User || 'root'; + const remoteUser = mergedConfig.remoteUser || runArgsUser || imageUser; if (remoteUser === 'root' || /^\d+$/.test(remoteUser)) { return null; } @@ -379,11 +412,11 @@ export async function getRemoteUserUIDUpdateDetails(params: DockerResolverParame }; } -export async function updateRemoteUserUID(params: DockerResolverParameters, config: DevContainerConfig, imageName: string, imageDetails: () => Promise, runArgsUser: string | undefined) { +export async function updateRemoteUserUID(params: DockerResolverParameters, mergedConfig: MergedDevContainerConfig, imageName: string, imageDetails: () => Promise, runArgsUser: string | undefined) { const { common } = params; const { cliHost } = common; - const updateDetails = await getRemoteUserUIDUpdateDetails(params, config, imageName, imageDetails, runArgsUser); + const updateDetails = await getRemoteUserUIDUpdateDetails(params, mergedConfig, imageName, imageDetails, runArgsUser); if (!updateDetails) { return imageName; } diff --git a/src/spec-node/devContainers.ts b/src/spec-node/devContainers.ts index b43dce7f7..94abdd1e3 100644 --- a/src/spec-node/devContainers.ts +++ b/src/spec-node/devContainers.ts @@ -19,6 +19,8 @@ import { Mount } from '../spec-configuration/containerFeaturesConfiguration'; import { getPackageConfig, PackageConfiguration } from '../spec-utils/product'; import { dockerBuildKitVersion } from '../spec-shutdown/dockerUtils'; +export const experimentalImageMetadataDefault = true; + export interface ProvisionOptions { dockerPath: string | undefined; dockerComposePath: string | undefined; @@ -52,6 +54,7 @@ export interface ProvisionOptions { buildxPush: boolean; skipFeatureAutoMapping: boolean; skipPostAttach: boolean; + experimentalImageMetadata: boolean; } export async function launch(options: ProvisionOptions, disposables: (() => Promise | undefined)[]) { @@ -124,6 +127,7 @@ export async function createDockerParams(options: ProvisionOptions, disposables: buildxPush: options.buildxPush, skipFeatureAutoMapping: options.skipFeatureAutoMapping, skipPostAttach: options.skipPostAttach, + experimentalImageMetadata: options.experimentalImageMetadata, }; const dockerPath = options.dockerPath || 'docker'; diff --git a/src/spec-node/devContainersSpecCLI.ts b/src/spec-node/devContainersSpecCLI.ts index 4532c58ce..51dff2e94 100644 --- a/src/spec-node/devContainersSpecCLI.ts +++ b/src/spec-node/devContainersSpecCLI.ts @@ -6,8 +6,8 @@ import * as path from 'path'; import yargs, { Argv } from 'yargs'; -import { createDockerParams, createLog, launch, ProvisionOptions } from './devContainers'; -import { createContainerProperties, createFeaturesTempFolder, envListToObj, isDockerFileConfig } from './utils'; +import { createDockerParams, createLog, experimentalImageMetadataDefault, launch, ProvisionOptions } from './devContainers'; +import { SubstitutedConfig, createContainerProperties, createFeaturesTempFolder, envListToObj, inspectDockerImage, isDockerFileConfig, SubstituteConfig } from './utils'; import { URI } from 'vscode-uri'; import { ContainerError } from '../spec-common/errors'; import { Log, LogLevel, makeLog, mapLogLevel } from '../spec-utils/log'; @@ -17,19 +17,20 @@ import { bailOut, buildNamedImageAndExtend, findDevContainer, hostFolderLabel } import { extendImage } from './containerFeatures'; import { DockerCLIParameters, dockerPtyCLI, inspectContainer } from '../spec-shutdown/dockerUtils'; import { buildAndExtendDockerCompose, dockerComposeCLIConfig, getDefaultImageName, getProjectName, readDockerComposeConfig } from './dockerCompose'; -import { getDockerComposeFilePaths } from '../spec-configuration/configuration'; +import { DevContainerConfig, DevContainerFromDockerComposeConfig, DevContainerFromDockerfileConfig, getDockerComposeFilePaths } from '../spec-configuration/configuration'; import { workspaceFromPath } from '../spec-utils/workspaces'; import { readDevContainerConfigFile } from './configContainer'; import { getDefaultDevContainerConfigPath, getDevContainerConfigPathIn, uriToFsPath } from '../spec-configuration/configurationCommonUtils'; import { getCLIHost } from '../spec-common/cliHost'; import { loadNativeModule } from '../spec-common/commonUtils'; -import { generateFeaturesConfig, getContainerFeaturesFolder } from '../spec-configuration/containerFeaturesConfiguration'; +import { FeaturesConfig, generateFeaturesConfig, getContainerFeaturesFolder } from '../spec-configuration/containerFeaturesConfiguration'; import { featuresTestOptions, featuresTestHandler } from './featuresCLI/test'; import { featuresPackageHandler, featuresPackageOptions } from './featuresCLI/package'; import { featuresPublishHandler, featuresPublishOptions } from './featuresCLI/publish'; import { featuresInfoHandler, featuresInfoOptions } from './featuresCLI/info'; import { containerSubstitute } from '../spec-common/variableSubstitution'; -import { getPackageConfig } from '../spec-utils/product'; +import { getPackageConfig, PackageConfiguration } from '../spec-utils/product'; +import { getDevcontainerMetadata, getImageBuildInfo, getImageMetadataFromContainer, ImageMetadataEntry, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata'; const defaultDefaultUserEnvProbe: UserEnvProbe = 'loginInteractiveShell'; @@ -102,6 +103,7 @@ function provisionOptions(y: Argv) { 'buildkit': { choices: ['auto' as 'auto', 'never' as 'never'], default: 'auto' as 'auto', description: 'Control whether BuildKit should be used' }, 'skip-feature-auto-mapping': { type: 'boolean', default: false, hidden: true, description: 'Temporary option for testing.' }, 'skip-post-attach': { type: 'boolean', default: false, description: 'Do not run postAttachCommand.' }, + 'experimental-image-metadata': { type: 'boolean', default: experimentalImageMetadataDefault, hidden: true, description: 'Temporary option for testing.' }, }) .check(argv => { const idLabels = (argv['id-label'] && (Array.isArray(argv['id-label']) ? argv['id-label'] : [argv['id-label']])) as string[] | undefined; @@ -162,6 +164,7 @@ async function provision({ 'buildkit': buildkit, 'skip-feature-auto-mapping': skipFeatureAutoMapping, 'skip-post-attach': skipPostAttach, + 'experimental-image-metadata': experimentalImageMetadata, }: ProvisionArgs) { const workspaceFolder = workspaceFolderArg ? path.resolve(process.cwd(), workspaceFolderArg) : undefined; @@ -207,6 +210,7 @@ async function provision({ buildxPush: false, skipFeatureAutoMapping, skipPostAttach, + experimentalImageMetadata, }; const result = await doProvision(options); @@ -267,6 +271,7 @@ function buildOptions(y: Argv) { 'platform': { type: 'string', description: 'Set target platforms.' }, 'push': { type: 'boolean', default: false, description: 'Push to a container registry.' }, 'skip-feature-auto-mapping': { type: 'boolean', default: false, hidden: true, description: 'Temporary option for testing.' }, + 'experimental-image-metadata': { type: 'boolean', default: experimentalImageMetadataDefault, hidden: true, description: 'Temporary option for testing.' }, }); } @@ -298,6 +303,7 @@ async function doBuild({ 'platform': buildxPlatform, 'push': buildxPush, 'skip-feature-auto-mapping': skipFeatureAutoMapping, + 'experimental-image-metadata': experimentalImageMetadata, }: BuildArgs) { const disposables: (() => Promise | undefined)[] = []; const dispose = async () => { @@ -339,6 +345,7 @@ async function doBuild({ buildxPush, skipFeatureAutoMapping, skipPostAttach: true, + experimentalImageMetadata, }, disposables); const { common, dockerCLI, dockerComposeCLI } = params; @@ -352,7 +359,8 @@ async function doBuild({ if (!configs) { throw new ContainerError({ description: `Dev container config (${uriToFsPath(configFile || getDefaultDevContainerConfigPath(cliHost, workspace!.configFolderPath), cliHost.platform)}) not found.` }); } - const { config } = configs; + const configWithRaw = configs.config; + const { config } = configWithRaw; let imageNameResult: string[] = ['']; // Support multiple use of `--image-name` @@ -361,7 +369,7 @@ async function doBuild({ if (isDockerFileConfig(config)) { // Build the base image and extend with features etc. - let { updatedImageName } = await buildNamedImageAndExtend(params, config, imageNames); + let { updatedImageName } = await buildNamedImageAndExtend(params, configWithRaw as SubstitutedConfig, imageNames); if (imageNames) { if (!buildxPush) { @@ -397,10 +405,10 @@ async function doBuild({ } const infoParams = { ...params, common: { ...params.common, output: makeLog(buildParams.output, LogLevel.Info) } }; - await buildAndExtendDockerCompose(config, projectName, infoParams, composeFiles, envFile, composeGlobalArgs, [config.service], params.buildNoCache || false, params.common.persistedFolder, 'docker-compose.devcontainer.build', addCacheFroms); + const { overrideImageName } = await buildAndExtendDockerCompose(configWithRaw as SubstitutedConfig, projectName, infoParams, composeFiles, envFile, composeGlobalArgs, [config.service], params.buildNoCache || false, params.common.persistedFolder, 'docker-compose.devcontainer.build', addCacheFroms); const service = composeConfig.services[config.service]; - const originalImageName = service.image || getDefaultImageName(await buildParams.dockerComposeCLI(), projectName, config.service); + const originalImageName = overrideImageName || service.image || getDefaultImageName(await buildParams.dockerComposeCLI(), projectName, config.service); if (imageNames) { await Promise.all(imageNames.map(imageName => dockerPtyCLI(params, 'tag', originalImageName, imageName))); @@ -410,8 +418,8 @@ async function doBuild({ } } else { - await dockerPtyCLI(params, 'pull', config.image); - const { updatedImageName } = await extendImage(params, config, config.image, 'image' in config); + await inspectDockerImage(params, config.image, true); + const { updatedImageName } = await extendImage(params, configWithRaw, config.image); if (buildxPlatform || buildxPush) { throw new ContainerError({ description: '--platform or --push require dockerfilePath.' }); @@ -471,6 +479,7 @@ function runUserCommandsOptions(y: Argv) { 'remote-env': { type: 'string', description: 'Remote environment variables of the format name=value. These will be added when executing the user commands.' }, 'skip-feature-auto-mapping': { type: 'boolean', default: false, hidden: true, description: 'Temporary option for testing.' }, 'skip-post-attach': { type: 'boolean', default: false, description: 'Do not run postAttachCommand.' }, + 'experimental-image-metadata': { type: 'boolean', default: experimentalImageMetadataDefault, hidden: true, description: 'Temporary option for testing.' }, }) .check(argv => { const idLabels = (argv['id-label'] && (Array.isArray(argv['id-label']) ? argv['id-label'] : [argv['id-label']])) as string[] | undefined; @@ -521,6 +530,7 @@ async function doRunUserCommands({ 'remote-env': addRemoteEnv, 'skip-feature-auto-mapping': skipFeatureAutoMapping, 'skip-post-attach': skipPostAttach, + 'experimental-image-metadata': experimentalImageMetadata, }: RunUserCommandsArgs) { const disposables: (() => Promise | undefined)[] = []; const dispose = async () => { @@ -563,6 +573,7 @@ async function doRunUserCommands({ buildxPush: false, skipFeatureAutoMapping, skipPostAttach, + experimentalImageMetadata, }, disposables); const { common } = params; @@ -582,8 +593,10 @@ async function doRunUserCommands({ if (!container) { bailOut(common.output, 'Dev container not found.'); } - const containerProperties = await createContainerProperties(params, container.Id, workspaceConfig.workspaceFolder, config.remoteUser); - const updatedConfig = containerSubstitute(cliHost.platform, config.configFilePath, containerProperties.env, config); + const imageMetadata = getImageMetadataFromContainer(container, config, [], experimentalImageMetadata, output).config; + const mergedConfig = mergeConfiguration(config.config, imageMetadata); + const containerProperties = await createContainerProperties(params, container.Id, workspaceConfig.workspaceFolder, mergedConfig.remoteUser); + const updatedConfig = containerSubstitute(cliHost.platform, config.config.configFilePath, containerProperties.env, mergedConfig); const remoteEnv = probeRemoteEnv(common, containerProperties, updatedConfig); const result = await runPostCreateCommands(common, containerProperties, updatedConfig, remoteEnv, stopForPersonalization); return { @@ -626,7 +639,9 @@ function readConfigurationOptions(y: Argv) { 'terminal-columns': { type: 'number', implies: ['terminal-rows'], description: 'Number of rows to render the output for. This is required for some of the subprocesses to correctly render their output.' }, 'terminal-rows': { type: 'number', implies: ['terminal-columns'], description: 'Number of columns to render the output for. This is required for some of the subprocesses to correctly render their output.' }, 'include-features-configuration': { type: 'boolean', default: false, description: 'Include features configuration.' }, + 'include-merged-configuration': { type: 'boolean', default: false, description: 'Include merged configuration.' }, 'skip-feature-auto-mapping': { type: 'boolean', default: false, hidden: true, description: 'Temporary option for testing.' }, + 'experimental-image-metadata': { type: 'boolean', default: experimentalImageMetadataDefault, hidden: true, description: 'Temporary option for testing.' }, }) .check(argv => { const idLabels = (argv['id-label'] && (Array.isArray(argv['id-label']) ? argv['id-label'] : [argv['id-label']])) as string[] | undefined; @@ -658,7 +673,9 @@ async function readConfiguration({ 'terminal-rows': terminalRows, 'terminal-columns': terminalColumns, 'include-features-configuration': includeFeaturesConfig, + 'include-merged-configuration': includeMergedConfig, 'skip-feature-auto-mapping': skipFeatureAutoMapping, + 'experimental-image-metadata': experimentalImageMetadata, }: ReadConfigurationArgs) { const disposables: (() => Promise | undefined)[] = []; const dispose = async () => { @@ -691,7 +708,7 @@ async function readConfiguration({ if (!configs) { throw new ContainerError({ description: `Dev container config (${uriToFsPath(configFile || getDefaultDevContainerConfigPath(cliHost, workspace!.configFolderPath), cliHost.platform)}) not found.` }); } - let { config: configuration } = configs; + let configuration = configs.config; const dockerCLI = dockerPath || 'docker'; const dockerComposeCLI = dockerComposeCLIConfig({ @@ -708,15 +725,35 @@ async function readConfiguration({ }; const container = containerId ? await inspectContainer(params, containerId) : await findDevContainer(params, idLabels); if (container) { - configuration = containerSubstitute(cliHost.platform, configuration.configFilePath, envListToObj(container.Config.Env), configuration); + const substitute2: SubstituteConfig = config => containerSubstitute(cliHost.platform, configuration.config.configFilePath, envListToObj(container.Config.Env), config); + configuration = { + config: substitute2(configuration.config), + raw: configuration.raw, + substitute: config => substitute2(configuration.substitute(config)), + }; } - const featuresConfiguration = includeFeaturesConfig ? await generateFeaturesConfig({ extensionPath, cwd, output, env: cliHost.env, skipFeatureAutoMapping }, (await createFeaturesTempFolder({ cliHost, package: pkg })), configs.config, getContainerFeaturesFolder) : undefined; + const needsFeaturesConfig = includeFeaturesConfig || (includeMergedConfig && (!container || !experimentalImageMetadata)); + const featuresConfiguration = needsFeaturesConfig ? await readFeaturesConfig(params, pkg, configuration.config, extensionPath, skipFeatureAutoMapping) : undefined; + let mergedConfig: MergedDevContainerConfig | undefined; + if (includeMergedConfig) { + let imageMetadata: ImageMetadataEntry[]; + if (container) { + imageMetadata = getImageMetadataFromContainer(container, configuration, featuresConfiguration || [], experimentalImageMetadata, output).config; + const substitute2: SubstituteConfig = config => containerSubstitute(cliHost.platform, configuration.config.configFilePath, envListToObj(container.Config.Env), config); + imageMetadata = imageMetadata.map(substitute2); + } else { + const imageBuildInfo = await getImageBuildInfo(params, configs.config, experimentalImageMetadata); + imageMetadata = getDevcontainerMetadata(imageBuildInfo.metadata, configs.config, featuresConfiguration || []).config; + } + mergedConfig = mergeConfiguration(configuration.config, imageMetadata); + } await new Promise((resolve, reject) => { process.stdout.write(JSON.stringify({ - configuration, + configuration: configuration.config, workspace: configs.workspaceConfig, featuresConfiguration, + mergedConfiguration: mergedConfig, }) + '\n', err => err ? reject(err) : resolve()); }); } catch (err) { @@ -732,6 +769,13 @@ async function readConfiguration({ process.exit(0); } +async function readFeaturesConfig(params: DockerCLIParameters, pkg: PackageConfiguration, config: DevContainerConfig, extensionPath: string, skipFeatureAutoMapping: boolean): Promise { + const { cliHost, output } = params; + const { cwd, env } = cliHost; + const featuresTmpFolder = await createFeaturesTempFolder({ cliHost, package: pkg }); + return generateFeaturesConfig({ extensionPath, cwd, output, env, skipFeatureAutoMapping }, featuresTmpFolder, config, getContainerFeaturesFolder); +} + function execOptions(y: Argv) { return y.options({ 'user-data-folder': { type: 'string', description: 'Host path to a directory that is intended to be persisted and share state between sessions.' }, @@ -752,6 +796,7 @@ function execOptions(y: Argv) { 'default-user-env-probe': { choices: ['none' as 'none', 'loginInteractiveShell' as 'loginInteractiveShell', 'interactiveShell' as 'interactiveShell', 'loginShell' as 'loginShell'], default: defaultDefaultUserEnvProbe, description: 'Default value for the devcontainer.json\'s "userEnvProbe".' }, 'remote-env': { type: 'string', description: 'Remote environment variables of the format name=value. These will be added when executing the user commands.' }, 'skip-feature-auto-mapping': { type: 'boolean', default: false, hidden: true, description: 'Temporary option for testing.' }, + 'experimental-image-metadata': { type: 'boolean', default: experimentalImageMetadataDefault, hidden: true, description: 'Temporary option for testing.' }, }) .positional('cmd', { type: 'string', @@ -809,6 +854,7 @@ export async function doExec({ 'default-user-env-probe': defaultUserEnvProbe, 'remote-env': addRemoteEnv, 'skip-feature-auto-mapping': skipFeatureAutoMapping, + 'experimental-image-metadata': experimentalImageMetadata, _: restArgs, }: ExecArgs & { _?: string[] }) { const disposables: (() => Promise | undefined)[] = []; @@ -853,6 +899,7 @@ export async function doExec({ buildxPush: false, skipFeatureAutoMapping, skipPostAttach: false, + experimentalImageMetadata, }, disposables); const { common } = params; @@ -872,8 +919,10 @@ export async function doExec({ if (!container) { bailOut(common.output, 'Dev container not found.'); } - const containerProperties = await createContainerProperties(params, container.Id, workspaceConfig.workspaceFolder, config.remoteUser); - const updatedConfig = containerSubstitute(cliHost.platform, config.configFilePath, containerProperties.env, config); + const imageMetadata = getImageMetadataFromContainer(container, config, [], experimentalImageMetadata, output).config; + const mergedConfig = mergeConfiguration(config.config, imageMetadata); + const containerProperties = await createContainerProperties(params, container.Id, workspaceConfig.workspaceFolder, mergedConfig.remoteUser); + const updatedConfig = containerSubstitute(cliHost.platform, config.config.configFilePath, containerProperties.env, mergedConfig); const remoteEnv = probeRemoteEnv(common, containerProperties, updatedConfig); const remoteCwd = containerProperties.remoteWorkspaceFolder || containerProperties.homeFolder; const infoOutput = makeLog(output, LogLevel.Info); diff --git a/src/spec-node/dockerCompose.ts b/src/spec-node/dockerCompose.ts index 84be4a99c..37c67be5d 100644 --- a/src/spec-node/dockerCompose.ts +++ b/src/spec-node/dockerCompose.ts @@ -6,7 +6,7 @@ import * as yaml from 'js-yaml'; import * as shellQuote from 'shell-quote'; -import { createContainerProperties, startEventSeen, ResolverResult, getTunnelInformation, DockerResolverParameters, inspectDockerImage, ensureDockerfileHasFinalStageName, getImageUser, getEmptyContextFolder } from './utils'; +import { createContainerProperties, startEventSeen, ResolverResult, getTunnelInformation, DockerResolverParameters, inspectDockerImage, getEmptyContextFolder, getFolderImageName, SubstitutedConfig } from './utils'; import { ContainerProperties, setupInContainer, ResolverProgress } from '../spec-common/injectHeadless'; import { ContainerError } from '../spec-common/errors'; import { Workspace } from '../spec-utils/workspaces'; @@ -15,23 +15,25 @@ import { ContainerDetails, inspectContainer, listContainers, DockerCLIParameters import { DevContainerFromDockerComposeConfig, getDockerComposeFilePaths } from '../spec-configuration/configuration'; import { Log, LogLevel, makeLog, terminalEscapeSequences } from '../spec-utils/log'; import { getExtendImageBuildInfo, updateRemoteUserUID } from './containerFeatures'; -import { Mount, CollapsedFeaturesConfig } from '../spec-configuration/containerFeaturesConfiguration'; -import { includeAllConfiguredFeatures } from '../spec-utils/product'; +import { Mount, parseMount } from '../spec-configuration/containerFeaturesConfiguration'; import path from 'path'; +import { getDevcontainerMetadata, getImageBuildInfoFromDockerfile, getImageMetadataFromContainer, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata'; +import { ensureDockerfileHasFinalStageName } from './dockerfileUtils'; const projectLabel = 'com.docker.compose.project'; const serviceLabel = 'com.docker.compose.service'; -export async function openDockerComposeDevContainer(params: DockerResolverParameters, workspace: Workspace, config: DevContainerFromDockerComposeConfig, idLabels: string[]): Promise { +export async function openDockerComposeDevContainer(params: DockerResolverParameters, workspace: Workspace, config: SubstitutedConfig, idLabels: string[]): Promise { const { common, dockerCLI, dockerComposeCLI } = params; const { cliHost, env, output } = common; const buildParams: DockerCLIParameters = { cliHost, dockerCLI, dockerComposeCLI, env, output }; - return _openDockerComposeDevContainer(params, buildParams, workspace, config, getRemoteWorkspaceFolder(config), idLabels); + return _openDockerComposeDevContainer(params, buildParams, workspace, config, getRemoteWorkspaceFolder(config.config), idLabels); } -async function _openDockerComposeDevContainer(params: DockerResolverParameters, buildParams: DockerCLIParameters, workspace: Workspace, config: DevContainerFromDockerComposeConfig, remoteWorkspaceFolder: string, idLabels: string[]): Promise { +async function _openDockerComposeDevContainer(params: DockerResolverParameters, buildParams: DockerCLIParameters, workspace: Workspace, configWithRaw: SubstitutedConfig, remoteWorkspaceFolder: string, idLabels: string[]): Promise { const { common } = params; const { cliHost: buildCLIHost } = buildParams; + const { config } = configWithRaw; let container: ContainerDetails | undefined; let containerProperties: ContainerProperties | undefined; @@ -57,7 +59,7 @@ async function _openDockerComposeDevContainer(params: DockerResolverParameters, // let collapsedFeaturesConfig: CollapsedFeaturesConfig | undefined; if (!container || container.State.Status !== 'running') { - const res = await startContainer(params, buildParams, config, projectName, composeFiles, envFile, container, idLabels); + const res = await startContainer(params, buildParams, configWithRaw, projectName, composeFiles, envFile, container, idLabels); container = await inspectContainer(params, res.containerId); // collapsedFeaturesConfig = res.collapsedFeaturesConfig; // } else { @@ -66,11 +68,13 @@ async function _openDockerComposeDevContainer(params: DockerResolverParameters, // collapsedFeaturesConfig = collapseFeaturesConfig(featuresConfig); } - containerProperties = await createContainerProperties(params, container.Id, remoteWorkspaceFolder, config.remoteUser); + const configs = getImageMetadataFromContainer(container, configWithRaw, [], common.experimentalImageMetadata, common.output).config; + const mergedConfig = mergeConfiguration(configWithRaw.config, configs); + containerProperties = await createContainerProperties(params, container.Id, remoteWorkspaceFolder, mergedConfig.remoteUser); const { remoteEnv: extensionHostEnv, - } = await setupInContainer(common, containerProperties, config); + } = await setupInContainer(common, containerProperties, mergedConfig); return { params: common, @@ -135,14 +139,16 @@ export function getBuildInfoForService(composeService: any, cliHostPath: typeof dockerfilePath: (composeBuild.dockerfile as string | undefined) ?? 'Dockerfile', context: (composeBuild.context as string | undefined) ?? cliHostPath.dirname(localComposeFiles[0]), target: composeBuild.target as string | undefined, + args: composeBuild.args as Record | undefined, } }; } -export async function buildAndExtendDockerCompose(config: DevContainerFromDockerComposeConfig, projectName: string, params: DockerResolverParameters, localComposeFiles: string[], envFile: string | undefined, composeGlobalArgs: string[], runServices: string[], noCache: boolean, overrideFilePath: string, overrideFilePrefix: string, additionalCacheFroms?: string[], noBuild?: boolean) { +export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConfig, projectName: string, params: DockerResolverParameters, localComposeFiles: string[], envFile: string | undefined, composeGlobalArgs: string[], runServices: string[], noCache: boolean, overrideFilePath: string, overrideFilePrefix: string, additionalCacheFroms?: string[], noBuild?: boolean) { const { common, dockerCLI, dockerComposeCLI: dockerComposeCLIFunc } = params; const { cliHost, env, output } = common; + const { config } = configWithRaw; const cliParams: DockerCLIParameters = { cliHost, dockerCLI, dockerComposeCLI: dockerComposeCLIFunc, env, output }; const composeConfig = await readDockerComposeConfig(cliParams, localComposeFiles, envFile); @@ -175,12 +181,19 @@ export async function buildAndExtendDockerCompose(config: DevContainerFromDocker // determine whether we need to extend with features const noBuildKitParams = { ...params, buildKitVersion: null }; // skip BuildKit -> can't set additional build contexts with compose - const extendImageBuildInfo = await getExtendImageBuildInfo(noBuildKitParams, config, baseName, () => getImageUser(params, originalDockerfile)); + const imageBuildInfo = await getImageBuildInfoFromDockerfile(params, originalDockerfile, serviceInfo.build?.args || {}, serviceInfo.build?.target, configWithRaw.substitute, common.experimentalImageMetadata); + const extendImageBuildInfo = await getExtendImageBuildInfo(noBuildKitParams, configWithRaw, baseName, imageBuildInfo); - let buildOverrideContent = null; + let overrideImageName: string | undefined; + let buildOverrideContent = ''; if (extendImageBuildInfo) { + // Avoid retagging a previously pulled image. + if (!serviceInfo.build) { + overrideImageName = getFolderImageName(common); + buildOverrideContent += ` image: ${overrideImageName}\n`; + } // Create overridden Dockerfile and generate docker-compose build override content - buildOverrideContent = ' build:\n'; + buildOverrideContent += ' build:\n'; const { featureBuildInfo } = extendImageBuildInfo; // We add a '# syntax' line at the start, so strip out any existing line const syntaxMatch = dockerfile.match(/^\s*#\s*syntax\s*=.*[\r\n]/g); @@ -225,6 +238,7 @@ export async function buildAndExtendDockerCompose(config: DevContainerFromDocker ${buildOverrideContent?.trimEnd()} ${cacheFromOverrideContent} `; + output.write(`Docker Compose override file for building image:\n${composeOverrideContent}`); await cliHost.writeFile(composeOverrideFile, Buffer.from(composeOverrideContent)); additionalComposeOverrideFiles.push(composeOverrideFile); args.push('-f', composeOverrideFile); @@ -259,8 +273,9 @@ ${cacheFromOverrideContent} } return { - collapsedFeaturesConfig: extendImageBuildInfo?.collapsedFeaturesConfig, + imageMetadata: getDevcontainerMetadata(imageBuildInfo.metadata, configWithRaw, extendImageBuildInfo?.collapsedFeaturesConfig?.allFeatures || []), additionalComposeOverrideFiles, + overrideImageName, }; } @@ -291,10 +306,11 @@ async function checkForPersistedFile(cliHost: CLIHost, output: Log, files: strin foundLabel: false }; } -async function startContainer(params: DockerResolverParameters, buildParams: DockerCLIParameters, config: DevContainerFromDockerComposeConfig, projectName: string, composeFiles: string[], envFile: string | undefined, container: ContainerDetails | undefined, idLabels: string[]) { +async function startContainer(params: DockerResolverParameters, buildParams: DockerCLIParameters, configWithRaw: SubstitutedConfig, projectName: string, composeFiles: string[], envFile: string | undefined, container: ContainerDetails | undefined, idLabels: string[]) { const { common } = params; const { persistedFolder, output } = common; const { cliHost: buildCLIHost } = buildParams; + const { config } = configWithRaw; const featuresBuildOverrideFilePrefix = 'docker-compose.devcontainer.build'; const featuresStartOverrideFilePrefix = 'docker-compose.devcontainer.containerFeatures'; @@ -355,17 +371,19 @@ async function startContainer(params: DockerResolverParameters, buildParams: Doc const noBuild = !!container; //if we have an existing container, just recreate override files but skip the build const infoParams = { ...params, common: { ...params.common, output: infoOutput } }; - const { collapsedFeaturesConfig, additionalComposeOverrideFiles } = await buildAndExtendDockerCompose(config, projectName, infoParams, localComposeFiles, envFile, composeGlobalArgs, config.runServices ?? [], params.buildNoCache ?? false, persistedFolder, featuresBuildOverrideFilePrefix, params.additionalCacheFroms, noBuild); + const { imageMetadata, additionalComposeOverrideFiles, overrideImageName } = await buildAndExtendDockerCompose(configWithRaw, projectName, infoParams, localComposeFiles, envFile, composeGlobalArgs, config.runServices ?? [], params.buildNoCache ?? false, persistedFolder, featuresBuildOverrideFilePrefix, params.additionalCacheFroms, noBuild); additionalComposeOverrideFiles.forEach(overrideFilePath => composeGlobalArgs.push('-f', overrideFilePath)); + const currentImageName = overrideImageName || originalImageName; let cache: Promise | undefined; - const imageDetails = () => cache || (cache = inspectDockerImage(params, originalImageName, true)); - const updatedImageName = noBuild ? originalImageName : await updateRemoteUserUID(params, config, originalImageName, imageDetails, service.user); + const imageDetails = () => cache || (cache = inspectDockerImage(params, currentImageName, true)); + const mergedConfig = mergeConfiguration(config, imageMetadata.config); + const updatedImageName = noBuild ? currentImageName : await updateRemoteUserUID(params, mergedConfig, currentImageName, imageDetails, service.user); // Save override docker-compose file to disk. // Persisted folder is a path that will be maintained between sessions // Note: As a fallback, persistedFolder is set to the build's tmpDir() directory - const overrideFilePath = await writeFeaturesComposeOverrideFile(updatedImageName, originalImageName, collapsedFeaturesConfig, config, buildParams, composeFiles, imageDetails, service, idLabels, params.additionalMounts, persistedFolder, featuresStartOverrideFilePrefix, buildCLIHost, output); + const overrideFilePath = await writeFeaturesComposeOverrideFile(updatedImageName, currentImageName, mergedConfig, config, buildParams, composeFiles, imageDetails, service, idLabels, params.additionalMounts, persistedFolder, featuresStartOverrideFilePrefix, buildCLIHost, output); if (overrideFilePath) { // Add file path to override file as parameter composeGlobalArgs.push('-f', overrideFilePath); @@ -409,7 +427,7 @@ export function getDefaultImageName(dockerComposeCLI: DockerComposeCLI, projectN async function writeFeaturesComposeOverrideFile( updatedImageName: string, originalImageName: string, - collapsedFeaturesConfig: CollapsedFeaturesConfig | undefined, + mergedConfig: MergedDevContainerConfig, config: DevContainerFromDockerComposeConfig, buildParams: DockerCLIParameters, composeFiles: string[], @@ -422,10 +440,10 @@ async function writeFeaturesComposeOverrideFile( buildCLIHost: CLIHost, output: Log, ) { - const composeOverrideContent = await generateFeaturesComposeOverrideContent(updatedImageName, originalImageName, collapsedFeaturesConfig, config, buildParams, composeFiles, imageDetails, service, additionalLabels, additionalMounts); + const composeOverrideContent = await generateFeaturesComposeOverrideContent(updatedImageName, originalImageName, mergedConfig, config, buildParams, composeFiles, imageDetails, service, additionalLabels, additionalMounts); const overrideFileHasContents = !!composeOverrideContent && composeOverrideContent.length > 0 && composeOverrideContent.trim() !== ''; if (overrideFileHasContents) { - output.write(`Docker Compose override file:\n${composeOverrideContent}`, LogLevel.Trace); + output.write(`Docker Compose override file for creating container:\n${composeOverrideContent}`); const fileName = `${overrideFilePrefix}-${Date.now()}.yml`; const composeFolder = buildCLIHost.path.join(overrideFilePath, 'docker-compose'); @@ -444,7 +462,7 @@ async function writeFeaturesComposeOverrideFile( async function generateFeaturesComposeOverrideContent( updatedImageName: string, originalImageName: string, - collapsedFeaturesConfig: CollapsedFeaturesConfig | undefined, + mergedConfig: MergedDevContainerConfig, config: DevContainerFromDockerComposeConfig, buildParams: DockerCLIParameters, composeFiles: string[], @@ -459,27 +477,22 @@ async function generateFeaturesComposeOverrideContent( const overrideImage = updatedImageName !== originalImageName; - const featureCaps = [...new Set(([] as string[]).concat(...(collapsedFeaturesConfig?.allFeatures || []) - .filter(f => (includeAllConfiguredFeatures || f.included) && f.value) - .map(f => f.capAdd || [])))]; - const featureSecurityOpts = [...new Set(([] as string[]).concat(...(collapsedFeaturesConfig?.allFeatures || []) - .filter(f => (includeAllConfiguredFeatures || f.included) && f.value) - .map(f => f.securityOpt || [])))]; - const featureMounts = ([] as Mount[]).concat( - ...(collapsedFeaturesConfig?.allFeatures || []) - .map(f => (includeAllConfiguredFeatures || f.included) && f.value && f.mounts) - .filter(Boolean) as Mount[][], - additionalMounts, - ); - const volumeMounts = featureMounts.filter(m => m.type === 'volume'); - const customEntrypoints = (collapsedFeaturesConfig?.allFeatures || []) - .map(f => (includeAllConfiguredFeatures || f.included) && f.value && f.entrypoint) - .filter(Boolean) as string[]; + const user = mergedConfig.containerUser; + const env = mergedConfig.containerEnv || {}; + const capAdd = mergedConfig.capAdd || []; + const securityOpts = mergedConfig.securityOpt || []; + const mounts = [ + ...mergedConfig.mounts || [], + ...additionalMounts, + ].map(m => typeof m === 'string' ? parseMount(m) : m); + const volumeMounts = mounts.filter(m => m.type === 'volume'); + const customEntrypoints = mergedConfig.entrypoints || []; const composeEntrypoint: string[] | undefined = typeof service.entrypoint === 'string' ? shellQuote.parse(service.entrypoint) : service.entrypoint; const composeCommand: string[] | undefined = typeof service.command === 'string' ? shellQuote.parse(service.command) : service.command; - const userEntrypoint = config.overrideCommand ? [] : composeEntrypoint /* $ already escaped. */ + const { overrideCommand } = mergedConfig; + const userEntrypoint = overrideCommand ? [] : composeEntrypoint /* $ already escaped. */ || ((await imageDetails()).Config.Entrypoint || []).map(c => c.replace(/\$/g, '$$$$')); // $ > $$ to escape docker-compose.yml's interpolation. - const userCommand = config.overrideCommand ? [] : composeCommand /* $ already escaped. */ + const userCommand = overrideCommand ? [] : composeCommand /* $ already escaped. */ || (composeEntrypoint ? [/* Ignore image CMD per docker-compose.yml spec. */] : ((await imageDetails()).Config.Cmd || []).map(c => c.replace(/\$/g, '$$$$'))); // $ > $$ to escape docker-compose.yml's interpolation. composeOverrideContent = `services: @@ -490,16 +503,19 @@ trap \\"exit 0\\" 15\\n ${customEntrypoints.join('\\n\n')}\\n exec \\"$$@\\"\\n while sleep 1 & wait $$!; do :; done", "-"${userEntrypoint.map(a => `, ${JSON.stringify(a)}`).join('')}]${userCommand !== composeCommand ? ` - command: ${JSON.stringify(userCommand)}` : ''}${(collapsedFeaturesConfig?.allFeatures || []).some(f => (includeAllConfiguredFeatures || f.included) && f.value && f.init) ? ` - init: true` : ''}${(collapsedFeaturesConfig?.allFeatures || []).some(f => (includeAllConfiguredFeatures || f.included) && f.value && f.privileged) ? ` - privileged: true` : ''}${featureCaps.length ? ` - cap_add:${featureCaps.map(cap => ` - - ${cap}`).join('')}` : ''}${featureSecurityOpts.length ? ` - security_opt:${featureSecurityOpts.map(securityOpt => ` + command: ${JSON.stringify(userCommand)}` : ''}${mergedConfig.init ? ` + init: true` : ''}${user ? ` + user: ${user}` : ''}${Object.keys(env).length ? ` + environment:${Object.keys(env).map(key => ` + - ${key}=${env[key]}`).join('')}` : ''}${mergedConfig.privileged ? ` + privileged: true` : ''}${capAdd.length ? ` + cap_add:${capAdd.map(cap => ` + - ${cap}`).join('')}` : ''}${securityOpts.length ? ` + security_opt:${securityOpts.map(securityOpt => ` - ${securityOpt}`).join('')}` : ''}${additionalLabels.length ? ` labels:${additionalLabels.map(label => ` - - ${label.replace(/\$/g, '$$$$')}`).join('')}` : ''}${featureMounts.length ? ` - volumes:${featureMounts.map(m => ` + - ${label.replace(/\$/g, '$$$$')}`).join('')}` : ''}${mounts.length ? ` + volumes:${mounts.map(m => ` - ${m.source}:${m.target}`).join('')}` : ''}${volumeMounts.length ? ` volumes:${volumeMounts.map(m => ` ${m.source}:${m.external ? '\n external: true' : ''}`).join('')}` : ''} diff --git a/src/spec-node/dockerfileUtils.ts b/src/spec-node/dockerfileUtils.ts new file mode 100644 index 000000000..45869fa64 --- /dev/null +++ b/src/spec-node/dockerfileUtils.ts @@ -0,0 +1,146 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export { getConfigFilePath, getDockerfilePath, isDockerFileConfig, resolveConfigFilePath } from '../spec-configuration/configuration'; +export { uriToFsPath, parentURI } from '../spec-configuration/configurationCommonUtils'; +export { CLIHostDocuments, Documents, createDocuments, Edit, fileDocuments, RemoteDocuments } from '../spec-configuration/editableFiles'; + + +const findFromLines = new RegExp(/^(?\s*FROM.*)/, 'gm'); +const parseFromLine = /FROM\s+(?--platform=\S+\s+)?(?\S+)(\s+[Aa][Ss]\s+(?