From 49aef844927380913e0c00201466d094de58084f Mon Sep 17 00:00:00 2001 From: Mathiyarasy <157102811+Mathiyarasy@users.noreply.github.com> Date: Wed, 13 May 2026 10:27:19 +0000 Subject: [PATCH] Include preprocessing file --- src/spec-node/dockerCompose.ts | 12 ++- src/spec-node/dockerfilePreprocess.ts | 84 +++++++++++++++++ src/spec-node/imageMetadata.ts | 7 +- src/spec-node/singleContainer.ts | 6 +- .../configs/podman-test/.devcontainer.json | 5 + .../configs/podman-test/cpp.Dockerfile.in | 2 + src/test/configs/podman-test/tools.Dockerfile | 2 + src/test/dockerfilePreprocess.test.ts | 93 +++++++++++++++++++ 8 files changed, 206 insertions(+), 5 deletions(-) create mode 100644 src/spec-node/dockerfilePreprocess.ts create mode 100644 src/test/configs/podman-test/.devcontainer.json create mode 100644 src/test/configs/podman-test/cpp.Dockerfile.in create mode 100644 src/test/configs/podman-test/tools.Dockerfile create mode 100644 src/test/dockerfilePreprocess.test.ts diff --git a/src/spec-node/dockerCompose.ts b/src/spec-node/dockerCompose.ts index 8093464cc..38201a188 100644 --- a/src/spec-node/dockerCompose.ts +++ b/src/spec-node/dockerCompose.ts @@ -19,6 +19,7 @@ import { Mount, parseMount } from '../spec-configuration/containerFeaturesConfig import path from 'path'; import { getDevcontainerMetadata, getImageBuildInfoFromDockerfile, getImageBuildInfoFromImage, getImageMetadataFromContainer, ImageBuildInfo, lifecycleCommandOriginMapFromMetadata, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata'; import { ensureDockerfileHasFinalStageName } from './dockerfileUtils'; +import { resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess'; import { randomUUID } from 'crypto'; const projectLabel = 'com.docker.compose.project'; @@ -163,11 +164,16 @@ export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConf let baseName = 'dev_container_auto_added_stage_label'; let dockerfile: string | undefined; let imageBuildInfo: ImageBuildInfo; + let preprocessedDockerfilePathForComposeBuild: string | undefined; const serviceInfo = getBuildInfoForService(composeService, cliHost.path, localComposeFiles); if (serviceInfo.build) { const { context, dockerfilePath, target } = serviceInfo.build; const resolvedDockerfilePath = cliHost.path.isAbsolute(dockerfilePath) ? dockerfilePath : path.resolve(context, dockerfilePath); - const originalDockerfile = (await cliHost.readFile(resolvedDockerfilePath)).toString(); + const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, resolvedDockerfilePath); + const originalDockerfile = resolvedDockerfile.effectiveDockerfileContent; + if (resolvedDockerfile.preprocessed) { + preprocessedDockerfilePathForComposeBuild = resolvedDockerfile.effectiveDockerfilePath; + } dockerfile = originalDockerfile; if (target) { // Explictly set build target for the dev container build features on that @@ -194,6 +200,10 @@ export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConf let overrideImageName: string | undefined; let buildOverrideContent = ''; + if (preprocessedDockerfilePathForComposeBuild && !extendImageBuildInfo?.featureBuildInfo) { + buildOverrideContent += ' build:\n'; + buildOverrideContent += ` dockerfile: ${preprocessedDockerfilePathForComposeBuild}\n`; + } if (extendImageBuildInfo?.featureBuildInfo) { // Avoid retagging a previously pulled image. if (!serviceInfo.build) { diff --git a/src/spec-node/dockerfilePreprocess.ts b/src/spec-node/dockerfilePreprocess.ts new file mode 100644 index 000000000..6c86e813e --- /dev/null +++ b/src/spec-node/dockerfilePreprocess.ts @@ -0,0 +1,84 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { randomUUID } from 'crypto'; +import { CLIHost } from '../spec-common/cliHost'; +import { ContainerError } from '../spec-common/errors'; + +const includeLine = /^\s*#include\s+"([^"]+)"\s*$/; + +export interface ResolvedDockerfile { + originalDockerfilePath: string; + effectiveDockerfilePath: string; + effectiveDockerfileContent: string; + preprocessed: boolean; +} + +export async function resolveDockerfileIncludesIfNeeded(cliHost: CLIHost, dockerfilePath: string): Promise { + const dockerfileText = (await cliHost.readFile(dockerfilePath)).toString(); + if (!dockerfilePath.toLowerCase().endsWith('.in')) { + return { + originalDockerfilePath: dockerfilePath, + effectiveDockerfilePath: dockerfilePath, + effectiveDockerfileContent: dockerfileText, + preprocessed: false, + }; + } + + const effectiveDockerfileContent = await preprocessDockerfileIncludes(cliHost, dockerfilePath, []); + const preprocessedDockerfilePath = await writePreprocessedDockerfile(cliHost, dockerfilePath, effectiveDockerfileContent); + + return { + originalDockerfilePath: dockerfilePath, + effectiveDockerfilePath: preprocessedDockerfilePath, + effectiveDockerfileContent, + preprocessed: true, + }; +} + +async function preprocessDockerfileIncludes(cliHost: CLIHost, currentPath: string, stack: string[]): Promise { + if (stack.includes(currentPath)) { + const chain = [...stack, currentPath].join(' -> '); + throw new ContainerError({ description: `Cyclic #include detected while preprocessing Dockerfile: ${chain}` }); + } + if (!(await cliHost.isFile(currentPath))) { + throw new ContainerError({ description: `Included Dockerfile not found: ${currentPath}` }); + } + + const currentText = (await cliHost.readFile(currentPath)).toString(); + const lines = currentText.split(/\r?\n/); + const expanded: string[] = []; + const nextStack = [...stack, currentPath]; + for (const line of lines) { + const match = includeLine.exec(line); + if (!match) { + expanded.push(line); + continue; + } + + const includePath = match[1]; + const resolvedIncludePath = cliHost.path.isAbsolute(includePath) + ? includePath + : cliHost.path.resolve(cliHost.path.dirname(currentPath), includePath); + expanded.push(await preprocessDockerfileIncludes(cliHost, resolvedIncludePath, nextStack)); + } + + return expanded.join('\n'); +} + +async function writePreprocessedDockerfile(cliHost: CLIHost, sourceDockerfilePath: string, content: string): Promise { + const cacheFolder = cliHost.path.join( + await cliHost.tmpdir(), + cliHost.platform === 'linux' ? `devcontainercli-${await cliHost.getUsername()}` : 'devcontainercli', + 'dockerfile-preprocess' + ); + await cliHost.mkdirp(cacheFolder); + + const sourceBasename = cliHost.path.basename(sourceDockerfilePath); + const targetBasename = sourceBasename.replace(/\.in$/i, '') || 'Dockerfile'; + const preprocessedDockerfilePath = cliHost.path.join(cacheFolder, `${Date.now()}-${randomUUID()}-${targetBasename}`); + await cliHost.writeFile(preprocessedDockerfilePath, Buffer.from(content)); + return preprocessedDockerfilePath; +} diff --git a/src/spec-node/imageMetadata.ts b/src/spec-node/imageMetadata.ts index 60884592e..b4839e2cc 100644 --- a/src/spec-node/imageMetadata.ts +++ b/src/spec-node/imageMetadata.ts @@ -12,6 +12,7 @@ import { ContainerDetails, DockerCLIParameters, ImageDetails } from '../spec-shu import { Log, LogLevel } from '../spec-utils/log'; import { getBuildInfoForService, readDockerComposeConfig } from './dockerCompose'; import { Dockerfile, extractDockerfile, findBaseImage, findUserStatement } from './dockerfileUtils'; +import { resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess'; import { SubstituteConfig, SubstitutedConfig, DockerResolverParameters, inspectDockerImage, uriToWSLFsPath, envListToObj } from './utils'; const pickConfigProperties: (keyof DevContainerConfig & keyof ImageMetadataEntry)[] = [ @@ -342,7 +343,8 @@ export async function getImageBuildInfo(params: DockerResolverParameters | Docke if (!cliHost.isFile(dockerfilePath)) { throw new ContainerError({ description: `Dockerfile (${dockerfilePath}) not found.` }); } - const dockerfile = (await cliHost.readFile(dockerfilePath)).toString(); + const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, dockerfilePath); + const dockerfile = resolvedDockerfile.effectiveDockerfileContent; return getImageBuildInfoFromDockerfile(params, dockerfile, config.build?.args || {}, config.build?.target, configWithRaw.substitute); } else if ('dockerComposeFile' in config) { @@ -363,7 +365,8 @@ export async function getImageBuildInfo(params: DockerResolverParameters | Docke if (serviceInfo.build) { const { context, dockerfilePath } = serviceInfo.build; const resolvedDockerfilePath = cliHost.path.isAbsolute(dockerfilePath) ? dockerfilePath : cliHost.path.resolve(context, dockerfilePath); - const dockerfile = (await cliHost.readFile(resolvedDockerfilePath)).toString(); + const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, resolvedDockerfilePath); + const dockerfile = resolvedDockerfile.effectiveDockerfileContent; return getImageBuildInfoFromDockerfile(params, dockerfile, serviceInfo.build.args || {}, serviceInfo.build.target, configWithRaw.substitute); } else { return getImageBuildInfoFromImage(params, composeService.image, configWithRaw.substitute); diff --git a/src/spec-node/singleContainer.ts b/src/spec-node/singleContainer.ts index 1c3669f74..ed926d91f 100644 --- a/src/spec-node/singleContainer.ts +++ b/src/spec-node/singleContainer.ts @@ -13,6 +13,7 @@ import { LogLevel, Log, makeLog } from '../spec-utils/log'; import { extendImage, getExtendImageBuildInfo, updateRemoteUserUID } from './containerFeatures'; import { getDevcontainerMetadata, getImageBuildInfoFromDockerfile, getImageMetadataFromContainer, ImageMetadataEntry, lifecycleCommandOriginMapFromMetadata, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata'; import { ensureDockerfileHasFinalStageName, generateMountCommand } from './dockerfileUtils'; +import { resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess'; export const hostFolderLabel = 'devcontainer.local_folder'; // used to label containers created from a workspace/folder export const configFileLabel = 'devcontainer.config_file'; @@ -130,7 +131,8 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config throw new ContainerError({ description: `Dockerfile (${dockerfilePath}) not found.` }); } - let dockerfile = (await cliHost.readFile(dockerfilePath)).toString(); + const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, dockerfilePath); + let dockerfile = resolvedDockerfile.effectiveDockerfileContent; const originalDockerfile = dockerfile; let baseName = 'dev_container_auto_added_stage_label'; if (config.build?.target) { @@ -149,7 +151,7 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config const imageBuildInfo = await getImageBuildInfoFromDockerfile(buildParams, originalDockerfile, config.build?.args || {}, config.build?.target, configWithRaw.substitute); const extendImageBuildInfo = await getExtendImageBuildInfo(buildParams, configWithRaw, baseName, imageBuildInfo, undefined, additionalFeatures, false); - let finalDockerfilePath = dockerfilePath; + let finalDockerfilePath = resolvedDockerfile.effectiveDockerfilePath; const additionalBuildArgs: string[] = []; if (extendImageBuildInfo?.featureBuildInfo) { const { featureBuildInfo } = extendImageBuildInfo; diff --git a/src/test/configs/podman-test/.devcontainer.json b/src/test/configs/podman-test/.devcontainer.json new file mode 100644 index 000000000..8bd3f065a --- /dev/null +++ b/src/test/configs/podman-test/.devcontainer.json @@ -0,0 +1,5 @@ +{ + "build": { + "dockerfile": "cpp.Dockerfile.in" + } +} diff --git a/src/test/configs/podman-test/cpp.Dockerfile.in b/src/test/configs/podman-test/cpp.Dockerfile.in new file mode 100644 index 000000000..fb7b042ed --- /dev/null +++ b/src/test/configs/podman-test/cpp.Dockerfile.in @@ -0,0 +1,2 @@ +#include "tools.Dockerfile" +RUN apt-get update && apt-get install -y clang \ No newline at end of file diff --git a/src/test/configs/podman-test/tools.Dockerfile b/src/test/configs/podman-test/tools.Dockerfile new file mode 100644 index 000000000..faccdf07a --- /dev/null +++ b/src/test/configs/podman-test/tools.Dockerfile @@ -0,0 +1,2 @@ +FROM docker.io/debian:latest +RUN apt-get update && apt-get install -y vim \ No newline at end of file diff --git a/src/test/dockerfilePreprocess.test.ts b/src/test/dockerfilePreprocess.test.ts new file mode 100644 index 000000000..e6a042093 --- /dev/null +++ b/src/test/dockerfilePreprocess.test.ts @@ -0,0 +1,93 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import { assert } from 'chai'; +import * as fs from 'fs'; +import * as path from 'path'; +import { CLIHost } from '../spec-common/cliHost'; +import { resolveDockerfileIncludesIfNeeded } from '../spec-node/dockerfilePreprocess'; + +function createMockCLIHost(files: Record, platform: NodeJS.Platform = 'linux'): CLIHost { + const pathModule = platform === 'win32' ? path.win32 : path.posix; + return { + type: 'local', + platform, + arch: 'x64', + path: pathModule, + cwd: platform === 'win32' ? 'C:\\' : '/', + env: {}, + exec: () => { throw new Error('Not implemented'); }, + ptyExec: () => { throw new Error('Not implemented'); }, + homedir: async () => platform === 'win32' ? 'C:\\Users\\test' : '/home/test', + tmpdir: async () => platform === 'win32' ? 'C:\\tmp' : '/tmp', + isFile: async (filepath: string) => filepath in files, + isFolder: async () => false, + readFile: async (filepath: string) => { + if (!(filepath in files)) { + throw new Error(`File not found: ${filepath}`); + } + return Buffer.from(files[filepath]); + }, + writeFile: async (filepath: string, content: Buffer) => { + files[filepath] = content.toString(); + }, + rename: async () => { }, + mkdirp: async () => { }, + readDir: async () => [], + getUsername: async () => 'test', + toCommonURI: async () => undefined, + connect: () => { throw new Error('Not implemented'); }, + }; +} + +describe('resolveDockerfileIncludesIfNeeded', () => { + it('returns source Dockerfile unchanged when not using .in extension', async () => { + const files: Record = { + '/workspace/Dockerfile': 'FROM debian:latest\nRUN echo ok', + }; + const cliHost = createMockCLIHost(files); + const result = await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/Dockerfile'); + assert.isFalse(result.preprocessed); + assert.equal(result.effectiveDockerfilePath, '/workspace/Dockerfile'); + assert.equal(result.effectiveDockerfileContent, files['/workspace/Dockerfile']); + }); + + it('expands #include lines and writes a generated Dockerfile for .in files', async () => { + const podmanTestConfigPath = path.resolve(__dirname, 'configs', 'podman-test'); + const sourceDockerfilePath = path.join(podmanTestConfigPath, 'cpp.Dockerfile.in'); + const includedDockerfilePath = path.join(podmanTestConfigPath, 'tools.Dockerfile'); + const sourceDockerfileContent = fs.readFileSync(sourceDockerfilePath).toString(); + const includedDockerfileContent = fs.readFileSync(includedDockerfilePath).toString(); + const files: Record = { + [sourceDockerfilePath]: sourceDockerfileContent, + [includedDockerfilePath]: includedDockerfileContent, + }; + const cliHost = createMockCLIHost(files); + const result = await resolveDockerfileIncludesIfNeeded(cliHost, sourceDockerfilePath); + assert.isTrue(result.preprocessed); + assert.notEqual(result.effectiveDockerfilePath, sourceDockerfilePath); + assert.include(result.effectiveDockerfilePath, '/tmp/devcontainercli-test/dockerfile-preprocess/'); + assert.equal( + result.effectiveDockerfileContent, + 'FROM docker.io/debian:latest\nRUN apt-get update && apt-get install -y vim\nRUN apt-get update && apt-get install -y clang' + ); + assert.equal(files[result.effectiveDockerfilePath], result.effectiveDockerfileContent); + }); + + it('fails with a clear error when #include has a cycle', async () => { + const files: Record = { + '/workspace/a.Dockerfile.in': '#include "b.Dockerfile"\nRUN echo a', + '/workspace/b.Dockerfile': '#include "a.Dockerfile.in"\nRUN echo b', + }; + const cliHost = createMockCLIHost(files); + let err: any; + try { + await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/a.Dockerfile.in'); + } catch (e) { + err = e; + } + assert.ok(err); + assert.include(String(err.message || err), 'Cyclic #include detected while preprocessing Dockerfile'); + }); +});