Include preprocessing file

src/spec-node/dockerCompose.ts

@@ -19,6 +19,7 @@ import { Mount, parseMount } from '../spec-configuration/containerFeaturesConfig
 import path from 'path';
 import { getDevcontainerMetadata, getImageBuildInfoFromDockerfile, getImageBuildInfoFromImage, getImageMetadataFromContainer, ImageBuildInfo, lifecycleCommandOriginMapFromMetadata, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata';
 import { ensureDockerfileHasFinalStageName } from './dockerfileUtils';
+import { resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess';
 import { randomUUID } from 'crypto';
 
 const projectLabel = 'com.docker.compose.project';
@@ -163,11 +164,16 @@ export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConf
 	let baseName = 'dev_container_auto_added_stage_label';
 	let dockerfile: string | undefined;
 	let imageBuildInfo: ImageBuildInfo;
+	let preprocessedDockerfilePathForComposeBuild: string | undefined;
 	const serviceInfo = getBuildInfoForService(composeService, cliHost.path, localComposeFiles);
 	if (serviceInfo.build) {
 		const { context, dockerfilePath, target } = serviceInfo.build;
 		const resolvedDockerfilePath = cliHost.path.isAbsolute(dockerfilePath) ? dockerfilePath : path.resolve(context, dockerfilePath);
-		const originalDockerfile = (await cliHost.readFile(resolvedDockerfilePath)).toString();
+		const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, resolvedDockerfilePath);
+		const originalDockerfile = resolvedDockerfile.effectiveDockerfileContent;
+		if (resolvedDockerfile.preprocessed) {
+			preprocessedDockerfilePathForComposeBuild = resolvedDockerfile.effectiveDockerfilePath;
+		}
 		dockerfile = originalDockerfile;
 		if (target) {
 			// Explictly set build target for the dev container build features on that
@@ -194,6 +200,10 @@ export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConf
 
 	let overrideImageName: string | undefined;
 	let buildOverrideContent = '';
+	if (preprocessedDockerfilePathForComposeBuild && !extendImageBuildInfo?.featureBuildInfo) {
+		buildOverrideContent += '    build:\n';
+		buildOverrideContent += `      dockerfile: ${preprocessedDockerfilePathForComposeBuild}\n`;
+	}
 	if (extendImageBuildInfo?.featureBuildInfo) {
 		// Avoid retagging a previously pulled image.
 		if (!serviceInfo.build) {

src/spec-node/dockerfilePreprocess.ts

@@ -0,0 +1,84 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { randomUUID } from 'crypto';
+import { CLIHost } from '../spec-common/cliHost';
+import { ContainerError } from '../spec-common/errors';
+
+const includeLine = /^\s*#include\s+"([^"]+)"\s*$/;
+
+export interface ResolvedDockerfile {
+	originalDockerfilePath: string;
+	effectiveDockerfilePath: string;
+	effectiveDockerfileContent: string;
+	preprocessed: boolean;
+}
+
+export async function resolveDockerfileIncludesIfNeeded(cliHost: CLIHost, dockerfilePath: string): Promise<ResolvedDockerfile> {
+	const dockerfileText = (await cliHost.readFile(dockerfilePath)).toString();
+	if (!dockerfilePath.toLowerCase().endsWith('.in')) {
+		return {
+			originalDockerfilePath: dockerfilePath,
+			effectiveDockerfilePath: dockerfilePath,
+			effectiveDockerfileContent: dockerfileText,
+			preprocessed: false,
+		};
+	}
+
+	const effectiveDockerfileContent = await preprocessDockerfileIncludes(cliHost, dockerfilePath, []);
+	const preprocessedDockerfilePath = await writePreprocessedDockerfile(cliHost, dockerfilePath, effectiveDockerfileContent);
+
+	return {
+		originalDockerfilePath: dockerfilePath,
+		effectiveDockerfilePath: preprocessedDockerfilePath,
+		effectiveDockerfileContent,
+		preprocessed: true,
+	};
+}
+
+async function preprocessDockerfileIncludes(cliHost: CLIHost, currentPath: string, stack: string[]): Promise<string> {
+	if (stack.includes(currentPath)) {
+		const chain = [...stack, currentPath].join(' -> ');
+		throw new ContainerError({ description: `Cyclic #include detected while preprocessing Dockerfile: ${chain}` });
+	}
+	if (!(await cliHost.isFile(currentPath))) {
+		throw new ContainerError({ description: `Included Dockerfile not found: ${currentPath}` });
+	}
+
+	const currentText = (await cliHost.readFile(currentPath)).toString();
+	const lines = currentText.split(/\r?\n/);
+	const expanded: string[] = [];
+	const nextStack = [...stack, currentPath];
+	for (const line of lines) {
+		const match = includeLine.exec(line);
+		if (!match) {
+			expanded.push(line);
+			continue;
+		}
+
+		const includePath = match[1];
+		const resolvedIncludePath = cliHost.path.isAbsolute(includePath)
+			? includePath
+			: cliHost.path.resolve(cliHost.path.dirname(currentPath), includePath);
+		expanded.push(await preprocessDockerfileIncludes(cliHost, resolvedIncludePath, nextStack));
+	}
+
+	return expanded.join('\n');
+}
+
+async function writePreprocessedDockerfile(cliHost: CLIHost, sourceDockerfilePath: string, content: string): Promise<string> {
+	const cacheFolder = cliHost.path.join(
+		await cliHost.tmpdir(),
+		cliHost.platform === 'linux' ? `devcontainercli-${await cliHost.getUsername()}` : 'devcontainercli',
+		'dockerfile-preprocess'
+	);
+	await cliHost.mkdirp(cacheFolder);
+
+	const sourceBasename = cliHost.path.basename(sourceDockerfilePath);
+	const targetBasename = sourceBasename.replace(/\.in$/i, '') || 'Dockerfile';
+	const preprocessedDockerfilePath = cliHost.path.join(cacheFolder, `${Date.now()}-${randomUUID()}-${targetBasename}`);
+	await cliHost.writeFile(preprocessedDockerfilePath, Buffer.from(content));
+	return preprocessedDockerfilePath;
+}

src/spec-node/imageMetadata.ts

@@ -12,6 +12,7 @@ import { ContainerDetails, DockerCLIParameters, ImageDetails } from '../spec-shu
 import { Log, LogLevel } from '../spec-utils/log';
 import { getBuildInfoForService, readDockerComposeConfig } from './dockerCompose';
 import { Dockerfile, extractDockerfile, findBaseImage, findUserStatement } from './dockerfileUtils';
+import { resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess';
 import { SubstituteConfig, SubstitutedConfig, DockerResolverParameters, inspectDockerImage, uriToWSLFsPath, envListToObj } from './utils';
 
 const pickConfigProperties: (keyof DevContainerConfig & keyof ImageMetadataEntry)[] = [
@@ -342,7 +343,8 @@ export async function getImageBuildInfo(params: DockerResolverParameters | Docke
 		if (!cliHost.isFile(dockerfilePath)) {
 			throw new ContainerError({ description: `Dockerfile (${dockerfilePath}) not found.` });
 		}
-		const dockerfile = (await cliHost.readFile(dockerfilePath)).toString();
+		const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, dockerfilePath);
+		const dockerfile = resolvedDockerfile.effectiveDockerfileContent;
 		return getImageBuildInfoFromDockerfile(params, dockerfile, config.build?.args || {}, config.build?.target, configWithRaw.substitute);
 
 	} else if ('dockerComposeFile' in config) {
@@ -363,7 +365,8 @@ export async function getImageBuildInfo(params: DockerResolverParameters | Docke
 		if (serviceInfo.build) {
 			const { context, dockerfilePath } = serviceInfo.build;
 			const resolvedDockerfilePath = cliHost.path.isAbsolute(dockerfilePath) ? dockerfilePath : cliHost.path.resolve(context, dockerfilePath);
-			const dockerfile = (await cliHost.readFile(resolvedDockerfilePath)).toString();
+			const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, resolvedDockerfilePath);
+			const dockerfile = resolvedDockerfile.effectiveDockerfileContent;
 			return getImageBuildInfoFromDockerfile(params, dockerfile, serviceInfo.build.args || {}, serviceInfo.build.target, configWithRaw.substitute);
 		} else {
 			return getImageBuildInfoFromImage(params, composeService.image, configWithRaw.substitute);

src/spec-node/singleContainer.ts

@@ -13,6 +13,7 @@ import { LogLevel, Log, makeLog } from '../spec-utils/log';
 import { extendImage, getExtendImageBuildInfo, updateRemoteUserUID } from './containerFeatures';
 import { getDevcontainerMetadata, getImageBuildInfoFromDockerfile, getImageMetadataFromContainer, ImageMetadataEntry, lifecycleCommandOriginMapFromMetadata, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata';
 import { ensureDockerfileHasFinalStageName, generateMountCommand } from './dockerfileUtils';
+import { resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess';
 
 export const hostFolderLabel = 'devcontainer.local_folder'; // used to label containers created from a workspace/folder
 export const configFileLabel = 'devcontainer.config_file';
@@ -130,7 +131,8 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config
 		throw new ContainerError({ description: `Dockerfile (${dockerfilePath}) not found.` });
 	}
 
-	let dockerfile = (await cliHost.readFile(dockerfilePath)).toString();
+	const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, dockerfilePath);
+	let dockerfile = resolvedDockerfile.effectiveDockerfileContent;
 	const originalDockerfile = dockerfile;
 	let baseName = 'dev_container_auto_added_stage_label';
 	if (config.build?.target) {
@@ -149,7 +151,7 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config
 	const imageBuildInfo = await getImageBuildInfoFromDockerfile(buildParams, originalDockerfile, config.build?.args || {}, config.build?.target, configWithRaw.substitute);
 	const extendImageBuildInfo = await getExtendImageBuildInfo(buildParams, configWithRaw, baseName, imageBuildInfo, undefined, additionalFeatures, false);
 
-	let finalDockerfilePath = dockerfilePath;
+	let finalDockerfilePath = resolvedDockerfile.effectiveDockerfilePath;
 	const additionalBuildArgs: string[] = [];
 	if (extendImageBuildInfo?.featureBuildInfo) {
 		const { featureBuildInfo } = extendImageBuildInfo;

src/test/configs/podman-test/.devcontainer.json

@@ -0,0 +1,5 @@
+{
+	"build": {
+		"dockerfile": "cpp.Dockerfile.in"
+	}
+}

src/test/configs/podman-test/cpp.Dockerfile.in

@@ -0,0 +1,2 @@
+#include "tools.Dockerfile"
+RUN apt-get update && apt-get install -y clang

src/test/configs/podman-test/tools.Dockerfile

@@ -0,0 +1,2 @@
+FROM docker.io/debian:latest
+RUN apt-get update && apt-get install -y vim

src/test/dockerfilePreprocess.test.ts

@@ -0,0 +1,93 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *--------------------------------------------------------------------------------------------*/
+
+import { assert } from 'chai';
+import * as fs from 'fs';
+import * as path from 'path';
+import { CLIHost } from '../spec-common/cliHost';
+import { resolveDockerfileIncludesIfNeeded } from '../spec-node/dockerfilePreprocess';
+
+function createMockCLIHost(files: Record<string, string>, platform: NodeJS.Platform = 'linux'): CLIHost {
+	const pathModule = platform === 'win32' ? path.win32 : path.posix;
+	return {
+		type: 'local',
+		platform,
+		arch: 'x64',
+		path: pathModule,
+		cwd: platform === 'win32' ? 'C:\\' : '/',
+		env: {},
+		exec: () => { throw new Error('Not implemented'); },
+		ptyExec: () => { throw new Error('Not implemented'); },
+		homedir: async () => platform === 'win32' ? 'C:\\Users\\test' : '/home/test',
+		tmpdir: async () => platform === 'win32' ? 'C:\\tmp' : '/tmp',
+		isFile: async (filepath: string) => filepath in files,
+		isFolder: async () => false,
+		readFile: async (filepath: string) => {
+			if (!(filepath in files)) {
+				throw new Error(`File not found: ${filepath}`);
+			}
+			return Buffer.from(files[filepath]);
+		},
+		writeFile: async (filepath: string, content: Buffer) => {
+			files[filepath] = content.toString();
+		},
+		rename: async () => { },
+		mkdirp: async () => { },
+		readDir: async () => [],
+		getUsername: async () => 'test',
+		toCommonURI: async () => undefined,
+		connect: () => { throw new Error('Not implemented'); },
+	};
+}
+
+describe('resolveDockerfileIncludesIfNeeded', () => {
+	it('returns source Dockerfile unchanged when not using .in extension', async () => {
+		const files: Record<string, string> = {
+			'/workspace/Dockerfile': 'FROM debian:latest\nRUN echo ok',
+		};
+		const cliHost = createMockCLIHost(files);
+		const result = await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/Dockerfile');
+		assert.isFalse(result.preprocessed);
+		assert.equal(result.effectiveDockerfilePath, '/workspace/Dockerfile');
+		assert.equal(result.effectiveDockerfileContent, files['/workspace/Dockerfile']);
+	});
+
+	it('expands #include lines and writes a generated Dockerfile for .in files', async () => {
+		const podmanTestConfigPath = path.resolve(__dirname, 'configs', 'podman-test');
+		const sourceDockerfilePath = path.join(podmanTestConfigPath, 'cpp.Dockerfile.in');
+		const includedDockerfilePath = path.join(podmanTestConfigPath, 'tools.Dockerfile');
+		const sourceDockerfileContent = fs.readFileSync(sourceDockerfilePath).toString();
+		const includedDockerfileContent = fs.readFileSync(includedDockerfilePath).toString();
+		const files: Record<string, string> = {
+			[sourceDockerfilePath]: sourceDockerfileContent,
+			[includedDockerfilePath]: includedDockerfileContent,
+		};
+		const cliHost = createMockCLIHost(files);
+		const result = await resolveDockerfileIncludesIfNeeded(cliHost, sourceDockerfilePath);
+		assert.isTrue(result.preprocessed);
+		assert.notEqual(result.effectiveDockerfilePath, sourceDockerfilePath);
+		assert.include(result.effectiveDockerfilePath, '/tmp/devcontainercli-test/dockerfile-preprocess/');
+		assert.equal(
+			result.effectiveDockerfileContent,
+			'FROM docker.io/debian:latest\nRUN apt-get update && apt-get install -y vim\nRUN apt-get update && apt-get install -y clang'
+		);
+		assert.equal(files[result.effectiveDockerfilePath], result.effectiveDockerfileContent);
+	});
+
+	it('fails with a clear error when #include has a cycle', async () => {
+		const files: Record<string, string> = {
+			'/workspace/a.Dockerfile.in': '#include "b.Dockerfile"\nRUN echo a',
+			'/workspace/b.Dockerfile': '#include "a.Dockerfile.in"\nRUN echo b',
+		};
+		const cliHost = createMockCLIHost(files);
+		let err: any;
+		try {
+			await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/a.Dockerfile.in');
+		} catch (e) {
+			err = e;
+		}
+		assert.ok(err);
+		assert.include(String(err.message || err), 'Cyclic #include detected while preprocessing Dockerfile');
+	});
+});