Adds AutoCopy plugin #3552

package.json

@@ -120,7 +120,7 @@
 		"@11ty/eleventy-utils": "2.0.0-alpha.2",
 		"@11ty/lodash-custom": "^4.17.21",
 		"@11ty/posthtml-urls": "^1.0.0",
-		"@11ty/recursive-copy": "^3.0.0",
+		"@11ty/recursive-copy": "^3.0.1",
 		"@sindresorhus/slugify": "^2.2.1",
 		"bcp-47-normalize": "^2.3.0",
 		"chardet": "^2.0.0",

src/Eleventy.js

@@ -40,6 +40,7 @@ import RenderPlugin, * as RenderPluginExtras from "./Plugins/RenderPlugin.js";
 import I18nPlugin, * as I18nPluginExtras from "./Plugins/I18nPlugin.js";
 import HtmlBasePlugin, * as HtmlBasePluginExtras from "./Plugins/HtmlBasePlugin.js";
 import { TransformPlugin as InputPathToUrlTransformPlugin } from "./Plugins/InputPathToUrl.js";
+import { AutoCopyPlugin } from "./Plugins/AutoCopyPlugin.js";
 import { IdAttributePlugin } from "./Plugins/IdAttributePlugin.js";
 
 const pkg = getEleventyPackageJson();
@@ -521,9 +522,9 @@ class Eleventy {
 		}
 		this.templateData.setFileSystemSearch(this.fileSystemSearch);
 
-		// TODO swap this to getters
 		this.passthroughManager = new TemplatePassthroughManager(this.eleventyConfig);
 		this.passthroughManager.setRunMode(this.runMode);
+		this.passthroughManager.setDryRun(this.isDryRun);
 		this.passthroughManager.extensionMap = this.extensionMap;
 		this.passthroughManager.setFileSystemSearch(this.fileSystemSearch);
 
@@ -1489,4 +1490,9 @@ export {
 	 * @type {module:11ty/eleventy/Plugins/IdAttributePlugin}
 	 */
 	IdAttributePlugin,
+
+	/**
+	 * @type {module:11ty/eleventy/Plugins/AutoCopyPlugin}
+	 */
+	AutoCopyPlugin,
 };

src/Plugins/AutoCopyPlugin.js

@@ -0,0 +1,50 @@
+import { AutoCopy } from "../Util/AutoCopy.js";
+
+function AutoCopyPlugin(eleventyConfig, defaultOptions = {}) {
+	let opts = Object.assign(
+		{
+			extensions: "html",
+			match: false,
+			paths: [], // directories to also look in for files
+			failOnError: true, // fails when a path matches (via `match`) but not found on file system
+			copyOptions: undefined,
+		},
+		defaultOptions,
+	);
+
+	let ac = eleventyConfig.autoCopy;
+	if (!ac) {
+		ac = new AutoCopy();
+		ac.setUserConfig(eleventyConfig);
+		eleventyConfig.autoCopy = ac;
+
+		ac.setFailOnError(opts.failOnError);
+		ac.setCopyOptions(opts.copyOptions);
+
+		eleventyConfig.htmlTransformer.addUrlTransform(
+			opts.extensions,
+			function (targetFilepathOrUrl) {
+				// @ts-ignore
+				ac.copy(targetFilepathOrUrl, this.page.inputPath, this.page.outputPath);
+
+				// TODO front matter option for manual copy
+
+				return targetFilepathOrUrl;
+			},
+			{
+				enabled: () => ac.isEnabled(),
+				// - MUST run after other plugins but BEFORE HtmlBase plugin
+				priority: -1,
+			},
+		);
+	}
+
+	ac.addMatchingGlob(opts.match);
+	ac.addPaths(opts.paths);
+}
+
+Object.defineProperty(AutoCopyPlugin, "eleventyPackage", {
+	value: "@11ty/eleventy/autocopy-transform-plugin",
+});
+
+export { AutoCopyPlugin };

src/Plugins/HtmlBasePlugin.js

@@ -137,7 +137,7 @@ function eleventyHtmlBasePlugin(eleventyConfig, defaultOptions = {}) {
 			});
 		},
 		{
-			priority: -1, // run last (especially after PathToUrl transform)
+			priority: -2, // priority is descending, so this runs last (especially after AutoCopy and InputPathToUrl transform)
 			enabled: function (context) {
 				// Enabled when pathPrefix is non-default or via renderTransforms
 				return Boolean(context.baseHref) || opts.baseHref !== "/";

src/TemplatePassthrough.js

@@ -19,12 +19,13 @@ class TemplatePassthrough {
 	#benchmarks;
 	#isAlreadyNormalized = false;
 
-	// paths already guaranteed (probably from the autocopy plugin)
-	static normalizedFactory(inputPath, outputPath, opts = {}) {
+	// paths already guaranteed from the autocopy plugin
+	static factory(inputPath, outputPath, opts = {}) {
 		let p = new TemplatePassthrough(
 			{
 				inputPath,
 				outputPath,
+				copyOptions: opts.copyOptions,
 			},
 			opts.templateConfig,
 		);
@@ -78,6 +79,7 @@ class TemplatePassthrough {
 		return this.templateConfig.directories.output;
 	}
 
+	// Skips `getFiles()` normalization
 	setIsAlreadyNormalized(isNormalized) {
 		this.#isAlreadyNormalized = Boolean(isNormalized);
 	}
@@ -255,6 +257,7 @@ class TemplatePassthrough {
 		let fileSizeCount = 0;
 		let map = {};
 		let b = this.benchmarks.aggregate.get("Passthrough Copy File");
+
 		// returns a promise
 		return copy(src, dest, copyOptions)
 			.on(copy.events.COPY_FILE_START, (copyOp) => {
@@ -268,13 +271,28 @@ class TemplatePassthrough {
 				fileSizeCount += copyOp.stats.size;
 				b.after();
 			})
-			.then(() => {
-				return {
-					count: fileCopyCount,
-					size: fileSizeCount,
-					map,
-				};
-			});
+			.then(
+				() => {
+					return {
+						count: fileCopyCount,
+						size: fileSizeCount,
+						map,
+					};
+				},
+				(error) => {
+					if (copyOptions.overwrite === false && error.code === "EEXIST") {
+						// just ignore if the output already exists and overwrite: false
+						debug("Overwrite error ignored: %O", error);
+						return {
+							count: 0,
+							size: 0,
+							map,
+						};
+					}
+
+					return Promise.reject(error);
+				},
+			);
 	}
 
 	async write() {

src/TemplatePassthroughManager.js

@@ -7,22 +7,39 @@ import EleventyBaseError from "./Errors/EleventyBaseError.js";
 import TemplatePassthrough from "./TemplatePassthrough.js";
 import checkPassthroughCopyBehavior from "./Util/PassthroughCopyBehaviorCheck.js";
 import { isGlobMatch } from "./Util/GlobMatcher.js";
+import { withResolvers } from "./Util/PromiseUtil.js";
 
 const debug = debugUtil("Eleventy:TemplatePassthroughManager");
 
-class TemplatePassthroughManagerConfigError extends EleventyBaseError {}
 class TemplatePassthroughManagerCopyError extends EleventyBaseError {}
 
 class TemplatePassthroughManager {
 	#isDryRun = false;
+	#afterBuild;
+	#queue = new Map();
 
 	constructor(templateConfig) {
 		if (!templateConfig || templateConfig.constructor.name !== "TemplateConfig") {
-			throw new TemplatePassthroughManagerConfigError("Missing or invalid `config` argument.");
+			throw new Error("Internal error: Missing or invalid `templateConfig` argument.");
 		}
 
 		this.templateConfig = templateConfig;
 		this.config = templateConfig.getConfig();
+
+		// eleventy# event listeners are removed on each build
+		this.config.events.on("eleventy#copy", ({ source, target, options }) => {
+			this.enqueueCopy(source, target, options);
+		});
+
+		this.config.events.on("eleventy#beforerender", () => {
+			this.#afterBuild = withResolvers();
+		});
+
+		this.config.events.on("eleventy#render", () => {
+			let { resolve } = this.#afterBuild;
+			resolve();
+		});
+
 		this.reset();
 	}
 
@@ -31,7 +48,8 @@ class TemplatePassthroughManager {
 		this.size = 0;
 		this.conflictMap = {};
 		this.incrementalFile;
-		debug("Resetting counts to 0");
+
+		this.#queue = new Map();
 	}
 
 	set extensionMap(extensionMap) {
@@ -288,16 +306,49 @@ class TemplatePassthroughManager {
 		return entries;
 	}
 
-	// Performance note: these can actually take a fair bit of time, but aren’t a
-	// bottleneck to eleventy. The copies are performed asynchronously and don’t affect eleventy
-	// write times in a significant way.
+	async #waitForTemplatesRendered() {
+		if (!this.#afterBuild) {
+			return Promise.resolve(); // immediately resolve
+		}
+
+		let { promise } = this.#afterBuild;
+		return promise;
+	}
+
+	enqueueCopy(source, target, copyOptions) {
+		let key = `${source}=>${target}`;
+
+		// light de-dupe the same source/target combo (might be in the same file, might be viaTransforms)
+		if (this.#queue.has(key)) {
+			return;
+		}
+
+		let passthrough = TemplatePassthrough.factory(source, target, {
+			templateConfig: this.templateConfig,
+			normalized: true,
+			copyOptions,
+		});
+
+		passthrough.setRunMode(this.runMode);
+		passthrough.setDryRun(this.#isDryRun);
+
+		this.#queue.set(key, this.copyPassthrough(passthrough));
+	}
+
 	async copyAll(templateExtensionPaths) {
 		debug("TemplatePassthrough copy started.");
 		let normalizedPaths = this.getAllNormalizedPaths(templateExtensionPaths);
 
 		let passthroughs = normalizedPaths.map((path) => this.getTemplatePassthroughForPath(path));
 
 		let promises = passthroughs.map((pass) => this.copyPassthrough(pass));
+
+		await this.#waitForTemplatesRendered();
+
+		for (let [key, afterBuildCopyPromises] of this.#queue) {
+			promises.push(afterBuildCopyPromises);
+		}
+
 		return Promise.all(promises).then(async (results) => {
 			let aliases = this.getAliasesFromPassthroughResults(results);
 			await this.config.events.emit("eleventy.passthrough", {

src/TemplateWriter.js

@@ -429,18 +429,20 @@ class TemplateWriter {
 
 	async write() {
 		let paths = await this._getAllPaths();
-		let promises = [];
 
-		// The ordering here is important to destructuring in Eleventy->_watch
-		promises.push(this.writePassthroughCopy(paths));
+		// This must happen before writePassthroughCopy
+		this.templateConfig.userConfig.emit("eleventy#beforerender");
 
-		promises.push(...(await this.generateTemplates(paths)));
+		let aggregatePassthroughCopyPromise = this.writePassthroughCopy(paths);
 
-		return Promise.all(promises).then(
-			async ([passthroughCopyResults, ...templateResults]) => {
-				// TODO wait for afterBuildCopy to finish
-				// console.log( "AFTER??", passthroughCopyResults );
+		let templatesPromise = Promise.all(await this.generateTemplates(paths)).then((results) => {
+			this.templateConfig.userConfig.emit("eleventy#render");
+
+			return results;
+		});
 
+		return Promise.all([aggregatePassthroughCopyPromise, templatesPromise]).then(
+			async ([passthroughCopyResults, templateResults]) => {
 				return {
 					passthroughCopy: passthroughCopyResults,
 					// New in 3.0: flatten and filter out falsy templates