perf: limit open files in generateFSTree (#2458)

Co-authored-by: Pooya Parsa <pooya@pi0.io>

src/core/prerender/prerender.ts

@@ -11,11 +11,11 @@ import { joinURL, withBase, withoutBase } from "ufo";
 import { build } from "../build/build";
 import { createNitro } from "../nitro";
 import { compressPublicAssets } from "../utils/compress";
+import { runParallel } from "../utils/parallel";
 import {
   extractLinks,
   formatPrerenderRoute,
   matchesIgnorePattern,
-  runParallel,
 } from "./utils";
 
 const JsonSigRx = /^\s*["[{]|^\s*-?\d{1,16}(\.\d{1,17})?([Ee][+-]?\d+)?\s*$/; // From unjs/destr

src/core/prerender/utils.ts

@@ -6,43 +6,6 @@ const allowedExtensions = new Set(["", ".json"]);
 
 const linkParents = new Map<string, Set<string>>();
 
-export async function runParallel<T>(
-  inputs: Set<T>,
-  cb: (input: T) => unknown | Promise<unknown>,
-  opts: { concurrency: number; interval: number }
-) {
-  const tasks = new Set<Promise<unknown>>();
-
-  function queueNext(): undefined | Promise<unknown> {
-    const route = inputs.values().next().value;
-    if (!route) {
-      return;
-    }
-
-    inputs.delete(route);
-    const task = new Promise((resolve) => setTimeout(resolve, opts.interval))
-      .then(() => cb(route))
-      .catch((error) => {
-        console.error(error);
-      });
-
-    tasks.add(task);
-    return task.then(() => {
-      tasks.delete(task);
-      if (inputs.size > 0) {
-        return refillQueue();
-      }
-    });
-  }
-
-  function refillQueue(): Promise<unknown> {
-    const workers = Math.min(opts.concurrency - tasks.size, inputs.size);
-    return Promise.all(Array.from({ length: workers }, () => queueNext()));
-  }
-
-  await refillQueue();
-}
-
 const LINK_REGEX = /(?<=\s)href=(?!&quot;)["']?([^"'>]+)/g;
 
 const HTML_ENTITIES = {

src/core/utils/fs-tree.ts

@@ -5,6 +5,7 @@ import { gzipSize } from "gzip-size";
 import { dirname, relative, resolve } from "pathe";
 import prettyBytes from "pretty-bytes";
 import { isTest } from "std-env";
+import { runParallel } from "./parallel";
 
 export async function generateFSTree(
   dir: string,
@@ -16,17 +17,22 @@ export async function generateFSTree(
 
   const files = await globby("**/*.*", { cwd: dir, ignore: ["*.map"] });
 
-  const items = (
-    await Promise.all(
-      files.map(async (file) => {
-        const path = resolve(dir, file);
-        const src = await fsp.readFile(path);
-        const size = src.byteLength;
-        const gzip = options.compressedSizes ? await gzipSize(src) : 0;
-        return { file, path, size, gzip };
-      })
-    )
-  ).sort((a, b) => a.path.localeCompare(b.path));
+  const items: { file: string; path: string; size: number; gzip: number }[] =
+    [];
+
+  await runParallel(
+    new Set(files),
+    async (file) => {
+      const path = resolve(dir, file);
+      const src = await fsp.readFile(path);
+      const size = src.byteLength;
+      const gzip = options.compressedSizes ? await gzipSize(src) : 0;
+      items.push({ file, path, size, gzip });
+    },
+    { concurrency: 10 }
+  );
+
+  items.sort((a, b) => a.path.localeCompare(b.path));
 
   let totalSize = 0;
   let totalGzip = 0;

src/core/utils/parallel.ts

@@ -0,0 +1,40 @@
+export async function runParallel<T>(
+  inputs: Set<T>,
+  cb: (input: T) => unknown | Promise<unknown>,
+  opts: { concurrency: number; interval?: number }
+) {
+  const tasks = new Set<Promise<unknown>>();
+
+  function queueNext(): undefined | Promise<unknown> {
+    const route = inputs.values().next().value;
+    if (!route) {
+      return;
+    }
+
+    inputs.delete(route);
+    const task = (
+      opts.interval
+        ? new Promise((resolve) => setTimeout(resolve, opts.interval))
+        : Promise.resolve()
+    )
+      .then(() => cb(route))
+      .catch((error) => {
+        console.error(error);
+      });
+
+    tasks.add(task);
+    return task.then(() => {
+      tasks.delete(task);
+      if (inputs.size > 0) {
+        return refillQueue();
+      }
+    });
+  }
+
+  function refillQueue(): Promise<unknown> {
+    const workers = Math.min(opts.concurrency - tasks.size, inputs.size);
+    return Promise.all(Array.from({ length: workers }, () => queueNext()));
+  }
+
+  await refillQueue();
+}