extend utils & update compilation process (#396)

Co-authored-by: Raid Ateir <ra@matterlabs.dev>

.gitignore

@@ -25,3 +25,4 @@ l1-contracts/out/*
 l1-contracts-foundry/broadcast/*
 l1-contracts-foundry/script-out/*
 !l1-contracts-foundry/script-out/.gitkeep
+*.timestamp

system-contracts/package.json

@@ -62,7 +62,7 @@
     "compile-zasm": "ts-node scripts/compile-zasm.ts",
     "deploy-preimages": "ts-node scripts/deploy-preimages.ts",
     "preprocess:bootloader": "rm -rf ./bootloader/build && yarn ts-node scripts/preprocess-bootloader.ts",
-    "preprocess:system-contracts": "rm -rf ./contracts-preprocessed && ts-node scripts/preprocess-system-contracts.ts",
+    "preprocess:system-contracts": "ts-node scripts/preprocess-system-contracts.ts",
     "test": "yarn build:test-system-contracts && hardhat test --network zkSyncTestNode",
     "test-node": "hardhat node-zksync --tag v0.0.1-vm1.5.0",
     "test:bootloader": "cd bootloader/test_infra && cargo run"

system-contracts/scripts/compile-yul.ts

@@ -1,11 +1,23 @@
 // hardhat import should be the first import in the file
 import type { CompilerPaths } from "./utils";
-import { spawn, compilerLocation, prepareCompilerPaths, getSolcLocation } from "./utils";
+import {
+  spawn,
+  compilerLocation,
+  prepareCompilerPaths,
+  getSolcLocation,
+  needsRecompilation,
+  setCompilationTime,
+} from "./utils";
 import * as fs from "fs";
 import { Command } from "commander";
+import * as _path from "path";
 
 const COMPILER_VERSION = "1.3.18";
 const IS_COMPILER_PRE_RELEASE = true;
+const CONTRACTS_DIR = "contracts-preprocessed";
+const BOOTLOADER_DIR = "bootloader";
+const TIMESTAMP_FILE_YUL = "last_compilation_yul.timestamp";
+const TIMESTAMP_FILE_BOOTLOADER = "last_compilation_bootloader.timestamp";
 
 export async function compileYul(paths: CompilerPaths, file: string) {
   const solcCompilerPath = await getSolcLocation();
@@ -30,14 +42,34 @@ async function main() {
   program.version("0.1.0").name("compile yul").description("publish preimages for the L2 contracts");
 
   program.command("compile-bootloader").action(async () => {
-    await compileYulFolder("bootloader/build");
-    await compileYulFolder("bootloader/tests");
+    const timestampFilePath = _path.join(process.cwd(), TIMESTAMP_FILE_BOOTLOADER);
+    const folderToCheck = _path.join(process.cwd(), BOOTLOADER_DIR);
+
+    if (needsRecompilation(folderToCheck, timestampFilePath)) {
+      console.log("Compilation needed.");
+      await compileYulFolder("bootloader/build");
+      await compileYulFolder("bootloader/tests");
+      setCompilationTime(timestampFilePath);
+    } else {
+      console.log("Compilation not needed.");
+      return;
+    }
   });
 
   program.command("compile-precompiles").action(async () => {
-    await compileYulFolder("contracts-preprocessed");
-    await compileYulFolder("contracts-preprocessed/precompiles");
-    await compileYulFolder("contracts-preprocessed/precompiles/test-contracts");
+    const timestampFilePath = _path.join(process.cwd(), TIMESTAMP_FILE_YUL);
+    const folderToCheck = _path.join(process.cwd(), CONTRACTS_DIR);
+
+    if (needsRecompilation(folderToCheck, timestampFilePath)) {
+      console.log("Compilation needed.");
+      await compileYulFolder("contracts-preprocessed");
+      await compileYulFolder("contracts-preprocessed/precompiles");
+      await compileYulFolder("contracts-preprocessed/precompiles/test-contracts");
+      setCompilationTime(timestampFilePath);
+    } else {
+      console.log("Compilation not needed.");
+      return;
+    }
   });
 
   await program.parseAsync(process.argv);

system-contracts/scripts/preprocess-system-contracts.ts

@@ -3,9 +3,11 @@ import path from "path";
 import { renderFile } from "template-file";
 import { glob } from "fast-glob";
 import { Command } from "commander";
+import { needsRecompilation, deleteDir, setCompilationTime, isFolderEmpty } from "./utils";
 
 const CONTRACTS_DIR = "contracts";
 const OUTPUT_DIR = "contracts-preprocessed";
+const TIMESTAMP_FILE = "last_compilation_preprocessing.timestamp"; // File to store the last compilation time
 
 const params = {
   SYSTEM_CONTRACTS_OFFSET: "0x8000",
@@ -17,6 +19,18 @@ async function preprocess(testMode: boolean) {
     params.SYSTEM_CONTRACTS_OFFSET = "0x9000";
   }
 
+  const timestampFilePath = path.join(process.cwd(), TIMESTAMP_FILE);
+  const folderToCheck = path.join(process.cwd(), CONTRACTS_DIR);
+
+  if ((await isFolderEmpty(OUTPUT_DIR)) || needsRecompilation(folderToCheck, timestampFilePath) || testMode) {
+    console.log("Preprocessing needed.");
+    deleteDir(OUTPUT_DIR);
+    setCompilationTime(timestampFilePath);
+  } else {
+    console.log("Preprocessing not needed.");
+    return;
+  }
+
   const contracts = await glob(
     [`${CONTRACTS_DIR}/**/*.sol`, `${CONTRACTS_DIR}/**/*.yul`, `${CONTRACTS_DIR}/**/*.zasm`],
     { onlyFiles: true }

system-contracts/scripts/utils.ts

@@ -7,6 +7,7 @@ import type { Deployer } from "@matterlabs/hardhat-zksync-deploy";
 import type { BigNumberish, BytesLike } from "ethers";
 import { BigNumber, ethers } from "ethers";
 import * as fs from "fs";
+import * as fsPr from "fs/promises";
 import { hashBytecode } from "zksync-web3/build/src/utils";
 import type { YulContractDescription, ZasmContractDescription } from "./constants";
 import { Language, SYSTEM_CONTRACTS } from "./constants";
@@ -253,3 +254,75 @@ export function prepareCompilerPaths(path: string): CompilerPaths {
 
   return new CompilerPaths(absolutePathSources, absolutePathArtifacts);
 }
+
+// Get the latest file modification time in the watched folder
+function getLatestModificationTime(folder: string): Date | null {
+  const files = fs.readdirSync(folder);
+  let latestTime: Date | null = null; // Initialize to null to avoid uninitialized variable
+
+  files.forEach((file) => {
+    const filePath = path.join(folder, file);
+    const stats = fs.statSync(filePath);
+    if (stats.isDirectory()) {
+      const dirLatestTime = getLatestModificationTime(filePath);
+      if (dirLatestTime && (!latestTime || dirLatestTime > latestTime)) {
+        latestTime = dirLatestTime;
+      }
+    } else if (stats.isFile()) {
+      if (!latestTime || stats.mtime > latestTime) {
+        latestTime = stats.mtime;
+      }
+    }
+  });
+
+  return latestTime;
+}
+
+// Read the last compilation timestamp from the file
+export function getLastCompilationTime(timestampFile: string): Date | null {
+  try {
+    if (fs.existsSync(timestampFile)) {
+      const timestamp = fs.readFileSync(timestampFile, "utf-8");
+      return new Date(parseInt(timestamp, 10));
+    }
+  } catch (error) {
+    const err = error as Error; // Cast `error` to `Error`
+    console.error(`Error reading timestamp: ${err.message}`);
+  }
+  return null;
+}
+
+// Write the current time to the timestamp file
+export function setCompilationTime(timestampFile: string) {
+  fs.writeFileSync(timestampFile, Date.now().toString());
+}
+
+// Determine if recompilation is needed
+export function needsRecompilation(folder: string, timestampFile: string): boolean {
+  const lastCompilationTime = getLastCompilationTime(timestampFile);
+  const latestModificationTime = getLatestModificationTime(folder);
+  if (!lastCompilationTime) {
+    return true; // If there's no history, always recompile
+  }
+
+  return latestModificationTime! > lastCompilationTime;
+}
+
+export function deleteDir(path: string): void {
+  try {
+    fs.rmSync(path, { recursive: true, force: true }); // 'recursive: true' deletes all contents, 'force: true' prevents errors if the directory doesn't exist
+    console.log(`Directory '${path}' deleted successfully.`);
+  } catch (error) {
+    console.error(`Error deleting directory '${path}':`, error);
+  }
+}
+
+export async function isFolderEmpty(folderPath: string): Promise<boolean> {
+  try {
+    const files = await fsPr.readdir(folderPath); // Get a list of files in the folder
+    return files.length === 0; // If there are no files, the folder is empty
+  } catch (error) {
+    console.error("No target folder with artifacts.");
+    return true; // Return true if an error, as folder doesn't exist.
+  }
+}