Use content-hash to invalidate cache

packages/hardhat-core/src/builtin-tasks/compile.ts

@@ -1,6 +1,7 @@
 import chalk from "chalk";
 import { exec } from "child_process";
 import debug from "debug";
+import fsExtra from "fs-extra";
 import path from "path";
 import semver from "semver";
 
@@ -69,6 +70,7 @@ import {
   TASK_COMPILE_SOLIDITY_LOG_RUN_COMPILER_END,
   TASK_COMPILE_SOLIDITY_LOG_RUN_COMPILER_START,
   TASK_COMPILE_SOLIDITY_MERGE_COMPILATION_JOBS,
+  TASK_COMPILE_SOLIDITY_READ_FILE,
   TASK_COMPILE_SOLIDITY_RUN_SOLC,
   TASK_COMPILE_SOLIDITY_RUN_SOLCJS,
 } from "./task-names";
@@ -137,6 +139,18 @@ subtask(TASK_COMPILE_SOLIDITY_GET_SOURCE_NAMES)
     }
   );
 
+subtask(TASK_COMPILE_SOLIDITY_READ_FILE)
+  .addParam("absolutePath", undefined, undefined, types.string)
+  .setAction(
+    async ({ absolutePath }: { absolutePath: string }): Promise<string> => {
+      const content = await fsExtra.readFile(absolutePath, {
+        encoding: "utf8",
+      });
+
+      return content;
+    }
+  );
+
 /**
  * Receives a list of source names and returns a dependency graph. This task
  * is responsible for both resolving dependencies (like getting files from
@@ -151,10 +165,15 @@ subtask(TASK_COMPILE_SOLIDITY_GET_DEPENDENCY_GRAPH)
         sourceNames,
         solidityFilesCache,
       }: { sourceNames: string[]; solidityFilesCache?: SolidityFilesCache },
-      { config }
+      { config, run }
     ): Promise<taskTypes.DependencyGraph> => {
       const parser = new Parser(solidityFilesCache);
-      const resolver = new Resolver(config.paths.root, parser);
+      const resolver = new Resolver(
+        config.paths.root,
+        parser,
+        (absolutePath: string) =>
+          run(TASK_COMPILE_SOLIDITY_READ_FILE, { absolutePath })
+      );
 
       const resolvedFiles = await Promise.all(
         sourceNames.map((sn) => resolver.resolveSourceName(sn))
@@ -1185,6 +1204,7 @@ subtask(TASK_COMPILE_SOLIDITY)
         for (const { file, artifactsEmitted } of artifactsEmittedPerFile) {
           solidityFilesCache.addFile(file.absolutePath, {
             lastModificationDate: file.lastModificationDate.valueOf(),
+            contentHash: file.contentHash,
             sourceName: file.sourceName,
             solcConfig: compilationJob.getSolcConfig(),
             imports: file.content.imports,
@@ -1287,7 +1307,7 @@ function needsCompilation(
   for (const file of job.getResolvedFiles()) {
     const hasChanged = cache.hasFileChanged(
       file.absolutePath,
-      file.lastModificationDate,
+      file.contentHash,
       // we only check if the solcConfig is different for files that
       // emit artifacts
       job.emitsArtifacts(file) ? job.getSolcConfig() : undefined

packages/hardhat-core/src/builtin-tasks/task-names.ts

@@ -11,6 +11,7 @@ export const TASK_COMPILE_SOLIDITY_GET_SOURCE_PATHS =
   "compile:solidity:get-source-paths";
 export const TASK_COMPILE_SOLIDITY_GET_SOURCE_NAMES =
   "compile:solidity:get-source-names";
+export const TASK_COMPILE_SOLIDITY_READ_FILE = "compile:solidity:read-file";
 export const TASK_COMPILE_SOLIDITY_GET_DEPENDENCY_GRAPH =
   "compile:solidity:get-dependency-graph";
 export const TASK_COMPILE_SOLIDITY_GET_COMPILATION_JOBS =

packages/hardhat-core/src/builtin-tasks/utils/solidity-files-cache.ts

@@ -1,3 +1,4 @@
+import debug from "debug";
 import fsExtra from "fs-extra";
 import * as t from "io-ts";
 import type { LoDashStatic } from "lodash";
@@ -6,10 +7,13 @@ import * as path from "path";
 import { SOLIDITY_FILES_CACHE_FILENAME } from "../../internal/constants";
 import type { ProjectPathsConfig, SolcConfig } from "../../types";
 
-const FORMAT_VERSION = "hh-sol-cache-1";
+const log = debug("hardhat:core:tasks:compile:cache");
+
+const FORMAT_VERSION = "hh-sol-cache-2";
 
 const CacheEntryCodec = t.type({
   lastModificationDate: t.number,
+  contentHash: t.string,
   sourceName: t.string,
   solcConfig: t.any,
   imports: t.array(t.string),
@@ -24,6 +28,7 @@ const CacheCodec = t.type({
 
 export interface CacheEntry {
   lastModificationDate: number;
+  contentHash: string;
   sourceName: string;
   solcConfig: SolcConfig;
   imports: string[];
@@ -52,30 +57,26 @@ export class SolidityFilesCache {
 
     if (result.isRight()) {
       const solidityFilesCache = new SolidityFilesCache(result.value);
-      await solidityFilesCache.removeModifiedFiles();
+      await solidityFilesCache.removeNonExistingFiles();
       return solidityFilesCache;
     }
 
-    // tslint:disable-next-line only-hardhat-error
-    throw new Error("Couldn't read cache file, try running the clean task"); // TODO use HardhatError
+    log("There was a problem reading the cache");
+
+    return new SolidityFilesCache({
+      _format: FORMAT_VERSION,
+      files: {},
+    });
   }
 
   constructor(private _cache: Cache) {}
 
-  public async removeModifiedFiles() {
-    for (const [absolutePath, cachedData] of Object.entries(
-      this._cache.files
-    )) {
+  public async removeNonExistingFiles() {
+    for (const absolutePath of Object.keys(this._cache.files)) {
       if (!fsExtra.existsSync(absolutePath)) {
         this.removeEntry(absolutePath);
         continue;
       }
-      const stats = await fsExtra.stat(absolutePath);
-      const lastModificationDate = new Date(stats.ctime);
-
-      if (lastModificationDate.valueOf() !== cachedData.lastModificationDate) {
-        this.removeEntry(absolutePath);
-      }
     }
   }
 
@@ -103,7 +104,7 @@ export class SolidityFilesCache {
 
   public hasFileChanged(
     absolutePath: string,
-    lastModificationDate: Date,
+    contentHash: string,
     solcConfig?: SolcConfig
   ): boolean {
     const { isEqual }: LoDashStatic = require("lodash");
@@ -115,7 +116,7 @@ export class SolidityFilesCache {
       return true;
     }
 
-    if (cacheEntry.lastModificationDate < lastModificationDate.valueOf()) {
+    if (cacheEntry.contentHash !== contentHash) {
       return true;
     }
 

packages/hardhat-core/src/internal/solidity/parse.ts

@@ -14,8 +14,12 @@ export class Parser {
 
   constructor(private _solidityFilesCache?: SolidityFilesCache) {}
 
-  public parse(fileContent: string, absolutePath: string): ParsedData {
-    const cacheResult = this._getFromCache(absolutePath);
+  public parse(
+    fileContent: string,
+    absolutePath: string,
+    contentHash: string
+  ): ParsedData {
+    const cacheResult = this._getFromCache(absolutePath, contentHash);
 
     if (cacheResult !== null) {
       return cacheResult;
@@ -55,12 +59,28 @@ export class Parser {
     return result;
   }
 
-  private _getFromCache(absolutePath: string): ParsedData | null {
-    const cacheEntry = this._solidityFilesCache?.getEntry(absolutePath);
+  /**
+   * Get parsed data from the internal cache, or from the solidity files cache.
+   *
+   * Returns null if cannot find it in either one.
+   */
+  private _getFromCache(
+    absolutePath: string,
+    contentHash: string
+  ): ParsedData | null {
+    if (this._solidityFilesCache === undefined) {
+      return this._cache.get(absolutePath) ?? null;
+    }
+
+    const cacheEntry = this._solidityFilesCache.getEntry(absolutePath);
+
+    if (cacheEntry === undefined) {
+      return this._cache.get(absolutePath) ?? null;
+    }
 
-    if (cacheEntry !== undefined) {
-      const { imports, versionPragmas } = cacheEntry;
+    const { imports, versionPragmas } = cacheEntry;
 
+    if (cacheEntry.contentHash === contentHash) {
       return { imports, versionPragmas };
     }
 

packages/hardhat-core/src/internal/solidity/resolver.ts

@@ -17,6 +17,7 @@ import {
 } from "../../utils/source-names";
 import { HardhatError } from "../core/errors";
 import { ERRORS } from "../core/errors-list";
+import { createNonCryptographicHashBasedIdentifier } from "../util/hash";
 
 import { Parser } from "./parse";
 
@@ -33,6 +34,7 @@ export class ResolvedFile implements IResolvedFile {
     public readonly sourceName: string,
     public readonly absolutePath: string,
     public readonly content: FileContent,
+    public readonly contentHash: string,
     public readonly lastModificationDate: Date,
     libraryName?: string,
     libraryVersion?: string
@@ -56,7 +58,8 @@ export class ResolvedFile implements IResolvedFile {
 export class Resolver {
   constructor(
     private readonly _projectRoot: string,
-    private readonly _parser: Parser
+    private readonly _parser: Parser,
+    private readonly _readFile: (absolutePath: string) => Promise<string>
   ) {}
 
   /**
@@ -294,13 +297,19 @@ export class Resolver {
     libraryName?: string,
     libraryVersion?: string
   ): Promise<ResolvedFile> {
-    const rawContent = await fsExtra.readFile(absolutePath, {
-      encoding: "utf8",
-    });
+    const rawContent = await this._readFile(absolutePath);
     const stats = await fsExtra.stat(absolutePath);
     const lastModificationDate = new Date(stats.ctime);
 
-    const parsedContent = this._parser.parse(rawContent, absolutePath);
+    const contentHash = createNonCryptographicHashBasedIdentifier(
+      Buffer.from(rawContent)
+    ).toString("hex");
+
+    const parsedContent = this._parser.parse(
+      rawContent,
+      absolutePath,
+      contentHash
+    );
 
     const content = {
       rawContent,
@@ -311,6 +320,7 @@ export class Resolver {
       sourceName,
       absolutePath,
       content,
+      contentHash,
       lastModificationDate,
       libraryName,
       libraryVersion

packages/hardhat-core/src/types/builtin-tasks/compile.ts

@@ -11,6 +11,7 @@ export interface ResolvedFile {
   // IMPORTANT: Mapped to ctime, NOT mtime. mtime isn't updated when the file
   // properties (e.g. its name) are changed, only when it's content changes.
   lastModificationDate: Date;
+  contentHash: string;
   getVersionedName(): string;
 }
 

packages/hardhat-core/test/builtin-tasks/utils/solidity-files-cache.ts

@@ -5,13 +5,17 @@ import {
   SolidityFilesCache,
 } from "../../../src/builtin-tasks/utils/solidity-files-cache";
 
+const UNMODIFIED_CONTENT_HASH = "<unmodified-content-hash>";
+const MODIFIED_CONTENT_HASH = "<modified-content-hash>";
+
 function mockCachedFile(
   sourceName: string,
   other: Partial<CacheEntry> = {}
 ): CacheEntry {
   return {
     sourceName,
     lastModificationDate: new Date().valueOf(),
+    contentHash: UNMODIFIED_CONTENT_HASH,
     solcConfig: { version: "0.6.6", settings: {} },
     imports: [],
     versionPragmas: [],
@@ -66,7 +70,7 @@ describe("SolidityFilesCache", function () {
 
     const hasChanged = solidityFilesCache.hasFileChanged(
       "/path/to/contracts/file.sol",
-      oneHourAgo,
+      UNMODIFIED_CONTENT_HASH,
       solcConfig
     );
 
@@ -88,7 +92,7 @@ describe("SolidityFilesCache", function () {
 
     const hasChanged = solidityFilesCache.hasFileChanged(
       "/path/to/contracts/file.sol",
-      now,
+      MODIFIED_CONTENT_HASH,
       solcConfig
     );
 
@@ -110,7 +114,7 @@ describe("SolidityFilesCache", function () {
 
     const hasChanged = solidityFilesCache.hasFileChanged(
       "/path/to/contracts/anotherFile.sol",
-      oneHourAgo,
+      UNMODIFIED_CONTENT_HASH,
       solcConfig
     );
 
@@ -132,7 +136,7 @@ describe("SolidityFilesCache", function () {
 
     const hasChanged = solidityFilesCache.hasFileChanged(
       "/path/to/contracts/file.sol",
-      oneHourAgo,
+      UNMODIFIED_CONTENT_HASH,
       { version: "0.6.7", settings: {} }
     );
 
@@ -154,7 +158,7 @@ describe("SolidityFilesCache", function () {
 
     const hasChanged = solidityFilesCache.hasFileChanged(
       "/path/to/contracts/file.sol",
-      oneHourAgo,
+      UNMODIFIED_CONTENT_HASH,
       { version: "0.6.6", settings: { optimizer: true, runs: 200 } }
     );
 
@@ -176,7 +180,7 @@ describe("SolidityFilesCache", function () {
 
     const hasChanged = solidityFilesCache.hasFileChanged(
       "/path/to/contracts/file.sol",
-      oneHourAgo,
+      UNMODIFIED_CONTENT_HASH,
       { version: "0.6.6", settings: {} }
     );
 
@@ -198,7 +202,7 @@ describe("SolidityFilesCache", function () {
 
     const hasChanged = solidityFilesCache.hasFileChanged(
       "/path/to/contracts/file.sol",
-      oneHourAgo
+      UNMODIFIED_CONTENT_HASH
     );
 
     assert.isFalse(hasChanged);
@@ -219,7 +223,7 @@ describe("SolidityFilesCache", function () {
 
     const hasChanged = solidityFilesCache.hasFileChanged(
       "/path/to/contracts/file.sol",
-      now
+      MODIFIED_CONTENT_HASH
     );
 
     assert.isTrue(hasChanged);

packages/hardhat-core/test/internal/solidity/compiler/compiler-input.ts

@@ -48,12 +48,14 @@ describe("compiler-input module", function () {
         sourceName1,
         path1,
         { rawContent: content1, imports: [], versionPragmas: [] },
+        "<content-hash-1>",
         new Date()
       ),
       new ResolvedFile(
         sourceName2,
         path2,
         { rawContent: content2, imports: [], versionPragmas: [] },
+        "<content-hash-2>",
         new Date()
       ),
     ];