diff --git a/main/maven/test/src/mill/main/maven/BuildGenTests.scala b/main/maven/test/src/mill/main/maven/BuildGenTests.scala
index 5f0ebac94e6..f95e59089f3 100644
--- a/main/maven/test/src/mill/main/maven/BuildGenTests.scala
+++ b/main/maven/test/src/mill/main/maven/BuildGenTests.scala
@@ -2,13 +2,19 @@ package mill.main.maven
 
 import mill.T
 import mill.api.PathRef
+import mill.main.client.OutFiles
 import mill.scalalib.scalafmt.ScalafmtModule
 import mill.testkit.{TestBaseModule, UnitTester}
 import utest.*
 import utest.framework.TestPath
 
+import java.nio.file.FileSystems
+
 object BuildGenTests extends TestSuite {
 
+  // Change this to true to update test data on disk
+  def updateSnapshots = false
+
   def tests: Tests = Tests {
     val resources = os.Path(sys.env("MILL_TEST_RESOURCE_DIR"))
     val scalafmtConfigFile = PathRef(resources / ".scalafmt.conf")
@@ -33,7 +39,7 @@ object BuildGenTests extends TestSuite {
       eval(module.reformat())
 
       // test
-      checkFiles(files, resources / expectedRel)
+      checkFiles(files.map(_.path.relativeTo(dest).asSubPath), dest, resources / expectedRel)
     }
 
     // multi level nested modules
@@ -125,15 +131,38 @@ object BuildGenTests extends TestSuite {
       .map(PathRef(_))
       .toSeq
 
-  def checkFiles(actualFiles: Seq[PathRef], expectedRoot: os.Path): Boolean = {
-    val expectedFiles = buildFiles(expectedRoot)
+  def checkFiles(actualFiles: Seq[os.SubPath], root: os.Path, expectedRoot: os.Path): Boolean = {
+    // Non *.mill files, that are not in test data, that we don't want
+    // to see in the diff
+    val toCleanUp = os.walk(root, skip = _.startsWith(root / OutFiles.defaultOut))
+      .filter(os.isFile)
+      .filter(!_.lastOpt.exists(_.endsWith(".mill")))
+    toCleanUp.foreach(os.remove)
+
+    // Try to normalize permissions while not touching those of committed test data
+    val supportsPerms = FileSystems.getDefault.supportedFileAttributeViews().contains("posix")
+    if (supportsPerms)
+      for {
+        testFile <- os.walk(expectedRoot)
+        if os.isFile(testFile)
+        targetFile = root / testFile.relativeTo(expectedRoot).asSubPath
+        if os.isFile(targetFile)
+      }
+        os.perms.set(targetFile, os.perms(testFile))
+
+    val diffExitCode = os.proc("git", "diff", "--no-index", expectedRoot, root)
+      .call(stdin = os.Inherit, stdout = os.Inherit, check = !updateSnapshots)
+      .exitCode
 
-    actualFiles.nonEmpty &&
-    actualFiles.length == expectedFiles.length &&
-    actualFiles.iterator.zip(expectedFiles.iterator).forall {
-      case (actual, expected) =>
-        actual.path.endsWith(expected.path.relativeTo(expectedRoot)) &&
-        os.read.lines(actual.path) == os.read.lines(expected.path)
+    if (updateSnapshots && diffExitCode != 0) {
+      System.err.println(
+        s"Expected and actual files differ, updating expected files in resources under $expectedRoot"
+      )
+
+      os.remove.all(expectedRoot)
+      os.copy(root, expectedRoot)
     }
+
+    diffExitCode == 0 || updateSnapshots
   }
 }