From 32edd8bd8d65ef09c8ef1eb293bcda4fa1de56ea Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Mon, 16 Dec 2024 08:32:48 +0800 Subject: [PATCH] Overhaul `out-dir.adoc` (#4139) * Converted it to an example test with `find` and `cat` to assert on the file layout and contents * Added `mill-dependency-tree.json` and `mill-invalidation-tree.json` introduced in https://github.com/com-lihaoyi/mill/pull/4136, so at least now it's somewhat discoverable --- .../ROOT/pages/fundamentals/out-dir.adoc | 113 +------- .../ROOT/pages/large/selective-execution.adoc | 11 + .../out-dir/1-out-files/build.mill | 247 ++++++++++++++++++ .../out-dir/1-out-files/foo/src/foo/Foo.java | 7 + .../{1-custom-out => 2-custom-out}/build.mill | 0 .../client/src/mill/main/client/OutFiles.java | 4 +- main/eval/src/mill/eval/EvaluatorCore.scala | 4 +- 7 files changed, 271 insertions(+), 115 deletions(-) create mode 100644 example/fundamentals/out-dir/1-out-files/build.mill create mode 100644 example/fundamentals/out-dir/1-out-files/foo/src/foo/Foo.java rename example/fundamentals/out-dir/{1-custom-out => 2-custom-out}/build.mill (100%) diff --git a/docs/modules/ROOT/pages/fundamentals/out-dir.adoc b/docs/modules/ROOT/pages/fundamentals/out-dir.adoc index a323d4d008d..5bd7501ccdc 100644 --- a/docs/modules/ROOT/pages/fundamentals/out-dir.adoc +++ b/docs/modules/ROOT/pages/fundamentals/out-dir.adoc @@ -3,118 +3,9 @@ include::partial$gtag-config.adoc[] -Mill puts all its output in the top-level `out/` folder. -== Structure of the `out/` Directory - -The `out/` folder contains all the generated files & metadata for your build. -It holds some files needed to manage Mill's longer running server instances (`out/mill-server/*`) as well as a directory and file structure resembling the project's module structure. - -.Example of the `out/` directory after running `mill main.compile` -[source,text] ----- -out/ -├── main/ <1> -│ ├── allScalacOptions.json -│ ├── allSourceFiles.json -│ ├── allSources.json -│ ├── compile.dest/ <2> -│ ├── compile.json -│ ├── compile.log <3> -│ ├── compileClasspath.json -│ ├── compileIvyDeps.json -│ ├── enablePluginScalacOptions.json -│ ├── generatedSources.json -│ ├── ivyDeps.json -│ ├── javacOptions.json -│ ├── mandatoryIvyDeps.json -│ ├── mandatoryIvyDeps.super/ <4> -│ ├── mandatoryScalacOptions.json -│ ├── platformSuffix.json -│ ├── resolvedIvyDeps.json -│ ├── resolvedIvyDeps.log <3> -│ ├── resources.json -│ ├── scalaCompilerClasspath.json -│ ├── scalaLibraryIvyDeps.json -│ ├── scalaOrganization.json -│ ├── scalaVersion.json -│ ├── scalacOptions.json -│ ├── scalacOptions.super/ <4> -│ ├── scalacPluginClasspath.json -│ ├── scalacPluginIvyDeps.json -│ ├── scalacPluginIvyDeps.super/ <4> -│ ├── sources.json -│ ├── transitiveCompileIvyDeps.json -│ ├── transitiveIvyDeps.json -│ ├── transitiveLocalClasspath.json -│ ├── unmanagedClasspath.json -│ └── upstreamCompileOutput.json -├── mill-profile.json -└── mill-server/VpZubuAK6LQHHN+3ojh1LsTZqWY=-1/ ----- - -<1> The `main` directory contains all files associated with tasks and submodules of the `main` module. -<2> The `compile` task has tried to access its scratch space via `Task.dest`. Here you will find the actual compile results. -<3> Two tasks printed something out while they ran. You can find these outputs in the `*.log` files. -<4> Three tasks are overridden but re-use the result of their `super`-tasks in some way. You can find these result under the `*.super/` path. - -== Task Metadata and Cached Files - -Each named task (``Target`` or ``Command``) that is run has a representation in the `out/` directory structure. - -The _module_ structure is reflected in the directories, so that each module of your project has a uniquely associated subdirectory under the `out/` directory. - -Each _task_ is associated with one or multiple files and directories under its module directory. -The following files can be found for a task `foo`: - -`foo.json`:: - the cache-key and JSON-serialized return-value of the -`Task`/`Command`. -The return-value can also be retrieved via `mill show foo.compile`. -Binary blobs are typically not included in `foo.json`, and instead stored as separate binary files in `foo.dest/` which are then referenced -by `foo.json` via `PathRef` references. - -`foo.dest/`:: - optional, a path for the `Task` to use either as a scratch space, or to place generated files that are returned -using `PathRef` references. -A `Task` should only output files within its own given `foo.dest/` folder (available as `Task.dest`) to avoid -conflicting with another `Task`, but can name files within `foo.dest/` arbitrarily. - -`foo.log`:: - optional, the `stdout`/`stderr` of the `Task`. This is also streamed to the console during evaluation. - -`foo.super/`:: - optional, holds task metadata for overridden tasks, so whenever you use a `super.foo()` in your `foo` task, you will find the metadata of the inherited task(s) under this directory. - - -The `out/` folder is intentionally kept simple and user-readable. -If your build is not behaving as you would expect, -feel free to poke around the various -`foo.dest/` folders to see what files are being created, or the `foo.json` files to see what is being returned by a -particular task. -You can also simply delete folders within `out/` if you want to force portions of your project to be -rebuilt, e.g. by deleting the `+out/main/+` or `+out/main/compile.*+` folders, but we strongly encourage you to use the xref:cli/builtin-commands.adoc#_clean[`clean` command] instead. - -[WARNING] --- -Cleaning some task state by manually deleting files under `out/` may be convenient, but you need to be careful to always delete the `foo.json` file whenever you delete a `foo.dest/` or `foo.super/`. Otherwise, you risk running into hard to diagnose issues later. - -Instead, you should always give the `clean` command a try before manually deleting some file under `out/`. --- -== Other files in the `out/` directory - -There are also top-level build-related files in the `out/` folder, prefixed as `mill-*`. - -`mill-profile.json`:: - Probably the most useful file for you. It logs the tasks run and time taken for the last Mill command you executed. -This is very useful if Mill is being unexpectedly slow, and you want to find out exactly what tasks are being run. - -`mill-chrome-profile.json`:: - This file is only written if you run Mill in parallel mode, e.g. `mill --jobs 4`. This file can be opened in Google Chrome with the built-in `tracing:` protocol even while Mill is still running, so you get a nice chart of what's going on in parallel. - -`mill-server/*`:: - Each Mill server instance needs to keep some temporary files in one of these directories. Deleting it will also terminate the associated server instance, if it is still running. +include::partial$example/fundamentals/out-dir/1-out-files.adoc[] == Using another location than the `out/` directory -include::partial$example/fundamentals/out-dir/1-custom-out.adoc[] +include::partial$example/fundamentals/out-dir/2-custom-out.adoc[] diff --git a/docs/modules/ROOT/pages/large/selective-execution.adoc b/docs/modules/ROOT/pages/large/selective-execution.adoc index 75d7c41965b..5b4d5e3ce8d 100644 --- a/docs/modules/ROOT/pages/large/selective-execution.adoc +++ b/docs/modules/ROOT/pages/large/selective-execution.adoc @@ -42,3 +42,14 @@ def myProjectVersion: T[String] = Task.Input { `upload-artifact`/`download-artifact` https://github.com/actions/download-artifact#permission-loss[does not preserve filesystem permissions]. If this is an issue, you can run `chmod -R . 777` before each of `selective.{prepare,run}` to ensure they have the exact same filesystem permissions. + + +== Debugging Selective Execution + +* Use `selective.resolve` before `selective.run`: this will print out what it was going to run, + and can give you a chance to eyeball if the list of targets to run makes sense or not + +* Look at xref:fundamentals/out-dir.adoc#_mill_invalidation_tree_json[out/mill-invalidation-tree.json], + whether on disk locally or printing it out (e.g via `cat`) on your CI machines to diagnose issues + there. This would give you a richer view of what source tasks or inputs are the ones actually + triggered the invalidation, and what tasks were just invalidated due to being downstream of them. \ No newline at end of file diff --git a/example/fundamentals/out-dir/1-out-files/build.mill b/example/fundamentals/out-dir/1-out-files/build.mill new file mode 100644 index 00000000000..90516cb1971 --- /dev/null +++ b/example/fundamentals/out-dir/1-out-files/build.mill @@ -0,0 +1,247 @@ +// Mill puts all its output in the top-level `out/` folder. +// +// The `out/` folder contains all the generated files & metadata for your build. +// It holds some files needed to manage Mill's longer running server instances +// (`out/mill-server/*`) as well as a directory and file structure resembling the +// project's module structure. +// +// For the purposes of this page, we will be using the following minimal Mill build + +package build +import mill._, javalib._ + +object foo extends JavaModule {} + +/** Usage + +> ./mill foo.compile # compile once + +> echo "" >> foo/src/foo/Foo.java + +> ./mill foo.compile # compile against after editing a file + +> find out +out/foo/compile.dest/... +out/foo/compile.dest/zinc +out/foo/compile.json +out/foo/compile.log +out/mill-build/... +out/mill-profile.json +out/mill-runner-state.json +out/mill-dependency-tree.json +out/mill-lock +out/mill-invalidation-tree.json +out/mill-chrome-profile.json +out/mill-server/... + +*/ + +// == Task Metadata and Cached Files +// +// Each named task (``Target`` or ``Command``) that is run has a representation in +// the `out/` directory structure. The _module_ structure is reflected in the directories, +// so that each module of your project has a uniquely associated subdirectory under the +//`out/` directory.For example, the `foo.compile` task we ran above places +// its files in the `out/foo/compile.*` paths: + +/** Usage + +> find out/foo/compile.* -maxdepth 0 +out/foo/compile.json +out/foo/compile.log +out/foo/compile.dest + +*/ + +// +// +// === `.json` +// +// The cache-key and JSON-serialized return-value of the `foo.compile` task. +// The return-value can also be retrieved via `mill show foo.compile`. +// Binary blobs are typically not included in `foo.json`, and instead stored as separate binary files in +// `.dest/` which are then referenced by `.json` file via `PathRef` references. +// +// === `.dest/` +// A path for the `Task` to use either as a scratch space, or to place generated files that are returned +// using `PathRef` references. +// A `Task` should only output files within its own given `foo.dest/` folder (available as `Task.dest`) to avoid +// conflicting with another `Task`, but can name files within `foo.dest/` arbitrarily. +// +// === `.log` +// The `stdout`/`stderr` of the `Task`, if any. This is also streamed to the console during evaluation. +// +// === `.super/` +// Holds task metadata for overridden tasks, if any. Whenever you use a `super.foo()` in your `foo` task, you +// will find the metadata of the `super.foo()` under this directory. +// +// +// The `out/` folder is intentionally kept simple and user-readable. +// If your build is not behaving as you would expect, +// feel free to poke around the various +// `foo.dest/` folders to see what files are being created, or the `foo.json` files to see what is being returned by a +// particular task. +// You can also simply delete folders within `out/` if you want to force portions of your project to be +// rebuilt, e.g. by deleting the `+out/main/+` or `+out/main/compile.*+` folders, but we strongly encourage you to use the xref:cli/builtin-commands.adoc#_clean[`clean` command] instead. +// +// [WARNING] +// -- +// Cleaning some task state by manually deleting files under `out/` may be convenient, but you need to be careful to always delete the `foo.json` file whenever you delete a `foo.dest/` or `foo.super/`. Otherwise, you risk running into hard to diagnose issues later. +// +// Instead, you should always give the `clean` command a try before manually deleting some file under `out/`. +// -- +// +// +// == Other files in the `out/` directory +// +// Apart from the build task-related files in the out folder, Mill itself places a variety +// of files in the outfolder under the `out/mill-*` prefix: + +/** Usage + +> find out/mill-* -maxdepth 0 +out/mill-chrome-profile.json +out/mill-dependency-tree.json +out/mill-invalidation-tree.json +out/mill-profile.json +out/mill-build +out/mill-server + +*/ + +// Files of note: +// +// === `mill-profile.json` +// +// Logs the tasks run and time taken for the last Mill command you executed. This is very useful +// if Mill is being unexpectedly slow, and you want to find out exactly what tasks are being run. +// This is useful to quickly look up tasks that were run to see how long they took, whether they +// were cached, and if not whether their outputs changed as a result of them being run: + +/** Usage + +> cat out/mill-profile.json +[ + { + "label": "mill.scalalib.ZincWorkerModule.worker", + "millis": 0, + "cached": true, + "valueHashChanged": false, + "dependencies": [ + ... + ], + "inputsHash": ... + }, + { + "label": "foo.compile", + "millis": ..., + "cached": false, + "valueHashChanged": false, + "dependencies": [ + ... + ], + "inputsHash": ..., + "previousInputsHash": ... + } +] + +*/ + +// === `mill-chrome-profile.json` +// This file can be opened in any Google Chrome browser with the built-in `chrome://tracing` +// URL to show you runtime profile of the last Mill command, so you can see what was executed +// when, sequentially or in parallel, and how long it took. This is very useful for +// understanding the performance of large parallel Mill builds +// +// image::basic/ChromeTracing.png[ChromeTracing.png] +// +// `mill-chrome-profile.json` complements `mill-profile.json`: where `mill-profile.json` is +// most useful for point lookups of facts about the last Mill evaluation, `mill-chrome-profile.json` +// is most useful to get a high-level overview of the runtime performance characteristics of +// the tasks that were +// +// === `mill-dependency-tree.json` +// A JSON file where the root keys are the tasks directly specified by the last +// `./mill ` command, and the tree structure shows the _upstream_ tasks and how +// the root tasks depend on them. You can use this to see why a task specified by +// `` is causing a particular upstream task to be selected. +// +// For example, when running `foo.compile` above, we get a tree structure (simplified below) +// that hsows how `foo.compile` depends on `foo.allSourceFiles`/`foo.allSources`/`foo.sources` +// (the files in the `src/` folder), `foo.compileClasspath`/`localCompileClasspath`/`compileResources` +// (i.e. the files in the `compile-resources/` folder: + +/** Usage + +> cat out/mill-dependency-tree.json +{ + "foo.compile": { + "foo.allSourceFiles": { + "foo.allSources": { + "foo.sources": {}, + "foo.generatedSources": {} + } + }, + "foo.compileClasspath": { + "foo.localCompileClasspath": { + "foo.unmanagedClasspath": {}, + "foo.compileResources": {} + } + }, + ... + } +} + + +*/ + +// If there are multiple paths through which one task depends on another, one path is chosen +// arbitrarily to be shown in the spanning tree +// +// === `mill-invalidation-tree.json` +// A JSON file where the root keys are the Mill inputs that were invalidated when the +// last command was run, and the tree structure shows the _downstream_ tasks that +// were invalidated due to those inputs changing. This is useful to see why a task +// that was selected was actually run rather than being cached. +// +// For example, above we edited the `foo/src/foo/Foo.java` file before running `foo.compile` +// a second time, and thus this file shows how `foo.sources` invalidated `foo.allSources`, +// `foo.allSourcesFiles`, and lastly `foo.compile`: + +/** Usage + +> cat out/mill-invalidation-tree.json +{ + "foo.sources": { + "foo.allSources": { + "foo.allSourceFiles": { + "foo.compile": {} + } + } + }, + ... +} + +*/ + +// Again, if there are multiple paths through which one task was invalidated by another, +// one path is chosen arbitrarily to be shown in the spanning tree +// +// Sometimes invalidation can be caused by a code change in your `build.mill`/`package.mill` +// files, rather than by a change in the project's source files or inputs. In such cases, +// the root tasks in `mill-invalidation-tree.json` may not necessarily be inputs. In such +// cases, you can look at `out/mill-build/methodCodeHashSignatures.dest/current/spanningInvalidationForest.json` +// to see an invalidation tree for how code changes in specfic methods propagate throughout +// the `build.mill` codebase. +// +// === `mill-build/` +// +// Contains the files related ot the xref:extending/meta-build.adoc[]. It contains many +// of thd same task-related and Mill-related files as the top-level `out/` folder, but +// related for compiling your `build.mill` rather than compiling your project's source files. +// +// === `mill-server/`, `mill-no-server/` +// +// Each Mill process needs to keep some temporary files in one of these directories. +// Deleting it will also terminate the associated server instance, if it is still running. +// diff --git a/example/fundamentals/out-dir/1-out-files/foo/src/foo/Foo.java b/example/fundamentals/out-dir/1-out-files/foo/src/foo/Foo.java new file mode 100644 index 00000000000..6bfe37e30c1 --- /dev/null +++ b/example/fundamentals/out-dir/1-out-files/foo/src/foo/Foo.java @@ -0,0 +1,7 @@ +package foo; + +public class Foo { + public static void main(String[] args) { + System.out.println("Hello World"); + } +} diff --git a/example/fundamentals/out-dir/1-custom-out/build.mill b/example/fundamentals/out-dir/2-custom-out/build.mill similarity index 100% rename from example/fundamentals/out-dir/1-custom-out/build.mill rename to example/fundamentals/out-dir/2-custom-out/build.mill diff --git a/main/client/src/mill/main/client/OutFiles.java b/main/client/src/mill/main/client/OutFiles.java index ddc226c12e4..2f056fc595a 100644 --- a/main/client/src/mill/main/client/OutFiles.java +++ b/main/client/src/mill/main/client/OutFiles.java @@ -74,6 +74,6 @@ public class OutFiles { */ public static final String millSelectiveExecution = "mill-selective-execution.json"; - public static final String millDependencyForest = "mill-dependency-forest.json"; - public static final String millInvalidationForest = "mill-invalidation-forest.json"; + public static final String millDependencyTree = "mill-dependency-tree.json"; + public static final String millInvalidationTree = "mill-invalidation-tree.json"; } diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index 62215659950..f9fda3ee42c 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -82,7 +82,7 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { val upstreamIndexEdges = indexToTerminal.map(t => interGroupDeps.getOrElse(t, Nil).map(terminalToIndex).toArray) os.write.over( - outPath / OutFiles.millDependencyForest, + outPath / OutFiles.millDependencyTree, SpanningForest.spanningTreeToJsonTree( SpanningForest(upstreamIndexEdges, indexToTerminal.indices.toSet, true), i => indexToTerminal(i).render @@ -287,7 +287,7 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { ) os.write.over( - outPath / OutFiles.millInvalidationForest, + outPath / OutFiles.millInvalidationTree, SpanningForest.spanningTreeToJsonTree( SpanningForest( downstreamIndexEdges,