Auto merge of #32016 - nikomatsakis:incr-comp-save, r=mw

Save/load incremental compilation dep graph

Contains the code to serialize/deserialize the dep graph to disk between executions. We also hash the item contents and compare to the new hashes. Also includes a unit test harness. There are definitely some known limitations, such as #32014 and #32015, but I am leaving those for follow-up work.

Note that this PR builds on #32007, so the overlapping commits can be excluded from review.

r? @michaelwoerister

mk/crates.mk

@@ -58,7 +58,7 @@ RUSTC_CRATES := rustc rustc_typeck rustc_mir rustc_borrowck rustc_resolve rustc_
                 rustc_trans rustc_back rustc_llvm rustc_privacy rustc_lint \
                 rustc_data_structures rustc_front rustc_platform_intrinsics \
                 rustc_plugin rustc_metadata rustc_passes rustc_save_analysis \
-                rustc_const_eval rustc_const_math
+                rustc_const_eval rustc_const_math rustc_incremental
 HOST_CRATES := syntax syntax_ext $(RUSTC_CRATES) rustdoc fmt_macros \
 		flate arena graphviz rbml log serialize
 TOOLS := compiletest rustdoc rustc rustbook error_index_generator
@@ -105,7 +105,8 @@ DEPS_rustc_data_structures := std log serialize
 DEPS_rustc_driver := arena flate getopts graphviz libc rustc rustc_back rustc_borrowck \
                      rustc_typeck rustc_mir rustc_resolve log syntax serialize rustc_llvm \
 	             rustc_trans rustc_privacy rustc_lint rustc_front rustc_plugin \
-                     rustc_metadata syntax_ext rustc_passes rustc_save_analysis rustc_const_eval
+                     rustc_metadata syntax_ext rustc_passes rustc_save_analysis rustc_const_eval \
+                     rustc_incremental
 DEPS_rustc_front := std syntax log serialize
 DEPS_rustc_lint := rustc log syntax rustc_const_eval
 DEPS_rustc_llvm := native:rustllvm libc std rustc_bitflags
@@ -118,7 +119,8 @@ DEPS_rustc_plugin := rustc rustc_metadata syntax rustc_mir
 DEPS_rustc_privacy := rustc rustc_front log syntax
 DEPS_rustc_trans := arena flate getopts graphviz libc rustc rustc_back rustc_mir \
                     log syntax serialize rustc_llvm rustc_front rustc_platform_intrinsics \
-                    rustc_const_math rustc_const_eval
+                    rustc_const_math rustc_const_eval rustc_incremental
+DEPS_rustc_incremental := rbml rustc rustc_front serialize rustc_data_structures
 DEPS_rustc_save_analysis := rustc log syntax rustc_front
 DEPS_rustc_typeck := rustc syntax rustc_front rustc_platform_intrinsics rustc_const_math \
                      rustc_const_eval

mk/tests.mk

@@ -305,6 +305,7 @@ check-stage$(1)-T-$(2)-H-$(3)-exec: \
 	check-stage$(1)-T-$(2)-H-$(3)-doc-crates-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-debuginfo-gdb-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-debuginfo-lldb-exec \
+	check-stage$(1)-T-$(2)-H-$(3)-incremental-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-doc-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-pretty-exec
 
@@ -481,6 +482,7 @@ DEBUGINFO_LLDB_RS := $(call rwildcard,$(S)src/test/debuginfo/,*.rs)
 CODEGEN_RS := $(call rwildcard,$(S)src/test/codegen/,*.rs)
 CODEGEN_CC := $(call rwildcard,$(S)src/test/codegen/,*.cc)
 CODEGEN_UNITS_RS := $(call rwildcard,$(S)src/test/codegen-units/,*.rs)
+INCREMENTAL_RS := $(call rwildcard,$(S)src/test/incremental/,*.rs)
 RUSTDOCCK_RS := $(call rwildcard,$(S)src/test/rustdoc/,*.rs)
 
 RPASS_TESTS := $(RPASS_RS)
@@ -496,6 +498,7 @@ DEBUGINFO_GDB_TESTS := $(DEBUGINFO_GDB_RS)
 DEBUGINFO_LLDB_TESTS := $(DEBUGINFO_LLDB_RS)
 CODEGEN_TESTS := $(CODEGEN_RS) $(CODEGEN_CC)
 CODEGEN_UNITS_TESTS := $(CODEGEN_UNITS_RS)
+INCREMENTAL_TESTS := $(INCREMENTAL_RS)
 RUSTDOCCK_TESTS := $(RUSTDOCCK_RS)
 
 CTEST_SRC_BASE_rpass = run-pass
@@ -558,6 +561,11 @@ CTEST_BUILD_BASE_codegen-units = codegen-units
 CTEST_MODE_codegen-units = codegen-units
 CTEST_RUNTOOL_codegen-units = $(CTEST_RUNTOOL)
 
+CTEST_SRC_BASE_incremental = incremental
+CTEST_BUILD_BASE_incremental = incremental
+CTEST_MODE_incremental = incremental
+CTEST_RUNTOOL_incremental = $(CTEST_RUNTOOL)
+
 CTEST_SRC_BASE_rustdocck = rustdoc
 CTEST_BUILD_BASE_rustdocck = rustdoc
 CTEST_MODE_rustdocck = rustdoc
@@ -681,6 +689,7 @@ CTEST_DEPS_debuginfo-lldb_$(1)-T-$(2)-H-$(3) = $$(DEBUGINFO_LLDB_TESTS) \
                                                $(S)src/etc/lldb_rust_formatters.py
 CTEST_DEPS_codegen_$(1)-T-$(2)-H-$(3) = $$(CODEGEN_TESTS)
 CTEST_DEPS_codegen-units_$(1)-T-$(2)-H-$(3) = $$(CODEGEN_UNITS_TESTS)
+CTEST_DEPS_incremental_$(1)-T-$(2)-H-$(3) = $$(INCREMENTAL_TESTS)
 CTEST_DEPS_rustdocck_$(1)-T-$(2)-H-$(3) = $$(RUSTDOCCK_TESTS) \
 		$$(HBIN$(1)_H_$(3))/rustdoc$$(X_$(3)) \
 		$(S)src/etc/htmldocck.py
@@ -747,7 +756,7 @@ endif
 endef
 
 CTEST_NAMES = rpass rpass-valgrind rpass-full rfail-full cfail-full rfail cfail pfail \
-	debuginfo-gdb debuginfo-lldb codegen codegen-units rustdocck
+	debuginfo-gdb debuginfo-lldb codegen codegen-units rustdocck incremental
 
 $(foreach host,$(CFG_HOST), \
  $(eval $(foreach target,$(CFG_TARGET), \
@@ -945,6 +954,7 @@ TEST_GROUPS = \
 	debuginfo-lldb \
 	codegen \
 	codegen-units \
+	incremental \
 	doc \
 	$(foreach docname,$(DOC_NAMES),doc-$(docname)) \
 	pretty \

src/compiletest/common.rs

@@ -25,7 +25,8 @@ pub enum Mode {
     DebugInfoLldb,
     Codegen,
     Rustdoc,
-    CodegenUnits
+    CodegenUnits,
+    Incremental,
 }
 
 impl FromStr for Mode {
@@ -43,6 +44,7 @@ impl FromStr for Mode {
           "codegen" => Ok(Codegen),
           "rustdoc" => Ok(Rustdoc),
           "codegen-units" => Ok(CodegenUnits),
+          "incremental" => Ok(Incremental),
           _ => Err(()),
         }
     }
@@ -62,6 +64,7 @@ impl fmt::Display for Mode {
             Codegen => "codegen",
             Rustdoc => "rustdoc",
             CodegenUnits => "codegen-units",
+            Incremental => "incremental",
         }, f)
     }
 }

src/compiletest/compiletest.rs

@@ -71,7 +71,8 @@ pub fn parse_config(args: Vec<String> ) -> Config {
           reqopt("", "aux-base", "directory to find auxiliary test files", "PATH"),
           reqopt("", "stage-id", "the target-stage identifier", "stageN-TARGET"),
           reqopt("", "mode", "which sort of compile tests to run",
-                 "(compile-fail|parse-fail|run-fail|run-pass|run-pass-valgrind|pretty|debug-info)"),
+                 "(compile-fail|parse-fail|run-fail|run-pass|\
+                  run-pass-valgrind|pretty|debug-info|incremental)"),
           optflag("", "ignored", "run tests marked as ignored"),
           optopt("", "runtool", "supervisor program to run tests under \
                                  (eg. emulator, valgrind)", "PROGRAM"),

src/compiletest/runtest.rs

@@ -11,6 +11,7 @@
 use common::Config;
 use common::{CompileFail, ParseFail, Pretty, RunFail, RunPass, RunPassValgrind};
 use common::{Codegen, DebugInfoLldb, DebugInfoGdb, Rustdoc, CodegenUnits};
+use common::{Incremental};
 use errors::{self, ErrorKind};
 use header::TestProps;
 use header;
@@ -59,6 +60,7 @@ pub fn run(config: Config, testpaths: &TestPaths) {
         Codegen => run_codegen_test(&config, &props, &testpaths),
         Rustdoc => run_rustdoc_test(&config, &props, &testpaths),
         CodegenUnits => run_codegen_units_test(&config, &props, &testpaths),
+        Incremental => run_incremental_test(&config, &props, &testpaths),
     }
 }
 
@@ -1966,3 +1968,67 @@ fn run_codegen_units_test(config: &Config, props: &TestProps, testpaths: &TestPa
         panic!();
     }
 }
+
+fn run_incremental_test(config: &Config, props: &TestProps, testpaths: &TestPaths) {
+    // Basic plan for a test incremental/foo/bar.rs:
+    // - load list of revisions pass1, fail2, pass3
+    //   - each should begin with `rpass`, `rfail`, or `cfail`
+    //   - if `rpass`, expect compile and execution to succeed
+    //   - if `cfail`, expect compilation to fail
+    //   - if `rfail`, expect execution to fail
+    // - create a directory build/foo/bar.incremental
+    // - compile foo/bar.rs with -Z incremental=.../foo/bar.incremental and -C pass1
+    //   - because name of revision starts with "pass", expect success
+    // - compile foo/bar.rs with -Z incremental=.../foo/bar.incremental and -C fail2
+    //   - because name of revision starts with "fail", expect an error
+    //   - load expected errors as usual, but filter for those that end in `[fail2]`
+    // - compile foo/bar.rs with -Z incremental=.../foo/bar.incremental and -C pass3
+    //   - because name of revision starts with "pass", expect success
+    // - execute build/foo/bar.exe and save output
+    //
+    // FIXME -- use non-incremental mode as an oracle? That doesn't apply
+    // to #[rustc_dirty] and clean tests I guess
+
+    assert!(!props.revisions.is_empty(), "incremental tests require a list of revisions");
+
+    let output_base_name = output_base_name(config, testpaths);
+
+    // Create the incremental workproduct directory.
+    let incremental_dir = output_base_name.with_extension("incremental");
+    if incremental_dir.exists() {
+        fs::remove_dir_all(&incremental_dir).unwrap();
+    }
+    fs::create_dir_all(&incremental_dir).unwrap();
+
+    if config.verbose {
+        print!("incremental_dir={}", incremental_dir.display());
+    }
+
+    for revision in &props.revisions {
+        let mut revision_props = props.clone();
+        header::load_props_into(&mut revision_props, &testpaths.file, Some(&revision));
+
+        revision_props.compile_flags.extend(vec![
+            format!("-Z"),
+            format!("incremental={}", incremental_dir.display()),
+            format!("--cfg"),
+            format!("{}", revision),
+        ]);
+
+        if config.verbose {
+            print!("revision={:?} revision_props={:#?}", revision, revision_props);
+        }
+
+        if revision.starts_with("rpass") {
+            run_rpass_test_revision(config, &revision_props, testpaths, Some(&revision));
+        } else if revision.starts_with("rfail") {
+            run_rfail_test_revision(config, &revision_props, testpaths, Some(&revision));
+        } else if revision.starts_with("cfail") {
+            run_cfail_test_revision(config, &revision_props, testpaths, Some(&revision));
+        } else {
+            fatal(
+                Some(revision),
+                "revision name must begin with rpass, rfail, or cfail");
+        }
+    }
+}