diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index 2390d7eccce76..04f71bd1e0664 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -204,13 +204,31 @@ impl DepGraph { where C: DepGraphSafe + StableHashingContextProvider<'gcx>, R: HashStable>, { - self.with_task_impl(key, cx, arg, false, task, + self.with_task_impl(key, cx, arg, false, true, task, |key| OpenTask::Regular(Lock::new(RegularOpenTask { node: key, reads: SmallVec::new(), read_set: FxHashSet(), })), - |data, key, task| data.borrow_mut().complete_task(key, task)) + |data, key, task| data.borrow_mut().complete_task(key, task, false)) + } + + pub fn with_forced_task<'gcx, C, A, R>(&self, + key: DepNode, + cx: C, + arg: A, + task: fn(C, A) -> R) + -> (R, DepNodeIndex) + where C: DepGraphSafe + StableHashingContextProvider<'gcx>, + R: HashStable>, + { + self.with_task_impl(key, cx, arg, false, false, task, + |key| OpenTask::Regular(Lock::new(RegularOpenTask { + node: key, + reads: SmallVec::new(), + read_set: FxHashSet(), + })), + |data, key, task| data.borrow_mut().complete_task(key, task, true)) } /// Creates a new dep-graph input with value `input` @@ -226,7 +244,7 @@ impl DepGraph { arg } - self.with_task_impl(key, cx, input, true, identity_fn, + self.with_task_impl(key, cx, input, true, true, identity_fn, |_| OpenTask::Ignore, |data, key, _| data.borrow_mut().alloc_node(key, SmallVec::new())) } @@ -237,6 +255,7 @@ impl DepGraph { cx: C, arg: A, no_tcx: bool, + do_fingerprinting: bool, task: fn(C, A) -> R, create_task: fn(DepNode) -> OpenTask, finish_task_and_alloc_depnode: fn(&Lock, @@ -282,41 +301,58 @@ impl DepGraph { let dep_node_index = finish_task_and_alloc_depnode(&data.current, key, open_task); - let mut stable_hasher = StableHasher::new(); - result.hash_stable(&mut hcx, &mut stable_hasher); + if do_fingerprinting { + let mut stable_hasher = StableHasher::new(); + result.hash_stable(&mut hcx, &mut stable_hasher); - let current_fingerprint = stable_hasher.finish(); + let current_fingerprint = stable_hasher.finish(); - // Store the current fingerprint - { - let mut fingerprints = self.fingerprints.borrow_mut(); + // Store the current fingerprint + { + let mut fingerprints = self.fingerprints.borrow_mut(); + + if dep_node_index.index() >= fingerprints.len() { + fingerprints.resize(dep_node_index.index() + 1, Fingerprint::ZERO); + } - if dep_node_index.index() >= fingerprints.len() { - fingerprints.resize(dep_node_index.index() + 1, Fingerprint::ZERO); + debug_assert!(fingerprints[dep_node_index] == Fingerprint::ZERO, + "DepGraph::with_task() - Duplicate fingerprint \ + insertion for {:?}", key); + fingerprints[dep_node_index] = current_fingerprint; } - debug_assert!(fingerprints[dep_node_index] == Fingerprint::ZERO, - "DepGraph::with_task() - Duplicate fingerprint \ - insertion for {:?}", key); - fingerprints[dep_node_index] = current_fingerprint; - } + // Determine the color of the new DepNode. + if let Some(prev_index) = data.previous.node_to_index_opt(&key) { + let prev_fingerprint = data.previous.fingerprint_by_index(prev_index); - // Determine the color of the new DepNode. - if let Some(prev_index) = data.previous.node_to_index_opt(&key) { - let prev_fingerprint = data.previous.fingerprint_by_index(prev_index); + let color = if current_fingerprint == prev_fingerprint { + DepNodeColor::Green(dep_node_index) + } else { + DepNodeColor::Red + }; - let color = if current_fingerprint == prev_fingerprint { - DepNodeColor::Green(dep_node_index) - } else { - DepNodeColor::Red - }; + let mut colors = data.colors.borrow_mut(); + debug_assert!(colors.get(prev_index).is_none(), + "DepGraph::with_task() - Duplicate DepNodeColor \ + insertion for {:?}", key); + + colors.insert(prev_index, color); + } + } else { + // Always store a ZERO fingerprint + { + let mut fingerprints = self.fingerprints.borrow_mut(); - let mut colors = data.colors.borrow_mut(); - debug_assert!(colors.get(prev_index).is_none(), - "DepGraph::with_task() - Duplicate DepNodeColor \ - insertion for {:?}", key); + if dep_node_index.index() >= fingerprints.len() { + fingerprints.resize(dep_node_index.index() + 1, Fingerprint::ZERO); + } + + fingerprints[dep_node_index] = Fingerprint::ZERO; + } - colors.insert(prev_index, color); + if let Some(prev_index) = data.previous.node_to_index_opt(&key) { + data.colors.borrow_mut().insert(prev_index, DepNodeColor::Red); + } } (result, dep_node_index) @@ -378,7 +414,7 @@ impl DepGraph { } /// Execute something within an "eval-always" task which is a task - // that runs whenever anything changes. + /// that runs whenever anything changes. pub fn with_eval_always_task<'gcx, C, A, R>(&self, key: DepNode, cx: C, @@ -388,7 +424,7 @@ impl DepGraph { where C: DepGraphSafe + StableHashingContextProvider<'gcx>, R: HashStable>, { - self.with_task_impl(key, cx, arg, false, task, + self.with_task_impl(key, cx, arg, false, true, task, |key| OpenTask::EvalAlways { node: key }, |data, key, task| data.borrow_mut().complete_eval_always_task(key, task)) } @@ -939,7 +975,11 @@ impl CurrentDepGraph { } } - fn complete_task(&mut self, key: DepNode, task: OpenTask) -> DepNodeIndex { + fn complete_task(&mut self, + key: DepNode, + task: OpenTask, + allow_existing_dep_node: bool) + -> DepNodeIndex { if let OpenTask::Regular(task) = task { let RegularOpenTask { node, @@ -970,7 +1010,16 @@ impl CurrentDepGraph { } } - self.alloc_node(node, reads) + if allow_existing_dep_node { + if let Some(&dep_node_index) = self.node_to_node_index.get(&node) { + self.edges[dep_node_index] = reads; + dep_node_index + } else { + self.alloc_node(node, reads) + } + } else { + self.alloc_node(node, reads) + } } else { bug!("complete_task() - Expected regular task to be popped") } diff --git a/src/librustc/session/config.rs b/src/librustc/session/config.rs index 55752141e30eb..cbc8b4fe68381 100644 --- a/src/librustc/session/config.rs +++ b/src/librustc/session/config.rs @@ -1973,7 +1973,7 @@ pub fn build_session_options_and_crate_config( (&None, &None) => None, }.map(|m| PathBuf::from(m)); - if cg.lto != Lto::No && incremental.is_some() { + if cg.lto == Lto::Fat && incremental.is_some() { early_error( error_format, "can't perform LTO when compiling incrementally", diff --git a/src/librustc/session/mod.rs b/src/librustc/session/mod.rs index c28b49756f096..327127bf144ea 100644 --- a/src/librustc/session/mod.rs +++ b/src/librustc/session/mod.rs @@ -583,11 +583,6 @@ impl Session { return config::Lto::No; } - // Right now ThinLTO isn't compatible with incremental compilation. - if self.opts.incremental.is_some() { - return config::Lto::No; - } - // Now we're in "defaults" territory. By default we enable ThinLTO for // optimized compiles (anything greater than O0). match self.opts.optimize { diff --git a/src/librustc/ty/query/config.rs b/src/librustc/ty/query/config.rs index bd6217e28c755..751d0a08f7d2c 100644 --- a/src/librustc/ty/query/config.rs +++ b/src/librustc/ty/query/config.rs @@ -702,12 +702,6 @@ impl<'tcx> QueryDescription<'tcx> for queries::codegen_unit<'tcx> { } } -impl<'tcx> QueryDescription<'tcx> for queries::compile_codegen_unit<'tcx> { - fn describe(_tcx: TyCtxt, _: InternedString) -> String { - format!("compile_codegen_unit") - } -} - impl<'tcx> QueryDescription<'tcx> for queries::output_filenames<'tcx> { fn describe(_tcx: TyCtxt, _: CrateNum) -> String { format!("output_filenames") diff --git a/src/librustc/ty/query/mod.rs b/src/librustc/ty/query/mod.rs index 3581dd87f6f8d..8fb0c5bbcae70 100644 --- a/src/librustc/ty/query/mod.rs +++ b/src/librustc/ty/query/mod.rs @@ -27,7 +27,7 @@ use middle::stability::{self, DeprecationEntry}; use middle::lang_items::{LanguageItems, LangItem}; use middle::exported_symbols::{SymbolExportLevel, ExportedSymbol}; use mir::interpret::ConstEvalResult; -use mir::mono::{CodegenUnit, Stats}; +use mir::mono::CodegenUnit; use mir; use mir::interpret::{GlobalId, Allocation}; use session::{CompileResult, CrateDisambiguator}; @@ -436,7 +436,6 @@ define_queries! { <'tcx> -> (Arc, Arc>>>), [] fn is_codegened_item: IsCodegenedItem(DefId) -> bool, [] fn codegen_unit: CodegenUnit(InternedString) -> Arc>, - [] fn compile_codegen_unit: CompileCodegenUnit(InternedString) -> Stats, [] fn output_filenames: output_filenames_node(CrateNum) -> Arc, diff --git a/src/librustc_codegen_llvm/back/lto.rs b/src/librustc_codegen_llvm/back/lto.rs index a68f22b2651f9..1aabd4facbe79 100644 --- a/src/librustc_codegen_llvm/back/lto.rs +++ b/src/librustc_codegen_llvm/back/lto.rs @@ -811,6 +811,10 @@ impl ThinLTOImports { } } + pub fn modules_imported_by(&self, llvm_module_name: &str) -> &[String] { + self.imports.get(llvm_module_name).map(|v| &v[..]).unwrap_or(&[]) + } + /// Load the ThinLTO import map from ThinLTOData. unsafe fn from_thin_lto_data(data: *const llvm::ThinLTOData) -> ThinLTOImports { let raw_data: *const llvm::ThinLTOModuleImports = diff --git a/src/librustc_codegen_llvm/base.rs b/src/librustc_codegen_llvm/base.rs index 603ee78585ea6..30b04e9ba988c 100644 --- a/src/librustc_codegen_llvm/base.rs +++ b/src/librustc_codegen_llvm/base.rs @@ -43,7 +43,6 @@ use rustc::middle::cstore::{EncodedMetadata}; use rustc::ty::{self, Ty, TyCtxt}; use rustc::ty::layout::{self, Align, TyLayout, LayoutOf}; use rustc::ty::query::Providers; -use rustc::dep_graph::{DepNode, DepConstructor}; use rustc::middle::cstore::{self, LinkMeta, LinkagePreference}; use rustc::middle::exported_symbols; use rustc::util::common::{time, print_time_passes_entry}; @@ -858,51 +857,50 @@ pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, let mut total_codegen_time = Duration::new(0, 0); let mut all_stats = Stats::default(); + let reusable_cgus = determine_cgu_reuse(tcx, &codegen_units); + for cgu in codegen_units.into_iter() { ongoing_codegen.wait_for_signal_to_codegen_item(); ongoing_codegen.check_for_errors(tcx.sess); - // First, if incremental compilation is enabled, we try to re-use the - // codegen unit from the cache. + debug!("starting codegen for CGU: {}", cgu.name()); + + let reused = if reusable_cgus.contains(cgu.name()) { + debug!("CGU `{}` can be re-used", cgu.name()); + + let work_product_id = &cgu.work_product_id(); + let work_product = tcx.dep_graph + .previous_work_product(work_product_id) + .expect("Could not find work-product for reuseable CGU"); + + let module = ModuleCodegen { + name: cgu.name().to_string(), + source: ModuleSource::Preexisting(work_product), + kind: ModuleKind::Regular, + }; + write::submit_codegened_module_to_llvm(tcx, module, 0); + true + } else { + debug!("CGU `{}` can NOT be re-used", cgu.name()); + + let _timing_guard = time_graph.as_ref().map(|time_graph| { + time_graph.start(write::CODEGEN_WORKER_TIMELINE, + write::CODEGEN_WORK_PACKAGE_KIND, + &format!("codegen {}", cgu.name())) + }); + let start_time = Instant::now(); + let stats = compile_codegen_unit(tcx, *cgu.name()); + all_stats.extend(stats); + total_codegen_time += start_time.elapsed(); + false + }; + if tcx.dep_graph.is_fully_enabled() { - let cgu_id = cgu.work_product_id(); - - // Check whether there is a previous work-product we can - // re-use. Not only must the file exist, and the inputs not - // be dirty, but the hash of the symbols we will generate must - // be the same. - if let Some(buf) = tcx.dep_graph.previous_work_product(&cgu_id) { - let dep_node = &DepNode::new(tcx, - DepConstructor::CompileCodegenUnit(cgu.name().clone())); - - // We try to mark the DepNode::CompileCodegenUnit green. If we - // succeed it means that none of the dependencies has changed - // and we can safely re-use. - if let Some(dep_node_index) = tcx.dep_graph.try_mark_green(tcx, dep_node) { - let module = ModuleCodegen { - name: cgu.name().to_string(), - source: ModuleSource::Preexisting(buf), - kind: ModuleKind::Regular, - }; - tcx.dep_graph.mark_loaded_from_cache(dep_node_index, true); - write::submit_codegened_module_to_llvm(tcx, module, 0); - // Continue to next cgu, this one is done. - continue - } - } else { - // This can happen if files were deleted from the cache - // directory for some reason. We just re-compile then. - } + let dep_node_index = + tcx.dep_graph.dep_node_index_of(&cgu.compilation_dep_node(tcx)); + tcx.dep_graph.mark_loaded_from_cache(dep_node_index, reused); } - let _timing_guard = time_graph.as_ref().map(|time_graph| { - time_graph.start(write::CODEGEN_WORKER_TIMELINE, - write::CODEGEN_WORK_PACKAGE_KIND, - &format!("codegen {}", cgu.name())) - }); - let start_time = Instant::now(); - all_stats.extend(tcx.compile_codegen_unit(*cgu.name())); - total_codegen_time += start_time.elapsed(); ongoing_codegen.check_for_errors(tcx.sess); } @@ -948,6 +946,67 @@ pub fn codegen_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, ongoing_codegen } +fn determine_cgu_reuse<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, + codegen_units: &[Arc>]) + -> FxHashSet { + if !tcx.dep_graph.is_fully_enabled() { + return FxHashSet() + } + + let mut green_cgus = FxHashSet(); + + for cgu in codegen_units { + let work_product_id = &cgu.work_product_id(); + + if tcx.dep_graph.previous_work_product(work_product_id).is_none() { + // We don't have anything cached for this CGU. This can happen + // if the CGU did not exist in the previous session. We are done + // for this CGU, skip to the next. + continue + }; + + // Try to mark the CGU as green + let dep_node = cgu.compilation_dep_node(tcx); + + assert!(!tcx.dep_graph.dep_node_exists(&dep_node), + "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.", + cgu.name()); + + if tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() { + green_cgus.insert(cgu.name().to_string()); + } else { + // We definitely cannot re-use this CGU + } + } + + let thin_lto_imports = load_thin_lto_imports(tcx.sess); + let mut reusable_cgus = FxHashSet(); + + // Now we know all CGUs that have not changed themselves. Next we need to + // check if anything they imported via ThinLTO has changed. + for cgu_name in &green_cgus { + let imported_cgus = thin_lto_imports.modules_imported_by(cgu_name); + + let all_imports_green = imported_cgus.iter().all(|imported_cgu| { + green_cgus.contains(&imported_cgu[..]) + }); + + if all_imports_green { + reusable_cgus.insert(cgu_name.clone()); + } + } + + codegen_units + .iter() + .filter_map(|cgu| { + if reusable_cgus.contains(&cgu.name().as_str()[..]) { + Some(cgu.name().clone()) + } else { + None + } + }).collect::>() +} + fn assert_and_save_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) { time(tcx.sess, "assert dep graph", @@ -1167,11 +1226,15 @@ fn is_codegened_item(tcx: TyCtxt, id: DefId) -> bool { } fn compile_codegen_unit<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, - cgu: InternedString) -> Stats { - let cgu = tcx.codegen_unit(cgu); - + cgu_name: InternedString) + -> Stats { let start_time = Instant::now(); - let (stats, module) = module_codegen(tcx, cgu); + + let dep_node = tcx.codegen_unit(cgu_name).compilation_dep_node(tcx); + let ((stats, module), _) = tcx.dep_graph.with_forced_task(dep_node, + tcx, + cgu_name, + module_codegen); let time_to_codegen = start_time.elapsed(); // We assume that the cost to run LLVM on a CGU is proportional to @@ -1180,22 +1243,22 @@ fn compile_codegen_unit<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, time_to_codegen.subsec_nanos() as u64; write::submit_codegened_module_to_llvm(tcx, - module, - cost); + module, + cost); return stats; fn module_codegen<'a, 'tcx>( tcx: TyCtxt<'a, 'tcx, 'tcx>, - cgu: Arc>) + cgu_name: InternedString) -> (Stats, ModuleCodegen) { - let cgu_name = cgu.name().to_string(); + let cgu = tcx.codegen_unit(cgu_name); // Instantiate monomorphizations without filling out definitions yet... let cx = CodegenCx::new(tcx, cgu); let module = { let mono_items = cx.codegen_unit - .items_in_deterministic_order(cx.tcx); + .items_in_deterministic_order(cx.tcx); for &(mono_item, (linkage, visibility)) in &mono_items { mono_item.predefine(&cx, linkage, visibility); } @@ -1247,7 +1310,7 @@ fn compile_codegen_unit<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, }; ModuleCodegen { - name: cgu_name, + name: cgu_name.to_string(), source: ModuleSource::Codegened(llvm_module), kind: ModuleKind::Regular, } @@ -1270,7 +1333,6 @@ pub fn provide(providers: &mut Providers) { .cloned() .expect(&format!("failed to find cgu with name {:?}", name)) }; - providers.compile_codegen_unit = compile_codegen_unit; provide_extern(providers); } @@ -1352,8 +1414,11 @@ mod temp_stable_hash_impls { } } -#[allow(unused)] fn load_thin_lto_imports(sess: &Session) -> lto::ThinLTOImports { + if sess.opts.incremental.is_none() { + return lto::ThinLTOImports::new_empty(); + } + let path = rustc_incremental::in_incr_comp_dir_sess( sess, lto::THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME diff --git a/src/librustc_mir/monomorphize/partitioning.rs b/src/librustc_mir/monomorphize/partitioning.rs index 73b430bc04112..f804cdc14e5f1 100644 --- a/src/librustc_mir/monomorphize/partitioning.rs +++ b/src/librustc_mir/monomorphize/partitioning.rs @@ -103,7 +103,7 @@ //! inlining, even when they are not marked #[inline]. use monomorphize::collector::InliningMap; -use rustc::dep_graph::WorkProductId; +use rustc::dep_graph::{WorkProductId, DepNode, DepConstructor}; use rustc::hir::def_id::{DefId, LOCAL_CRATE}; use rustc::hir::map::DefPathData; use rustc::mir::mono::{Linkage, Visibility}; @@ -194,6 +194,10 @@ pub trait CodegenUnitExt<'tcx> { items.sort_by_cached_key(|&(i, _)| item_sort_key(tcx, i)); items } + + fn compilation_dep_node(&self, tcx: TyCtxt<'_, 'tcx, 'tcx>) -> DepNode { + DepNode::new(tcx, DepConstructor::CompileCodegenUnit(self.name().clone())) + } } impl<'tcx> CodegenUnitExt<'tcx> for CodegenUnit<'tcx> { diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index e5fb472afd635..1f63ae241b66c 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -1658,6 +1658,9 @@ impl<'test> TestCx<'test> { rustc.args(&["-C", &format!("incremental={}", incremental_dir.display())]); rustc.args(&["-Z", "incremental-verify-ich"]); rustc.args(&["-Z", "incremental-queries"]); + + // Force thinlto off for now.. + rustc.args(&["-Zthinlto=no"]); } if self.config.mode == CodegenUnits {