From 0f973ac53df631ad2abdf85dbe2453e528c7e6c3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 17 Jun 2023 11:02:28 +0200 Subject: [PATCH] gix-corpus now respects the --trace flag --- Cargo.lock | 1 + gitoxide-core/Cargo.toml | 3 ++- gitoxide-core/src/corpus/engine.rs | 24 ++++++++++++++---- gitoxide-core/src/corpus/mod.rs | 2 ++ gitoxide-core/src/corpus/trace.rs | 35 ++++++++++++++++++++++++--- src/plumbing/main.rs | 39 ++++++++++++++++++------------ src/shared.rs | 2 +- 7 files changed, 80 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e23021c2232..97b4c4caafc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1268,6 +1268,7 @@ dependencies = [ "jwalk", "layout-rs", "open", + "parking_lot", "rusqlite", "serde", "serde_json", diff --git a/gitoxide-core/Cargo.toml b/gitoxide-core/Cargo.toml index b814f130a42..64eb4b707ac 100644 --- a/gitoxide-core/Cargo.toml +++ b/gitoxide-core/Cargo.toml @@ -23,7 +23,7 @@ estimate-hours = ["dep:itertools", "dep:fs-err", "dep:crossbeam-channel", "dep:s query = ["dep:rusqlite"] ## Run algorithms on a corpus of repositories and store their results for later comparison and intelligence gathering. ## *Note that* `organize` we need for finding git repositories fast. -corpus = [ "dep:rusqlite", "dep:sysinfo", "organize", "dep:crossbeam-channel", "dep:serde_json", "dep:tracing-forest", "dep:tracing-subscriber", "dep:tracing" ] +corpus = [ "dep:rusqlite", "dep:sysinfo", "organize", "dep:crossbeam-channel", "dep:serde_json", "dep:tracing-forest", "dep:tracing-subscriber", "dep:tracing", "dep:parking_lot" ] #! ### Mutually Exclusive Networking #! If both are set, _blocking-client_ will take precedence, allowing `--all-features` to be used. @@ -72,6 +72,7 @@ smallvec = { version = "1.10.0", optional = true } rusqlite = { version = "0.29.0", optional = true, features = ["bundled"] } # for 'corpus' +parking_lot = { version = "0.12.1", optional = true } sysinfo = { version = "0.29.2", optional = true, default-features = false } serde_json = { version = "1.0.65", optional = true } tracing-forest = { version = "0.1.5", features = ["serde"], optional = true } diff --git a/gitoxide-core/src/corpus/engine.rs b/gitoxide-core/src/corpus/engine.rs index 85c43236b02..bf4bfc87844 100644 --- a/gitoxide-core/src/corpus/engine.rs +++ b/gitoxide-core/src/corpus/engine.rs @@ -12,12 +12,20 @@ use std::time::{Duration, Instant}; impl Engine { /// Open the corpus DB or create it. - pub fn open_or_create(db: PathBuf, gitoxide_version: String, progress: corpus::Progress) -> anyhow::Result { + pub fn open_or_create( + db: PathBuf, + gitoxide_version: String, + progress: corpus::Progress, + trace_to_progress: bool, + reverse_trace_lines: bool, + ) -> anyhow::Result { let con = crate::corpus::db::create(db).context("Could not open or create database")?; Ok(Engine { progress, con, gitoxide_version, + trace_to_progress, + reverse_trace_lines, }) } @@ -66,7 +74,11 @@ impl Engine { if task.execute_exclusive || threads == 1 { let mut run_progress = repo_progress.add_child("set later"); - let (_guard, current_id) = corpus::trace::override_thread_subscriber(db_path.as_str())?; + let (_guard, current_id) = corpus::trace::override_thread_subscriber( + db_path.as_str(), + self.trace_to_progress.then(|| task_progress.add_child("trace")), + self.reverse_trace_lines, + )?; for repo in &repos { if gix::interrupt::is_triggered() { @@ -80,7 +92,7 @@ impl Engine { .display() )); - // TODO: wait for new release to be able to provide run_id via span attributes + // TODO: wait for new release of `tracing-forest` to be able to provide run_id via span attributes let mut run = Self::insert_run(&self.con, gitoxide_id, runner_id, *task_id, repo.id)?; current_id.store(run.id, Ordering::SeqCst); tracing::info_span!("run", run_id = run.id).in_scope(|| { @@ -106,9 +118,11 @@ impl Engine { let shared_repo_progress = repo_progress.clone(); let db_path = db_path.clone(); move |tid| { + let mut progress = gix::threading::lock(&shared_repo_progress); ( - corpus::trace::override_thread_subscriber(db_path.as_str()), - gix::threading::lock(&shared_repo_progress).add_child(format!("{tid}")), + // threaded printing is usually spammy, and lines interleave so it's useless. + corpus::trace::override_thread_subscriber(db_path.as_str(), None, false), + progress.add_child(format!("{tid}")), rusqlite::Connection::open(&db_path), ) } diff --git a/gitoxide-core/src/corpus/mod.rs b/gitoxide-core/src/corpus/mod.rs index 589a574d3a7..8e49b9a05c3 100644 --- a/gitoxide-core/src/corpus/mod.rs +++ b/gitoxide-core/src/corpus/mod.rs @@ -5,6 +5,8 @@ pub struct Engine { progress: Progress, con: rusqlite::Connection, gitoxide_version: String, + trace_to_progress: bool, + reverse_trace_lines: bool, } pub struct RunOutcome { diff --git a/gitoxide-core/src/corpus/trace.rs b/gitoxide-core/src/corpus/trace.rs index 8a7be071636..614d4b27f0c 100644 --- a/gitoxide-core/src/corpus/trace.rs +++ b/gitoxide-core/src/corpus/trace.rs @@ -1,17 +1,25 @@ +use gix::progress::DoOrDiscard; +use parking_lot::Mutex; use rusqlite::params; use std::path::Path; use std::sync::atomic::{AtomicU32, Ordering}; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use tracing_forest::tree::Tree; use tracing_subscriber::layer::SubscriberExt; +type ProgressItem = DoOrDiscard; + pub fn override_thread_subscriber( db_path: impl AsRef, + progress: Option, + reverse_lines: bool, ) -> anyhow::Result<(tracing::subscriber::DefaultGuard, Arc)> { let current_id = Arc::new(AtomicU32::default()); let processor = tracing_forest::Printer::new().formatter(StoreTreeToDb { con: Arc::new(Mutex::new(rusqlite::Connection::open(&db_path)?)), run_id: current_id.clone(), + progress: progress.map(Mutex::new), + reverse_lines, }); let subscriber = tracing_subscriber::Registry::default().with(tracing_forest::ForestLayer::from(processor)); let guard = tracing::subscriber::set_default(subscriber); @@ -19,19 +27,38 @@ pub fn override_thread_subscriber( } pub struct StoreTreeToDb { - pub con: Arc>, - pub run_id: Arc, + con: Arc>, + run_id: Arc, + progress: Option>, + reverse_lines: bool, } impl tracing_forest::printer::Formatter for StoreTreeToDb { type Error = rusqlite::Error; fn fmt(&self, tree: &Tree) -> Result { + if let Some((progress, tree)) = self + .progress + .as_ref() + .map(Mutex::lock) + .zip(tracing_forest::printer::Pretty.fmt(tree).ok()) + { + use gix::Progress; + if self.reverse_lines { + for line in tree.lines().rev() { + progress.info(line); + } + } else { + for line in tree.lines() { + progress.info(line); + } + } + } + // TODO: wait for new release of `tracing-forest` and load the ID from span fields. let json = serde_json::to_string_pretty(&tree).expect("serialization to string always works"); let run_id = self.run_id.load(Ordering::SeqCst); self.con .lock() - .unwrap() .execute("UPDATE run SET spans_json = ?1 WHERE id = ?2", params![json, run_id])?; Ok(String::new()) } diff --git a/src/plumbing/main.rs b/src/plumbing/main.rs index 306b9eae51a..bdcb9013f64 100644 --- a/src/plumbing/main.rs +++ b/src/plumbing/main.rs @@ -136,21 +136,30 @@ pub fn main() -> Result<()> { match cmd { #[cfg(feature = "gitoxide-core-tools-corpus")] - Subcommands::Corpus(crate::plumbing::options::corpus::Platform { db, path, cmd }) => prepare_and_run( - "corpus", - trace, - auto_verbose, - progress, - progress_keep_open, - core::corpus::PROGRESS_RANGE, - move |progress, _out, _err| { - let mut engine = core::corpus::Engine::open_or_create(db, env!("GITOXIDE_VERSION").into(), progress)?; - match cmd { - crate::plumbing::options::corpus::SubCommands::Run => engine.run(path, thread_limit), - crate::plumbing::options::corpus::SubCommands::Refresh => engine.refresh(path), - } - }, - ), + Subcommands::Corpus(crate::plumbing::options::corpus::Platform { db, path, cmd }) => { + let reverse_trace_lines = progress; + prepare_and_run( + "corpus", + trace, + auto_verbose, + progress, + progress_keep_open, + core::corpus::PROGRESS_RANGE, + move |progress, _out, _err| { + let mut engine = core::corpus::Engine::open_or_create( + db, + env!("GITOXIDE_VERSION").into(), + progress, + trace, + reverse_trace_lines, + )?; + match cmd { + crate::plumbing::options::corpus::SubCommands::Run => engine.run(path, thread_limit), + crate::plumbing::options::corpus::SubCommands::Refresh => engine.refresh(path), + } + }, + ) + } Subcommands::CommitGraph(cmd) => match cmd { commitgraph::Subcommands::List { spec } => prepare_and_run( "commitgraph-list", diff --git a/src/shared.rs b/src/shared.rs index 8b5e640a4fe..9b775d9b66d 100644 --- a/src/shared.rs +++ b/src/shared.rs @@ -108,8 +108,8 @@ pub mod pretty { move |tree: &tracing_forest::tree::Tree| -> Result { use gix::Progress; use tracing_forest::Formatter; - let tree = tracing_forest::printer::Pretty.fmt(tree)?; let progress = &mut progress.lock().unwrap(); + let tree = tracing_forest::printer::Pretty.fmt(tree)?; if reverse_lines { for line in tree.lines().rev() { progress.info(line);