From fcbda1d2fee735c8bf3e7357a8691af2f7637922 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 14 Jun 2023 10:10:21 +0200 Subject: [PATCH] provide all the meta-data that is needed to make a run (and associate it with that) --- Cargo.lock | 28 ++++++++- gitoxide-core/Cargo.toml | 5 +- gitoxide-core/src/corpus/mod.rs | 108 +++++++++++++++++++++++++++++--- src/plumbing/main.rs | 2 +- 4 files changed, 132 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6cb9c1c95da..47e5add8698 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1275,6 +1275,7 @@ dependencies = [ "serde", "serde_json", "smallvec", + "sysinfo", "tempfile", "thiserror", ] @@ -2904,9 +2905,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.142" +version = "0.2.146" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" +checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" [[package]] name = "libgit2-sys" @@ -3150,6 +3151,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "num-traits" version = "0.2.15" @@ -4004,6 +4014,20 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sysinfo" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9557d0845b86eea8182f7b10dff120214fb6cd9fd937b6f4917714e546a38695" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "winapi", +] + [[package]] name = "tabled" version = "0.10.0" diff --git a/gitoxide-core/Cargo.toml b/gitoxide-core/Cargo.toml index 00d8ce20eb9..70b1c421961 100644 --- a/gitoxide-core/Cargo.toml +++ b/gitoxide-core/Cargo.toml @@ -22,7 +22,7 @@ estimate-hours = ["dep:itertools", "dep:fs-err", "dep:crossbeam-channel", "dep:s ## Gather information about repositories and store it in a database for easy querying. query = ["dep:rusqlite"] ## Run algorithms on a corpus of repositories and store their results for later comparison and intelligence gathering. -corpus = ["dep:rusqlite"] +corpus = ["dep:rusqlite", "dep:sysinfo"] #! ### Mutually Exclusive Networking #! If both are set, _blocking-client_ will take precedence, allowing `--all-features` to be used. @@ -71,6 +71,9 @@ smallvec = { version = "1.10.0", optional = true } # for 'query' and 'corpus' rusqlite = { version = "0.29.0", optional = true, features = ["bundled"] } +# for 'corpus' +sysinfo = { version = "0.29.2", optional = true, default-features = false } + # for svg graph output layout-rs = "0.1.1" open = "4.1.0" diff --git a/gitoxide-core/src/corpus/mod.rs b/gitoxide-core/src/corpus/mod.rs index 894905acc83..bc0c323c704 100644 --- a/gitoxide-core/src/corpus/mod.rs +++ b/gitoxide-core/src/corpus/mod.rs @@ -1,6 +1,7 @@ pub struct Engine

{ progress: P, con: rusqlite::Connection, + gitoxide_version: String, } pub mod engine { @@ -8,26 +9,39 @@ pub mod engine { use anyhow::Context; use std::path::PathBuf; + pub(crate) type Id = u32; + impl

Engine

where P: gix::Progress, { /// Open the corpus DB or create it. - pub fn open_or_create(db: PathBuf, progress: P) -> anyhow::Result> { + pub fn open_or_create(db: PathBuf, gitoxide_version: String, progress: P) -> anyhow::Result> { let con = crate::corpus::db::create(db).context("Could not open or create database")?; - Ok(Engine { progress, con }) + Ok(Engine { + progress, + con, + gitoxide_version, + }) } /// Run on the existing set of repositories we have already seen or obtain them from `path` if there is none yet. - pub fn run(&self, _path: PathBuf) -> anyhow::Result<()> { + pub fn run(&self, path: PathBuf) -> anyhow::Result<()> { + let _corpus_id = self.corpus_id_or_insert(&path)?; + let _gitoxide_id = self.gitoxide_version_id_or_insert()?; + let _runner_id = self.runner_id_or_insert()?; todo!() } } } pub mod db { - use anyhow::bail; + use crate::corpus::engine::Id; + use crate::corpus::Engine; + use anyhow::{bail, Context}; use rusqlite::{params, OptionalExtension}; + use std::path::Path; + use sysinfo::{CpuExt, CpuRefreshKind, RefreshKind, SystemExt}; /// A version to be incremented whenever the database layout is changed, to refresh it automatically. const VERSION: usize = 1; @@ -81,7 +95,7 @@ pub mod db { )?; con.execute_batch( r#" - CREATE TABLE if not exists gix_version( + CREATE TABLE if not exists gitoxide_version( version text UNIQUE -- the unique git version via gix describe ) "#, @@ -91,17 +105,97 @@ pub mod db { CREATE TABLE if not exists run( repository integer, runner integer, - gix_version integer, + gitoxide_version integer, start_time integer, end_time integer, -- or NULL if not yet finished (either successfull or with failure) error text, -- or NULL if there was on error FOREIGN KEY (repository) REFERENCES repository (rowid), FOREIGN KEY (runner) REFERENCES runner (rowid), - FOREIGN KEY (gix_version) REFERENCES gix_version (rowid) + FOREIGN KEY (gitoxide_version) REFERENCES gitoxide_version (rowid) ) "#, )?; Ok(con) } + + /// Utilities + impl

Engine

{ + pub(crate) fn runner_id_or_insert(&self) -> anyhow::Result { + let sys = sysinfo::System::new_with_specifics( + RefreshKind::new().with_cpu(CpuRefreshKind::new().with_frequency()), + ); + let cpu = &sys.cpus()[0]; + let vendor = Some(cpu.vendor_id().to_owned()); + let host = sys.host_name(); + let brand = Some(cpu.brand().to_owned()); + Ok( + match self + .con + .query_row( + "SELECT rowid FROM runner WHERE vendor = ?1 AND brand = ?2", + [vendor.as_deref(), brand.as_deref()], + |r| r.get(0), + ) + .optional()? + { + Some(existing) => existing, + None => { + self.con.execute( + "INSERT INTO runner (vendor, brand, host_name) VALUES (?1, ?2, ?3)", + [vendor.as_deref(), brand.as_deref(), host.as_deref()], + )?; + self.con.query_row( + "SELECT rowid FROM runner WHERE vendor = ?1 AND brand = ?2", + [vendor, brand], + |r| r.get(0), + )? + } + }, + ) + } + pub(crate) fn corpus_id_or_insert(&self, path: &Path) -> anyhow::Result { + let path = path.to_str().context("corpus root cannot contain illformed UTF-8")?; + Ok( + match self + .con + .query_row("SELECT rowid FROM corpus WHERE root = ?1", [path], |r| r.get(0)) + .optional()? + { + Some(existing) => existing, + None => { + self.con.execute("INSERT INTO corpus (root) VALUES (?1)", [path])?; + self.con + .query_row("SELECT rowid FROM corpus WHERE root = ?1", [path], |r| r.get(0))? + } + }, + ) + } + pub(crate) fn gitoxide_version_id_or_insert(&self) -> anyhow::Result { + Ok( + match self + .con + .query_row( + "SELECT rowid FROM gitoxide_version WHERE version = ?1", + [&self.gitoxide_version], + |r| r.get(0), + ) + .optional()? + { + Some(existing) => existing, + None => { + self.con.execute( + "INSERT INTO gitoxide_version (version) VALUES (?1)", + [&self.gitoxide_version], + )?; + self.con.query_row( + "SELECT rowid FROM gitoxide_version WHERE version = ?1", + [&self.gitoxide_version], + |r| r.get(0), + )? + } + }, + ) + } + } } diff --git a/src/plumbing/main.rs b/src/plumbing/main.rs index 6cc04931c83..0fd667d56c6 100644 --- a/src/plumbing/main.rs +++ b/src/plumbing/main.rs @@ -136,7 +136,7 @@ pub fn main() -> Result<()> { progress_keep_open, None, move |progress, _out, _err| { - let engine = core::corpus::Engine::open_or_create(db, progress)?; + let engine = core::corpus::Engine::open_or_create(db, env!("GITOXIDE_VERSION").into(), progress)?; match cmd { crate::plumbing::options::corpus::SubCommands::Run => engine.run(path), }