diff --git a/Cargo.lock b/Cargo.lock index 71c0b45a..5795378a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -224,9 +224,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.4.0" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d5f1946157a96594eb2d2c10eb7ad9a2b27518cb3000209dec700c35df9197d" +checksum = "7c8d502cbaec4595d2e7d5f61e318f05417bd2b66fdc3809498f0d3fdf0bea27" dependencies = [ "clap_builder", "clap_derive", @@ -235,9 +235,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.0" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78116e32a042dd73c2901f0dc30790d20ff3447f3e3472fad359e8c3d282bcd6" +checksum = "5891c7bc0edb3e1c2204fc5e94009affabeb1821c9e5fdc3959536c5c0bb984d" dependencies = [ "anstream", "anstyle", @@ -431,6 +431,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -936,7 +942,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi 0.3.2", - "rustix 0.38.8", + "rustix 0.38.9", "windows-sys 0.48.0", ] @@ -1032,9 +1038,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "76fc44e2588d5b436dbc3c6cf62aef290f90dab6235744a93dfe1cc18f451e2c" [[package]] name = "memmap2" @@ -1124,16 +1130,15 @@ dependencies = [ [[package]] name = "nix" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +checksum = "abbbc55ad7b13aac85f9401c796dcda1b864e07fcad40ad47792eaa8932ea502" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "cfg-if", "libc", "memoffset", "pin-utils", - "static_assertions", ] [[package]] @@ -1491,9 +1496,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.8" +version = "0.38.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" +checksum = "9bfe0f2582b4931a45d1fa608f8a8722e8b3c7ac54dd6d5f3b3212791fedef49" dependencies = [ "bitflags 2.4.0", "errno", @@ -1525,18 +1530,18 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.186" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f5db24220c009de9bd45e69fb2938f4b6d2df856aa9304ce377b3180f83b7c1" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.186" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ad697f7e0b65af4983a4ce8f56ed5b357e8d3c36651bf6a7e13639c17b8e670" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", @@ -1562,9 +1567,9 @@ checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" [[package]] name = "sharded-slab" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +checksum = "b6805d8ff0f66aa61fb79a97a51ba210dcae753a797336dea8a36a3168196fab" dependencies = [ "lazy_static", ] @@ -1621,12 +1626,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "strip-ansi-escapes" version = "0.2.0" @@ -1706,7 +1705,7 @@ dependencies = [ "cfg-if", "fastrand", "redox_syscall 0.3.5", - "rustix 0.38.8", + "rustix 0.38.9", "windows-sys 0.48.0", ] @@ -1748,15 +1747,33 @@ name = "test-harness" version = "0.1.0" dependencies = [ "backoff", + "fs_extra", "itertools", "miette", "regex", "serde", "serde_json", + "tempfile", + "test-harness-macro", + "test_bin", "tokio", "tracing", ] +[[package]] +name = "test-harness-macro" +version = "0.1.0" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "test_bin" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e7a7de15468c6e65dd7db81cf3822c1ec94c71b2a3c1a976ea8e4696c91115c" + [[package]] name = "textwrap" version = "0.15.2" @@ -1812,9 +1829,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb39ee79a6d8de55f48f2293a830e040392f1c5f16e336bdd1788cd0aadce07" +checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" dependencies = [ "deranged", "itoa", @@ -1833,9 +1850,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733d258752e9303d392b94b75230d07b0b9c489350c69b851fc6c065fde3e8f9" +checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" dependencies = [ "time-core", ] diff --git a/Cargo.toml b/Cargo.toml index 10bd85ff..061bb190 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "ghcid-ng", "test-harness", + "test-harness-macro", ] resolver = "2" diff --git a/test-harness-macro/Cargo.toml b/test-harness-macro/Cargo.toml new file mode 100644 index 00000000..13f50937 --- /dev/null +++ b/test-harness-macro/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "test-harness-macro" +version = "0.1.0" +edition = "2021" + +description = "Test attribute for ghcid-ng" + +publish = false + +[lib] +proc-macro = true + +[dependencies] +quote = "1.0.33" +syn = { version = "2.0.29", features = ["full"] } diff --git a/test-harness-macro/src/lib.rs b/test-harness-macro/src/lib.rs new file mode 100644 index 00000000..7a3f2f30 --- /dev/null +++ b/test-harness-macro/src/lib.rs @@ -0,0 +1,106 @@ +use proc_macro::TokenStream; + +use quote::quote; +use quote::ToTokens; +use syn::parse; +use syn::parse::Parse; +use syn::parse::ParseStream; +use syn::Attribute; +use syn::Block; +use syn::Ident; +use syn::ItemFn; + +/// GHC versions to run each test under. +/// +/// Keep this synced with `../../flake.nix`. +const GHC_VERSIONS: [&str; 4] = ["9.0.2", "9.2.8", "9.4.6", "9.6.2"]; + +#[proc_macro_attribute] +pub fn test(_attr: TokenStream, item: TokenStream) -> TokenStream { + // Parse annotated function + let mut function: ItemFn = parse(item).expect("Could not parse item as function"); + + function.attrs.extend( + parse::( + quote! { + #[tokio::test] + #[tracing_test::traced_test] + #[allow(non_snake_case)] + } + .into(), + ) + .expect("Could not parse quoted attributes #[tokio::test] and #[tracing_test::traced_test]") + .0, + ); + + // Generate functions for each GHC version we want to test. + let mut ret = TokenStream::new(); + for ghc_version in GHC_VERSIONS { + ret.extend::( + make_test_fn(function.clone(), ghc_version) + .to_token_stream() + .into(), + ); + } + ret +} + +struct Attributes(Vec); + +impl Parse for Attributes { + fn parse(input: ParseStream) -> syn::Result { + Ok(Self(input.call(Attribute::parse_outer)?)) + } +} + +fn make_test_fn(mut function: ItemFn, ghc_version: &str) -> ItemFn { + let ghc_version_ident = ghc_version.replace('.', ""); + let stmts = function.block.stmts; + let test_name_base = function.sig.ident.to_string(); + let test_name = format!("{test_name_base}_{ghc_version_ident}"); + function.sig.ident = Ident::new(&test_name, function.sig.ident.span()); + + let new_body = parse::( + quote! { + { + ::test_harness::internal::IN_CUSTOM_TEST_HARNESS.with(|value| { + value.store(true, ::std::sync::atomic::Ordering::SeqCst); + }); + ::test_harness::internal::CARGO_TARGET_TMPDIR.with(|tmpdir| { + *tmpdir.borrow_mut() = Some(::std::path::PathBuf::from(env!("CARGO_TARGET_TMPDIR"))); + }); + ::test_harness::internal::GHC_VERSION.with(|tmpdir| { + *tmpdir.borrow_mut() = #ghc_version.to_owned(); + }); + + match ::tokio::task::spawn(async { + #(#stmts);* + }).await { + Err(err) => { + // Copy out temp files + ::test_harness::internal::save_test_logs( + format!("{}::{}", module_path!(), #test_name) + ); + ::test_harness::internal::cleanup_tempdir(); + + if err.is_panic() { + ::std::panic::resume_unwind(err.into_panic()); + } else { + panic!("Test cancelled? {err:?}"); + } + } + Ok(()) => { + ::test_harness::internal::cleanup_tempdir(); + } + }; + } + } + .into(), + ) + .expect("Could not parse function body"); + + // Replace function body + *function.block = new_body; + + function +} diff --git a/test-harness/Cargo.toml b/test-harness/Cargo.toml index 7c0cefc2..ab26f574 100644 --- a/test-harness/Cargo.toml +++ b/test-harness/Cargo.toml @@ -7,10 +7,14 @@ publish = false [dependencies] backoff = { version = "0.4.0", default-features = false } +fs_extra = "1.3.0" itertools = "0.11.0" miette = { version = "5.9.0", features = ["fancy"] } regex = "1.9.4" serde = { version = "1.0.186", features = ["derive"] } serde_json = "1.0.105" +tempfile = "3.8.0" +test-harness-macro = { path = "../test-harness-macro" } +test_bin = "0.4.0" tokio = { version = "1.28.2", features = ["full", "tracing"] } tracing = "0.1.37" diff --git a/test-harness/src/ghcid_ng.rs b/test-harness/src/ghcid_ng.rs new file mode 100644 index 00000000..5270ba68 --- /dev/null +++ b/test-harness/src/ghcid_ng.rs @@ -0,0 +1,186 @@ +use std::ffi::OsStr; +use std::path::Path; +use std::path::PathBuf; +use std::process::Stdio; +use std::time::Duration; + +use miette::miette; +use miette::Context; +use miette::IntoDiagnostic; +use tokio::process::Child; +use tokio::process::Command; +use tokio::sync::mpsc; +use tokio::task; +use tokio::task::JoinHandle; + +use crate::tracing_reader::TracingReader; +use crate::Event; +use crate::IntoMatcher; + +/// Where to write `ghcid-ng` logs written by integration tests, relative to the temporary +/// directory created for the test. +pub(crate) const LOG_FILENAME: &str = "ghcid-ng.json"; + +/// `ghcid-ng` session for integration testing. +/// +/// This handles copying a directory of files to a temporary directory, starting a `ghcid-ng` +/// session, and asynchronously reading a stream of log events from its JSON log output. +pub struct GhcidNg { + /// The current working directory of the `ghcid-ng` session. + cwd: PathBuf, + /// The `ghcid-ng` child process. + #[allow(dead_code)] + child: Child, + #[allow(dead_code)] + tracing_reader_handle: JoinHandle>, + /// A stream of tracing events from `ghcid-ng`. + log_receiver: mpsc::Receiver, +} + +impl GhcidNg { + /// Start a new `ghcid-ng` session in a copy of the given path. + pub async fn new(project_directory: impl AsRef) -> miette::Result { + Self::new_with_args(project_directory, std::iter::empty::<&str>()).await + } + + /// Start a new `ghcid-ng` session in a copy of the given path. + /// + /// Also add the given arguments to the `ghcid-ng` invocation. + pub async fn new_with_args( + project_directory: impl AsRef, + args: impl IntoIterator>, + ) -> miette::Result { + crate::internal::ensure_in_custom_test_harness()?; + let ghc_version = crate::internal::get_ghc_version()?; + let tempdir = crate::internal::set_tempdir()?; + write_cabal_config(&tempdir).await?; + check_ghc_version(&tempdir, &ghc_version).await?; + + let project_directory = project_directory.as_ref(); + tracing::info!("Copying project files"); + fs_extra::copy_items(&[project_directory], &tempdir, &Default::default()) + .into_diagnostic() + .wrap_err("Failed to copy project files")?; + + let project_directory_name = project_directory + .file_name() + .ok_or_else(|| miette!("Path has no directory name: {project_directory:?}"))?; + + let cwd = tempdir.join(project_directory_name); + + let log_path = tempdir.join(LOG_FILENAME); + + tracing::info!("Starting ghcid-ng"); + let child = Command::new(test_bin::get_test_bin("ghcid-ng").get_program()) + .arg("--log-json") + .arg(&log_path) + .args([ + "--command", + &format!("cabal --offline v2-repl --with-compiler ghc-{ghc_version}"), + "--tracing-filter", + "ghcid_ng=debug", + "--trace-spans", + "new,close", + ]) + .args(args) + .current_dir(&cwd) + .env("HOME", &tempdir) + .stdin(Stdio::piped()) + .stderr(Stdio::piped()) + .stdout(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .into_diagnostic() + .wrap_err("Failed to start `ghcid-ng`")?; + + // Wait for `ghcid-ng` to create the `log_path` + tokio::time::timeout(Duration::from_secs(10), crate::fs::wait_for_path(&log_path)) + .await + .into_diagnostic() + .wrap_err_with(|| { + format!("`ghcid-ng` didn't create log path {log_path:?} fast enough") + })?; + + let (sender, receiver) = mpsc::channel(128); + + let tracing_reader_handle = + task::spawn(TracingReader::new(sender, log_path.clone()).await?.run()); + + Ok(Self { + cwd, + child, + log_receiver: receiver, + tracing_reader_handle, + }) + } + + /// Wait until a matching log event is found. + /// + /// Errors if waiting for the event takes longer than the given `timeout`. + pub async fn get_log_with_timeout( + &mut self, + matcher: impl IntoMatcher, + timeout_duration: Duration, + ) -> miette::Result { + let matcher = matcher.into_matcher()?; + + match tokio::time::timeout(timeout_duration, async { + while let Some(event) = self.log_receiver.recv().await { + println!("{event}"); + if matcher.matches(&event) { + return Some(event); + } + } + None + }) + .await + { + Ok(Some(event)) => Ok(event), + Ok(None) => Err(miette!("Log task exited")), + Err(_) => Err(miette!("Waiting for a log message timed out")), + } + } + + /// Wait until a matching log event is found, with a default 1-minute timeout. + pub async fn get_log(&mut self, matcher: impl IntoMatcher) -> miette::Result { + self.get_log_with_timeout(matcher, Duration::from_secs(60)) + .await + } + + /// Get a path relative to the project root. + pub fn path(&self, path: impl AsRef) -> PathBuf { + self.cwd.join(path) + } +} + +/// Write an empty `~/.cabal/config` so that `cabal` doesn't try to access the internet. +/// +/// See: +async fn write_cabal_config(home: &Path) -> miette::Result<()> { + std::fs::create_dir_all(home.join(".cabal")) + .into_diagnostic() + .wrap_err("Failed to create `.cabal` directory")?; + crate::fs::touch(home.join(".cabal/config")) + .await + .wrap_err("Failed to write empty `.cabal/config`")?; + Ok(()) +} + +/// Check that `ghc-{ghc_version} --version` executes successfully. +/// +/// This is a nice check that the given GHC version is present in the environment, to fail tests +/// early without waiting for `ghcid-ng` to fail. +async fn check_ghc_version(home: &Path, ghc_version: &str) -> miette::Result<()> { + if Command::new(format!("ghc-{ghc_version}")) + .env("HOME", home) + .status() + .await + .into_diagnostic() + .wrap_err_with(|| format!("Failed to find GHC {ghc_version}"))? + .success() + { + Ok(()) + } else { + Err(miette!("Could not execute `ghc-{ghc_version} --version`. Are you running the integration tests in the `nix develop` shell?")) + } +} diff --git a/test-harness/src/internal.rs b/test-harness/src/internal.rs new file mode 100644 index 00000000..e5787b8d --- /dev/null +++ b/test-harness/src/internal.rs @@ -0,0 +1,119 @@ +//! Internal functions, exposed for the `#[test]` attribute macro. + +use std::cell::RefCell; +use std::path::PathBuf; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering::SeqCst; + +use miette::miette; +use miette::Context; +use miette::IntoDiagnostic; + +thread_local! { + /// The temporary directory where `ghcid-ng` is run. Note that because tests are run with the + /// `tokio` current-thread runtime, this is unique per-test. + pub static TEMPDIR: RefCell> = RefCell::new(None); + + /// Directory to put failed test logs in. The `#[test]` attribute sets this at the start of the + /// test to the value of the compile-time environment variable `$CARGO_TARGET_TMPDIR`. + /// See: + pub static CARGO_TARGET_TMPDIR: RefCell> = RefCell::new(None); + + /// The GHC version to use for this test. This should be a string like `ghc962`. + /// This is used to open a corresponding (e.g.) `nix develop .#ghc962` shell to run `ghcid-ng` + /// in. + pub static GHC_VERSION: RefCell = RefCell::new(String::new()); + + /// Is this thread running in the custom test harness? + /// If `GhcidNg::new` was used outside of our custom test harness, the temporary directory + /// wouldn't be cleaned up -- this lets us detect that case and error to avoid it. + pub static IN_CUSTOM_TEST_HARNESS: AtomicBool = const { AtomicBool::new(false) }; +} + +/// Save the test logs in `TEMPDIR` to `CARGO_TARGET_TMPDIR`. +/// +/// This is called when a `#[test]`-annotated function panics, to persist the logs for further +/// analysis. +pub fn save_test_logs(test_name: String) { + let log_path: PathBuf = TEMPDIR.with(|tempdir| { + tempdir + .borrow() + .as_deref() + .map(|path| path.join(crate::ghcid_ng::LOG_FILENAME)) + .expect("`test_harness::TEMPDIR` is not set") + }); + let persist_to = CARGO_TARGET_TMPDIR.with(|dir| { + dir.borrow() + .clone() + .expect("`CARGO_TARGET_TMPDIR` is not set") + }); + + let test_name = test_name.replace("::", "-"); + let persist_log_path = persist_to.join(format!("{test_name}.json")); + if persist_log_path.exists() { + // Cargo doesn't manage `CARGO_TARGET_TMPDIR` for us, so we remove output from old tests + // ourself. + std::fs::remove_file(&persist_log_path).expect("Failed to remove log output"); + } + + if !log_path.exists() { + eprintln!("No logs were written"); + } else { + let logs = std::fs::read_to_string(log_path).expect("Failed to read logs"); + std::fs::write(&persist_log_path, logs).expect("Failed to write logs"); + eprintln!("Wrote logs to {}", persist_log_path.display()); + } +} + +/// Remove the [`TEMPDIR`] from the filesystem. This is called at the end of `#[test]`-annotated +/// functions. +pub fn cleanup_tempdir() { + TEMPDIR.with(|path| { + std::fs::remove_dir_all(path.borrow().as_deref().expect("`TEMPDIR` is not set")) + .expect("Failed to clean up `TEMPDIR`"); + }); +} + +/// Fail if [`IN_CUSTOM_TEST_HARNESS`] has not been set. +pub(crate) fn ensure_in_custom_test_harness() -> miette::Result<()> { + if IN_CUSTOM_TEST_HARNESS.with(|value| value.load(SeqCst)) { + Ok(()) + } else { + Err(miette!( + "`GhcidNg` can only be used in `#[test_harness::test]` functions" + )) + } +} + +/// Get the GHC version as given by [`GHC_VERSION`]. +pub(crate) fn get_ghc_version() -> miette::Result { + let ghc_version = GHC_VERSION.with(|version| version.borrow().to_owned()); + if ghc_version.is_empty() { + Err(miette!("`GHC_VERSION` should be set")) + } else { + Ok(ghc_version) + } +} + +/// Create a new temporary directory and set [`TEMPDIR`] to it, persisting it to disk. +/// +/// Fails if [`TEMPDIR`] is already set. +pub(crate) fn set_tempdir() -> miette::Result { + let tempdir = tempfile::tempdir() + .into_diagnostic() + .wrap_err("Failed to create temporary directory")?; + + // Set the thread-local tempdir for cleanup later. + TEMPDIR.with(|thread_tempdir| { + if thread_tempdir.borrow().is_some() { + return Err(miette!( + "`GhcidNg` can only be constructed once per `#[test_harness::test]` function" + )); + } + *thread_tempdir.borrow_mut() = Some(tempdir.path().to_path_buf()); + Ok(()) + })?; + + // Now we can persist the tempdir to disk, knowing the test harness will clean it up later. + Ok(tempdir.into_path()) +} diff --git a/test-harness/src/lib.rs b/test-harness/src/lib.rs index 3a355475..3e7807dc 100644 --- a/test-harness/src/lib.rs +++ b/test-harness/src/lib.rs @@ -1,3 +1,5 @@ +//! Test harness library for `ghcid-ng` integration tests. + mod tracing_json; pub use tracing_json::Event; @@ -8,3 +10,10 @@ pub use matcher::IntoMatcher; pub use matcher::Matcher; pub mod fs; + +pub mod internal; + +pub use test_harness_macro::test; + +mod ghcid_ng; +pub use ghcid_ng::GhcidNg; diff --git a/test-harness/src/tracing_reader.rs b/test-harness/src/tracing_reader.rs index b2e33c79..3809cd2c 100644 --- a/test-harness/src/tracing_reader.rs +++ b/test-harness/src/tracing_reader.rs @@ -15,7 +15,6 @@ use tracing::instrument; use super::Event; /// A task to read JSON tracing log events output by `ghid-ng` and send them over a channel. -#[allow(dead_code)] pub struct TracingReader { sender: mpsc::Sender, lines: Lines>,