diff --git a/bundle/src/bundler.rs b/bundle/src/bundler.rs index 5c04fffc..4e86e600 100644 --- a/bundle/src/bundler.rs +++ b/bundle/src/bundler.rs @@ -1,9 +1,13 @@ use async_compression::futures::bufread::ZstdDecoder; use async_std::{io::ReadExt, stream::StreamExt}; use async_tar_wasm::Archive; +use context::bazel_bep::parser::BepParseResult; use futures_io::AsyncBufRead; -use std::io::{Seek, Write}; -use std::path::PathBuf; +use std::{ + fs::File, + io::{Seek, Write}, + path::PathBuf, +}; #[cfg(feature = "wasm")] use tsify_next::Tsify; #[cfg(feature = "wasm")] @@ -17,7 +21,8 @@ use crate::bundle_meta::{BundleMeta, VersionedBundle}; /// #[cfg_attr(feature = "wasm", derive(Tsify))] pub struct BundlerUtil { - pub meta: BundleMeta, + meta: BundleMeta, + bep_result: Option, } const META_FILENAME: &'static str = "meta.json"; @@ -25,8 +30,8 @@ const META_FILENAME: &'static str = "meta.json"; impl BundlerUtil { const ZSTD_COMPRESSION_LEVEL: i32 = 15; // This gives roughly 10x compression for text, 22 gives 11x. - pub fn new(meta: BundleMeta) -> Self { - Self { meta } + pub fn new(meta: BundleMeta, bep_result: Option) -> Self { + Self { meta, bep_result } } /// Writes compressed tarball to disk. @@ -34,7 +39,7 @@ impl BundlerUtil { pub fn make_tarball(&self, bundle_path: &PathBuf) -> anyhow::Result<()> { let mut total_bytes_in: u64 = 0; - let tar_file = std::fs::File::create(bundle_path)?; + let tar_file = File::create(bundle_path)?; let zstd_encoder = zstd::Encoder::new(tar_file, Self::ZSTD_COMPRESSION_LEVEL)?; let mut tar = tar::Builder::new(zstd_encoder); @@ -56,7 +61,7 @@ impl BundlerUtil { .try_for_each(|file_set| { file_set.files.iter().try_for_each(|bundled_file| { let path = std::path::Path::new(&bundled_file.original_path); - let mut file = std::fs::File::open(path)?; + let mut file = File::open(path)?; tar.append_file(&bundled_file.path, &mut file)?; total_bytes_in += std::fs::metadata(path)?.len(); Ok::<(), anyhow::Error>(()) @@ -65,11 +70,28 @@ impl BundlerUtil { })?; if let Some(CodeOwners { ref path, .. }) = self.meta.base_props.codeowners { - let mut file = std::fs::File::open(path)?; + let mut file = File::open(path)?; tar.append_file("CODEOWNERS", &mut file)?; total_bytes_in += std::fs::metadata(path)?.len(); } + if let Some(bep_result) = self.bep_result.as_ref() { + let mut bep_events_file = + bep_result + .bep_test_events + .iter() + .fold(tempfile::tempfile()?, |f, event| { + if let Err(e) = serde_json::to_writer(&f, event) { + log::error!("Failed to write BEP event: {}", e); + } + f + }); + bep_events_file.flush()?; + bep_events_file.seek(std::io::SeekFrom::Start(0))?; + tar.append_file("bazel_bep.json", &mut bep_events_file)?; + total_bytes_in += bep_events_file.seek(std::io::SeekFrom::End(0))?; + } + // Flush to disk. tar.into_inner()?.finish()?; diff --git a/cli-tests/src/upload.rs b/cli-tests/src/upload.rs index bede1c7c..81ff98a3 100644 --- a/cli-tests/src/upload.rs +++ b/cli-tests/src/upload.rs @@ -9,7 +9,9 @@ use assert_cmd::Command; use assert_matches::assert_matches; use bundle::{BundleMeta, FileSetType}; use codeowners::CodeOwners; -use context::{junit::parser::JunitParser, repo::RepoUrlParts as Repo}; +use context::{ + bazel_bep::parser::BazelBepParser, junit::parser::JunitParser, repo::RepoUrlParts as Repo, +}; use predicates::prelude::*; use tempfile::tempdir; use test_utils::{ @@ -228,14 +230,19 @@ async fn upload_bundle_using_bep() { let tar_extract_directory = assert_matches!(&requests[3], RequestPayload::S3Upload(d) => d); - let file = fs::File::open(tar_extract_directory.join("junit/0")).unwrap(); - let reader = BufReader::new(file); + let junit_file = fs::File::open(tar_extract_directory.join("junit/0")).unwrap(); + let junit_reader = BufReader::new(junit_file); // Uploaded file is a junit, even when using BEP let mut junit_parser = JunitParser::new(); - assert!(junit_parser.parse(reader).is_ok()); + assert!(junit_parser.parse(junit_reader).is_ok()); assert!(junit_parser.errors().is_empty()); + let mut bazel_bep_parser = BazelBepParser::new(tar_extract_directory.join("bazel_bep.json")); + let parse_result = bazel_bep_parser.parse().ok().unwrap(); + assert!(parse_result.errors.is_empty()); + assert_eq!(parse_result.xml_file_counts(), (1, 0)); + // HINT: View CLI output with `cargo test -- --nocapture` println!("{assert}"); } diff --git a/cli/src/main.rs b/cli/src/main.rs index 5ee12aeb..d88df7cf 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -231,9 +231,9 @@ async fn run(cli: Cli) -> anyhow::Result { let junit_file_paths = match bazel_bep_path { Some(bazel_bep_path) => { let mut parser = BazelBepParser::new(bazel_bep_path); - parser.parse()?; - print_bep_results(&parser); - parser.uncached_xml_files() + let bep_result = parser.parse()?; + print_bep_results(&bep_result); + bep_result.uncached_xml_files() } None => junit_paths, }; diff --git a/cli/src/print.rs b/cli/src/print.rs index 59684bab..7e518cac 100644 --- a/cli/src/print.rs +++ b/cli/src/print.rs @@ -1,14 +1,14 @@ -use context::bazel_bep::parser::BazelBepParser; +use context::bazel_bep::parser::BepParseResult; -pub fn print_bep_results(parser: &BazelBepParser) { - if !parser.errors().is_empty() { - log::warn!("Errors parsing BEP file: {:?}", &parser.errors()); +pub fn print_bep_results(bep_result: &BepParseResult) { + if !bep_result.errors.is_empty() { + log::warn!("Errors parsing BEP file: {:?}", &bep_result.errors); } - let (test_count, cached_count) = parser.test_counts(); + let (xml_count, cached_xml_count) = bep_result.xml_file_counts(); log::info!( "Parsed {} ({} cached) test results from BEP file", - test_count, - cached_count + xml_count, + cached_xml_count ); } diff --git a/cli/src/runner.rs b/cli/src/runner.rs index 80fe862f..d8308481 100644 --- a/cli/src/runner.rs +++ b/cli/src/runner.rs @@ -35,9 +35,9 @@ pub async fn run_test_command( JunitSpec::Paths(paths) => paths, JunitSpec::BazelBep(bep_path) => { let mut parser = BazelBepParser::new(bep_path); - parser.parse()?; - print_bep_results(&parser); - parser.uncached_xml_files() + let bep_result = parser.parse()?; + print_bep_results(&bep_result); + bep_result.uncached_xml_files() } }; diff --git a/cli/src/upload.rs b/cli/src/upload.rs index f6c1aff8..f86de44f 100644 --- a/cli/src/upload.rs +++ b/cli/src/upload.rs @@ -18,7 +18,11 @@ use codeowners::CodeOwners; use constants::{EXIT_FAILURE, EXIT_SUCCESS}; #[cfg(target_os = "macos")] use context::repo::RepoUrlParts; -use context::{bazel_bep::parser::BazelBepParser, junit::parser::JunitParser, repo::BundleRepo}; +use context::{ + bazel_bep::parser::{BazelBepParser, BepParseResult}, + junit::parser::JunitParser, + repo::BundleRepo, +}; use crate::{ api_client::ApiClient, @@ -154,11 +158,13 @@ pub async fn run_upload( let codeowners = codeowners.or_else(|| CodeOwners::find_file(&repo.repo_root, &codeowners_path)); + let mut bep_result: Option = None; if let Some(bazel_bep_path) = bazel_bep_path { let mut parser = BazelBepParser::new(bazel_bep_path); - parser.parse()?; - print_bep_results(&parser); - junit_paths = parser.uncached_xml_files(); + let bep_parse_result = parser.parse()?; + print_bep_results(&bep_parse_result); + junit_paths = bep_parse_result.uncached_xml_files(); + bep_result = Some(bep_parse_result); } let tags = parse_custom_tags(&tags)?; @@ -308,7 +314,7 @@ pub async fn run_upload( let bundle_temp_dir = tempfile::tempdir()?; let bundle_time_file = bundle_temp_dir.path().join("bundle.tar.zstd"); - let bundle = BundlerUtil::new(meta); + let bundle = BundlerUtil::new(meta, bep_result); bundle.make_tarball(&bundle_time_file)?; log::info!("Flushed temporary tarball to {:?}", bundle_time_file); diff --git a/context/src/bazel_bep/parser.rs b/context/src/bazel_bep/parser.rs index be4af13c..160606a9 100644 --- a/context/src/bazel_bep/parser.rs +++ b/context/src/bazel_bep/parser.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use anyhow::Ok; use bazel_bep::types::build_event_stream::{build_event::Payload, file::File::Uri, BuildEvent}; use serde_json::Deserializer; @@ -10,30 +12,16 @@ pub struct TestResult { const FILE_URI_PREFIX: &str = "file://"; -/// Uses proto spec -/// https://github.com/TylerJang27/bazel-bep/blob/master/proto/build_event_stream.proto based on -/// https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto #[derive(Debug, Clone, Default)] -pub struct BazelBepParser { - bazel_bep_path: String, - test_results: Vec, - errors: Vec, +pub struct BepParseResult { + pub bep_test_events: Vec, + pub errors: Vec, + pub test_results: Vec, } -impl BazelBepParser { - pub fn new(bazel_bep_path: String) -> Self { - Self { - bazel_bep_path, - ..Default::default() - } - } - - pub fn errors(&self) -> &Vec { - &self.errors - } - - pub fn test_counts(&self) -> (usize, usize) { - let (test_count, cached_count) = self.test_results.iter().fold( +impl BepParseResult { + pub fn xml_file_counts(&self) -> (usize, usize) { + let (xml_count, cached_xml_count) = self.test_results.iter().fold( (0, 0), |(mut test_count, mut cached_count), test_result| { test_count += test_result.xml_files.len(); @@ -43,7 +31,7 @@ impl BazelBepParser { (test_count, cached_count) }, ); - (test_count, cached_count) + (xml_count, cached_xml_count) } pub fn uncached_xml_files(&self) -> Vec { @@ -58,28 +46,49 @@ impl BazelBepParser { .flatten() .collect() } +} + +/// Uses proto spec +/// https://github.com/TylerJang27/bazel-bep/blob/master/proto/build_event_stream.proto based on +/// https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto +#[derive(Debug, Clone, Default)] +pub struct BazelBepParser { + bazel_bep_path: PathBuf, +} + +impl BazelBepParser { + pub fn new>(bazel_bep_path: T) -> Self { + Self { + bazel_bep_path: bazel_bep_path.into(), + ..Default::default() + } + } - pub fn parse(&mut self) -> anyhow::Result<()> { + pub fn parse(&mut self) -> anyhow::Result { let file = std::fs::File::open(&self.bazel_bep_path)?; let reader = std::io::BufReader::new(file); - let (errors, test_results) = Deserializer::from_reader(reader) + let (errors, test_results, bep_test_events) = Deserializer::from_reader(reader) .into_iter::() .fold( - (Vec::::new(), Vec::::new()), - |(mut errors, mut test_results), parse_event| { + ( + Vec::::new(), + Vec::::new(), + Vec::::new(), + ), + |(mut errors, mut test_results, mut bep_test_events), parse_event| { match parse_event { Result::Err(ref err) => { errors.push(format!("Error parsing build event: {}", err)); } Result::Ok(build_event) => { - if let Some(Payload::TestResult(test_result)) = build_event.payload { - let xml_files = test_result + if let Some(Payload::TestResult(test_result)) = &build_event.payload { + let xml_files: Vec = test_result .test_action_output - .into_iter() + .iter() .filter_map(|action_output| { if action_output.name.ends_with(".xml") { - action_output.file.and_then(|f| { + action_output.file.clone().and_then(|f| { if let Uri(uri) = f { Some( uri.strip_prefix(FILE_URI_PREFIX) @@ -97,24 +106,27 @@ impl BazelBepParser { .collect(); let cached = - if let Some(execution_info) = test_result.execution_info { + if let Some(execution_info) = &test_result.execution_info { execution_info.cached_remotely || test_result.cached_locally } else { test_result.cached_locally }; + bep_test_events.push(build_event); test_results.push(TestResult { cached, xml_files }); } } } - (errors, test_results) + (errors, test_results, bep_test_events) }, ); - self.errors = errors; - self.test_results = test_results; - Ok(()) + Ok(BepParseResult { + bep_test_events, + errors, + test_results, + }) } } @@ -131,39 +143,42 @@ mod tests { fn test_parse_simple_bep() { let input_file = get_test_file_path(SIMPLE_EXAMPLE); let mut parser = BazelBepParser::new(input_file); - parser.parse().unwrap(); + let parse_result = parser.parse().unwrap(); let empty_vec: Vec = Vec::new(); assert_eq!( - parser.uncached_xml_files(), + parse_result.uncached_xml_files(), vec!["/tmp/hello_test/test.xml"] ); - assert_eq!(*parser.errors(), empty_vec); + assert_eq!(parse_result.xml_file_counts(), (1, 0)); + assert_eq!(*parse_result.errors, empty_vec); } #[test] fn test_parse_empty_bep() { let input_file = get_test_file_path(EMPTY_EXAMPLE); let mut parser = BazelBepParser::new(input_file); - parser.parse().unwrap(); + let parse_result = parser.parse().unwrap(); let empty_vec: Vec = Vec::new(); - assert_eq!(parser.uncached_xml_files(), empty_vec); - assert_eq!(*parser.errors(), empty_vec); + assert_eq!(parse_result.uncached_xml_files(), empty_vec); + assert_eq!(parse_result.xml_file_counts(), (0, 0)); + assert_eq!(*parse_result.errors, empty_vec); } #[test] fn test_parse_partial_bep() { let input_file = get_test_file_path(PARTIAL_EXAMPLE); let mut parser = BazelBepParser::new(input_file); - parser.parse().unwrap(); + let parse_result = parser.parse().unwrap(); assert_eq!( - parser.uncached_xml_files(), + parse_result.uncached_xml_files(), vec!["/tmp/hello_test/test.xml", "/tmp/client_test/test.xml"] ); + assert_eq!(parse_result.xml_file_counts(), (3, 1)); assert_eq!( - *parser.errors(), + *parse_result.errors, vec!["Error parsing build event: EOF while parsing a value at line 108 column 0"] ); }