From d24f9af31c4e96e5158677408b513d3e33be7357 Mon Sep 17 00:00:00 2001 From: Gilad Naaman Date: Wed, 8 Nov 2017 06:06:16 +0200 Subject: [PATCH 01/10] Refactoring needed in order to have test json output. --- src/libtest/formatters.rs | 246 +++++++++++++++++++++++++++++++ src/libtest/lib.rs | 294 ++++++++------------------------------ 2 files changed, 308 insertions(+), 232 deletions(-) create mode 100644 src/libtest/formatters.rs diff --git a/src/libtest/formatters.rs b/src/libtest/formatters.rs new file mode 100644 index 0000000000000..4454e7ed115c2 --- /dev/null +++ b/src/libtest/formatters.rs @@ -0,0 +1,246 @@ +// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use super::*; + +pub(crate) trait OutputFormatter { + fn write_run_start(&mut self, len: usize) -> io::Result<()>; + fn write_test_start(&mut self, + test: &TestDesc, + align: NamePadding, + max_name_len: usize) -> io::Result<()>; + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()>; + fn write_result(&mut self, result: &TestResult) -> io::Result<()>; + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result; +} + +pub(crate) struct HumanFormatter { + out: OutputLocation, + terse: bool, + use_color: bool, + test_count: usize, +} + +impl HumanFormatter { + pub fn new(out: OutputLocation, use_color: bool, terse: bool) -> Self { + HumanFormatter { + out, + terse, + use_color, + test_count: 0, + } + } + + #[cfg(test)] + pub fn output_location(&self) -> &OutputLocation { + &self.out + } + + pub fn write_ok(&mut self) -> io::Result<()> { + self.write_short_result("ok", ".", term::color::GREEN) + } + + pub fn write_failed(&mut self) -> io::Result<()> { + self.write_short_result("FAILED", "F", term::color::RED) + } + + pub fn write_ignored(&mut self) -> io::Result<()> { + self.write_short_result("ignored", "i", term::color::YELLOW) + } + + pub fn write_allowed_fail(&mut self) -> io::Result<()> { + self.write_short_result("FAILED (allowed)", "a", term::color::YELLOW) + } + + pub fn write_bench(&mut self) -> io::Result<()> { + self.write_pretty("bench", term::color::CYAN) + } + + pub fn write_short_result(&mut self, verbose: &str, quiet: &str, color: term::color::Color) + -> io::Result<()> { + if self.terse { + self.write_pretty(quiet, color)?; + if self.test_count % QUIET_MODE_MAX_COLUMN == QUIET_MODE_MAX_COLUMN - 1 { + // we insert a new line every 100 dots in order to flush the + // screen when dealing with line-buffered output (e.g. piping to + // `stamp` in the rust CI). + self.write_plain("\n")?; + } + + self.test_count += 1; + Ok(()) + } else { + self.write_pretty(verbose, color)?; + self.write_plain("\n") + } + } + + pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> { + match self.out { + Pretty(ref mut term) => { + if self.use_color { + term.fg(color)?; + } + term.write_all(word.as_bytes())?; + if self.use_color { + term.reset()?; + } + term.flush() + } + Raw(ref mut stdout) => { + stdout.write_all(word.as_bytes())?; + stdout.flush() + } + } + } + + pub fn write_plain>(&mut self, s: S) -> io::Result<()> { + let s = s.as_ref(); + self.out.write_all(s.as_bytes())?; + self.out.flush() + } + + pub fn write_outputs(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_plain("\nsuccesses:\n")?; + let mut successes = Vec::new(); + let mut stdouts = String::new(); + for &(ref f, ref stdout) in &state.not_failures { + successes.push(f.name.to_string()); + if !stdout.is_empty() { + stdouts.push_str(&format!("---- {} stdout ----\n\t", f.name)); + let output = String::from_utf8_lossy(stdout); + stdouts.push_str(&output); + stdouts.push_str("\n"); + } + } + if !stdouts.is_empty() { + self.write_plain("\n")?; + self.write_plain(&stdouts)?; + } + + self.write_plain("\nsuccesses:\n")?; + successes.sort(); + for name in &successes { + self.write_plain(&format!(" {}\n", name))?; + } + Ok(()) + } + + pub fn write_failures(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_plain("\nfailures:\n")?; + let mut failures = Vec::new(); + let mut fail_out = String::new(); + for &(ref f, ref stdout) in &state.failures { + failures.push(f.name.to_string()); + if !stdout.is_empty() { + fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name)); + let output = String::from_utf8_lossy(stdout); + fail_out.push_str(&output); + fail_out.push_str("\n"); + } + } + if !fail_out.is_empty() { + self.write_plain("\n")?; + self.write_plain(&fail_out)?; + } + + self.write_plain("\nfailures:\n")?; + failures.sort(); + for name in &failures { + self.write_plain(&format!(" {}\n", name))?; + } + Ok(()) + } +} + +impl OutputFormatter for HumanFormatter { + fn write_run_start(&mut self, len: usize) -> io::Result<()> { + let noun = if len != 1 { + "tests" + } else { + "test" + }; + self.write_plain(&format!("\nrunning {} {}\n", len, noun)) + } + + fn write_test_start(&mut self, + test: &TestDesc, + align: NamePadding, + max_name_len: usize) -> io::Result<()> { + if self.terse && align != PadOnRight { + Ok(()) + } + else { + let name = test.padded_name(max_name_len, align); + self.write_plain(&format!("test {} ... ", name)) + } + } + + fn write_result(&mut self, result: &TestResult) -> io::Result<()> { + match *result { + TrOk => self.write_ok(), + TrFailed | TrFailedMsg(_) => self.write_failed(), + TrIgnored => self.write_ignored(), + TrAllowedFail => self.write_allowed_fail(), + TrBench(ref bs) => { + self.write_bench()?; + self.write_plain(&format!(": {}\n", fmt_bench_samples(bs))) + } + } + } + + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { + self.write_plain(&format!("test {} has been running for over {} seconds\n", + desc.name, + TEST_WARN_TIMEOUT_S)) + } + + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { + if state.options.display_output { + self.write_outputs(state)?; + } + let success = state.failed == 0; + if !success { + self.write_failures(state)?; + } + + self.write_plain("\ntest result: ")?; + + if success { + // There's no parallelism at this point so it's safe to use color + self.write_pretty("ok", term::color::GREEN)?; + } else { + self.write_pretty("FAILED", term::color::RED)?; + } + + let s = if state.allowed_fail > 0 { + format!( + ". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out\n\n", + state.passed, + state.failed + state.allowed_fail, + state.allowed_fail, + state.ignored, + state.measured, + state.filtered_out) + } else { + format!( + ". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n", + state.passed, + state.failed, + state.ignored, + state.measured, + state.filtered_out) + }; + + self.write_plain(&s)?; + + Ok(success) + } +} diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index f7880d3c4d854..2fb62c832f802 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -84,6 +84,9 @@ pub mod test { } pub mod stats; +mod formatters; + +use formatters::*; // The name of a test. By convention this follows the rules for rust // paths; i.e. it should be a series of identifiers separated by double @@ -359,7 +362,8 @@ fn optgroups() -> getopts::Options { in parallel", "n_threads") .optmulti("", "skip", "Skip tests whose names contain FILTER (this flag can \ be used multiple times)","FILTER") - .optflag("q", "quiet", "Display one character per test instead of one line") + .optflag("q", "quiet", "Display one character per test instead of one line.\ + Equivalent to --format=terse") .optflag("", "exact", "Exactly match filters rather than by substring") .optopt("", "color", "Configure coloring of output: auto = colorize if stdout is a tty and tests are run on serially (default); @@ -507,11 +511,24 @@ enum OutputLocation { Raw(T), } -struct ConsoleTestState { +impl Write for OutputLocation { + fn write(&mut self, buf: &[u8]) -> io::Result { + match *self { + Pretty(ref mut term) => term.write(buf), + Raw(ref mut stdout) => stdout.write(buf) + } + } + + fn flush(&mut self) -> io::Result<()> { + match *self { + Pretty(ref mut term) => term.flush(), + Raw(ref mut stdout) => stdout.flush() + } + } +} + +struct ConsoleTestState { log_out: Option, - out: OutputLocation, - use_color: bool, - quiet: bool, total: usize, passed: usize, failed: usize, @@ -526,22 +543,15 @@ struct ConsoleTestState { options: Options, } -impl ConsoleTestState { - pub fn new(opts: &TestOpts, _: Option) -> io::Result> { +impl ConsoleTestState { + pub fn new(opts: &TestOpts) -> io::Result { let log_out = match opts.logfile { Some(ref path) => Some(File::create(path)?), None => None, }; - let out = match term::stdout() { - None => Raw(io::stdout()), - Some(t) => Pretty(t), - }; Ok(ConsoleTestState { - out, log_out, - use_color: use_color(opts), - quiet: opts.quiet, total: 0, passed: 0, failed: 0, @@ -557,114 +567,6 @@ impl ConsoleTestState { }) } - pub fn write_ok(&mut self) -> io::Result<()> { - self.write_short_result("ok", ".", term::color::GREEN) - } - - pub fn write_failed(&mut self) -> io::Result<()> { - self.write_short_result("FAILED", "F", term::color::RED) - } - - pub fn write_ignored(&mut self) -> io::Result<()> { - self.write_short_result("ignored", "i", term::color::YELLOW) - } - - pub fn write_allowed_fail(&mut self) -> io::Result<()> { - self.write_short_result("FAILED (allowed)", "a", term::color::YELLOW) - } - - pub fn write_bench(&mut self) -> io::Result<()> { - self.write_pretty("bench", term::color::CYAN) - } - - pub fn write_short_result(&mut self, verbose: &str, quiet: &str, color: term::color::Color) - -> io::Result<()> { - if self.quiet { - self.write_pretty(quiet, color)?; - if self.current_test_count() % QUIET_MODE_MAX_COLUMN == QUIET_MODE_MAX_COLUMN - 1 { - // we insert a new line every 100 dots in order to flush the - // screen when dealing with line-buffered output (e.g. piping to - // `stamp` in the rust CI). - self.write_plain("\n")?; - } - Ok(()) - } else { - self.write_pretty(verbose, color)?; - self.write_plain("\n") - } - } - - pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> { - match self.out { - Pretty(ref mut term) => { - if self.use_color { - term.fg(color)?; - } - term.write_all(word.as_bytes())?; - if self.use_color { - term.reset()?; - } - term.flush() - } - Raw(ref mut stdout) => { - stdout.write_all(word.as_bytes())?; - stdout.flush() - } - } - } - - pub fn write_plain>(&mut self, s: S) -> io::Result<()> { - let s = s.as_ref(); - match self.out { - Pretty(ref mut term) => { - term.write_all(s.as_bytes())?; - term.flush() - } - Raw(ref mut stdout) => { - stdout.write_all(s.as_bytes())?; - stdout.flush() - } - } - } - - pub fn write_run_start(&mut self, len: usize) -> io::Result<()> { - self.total = len; - let noun = if len != 1 { - "tests" - } else { - "test" - }; - self.write_plain(&format!("\nrunning {} {}\n", len, noun)) - } - - pub fn write_test_start(&mut self, test: &TestDesc, align: NamePadding) -> io::Result<()> { - if self.quiet && align != PadOnRight { - Ok(()) - } else { - let name = test.padded_name(self.max_name_len, align); - self.write_plain(&format!("test {} ... ", name)) - } - } - - pub fn write_result(&mut self, result: &TestResult) -> io::Result<()> { - match *result { - TrOk => self.write_ok(), - TrFailed | TrFailedMsg(_) => self.write_failed(), - TrIgnored => self.write_ignored(), - TrAllowedFail => self.write_allowed_fail(), - TrBench(ref bs) => { - self.write_bench()?; - self.write_plain(&format!(": {}\n", fmt_bench_samples(bs))) - } - } - } - - pub fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { - self.write_plain(&format!("test {} has been running for over {} seconds\n", - desc.name, - TEST_WARN_TIMEOUT_S)) - } - pub fn write_log>(&mut self, msg: S) -> io::Result<()> { let msg = msg.as_ref(); match self.log_out { @@ -687,101 +589,9 @@ impl ConsoleTestState { test.name)) } - pub fn write_failures(&mut self) -> io::Result<()> { - self.write_plain("\nfailures:\n")?; - let mut failures = Vec::new(); - let mut fail_out = String::new(); - for &(ref f, ref stdout) in &self.failures { - failures.push(f.name.to_string()); - if !stdout.is_empty() { - fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name)); - let output = String::from_utf8_lossy(stdout); - fail_out.push_str(&output); - fail_out.push_str("\n"); - } - } - if !fail_out.is_empty() { - self.write_plain("\n")?; - self.write_plain(&fail_out)?; - } - - self.write_plain("\nfailures:\n")?; - failures.sort(); - for name in &failures { - self.write_plain(&format!(" {}\n", name))?; - } - Ok(()) - } - - pub fn write_outputs(&mut self) -> io::Result<()> { - self.write_plain("\nsuccesses:\n")?; - let mut successes = Vec::new(); - let mut stdouts = String::new(); - for &(ref f, ref stdout) in &self.not_failures { - successes.push(f.name.to_string()); - if !stdout.is_empty() { - stdouts.push_str(&format!("---- {} stdout ----\n\t", f.name)); - let output = String::from_utf8_lossy(stdout); - stdouts.push_str(&output); - stdouts.push_str("\n"); - } - } - if !stdouts.is_empty() { - self.write_plain("\n")?; - self.write_plain(&stdouts)?; - } - - self.write_plain("\nsuccesses:\n")?; - successes.sort(); - for name in &successes { - self.write_plain(&format!(" {}\n", name))?; - } - Ok(()) - } - fn current_test_count(&self) -> usize { self.passed + self.failed + self.ignored + self.measured + self.allowed_fail } - - pub fn write_run_finish(&mut self) -> io::Result { - assert!(self.current_test_count() == self.total); - - if self.options.display_output { - self.write_outputs()?; - } - let success = self.failed == 0; - if !success { - self.write_failures()?; - } - - self.write_plain("\ntest result: ")?; - if success { - // There's no parallelism at this point so it's safe to use color - self.write_pretty("ok", term::color::GREEN)?; - } else { - self.write_pretty("FAILED", term::color::RED)?; - } - let s = if self.allowed_fail > 0 { - format!( - ". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out\n\n", - self.passed, - self.failed + self.allowed_fail, - self.allowed_fail, - self.ignored, - self.measured, - self.filtered_out) - } else { - format!( - ". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n", - self.passed, - self.failed, - self.ignored, - self.measured, - self.filtered_out) - }; - self.write_plain(&s)?; - return Ok(success); - } } // Format a number with thousands separators @@ -827,7 +637,12 @@ pub fn fmt_bench_samples(bs: &BenchSamples) -> String { // List the tests to console, and optionally to logfile. Filters are honored. pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Result<()> { - let mut st = ConsoleTestState::new(opts, None::)?; + let output = match term::stdout() { + None => Raw(io::stdout()), + Some(t) => Pretty(t), + }; + let mut out = HumanFormatter::new(output, use_color(opts), opts.quiet); + let mut st = ConsoleTestState::new(opts)?; let mut ntest = 0; let mut nbench = 0; @@ -842,7 +657,7 @@ pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Res StaticBenchFn(..) | DynBenchFn(..) => { nbench += 1; "benchmark" }, }; - st.write_plain(format!("{}: {}\n", name, fntype))?; + out.write_plain(format!("{}: {}\n", name, fntype))?; st.write_log(format!("{} {}\n", fntype, name))?; } @@ -868,15 +683,21 @@ pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Res // A simple console test runner pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Result { - fn callback(event: &TestEvent, st: &mut ConsoleTestState) -> io::Result<()> { + fn callback(event: &TestEvent, + st: &mut ConsoleTestState, + out: &mut OutputFormatter) -> io::Result<()> { + match (*event).clone() { - TeFiltered(ref filtered_tests) => st.write_run_start(filtered_tests.len()), + TeFiltered(ref filtered_tests) => { + st.total = filtered_tests.len(); + out.write_run_start(filtered_tests.len()) + }, TeFilteredOut(filtered_out) => Ok(st.filtered_out = filtered_out), - TeWait(ref test, padding) => st.write_test_start(test, padding), - TeTimeout(ref test) => st.write_timeout(test), + TeWait(ref test, padding) => out.write_test_start(test, padding, st.max_name_len), + TeTimeout(ref test) => out.write_timeout(test), TeResult(test, result, stdout) => { st.write_log_result(&test, &result)?; - st.write_result(&result)?; + out.write_result(&result)?; match result { TrOk => { st.passed += 1; @@ -908,7 +729,14 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu } } - let mut st = ConsoleTestState::new(opts, None::)?; + let output = match term::stdout() { + None => Raw(io::stdout()), + Some(t) => Pretty(t), + }; + + let mut out = HumanFormatter::new(output, use_color(opts), opts.quiet); + + let mut st = ConsoleTestState::new(opts)?; fn len_if_padded(t: &TestDescAndFn) -> usize { match t.testfn.padding() { PadNone => 0, @@ -919,8 +747,11 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu let n = t.desc.name.as_slice(); st.max_name_len = n.len(); } - run_tests(opts, tests, |x| callback(&x, &mut st))?; - return st.write_run_finish(); + run_tests(opts, tests, |x| callback(&x, &mut st, &mut out))?; + + assert!(st.current_test_count() == st.total); + + return out.write_run_finish(&st); } #[test] @@ -939,11 +770,10 @@ fn should_sort_failures_before_printing_them() { allow_fail: false, }; - let mut st = ConsoleTestState { + let mut out = HumanFormatter::new(Raw(Vec::new()), false, false); + + let st = ConsoleTestState { log_out: None, - out: Raw(Vec::new()), - use_color: false, - quiet: false, total: 0, passed: 0, failed: 0, @@ -958,10 +788,10 @@ fn should_sort_failures_before_printing_them() { not_failures: Vec::new(), }; - st.write_failures().unwrap(); - let s = match st.out { - Raw(ref m) => String::from_utf8_lossy(&m[..]), - Pretty(_) => unreachable!(), + out.write_failures(&st).unwrap(); + let s = match out.output_location() { + &Raw(ref m) => String::from_utf8_lossy(&m[..]), + &Pretty(_) => unreachable!(), }; let apos = s.find("a").unwrap(); From 588a6a35be4446fbaaa792d08efee51e04e61fe8 Mon Sep 17 00:00:00 2001 From: Gilad Naaman Date: Fri, 10 Nov 2017 14:00:55 +0200 Subject: [PATCH 02/10] Added JSON output to libtest. libtest: Json format now outputs failed tests' stdouts. libtest: Json format now outputs failed tests' stdouts. libtest: Json formatter now spews individiual events, not as an array libtest: JSON fixes libtest: Better JSON escaping libtest: Test start event is printed on time --- src/libtest/formatters.rs | 238 +++++++++++++++++++++++++++--- src/libtest/lib.rs | 124 ++++++++++++---- src/tools/compiletest/src/main.rs | 2 +- 3 files changed, 313 insertions(+), 51 deletions(-) diff --git a/src/libtest/formatters.rs b/src/libtest/formatters.rs index 4454e7ed115c2..08d87b9097896 100644 --- a/src/libtest/formatters.rs +++ b/src/libtest/formatters.rs @@ -12,12 +12,12 @@ use super::*; pub(crate) trait OutputFormatter { fn write_run_start(&mut self, len: usize) -> io::Result<()>; - fn write_test_start(&mut self, - test: &TestDesc, - align: NamePadding, - max_name_len: usize) -> io::Result<()>; + fn write_test_start(&mut self, test: &TestDesc) -> io::Result<()>; fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()>; - fn write_result(&mut self, result: &TestResult) -> io::Result<()>; + fn write_result(&mut self, + desc: &TestDesc, + result: &TestResult, + stdout: &[u8]) -> io::Result<()>; fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result; } @@ -26,15 +26,17 @@ pub(crate) struct HumanFormatter { terse: bool, use_color: bool, test_count: usize, + max_name_len: usize, // number of columns to fill when aligning names } impl HumanFormatter { - pub fn new(out: OutputLocation, use_color: bool, terse: bool) -> Self { + pub fn new(out: OutputLocation, use_color: bool, terse: bool, max_name_len: usize) -> Self { HumanFormatter { out, terse, use_color, test_count: 0, + max_name_len, } } @@ -73,7 +75,7 @@ impl HumanFormatter { // `stamp` in the rust CI). self.write_plain("\n")?; } - + self.test_count += 1; Ok(()) } else { @@ -170,20 +172,18 @@ impl OutputFormatter for HumanFormatter { self.write_plain(&format!("\nrunning {} {}\n", len, noun)) } - fn write_test_start(&mut self, - test: &TestDesc, - align: NamePadding, - max_name_len: usize) -> io::Result<()> { - if self.terse && align != PadOnRight { - Ok(()) - } - else { - let name = test.padded_name(max_name_len, align); - self.write_plain(&format!("test {} ... ", name)) - } + fn write_test_start(&mut self, _desc: &TestDesc) -> io::Result<()> { + // Do not print header, as priting it at this point will result in + // an unreadable output when running tests concurrently. + Ok(()) } - fn write_result(&mut self, result: &TestResult) -> io::Result<()> { + fn write_result(&mut self, desc: &TestDesc, result: &TestResult, _: &[u8]) -> io::Result<()> { + if !(self.terse && desc.name.padding() != PadOnRight) { + let name = desc.padded_name(self.max_name_len, desc.name.padding()); + self.write_plain(&format!("test {} ... ", name))?; + } + match *result { TrOk => self.write_ok(), TrFailed | TrFailedMsg(_) => self.write_failed(), @@ -244,3 +244,203 @@ impl OutputFormatter for HumanFormatter { Ok(success) } } + +pub(crate) struct JsonFormatter { + out: OutputLocation +} + +impl JsonFormatter { + pub fn new(out: OutputLocation) -> Self { + Self { + out, } + } + + fn write_str>(&mut self, s: S) -> io::Result<()> { + self.out.write_all(s.as_ref().as_ref())?; + self.out.write_all("\n".as_ref()) + } + + fn write_event(&mut self, + ty: &str, + name: &str, + evt: &str, + extra: Option) -> io::Result<()> { + if let Some(extras) = extra { + self.write_str(&*format!(r#"{{ "type": "{}", "name": "{}", "event": "{}", {} }}"#, + ty, + name, + evt, + extras)) + } + else { + self.write_str(&*format!(r#"{{ "type": "{}", "name": "{}", "event": "{}" }}"#, + ty, + name, + evt)) + } + } +} + +impl OutputFormatter for JsonFormatter { + fn write_run_start(&mut self, len: usize) -> io::Result<()> { + self.write_str( + &*format!(r#"{{ "type": "suite", "event": "started", "test_count": "{}" }}"#, len)) + } + + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { + self.write_str(&*format!(r#"{{ "type": "test", "event": "started", "name": "{}" }}"#, + desc.name)) + } + + fn write_result(&mut self, + desc: &TestDesc, + result: &TestResult, + stdout: &[u8]) -> io::Result<()> { + match *result { + TrOk => { + self.write_event("test", desc.name.as_slice(), "ok", None) + }, + + TrFailed => { + let extra_data = if stdout.len() > 0 { + Some(format!(r#""stdout": "{}""#, + EscapedString(String::from_utf8_lossy(stdout)))) + } + else { + None + }; + + self.write_event("test", desc.name.as_slice(), "failed", extra_data) + }, + + TrFailedMsg(ref m) => { + self.write_event("test", + desc.name.as_slice(), + "failed", + Some(format!(r#""message": "{}""#, EscapedString(m)))) + }, + + TrIgnored => { + self.write_event("test", desc.name.as_slice(), "ignored", None) + }, + + TrAllowedFail => { + self.write_event("test", desc.name.as_slice(), "allowed_failure", None) + }, + + TrBench(ref bs) => { + let median = bs.ns_iter_summ.median as usize; + let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize; + + let mbps = if bs.mb_s == 0 { + "".into() + } + else { + format!(r#", "mib_per_second": {}"#, bs.mb_s) + }; + + let line = format!("{{ \"type\": \"bench\", \ + \"name\": \"{}\", \ + \"median\": {}, \ + \"deviation\": {}{} }}", + desc.name, + median, + deviation, + mbps); + + self.write_str(&*line) + }, + } + } + + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { + self.write_str(&*format!(r#"{{ "type": "test", "event": "timeout", "name": "{}" }}"#, + desc.name)) + } + + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { + + self.write_str(&*format!("{{ \"type\": \"suite\", \ + \"event\": \"{}\", \ + \"passed\": {}, \ + \"failed\": {}, \ + \"allowed_fail\": {}, \ + \"ignored\": {}, \ + \"measured\": {}, \ + \"filtered_out\": \"{}\" }}", + if state.failed == 0 { "ok" } else { "failed" }, + state.passed, + state.failed + state.allowed_fail, + state.allowed_fail, + state.ignored, + state.measured, + state.filtered_out))?; + + Ok(state.failed == 0) + } +} + +/// A formatting utility used to print strings with characters in need of escaping. +/// Base code taken form `libserialize::json::escape_str` +struct EscapedString>(S); + +impl> ::std::fmt::Display for EscapedString { + fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { + let mut start = 0; + + for (i, byte) in self.0.as_ref().bytes().enumerate() { + let escaped = match byte { + b'"' => "\\\"", + b'\\' => "\\\\", + b'\x00' => "\\u0000", + b'\x01' => "\\u0001", + b'\x02' => "\\u0002", + b'\x03' => "\\u0003", + b'\x04' => "\\u0004", + b'\x05' => "\\u0005", + b'\x06' => "\\u0006", + b'\x07' => "\\u0007", + b'\x08' => "\\b", + b'\t' => "\\t", + b'\n' => "\\n", + b'\x0b' => "\\u000b", + b'\x0c' => "\\f", + b'\r' => "\\r", + b'\x0e' => "\\u000e", + b'\x0f' => "\\u000f", + b'\x10' => "\\u0010", + b'\x11' => "\\u0011", + b'\x12' => "\\u0012", + b'\x13' => "\\u0013", + b'\x14' => "\\u0014", + b'\x15' => "\\u0015", + b'\x16' => "\\u0016", + b'\x17' => "\\u0017", + b'\x18' => "\\u0018", + b'\x19' => "\\u0019", + b'\x1a' => "\\u001a", + b'\x1b' => "\\u001b", + b'\x1c' => "\\u001c", + b'\x1d' => "\\u001d", + b'\x1e' => "\\u001e", + b'\x1f' => "\\u001f", + b'\x7f' => "\\u007f", + _ => { continue; } + }; + + if start < i { + f.write_str(&self.0.as_ref()[start..i])?; + } + + f.write_str(escaped)?; + + start = i + 1; + } + + if start != self.0.as_ref().len() { + f.write_str(&self.0.as_ref()[start..])?; + } + + Ok(()) + } +} diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index 2fb62c832f802..b11f783770f83 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -71,6 +71,7 @@ use std::sync::mpsc::{channel, Sender}; use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Instant, Duration}; +use std::borrow::Cow; const TEST_WARN_TIMEOUT_S: u64 = 60; const QUIET_MODE_MAX_COLUMN: usize = 100; // insert a '\n' after 100 tests in quiet mode @@ -97,14 +98,33 @@ use formatters::*; pub enum TestName { StaticTestName(&'static str), DynTestName(String), + AlignedTestName(Cow<'static, str>, NamePadding), } impl TestName { fn as_slice(&self) -> &str { match *self { StaticTestName(s) => s, DynTestName(ref s) => s, + AlignedTestName(ref s, _) => &*s, } } + + fn padding(&self) -> NamePadding { + match self { + &AlignedTestName(_, p) => p, + _ => PadNone, + } + } + + fn with_padding(&self, padding: NamePadding) -> TestName { + let name = match self { + &TestName::StaticTestName(name) => Cow::Borrowed(name), + &TestName::DynTestName(ref name) => Cow::Owned(name.clone()), + &TestName::AlignedTestName(ref name, _) => name.clone(), + }; + + TestName::AlignedTestName(name, padding) + } } impl fmt::Display for TestName { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { @@ -112,7 +132,7 @@ impl fmt::Display for TestName { } } -#[derive(Clone, Copy, PartialEq, Eq)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] pub enum NamePadding { PadNone, PadOnRight, @@ -306,6 +326,13 @@ pub enum ColorConfig { NeverColor, } +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum OutputFormat { + Pretty, + Terse, + Json +} + #[derive(Debug)] pub struct TestOpts { pub list: bool, @@ -317,7 +344,7 @@ pub struct TestOpts { pub logfile: Option, pub nocapture: bool, pub color: ColorConfig, - pub quiet: bool, + pub format: OutputFormat, pub test_threads: Option, pub skip: Vec, pub options: Options, @@ -336,7 +363,7 @@ impl TestOpts { logfile: None, nocapture: false, color: AutoColor, - quiet: false, + format: OutputFormat::Pretty, test_threads: None, skip: vec![], options: Options::new(), @@ -362,13 +389,17 @@ fn optgroups() -> getopts::Options { in parallel", "n_threads") .optmulti("", "skip", "Skip tests whose names contain FILTER (this flag can \ be used multiple times)","FILTER") - .optflag("q", "quiet", "Display one character per test instead of one line.\ - Equivalent to --format=terse") + .optflag("q", "quiet", "Display one character per test instead of one line. \ + Alias to --format=terse") .optflag("", "exact", "Exactly match filters rather than by substring") .optopt("", "color", "Configure coloring of output: auto = colorize if stdout is a tty and tests are run on serially (default); always = always colorize output; - never = never colorize output;", "auto|always|never"); + never = never colorize output;", "auto|always|never") + .optopt("", "format", "Configure formatting of output: + pretty = Print verbose output; + terse = Display one character per test; + json = Output a json document", "pretty|terse|json"); return opts } @@ -469,6 +500,19 @@ pub fn parse_opts(args: &[String]) -> Option { } }; + let format = match matches.opt_str("format").as_ref().map(|s| &**s) { + None if quiet => OutputFormat::Terse, + Some("pretty") | None => OutputFormat::Pretty, + Some("terse") => OutputFormat::Terse, + Some("json") => OutputFormat::Json, + + Some(v) => { + return Some(Err(format!("argument for --format must be pretty, terse, or json (was \ + {})", + v))) + } + }; + let test_opts = TestOpts { list, filter, @@ -479,7 +523,7 @@ pub fn parse_opts(args: &[String]) -> Option { logfile, nocapture, color, - quiet, + format, test_threads, skip: matches.opt_strs("skip"), options: Options::new(), @@ -539,7 +583,6 @@ struct ConsoleTestState { metrics: MetricMap, failures: Vec<(TestDesc, Vec)>, not_failures: Vec<(TestDesc, Vec)>, - max_name_len: usize, // number of columns to fill when aligning names options: Options, } @@ -562,7 +605,6 @@ impl ConsoleTestState { metrics: MetricMap::new(), failures: Vec::new(), not_failures: Vec::new(), - max_name_len: 0, options: opts.options, }) } @@ -641,7 +683,9 @@ pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Res None => Raw(io::stdout()), Some(t) => Pretty(t), }; - let mut out = HumanFormatter::new(output, use_color(opts), opts.quiet); + + let quiet = opts.format == OutputFormat::Terse; + let mut out = HumanFormatter::new(output, use_color(opts), quiet, 0); let mut st = ConsoleTestState::new(opts)?; let mut ntest = 0; @@ -668,11 +712,11 @@ pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Res } } - if !opts.quiet { + if !quiet { if ntest != 0 || nbench != 0 { - st.write_plain("\n")?; + out.write_plain("\n")?; } - st.write_plain(format!("{}, {}\n", + out.write_plain(format!("{}, {}\n", plural(ntest, "test"), plural(nbench, "benchmark")))?; } @@ -682,6 +726,14 @@ pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Res // A simple console test runner pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Result { + let tests = { + let mut tests = tests; + for test in tests.iter_mut() { + test.desc.name = test.desc.name.with_padding(test.testfn.padding()); + } + + tests + }; fn callback(event: &TestEvent, st: &mut ConsoleTestState, @@ -693,11 +745,11 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu out.write_run_start(filtered_tests.len()) }, TeFilteredOut(filtered_out) => Ok(st.filtered_out = filtered_out), - TeWait(ref test, padding) => out.write_test_start(test, padding, st.max_name_len), + TeWait(ref test) => out.write_test_start(test), TeTimeout(ref test) => out.write_timeout(test), TeResult(test, result, stdout) => { st.write_log_result(&test, &result)?; - out.write_result(&result)?; + out.write_result(&test, &result, &*stdout)?; match result { TrOk => { st.passed += 1; @@ -734,8 +786,25 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu Some(t) => Pretty(t), }; - let mut out = HumanFormatter::new(output, use_color(opts), opts.quiet); + let max_name_len = if let Some(t) = tests.iter().max_by_key(|t| len_if_padded(*t)) { + let n = t.desc.name.as_slice(); + n.len() + } + else { + 0 + }; + let mut out: Box = match opts.format { + OutputFormat::Pretty => Box::new(HumanFormatter::new(output, + use_color(opts), + false, + max_name_len)), + OutputFormat::Terse => Box::new(HumanFormatter::new(output, + use_color(opts), + true, + max_name_len)), + OutputFormat::Json => Box::new(JsonFormatter::new(output)), + }; let mut st = ConsoleTestState::new(opts)?; fn len_if_padded(t: &TestDescAndFn) -> usize { match t.testfn.padding() { @@ -743,11 +812,8 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu PadOnRight => t.desc.name.as_slice().len(), } } - if let Some(t) = tests.iter().max_by_key(|t| len_if_padded(*t)) { - let n = t.desc.name.as_slice(); - st.max_name_len = n.len(); - } - run_tests(opts, tests, |x| callback(&x, &mut st, &mut out))?; + + run_tests(opts, tests, |x| callback(&x, &mut st, &mut *out))?; assert!(st.current_test_count() == st.total); @@ -770,7 +836,7 @@ fn should_sort_failures_before_printing_them() { allow_fail: false, }; - let mut out = HumanFormatter::new(Raw(Vec::new()), false, false); + let mut out = HumanFormatter::new(Raw(Vec::new()), false, false, 10); let st = ConsoleTestState { log_out: None, @@ -781,7 +847,6 @@ fn should_sort_failures_before_printing_them() { allowed_fail: 0, filtered_out: 0, measured: 0, - max_name_len: 10, metrics: MetricMap::new(), failures: vec![(test_b, Vec::new()), (test_a, Vec::new())], options: Options::new(), @@ -839,7 +904,7 @@ fn stdout_isatty() -> bool { #[derive(Clone)] pub enum TestEvent { TeFiltered(Vec), - TeWait(TestDesc, NamePadding), + TeWait(TestDesc), TeResult(TestDesc, TestResult, Vec), TeTimeout(TestDesc), TeFilteredOut(usize), @@ -915,7 +980,7 @@ pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) if concurrency == 1 { while !remaining.is_empty() { let test = remaining.pop().unwrap(); - callback(TeWait(test.desc.clone(), test.testfn.padding()))?; + callback(TeWait(test.desc.clone()))?; run_test(opts, !opts.run_tests, test, tx.clone()); let (test, result, stdout) = rx.recv().unwrap(); callback(TeResult(test, result, stdout))?; @@ -926,6 +991,7 @@ pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) let test = remaining.pop().unwrap(); let timeout = Instant::now() + Duration::from_secs(TEST_WARN_TIMEOUT_S); running_tests.insert(test.desc.clone(), timeout); + callback(TeWait(test.desc.clone()))?; //here no pad run_test(opts, !opts.run_tests, test, tx.clone()); pending += 1; } @@ -949,7 +1015,6 @@ pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) let (desc, result, stdout) = res.unwrap(); running_tests.remove(&desc); - callback(TeWait(desc.clone(), PadNone))?; callback(TeResult(desc, result, stdout))?; pending -= 1; } @@ -958,7 +1023,7 @@ pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) if opts.bench_benchmarks { // All benchmarks run at the end, in serial. for b in filtered_benchs { - callback(TeWait(b.desc.clone(), b.testfn.padding()))?; + callback(TeWait(b.desc.clone()))?; run_test(opts, false, b, tx.clone()); let (test, result, stdout) = rx.recv().unwrap(); callback(TeResult(test, result, stdout))?; @@ -1239,10 +1304,7 @@ pub fn run_test(opts: &TestOpts, !cfg!(target_os = "emscripten") && !cfg!(target_arch = "wasm32"); if supports_threads { - let cfg = thread::Builder::new().name(match name { - DynTestName(ref name) => name.clone(), - StaticTestName(name) => name.to_owned(), - }); + let cfg = thread::Builder::new().name(name.as_slice().to_owned()); cfg.spawn(runtest).unwrap(); } else { runtest(); diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs index 533aaf9cd2735..1c52ebd7dc54e 100644 --- a/src/tools/compiletest/src/main.rs +++ b/src/tools/compiletest/src/main.rs @@ -485,7 +485,7 @@ pub fn test_opts(config: &Config) -> test::TestOpts { filter: config.filter.clone(), filter_exact: config.filter_exact, run_ignored: config.run_ignored, - quiet: config.quiet, + format: if config.quiet { test::OutputFormat::Terse } else { test::OutputFormat::Pretty }, logfile: config.logfile.clone(), run_tests: true, bench_benchmarks: true, From e570e9e79aa4ebfa5c1f9f9b2345dfb3525e42e7 Mon Sep 17 00:00:00 2001 From: Gilad Naaman Date: Wed, 13 Dec 2017 21:12:19 +0200 Subject: [PATCH 03/10] libtest: JSON formatting is now only available in unstable builds libtest: Added the -Z option for unstable options --- src/libtest/lib.rs | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index b11f783770f83..caaa4f7e2b7e5 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -399,7 +399,9 @@ fn optgroups() -> getopts::Options { .optopt("", "format", "Configure formatting of output: pretty = Print verbose output; terse = Display one character per test; - json = Output a json document", "pretty|terse|json"); + json = Output a json document", "pretty|terse|json") + .optopt("Z", "", "Enable nightly-only flags: + unstable-options = Allow use of experimental features", "unstable-options"); return opts } @@ -435,8 +437,19 @@ Test Attributes: usage = options.usage(&message)); } +// FIXME: Copied from libsyntax until linkage errors are resolved. +fn is_nightly() -> bool { + // Whether this is a feature-staged build, i.e. on the beta or stable channel + let disable_unstable_features = option_env!("CFG_DISABLE_UNSTABLE_FEATURES").is_some(); + // Whether we should enable unstable features for bootstrapping + let bootstrap = env::var("RUSTC_BOOTSTRAP").is_ok(); + + bootstrap || !disable_unstable_features +} + // Parses command line arguments into test options pub fn parse_opts(args: &[String]) -> Option { + let mut allow_unstable = false; let opts = optgroups(); let args = args.get(1..).unwrap_or(args); let matches = match opts.parse(args) { @@ -444,6 +457,21 @@ pub fn parse_opts(args: &[String]) -> Option { Err(f) => return Some(Err(f.to_string())), }; + if let Some(opt) = matches.opt_str("Z") { + if !is_nightly() { + return Some(Err("the option `Z` is only accepted on the nightly compiler".into())); + } + + match &*opt { + "unstable-options" => { + allow_unstable = true; + } + _ => { + return Some(Err("Unrecognized option to `Z`".into())); + } + } + }; + if matches.opt_present("h") { usage(&args[0], &opts); return None; @@ -504,7 +532,13 @@ pub fn parse_opts(args: &[String]) -> Option { None if quiet => OutputFormat::Terse, Some("pretty") | None => OutputFormat::Pretty, Some("terse") => OutputFormat::Terse, - Some("json") => OutputFormat::Json, + Some("json") => { + if !allow_unstable { + return Some( + Err("The \"json\" format is only accepted on the nightly compiler".into())); + } + OutputFormat::Json + }, Some(v) => { return Some(Err(format!("argument for --format must be pretty, terse, or json (was \ From 94bd1216bb735514118670878d28081f8493d1ac Mon Sep 17 00:00:00 2001 From: Gilad Naaman Date: Fri, 15 Dec 2017 17:25:44 +0200 Subject: [PATCH 04/10] libtest: Fixed pretty-printing of test names in single-threaded code. --- src/libtest/formatters.rs | 83 ++++++++++++++++++++++++++------------- src/libtest/lib.rs | 22 ++++++----- 2 files changed, 69 insertions(+), 36 deletions(-) diff --git a/src/libtest/formatters.rs b/src/libtest/formatters.rs index 08d87b9097896..f45ae3a7c2cc1 100644 --- a/src/libtest/formatters.rs +++ b/src/libtest/formatters.rs @@ -11,8 +11,8 @@ use super::*; pub(crate) trait OutputFormatter { - fn write_run_start(&mut self, len: usize) -> io::Result<()>; - fn write_test_start(&mut self, test: &TestDesc) -> io::Result<()>; + fn write_run_start(&mut self, test_count: usize) -> io::Result<()>; + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()>; fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()>; fn write_result(&mut self, desc: &TestDesc, @@ -26,17 +26,26 @@ pub(crate) struct HumanFormatter { terse: bool, use_color: bool, test_count: usize, - max_name_len: usize, // number of columns to fill when aligning names + + /// Number of columns to fill when aligning names + max_name_len: usize, + + is_multithreaded: bool, } impl HumanFormatter { - pub fn new(out: OutputLocation, use_color: bool, terse: bool, max_name_len: usize) -> Self { + pub fn new(out: OutputLocation, + use_color: bool, + terse: bool, + max_name_len: usize, + is_multithreaded: bool) -> Self { HumanFormatter { out, terse, use_color, test_count: 0, max_name_len, + is_multithreaded, } } @@ -160,28 +169,42 @@ impl HumanFormatter { } Ok(()) } + + fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> { + if !(self.terse && desc.name.padding() != PadOnRight) { + let name = desc.padded_name(self.max_name_len, desc.name.padding()); + self.write_plain(&format!("test {} ... ", name))?; + } + + Ok(()) + } } impl OutputFormatter for HumanFormatter { - fn write_run_start(&mut self, len: usize) -> io::Result<()> { - let noun = if len != 1 { + fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { + let noun = if test_count != 1 { "tests" } else { "test" }; - self.write_plain(&format!("\nrunning {} {}\n", len, noun)) + self.write_plain(&format!("\nrunning {} {}\n", test_count, noun)) } - fn write_test_start(&mut self, _desc: &TestDesc) -> io::Result<()> { - // Do not print header, as priting it at this point will result in - // an unreadable output when running tests concurrently. + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { + // When running tests concurrently, we should not print + // the test's name as the result will be mis-aligned. + // When running the tests serially, we print the name here so + // that the user can see which test hangs. + if !self.is_multithreaded { + self.write_test_name(desc)?; + } + Ok(()) } fn write_result(&mut self, desc: &TestDesc, result: &TestResult, _: &[u8]) -> io::Result<()> { - if !(self.terse && desc.name.padding() != PadOnRight) { - let name = desc.padded_name(self.max_name_len, desc.name.padding()); - self.write_plain(&format!("test {} ... ", name))?; + if self.is_multithreaded { + self.write_test_name(desc)?; } match *result { @@ -197,6 +220,10 @@ impl OutputFormatter for HumanFormatter { } fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { + if self.is_multithreaded { + self.write_test_name(desc)?; + } + self.write_plain(&format!("test {} has been running for over {} seconds\n", desc.name, TEST_WARN_TIMEOUT_S)) @@ -251,13 +278,14 @@ pub(crate) struct JsonFormatter { impl JsonFormatter { pub fn new(out: OutputLocation) -> Self { - Self { - out, } + Self { out } } - fn write_str>(&mut self, s: S) -> io::Result<()> { - self.out.write_all(s.as_ref().as_ref())?; - self.out.write_all("\n".as_ref()) + fn write_message(&mut self, s: &str) -> io::Result<()> { + assert!(!s.contains('\n')); + + self.out.write_all(s.as_ref())?; + self.out.write_all(b"\n") } fn write_event(&mut self, @@ -266,14 +294,14 @@ impl JsonFormatter { evt: &str, extra: Option) -> io::Result<()> { if let Some(extras) = extra { - self.write_str(&*format!(r#"{{ "type": "{}", "name": "{}", "event": "{}", {} }}"#, + self.write_message(&*format!(r#"{{ "type": "{}", "name": "{}", "event": "{}", {} }}"#, ty, name, evt, extras)) } else { - self.write_str(&*format!(r#"{{ "type": "{}", "name": "{}", "event": "{}" }}"#, + self.write_message(&*format!(r#"{{ "type": "{}", "name": "{}", "event": "{}" }}"#, ty, name, evt)) @@ -282,13 +310,14 @@ impl JsonFormatter { } impl OutputFormatter for JsonFormatter { - fn write_run_start(&mut self, len: usize) -> io::Result<()> { - self.write_str( - &*format!(r#"{{ "type": "suite", "event": "started", "test_count": "{}" }}"#, len)) + fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { + self.write_message( + &*format!(r#"{{ "type": "suite", "event": "started", "test_count": "{}" }}"#, + test_count)) } fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { - self.write_str(&*format!(r#"{{ "type": "test", "event": "started", "name": "{}" }}"#, + self.write_message(&*format!(r#"{{ "type": "test", "event": "started", "name": "{}" }}"#, desc.name)) } @@ -348,19 +377,19 @@ impl OutputFormatter for JsonFormatter { deviation, mbps); - self.write_str(&*line) + self.write_message(&*line) }, } } fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { - self.write_str(&*format!(r#"{{ "type": "test", "event": "timeout", "name": "{}" }}"#, + self.write_message(&*format!(r#"{{ "type": "test", "event": "timeout", "name": "{}" }}"#, desc.name)) } fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { - self.write_str(&*format!("{{ \"type\": \"suite\", \ + self.write_message(&*format!("{{ \"type\": \"suite\", \ \"event\": \"{}\", \ \"passed\": {}, \ \"failed\": {}, \ diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index caaa4f7e2b7e5..04c0734b5241b 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -719,7 +719,7 @@ pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Res }; let quiet = opts.format == OutputFormat::Terse; - let mut out = HumanFormatter::new(output, use_color(opts), quiet, 0); + let mut out = HumanFormatter::new(output, use_color(opts), quiet, 0, false); let mut st = ConsoleTestState::new(opts)?; let mut ntest = 0; @@ -820,23 +820,27 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu Some(t) => Pretty(t), }; - let max_name_len = if let Some(t) = tests.iter().max_by_key(|t| len_if_padded(*t)) { - let n = t.desc.name.as_slice(); - n.len() - } - else { - 0 + let max_name_len = tests.iter() + .max_by_key(|t| len_if_padded(*t)) + .map(|t| t.desc.name.as_slice().len()) + .unwrap_or(0); + + let is_multithreaded = match opts.test_threads { + Some(n) => n > 1, + None => get_concurrency() > 1, }; let mut out: Box = match opts.format { OutputFormat::Pretty => Box::new(HumanFormatter::new(output, use_color(opts), false, - max_name_len)), + max_name_len, + is_multithreaded)), OutputFormat::Terse => Box::new(HumanFormatter::new(output, use_color(opts), true, - max_name_len)), + max_name_len, + is_multithreaded)), OutputFormat::Json => Box::new(JsonFormatter::new(output)), }; let mut st = ConsoleTestState::new(opts)?; From 8b3fd98f4c12aca4e00f6cf2f00540182eef8a92 Mon Sep 17 00:00:00 2001 From: Gilad Naaman Date: Fri, 15 Dec 2017 17:32:00 +0200 Subject: [PATCH 05/10] libtest: rustfmt run libtest: Whoops --- src/libtest/formatters.rs | 178 ++++++----- src/libtest/lib.rs | 641 +++++++++++++++++++++++--------------- src/libtest/stats.rs | 504 ++++++++++++++++-------------- 3 files changed, 747 insertions(+), 576 deletions(-) diff --git a/src/libtest/formatters.rs b/src/libtest/formatters.rs index f45ae3a7c2cc1..59228146e6be4 100644 --- a/src/libtest/formatters.rs +++ b/src/libtest/formatters.rs @@ -14,10 +14,12 @@ pub(crate) trait OutputFormatter { fn write_run_start(&mut self, test_count: usize) -> io::Result<()>; fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()>; fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()>; - fn write_result(&mut self, - desc: &TestDesc, - result: &TestResult, - stdout: &[u8]) -> io::Result<()>; + fn write_result( + &mut self, + desc: &TestDesc, + result: &TestResult, + stdout: &[u8], + ) -> io::Result<()>; fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result; } @@ -34,11 +36,13 @@ pub(crate) struct HumanFormatter { } impl HumanFormatter { - pub fn new(out: OutputLocation, - use_color: bool, - terse: bool, - max_name_len: usize, - is_multithreaded: bool) -> Self { + pub fn new( + out: OutputLocation, + use_color: bool, + terse: bool, + max_name_len: usize, + is_multithreaded: bool, + ) -> Self { HumanFormatter { out, terse, @@ -74,8 +78,12 @@ impl HumanFormatter { self.write_pretty("bench", term::color::CYAN) } - pub fn write_short_result(&mut self, verbose: &str, quiet: &str, color: term::color::Color) - -> io::Result<()> { + pub fn write_short_result( + &mut self, + verbose: &str, + quiet: &str, + color: term::color::Color, + ) -> io::Result<()> { if self.terse { self.write_pretty(quiet, color)?; if self.test_count % QUIET_MODE_MAX_COLUMN == QUIET_MODE_MAX_COLUMN - 1 { @@ -182,11 +190,7 @@ impl HumanFormatter { impl OutputFormatter for HumanFormatter { fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { - let noun = if test_count != 1 { - "tests" - } else { - "test" - }; + let noun = if test_count != 1 { "tests" } else { "test" }; self.write_plain(&format!("\nrunning {} {}\n", test_count, noun)) } @@ -224,9 +228,11 @@ impl OutputFormatter for HumanFormatter { self.write_test_name(desc)?; } - self.write_plain(&format!("test {} has been running for over {} seconds\n", - desc.name, - TEST_WARN_TIMEOUT_S)) + self.write_plain(&format!( + "test {} has been running for over {} seconds\n", + desc.name, + TEST_WARN_TIMEOUT_S + )) } fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { @@ -255,7 +261,8 @@ impl OutputFormatter for HumanFormatter { state.allowed_fail, state.ignored, state.measured, - state.filtered_out) + state.filtered_out + ) } else { format!( ". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n", @@ -263,7 +270,8 @@ impl OutputFormatter for HumanFormatter { state.failed, state.ignored, state.measured, - state.filtered_out) + state.filtered_out + ) }; self.write_plain(&s)?; @@ -273,7 +281,7 @@ impl OutputFormatter for HumanFormatter { } pub(crate) struct JsonFormatter { - out: OutputLocation + out: OutputLocation, } impl JsonFormatter { @@ -288,74 +296,83 @@ impl JsonFormatter { self.out.write_all(b"\n") } - fn write_event(&mut self, - ty: &str, - name: &str, - evt: &str, - extra: Option) -> io::Result<()> { + fn write_event( + &mut self, + ty: &str, + name: &str, + evt: &str, + extra: Option, + ) -> io::Result<()> { if let Some(extras) = extra { - self.write_message(&*format!(r#"{{ "type": "{}", "name": "{}", "event": "{}", {} }}"#, - ty, - name, - evt, - extras)) - } - else { - self.write_message(&*format!(r#"{{ "type": "{}", "name": "{}", "event": "{}" }}"#, - ty, - name, - evt)) + self.write_message(&*format!( + r#"{{ "type": "{}", "name": "{}", "event": "{}", {} }}"#, + ty, + name, + evt, + extras + )) + } else { + self.write_message(&*format!( + r#"{{ "type": "{}", "name": "{}", "event": "{}" }}"#, + ty, + name, + evt + )) } } } impl OutputFormatter for JsonFormatter { fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { - self.write_message( - &*format!(r#"{{ "type": "suite", "event": "started", "test_count": "{}" }}"#, - test_count)) + self.write_message(&*format!( + r#"{{ "type": "suite", "event": "started", "test_count": "{}" }}"#, + test_count + )) } fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { - self.write_message(&*format!(r#"{{ "type": "test", "event": "started", "name": "{}" }}"#, - desc.name)) + self.write_message(&*format!( + r#"{{ "type": "test", "event": "started", "name": "{}" }}"#, + desc.name + )) } - fn write_result(&mut self, - desc: &TestDesc, - result: &TestResult, - stdout: &[u8]) -> io::Result<()> { + fn write_result( + &mut self, + desc: &TestDesc, + result: &TestResult, + stdout: &[u8], + ) -> io::Result<()> { match *result { - TrOk => { - self.write_event("test", desc.name.as_slice(), "ok", None) - }, + TrOk => self.write_event("test", desc.name.as_slice(), "ok", None), TrFailed => { let extra_data = if stdout.len() > 0 { - Some(format!(r#""stdout": "{}""#, - EscapedString(String::from_utf8_lossy(stdout)))) - } - else { + Some(format!( + r#""stdout": "{}""#, + EscapedString(String::from_utf8_lossy(stdout)) + )) + } else { None }; self.write_event("test", desc.name.as_slice(), "failed", extra_data) - }, + } TrFailedMsg(ref m) => { - self.write_event("test", - desc.name.as_slice(), - "failed", - Some(format!(r#""message": "{}""#, EscapedString(m)))) - }, + self.write_event( + "test", + desc.name.as_slice(), + "failed", + Some(format!(r#""message": "{}""#, EscapedString(m))), + ) + } - TrIgnored => { - self.write_event("test", desc.name.as_slice(), "ignored", None) - }, + TrIgnored => self.write_event("test", desc.name.as_slice(), "ignored", None), TrAllowedFail => { self.write_event("test", desc.name.as_slice(), "allowed_failure", None) - }, + } TrBench(ref bs) => { let median = bs.ns_iter_summ.median as usize; @@ -363,33 +380,37 @@ impl OutputFormatter for JsonFormatter { let mbps = if bs.mb_s == 0 { "".into() - } - else { + } else { format!(r#", "mib_per_second": {}"#, bs.mb_s) }; - let line = format!("{{ \"type\": \"bench\", \ + let line = format!( + "{{ \"type\": \"bench\", \ \"name\": \"{}\", \ \"median\": {}, \ \"deviation\": {}{} }}", - desc.name, - median, - deviation, - mbps); + desc.name, + median, + deviation, + mbps + ); self.write_message(&*line) - }, + } } } fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { - self.write_message(&*format!(r#"{{ "type": "test", "event": "timeout", "name": "{}" }}"#, - desc.name)) + self.write_message(&*format!( + r#"{{ "type": "test", "event": "timeout", "name": "{}" }}"#, + desc.name + )) } fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { - self.write_message(&*format!("{{ \"type\": \"suite\", \ + self.write_message(&*format!( + "{{ \"type\": \"suite\", \ \"event\": \"{}\", \ \"passed\": {}, \ \"failed\": {}, \ @@ -403,7 +424,8 @@ impl OutputFormatter for JsonFormatter { state.allowed_fail, state.ignored, state.measured, - state.filtered_out))?; + state.filtered_out + ))?; Ok(state.failed == 0) } @@ -454,7 +476,9 @@ impl> ::std::fmt::Display for EscapedString { b'\x1e' => "\\u001e", b'\x1f' => "\\u001f", b'\x7f' => "\\u007f", - _ => { continue; } + _ => { + continue; + } }; if start < i { diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index 04c0734b5241b..4da89f5ab4b3c 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -240,10 +240,7 @@ pub struct Metric { impl Metric { pub fn new(value: f64, noise: f64) -> Metric { - Metric { - value, - noise, - } + Metric { value, noise } } } @@ -255,9 +252,7 @@ pub struct Options { impl Options { pub fn new() -> Options { - Options { - display_output: false, - } + Options { display_output: false } } pub fn display_output(mut self, display_output: bool) -> Options { @@ -297,25 +292,24 @@ pub fn test_main(args: &[String], tests: Vec, options: Options) { // rather than a &[]. pub fn test_main_static(tests: &[TestDescAndFn]) { let args = env::args().collect::>(); - let owned_tests = tests.iter() - .map(|t| { - match t.testfn { - StaticTestFn(f) => { - TestDescAndFn { - testfn: StaticTestFn(f), - desc: t.desc.clone(), - } - } - StaticBenchFn(f) => { - TestDescAndFn { - testfn: StaticBenchFn(f), - desc: t.desc.clone(), - } - } - _ => panic!("non-static tests passed to test::test_main_static"), - } - }) - .collect(); + let owned_tests = tests + .iter() + .map(|t| match t.testfn { + StaticTestFn(f) => { + TestDescAndFn { + testfn: StaticTestFn(f), + desc: t.desc.clone(), + } + } + StaticBenchFn(f) => { + TestDescAndFn { + testfn: StaticBenchFn(f), + desc: t.desc.clone(), + } + } + _ => panic!("non-static tests passed to test::test_main_static"), + }) + .collect(); test_main(&args, owned_tests, Options::new()) } @@ -330,7 +324,7 @@ pub enum ColorConfig { pub enum OutputFormat { Pretty, Terse, - Json + Json, } #[derive(Debug)] @@ -381,33 +375,76 @@ fn optgroups() -> getopts::Options { .optflag("", "bench", "Run benchmarks instead of tests") .optflag("", "list", "List all tests and benchmarks") .optflag("h", "help", "Display this message (longer with --help)") - .optopt("", "logfile", "Write logs to the specified file instead \ - of stdout", "PATH") - .optflag("", "nocapture", "don't capture stdout/stderr of each \ - task, allow printing directly") - .optopt("", "test-threads", "Number of threads used for running tests \ - in parallel", "n_threads") - .optmulti("", "skip", "Skip tests whose names contain FILTER (this flag can \ - be used multiple times)","FILTER") - .optflag("q", "quiet", "Display one character per test instead of one line. \ - Alias to --format=terse") - .optflag("", "exact", "Exactly match filters rather than by substring") - .optopt("", "color", "Configure coloring of output: + .optopt( + "", + "logfile", + "Write logs to the specified file instead \ + of stdout", + "PATH", + ) + .optflag( + "", + "nocapture", + "don't capture stdout/stderr of each \ + task, allow printing directly", + ) + .optopt( + "", + "test-threads", + "Number of threads used for running tests \ + in parallel", + "n_threads", + ) + .optmulti( + "", + "skip", + "Skip tests whose names contain FILTER (this flag can \ + be used multiple times)", + "FILTER", + ) + .optflag( + "q", + "quiet", + "Display one character per test instead of one line. \ + Alias to --format=terse", + ) + .optflag( + "", + "exact", + "Exactly match filters rather than by substring", + ) + .optopt( + "", + "color", + "Configure coloring of output: auto = colorize if stdout is a tty and tests are run on serially (default); always = always colorize output; - never = never colorize output;", "auto|always|never") - .optopt("", "format", "Configure formatting of output: + never = never colorize output;", + "auto|always|never", + ) + .optopt( + "", + "format", + "Configure formatting of output: pretty = Print verbose output; terse = Display one character per test; - json = Output a json document", "pretty|terse|json") - .optopt("Z", "", "Enable nightly-only flags: - unstable-options = Allow use of experimental features", "unstable-options"); - return opts + json = Output a json document", + "pretty|terse|json", + ) + .optopt( + "Z", + "", + "Enable nightly-only flags: + unstable-options = Allow use of experimental features", + "unstable-options", + ); + return opts; } fn usage(binary: &str, options: &getopts::Options) { let message = format!("Usage: {} [OPTIONS] [FILTER]", binary); - println!(r#"{usage} + println!( + r#"{usage} The FILTER string is tested against the name of all tests, and only those tests whose names contain the filter are run. @@ -434,7 +471,8 @@ Test Attributes: test, then the test runner will ignore these tests during normal test runs. Running with --ignored will run these tests."#, - usage = options.usage(&message)); + usage = options.usage(&message) + ); } // FIXME: Copied from libsyntax until linkage errors are resolved. @@ -459,7 +497,10 @@ pub fn parse_opts(args: &[String]) -> Option { if let Some(opt) = matches.opt_str("Z") { if !is_nightly() { - return Some(Err("the option `Z` is only accepted on the nightly compiler".into())); + return Some(Err( + "the option `Z` is only accepted on the nightly compiler" + .into(), + )); } match &*opt { @@ -498,22 +539,25 @@ pub fn parse_opts(args: &[String]) -> Option { if !nocapture { nocapture = match env::var("RUST_TEST_NOCAPTURE") { Ok(val) => &val != "0", - Err(_) => false + Err(_) => false, }; } let test_threads = match matches.opt_str("test-threads") { - Some(n_str) => + Some(n_str) => { match n_str.parse::() { - Ok(0) => - return Some(Err(format!("argument for --test-threads must not be 0"))), + Ok(0) => return Some(Err(format!("argument for --test-threads must not be 0"))), Ok(n) => Some(n), - Err(e) => - return Some(Err(format!("argument for --test-threads must be a number > 0 \ - (error: {})", e))) - }, - None => - None, + Err(e) => { + return Some(Err(format!( + "argument for --test-threads must be a number > 0 \ + (error: {})", + e + ))) + } + } + } + None => None, }; let color = match matches.opt_str("color").as_ref().map(|s| &**s) { @@ -522,9 +566,11 @@ pub fn parse_opts(args: &[String]) -> Option { Some("never") => NeverColor, Some(v) => { - return Some(Err(format!("argument for --color must be auto, always, or never (was \ + return Some(Err(format!( + "argument for --color must be auto, always, or never (was \ {})", - v))) + v + ))) } }; @@ -534,16 +580,20 @@ pub fn parse_opts(args: &[String]) -> Option { Some("terse") => OutputFormat::Terse, Some("json") => { if !allow_unstable { - return Some( - Err("The \"json\" format is only accepted on the nightly compiler".into())); + return Some(Err( + "The \"json\" format is only accepted on the nightly compiler" + .into(), + )); } OutputFormat::Json - }, + } Some(v) => { - return Some(Err(format!("argument for --format must be pretty, terse, or json (was \ + return Some(Err(format!( + "argument for --format must be pretty, terse, or json (was \ {})", - v))) + v + ))) } }; @@ -593,14 +643,14 @@ impl Write for OutputLocation { fn write(&mut self, buf: &[u8]) -> io::Result { match *self { Pretty(ref mut term) => term.write(buf), - Raw(ref mut stdout) => stdout.write(buf) + Raw(ref mut stdout) => stdout.write(buf), } } fn flush(&mut self) -> io::Result<()> { match *self { Pretty(ref mut term) => term.flush(), - Raw(ref mut stdout) => stdout.flush() + Raw(ref mut stdout) => stdout.flush(), } } } @@ -652,17 +702,18 @@ impl ConsoleTestState { } pub fn write_log_result(&mut self, test: &TestDesc, result: &TestResult) -> io::Result<()> { - self.write_log( - format!("{} {}\n", - match *result { - TrOk => "ok".to_owned(), - TrFailed => "failed".to_owned(), - TrFailedMsg(ref msg) => format!("failed: {}", msg), - TrIgnored => "ignored".to_owned(), - TrAllowedFail => "failed (allowed)".to_owned(), - TrBench(ref bs) => fmt_bench_samples(bs), - }, - test.name)) + self.write_log(format!( + "{} {}\n", + match *result { + TrOk => "ok".to_owned(), + TrFailed => "failed".to_owned(), + TrFailedMsg(ref msg) => format!("failed: {}", msg), + TrIgnored => "ignored".to_owned(), + TrAllowedFail => "failed (allowed)".to_owned(), + TrBench(ref bs) => fmt_bench_samples(bs), + }, + test.name + )) } fn current_test_count(&self) -> usize { @@ -701,12 +752,17 @@ pub fn fmt_bench_samples(bs: &BenchSamples) -> String { let median = bs.ns_iter_summ.median as usize; let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize; - output.write_fmt(format_args!("{:>11} ns/iter (+/- {})", - fmt_thousands_sep(median, ','), - fmt_thousands_sep(deviation, ','))) - .unwrap(); + output + .write_fmt(format_args!( + "{:>11} ns/iter (+/- {})", + fmt_thousands_sep(median, ','), + fmt_thousands_sep(deviation, ',') + )) + .unwrap(); if bs.mb_s != 0 { - output.write_fmt(format_args!(" = {} MB/s", bs.mb_s)).unwrap(); + output + .write_fmt(format_args!(" = {} MB/s", bs.mb_s)) + .unwrap(); } output } @@ -728,11 +784,21 @@ pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Res for test in filter_tests(&opts, tests) { use TestFn::*; - let TestDescAndFn { desc: TestDesc { name, .. }, testfn } = test; + let TestDescAndFn { + desc: TestDesc { name, .. }, + testfn, + } = test; let fntype = match testfn { - StaticTestFn(..) | DynTestFn(..) => { ntest += 1; "test" }, - StaticBenchFn(..) | DynBenchFn(..) => { nbench += 1; "benchmark" }, + StaticTestFn(..) | DynTestFn(..) => { + ntest += 1; + "test" + } + StaticBenchFn(..) | + DynBenchFn(..) => { + nbench += 1; + "benchmark" + } }; out.write_plain(format!("{}: {}\n", name, fntype))?; @@ -750,9 +816,11 @@ pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Res if ntest != 0 || nbench != 0 { out.write_plain("\n")?; } - out.write_plain(format!("{}, {}\n", + out.write_plain(format!( + "{}, {}\n", plural(ntest, "test"), - plural(nbench, "benchmark")))?; + plural(nbench, "benchmark") + ))?; } Ok(()) @@ -769,15 +837,17 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu tests }; - fn callback(event: &TestEvent, - st: &mut ConsoleTestState, - out: &mut OutputFormatter) -> io::Result<()> { + fn callback( + event: &TestEvent, + st: &mut ConsoleTestState, + out: &mut OutputFormatter, + ) -> io::Result<()> { match (*event).clone() { TeFiltered(ref filtered_tests) => { st.total = filtered_tests.len(); out.write_run_start(filtered_tests.len()) - }, + } TeFilteredOut(filtered_out) => Ok(st.filtered_out = filtered_out), TeWait(ref test) => out.write_test_start(test), TeTimeout(ref test) => out.write_timeout(test), @@ -792,9 +862,11 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu TrIgnored => st.ignored += 1, TrAllowedFail => st.allowed_fail += 1, TrBench(bs) => { - st.metrics.insert_metric(test.name.as_slice(), - bs.ns_iter_summ.median, - bs.ns_iter_summ.max - bs.ns_iter_summ.min); + st.metrics.insert_metric( + test.name.as_slice(), + bs.ns_iter_summ.median, + bs.ns_iter_summ.max - bs.ns_iter_summ.min, + ); st.measured += 1 } TrFailed => { @@ -804,9 +876,7 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu TrFailedMsg(msg) => { st.failed += 1; let mut stdout = stdout; - stdout.extend_from_slice( - format!("note: {}", msg).as_bytes() - ); + stdout.extend_from_slice(format!("note: {}", msg).as_bytes()); st.failures.push((test, stdout)); } } @@ -820,10 +890,11 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu Some(t) => Pretty(t), }; - let max_name_len = tests.iter() - .max_by_key(|t| len_if_padded(*t)) - .map(|t| t.desc.name.as_slice().len()) - .unwrap_or(0); + let max_name_len = tests + .iter() + .max_by_key(|t| len_if_padded(*t)) + .map(|t| t.desc.name.as_slice().len()) + .unwrap_or(0); let is_multithreaded = match opts.test_threads { Some(n) => n > 1, @@ -831,16 +902,20 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu }; let mut out: Box = match opts.format { - OutputFormat::Pretty => Box::new(HumanFormatter::new(output, - use_color(opts), - false, - max_name_len, - is_multithreaded)), - OutputFormat::Terse => Box::new(HumanFormatter::new(output, - use_color(opts), - true, - max_name_len, - is_multithreaded)), + OutputFormat::Pretty => Box::new(HumanFormatter::new( + output, + use_color(opts), + false, + max_name_len, + is_multithreaded, + )), + OutputFormat::Terse => Box::new(HumanFormatter::new( + output, + use_color(opts), + true, + max_name_len, + is_multithreaded, + )), OutputFormat::Json => Box::new(JsonFormatter::new(output)), }; let mut st = ConsoleTestState::new(opts)?; @@ -874,7 +949,7 @@ fn should_sort_failures_before_printing_them() { allow_fail: false, }; - let mut out = HumanFormatter::new(Raw(Vec::new()), false, false, 10); + let mut out = HumanFormatter::new(Raw(Vec::new()), false, false, 10, false); let st = ConsoleTestState { log_out: None, @@ -952,7 +1027,8 @@ pub type MonitorMsg = (TestDesc, TestResult, Vec); pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) -> io::Result<()> - where F: FnMut(TestEvent) -> io::Result<()> +where + F: FnMut(TestEvent) -> io::Result<()>, { use std::collections::HashMap; use std::sync::mpsc::RecvTimeoutError; @@ -967,18 +1043,14 @@ pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) let filtered_out = tests_len - filtered_tests.len(); callback(TeFilteredOut(filtered_out))?; - let filtered_descs = filtered_tests.iter() - .map(|t| t.desc.clone()) - .collect(); + let filtered_descs = filtered_tests.iter().map(|t| t.desc.clone()).collect(); callback(TeFiltered(filtered_descs))?; let (filtered_tests, filtered_benchs): (Vec<_>, _) = - filtered_tests.into_iter().partition(|e| { - match e.testfn { - StaticTestFn(_) | DynTestFn(_) => true, - _ => false, - } + filtered_tests.into_iter().partition(|e| match e.testfn { + StaticTestFn(_) | DynTestFn(_) => true, + _ => false, }); let concurrency = match opts.test_threads { @@ -996,8 +1068,13 @@ pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) fn get_timed_out_tests(running_tests: &mut HashMap) -> Vec { let now = Instant::now(); - let timed_out = running_tests.iter() - .filter_map(|(desc, timeout)| if &now >= timeout { Some(desc.clone())} else { None }) + let timed_out = running_tests + .iter() + .filter_map(|(desc, timeout)| if &now >= timeout { + Some(desc.clone()) + } else { + None + }) .collect(); for test in &timed_out { running_tests.remove(test); @@ -1012,7 +1089,8 @@ pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) *next_timeout - now } else { Duration::new(0, 0) - }}) + } + }) }; if concurrency == 1 { @@ -1078,8 +1156,10 @@ fn get_concurrency() -> usize { match opt_n { Some(n) if n > 0 => n, _ => { - panic!("RUST_TEST_THREADS is `{}`, should be a positive integer.", - s) + panic!( + "RUST_TEST_THREADS is `{}`, should be a positive integer.", + s + ) } } } @@ -1136,10 +1216,8 @@ fn get_concurrency() -> usize { unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) as usize } } - #[cfg(any(target_os = "freebsd", - target_os = "dragonfly", - target_os = "bitrig", - target_os = "netbsd"))] + #[cfg(any(target_os = "freebsd", target_os = "dragonfly", target_os = "bitrig", + target_os = "netbsd"))] fn num_cpus() -> usize { use std::ptr; @@ -1152,12 +1230,14 @@ fn get_concurrency() -> usize { if cpus < 1 { let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0]; unsafe { - libc::sysctl(mib.as_mut_ptr(), - 2, - &mut cpus as *mut _ as *mut _, - &mut cpus_size as *mut _ as *mut _, - ptr::null_mut(), - 0); + libc::sysctl( + mib.as_mut_ptr(), + 2, + &mut cpus as *mut _ as *mut _, + &mut cpus_size as *mut _ as *mut _, + ptr::null_mut(), + 0, + ); } if cpus < 1 { cpus = 1; @@ -1175,12 +1255,14 @@ fn get_concurrency() -> usize { let mut mib = [libc::CTL_HW, libc::HW_NCPU, 0, 0]; unsafe { - libc::sysctl(mib.as_mut_ptr(), - 2, - &mut cpus as *mut _ as *mut _, - &mut cpus_size as *mut _ as *mut _, - ptr::null_mut(), - 0); + libc::sysctl( + mib.as_mut_ptr(), + 2, + &mut cpus as *mut _ as *mut _, + &mut cpus_size as *mut _ as *mut _, + ptr::null_mut(), + 0, + ); } if cpus < 1 { cpus = 1; @@ -1202,27 +1284,27 @@ pub fn filter_tests(opts: &TestOpts, tests: Vec) -> Vec filtered, Some(ref filter) => { - filtered.into_iter() - .filter(|test| { - if opts.filter_exact { - test.desc.name.as_slice() == &filter[..] - } else { - test.desc.name.as_slice().contains(&filter[..]) - } - }) - .collect() + filtered + .into_iter() + .filter(|test| if opts.filter_exact { + test.desc.name.as_slice() == &filter[..] + } else { + test.desc.name.as_slice().contains(&filter[..]) + }) + .collect() } }; // Skip tests that match any of the skip filters - filtered = filtered.into_iter() - .filter(|t| !opts.skip.iter().any(|sf| { - if opts.filter_exact { - t.desc.name.as_slice() == &sf[..] - } else { - t.desc.name.as_slice().contains(&sf[..]) - } - })) + filtered = filtered + .into_iter() + .filter(|t| { + !opts.skip.iter().any(|sf| if opts.filter_exact { + t.desc.name.as_slice() == &sf[..] + } else { + t.desc.name.as_slice().contains(&sf[..]) + }) + }) .collect(); // Maybe pull out the ignored test and unignore them @@ -1231,9 +1313,12 @@ pub fn filter_tests(opts: &TestOpts, tests: Vec) -> Vec Option { if test.desc.ignore { - let TestDescAndFn {desc, testfn} = test; + let TestDescAndFn { desc, testfn } = test; Some(TestDescAndFn { - desc: TestDesc { ignore: false, ..desc }, + desc: TestDesc { + ignore: false, + ..desc + }, testfn, }) } else { @@ -1244,7 +1329,9 @@ pub fn filter_tests(opts: &TestOpts, tests: Vec) -> Vec) -> Vec f, - }; - TestDescAndFn { - desc: x.desc, - testfn, - } - }).collect() + f => f, + }; + TestDescAndFn { + desc: x.desc, + testfn, + } + }) + .collect() } -pub fn run_test(opts: &TestOpts, - force_ignore: bool, - test: TestDescAndFn, - monitor_ch: Sender) { +pub fn run_test( + opts: &TestOpts, + force_ignore: bool, + test: TestDescAndFn, + monitor_ch: Sender, +) { - let TestDescAndFn {desc, testfn} = test; + let TestDescAndFn { desc, testfn } = test; - let ignore_because_panic_abort = - cfg!(target_arch = "wasm32") && + let ignore_because_panic_abort = cfg!(target_arch = "wasm32") && !cfg!(target_os = "emscripten") && desc.should_panic != ShouldPanic::No; @@ -1316,7 +1405,7 @@ pub fn run_test(opts: &TestOpts, let oldio = if !nocapture { Some(( io::set_print(Some(Box::new(Sink(data2.clone())))), - io::set_panic(Some(Box::new(Sink(data2)))) + io::set_panic(Some(Box::new(Sink(data2)))), )) } else { None @@ -1331,16 +1420,16 @@ pub fn run_test(opts: &TestOpts, let test_result = calc_result(&desc, result); let stdout = data.lock().unwrap().to_vec(); - monitor_ch.send((desc.clone(), test_result, stdout)).unwrap(); + monitor_ch + .send((desc.clone(), test_result, stdout)) + .unwrap(); }; // If the platform is single-threaded we're just going to run // the test synchronously, regardless of the concurrency // level. - let supports_threads = - !cfg!(target_os = "emscripten") && - !cfg!(target_arch = "wasm32"); + let supports_threads = !cfg!(target_os = "emscripten") && !cfg!(target_arch = "wasm32"); if supports_threads { let cfg = thread::Builder::new().name(name.as_slice().to_owned()); cfg.spawn(runtest).unwrap(); @@ -1382,12 +1471,13 @@ fn calc_result(desc: &TestDesc, task_result: Result<(), Box>) -> Tes match (&desc.should_panic, task_result) { (&ShouldPanic::No, Ok(())) | (&ShouldPanic::Yes, Err(_)) => TrOk, - (&ShouldPanic::YesWithMessage(msg), Err(ref err)) => + (&ShouldPanic::YesWithMessage(msg), Err(ref err)) => { if err.downcast_ref::() - .map(|e| &**e) - .or_else(|| err.downcast_ref::<&'static str>().map(|e| *e)) - .map(|e| e.contains(msg)) - .unwrap_or(false) { + .map(|e| &**e) + .or_else(|| err.downcast_ref::<&'static str>().map(|e| *e)) + .map(|e| e.contains(msg)) + .unwrap_or(false) + { TrOk } else { if desc.allow_fail { @@ -1395,7 +1485,8 @@ fn calc_result(desc: &TestDesc, task_result: Result<(), Box>) -> Tes } else { TrFailedMsg(format!("Panic did not include expected string '{}'", msg)) } - }, + } + } _ if desc.allow_fail => TrAllowedFail, _ => TrFailed, } @@ -1423,18 +1514,15 @@ impl MetricMap { /// you want to see grow larger, so a change larger than `noise` in the /// negative direction represents a regression. pub fn insert_metric(&mut self, name: &str, value: f64, noise: f64) { - let m = Metric { - value, - noise, - }; + let m = Metric { value, noise }; self.0.insert(name.to_owned(), m); } pub fn fmt_metrics(&self) -> String { let v = self.0 - .iter() - .map(|(k, v)| format!("{}: {} (+/- {})", *k, v.value, v.noise)) - .collect::>(); + .iter() + .map(|(k, v)| format!("{}: {} (+/- {})", *k, v.value, v.noise)) + .collect::>(); v.join(", ") } } @@ -1464,7 +1552,8 @@ pub fn black_box(dummy: T) -> T { impl Bencher { /// Callback for benchmark functions to run in their body. pub fn iter(&mut self, mut inner: F) - where F: FnMut() -> T + where + F: FnMut() -> T, { if self.mode == BenchMode::Single { ns_iter_inner(&mut inner, 1); @@ -1475,7 +1564,8 @@ impl Bencher { } pub fn bench(&mut self, mut f: F) -> Option - where F: FnMut(&mut Bencher) + where + F: FnMut(&mut Bencher), { f(self); return self.summary; @@ -1487,7 +1577,8 @@ fn ns_from_dur(dur: Duration) -> u64 { } fn ns_iter_inner(inner: &mut F, k: u64) -> u64 - where F: FnMut() -> T +where + F: FnMut() -> T, { let start = Instant::now(); for _ in 0..k { @@ -1498,7 +1589,8 @@ fn ns_iter_inner(inner: &mut F, k: u64) -> u64 pub fn iter(inner: &mut F) -> stats::Summary - where F: FnMut() -> T +where + F: FnMut() -> T, { // Initial bench run to get ballpark figure. let ns_single = ns_iter_inner(inner, 1); @@ -1540,7 +1632,8 @@ pub fn iter(inner: &mut F) -> stats::Summary // If we've run for 100ms and seem to have converged to a // stable median. if loop_run > Duration::from_millis(100) && summ.median_abs_dev_pct < 1.0 && - summ.median - summ5.median < summ5.median_abs_dev { + summ.median - summ5.median < summ5.median_abs_dev + { return summ5; } @@ -1569,7 +1662,8 @@ pub mod bench { use super::{Bencher, BenchSamples, BenchMode}; pub fn benchmark(f: F) -> BenchSamples - where F: FnMut(&mut Bencher) + where + F: FnMut(&mut Bencher), { let mut bs = Bencher { mode: BenchMode::Auto, @@ -1600,7 +1694,8 @@ pub mod bench { } pub fn run_once(f: F) - where F: FnMut(&mut Bencher) + where + F: FnMut(&mut Bencher), { let mut bs = Bencher { mode: BenchMode::Single, @@ -1740,7 +1835,11 @@ mod tests { #[test] fn parse_ignored_flag() { - let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()]; + let args = vec![ + "progname".to_string(), + "filter".to_string(), + "--ignored".to_string(), + ]; let opts = match parse_opts(&args) { Some(Ok(o)) => o, _ => panic!("Malformed arg in parse_ignored_flag"), @@ -1757,7 +1856,8 @@ mod tests { opts.run_tests = true; opts.run_ignored = true; - let tests = vec![TestDescAndFn { + let tests = + vec![TestDescAndFn { desc: TestDesc { name: StaticTestName("1"), ignore: true, @@ -1785,72 +1885,95 @@ mod tests { #[test] pub fn exact_filter_match() { fn tests() -> Vec { - vec!["base", - "base::test", - "base::test1", - "base::test2", - ].into_iter() - .map(|name| TestDescAndFn { - desc: TestDesc { - name: StaticTestName(name), - ignore: false, - should_panic: ShouldPanic::No, - allow_fail: false, - }, - testfn: DynTestFn(Box::new(move || {})) - }) - .collect() + vec!["base", "base::test", "base::test1", "base::test2"] + .into_iter() + .map(|name| { + TestDescAndFn { + desc: TestDesc { + name: StaticTestName(name), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + }, + testfn: DynTestFn(Box::new(move || {})) + } + }).collect() } - let substr = filter_tests(&TestOpts { + let substr = filter_tests( + &TestOpts { filter: Some("base".into()), ..TestOpts::new() - }, tests()); + }, + tests(), + ); assert_eq!(substr.len(), 4); - let substr = filter_tests(&TestOpts { + let substr = filter_tests( + &TestOpts { filter: Some("bas".into()), ..TestOpts::new() - }, tests()); + }, + tests(), + ); assert_eq!(substr.len(), 4); - let substr = filter_tests(&TestOpts { + let substr = filter_tests( + &TestOpts { filter: Some("::test".into()), ..TestOpts::new() - }, tests()); + }, + tests(), + ); assert_eq!(substr.len(), 3); - let substr = filter_tests(&TestOpts { + let substr = filter_tests( + &TestOpts { filter: Some("base::test".into()), ..TestOpts::new() - }, tests()); + }, + tests(), + ); assert_eq!(substr.len(), 3); - let exact = filter_tests(&TestOpts { + let exact = filter_tests( + &TestOpts { filter: Some("base".into()), - filter_exact: true, ..TestOpts::new() - }, tests()); + filter_exact: true, + ..TestOpts::new() + }, + tests(), + ); assert_eq!(exact.len(), 1); - let exact = filter_tests(&TestOpts { + let exact = filter_tests( + &TestOpts { filter: Some("bas".into()), filter_exact: true, ..TestOpts::new() - }, tests()); + }, + tests(), + ); assert_eq!(exact.len(), 0); - let exact = filter_tests(&TestOpts { + let exact = filter_tests( + &TestOpts { filter: Some("::test".into()), filter_exact: true, ..TestOpts::new() - }, tests()); + }, + tests(), + ); assert_eq!(exact.len(), 0); - let exact = filter_tests(&TestOpts { + let exact = filter_tests( + &TestOpts { filter: Some("base::test".into()), filter_exact: true, ..TestOpts::new() - }, tests()); + }, + tests(), + ); assert_eq!(exact.len(), 1); } @@ -1859,15 +1982,17 @@ mod tests { let mut opts = TestOpts::new(); opts.run_tests = true; - let names = vec!["sha1::test".to_string(), - "isize::test_to_str".to_string(), - "isize::test_pow".to_string(), - "test::do_not_run_ignored_tests".to_string(), - "test::ignored_tests_result_in_ignored".to_string(), - "test::first_free_arg_should_be_a_filter".to_string(), - "test::parse_ignored_flag".to_string(), - "test::filter_for_ignored_option".to_string(), - "test::sort_tests".to_string()]; + let names = vec![ + "sha1::test".to_string(), + "isize::test_to_str".to_string(), + "isize::test_pow".to_string(), + "test::do_not_run_ignored_tests".to_string(), + "test::ignored_tests_result_in_ignored".to_string(), + "test::first_free_arg_should_be_a_filter".to_string(), + "test::parse_ignored_flag".to_string(), + "test::filter_for_ignored_option".to_string(), + "test::sort_tests".to_string(), + ]; let tests = { fn testfn() {} let mut tests = Vec::new(); @@ -1887,15 +2012,17 @@ mod tests { }; let filtered = filter_tests(&opts, tests); - let expected = vec!["isize::test_pow".to_string(), - "isize::test_to_str".to_string(), - "sha1::test".to_string(), - "test::do_not_run_ignored_tests".to_string(), - "test::filter_for_ignored_option".to_string(), - "test::first_free_arg_should_be_a_filter".to_string(), - "test::ignored_tests_result_in_ignored".to_string(), - "test::parse_ignored_flag".to_string(), - "test::sort_tests".to_string()]; + let expected = vec![ + "isize::test_pow".to_string(), + "isize::test_to_str".to_string(), + "sha1::test".to_string(), + "test::do_not_run_ignored_tests".to_string(), + "test::filter_for_ignored_option".to_string(), + "test::first_free_arg_should_be_a_filter".to_string(), + "test::ignored_tests_result_in_ignored".to_string(), + "test::parse_ignored_flag".to_string(), + "test::sort_tests".to_string(), + ]; for (a, b) in expected.iter().zip(filtered) { assert!(*a == b.desc.name.to_string()); @@ -1934,8 +2061,7 @@ mod tests { #[test] pub fn test_bench_once_iter() { fn f(b: &mut Bencher) { - b.iter(|| { - }) + b.iter(|| {}) } bench::run_once(f); } @@ -1949,8 +2075,7 @@ mod tests { #[test] pub fn test_bench_iter() { fn f(b: &mut Bencher) { - b.iter(|| { - }) + b.iter(|| {}) } bench::benchmark(f); } diff --git a/src/libtest/stats.rs b/src/libtest/stats.rs index 9f8b4a73d0cc0..e22fdf77fc171 100644 --- a/src/libtest/stats.rs +++ b/src/libtest/stats.rs @@ -400,16 +400,18 @@ mod tests { } #[test] fn test_norm10narrow() { - let val = &[966.0000000000, - 985.0000000000, - 1110.0000000000, - 848.0000000000, - 821.0000000000, - 975.0000000000, - 962.0000000000, - 1157.0000000000, - 1217.0000000000, - 955.0000000000]; + let val = &[ + 966.0000000000, + 985.0000000000, + 1110.0000000000, + 848.0000000000, + 821.0000000000, + 975.0000000000, + 962.0000000000, + 1157.0000000000, + 1217.0000000000, + 955.0000000000, + ]; let summ = &Summary { sum: 9996.0000000000, min: 821.0000000000, @@ -428,16 +430,18 @@ mod tests { } #[test] fn test_norm10medium() { - let val = &[954.0000000000, - 1064.0000000000, - 855.0000000000, - 1000.0000000000, - 743.0000000000, - 1084.0000000000, - 704.0000000000, - 1023.0000000000, - 357.0000000000, - 869.0000000000]; + let val = &[ + 954.0000000000, + 1064.0000000000, + 855.0000000000, + 1000.0000000000, + 743.0000000000, + 1084.0000000000, + 704.0000000000, + 1023.0000000000, + 357.0000000000, + 869.0000000000, + ]; let summ = &Summary { sum: 8653.0000000000, min: 357.0000000000, @@ -456,16 +460,18 @@ mod tests { } #[test] fn test_norm10wide() { - let val = &[505.0000000000, - 497.0000000000, - 1591.0000000000, - 887.0000000000, - 1026.0000000000, - 136.0000000000, - 1580.0000000000, - 940.0000000000, - 754.0000000000, - 1433.0000000000]; + let val = &[ + 505.0000000000, + 497.0000000000, + 1591.0000000000, + 887.0000000000, + 1026.0000000000, + 136.0000000000, + 1580.0000000000, + 940.0000000000, + 754.0000000000, + 1433.0000000000, + ]; let summ = &Summary { sum: 9349.0000000000, min: 136.0000000000, @@ -484,31 +490,33 @@ mod tests { } #[test] fn test_norm25verynarrow() { - let val = &[991.0000000000, - 1018.0000000000, - 998.0000000000, - 1013.0000000000, - 974.0000000000, - 1007.0000000000, - 1014.0000000000, - 999.0000000000, - 1011.0000000000, - 978.0000000000, - 985.0000000000, - 999.0000000000, - 983.0000000000, - 982.0000000000, - 1015.0000000000, - 1002.0000000000, - 977.0000000000, - 948.0000000000, - 1040.0000000000, - 974.0000000000, - 996.0000000000, - 989.0000000000, - 1015.0000000000, - 994.0000000000, - 1024.0000000000]; + let val = &[ + 991.0000000000, + 1018.0000000000, + 998.0000000000, + 1013.0000000000, + 974.0000000000, + 1007.0000000000, + 1014.0000000000, + 999.0000000000, + 1011.0000000000, + 978.0000000000, + 985.0000000000, + 999.0000000000, + 983.0000000000, + 982.0000000000, + 1015.0000000000, + 1002.0000000000, + 977.0000000000, + 948.0000000000, + 1040.0000000000, + 974.0000000000, + 996.0000000000, + 989.0000000000, + 1015.0000000000, + 994.0000000000, + 1024.0000000000, + ]; let summ = &Summary { sum: 24926.0000000000, min: 948.0000000000, @@ -527,16 +535,18 @@ mod tests { } #[test] fn test_exp10a() { - let val = &[23.0000000000, - 11.0000000000, - 2.0000000000, - 57.0000000000, - 4.0000000000, - 12.0000000000, - 5.0000000000, - 29.0000000000, - 3.0000000000, - 21.0000000000]; + let val = &[ + 23.0000000000, + 11.0000000000, + 2.0000000000, + 57.0000000000, + 4.0000000000, + 12.0000000000, + 5.0000000000, + 29.0000000000, + 3.0000000000, + 21.0000000000, + ]; let summ = &Summary { sum: 167.0000000000, min: 2.0000000000, @@ -555,16 +565,18 @@ mod tests { } #[test] fn test_exp10b() { - let val = &[24.0000000000, - 17.0000000000, - 6.0000000000, - 38.0000000000, - 25.0000000000, - 7.0000000000, - 51.0000000000, - 2.0000000000, - 61.0000000000, - 32.0000000000]; + let val = &[ + 24.0000000000, + 17.0000000000, + 6.0000000000, + 38.0000000000, + 25.0000000000, + 7.0000000000, + 51.0000000000, + 2.0000000000, + 61.0000000000, + 32.0000000000, + ]; let summ = &Summary { sum: 263.0000000000, min: 2.0000000000, @@ -583,16 +595,18 @@ mod tests { } #[test] fn test_exp10c() { - let val = &[71.0000000000, - 2.0000000000, - 32.0000000000, - 1.0000000000, - 6.0000000000, - 28.0000000000, - 13.0000000000, - 37.0000000000, - 16.0000000000, - 36.0000000000]; + let val = &[ + 71.0000000000, + 2.0000000000, + 32.0000000000, + 1.0000000000, + 6.0000000000, + 28.0000000000, + 13.0000000000, + 37.0000000000, + 16.0000000000, + 36.0000000000, + ]; let summ = &Summary { sum: 242.0000000000, min: 1.0000000000, @@ -611,31 +625,33 @@ mod tests { } #[test] fn test_exp25() { - let val = &[3.0000000000, - 24.0000000000, - 1.0000000000, - 19.0000000000, - 7.0000000000, - 5.0000000000, - 30.0000000000, - 39.0000000000, - 31.0000000000, - 13.0000000000, - 25.0000000000, - 48.0000000000, - 1.0000000000, - 6.0000000000, - 42.0000000000, - 63.0000000000, - 2.0000000000, - 12.0000000000, - 108.0000000000, - 26.0000000000, - 1.0000000000, - 7.0000000000, - 44.0000000000, - 25.0000000000, - 11.0000000000]; + let val = &[ + 3.0000000000, + 24.0000000000, + 1.0000000000, + 19.0000000000, + 7.0000000000, + 5.0000000000, + 30.0000000000, + 39.0000000000, + 31.0000000000, + 13.0000000000, + 25.0000000000, + 48.0000000000, + 1.0000000000, + 6.0000000000, + 42.0000000000, + 63.0000000000, + 2.0000000000, + 12.0000000000, + 108.0000000000, + 26.0000000000, + 1.0000000000, + 7.0000000000, + 44.0000000000, + 25.0000000000, + 11.0000000000, + ]; let summ = &Summary { sum: 593.0000000000, min: 1.0000000000, @@ -654,31 +670,33 @@ mod tests { } #[test] fn test_binom25() { - let val = &[18.0000000000, - 17.0000000000, - 27.0000000000, - 15.0000000000, - 21.0000000000, - 25.0000000000, - 17.0000000000, - 24.0000000000, - 25.0000000000, - 24.0000000000, - 26.0000000000, - 26.0000000000, - 23.0000000000, - 15.0000000000, - 23.0000000000, - 17.0000000000, - 18.0000000000, - 18.0000000000, - 21.0000000000, - 16.0000000000, - 15.0000000000, - 31.0000000000, - 20.0000000000, - 17.0000000000, - 15.0000000000]; + let val = &[ + 18.0000000000, + 17.0000000000, + 27.0000000000, + 15.0000000000, + 21.0000000000, + 25.0000000000, + 17.0000000000, + 24.0000000000, + 25.0000000000, + 24.0000000000, + 26.0000000000, + 26.0000000000, + 23.0000000000, + 15.0000000000, + 23.0000000000, + 17.0000000000, + 18.0000000000, + 18.0000000000, + 21.0000000000, + 16.0000000000, + 15.0000000000, + 31.0000000000, + 20.0000000000, + 17.0000000000, + 15.0000000000, + ]; let summ = &Summary { sum: 514.0000000000, min: 15.0000000000, @@ -697,31 +715,33 @@ mod tests { } #[test] fn test_pois25lambda30() { - let val = &[27.0000000000, - 33.0000000000, - 34.0000000000, - 34.0000000000, - 24.0000000000, - 39.0000000000, - 28.0000000000, - 27.0000000000, - 31.0000000000, - 28.0000000000, - 38.0000000000, - 21.0000000000, - 33.0000000000, - 36.0000000000, - 29.0000000000, - 37.0000000000, - 32.0000000000, - 34.0000000000, - 31.0000000000, - 39.0000000000, - 25.0000000000, - 31.0000000000, - 32.0000000000, - 40.0000000000, - 24.0000000000]; + let val = &[ + 27.0000000000, + 33.0000000000, + 34.0000000000, + 34.0000000000, + 24.0000000000, + 39.0000000000, + 28.0000000000, + 27.0000000000, + 31.0000000000, + 28.0000000000, + 38.0000000000, + 21.0000000000, + 33.0000000000, + 36.0000000000, + 29.0000000000, + 37.0000000000, + 32.0000000000, + 34.0000000000, + 31.0000000000, + 39.0000000000, + 25.0000000000, + 31.0000000000, + 32.0000000000, + 40.0000000000, + 24.0000000000, + ]; let summ = &Summary { sum: 787.0000000000, min: 21.0000000000, @@ -740,31 +760,33 @@ mod tests { } #[test] fn test_pois25lambda40() { - let val = &[42.0000000000, - 50.0000000000, - 42.0000000000, - 46.0000000000, - 34.0000000000, - 45.0000000000, - 34.0000000000, - 49.0000000000, - 39.0000000000, - 28.0000000000, - 40.0000000000, - 35.0000000000, - 37.0000000000, - 39.0000000000, - 46.0000000000, - 44.0000000000, - 32.0000000000, - 45.0000000000, - 42.0000000000, - 37.0000000000, - 48.0000000000, - 42.0000000000, - 33.0000000000, - 42.0000000000, - 48.0000000000]; + let val = &[ + 42.0000000000, + 50.0000000000, + 42.0000000000, + 46.0000000000, + 34.0000000000, + 45.0000000000, + 34.0000000000, + 49.0000000000, + 39.0000000000, + 28.0000000000, + 40.0000000000, + 35.0000000000, + 37.0000000000, + 39.0000000000, + 46.0000000000, + 44.0000000000, + 32.0000000000, + 45.0000000000, + 42.0000000000, + 37.0000000000, + 48.0000000000, + 42.0000000000, + 33.0000000000, + 42.0000000000, + 48.0000000000, + ]; let summ = &Summary { sum: 1019.0000000000, min: 28.0000000000, @@ -783,31 +805,33 @@ mod tests { } #[test] fn test_pois25lambda50() { - let val = &[45.0000000000, - 43.0000000000, - 44.0000000000, - 61.0000000000, - 51.0000000000, - 53.0000000000, - 59.0000000000, - 52.0000000000, - 49.0000000000, - 51.0000000000, - 51.0000000000, - 50.0000000000, - 49.0000000000, - 56.0000000000, - 42.0000000000, - 52.0000000000, - 51.0000000000, - 43.0000000000, - 48.0000000000, - 48.0000000000, - 50.0000000000, - 42.0000000000, - 43.0000000000, - 42.0000000000, - 60.0000000000]; + let val = &[ + 45.0000000000, + 43.0000000000, + 44.0000000000, + 61.0000000000, + 51.0000000000, + 53.0000000000, + 59.0000000000, + 52.0000000000, + 49.0000000000, + 51.0000000000, + 51.0000000000, + 50.0000000000, + 49.0000000000, + 56.0000000000, + 42.0000000000, + 52.0000000000, + 51.0000000000, + 43.0000000000, + 48.0000000000, + 48.0000000000, + 50.0000000000, + 42.0000000000, + 43.0000000000, + 42.0000000000, + 60.0000000000, + ]; let summ = &Summary { sum: 1235.0000000000, min: 42.0000000000, @@ -826,31 +850,33 @@ mod tests { } #[test] fn test_unif25() { - let val = &[99.0000000000, - 55.0000000000, - 92.0000000000, - 79.0000000000, - 14.0000000000, - 2.0000000000, - 33.0000000000, - 49.0000000000, - 3.0000000000, - 32.0000000000, - 84.0000000000, - 59.0000000000, - 22.0000000000, - 86.0000000000, - 76.0000000000, - 31.0000000000, - 29.0000000000, - 11.0000000000, - 41.0000000000, - 53.0000000000, - 45.0000000000, - 44.0000000000, - 98.0000000000, - 98.0000000000, - 7.0000000000]; + let val = &[ + 99.0000000000, + 55.0000000000, + 92.0000000000, + 79.0000000000, + 14.0000000000, + 2.0000000000, + 33.0000000000, + 49.0000000000, + 3.0000000000, + 32.0000000000, + 84.0000000000, + 59.0000000000, + 22.0000000000, + 86.0000000000, + 76.0000000000, + 31.0000000000, + 29.0000000000, + 11.0000000000, + 41.0000000000, + 53.0000000000, + 45.0000000000, + 44.0000000000, + 98.0000000000, + 98.0000000000, + 7.0000000000, + ]; let summ = &Summary { sum: 1242.0000000000, min: 2.0000000000, @@ -885,18 +911,14 @@ mod bench { #[bench] pub fn sum_three_items(b: &mut Bencher) { - b.iter(|| { - [1e20f64, 1.5f64, -1e20f64].sum(); - }) + b.iter(|| { [1e20f64, 1.5f64, -1e20f64].sum(); }) } #[bench] pub fn sum_many_f64(b: &mut Bencher) { let nums = [-1e30f64, 1e60, 1e30, 1.0, -1e60]; let v = (0..500).map(|i| nums[i % 5]).collect::>(); - b.iter(|| { - v.sum(); - }) + b.iter(|| { v.sum(); }) } #[bench] From adddb0f41ae70be84c5e3bc30fbe7349462d22b9 Mon Sep 17 00:00:00 2001 From: Gilad Naaman Date: Tue, 19 Dec 2017 14:44:18 +0200 Subject: [PATCH 06/10] libtest: Added UI tests for --format=json libtest: Remove usage of jq libtest: Fixed UI tests - Now comparing to the right file. - A python script checks for validity of JSON documents --- src/test/run-make/libtest-json/Makefile | 14 +++++++++ src/test/run-make/libtest-json/f.rs | 31 +++++++++++++++++++ src/test/run-make/libtest-json/output.json | 10 ++++++ .../run-make/libtest-json/validate_json.py | 8 +++++ 4 files changed, 63 insertions(+) create mode 100644 src/test/run-make/libtest-json/Makefile create mode 100644 src/test/run-make/libtest-json/f.rs create mode 100644 src/test/run-make/libtest-json/output.json create mode 100755 src/test/run-make/libtest-json/validate_json.py diff --git a/src/test/run-make/libtest-json/Makefile b/src/test/run-make/libtest-json/Makefile new file mode 100644 index 0000000000000..9a62bc2666e9e --- /dev/null +++ b/src/test/run-make/libtest-json/Makefile @@ -0,0 +1,14 @@ +-include ../tools.mk + +# Test expected libtest's JSON output + +OUTPUT_FILE := $(TMPDIR)/libtest-json-output.json + +all: + $(RUSTC) --test f.rs + $(call RUN,f) -Z unstable-options --test-threads=1 --format=json > $(OUTPUT_FILE) || true + + cat $(OUTPUT_FILE) | $(PYTHON) validate_json.py + + # Compare to output file + diff output.json $(OUTPUT_FILE) diff --git a/src/test/run-make/libtest-json/f.rs b/src/test/run-make/libtest-json/f.rs new file mode 100644 index 0000000000000..ce44bfd9107ae --- /dev/null +++ b/src/test/run-make/libtest-json/f.rs @@ -0,0 +1,31 @@ +// Copyright 2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#[test] +fn a() { + // Should pass +} + +#[test] +fn b() { + assert!(false) +} + +#[test] +#[should_panic] +fn c() { + assert!(false); +} + +#[test] +#[ignore] +fn d() { + assert!(false); +} \ No newline at end of file diff --git a/src/test/run-make/libtest-json/output.json b/src/test/run-make/libtest-json/output.json new file mode 100644 index 0000000000000..dbec70e4a7ad3 --- /dev/null +++ b/src/test/run-make/libtest-json/output.json @@ -0,0 +1,10 @@ +{ "type": "suite", "event": "started", "test_count": "4" } +{ "type": "test", "event": "started", "name": "a" } +{ "type": "test", "name": "a", "event": "ok" } +{ "type": "test", "event": "started", "name": "b" } +{ "type": "test", "name": "b", "event": "failed", "stdout": "thread 'b' panicked at 'assertion failed: false', f.rs:18:4\nnote: Run with `RUST_BACKTRACE=1` for a backtrace.\n" } +{ "type": "test", "event": "started", "name": "c" } +{ "type": "test", "name": "c", "event": "ok" } +{ "type": "test", "event": "started", "name": "d" } +{ "type": "test", "name": "d", "event": "ignored" } +{ "type": "suite", "event": "failed", "passed": 2, "failed": 1, "allowed_fail": 0, "ignored": 1, "measured": 0, "filtered_out": "0" } diff --git a/src/test/run-make/libtest-json/validate_json.py b/src/test/run-make/libtest-json/validate_json.py new file mode 100755 index 0000000000000..657f732f2bffd --- /dev/null +++ b/src/test/run-make/libtest-json/validate_json.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python + +import sys +import json + +# Try to decode line in order to ensure it is a valid JSON document +for line in sys.stdin: + json.loads(line) From 360b26389c90716650d20fedf329c1a5332bbdbb Mon Sep 17 00:00:00 2001 From: Gilad Naaman Date: Tue, 19 Dec 2017 23:12:24 +0200 Subject: [PATCH 07/10] libtest: Split-up formatters.rs into smaller modules libtest: Split HumanFormatter into {Pretty,Terse} libtest: Fixed padding of benchmarks when not benchmarking libtest: Fixed benchmarks' names not showing in terse-mode libtest: Formatting --- src/libtest/formatters.rs | 499 ------------------ src/libtest/formatters/json.rs | 229 ++++++++ src/libtest/formatters/mod.rs | 32 ++ src/libtest/formatters/pretty.rs | 247 +++++++++ src/libtest/formatters/terse.rs | 246 +++++++++ src/libtest/lib.rs | 46 +- src/test/run-make/libtest-json/f.rs | 3 +- src/test/run-make/libtest-json/output.json | 2 +- .../run-make/libtest-json/validate_json.py | 10 + 9 files changed, 789 insertions(+), 525 deletions(-) delete mode 100644 src/libtest/formatters.rs create mode 100644 src/libtest/formatters/json.rs create mode 100644 src/libtest/formatters/mod.rs create mode 100644 src/libtest/formatters/pretty.rs create mode 100644 src/libtest/formatters/terse.rs diff --git a/src/libtest/formatters.rs b/src/libtest/formatters.rs deleted file mode 100644 index 59228146e6be4..0000000000000 --- a/src/libtest/formatters.rs +++ /dev/null @@ -1,499 +0,0 @@ -// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -use super::*; - -pub(crate) trait OutputFormatter { - fn write_run_start(&mut self, test_count: usize) -> io::Result<()>; - fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()>; - fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()>; - fn write_result( - &mut self, - desc: &TestDesc, - result: &TestResult, - stdout: &[u8], - ) -> io::Result<()>; - fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result; -} - -pub(crate) struct HumanFormatter { - out: OutputLocation, - terse: bool, - use_color: bool, - test_count: usize, - - /// Number of columns to fill when aligning names - max_name_len: usize, - - is_multithreaded: bool, -} - -impl HumanFormatter { - pub fn new( - out: OutputLocation, - use_color: bool, - terse: bool, - max_name_len: usize, - is_multithreaded: bool, - ) -> Self { - HumanFormatter { - out, - terse, - use_color, - test_count: 0, - max_name_len, - is_multithreaded, - } - } - - #[cfg(test)] - pub fn output_location(&self) -> &OutputLocation { - &self.out - } - - pub fn write_ok(&mut self) -> io::Result<()> { - self.write_short_result("ok", ".", term::color::GREEN) - } - - pub fn write_failed(&mut self) -> io::Result<()> { - self.write_short_result("FAILED", "F", term::color::RED) - } - - pub fn write_ignored(&mut self) -> io::Result<()> { - self.write_short_result("ignored", "i", term::color::YELLOW) - } - - pub fn write_allowed_fail(&mut self) -> io::Result<()> { - self.write_short_result("FAILED (allowed)", "a", term::color::YELLOW) - } - - pub fn write_bench(&mut self) -> io::Result<()> { - self.write_pretty("bench", term::color::CYAN) - } - - pub fn write_short_result( - &mut self, - verbose: &str, - quiet: &str, - color: term::color::Color, - ) -> io::Result<()> { - if self.terse { - self.write_pretty(quiet, color)?; - if self.test_count % QUIET_MODE_MAX_COLUMN == QUIET_MODE_MAX_COLUMN - 1 { - // we insert a new line every 100 dots in order to flush the - // screen when dealing with line-buffered output (e.g. piping to - // `stamp` in the rust CI). - self.write_plain("\n")?; - } - - self.test_count += 1; - Ok(()) - } else { - self.write_pretty(verbose, color)?; - self.write_plain("\n") - } - } - - pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> { - match self.out { - Pretty(ref mut term) => { - if self.use_color { - term.fg(color)?; - } - term.write_all(word.as_bytes())?; - if self.use_color { - term.reset()?; - } - term.flush() - } - Raw(ref mut stdout) => { - stdout.write_all(word.as_bytes())?; - stdout.flush() - } - } - } - - pub fn write_plain>(&mut self, s: S) -> io::Result<()> { - let s = s.as_ref(); - self.out.write_all(s.as_bytes())?; - self.out.flush() - } - - pub fn write_outputs(&mut self, state: &ConsoleTestState) -> io::Result<()> { - self.write_plain("\nsuccesses:\n")?; - let mut successes = Vec::new(); - let mut stdouts = String::new(); - for &(ref f, ref stdout) in &state.not_failures { - successes.push(f.name.to_string()); - if !stdout.is_empty() { - stdouts.push_str(&format!("---- {} stdout ----\n\t", f.name)); - let output = String::from_utf8_lossy(stdout); - stdouts.push_str(&output); - stdouts.push_str("\n"); - } - } - if !stdouts.is_empty() { - self.write_plain("\n")?; - self.write_plain(&stdouts)?; - } - - self.write_plain("\nsuccesses:\n")?; - successes.sort(); - for name in &successes { - self.write_plain(&format!(" {}\n", name))?; - } - Ok(()) - } - - pub fn write_failures(&mut self, state: &ConsoleTestState) -> io::Result<()> { - self.write_plain("\nfailures:\n")?; - let mut failures = Vec::new(); - let mut fail_out = String::new(); - for &(ref f, ref stdout) in &state.failures { - failures.push(f.name.to_string()); - if !stdout.is_empty() { - fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name)); - let output = String::from_utf8_lossy(stdout); - fail_out.push_str(&output); - fail_out.push_str("\n"); - } - } - if !fail_out.is_empty() { - self.write_plain("\n")?; - self.write_plain(&fail_out)?; - } - - self.write_plain("\nfailures:\n")?; - failures.sort(); - for name in &failures { - self.write_plain(&format!(" {}\n", name))?; - } - Ok(()) - } - - fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> { - if !(self.terse && desc.name.padding() != PadOnRight) { - let name = desc.padded_name(self.max_name_len, desc.name.padding()); - self.write_plain(&format!("test {} ... ", name))?; - } - - Ok(()) - } -} - -impl OutputFormatter for HumanFormatter { - fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { - let noun = if test_count != 1 { "tests" } else { "test" }; - self.write_plain(&format!("\nrunning {} {}\n", test_count, noun)) - } - - fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { - // When running tests concurrently, we should not print - // the test's name as the result will be mis-aligned. - // When running the tests serially, we print the name here so - // that the user can see which test hangs. - if !self.is_multithreaded { - self.write_test_name(desc)?; - } - - Ok(()) - } - - fn write_result(&mut self, desc: &TestDesc, result: &TestResult, _: &[u8]) -> io::Result<()> { - if self.is_multithreaded { - self.write_test_name(desc)?; - } - - match *result { - TrOk => self.write_ok(), - TrFailed | TrFailedMsg(_) => self.write_failed(), - TrIgnored => self.write_ignored(), - TrAllowedFail => self.write_allowed_fail(), - TrBench(ref bs) => { - self.write_bench()?; - self.write_plain(&format!(": {}\n", fmt_bench_samples(bs))) - } - } - } - - fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { - if self.is_multithreaded { - self.write_test_name(desc)?; - } - - self.write_plain(&format!( - "test {} has been running for over {} seconds\n", - desc.name, - TEST_WARN_TIMEOUT_S - )) - } - - fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { - if state.options.display_output { - self.write_outputs(state)?; - } - let success = state.failed == 0; - if !success { - self.write_failures(state)?; - } - - self.write_plain("\ntest result: ")?; - - if success { - // There's no parallelism at this point so it's safe to use color - self.write_pretty("ok", term::color::GREEN)?; - } else { - self.write_pretty("FAILED", term::color::RED)?; - } - - let s = if state.allowed_fail > 0 { - format!( - ". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out\n\n", - state.passed, - state.failed + state.allowed_fail, - state.allowed_fail, - state.ignored, - state.measured, - state.filtered_out - ) - } else { - format!( - ". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n", - state.passed, - state.failed, - state.ignored, - state.measured, - state.filtered_out - ) - }; - - self.write_plain(&s)?; - - Ok(success) - } -} - -pub(crate) struct JsonFormatter { - out: OutputLocation, -} - -impl JsonFormatter { - pub fn new(out: OutputLocation) -> Self { - Self { out } - } - - fn write_message(&mut self, s: &str) -> io::Result<()> { - assert!(!s.contains('\n')); - - self.out.write_all(s.as_ref())?; - self.out.write_all(b"\n") - } - - fn write_event( - &mut self, - ty: &str, - name: &str, - evt: &str, - extra: Option, - ) -> io::Result<()> { - if let Some(extras) = extra { - self.write_message(&*format!( - r#"{{ "type": "{}", "name": "{}", "event": "{}", {} }}"#, - ty, - name, - evt, - extras - )) - } else { - self.write_message(&*format!( - r#"{{ "type": "{}", "name": "{}", "event": "{}" }}"#, - ty, - name, - evt - )) - } - } -} - -impl OutputFormatter for JsonFormatter { - fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { - self.write_message(&*format!( - r#"{{ "type": "suite", "event": "started", "test_count": "{}" }}"#, - test_count - )) - } - - fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { - self.write_message(&*format!( - r#"{{ "type": "test", "event": "started", "name": "{}" }}"#, - desc.name - )) - } - - fn write_result( - &mut self, - desc: &TestDesc, - result: &TestResult, - stdout: &[u8], - ) -> io::Result<()> { - match *result { - TrOk => self.write_event("test", desc.name.as_slice(), "ok", None), - - TrFailed => { - let extra_data = if stdout.len() > 0 { - Some(format!( - r#""stdout": "{}""#, - EscapedString(String::from_utf8_lossy(stdout)) - )) - } else { - None - }; - - self.write_event("test", desc.name.as_slice(), "failed", extra_data) - } - - TrFailedMsg(ref m) => { - self.write_event( - "test", - desc.name.as_slice(), - "failed", - Some(format!(r#""message": "{}""#, EscapedString(m))), - ) - } - - TrIgnored => self.write_event("test", desc.name.as_slice(), "ignored", None), - - TrAllowedFail => { - self.write_event("test", desc.name.as_slice(), "allowed_failure", None) - } - - TrBench(ref bs) => { - let median = bs.ns_iter_summ.median as usize; - let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize; - - let mbps = if bs.mb_s == 0 { - "".into() - } else { - format!(r#", "mib_per_second": {}"#, bs.mb_s) - }; - - let line = format!( - "{{ \"type\": \"bench\", \ - \"name\": \"{}\", \ - \"median\": {}, \ - \"deviation\": {}{} }}", - desc.name, - median, - deviation, - mbps - ); - - self.write_message(&*line) - } - } - } - - fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { - self.write_message(&*format!( - r#"{{ "type": "test", "event": "timeout", "name": "{}" }}"#, - desc.name - )) - } - - fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { - - self.write_message(&*format!( - "{{ \"type\": \"suite\", \ - \"event\": \"{}\", \ - \"passed\": {}, \ - \"failed\": {}, \ - \"allowed_fail\": {}, \ - \"ignored\": {}, \ - \"measured\": {}, \ - \"filtered_out\": \"{}\" }}", - if state.failed == 0 { "ok" } else { "failed" }, - state.passed, - state.failed + state.allowed_fail, - state.allowed_fail, - state.ignored, - state.measured, - state.filtered_out - ))?; - - Ok(state.failed == 0) - } -} - -/// A formatting utility used to print strings with characters in need of escaping. -/// Base code taken form `libserialize::json::escape_str` -struct EscapedString>(S); - -impl> ::std::fmt::Display for EscapedString { - fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { - let mut start = 0; - - for (i, byte) in self.0.as_ref().bytes().enumerate() { - let escaped = match byte { - b'"' => "\\\"", - b'\\' => "\\\\", - b'\x00' => "\\u0000", - b'\x01' => "\\u0001", - b'\x02' => "\\u0002", - b'\x03' => "\\u0003", - b'\x04' => "\\u0004", - b'\x05' => "\\u0005", - b'\x06' => "\\u0006", - b'\x07' => "\\u0007", - b'\x08' => "\\b", - b'\t' => "\\t", - b'\n' => "\\n", - b'\x0b' => "\\u000b", - b'\x0c' => "\\f", - b'\r' => "\\r", - b'\x0e' => "\\u000e", - b'\x0f' => "\\u000f", - b'\x10' => "\\u0010", - b'\x11' => "\\u0011", - b'\x12' => "\\u0012", - b'\x13' => "\\u0013", - b'\x14' => "\\u0014", - b'\x15' => "\\u0015", - b'\x16' => "\\u0016", - b'\x17' => "\\u0017", - b'\x18' => "\\u0018", - b'\x19' => "\\u0019", - b'\x1a' => "\\u001a", - b'\x1b' => "\\u001b", - b'\x1c' => "\\u001c", - b'\x1d' => "\\u001d", - b'\x1e' => "\\u001e", - b'\x1f' => "\\u001f", - b'\x7f' => "\\u007f", - _ => { - continue; - } - }; - - if start < i { - f.write_str(&self.0.as_ref()[start..i])?; - } - - f.write_str(escaped)?; - - start = i + 1; - } - - if start != self.0.as_ref().len() { - f.write_str(&self.0.as_ref()[start..])?; - } - - Ok(()) - } -} diff --git a/src/libtest/formatters/json.rs b/src/libtest/formatters/json.rs new file mode 100644 index 0000000000000..d323d50f702ba --- /dev/null +++ b/src/libtest/formatters/json.rs @@ -0,0 +1,229 @@ +// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use super::*; + +pub(crate) struct JsonFormatter { + out: OutputLocation, +} + +impl JsonFormatter { + pub fn new(out: OutputLocation) -> Self { + Self { out } + } + + fn write_message(&mut self, s: &str) -> io::Result<()> { + assert!(!s.contains('\n')); + + self.out.write_all(s.as_ref())?; + self.out.write_all(b"\n") + } + + fn write_event( + &mut self, + ty: &str, + name: &str, + evt: &str, + extra: Option, + ) -> io::Result<()> { + if let Some(extras) = extra { + self.write_message(&*format!( + r#"{{ "type": "{}", "name": "{}", "event": "{}", {} }}"#, + ty, + name, + evt, + extras + )) + } else { + self.write_message(&*format!( + r#"{{ "type": "{}", "name": "{}", "event": "{}" }}"#, + ty, + name, + evt + )) + } + } +} + +impl OutputFormatter for JsonFormatter { + fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { + self.write_message(&*format!( + r#"{{ "type": "suite", "event": "started", "test_count": "{}" }}"#, + test_count + )) + } + + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { + self.write_message(&*format!( + r#"{{ "type": "test", "event": "started", "name": "{}" }}"#, + desc.name + )) + } + + fn write_result( + &mut self, + desc: &TestDesc, + result: &TestResult, + stdout: &[u8], + ) -> io::Result<()> { + match *result { + TrOk => self.write_event("test", desc.name.as_slice(), "ok", None), + + TrFailed => { + let extra_data = if stdout.len() > 0 { + Some(format!( + r#""stdout": "{}""#, + EscapedString(String::from_utf8_lossy(stdout)) + )) + } else { + None + }; + + self.write_event("test", desc.name.as_slice(), "failed", extra_data) + } + + TrFailedMsg(ref m) => { + self.write_event( + "test", + desc.name.as_slice(), + "failed", + Some(format!(r#""message": "{}""#, EscapedString(m))), + ) + } + + TrIgnored => self.write_event("test", desc.name.as_slice(), "ignored", None), + + TrAllowedFail => { + self.write_event("test", desc.name.as_slice(), "allowed_failure", None) + } + + TrBench(ref bs) => { + let median = bs.ns_iter_summ.median as usize; + let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize; + + let mbps = if bs.mb_s == 0 { + "".into() + } else { + format!(r#", "mib_per_second": {}"#, bs.mb_s) + }; + + let line = format!( + "{{ \"type\": \"bench\", \ + \"name\": \"{}\", \ + \"median\": {}, \ + \"deviation\": {}{} }}", + desc.name, + median, + deviation, + mbps + ); + + self.write_message(&*line) + } + } + } + + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { + self.write_message(&*format!( + r#"{{ "type": "test", "event": "timeout", "name": "{}" }}"#, + desc.name + )) + } + + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { + + self.write_message(&*format!( + "{{ \"type\": \"suite\", \ + \"event\": \"{}\", \ + \"passed\": {}, \ + \"failed\": {}, \ + \"allowed_fail\": {}, \ + \"ignored\": {}, \ + \"measured\": {}, \ + \"filtered_out\": \"{}\" }}", + if state.failed == 0 { "ok" } else { "failed" }, + state.passed, + state.failed + state.allowed_fail, + state.allowed_fail, + state.ignored, + state.measured, + state.filtered_out + ))?; + + Ok(state.failed == 0) + } +} + +/// A formatting utility used to print strings with characters in need of escaping. +/// Base code taken form `libserialize::json::escape_str` +struct EscapedString>(S); + +impl> ::std::fmt::Display for EscapedString { + fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { + let mut start = 0; + + for (i, byte) in self.0.as_ref().bytes().enumerate() { + let escaped = match byte { + b'"' => "\\\"", + b'\\' => "\\\\", + b'\x00' => "\\u0000", + b'\x01' => "\\u0001", + b'\x02' => "\\u0002", + b'\x03' => "\\u0003", + b'\x04' => "\\u0004", + b'\x05' => "\\u0005", + b'\x06' => "\\u0006", + b'\x07' => "\\u0007", + b'\x08' => "\\b", + b'\t' => "\\t", + b'\n' => "\\n", + b'\x0b' => "\\u000b", + b'\x0c' => "\\f", + b'\r' => "\\r", + b'\x0e' => "\\u000e", + b'\x0f' => "\\u000f", + b'\x10' => "\\u0010", + b'\x11' => "\\u0011", + b'\x12' => "\\u0012", + b'\x13' => "\\u0013", + b'\x14' => "\\u0014", + b'\x15' => "\\u0015", + b'\x16' => "\\u0016", + b'\x17' => "\\u0017", + b'\x18' => "\\u0018", + b'\x19' => "\\u0019", + b'\x1a' => "\\u001a", + b'\x1b' => "\\u001b", + b'\x1c' => "\\u001c", + b'\x1d' => "\\u001d", + b'\x1e' => "\\u001e", + b'\x1f' => "\\u001f", + b'\x7f' => "\\u007f", + _ => { + continue; + } + }; + + if start < i { + f.write_str(&self.0.as_ref()[start..i])?; + } + + f.write_str(escaped)?; + + start = i + 1; + } + + if start != self.0.as_ref().len() { + f.write_str(&self.0.as_ref()[start..])?; + } + + Ok(()) + } +} diff --git a/src/libtest/formatters/mod.rs b/src/libtest/formatters/mod.rs new file mode 100644 index 0000000000000..24c7929076c1d --- /dev/null +++ b/src/libtest/formatters/mod.rs @@ -0,0 +1,32 @@ +// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use super::*; + +mod pretty; +mod json; +mod terse; + +pub(crate) use self::pretty::PrettyFormatter; +pub(crate) use self::json::JsonFormatter; +pub(crate) use self::terse::TerseFormatter; + +pub(crate) trait OutputFormatter { + fn write_run_start(&mut self, test_count: usize) -> io::Result<()>; + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()>; + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()>; + fn write_result( + &mut self, + desc: &TestDesc, + result: &TestResult, + stdout: &[u8], + ) -> io::Result<()>; + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result; +} diff --git a/src/libtest/formatters/pretty.rs b/src/libtest/formatters/pretty.rs new file mode 100644 index 0000000000000..f2064deefce62 --- /dev/null +++ b/src/libtest/formatters/pretty.rs @@ -0,0 +1,247 @@ +// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use super::*; + +pub(crate) struct PrettyFormatter { + out: OutputLocation, + use_color: bool, + + /// Number of columns to fill when aligning names + max_name_len: usize, + + is_multithreaded: bool, +} + +impl PrettyFormatter { + pub fn new( + out: OutputLocation, + use_color: bool, + max_name_len: usize, + is_multithreaded: bool, + ) -> Self { + PrettyFormatter { + out, + use_color, + max_name_len, + is_multithreaded, + } + } + + #[cfg(test)] + pub fn output_location(&self) -> &OutputLocation { + &self.out + } + + pub fn write_ok(&mut self) -> io::Result<()> { + self.write_short_result("ok", term::color::GREEN) + } + + pub fn write_failed(&mut self) -> io::Result<()> { + self.write_short_result("FAILED", term::color::RED) + } + + pub fn write_ignored(&mut self) -> io::Result<()> { + self.write_short_result("ignored", term::color::YELLOW) + } + + pub fn write_allowed_fail(&mut self) -> io::Result<()> { + self.write_short_result("FAILED (allowed)", term::color::YELLOW) + } + + pub fn write_bench(&mut self) -> io::Result<()> { + self.write_pretty("bench", term::color::CYAN) + } + + pub fn write_short_result( + &mut self, + result: &str, + color: term::color::Color, + ) -> io::Result<()> { + self.write_pretty(result, color)?; + self.write_plain("\n") + } + + pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> { + match self.out { + Pretty(ref mut term) => { + if self.use_color { + term.fg(color)?; + } + term.write_all(word.as_bytes())?; + if self.use_color { + term.reset()?; + } + term.flush() + } + Raw(ref mut stdout) => { + stdout.write_all(word.as_bytes())?; + stdout.flush() + } + } + } + + pub fn write_plain>(&mut self, s: S) -> io::Result<()> { + let s = s.as_ref(); + self.out.write_all(s.as_bytes())?; + self.out.flush() + } + + pub fn write_successes(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_plain("\nsuccesses:\n")?; + let mut successes = Vec::new(); + let mut stdouts = String::new(); + for &(ref f, ref stdout) in &state.not_failures { + successes.push(f.name.to_string()); + if !stdout.is_empty() { + stdouts.push_str(&format!("---- {} stdout ----\n\t", f.name)); + let output = String::from_utf8_lossy(stdout); + stdouts.push_str(&output); + stdouts.push_str("\n"); + } + } + if !stdouts.is_empty() { + self.write_plain("\n")?; + self.write_plain(&stdouts)?; + } + + self.write_plain("\nsuccesses:\n")?; + successes.sort(); + for name in &successes { + self.write_plain(&format!(" {}\n", name))?; + } + Ok(()) + } + + pub fn write_failures(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_plain("\nfailures:\n")?; + let mut failures = Vec::new(); + let mut fail_out = String::new(); + for &(ref f, ref stdout) in &state.failures { + failures.push(f.name.to_string()); + if !stdout.is_empty() { + fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name)); + let output = String::from_utf8_lossy(stdout); + fail_out.push_str(&output); + fail_out.push_str("\n"); + } + } + if !fail_out.is_empty() { + self.write_plain("\n")?; + self.write_plain(&fail_out)?; + } + + self.write_plain("\nfailures:\n")?; + failures.sort(); + for name in &failures { + self.write_plain(&format!(" {}\n", name))?; + } + Ok(()) + } + + fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> { + let name = desc.padded_name(self.max_name_len, desc.name.padding()); + self.write_plain(&format!("test {} ... ", name))?; + + Ok(()) + } +} + +impl OutputFormatter for PrettyFormatter { + fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { + let noun = if test_count != 1 { "tests" } else { "test" }; + self.write_plain(&format!("\nrunning {} {}\n", test_count, noun)) + } + + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { + // When running tests concurrently, we should not print + // the test's name as the result will be mis-aligned. + // When running the tests serially, we print the name here so + // that the user can see which test hangs. + if !self.is_multithreaded { + self.write_test_name(desc)?; + } + + Ok(()) + } + + fn write_result(&mut self, desc: &TestDesc, result: &TestResult, _: &[u8]) -> io::Result<()> { + if self.is_multithreaded { + self.write_test_name(desc)?; + } + + match *result { + TrOk => self.write_ok(), + TrFailed | TrFailedMsg(_) => self.write_failed(), + TrIgnored => self.write_ignored(), + TrAllowedFail => self.write_allowed_fail(), + TrBench(ref bs) => { + self.write_bench()?; + self.write_plain(&format!(": {}\n", fmt_bench_samples(bs))) + } + } + } + + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { + if self.is_multithreaded { + self.write_test_name(desc)?; + } + + self.write_plain(&format!( + "test {} has been running for over {} seconds\n", + desc.name, + TEST_WARN_TIMEOUT_S + )) + } + + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { + if state.options.display_output { + self.write_successes(state)?; + } + let success = state.failed == 0; + if !success { + self.write_failures(state)?; + } + + self.write_plain("\ntest result: ")?; + + if success { + // There's no parallelism at this point so it's safe to use color + self.write_pretty("ok", term::color::GREEN)?; + } else { + self.write_pretty("FAILED", term::color::RED)?; + } + + let s = if state.allowed_fail > 0 { + format!( + ". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out\n\n", + state.passed, + state.failed + state.allowed_fail, + state.allowed_fail, + state.ignored, + state.measured, + state.filtered_out + ) + } else { + format!( + ". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n", + state.passed, + state.failed, + state.ignored, + state.measured, + state.filtered_out + ) + }; + + self.write_plain(&s)?; + + Ok(success) + } +} diff --git a/src/libtest/formatters/terse.rs b/src/libtest/formatters/terse.rs new file mode 100644 index 0000000000000..88689485144c0 --- /dev/null +++ b/src/libtest/formatters/terse.rs @@ -0,0 +1,246 @@ +// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use super::*; + +pub(crate) struct TerseFormatter { + out: OutputLocation, + use_color: bool, + is_multithreaded: bool, + /// Number of columns to fill when aligning names + max_name_len: usize, + + test_count: usize, +} + +impl TerseFormatter { + pub fn new( + out: OutputLocation, + use_color: bool, + max_name_len: usize, + is_multithreaded: bool, + ) -> Self { + TerseFormatter { + out, + use_color, + max_name_len, + is_multithreaded, + test_count: 0, + } + } + + pub fn write_ok(&mut self) -> io::Result<()> { + self.write_short_result(".", term::color::GREEN) + } + + pub fn write_failed(&mut self) -> io::Result<()> { + self.write_short_result("F", term::color::RED) + } + + pub fn write_ignored(&mut self) -> io::Result<()> { + self.write_short_result("i", term::color::YELLOW) + } + + pub fn write_allowed_fail(&mut self) -> io::Result<()> { + self.write_short_result("a", term::color::YELLOW) + } + + pub fn write_bench(&mut self) -> io::Result<()> { + self.write_pretty("bench", term::color::CYAN) + } + + pub fn write_short_result( + &mut self, + result: &str, + color: term::color::Color, + ) -> io::Result<()> { + self.write_pretty(result, color)?; + if self.test_count % QUIET_MODE_MAX_COLUMN == QUIET_MODE_MAX_COLUMN - 1 { + // we insert a new line every 100 dots in order to flush the + // screen when dealing with line-buffered output (e.g. piping to + // `stamp` in the rust CI). + self.write_plain("\n")?; + } + + self.test_count += 1; + Ok(()) + } + + pub fn write_pretty(&mut self, word: &str, color: term::color::Color) -> io::Result<()> { + match self.out { + Pretty(ref mut term) => { + if self.use_color { + term.fg(color)?; + } + term.write_all(word.as_bytes())?; + if self.use_color { + term.reset()?; + } + term.flush() + } + Raw(ref mut stdout) => { + stdout.write_all(word.as_bytes())?; + stdout.flush() + } + } + } + + pub fn write_plain>(&mut self, s: S) -> io::Result<()> { + let s = s.as_ref(); + self.out.write_all(s.as_bytes())?; + self.out.flush() + } + + pub fn write_outputs(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_plain("\nsuccesses:\n")?; + let mut successes = Vec::new(); + let mut stdouts = String::new(); + for &(ref f, ref stdout) in &state.not_failures { + successes.push(f.name.to_string()); + if !stdout.is_empty() { + stdouts.push_str(&format!("---- {} stdout ----\n\t", f.name)); + let output = String::from_utf8_lossy(stdout); + stdouts.push_str(&output); + stdouts.push_str("\n"); + } + } + if !stdouts.is_empty() { + self.write_plain("\n")?; + self.write_plain(&stdouts)?; + } + + self.write_plain("\nsuccesses:\n")?; + successes.sort(); + for name in &successes { + self.write_plain(&format!(" {}\n", name))?; + } + Ok(()) + } + + pub fn write_failures(&mut self, state: &ConsoleTestState) -> io::Result<()> { + self.write_plain("\nfailures:\n")?; + let mut failures = Vec::new(); + let mut fail_out = String::new(); + for &(ref f, ref stdout) in &state.failures { + failures.push(f.name.to_string()); + if !stdout.is_empty() { + fail_out.push_str(&format!("---- {} stdout ----\n\t", f.name)); + let output = String::from_utf8_lossy(stdout); + fail_out.push_str(&output); + fail_out.push_str("\n"); + } + } + if !fail_out.is_empty() { + self.write_plain("\n")?; + self.write_plain(&fail_out)?; + } + + self.write_plain("\nfailures:\n")?; + failures.sort(); + for name in &failures { + self.write_plain(&format!(" {}\n", name))?; + } + Ok(()) + } + + fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> { + let name = desc.padded_name(self.max_name_len, desc.name.padding()); + self.write_plain(&format!("test {} ... ", name))?; + + Ok(()) + } +} + +impl OutputFormatter for TerseFormatter { + fn write_run_start(&mut self, test_count: usize) -> io::Result<()> { + let noun = if test_count != 1 { "tests" } else { "test" }; + self.write_plain(&format!("\nrunning {} {}\n", test_count, noun)) + } + + fn write_test_start(&mut self, desc: &TestDesc) -> io::Result<()> { + // Remnants from old libtest code that used the padding value + // in order to indicate benchmarks. + // When running benchmarks, terse-mode should still print their name as if + // it is the Pretty formatter. + if !self.is_multithreaded && desc.name.padding() == PadOnRight { + self.write_test_name(desc)?; + } + + Ok(()) + } + + fn write_result(&mut self, desc: &TestDesc, result: &TestResult, _: &[u8]) -> io::Result<()> { + match *result { + TrOk => self.write_ok(), + TrFailed | TrFailedMsg(_) => self.write_failed(), + TrIgnored => self.write_ignored(), + TrAllowedFail => self.write_allowed_fail(), + TrBench(ref bs) => { + if self.is_multithreaded { + self.write_test_name(desc)?; + } + self.write_bench()?; + self.write_plain(&format!(": {}\n", fmt_bench_samples(bs))) + } + } + } + + fn write_timeout(&mut self, desc: &TestDesc) -> io::Result<()> { + self.write_plain(&format!( + "test {} has been running for over {} seconds\n", + desc.name, + TEST_WARN_TIMEOUT_S + )) + } + + fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result { + if state.options.display_output { + self.write_outputs(state)?; + } + let success = state.failed == 0; + if !success { + self.write_failures(state)?; + } + + self.write_plain("\ntest result: ")?; + + if success { + // There's no parallelism at this point so it's safe to use color + self.write_pretty("ok", term::color::GREEN)?; + } else { + self.write_pretty("FAILED", term::color::RED)?; + } + + let s = if state.allowed_fail > 0 { + format!( + ". {} passed; {} failed ({} allowed); {} ignored; {} measured; {} filtered out\n\n", + state.passed, + state.failed + state.allowed_fail, + state.allowed_fail, + state.ignored, + state.measured, + state.filtered_out + ) + } else { + format!( + ". {} passed; {} failed; {} ignored; {} measured; {} filtered out\n\n", + state.passed, + state.failed, + state.ignored, + state.measured, + state.filtered_out + ) + }; + + self.write_plain(&s)?; + + Ok(success) + } +} diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index 4da89f5ab4b3c..d69a9f493f055 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT +// Copyright 2012-2017 The Rust Project Developers. See the COPYRIGHT // file at the top-level directory of this distribution and at // http://rust-lang.org/COPYRIGHT. // @@ -87,7 +87,7 @@ pub mod test { pub mod stats; mod formatters; -use formatters::*; +use formatters::{OutputFormatter, PrettyFormatter, TerseFormatter, JsonFormatter}; // The name of a test. By convention this follows the rules for rust // paths; i.e. it should be a series of identifiers separated by double @@ -475,7 +475,7 @@ Test Attributes: ); } -// FIXME: Copied from libsyntax until linkage errors are resolved. +// FIXME: Copied from libsyntax until linkage errors are resolved. Issue #47566 fn is_nightly() -> bool { // Whether this is a feature-staged build, i.e. on the beta or stable channel let disable_unstable_features = option_env!("CFG_DISABLE_UNSTABLE_FEATURES").is_some(); @@ -769,13 +769,12 @@ pub fn fmt_bench_samples(bs: &BenchSamples) -> String { // List the tests to console, and optionally to logfile. Filters are honored. pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Result<()> { - let output = match term::stdout() { + let mut output = match term::stdout() { None => Raw(io::stdout()), Some(t) => Pretty(t), }; let quiet = opts.format == OutputFormat::Terse; - let mut out = HumanFormatter::new(output, use_color(opts), quiet, 0, false); let mut st = ConsoleTestState::new(opts)?; let mut ntest = 0; @@ -801,7 +800,7 @@ pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Res } }; - out.write_plain(format!("{}: {}\n", name, fntype))?; + writeln!(output, "{}: {}", name, fntype)?; st.write_log(format!("{} {}\n", fntype, name))?; } @@ -814,13 +813,14 @@ pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Res if !quiet { if ntest != 0 || nbench != 0 { - out.write_plain("\n")?; + writeln!(output, "")?; } - out.write_plain(format!( - "{}, {}\n", + + writeln!(output, + "{}, {}", plural(ntest, "test"), plural(nbench, "benchmark") - ))?; + )?; } Ok(()) @@ -828,15 +828,6 @@ pub fn list_tests_console(opts: &TestOpts, tests: Vec) -> io::Res // A simple console test runner pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Result { - let tests = { - let mut tests = tests; - for test in tests.iter_mut() { - test.desc.name = test.desc.name.with_padding(test.testfn.padding()); - } - - tests - }; - fn callback( event: &TestEvent, st: &mut ConsoleTestState, @@ -902,17 +893,15 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu }; let mut out: Box = match opts.format { - OutputFormat::Pretty => Box::new(HumanFormatter::new( + OutputFormat::Pretty => Box::new(PrettyFormatter::new( output, use_color(opts), - false, max_name_len, is_multithreaded, )), - OutputFormat::Terse => Box::new(HumanFormatter::new( + OutputFormat::Terse => Box::new(TerseFormatter::new( output, use_color(opts), - true, max_name_len, is_multithreaded, )), @@ -949,7 +938,7 @@ fn should_sort_failures_before_printing_them() { allow_fail: false, }; - let mut out = HumanFormatter::new(Raw(Vec::new()), false, false, 10, false); + let mut out = PrettyFormatter::new(Raw(Vec::new()), false, 10, false); let st = ConsoleTestState { log_out: None, @@ -1040,6 +1029,15 @@ where filtered_tests = convert_benchmarks_to_tests(filtered_tests); } + let filtered_tests = { + let mut filtered_tests = filtered_tests; + for test in filtered_tests.iter_mut() { + test.desc.name = test.desc.name.with_padding(test.testfn.padding()); + } + + filtered_tests + }; + let filtered_out = tests_len - filtered_tests.len(); callback(TeFilteredOut(filtered_out))?; diff --git a/src/test/run-make/libtest-json/f.rs b/src/test/run-make/libtest-json/f.rs index ce44bfd9107ae..5cff1f1a5b1af 100644 --- a/src/test/run-make/libtest-json/f.rs +++ b/src/test/run-make/libtest-json/f.rs @@ -28,4 +28,5 @@ fn c() { #[ignore] fn d() { assert!(false); -} \ No newline at end of file +} + diff --git a/src/test/run-make/libtest-json/output.json b/src/test/run-make/libtest-json/output.json index dbec70e4a7ad3..235f8cd7c7257 100644 --- a/src/test/run-make/libtest-json/output.json +++ b/src/test/run-make/libtest-json/output.json @@ -2,7 +2,7 @@ { "type": "test", "event": "started", "name": "a" } { "type": "test", "name": "a", "event": "ok" } { "type": "test", "event": "started", "name": "b" } -{ "type": "test", "name": "b", "event": "failed", "stdout": "thread 'b' panicked at 'assertion failed: false', f.rs:18:4\nnote: Run with `RUST_BACKTRACE=1` for a backtrace.\n" } +{ "type": "test", "name": "b", "event": "failed", "stdout": "thread 'b' panicked at 'assertion failed: false', f.rs:18:5\nnote: Run with `RUST_BACKTRACE=1` for a backtrace.\n" } { "type": "test", "event": "started", "name": "c" } { "type": "test", "name": "c", "event": "ok" } { "type": "test", "event": "started", "name": "d" } diff --git a/src/test/run-make/libtest-json/validate_json.py b/src/test/run-make/libtest-json/validate_json.py index 657f732f2bffd..1e97639b524e8 100755 --- a/src/test/run-make/libtest-json/validate_json.py +++ b/src/test/run-make/libtest-json/validate_json.py @@ -1,5 +1,15 @@ #!/usr/bin/env python +# Copyright 2016 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + import sys import json From cb86f38497db255efbfc018685ee92b1d24acc77 Mon Sep 17 00:00:00 2001 From: Gilad Naaman Date: Mon, 22 Jan 2018 22:29:41 +0200 Subject: [PATCH 08/10] libtest: Failing benchmarks no longer crash the harness. --- src/libtest/lib.rs | 128 ++++++++++++++++++++++++++++++++------------- 1 file changed, 91 insertions(+), 37 deletions(-) diff --git a/src/libtest/lib.rs b/src/libtest/lib.rs index d69a9f493f055..ffa27688cf1a7 100644 --- a/src/libtest/lib.rs +++ b/src/libtest/lib.rs @@ -887,10 +887,7 @@ pub fn run_tests_console(opts: &TestOpts, tests: Vec) -> io::Resu .map(|t| t.desc.name.as_slice().len()) .unwrap_or(0); - let is_multithreaded = match opts.test_threads { - Some(n) => n > 1, - None => get_concurrency() > 1, - }; + let is_multithreaded = opts.test_threads.unwrap_or_else(get_concurrency) > 1; let mut out: Box = match opts.format { OutputFormat::Pretty => Box::new(PrettyFormatter::new( @@ -1014,6 +1011,15 @@ pub enum TestEvent { pub type MonitorMsg = (TestDesc, TestResult, Vec); +struct Sink(Arc>>); +impl Write for Sink { + fn write(&mut self, data: &[u8]) -> io::Result { + Write::write(&mut *self.0.lock().unwrap(), data) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} pub fn run_tests(opts: &TestOpts, tests: Vec, mut callback: F) -> io::Result<()> where @@ -1051,10 +1057,7 @@ where _ => false, }); - let concurrency = match opts.test_threads { - Some(n) => n, - None => get_concurrency(), - }; + let concurrency = opts.test_threads.unwrap_or_else(get_concurrency); let mut remaining = filtered_tests; remaining.reverse(); @@ -1384,16 +1387,6 @@ pub fn run_test( monitor_ch: Sender, nocapture: bool, testfn: Box) { - struct Sink(Arc>>); - impl Write for Sink { - fn write(&mut self, data: &[u8]) -> io::Result { - Write::write(&mut *self.0.lock().unwrap(), data) - } - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } - } - // Buffer for capturing standard I/O let data = Arc::new(Mutex::new(Vec::new())); let data2 = data.clone(); @@ -1438,14 +1431,16 @@ pub fn run_test( match testfn { DynBenchFn(bencher) => { - let bs = ::bench::benchmark(|harness| bencher.run(harness)); - monitor_ch.send((desc, TrBench(bs), Vec::new())).unwrap(); - return; + ::bench::benchmark(desc, + monitor_ch, + opts.nocapture, + |harness| bencher.run(harness)); } StaticBenchFn(benchfn) => { - let bs = ::bench::benchmark(|harness| (benchfn.clone())(harness)); - monitor_ch.send((desc, TrBench(bs), Vec::new())).unwrap(); - return; + ::bench::benchmark(desc, + monitor_ch, + opts.nocapture, + |harness| (benchfn.clone())(harness)); } DynTestFn(f) => { let cb = move || { @@ -1453,9 +1448,10 @@ pub fn run_test( }; run_test_inner(desc, monitor_ch, opts.nocapture, Box::new(cb)) } - StaticTestFn(f) => + StaticTestFn(f) => { run_test_inner(desc, monitor_ch, opts.nocapture, - Box::new(move || __rust_begin_short_backtrace(f))), + Box::new(move || __rust_begin_short_backtrace(f))) + } } } @@ -1655,11 +1651,14 @@ where } pub mod bench { + use std::panic::{catch_unwind, AssertUnwindSafe}; use std::cmp; + use std::io; + use std::sync::{Arc, Mutex}; use stats; - use super::{Bencher, BenchSamples, BenchMode}; + use super::{Bencher, BenchSamples, BenchMode, Sink, MonitorMsg, TestDesc, Sender, TestResult}; - pub fn benchmark(f: F) -> BenchSamples + pub fn benchmark(desc: TestDesc, monitor_ch: Sender, nocapture: bool, f: F) where F: FnMut(&mut Bencher), { @@ -1669,26 +1668,53 @@ pub mod bench { bytes: 0, }; - return match bs.bench(f) { - Some(ns_iter_summ) => { + let data = Arc::new(Mutex::new(Vec::new())); + let data2 = data.clone(); + + let oldio = if !nocapture { + Some(( + io::set_print(Some(Box::new(Sink(data2.clone())))), + io::set_panic(Some(Box::new(Sink(data2)))), + )) + } else { + None + }; + + let result = catch_unwind(AssertUnwindSafe(|| bs.bench(f))); + + if let Some((printio, panicio)) = oldio { + io::set_print(printio); + io::set_panic(panicio); + }; + + let test_result = match result { //bs.bench(f) { + Ok(Some(ns_iter_summ)) => { let ns_iter = cmp::max(ns_iter_summ.median as u64, 1); let mb_s = bs.bytes * 1000 / ns_iter; - BenchSamples { + let bs = BenchSamples { ns_iter_summ, mb_s: mb_s as usize, - } + }; + TestResult::TrBench(bs) } - None => { + Ok(None) => { // iter not called, so no data. // FIXME: error in this case? let samples: &mut [f64] = &mut [0.0_f64; 1]; - BenchSamples { + let bs = BenchSamples { ns_iter_summ: stats::Summary::new(samples), mb_s: 0, - } + }; + TestResult::TrBench(bs) + } + Err(_) => { + TestResult::TrFailed } }; + + let stdout = data.lock().unwrap().to_vec(); + monitor_ch.send((desc, test_result, stdout)).unwrap(); } pub fn run_once(f: F) @@ -2067,7 +2093,21 @@ mod tests { #[test] pub fn test_bench_no_iter() { fn f(_: &mut Bencher) {} - bench::benchmark(f); + + let (tx, rx) = channel(); + + let desc = TestDesc { + name: StaticTestName("f"), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + }; + + ::bench::benchmark(desc, + tx, + true, + f); + rx.recv().unwrap(); } #[test] @@ -2075,6 +2115,20 @@ mod tests { fn f(b: &mut Bencher) { b.iter(|| {}) } - bench::benchmark(f); + + let (tx, rx) = channel(); + + let desc = TestDesc { + name: StaticTestName("f"), + ignore: false, + should_panic: ShouldPanic::No, + allow_fail: false, + }; + + ::bench::benchmark(desc, + tx, + true, + f); + rx.recv().unwrap(); } } From 100ead353aecbaee2b710912e363e80184378690 Mon Sep 17 00:00:00 2001 From: Gilad Naaman Date: Fri, 26 Jan 2018 19:57:02 +0200 Subject: [PATCH 09/10] Updated Cargo commit hash --- src/tools/cargo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/cargo b/src/tools/cargo index 91e36aa86c703..1d6dfea44f971 160000 --- a/src/tools/cargo +++ b/src/tools/cargo @@ -1 +1 @@ -Subproject commit 91e36aa86c7037de50642f2fec1cf47c3d18af02 +Subproject commit 1d6dfea44f97199d5d5c177c7dadcde393eaff9a From 8b7f1d0cec3884e07e2dd9ea4d0d4faeda5b95ed Mon Sep 17 00:00:00 2001 From: Gilad Naaman Date: Sat, 27 Jan 2018 11:50:01 +0200 Subject: [PATCH 10/10] libtest: Fixed call to python in run-make --- src/test/run-make/libtest-json/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/run-make/libtest-json/Makefile b/src/test/run-make/libtest-json/Makefile index 9a62bc2666e9e..ec91ddfb9f917 100644 --- a/src/test/run-make/libtest-json/Makefile +++ b/src/test/run-make/libtest-json/Makefile @@ -8,7 +8,7 @@ all: $(RUSTC) --test f.rs $(call RUN,f) -Z unstable-options --test-threads=1 --format=json > $(OUTPUT_FILE) || true - cat $(OUTPUT_FILE) | $(PYTHON) validate_json.py + cat $(OUTPUT_FILE) | "$(PYTHON)" validate_json.py # Compare to output file diff output.json $(OUTPUT_FILE)