From c22ee7c2730b3827472054d7e2676d3f4403a333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Mon, 24 Apr 2023 23:41:22 +0200 Subject: [PATCH 01/10] capture screenshot at a given frame, with fixed time updates --- crates/bevy_app/src/ci_testing.rs | 4 +++ crates/bevy_app/src/lib.rs | 2 +- crates/bevy_internal/Cargo.toml | 2 +- crates/bevy_render/Cargo.toml | 1 + .../bevy_render/src/view/window/screenshot.rs | 28 +++++++++++++++++++ crates/bevy_time/Cargo.toml | 1 + crates/bevy_time/src/lib.rs | 16 ++++++++++- 7 files changed, 51 insertions(+), 3 deletions(-) diff --git a/crates/bevy_app/src/ci_testing.rs b/crates/bevy_app/src/ci_testing.rs index f662a350a4177..1f537fe119921 100644 --- a/crates/bevy_app/src/ci_testing.rs +++ b/crates/bevy_app/src/ci_testing.rs @@ -13,6 +13,10 @@ use bevy_utils::tracing::info; pub struct CiTestingConfig { /// The number of frames after which Bevy should exit. pub exit_after: Option, + /// The time in seconds to update for each frame. + pub frame_time: Option, + /// Frames at which to capture a screenshot. + pub screenshot_frames: Vec, } fn ci_testing_exit_after( diff --git a/crates/bevy_app/src/lib.rs b/crates/bevy_app/src/lib.rs index ee409da3d1532..f9ecd22569655 100644 --- a/crates/bevy_app/src/lib.rs +++ b/crates/bevy_app/src/lib.rs @@ -10,7 +10,7 @@ mod plugin_group; mod schedule_runner; #[cfg(feature = "bevy_ci_testing")] -mod ci_testing; +pub mod ci_testing; pub use app::*; pub use bevy_derive::DynamicPlugin; diff --git a/crates/bevy_internal/Cargo.toml b/crates/bevy_internal/Cargo.toml index 51fe644c92969..77c2f849439d9 100644 --- a/crates/bevy_internal/Cargo.toml +++ b/crates/bevy_internal/Cargo.toml @@ -73,7 +73,7 @@ subpixel_glyph_atlas = ["bevy_text/subpixel_glyph_atlas"] webgl = ["bevy_core_pipeline?/webgl", "bevy_pbr?/webgl", "bevy_render?/webgl"] # enable systems that allow for automated testing on CI -bevy_ci_testing = ["bevy_app/bevy_ci_testing", "bevy_render?/ci_limits"] +bevy_ci_testing = ["bevy_app/bevy_ci_testing", "bevy_time/bevy_ci_testing", "bevy_render?/bevy_ci_testing", "bevy_render?/ci_limits"] # Enable animation support, and glTF animation loading animation = ["bevy_animation", "bevy_gltf?/bevy_animation"] diff --git a/crates/bevy_render/Cargo.toml b/crates/bevy_render/Cargo.toml index e10f7f6da5841..810f12c446871 100644 --- a/crates/bevy_render/Cargo.toml +++ b/crates/bevy_render/Cargo.toml @@ -17,6 +17,7 @@ jpeg = ["image/jpeg"] bmp = ["image/bmp"] webp = ["image/webp"] dds = ["ddsfile"] +bevy_ci_testing = ["bevy_app/bevy_ci_testing"] # For ktx2 supercompression zlib = ["flate2"] diff --git a/crates/bevy_render/src/view/window/screenshot.rs b/crates/bevy_render/src/view/window/screenshot.rs index 2a67f43005a7a..1971bf019c8cd 100644 --- a/crates/bevy_render/src/view/window/screenshot.rs +++ b/crates/bevy_render/src/view/window/screenshot.rs @@ -102,7 +102,35 @@ impl Plugin for ScreenshotPlugin { if let Ok(render_app) = app.get_sub_app_mut(RenderApp) { render_app.init_resource::>(); } + + #[cfg(feature = "bevy_ci_testing")] + if app + .world + .contains_resource::() + { + app.add_systems(bevy_app::Update, ci_testing_screenshot_at); + } + } +} + +#[cfg(feature = "bevy_ci_testing")] +fn ci_testing_screenshot_at( + mut current_frame: bevy_ecs::prelude::Local, + ci_testing_config: bevy_ecs::prelude::Res, + mut screenshot_manager: ResMut, + main_window: Query>, +) { + if ci_testing_config + .screenshot_frames + .contains(&*current_frame) + { + info!("Taking a screenshot at frame {}.", *current_frame); + let path = format!("./screenshot-{}.png", *current_frame); + screenshot_manager + .save_screenshot_to_disk(main_window.single(), path) + .unwrap(); } + *current_frame += 1; } pub(crate) fn align_byte_size(value: u32) -> u32 { diff --git a/crates/bevy_time/Cargo.toml b/crates/bevy_time/Cargo.toml index 3a7219ae042ca..7d4743b032834 100644 --- a/crates/bevy_time/Cargo.toml +++ b/crates/bevy_time/Cargo.toml @@ -11,6 +11,7 @@ keywords = ["bevy"] [features] default = [] serialize = ["serde"] +bevy_ci_testing = ["bevy_app/bevy_ci_testing"] [dependencies] # bevy diff --git a/crates/bevy_time/src/lib.rs b/crates/bevy_time/src/lib.rs index ee923f7b964ce..c0ea26fce6948 100644 --- a/crates/bevy_time/src/lib.rs +++ b/crates/bevy_time/src/lib.rs @@ -47,6 +47,19 @@ impl Plugin for TimePlugin { .init_resource::() .add_systems(First, time_system.in_set(TimeSystem)) .add_systems(RunFixedUpdateLoop, run_fixed_update_schedule); + + #[cfg(feature = "bevy_ci_testing")] + if let Some(ci_testing_config) = app + .world + .get_resource::() + { + if let Some(frame_time) = ci_testing_config.frame_time { + app.world + .insert_resource(TimeUpdateStrategy::ManualDuration(Duration::from_secs_f32( + frame_time, + ))); + } + } } } @@ -107,7 +120,8 @@ fn time_system( TimeUpdateStrategy::Automatic => time.update_with_instant(new_time), TimeUpdateStrategy::ManualInstant(instant) => time.update_with_instant(*instant), TimeUpdateStrategy::ManualDuration(duration) => { - time.update_with_instant(Instant::now() + *duration); + let last_update = time.last_update().unwrap_or_else(|| time.startup()); + time.update_with_instant(last_update + *duration); } } } From 0dde5c9f62bdcfba56b5ace519a35ecb8117c663 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 25 Apr 2023 13:20:16 +0200 Subject: [PATCH 02/10] save some screenshots in CI --- .github/example-run/load_gltf.ron | 4 +++- .github/workflows/ci.yml | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/example-run/load_gltf.ron b/.github/example-run/load_gltf.ron index d170958d73bad..13f79f298c316 100644 --- a/.github/example-run/load_gltf.ron +++ b/.github/example-run/load_gltf.ron @@ -1,3 +1,5 @@ ( - exit_after: Some(300) + exit_after: Some(300), + frame_time: Some(0.03), + screenshot_frames: [100], ) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4b890ff8bf33..f585ac0f4c4a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,13 +199,21 @@ jobs: echo "running $example_name - "`date` time TRACE_CHROME=trace-$example_name.json CI_TESTING_CONFIG=$example xvfb-run cargo run --example $example_name --features "bevy_ci_testing,trace,trace_chrome" sleep 10 + mkdir screenshots-$example_name + mv screenshots*.png screenshots-$example_name done zip traces.zip trace*.json + zip screenshots.zip screenshots-* - name: save traces uses: actions/upload-artifact@v3 with: name: example-traces.zip path: traces.zip + - name: save screenshots + uses: actions/upload-artifact@v3 + with: + name: screenshots.zip + path: screenshots.zip - name: Save PR number if: ${{ failure() && github.event_name == 'pull_request' }} run: | From 289fabdd5f842b85024f7a2ca8efaec6e6fdafb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 25 Apr 2023 13:53:08 +0200 Subject: [PATCH 03/10] accept default value --- crates/bevy_app/src/ci_testing.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/bevy_app/src/ci_testing.rs b/crates/bevy_app/src/ci_testing.rs index 1f537fe119921..cc3fee7a0e515 100644 --- a/crates/bevy_app/src/ci_testing.rs +++ b/crates/bevy_app/src/ci_testing.rs @@ -16,6 +16,7 @@ pub struct CiTestingConfig { /// The time in seconds to update for each frame. pub frame_time: Option, /// Frames at which to capture a screenshot. + #[serde(default)] pub screenshot_frames: Vec, } From ccd12eff8d56c68af32f7ab48ef6c648b466a93b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 25 Apr 2023 13:53:22 +0200 Subject: [PATCH 04/10] module documentation --- crates/bevy_app/src/ci_testing.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/bevy_app/src/ci_testing.rs b/crates/bevy_app/src/ci_testing.rs index cc3fee7a0e515..3ba0dde78e307 100644 --- a/crates/bevy_app/src/ci_testing.rs +++ b/crates/bevy_app/src/ci_testing.rs @@ -1,3 +1,5 @@ +//! Utilities for testing in CI environments. + use crate::{app::AppExit, App, Update}; use serde::Deserialize; From d4f9244b7896c3e22bae4a26abacefab398b25d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 25 Apr 2023 14:01:12 +0200 Subject: [PATCH 05/10] update time doc --- crates/bevy_time/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bevy_time/src/lib.rs b/crates/bevy_time/src/lib.rs index c0ea26fce6948..704d54d860452 100644 --- a/crates/bevy_time/src/lib.rs +++ b/crates/bevy_time/src/lib.rs @@ -73,7 +73,7 @@ pub enum TimeUpdateStrategy { Automatic, // Update [`Time`] with an exact `Instant` value ManualInstant(Instant), - // Update [`Time`] with the current time + a specified `Duration` + // Update [`Time`] with the last update time + a specified `Duration` ManualDuration(Duration), } From 7302b77bcf02acb90d2cb13c46b3119c4eeb07dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 25 Apr 2023 14:01:20 +0200 Subject: [PATCH 06/10] check for file in ci --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f585ac0f4c4a1..ed3c1bbc55f4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,8 +199,10 @@ jobs: echo "running $example_name - "`date` time TRACE_CHROME=trace-$example_name.json CI_TESTING_CONFIG=$example xvfb-run cargo run --example $example_name --features "bevy_ci_testing,trace,trace_chrome" sleep 10 - mkdir screenshots-$example_name - mv screenshots*.png screenshots-$example_name + if [ -f "screenshot-*.png" ]; then + mkdir screenshots-$example_name + mv screenshots-*.png screenshots-$example_name + fi done zip traces.zip trace*.json zip screenshots.zip screenshots-* From 9b45e4210aedabe73bf9ed5a494fecd1f32487d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 25 Apr 2023 14:30:08 +0200 Subject: [PATCH 07/10] fix condition --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed3c1bbc55f4a..33e2f7567353f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,7 +199,7 @@ jobs: echo "running $example_name - "`date` time TRACE_CHROME=trace-$example_name.json CI_TESTING_CONFIG=$example xvfb-run cargo run --example $example_name --features "bevy_ci_testing,trace,trace_chrome" sleep 10 - if [ -f "screenshot-*.png" ]; then + if [ `find ./ -maxdepth 1 -name 'screenshot-*.png' -print -quit` ]; then mkdir screenshots-$example_name mv screenshots-*.png screenshots-$example_name fi From beb02b545433dee87e4f7008c535f746723dcd51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 25 Apr 2023 14:46:52 +0200 Subject: [PATCH 08/10] fix filename --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33e2f7567353f..96d5fd0a5f13c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -201,7 +201,7 @@ jobs: sleep 10 if [ `find ./ -maxdepth 1 -name 'screenshot-*.png' -print -quit` ]; then mkdir screenshots-$example_name - mv screenshots-*.png screenshots-$example_name + mv screenshot-*.png screenshots-$example_name/ fi done zip traces.zip trace*.json From 8b20a1263f50286f88edca5a328e741e67a2d173 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 25 Apr 2023 15:06:28 +0200 Subject: [PATCH 09/10] recursively zip folders --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96d5fd0a5f13c..1db57174ae857 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -205,7 +205,7 @@ jobs: fi done zip traces.zip trace*.json - zip screenshots.zip screenshots-* + zip -r screenshots.zip screenshots-* - name: save traces uses: actions/upload-artifact@v3 with: From cdb391bc502034d0b62420dc0601b8f400875b1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Tue, 25 Apr 2023 19:14:30 +0200 Subject: [PATCH 10/10] adding one more example --- .github/example-run/breakout.ron | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/example-run/breakout.ron b/.github/example-run/breakout.ron index 1d78f6a73ad80..f2036f4a4986c 100644 --- a/.github/example-run/breakout.ron +++ b/.github/example-run/breakout.ron @@ -1,3 +1,5 @@ ( - exit_after: Some(900) + exit_after: Some(900), + frame_time: Some(0.03), + screenshot_frames: [200], )