Initial world space UI support (#304)

* implement worldspace ui

Signed-off-by: Schmarni <marnistromer@gmail.com>

* clean render to texture logic from EguiNode

Signed-off-by: Schmarni <marnistromer@gmail.com>

* fix warnings

Signed-off-by: Schmarni <marnistromer@gmail.com>

* require render feature for rtt example

Signed-off-by: Schmarni <marnistromer@gmail.com>

* Implement paint callbacks for rendering to a texture

* Fix compilation of not(feature = render) flag

---------

Signed-off-by: Schmarni <marnistromer@gmail.com>
Co-authored-by: Schmarni <marnistromer@gmail.com>
Co-authored-by: Schmarni <51007916+Schmarni-Dev@users.noreply.github.com>

Cargo.toml

@@ -40,6 +40,9 @@ required-features = ["render"]
 [[example]]
 name = "ui"
 required-features = ["render"]
+[[example]]
+name = "render_egui_to_texture"
+required-features = ["render"]
 
 [dependencies]
 bevy = { version = "0.14.0", default-features = false, features = [
@@ -48,6 +51,7 @@ bevy = { version = "0.14.0", default-features = false, features = [
 egui = { version = "0.28", default-features = false, features = ["bytemuck"] }
 bytemuck = "1"
 webbrowser = { version = "1.0.1", optional = true }
+wgpu-types = "0.20"
 
 [target.'cfg(not(any(target_arch = "wasm32", target_os = "android")))'.dependencies]
 arboard = { version = "3.2.0", optional = true }

examples/paint_callback.rs

@@ -13,14 +13,19 @@ use bevy::{
 };
 use bevy_egui::{
     egui_node::{EguiBevyPaintCallback, EguiBevyPaintCallbackImpl, EguiPipelineKey},
-    EguiContexts, EguiPlugin,
+    EguiContexts, EguiPlugin, EguiRenderToTextureHandle,
 };
 use std::path::Path;
+use wgpu_types::{Extent3d, TextureUsages};
 
 fn main() {
     App::new()
         .add_plugins((DefaultPlugins, EguiPlugin, CustomPipelinePlugin))
-        .add_systems(Update, ui_example_system)
+        .add_systems(Startup, setup_worldspace)
+        .add_systems(
+            Update,
+            (ui_example_system, ui_render_to_texture_example_system),
+        )
         .run();
 }
 
@@ -170,3 +175,64 @@ fn ui_example_system(mut ctx: EguiContexts) {
         });
     }
 }
+
+// The following systems are used to render UI in world space to demonstrate that paint callbacks
+// work for them as well (they aren't needed to set up pain callbacks for regular screen-space UI,
+// so feel free to skip them):
+
+fn setup_worldspace(
+    mut images: ResMut<Assets<Image>>,
+    mut meshes: ResMut<Assets<Mesh>>,
+    mut materials: ResMut<Assets<StandardMaterial>>,
+    mut commands: Commands,
+) {
+    let output_texture = images.add({
+        let size = Extent3d {
+            width: 256,
+            height: 256,
+            depth_or_array_layers: 1,
+        };
+        let mut output_texture = Image {
+            // You should use `0` so that the pixels are transparent.
+            data: vec![0; (size.width * size.height * 4) as usize],
+            ..default()
+        };
+        output_texture.texture_descriptor.usage |= TextureUsages::RENDER_ATTACHMENT;
+        output_texture.texture_descriptor.size = size;
+        output_texture
+    });
+
+    commands.spawn(PbrBundle {
+        mesh: meshes.add(Cuboid::new(1.0, 1.0, 1.0).mesh()),
+        material: materials.add(StandardMaterial {
+            base_color: Color::WHITE,
+            base_color_texture: Some(Handle::clone(&output_texture)),
+            alpha_mode: AlphaMode::Blend,
+            // Remove this if you want it to use the world's lighting.
+            unlit: true,
+            ..default()
+        }),
+        ..default()
+    });
+    commands.spawn(EguiRenderToTextureHandle(output_texture));
+    commands.spawn(Camera3dBundle {
+        transform: Transform::from_xyz(1.5, 1.5, 1.5).looking_at(Vec3::new(0., 0., 0.), Vec3::Y),
+        ..default()
+    });
+}
+
+fn ui_render_to_texture_example_system(
+    mut contexts: Query<&mut bevy_egui::EguiContext, With<EguiRenderToTextureHandle>>,
+) {
+    for mut ctx in contexts.iter_mut() {
+        egui::Window::new("Worldspace UI").show(ctx.get_mut(), |ui| {
+            let (resp, painter) =
+                ui.allocate_painter(egui::Vec2 { x: 200., y: 200. }, egui::Sense::hover());
+
+            painter.add(EguiBevyPaintCallback::new_paint_callback(
+                resp.rect,
+                CustomPaintCallback,
+            ));
+        });
+    }
+}

examples/render_egui_to_texture.rs

@@ -0,0 +1,69 @@
+use bevy::prelude::*;
+use bevy_egui::{EguiContexts, EguiPlugin, EguiRenderToTextureHandle};
+use wgpu_types::{Extent3d, TextureUsages};
+
+fn main() {
+    let mut app = App::new();
+    app.add_plugins(DefaultPlugins);
+    app.add_plugins(EguiPlugin);
+    app.add_systems(Startup, setup_worldspace);
+    app.add_systems(Update, (update_screenspace, update_worldspace));
+    app.run();
+}
+
+fn update_screenspace(mut contexts: EguiContexts) {
+    egui::Window::new("Screenspace UI").show(contexts.ctx_mut(), |ui| {
+        ui.label("I'm rendering to screenspace!");
+    });
+}
+
+fn update_worldspace(
+    mut contexts: Query<&mut bevy_egui::EguiContext, With<EguiRenderToTextureHandle>>,
+) {
+    for mut ctx in contexts.iter_mut() {
+        egui::Window::new("Worldspace UI").show(ctx.get_mut(), |ui| {
+            ui.label("I'm rendering to a texture in worldspace!");
+        });
+    }
+}
+
+fn setup_worldspace(
+    mut images: ResMut<Assets<Image>>,
+    mut meshes: ResMut<Assets<Mesh>>,
+    mut materials: ResMut<Assets<StandardMaterial>>,
+    mut commands: Commands,
+) {
+    let output_texture = images.add({
+        let size = Extent3d {
+            width: 256,
+            height: 256,
+            depth_or_array_layers: 1,
+        };
+        let mut output_texture = Image {
+            // You should use `0` so that the pixels are transparent.
+            data: vec![0; (size.width * size.height * 4) as usize],
+            ..default()
+        };
+        output_texture.texture_descriptor.usage |= TextureUsages::RENDER_ATTACHMENT;
+        output_texture.texture_descriptor.size = size;
+        output_texture
+    });
+
+    commands.spawn(PbrBundle {
+        mesh: meshes.add(Cuboid::new(1.0, 1.0, 1.0).mesh()),
+        material: materials.add(StandardMaterial {
+            base_color: Color::WHITE,
+            base_color_texture: Some(Handle::clone(&output_texture)),
+            alpha_mode: AlphaMode::Blend,
+            // Remove this if you want it to use the world's lighting.
+            unlit: true,
+            ..default()
+        }),
+        ..default()
+    });
+    commands.spawn(EguiRenderToTextureHandle(output_texture));
+    commands.spawn(Camera3dBundle {
+        transform: Transform::from_xyz(1.5, 1.5, 1.5).looking_at(Vec3::new(0., 0., 0.), Vec3::Y),
+        ..default()
+    });
+}

src/egui_node.rs

@@ -2,7 +2,7 @@ use crate::{
     render_systems::{
         EguiPipelines, EguiTextureBindGroups, EguiTextureId, EguiTransform, EguiTransforms,
     },
-    EguiRenderOutput, EguiSettings, WindowSize,
+    EguiRenderOutput, EguiSettings, RenderTargetSize,
 };
 use bevy::{
     ecs::world::{FromWorld, World},
@@ -22,7 +22,10 @@ use bevy::{
             VertexBufferLayout, VertexFormat, VertexState, VertexStepMode,
         },
         renderer::{RenderContext, RenderDevice, RenderQueue},
-        texture::{Image, ImageAddressMode, ImageFilterMode, ImageSampler, ImageSamplerDescriptor},
+        texture::{
+            GpuImage, Image, ImageAddressMode, ImageFilterMode, ImageSampler,
+            ImageSamplerDescriptor,
+        },
         view::{ExtractedWindow, ExtractedWindows},
     },
 };
@@ -96,12 +99,19 @@ pub struct EguiPipelineKey {
 }
 
 impl EguiPipelineKey {
-    /// Extracts target texture format in egui renderpass
+    /// Constructs a pipeline key from a window.
     pub fn from_extracted_window(window: &ExtractedWindow) -> Option<Self> {
         Some(Self {
             texture_format: window.swap_chain_texture_format?.add_srgb_suffix(),
         })
     }
+
+    /// Constructs a pipeline key from a gpu image.
+    pub fn from_gpu_image(image: &GpuImage) -> Self {
+        EguiPipelineKey {
+            texture_format: image.texture_format.add_srgb_suffix(),
+        }
+    }
 }
 
 impl SpecializedRenderPipeline for EguiPipeline {
@@ -160,25 +170,24 @@ impl SpecializedRenderPipeline for EguiPipeline {
     }
 }
 
-struct DrawCommand {
-    clip_rect: egui::Rect,
-    primitive: DrawPrimitive,
+pub(crate) struct DrawCommand {
+    pub(crate) clip_rect: egui::Rect,
+    pub(crate) primitive: DrawPrimitive,
 }
 
-enum DrawPrimitive {
+pub(crate) enum DrawPrimitive {
     Egui(EguiDraw),
     PaintCallback(PaintCallbackDraw),
 }
 
-struct PaintCallbackDraw {
-    callback: std::sync::Arc<EguiBevyPaintCallback>,
-    rect: egui::Rect,
+pub(crate) struct PaintCallbackDraw {
+    pub(crate) callback: std::sync::Arc<EguiBevyPaintCallback>,
+    pub(crate) rect: egui::Rect,
 }
 
-#[derive(Debug)]
-struct EguiDraw {
-    vertices_count: usize,
-    egui_texture: EguiTextureId,
+pub(crate) struct EguiDraw {
+    pub(crate) vertices_count: usize,
+    pub(crate) egui_texture: EguiTextureId,
 }
 
 /// Egui render node.
@@ -223,9 +232,10 @@ impl Node for EguiNode {
             return;
         };
 
-        let mut window_sizes = world.query::<(&WindowSize, &mut EguiRenderOutput)>();
+        let mut render_target_size = world.query::<(&RenderTargetSize, &mut EguiRenderOutput)>();
 
-        let Ok((window_size, mut render_output)) = window_sizes.get_mut(world, self.window_entity)
+        let Ok((window_size, mut render_output)) =
+            render_target_size.get_mut(world, self.window_entity)
         else {
             return;
         };
@@ -382,21 +392,13 @@ impl Node for EguiNode {
         let pipeline_cache = world.get_resource::<PipelineCache>().unwrap();
 
         let extracted_windows = &world.get_resource::<ExtractedWindows>().unwrap().windows;
-        let extracted_window =
-            if let Some(extracted_window) = extracted_windows.get(&self.window_entity) {
-                extracted_window
-            } else {
-                return Ok(()); // No window
+        let extracted_window = extracted_windows.get(&self.window_entity);
+        let swap_chain_texture_view =
+            match extracted_window.and_then(|v| v.swap_chain_texture_view.as_ref()) {
+                None => return Ok(()),
+                Some(window) => window,
             };
 
-        let swap_chain_texture_view = if let Some(swap_chain_texture_view) =
-            extracted_window.swap_chain_texture_view.as_ref()
-        {
-            swap_chain_texture_view
-        } else {
-            return Ok(()); // No swapchain texture
-        };
-
         let render_queue = world.get_resource::<RenderQueue>().unwrap();
 
         let (vertex_buffer, index_buffer) = match (&self.vertex_buffer, &self.index_buffer) {
@@ -432,13 +434,19 @@ impl Node for EguiNode {
                 });
         let mut render_pass = TrackedRenderPass::new(device, render_pass);
 
-        let Some(key) = EguiPipelineKey::from_extracted_window(extracted_window) else {
-            return Ok(());
+        let (physical_width, physical_height, pipeline_key) = match extracted_window {
+            Some(window) => (
+                window.physical_width,
+                window.physical_height,
+                EguiPipelineKey::from_extracted_window(window),
+            ),
+            None => unreachable!(),
         };
-
-        let Some(pipeline_id) = egui_pipelines.get(&extracted_window.entity) else {
+        let Some(key) = pipeline_key else {
             return Ok(());
         };
+
+        let pipeline_id = egui_pipelines.get(&self.window_entity).unwrap();
         let Some(pipeline) = pipeline_cache.get_render_pipeline(*pipeline_id) else {
             return Ok(());
         };
@@ -454,8 +462,8 @@ impl Node for EguiNode {
                 render_pass.set_viewport(
                     0.,
                     0.,
-                    extracted_window.physical_width as f32,
-                    extracted_window.physical_height as f32,
+                    physical_width as f32,
+                    physical_height as f32,
                     0.,
                     1.,
                 );
@@ -479,11 +487,12 @@ impl Node for EguiNode {
                     y: (draw_command.clip_rect.max.y * self.pixels_per_point).round() as u32,
                 },
             };
+
             let scrissor_rect = clip_urect.intersect(bevy::math::URect::new(
                 0,
                 0,
-                extracted_window.physical_width,
-                extracted_window.physical_width,
+                physical_width,
+                physical_height,
             ));
             if scrissor_rect.is_empty() {
                 continue;
@@ -529,10 +538,7 @@ impl Node for EguiNode {
                         viewport: command.rect,
                         clip_rect: draw_command.clip_rect,
                         pixels_per_point: self.pixels_per_point,
-                        screen_size_px: [
-                            extracted_window.physical_width,
-                            extracted_window.physical_height,
-                        ],
+                        screen_size_px: [physical_width, physical_height],
                     };
 
                     let viewport = info.viewport_in_pixels();
@@ -649,7 +655,7 @@ impl EguiBevyPaintCallback {
         }
     }
 
-    fn cb(&self) -> &dyn EguiBevyPaintCallbackImpl {
+    pub(crate) fn cb(&self) -> &dyn EguiBevyPaintCallbackImpl {
         self.0.as_ref()
     }
 }