Improve Queue Phase parallelization and other small optimizations

crates/bevy_core_pipeline/src/core_2d/main_pass_2d_node.rs

@@ -79,7 +79,7 @@ impl Node for MainPass2dNode {
 
         let mut draw_functions = draw_functions.write();
         let mut tracked_pass = TrackedRenderPass::new(render_pass);
-        for item in &transparent_phase.items {
+        for item in &transparent_phase.sorted {
             let draw_function = draw_functions.get_mut(item.draw_function).unwrap();
             draw_function.draw(world, &mut tracked_pass, view_entity, item);
         }

crates/bevy_core_pipeline/src/core_3d/main_pass_3d_node.rs

@@ -100,13 +100,13 @@ impl Node for MainPass3dNode {
                 .begin_render_pass(&pass_descriptor);
             let mut draw_functions = draw_functions.write();
             let mut tracked_pass = TrackedRenderPass::new(render_pass);
-            for item in &opaque_phase.items {
+            for item in &opaque_phase.sorted {
                 let draw_function = draw_functions.get_mut(item.draw_function).unwrap();
                 draw_function.draw(world, &mut tracked_pass, view_entity, item);
             }
         }
 
-        if !alpha_mask_phase.items.is_empty() {
+        if !alpha_mask_phase.sorted.is_empty() {
             // Run the alpha mask pass, sorted front-to-back
             // NOTE: Scoped to drop the mutable borrow of render_context
             #[cfg(feature = "trace")]
@@ -136,13 +136,13 @@ impl Node for MainPass3dNode {
                 .begin_render_pass(&pass_descriptor);
             let mut draw_functions = draw_functions.write();
             let mut tracked_pass = TrackedRenderPass::new(render_pass);
-            for item in &alpha_mask_phase.items {
+            for item in &alpha_mask_phase.sorted {
                 let draw_function = draw_functions.get_mut(item.draw_function).unwrap();
                 draw_function.draw(world, &mut tracked_pass, view_entity, item);
             }
         }
 
-        if !transparent_phase.items.is_empty() {
+        if !transparent_phase.sorted.is_empty() {
             // Run the transparent pass, sorted back-to-front
             // NOTE: Scoped to drop the mutable borrow of render_context
             #[cfg(feature = "trace")]
@@ -177,7 +177,7 @@ impl Node for MainPass3dNode {
                 .begin_render_pass(&pass_descriptor);
             let mut draw_functions = draw_functions.write();
             let mut tracked_pass = TrackedRenderPass::new(render_pass);
-            for item in &transparent_phase.items {
+            for item in &transparent_phase.sorted {
                 let draw_function = draw_functions.get_mut(item.draw_function).unwrap();
                 draw_function.draw(world, &mut tracked_pass, view_entity, item);
             }

crates/bevy_pbr/Cargo.toml

@@ -28,3 +28,4 @@ bevy_window = { path = "../bevy_window", version = "0.8.0-dev" }
 bitflags = "1.2"
 # direct dependency required for derive macro
 bytemuck = { version = "1", features = ["derive"] }
+copyless = "0.1"

crates/bevy_pbr/src/material.rs

@@ -23,14 +23,15 @@ use bevy_render::{
         SetItemPipeline, TrackedRenderPass,
     },
     render_resource::{
-        BindGroup, BindGroupLayout, PipelineCache, RenderPipelineDescriptor, Shader,
+        BindGroup, BindGroupLayout, LockablePipelineCache, RenderPipelineDescriptor, Shader,
         SpecializedMeshPipeline, SpecializedMeshPipelineError, SpecializedMeshPipelines,
     },
     renderer::RenderDevice,
     view::{ExtractedView, Msaa, VisibleEntities},
     RenderApp, RenderStage,
 };
 use bevy_utils::tracing::error;
+use copyless::VecHelper;
 use std::hash::Hash;
 use std::marker::PhantomData;
 
@@ -330,21 +331,20 @@ pub fn queue_material_meshes<M: SpecializedMaterial>(
     transparent_draw_functions: Res<DrawFunctions<Transparent3d>>,
     material_pipeline: Res<MaterialPipeline<M>>,
     mut pipelines: ResMut<SpecializedMeshPipelines<MaterialPipeline<M>>>,
-    mut pipeline_cache: ResMut<PipelineCache>,
+    pipeline_cache: Res<LockablePipelineCache>,
     msaa: Res<Msaa>,
     render_meshes: Res<RenderAssets<Mesh>>,
     render_materials: Res<RenderAssets<M>>,
     material_meshes: Query<(&Handle<M>, &Handle<Mesh>, &MeshUniform)>,
-    mut views: Query<(
+    views: Query<(
         &ExtractedView,
         &VisibleEntities,
-        &mut RenderPhase<Opaque3d>,
-        &mut RenderPhase<AlphaMask3d>,
-        &mut RenderPhase<Transparent3d>,
+        &RenderPhase<Opaque3d>,
+        &RenderPhase<AlphaMask3d>,
+        &RenderPhase<Transparent3d>,
     )>,
 ) {
-    for (view, visible_entities, mut opaque_phase, mut alpha_mask_phase, mut transparent_phase) in
-        views.iter_mut()
+    for (view, visible_entities, opaque_phase, alpha_mask_phase, transparent_phase) in views.iter()
     {
         let draw_opaque_pbr = opaque_draw_functions
             .read()
@@ -363,6 +363,14 @@ pub fn queue_material_meshes<M: SpecializedMaterial>(
         let inverse_view_row_2 = inverse_view_matrix.row(2);
         let msaa_key = MeshPipelineKey::from_msaa_samples(msaa.samples);
 
+        let opaque_phase_cell = opaque_phase.get();
+        let alpha_mask_phase_cell = alpha_mask_phase.get();
+        let transparent_phase_cell = transparent_phase.get();
+
+        let mut opaque_phase_queue = opaque_phase_cell.take();
+        let mut alpha_mask_phase_queue = alpha_mask_phase_cell.take();
+        let mut transparent_phase_queue = transparent_phase_cell.take();
+
         for visible_entity in &visible_entities.entities {
             if let Ok((material_handle, mesh_handle, mesh_uniform)) =
                 material_meshes.get(*visible_entity)
@@ -380,7 +388,7 @@ pub fn queue_material_meshes<M: SpecializedMaterial>(
                         let material_key = M::key(material);
 
                         let pipeline_id = pipelines.specialize(
-                            &mut pipeline_cache,
+                            &pipeline_cache,
                             &material_pipeline,
                             MaterialPipelineKey {
                                 mesh_key,
@@ -402,7 +410,7 @@ pub fn queue_material_meshes<M: SpecializedMaterial>(
                         let mesh_z = inverse_view_row_2.dot(mesh_uniform.transform.col(3)) + bias;
                         match alpha_mode {
                             AlphaMode::Opaque => {
-                                opaque_phase.add(Opaque3d {
+                                opaque_phase_queue.alloc().init(Opaque3d {
                                     entity: *visible_entity,
                                     draw_function: draw_opaque_pbr,
                                     pipeline: pipeline_id,
@@ -414,7 +422,7 @@ pub fn queue_material_meshes<M: SpecializedMaterial>(
                                 });
                             }
                             AlphaMode::Mask(_) => {
-                                alpha_mask_phase.add(AlphaMask3d {
+                                alpha_mask_phase_queue.alloc().init(AlphaMask3d {
                                     entity: *visible_entity,
                                     draw_function: draw_alpha_mask_pbr,
                                     pipeline: pipeline_id,
@@ -426,7 +434,7 @@ pub fn queue_material_meshes<M: SpecializedMaterial>(
                                 });
                             }
                             AlphaMode::Blend => {
-                                transparent_phase.add(Transparent3d {
+                                transparent_phase_queue.alloc().init(Transparent3d {
                                     entity: *visible_entity,
                                     draw_function: draw_transparent_pbr,
                                     pipeline: pipeline_id,
@@ -442,5 +450,9 @@ pub fn queue_material_meshes<M: SpecializedMaterial>(
                 }
             }
         }
+
+        opaque_phase_cell.set(opaque_phase_queue);
+        alpha_mask_phase_cell.set(alpha_mask_phase_queue);
+        transparent_phase_cell.set(transparent_phase_queue);
     }
 }

crates/bevy_pbr/src/render/light.rs

@@ -1341,9 +1341,9 @@ pub fn queue_shadows(
     casting_meshes: Query<&Handle<Mesh>, Without<NotShadowCaster>>,
     render_meshes: Res<RenderAssets<Mesh>>,
     mut pipelines: ResMut<SpecializedMeshPipelines<ShadowPipeline>>,
-    mut pipeline_cache: ResMut<PipelineCache>,
+    pipeline_cache: Res<LockablePipelineCache>,
     view_lights: Query<&ViewLightEntities>,
-    mut view_light_shadow_phases: Query<(&LightEntity, &mut RenderPhase<Shadow>)>,
+    view_light_shadow_phases: Query<(&LightEntity, &RenderPhase<Shadow>)>,
     point_light_entities: Query<&CubemapVisibleEntities, With<ExtractedPointLight>>,
     directional_light_entities: Query<&VisibleEntities, With<ExtractedDirectionalLight>>,
 ) {
@@ -1353,8 +1353,8 @@ pub fn queue_shadows(
             .get_id::<DrawShadowMesh>()
             .unwrap();
         for view_light_entity in view_lights.lights.iter().copied() {
-            let (light_entity, mut shadow_phase) =
-                view_light_shadow_phases.get_mut(view_light_entity).unwrap();
+            let (light_entity, shadow_phase) =
+                view_light_shadow_phases.get(view_light_entity).unwrap();
             let visible_entities = match light_entity {
                 LightEntity::Directional { light_entity } => directional_light_entities
                     .get(*light_entity)
@@ -1367,37 +1367,39 @@ pub fn queue_shadows(
                     .expect("Failed to get point light visible entities")
                     .get(*face_index),
             };
-            // NOTE: Lights with shadow mapping disabled will have no visible entities
-            // so no meshes will be queued
-            for entity in visible_entities.iter().copied() {
-                if let Ok(mesh_handle) = casting_meshes.get(entity) {
-                    if let Some(mesh) = render_meshes.get(mesh_handle) {
-                        let key =
-                            ShadowPipelineKey::from_primitive_topology(mesh.primitive_topology);
-                        let pipeline_id = pipelines.specialize(
-                            &mut pipeline_cache,
-                            &shadow_pipeline,
-                            key,
-                            &mesh.layout,
-                        );
-
-                        let pipeline_id = match pipeline_id {
-                            Ok(id) => id,
-                            Err(err) => {
-                                error!("{}", err);
-                                continue;
-                            }
-                        };
-
-                        shadow_phase.add(Shadow {
-                            draw_function: draw_shadow_mesh,
-                            pipeline: pipeline_id,
-                            entity,
-                            distance: 0.0, // TODO: sort back-to-front
-                        });
+            shadow_phase.phase_scope(|mut phase| {
+                // NOTE: Lights with shadow mapping disabled will have no visible entities
+                // so no meshes will be queued
+                for entity in visible_entities.iter().copied() {
+                    if let Ok(mesh_handle) = casting_meshes.get(entity) {
+                        if let Some(mesh) = render_meshes.get(mesh_handle) {
+                            let key =
+                                ShadowPipelineKey::from_primitive_topology(mesh.primitive_topology);
+                            let pipeline_id = pipelines.specialize(
+                                &pipeline_cache,
+                                &shadow_pipeline,
+                                key,
+                                &mesh.layout,
+                            );
+
+                            let pipeline_id = match pipeline_id {
+                                Ok(id) => id,
+                                Err(err) => {
+                                    error!("{}", err);
+                                    continue;
+                                }
+                            };
+
+                            phase.add(Shadow {
+                                draw_function: draw_shadow_mesh,
+                                pipeline: pipeline_id,
+                                entity,
+                                distance: 0.0, // TODO: sort back-to-front
+                            });
+                        }
                     }
                 }
-            }
+            });
         }
     }
 }
@@ -1476,7 +1478,7 @@ impl Node for ShadowPassNode {
                     .get_manual(world, view_light_entity)
                     .unwrap();
 
-                if shadow_phase.items.is_empty() {
+                if shadow_phase.sorted.is_empty() {
                     continue;
                 }
 
@@ -1499,7 +1501,7 @@ impl Node for ShadowPassNode {
                     .begin_render_pass(&pass_descriptor);
                 let mut draw_functions = draw_functions.write();
                 let mut tracked_pass = TrackedRenderPass::new(render_pass);
-                for item in &shadow_phase.items {
+                for item in &shadow_phase.sorted {
                     let draw_function = draw_functions.get_mut(item.draw_function).unwrap();
                     draw_function.draw(world, &mut tracked_pass, view_light_entity, item);
                 }

crates/bevy_pbr/src/wireframe.rs

@@ -12,8 +12,8 @@ use bevy_render::{
     render_asset::RenderAssets,
     render_phase::{AddRenderCommand, DrawFunctions, RenderPhase, SetItemPipeline},
     render_resource::{
-        PipelineCache, PolygonMode, RenderPipelineDescriptor, Shader, SpecializedMeshPipeline,
-        SpecializedMeshPipelineError, SpecializedMeshPipelines,
+        LockablePipelineCache, PolygonMode, RenderPipelineDescriptor, Shader,
+        SpecializedMeshPipeline, SpecializedMeshPipelineError, SpecializedMeshPipelines,
     },
     view::{ExtractedView, Msaa, VisibleEntities},
     RenderApp, RenderStage,
@@ -103,65 +103,67 @@ fn queue_wireframes(
     wireframe_config: Res<WireframeConfig>,
     wireframe_pipeline: Res<WireframePipeline>,
     mut pipelines: ResMut<SpecializedMeshPipelines<WireframePipeline>>,
-    mut pipeline_cache: ResMut<PipelineCache>,
+    pipeline_cache: Res<LockablePipelineCache>,
     msaa: Res<Msaa>,
     mut material_meshes: ParamSet<(
         Query<(Entity, &Handle<Mesh>, &MeshUniform)>,
         Query<(Entity, &Handle<Mesh>, &MeshUniform), With<Wireframe>>,
     )>,
-    mut views: Query<(&ExtractedView, &VisibleEntities, &mut RenderPhase<Opaque3d>)>,
+    views: Query<(&ExtractedView, &VisibleEntities, &RenderPhase<Opaque3d>)>,
 ) {
     let draw_custom = opaque_3d_draw_functions
         .read()
         .get_id::<DrawWireframes>()
         .unwrap();
     let msaa_key = MeshPipelineKey::from_msaa_samples(msaa.samples);
-    for (view, visible_entities, mut opaque_phase) in views.iter_mut() {
+    for (view, visible_entities, opaque_phase) in views.iter() {
         let view_matrix = view.transform.compute_matrix();
         let view_row_2 = view_matrix.row(2);
 
-        let add_render_phase =
-            |(entity, mesh_handle, mesh_uniform): (Entity, &Handle<Mesh>, &MeshUniform)| {
-                if let Some(mesh) = render_meshes.get(mesh_handle) {
-                    let key = msaa_key
-                        | MeshPipelineKey::from_primitive_topology(mesh.primitive_topology);
-                    let pipeline_id = pipelines.specialize(
-                        &mut pipeline_cache,
-                        &wireframe_pipeline,
-                        key,
-                        &mesh.layout,
-                    );
-                    let pipeline_id = match pipeline_id {
-                        Ok(id) => id,
-                        Err(err) => {
-                            error!("{}", err);
-                            return;
-                        }
-                    };
-                    opaque_phase.add(Opaque3d {
-                        entity,
-                        pipeline: pipeline_id,
-                        draw_function: draw_custom,
-                        distance: view_row_2.dot(mesh_uniform.transform.col(3)),
-                    });
-                }
-            };
+        opaque_phase.phase_scope(|mut phase| {
+            let add_render_phase =
+                |(entity, mesh_handle, mesh_uniform): (Entity, &Handle<Mesh>, &MeshUniform)| {
+                    if let Some(mesh) = render_meshes.get(mesh_handle) {
+                        let key = msaa_key
+                            | MeshPipelineKey::from_primitive_topology(mesh.primitive_topology);
+                        let pipeline_id = pipelines.specialize(
+                            &pipeline_cache,
+                            &wireframe_pipeline,
+                            key,
+                            &mesh.layout,
+                        );
+                        let pipeline_id = match pipeline_id {
+                            Ok(id) => id,
+                            Err(err) => {
+                                error!("{}", err);
+                                return;
+                            }
+                        };
+                        phase.add(Opaque3d {
+                            entity,
+                            pipeline: pipeline_id,
+                            draw_function: draw_custom,
+                            distance: view_row_2.dot(mesh_uniform.transform.col(3)),
+                        });
+                    }
+                };
 
-        if wireframe_config.global {
-            let query = material_meshes.p0();
-            visible_entities
-                .entities
-                .iter()
-                .filter_map(|visible_entity| query.get(*visible_entity).ok())
-                .for_each(add_render_phase);
-        } else {
-            let query = material_meshes.p1();
-            visible_entities
-                .entities
-                .iter()
-                .filter_map(|visible_entity| query.get(*visible_entity).ok())
-                .for_each(add_render_phase);
-        }
+            if wireframe_config.global {
+                let query = material_meshes.p0();
+                visible_entities
+                    .entities
+                    .iter()
+                    .filter_map(|visible_entity| query.get(*visible_entity).ok())
+                    .for_each(add_render_phase);
+            } else {
+                let query = material_meshes.p1();
+                visible_entities
+                    .entities
+                    .iter()
+                    .filter_map(|visible_entity| query.get(*visible_entity).ok())
+                    .for_each(add_render_phase);
+            }
+        });
     }
 }
 

crates/bevy_render/Cargo.toml

@@ -70,3 +70,4 @@ ruzstd = { version = "0.2.4", optional = true }
 # For transcoding of UASTC/ETC1S universal formats, and for .basis file support
 basis-universal = { version = "0.2.0", optional = true }
 encase = { version = "0.2", features = ["glam"] }
+thread_local = "1.1"