OpenXR: Use the XR_FB_foveation_vulkan extension to generate the de…

…nsity map for VRS

modules/openxr/extensions/openxr_extension_wrapper.h

@@ -123,6 +123,7 @@ class OpenXRGraphicsExtensionWrapper : public OpenXRExtensionWrapper {
 	virtual void cleanup_swapchain_graphics_data(void **p_swapchain_graphics_data) = 0; // `cleanup_swapchain_graphics_data` cleans up the data held in our implementation dependent data structure and should free up its memory.
 	virtual bool create_projection_fov(const XrFovf p_fov, double p_z_near, double p_z_far, Projection &r_camera_matrix) = 0; // `create_projection_fov` creates a proper projection matrix based on asymmetric FOV data provided by OpenXR.
 	virtual RID get_texture(void *p_swapchain_graphics_data, int p_image_index) = 0; // `get_texture` returns a Godot texture RID for the current active texture in our swapchain.
+	virtual RID get_density_map(void *p_swapchain_graphics_data, int p_image_index) = 0; // `get_density_map` returns a Godot texture RID for the current active density map in our swapchain (if any).
 };
 
 #endif // OPENXR_EXTENSION_WRAPPER_H

modules/openxr/extensions/openxr_fb_foveation_extension.cpp

@@ -31,6 +31,8 @@
 #include "openxr_fb_foveation_extension.h"
 #include "core/config/project_settings.h"
 
+#include "../openxr_platform_inc.h"
+
 OpenXRFBFoveationExtension *OpenXRFBFoveationExtension::singleton = nullptr;
 
 OpenXRFBFoveationExtension *OpenXRFBFoveationExtension::get_singleton() {
@@ -51,6 +53,12 @@ OpenXRFBFoveationExtension::OpenXRFBFoveationExtension(const String &p_rendering
 	swapchain_create_info_foveation_fb.type = XR_TYPE_SWAPCHAIN_CREATE_INFO_FOVEATION_FB;
 	swapchain_create_info_foveation_fb.next = nullptr;
 	swapchain_create_info_foveation_fb.flags = 0;
+
+	if (rendering_driver == "opengl3") {
+		swapchain_create_info_foveation_fb.flags = XR_SWAPCHAIN_CREATE_FOVEATION_SCALED_BIN_BIT_FB;
+	} else if (rendering_driver == "vulkan") {
+		swapchain_create_info_foveation_fb.flags = XR_SWAPCHAIN_CREATE_FOVEATION_FRAGMENT_DENSITY_MAP_BIT_FB;
+	}
 }
 
 OpenXRFBFoveationExtension::~OpenXRFBFoveationExtension() {
@@ -61,12 +69,14 @@ OpenXRFBFoveationExtension::~OpenXRFBFoveationExtension() {
 HashMap<String, bool *> OpenXRFBFoveationExtension::get_requested_extensions() {
 	HashMap<String, bool *> request_extensions;
 
-	if (rendering_driver == "vulkan") {
-		// This is currently only supported on OpenGL, but we may add Vulkan support in the future...
+	request_extensions[XR_FB_FOVEATION_EXTENSION_NAME] = &fb_foveation_ext;
+	request_extensions[XR_FB_FOVEATION_CONFIGURATION_EXTENSION_NAME] = &fb_foveation_configuration_ext;
 
-	} else if (rendering_driver == "opengl3") {
-		request_extensions[XR_FB_FOVEATION_EXTENSION_NAME] = &fb_foveation_ext;
-		request_extensions[XR_FB_FOVEATION_CONFIGURATION_EXTENSION_NAME] = &fb_foveation_configuration_ext;
+	// Only works on Vulkan if the OpenXR 'XR_FB_foveation_vulkan' extension and the Vulkan 'VK_EXT_fragment_density_map' extension is enabled.
+	// @todo Can't do this here, because RD::get_singleton() is still NULL.
+	//if (rendering_driver == "vulkan" && RD::get_singleton()->has_feature(RD::SUPPORTS_FRAGMENT_DENSITY_MAP)) {
+	if (rendering_driver == "vulkan") {
+		request_extensions[XR_FB_FOVEATION_VULKAN_EXTENSION_NAME] = &fb_foveation_vulkan_ext;
 	}
 
 	return request_extensions;
@@ -89,7 +99,11 @@ void OpenXRFBFoveationExtension::on_instance_destroyed() {
 }
 
 bool OpenXRFBFoveationExtension::is_enabled() const {
-	return swapchain_update_state_ext != nullptr && swapchain_update_state_ext->is_enabled() && fb_foveation_ext && fb_foveation_configuration_ext;
+	bool enabled = swapchain_update_state_ext != nullptr && swapchain_update_state_ext->is_enabled() && fb_foveation_ext && fb_foveation_configuration_ext;
+	if (rendering_driver == "vulkan") {
+		enabled = enabled && fb_foveation_vulkan_ext;
+	}
+	return enabled;
 }
 
 void *OpenXRFBFoveationExtension::set_swapchain_create_info_and_get_next_pointer(void *p_next_pointer) {

modules/openxr/extensions/openxr_fb_foveation_extension.h

@@ -36,11 +36,6 @@
 // Other Android based devices are implementing this as well, see:
 // https://github.khronos.org/OpenXR-Inventory/extension_support.html#XR_FB_foveation
 
-// Note: Currently we only support this for OpenGL.
-// This extension works on enabling foveated rendering on the swapchain.
-// Vulkan does not render 3D content directly to the swapchain image
-// hence this extension can't be used.
-
 #include "../openxr_api.h"
 #include "../util.h"
 #include "openxr_extension_wrapper.h"
@@ -77,6 +72,7 @@ class OpenXRFBFoveationExtension : public OpenXRExtensionWrapper {
 	String rendering_driver;
 	bool fb_foveation_ext = false;
 	bool fb_foveation_configuration_ext = false;
+	bool fb_foveation_vulkan_ext = false;
 
 	// Configuration
 	XrFoveationLevelFB foveation_level = XR_FOVEATION_LEVEL_NONE_FB;

modules/openxr/extensions/platform/openxr_opengl_extension.h

@@ -56,6 +56,7 @@ class OpenXROpenGLExtension : public OpenXRGraphicsExtensionWrapper {
 	virtual void cleanup_swapchain_graphics_data(void **p_swapchain_graphics_data) override;
 	virtual bool create_projection_fov(const XrFovf p_fov, double p_z_near, double p_z_far, Projection &r_camera_matrix) override;
 	virtual RID get_texture(void *p_swapchain_graphics_data, int p_image_index) override;
+	virtual RID get_density_map(void *p_swapchain_graphics_data, int p_image_index) override { return RID(); }
 
 private:
 	static OpenXROpenGLExtension *singleton;

modules/openxr/extensions/platform/openxr_vulkan_extension.cpp

@@ -31,6 +31,7 @@
 #include "openxr_vulkan_extension.h"
 
 #include "../../openxr_util.h"
+#include "../openxr_fb_foveation_extension.h"
 
 #include "core/string/print_string.h"
 #include "servers/rendering/renderer_rd/effects/copy_effects.h"
@@ -240,6 +241,7 @@ void OpenXRVulkanExtension::get_usable_depth_formats(Vector<int64_t> &p_usable_s
 
 bool OpenXRVulkanExtension::get_swapchain_image_data(XrSwapchain p_swapchain, int64_t p_swapchain_format, uint32_t p_width, uint32_t p_height, uint32_t p_sample_count, uint32_t p_array_size, void **r_swapchain_graphics_data) {
 	XrSwapchainImageVulkanKHR *images = nullptr;
+	XrSwapchainImageFoveationVulkanFB *density_images = nullptr;
 
 	RenderingServer *rendering_server = RenderingServer::get_singleton();
 	ERR_FAIL_NULL_V(rendering_server, false);
@@ -262,10 +264,27 @@ bool OpenXRVulkanExtension::get_swapchain_image_data(XrSwapchain p_swapchain, in
 		images[i].image = VK_NULL_HANDLE;
 	}
 
+	if (OpenXRFBFoveationExtension::get_singleton()->is_enabled()) {
+		density_images = (XrSwapchainImageFoveationVulkanFB *)memalloc(sizeof(XrSwapchainImageFoveationVulkanFB) * swapchain_length);
+
+		for (uint64_t i = 0; i < swapchain_length; i++) {
+			density_images[i].type = XR_TYPE_SWAPCHAIN_IMAGE_FOVEATION_VULKAN_FB;
+			density_images[i].next = nullptr;
+			density_images[i].image = VK_NULL_HANDLE;
+			density_images[i].width = 0;
+			density_images[i].height = 0;
+
+			images[i].next = &density_images[i];
+		}
+	}
+
 	result = xrEnumerateSwapchainImages(p_swapchain, swapchain_length, &swapchain_length, (XrSwapchainImageBaseHeader *)images);
 	if (XR_FAILED(result)) {
-		print_line("OpenXR: Failed to get swapchaim images [", OpenXRAPI::get_singleton()->get_error_string(result), "]");
+		print_line("OpenXR: Failed to get swapchain images [", OpenXRAPI::get_singleton()->get_error_string(result), "]");
 		memfree(images);
+		if (density_images) {
+			memfree(density_images);
+		}
 		return false;
 	}
 
@@ -274,6 +293,9 @@ bool OpenXRVulkanExtension::get_swapchain_image_data(XrSwapchain p_swapchain, in
 	if (data == nullptr) {
 		print_line("OpenXR: Failed to allocate memory for swapchain data");
 		memfree(images);
+		if (density_images) {
+			memfree(density_images);
+		}
 		return false;
 	}
 	*r_swapchain_graphics_data = data;
@@ -355,6 +377,7 @@ bool OpenXRVulkanExtension::get_swapchain_image_data(XrSwapchain p_swapchain, in
 	}
 
 	Vector<RID> texture_rids;
+	Vector<RID> density_map_rids;
 
 	// create Godot texture objects for each entry in our swapchain
 	for (uint64_t i = 0; i < swapchain_length; i++) {
@@ -370,11 +393,30 @@ bool OpenXRVulkanExtension::get_swapchain_image_data(XrSwapchain p_swapchain, in
 				p_array_size);
 
 		texture_rids.push_back(image_rid);
+
+		if (density_images && density_images[i].image != VK_NULL_HANDLE) {
+			RID density_map_rid = rendering_device->texture_create_from_extension(
+					p_array_size == 1 ? RenderingDevice::TEXTURE_TYPE_2D : RenderingDevice::TEXTURE_TYPE_2D_ARRAY,
+					RD::DATA_FORMAT_R8G8_UNORM,
+					RenderingDevice::TEXTURE_SAMPLES_1,
+					RD::TEXTURE_USAGE_COLOR_ATTACHMENT_BIT | RD::TEXTURE_USAGE_SAMPLING_BIT | RD::TEXTURE_USAGE_STORAGE_BIT | RD::TEXTURE_USAGE_VRS_ATTACHMENT_BIT,
+					(uint64_t)density_images[i].image,
+					density_images[i].width,
+					density_images[i].height,
+					1,
+					p_array_size);
+
+			density_map_rids.push_back(density_map_rid);
+		} else {
+			density_map_rids.push_back(RID());
+		}
 	}
 
 	data->texture_rids = texture_rids;
+	data->density_map_rids = density_map_rids;
 
 	memfree(images);
+	memfree(density_images);
 
 	return true;
 }
@@ -401,6 +443,14 @@ RID OpenXRVulkanExtension::get_texture(void *p_swapchain_graphics_data, int p_im
 	return data->texture_rids[p_image_index];
 }
 
+RID OpenXRVulkanExtension::get_density_map(void *p_swapchain_graphics_data, int p_image_index) {
+	SwapchainGraphicsData *data = (SwapchainGraphicsData *)p_swapchain_graphics_data;
+	ERR_FAIL_NULL_V(data, RID());
+
+	ERR_FAIL_INDEX_V(p_image_index, data->density_map_rids.size(), RID());
+	return data->density_map_rids[p_image_index];
+}
+
 void OpenXRVulkanExtension::cleanup_swapchain_graphics_data(void **p_swapchain_graphics_data) {
 	if (*p_swapchain_graphics_data == nullptr) {
 		return;
@@ -419,6 +469,13 @@ void OpenXRVulkanExtension::cleanup_swapchain_graphics_data(void **p_swapchain_g
 	}
 	data->texture_rids.clear();
 
+	for (int i = 0; i < data->density_map_rids.size(); i++) {
+		if (data->density_map_rids[i].is_valid()) {
+			rendering_device->free(data->density_map_rids[i]);
+		}
+	}
+	data->density_map_rids.clear();
+
 	memdelete(data);
 	*p_swapchain_graphics_data = nullptr;
 }

modules/openxr/extensions/platform/openxr_vulkan_extension.h

@@ -63,6 +63,7 @@ class OpenXRVulkanExtension : public OpenXRGraphicsExtensionWrapper, VulkanHooks
 	virtual void cleanup_swapchain_graphics_data(void **p_swapchain_graphics_data) override;
 	virtual bool create_projection_fov(const XrFovf p_fov, double p_z_near, double p_z_far, Projection &r_camera_matrix) override;
 	virtual RID get_texture(void *p_swapchain_graphics_data, int p_image_index) override;
+	virtual RID get_density_map(void *p_swapchain_graphics_data, int p_image_index) override;
 
 private:
 	static OpenXRVulkanExtension *singleton;
@@ -71,6 +72,7 @@ class OpenXRVulkanExtension : public OpenXRGraphicsExtensionWrapper, VulkanHooks
 	struct SwapchainGraphicsData {
 		bool is_multiview;
 		Vector<RID> texture_rids;
+		Vector<RID> density_map_rids;
 	};
 
 	bool check_graphics_api_support(XrVersion p_desired_version);

modules/openxr/openxr_api.cpp

@@ -264,6 +264,16 @@ RID OpenXRAPI::OpenXRSwapChainInfo::get_image() {
 	}
 }
 
+RID OpenXRAPI::OpenXRSwapChainInfo::get_density_map() {
+	OpenXRAPI *openxr_api = OpenXRAPI::get_singleton();
+
+	if (image_acquired && openxr_api && openxr_api->get_graphics_extension()) {
+		return OpenXRAPI::get_singleton()->get_graphics_extension()->get_density_map(swapchain_graphics_data, image_index);
+	} else {
+		return RID();
+	}
+}
+
 ////////////////////////////////////
 // OpenXRAPI
 
@@ -2343,6 +2353,17 @@ RID OpenXRAPI::get_depth_texture() {
 	}
 }
 
+RID OpenXRAPI::get_density_map_texture() {
+	ERR_NOT_ON_RENDER_THREAD_V(RID());
+
+	OpenXRFBFoveationExtension *fov_ext = OpenXRFBFoveationExtension::get_singleton();
+	if (fov_ext && fov_ext->is_enabled()) {
+		return render_state.main_swapchains[OPENXR_SWAPCHAIN_COLOR].get_density_map();
+	}
+
+	return RID();
+}
+
 void OpenXRAPI::post_draw_viewport(RID p_render_target) {
 	// Must be called from rendering thread!
 	ERR_NOT_ON_RENDER_THREAD;

modules/openxr/openxr_api.h

@@ -78,6 +78,7 @@ class OpenXRAPI {
 		bool acquire(bool &p_should_render);
 		bool release();
 		RID get_image();
+		RID get_density_map();
 	};
 
 private:
@@ -477,6 +478,7 @@ class OpenXRAPI {
 	XrSwapchain get_color_swapchain();
 	RID get_color_texture();
 	RID get_depth_texture();
+	RID get_density_map_texture();
 	void post_draw_viewport(RID p_render_target);
 	void end_frame();
 

modules/openxr/openxr_interface.cpp

@@ -1476,6 +1476,11 @@ RID OpenXRInterface::get_vrs_texture() {
 		return RID();
 	}
 
+	RID density_map = openxr_api->get_density_map_texture();
+	if (density_map.is_valid()) {
+		return density_map;
+	}
+
 	PackedVector2Array eye_foci;
 
 	Size2 target_size = get_render_target_size();

servers/rendering/renderer_rd/forward_mobile/render_forward_mobile.cpp

@@ -173,9 +173,17 @@ RID RenderForwardMobile::RenderBufferDataForwardMobile::get_color_fbs(Framebuffe
 	uint32_t view_count = render_buffers->get_view_count();
 
 	RID vrs_texture;
+#if 0
 	if (render_buffers->has_texture(RB_SCOPE_VRS, RB_TEXTURE)) {
 		vrs_texture = render_buffers->get_texture(RB_SCOPE_VRS, RB_TEXTURE);
 	}
+#else
+	// HACK: Directly use the VRS texture from the XRInterface.
+	Ref<XRInterface> interface = XRServer::get_singleton()->get_primary_interface();
+	if (interface.is_valid()) {
+		vrs_texture = interface->get_vrs_texture();
+	}
+#endif
 
 	Vector<RID> textures;
 	int color_buffer_id = 0;
@@ -730,7 +738,10 @@ void RenderForwardMobile::_render_scene(RenderDataRD *p_render_data, const Color
 
 	RENDER_TIMESTAMP("Prepare 3D Scene");
 
+#if 0
+	// HACK: Skip updating the VRS texture.
 	_update_vrs(rb);
+#endif
 
 	RENDER_TIMESTAMP("Setup 3D Scene");