Artifacts in post-processing depth shader on 4.3-RC1

[TranquilMarmot]

Tested versions

Reproduceable in:
Godot v4.3.rc1.mono

Not reproduceable in:
Godot v4.2.2.stable.mono

System information

macOS 14.5.0 - Vulkan (Forward+) - integrated Apple M1 - Apple M1 (8 Threads)

Issue description

I'm using this shader: https://godotshaders.com/shader/high-quality-post-process-outline/

Here it is for Godot 4.2 with all of the irrelevant mobile-related code stripped out:

shader_type spatial;
render_mode unshaded, blend_mix, depth_draw_never, depth_test_disabled;

/*
	AUTHOR: Hannah "EMBYR" Crawford
	ENGINE_VERSION: 4.0.3
	
	HOW TO USE:
		1. Create a MeshInstance3D node and place it in your scene.
		2. Set its size to 2x2.
		3. Enable the "Flip Faces" option.
		4. Create a new shader material with this shader.
		5. Assign the material to the MeshInstance3D
	
	LIMITATIONS:
		Does not work well with TAA enabled.
*/

group_uniforms outline;
uniform vec4 outlineColor: source_color = vec4(0.0, 0.0, 0.0, 0.78);
uniform float depth_threshold = 0.025;
uniform float normal_threshold : hint_range(0.0, 1.5) = 0.5;
uniform float normal_smoothing : hint_range(0.0, 1.0) = 0.25;

group_uniforms thickness;
uniform float max_thickness: hint_range(0.0, 5.0) = 1.3;
uniform float min_thickness = 0.5;
uniform float max_distance = 75.0;
uniform float min_distance = 2.0;

group_uniforms grazing_prevention;
uniform float grazing_fresnel_power = 5.0;
uniform float grazing_angle_mask_power = 1.0;
uniform float grazing_angle_modulation_factor = 50.0;

uniform sampler2D DEPTH_TEXTURE : hint_depth_texture, filter_linear, repeat_disable;
uniform sampler2D NORMR_TEXTURE : hint_normal_roughness_texture, filter_linear, repeat_disable;

struct UVNeighbors {
	vec2 center; 
	vec2 left;     vec2 right;     vec2 up;          vec2 down;
	vec2 top_left; vec2 top_right; vec2 bottom_left; vec2 bottom_right;
};

struct NeighborDepthSamples {
	float c_d; 
	float l_d;  float r_d;  float u_d;  float d_d; 
	float tl_d; float tr_d; float bl_d; float br_d;
};

UVNeighbors getNeighbors(vec2 center, float width, float aspect) {
	vec2 h_offset = vec2(width * aspect * 0.001, 0.0);
	vec2 v_offset = vec2(0.0, width * 0.001);
	UVNeighbors n;
	n.center = center;
	n.left   = center - h_offset;
	n.right  = center + h_offset;
	n.up     = center - v_offset;
	n.down   = center + v_offset;
	n.top_left     = center - (h_offset - v_offset);
	n.top_right    = center + (h_offset - v_offset);
	n.bottom_left  = center - (h_offset + v_offset);
	n.bottom_right = center + (h_offset + v_offset);
	return n;
}

float getMinimumDepth(NeighborDepthSamples ds){
	return min(ds.c_d, min(ds.l_d, min(ds.r_d, min(ds.u_d, min(ds.d_d, min(ds.tl_d, min(ds.tr_d, min(ds.bl_d, ds.br_d))))))));
}

float getLinearDepth(float depth, vec2 uv, mat4 inv_proj) {
	vec3 ndc = vec3(uv * 2.0 - 1.0, depth);
	vec4 view = inv_proj * vec4(ndc, 1.0);
	view.xyz /= view.w;
	return -view.z;
}

NeighborDepthSamples getLinearDepthSamples(UVNeighbors uvs, sampler2D depth_tex, mat4 invProjMat) {
	NeighborDepthSamples result;
	result.c_d  = getLinearDepth(texture(depth_tex, uvs.center).r, uvs.center, invProjMat);
	result.l_d  = getLinearDepth(texture(depth_tex, uvs.left).r  , uvs.left  , invProjMat);
	result.r_d  = getLinearDepth(texture(depth_tex, uvs.right).r , uvs.right , invProjMat);
	result.u_d  = getLinearDepth(texture(depth_tex, uvs.up).r    , uvs.up    , invProjMat);
	result.d_d  = getLinearDepth(texture(depth_tex, uvs.down).r  , uvs.down  , invProjMat);
	result.tl_d = getLinearDepth(texture(depth_tex, uvs.top_left).r, uvs.top_left, invProjMat);
	result.tr_d = getLinearDepth(texture(depth_tex, uvs.top_right).r, uvs.top_right, invProjMat);
	result.bl_d = getLinearDepth(texture(depth_tex, uvs.bottom_left).r, uvs.bottom_left, invProjMat);
	result.br_d = getLinearDepth(texture(depth_tex, uvs.bottom_right).r, uvs.bottom_right, invProjMat);
	return result;
}

float remap(float v, float from1, float to1, float from2, float to2) {
	return (v - from1) / (to1 - from1) * (to2 - from2) + from2;
}

float fresnel(float amount, vec3 normal, vec3 view) {
	return pow((1.0 - clamp(dot(normalize(normal), normalize(view)), 0.0, 1.0 )), amount);
}

float getGrazingAngleModulation(vec3 pixel_normal, vec3 view) {
	float x = clamp(((fresnel(grazing_fresnel_power, pixel_normal, view) - 1.0) / grazing_angle_mask_power) + 1.0, 0.0, 1.0);
	return (x + grazing_angle_modulation_factor) + 1.0;
}

float detectEdgesDepth(NeighborDepthSamples depth_samples, vec3 pixel_normal, vec3 view) {
	float n_total = 
		depth_samples.l_d + 
		depth_samples.r_d + 
		depth_samples.u_d + 
		depth_samples.d_d + 
		depth_samples.tl_d + 
		depth_samples.tr_d + 
		depth_samples.bl_d + 
		depth_samples.br_d;
	
	float t = depth_threshold * getGrazingAngleModulation(pixel_normal, view);
	return step(t, n_total - (depth_samples.c_d * 8.0));
}

float detectEdgesNormal(UVNeighbors uvs, sampler2D normTex, vec3 camDirWorld){
	vec3 n_u = texture(normTex, uvs.up).xyz;
	vec3 n_d = texture(normTex, uvs.down).xyz;
	vec3 n_l = texture(normTex, uvs.left).xyz;
	vec3 n_r = texture(normTex, uvs.right).xyz;
	vec3 n_tl = texture(normTex, uvs.top_left).xyz;
	vec3 n_tr = texture(normTex, uvs.top_right).xyz;
	vec3 n_bl = texture(normTex, uvs.bottom_left).xyz;
	vec3 n_br = texture(normTex, uvs.bottom_right).xyz;
	
	vec3 normalFiniteDifference0 = n_tr - n_bl;
	vec3 normalFiniteDifference1 = n_tl - n_br;
	vec3 normalFiniteDifference2 = n_l - n_r;
	vec3 normalFiniteDifference3 = n_u - n_d;
	
	float edgeNormal = sqrt(
		dot(normalFiniteDifference0, normalFiniteDifference0) + 
		dot(normalFiniteDifference1, normalFiniteDifference1) + 
		dot(normalFiniteDifference2, normalFiniteDifference2) + 
		dot(normalFiniteDifference3, normalFiniteDifference3)
	);
	
	return smoothstep(normal_threshold - normal_smoothing, normal_threshold + normal_smoothing, edgeNormal);
}

void vertex() {
	POSITION = vec4(VERTEX, 1.0);
}

void fragment() {
	float aspect = float(VIEWPORT_SIZE.y) / float(VIEWPORT_SIZE.x);
	
	UVNeighbors n = getNeighbors(SCREEN_UV, max_thickness, aspect);
	NeighborDepthSamples depth_samples = getLinearDepthSamples(n, DEPTH_TEXTURE, INV_PROJECTION_MATRIX);
	
	float min_d = getMinimumDepth(depth_samples);
	float thickness = clamp(remap(min_d, min_distance, max_distance, max_thickness, min_thickness), min_thickness, max_thickness);
	float fade_a = clamp(remap(min_d, min_distance, max_distance, 1.0, 0.0), 0.0, 1.0);
	
	n = getNeighbors(SCREEN_UV, thickness, aspect);
	depth_samples = getLinearDepthSamples(n, DEPTH_TEXTURE, INV_PROJECTION_MATRIX);
	
	vec3 pixel_normal = texture(NORMR_TEXTURE, SCREEN_UV).xyz;
	
	float depthEdges = detectEdgesDepth(depth_samples, pixel_normal, VIEW);
	
	float normEdges = min(detectEdgesNormal(n, NORMR_TEXTURE, CAMERA_DIRECTION_WORLD), 1.0);
	
	ALBEDO.rgb = outlineColor.rgb;
	ALPHA = max(depthEdges, normEdges) * outlineColor.a * fade_a;
}

Behavior in 4.2

Nice outlines, no artifacts 👍

Screen.Recording.2024-07-25.at.10.58.37.PM.mov

Behavior in 4.3

After reading through the Introducing Reverse Z (AKA I'm sorry I broke your shader) article, I made the following changes:

void vertex() {
-	POSITION = vec4(VERTEX, 1.0);
+       POSITION = vec4(VERTEX.xy, 1.0, 1.0);
}

float detectEdgesDepth(NeighborDepthSamples depth_samples, vec3 pixel_normal, vec3 view) {
	float n_total =
		depth_samples.l_d +
		depth_samples.r_d +
		depth_samples.u_d +
		depth_samples.d_d +
		depth_samples.tl_d +
		depth_samples.tr_d +
		depth_samples.bl_d +
		depth_samples.br_d;

	float t = depth_threshold * getGrazingAngleModulation(pixel_normal, view);
- 	return step(t, n_total - (depth_samples.c_d * 8.0));
+       return step(t, (depth_samples.c_d * 8.0) - n_total); // Invert the comparison for reverse Z
}

However, running this in Godot 4.3 there are artifacts on sloped surfaces.

Screen.Recording.2024-07-25.at.11.01.34.PM.mov

Steps to reproduce

• Use the shader from above
• Create a MeshInstance3D in the scene
  • Give it a QuadMesh
  • Set the material override to a ShaderMaterial using the shader
• Look at a sloped surface

Minimal reproduction project (MRP)

Here is a repo with a reproduction:

https://github.com/TranquilMarmot/godot_4.3_shader_artifacts

There are two folders:

• 4.2: Open with Godot 4.2 to see intended behavior
• 4.3: Open with Godot 4.3 to see broken behavior

[TranquilMarmot]

It looks like this is the same issue mentioned in godotengine/godot-docs#9591 (see also #86316)

Applying this function to each of the normal samples fixes the issue:

vec4 normal_roughness_compatibility(vec4 p_normal_roughness) {
    float roughness = p_normal_roughness.w;

    if (roughness > 0.5) {
        roughness = 1.0 - roughness;
    }

    roughness /= (127.0 / 255.0);
    return vec4(normalize(p_normal_roughness.xyz * 2.0 - 1.0) * 0.5 + 0.5, roughness);
}

float detectEdgesNormal(UVNeighbors uvs, sampler2D normTex, vec3 camDirWorld){
	vec3 n_u = normal_roughness_compatibility(texture(normTex, uvs.up)).xyz;
	vec3 n_d = normal_roughness_compatibility(texture(normTex, uvs.down)).xyz;
	vec3 n_l = normal_roughness_compatibility(texture(normTex, uvs.left)).xyz;
	vec3 n_r = normal_roughness_compatibility(texture(normTex, uvs.right)).xyz;
	vec3 n_tl = normal_roughness_compatibility(texture(normTex, uvs.top_left)).xyz;
	vec3 n_tr = normal_roughness_compatibility(texture(normTex, uvs.top_right)).xyz;
	vec3 n_bl = normal_roughness_compatibility(texture(normTex, uvs.bottom_left)).xyz;
	vec3 n_br = normal_roughness_compatibility(texture(normTex, uvs.bottom_right)).xyz;

	vec3 normalFiniteDifference0 = n_tr - n_bl;
	vec3 normalFiniteDifference1 = n_tl - n_br;
	vec3 normalFiniteDifference2 = n_l - n_r;
	vec3 normalFiniteDifference3 = n_u - n_d;

	float edgeNormal = sqrt(
		dot(normalFiniteDifference0, normalFiniteDifference0) +
		dot(normalFiniteDifference1, normalFiniteDifference1) +
		dot(normalFiniteDifference2, normalFiniteDifference2) +
		dot(normalFiniteDifference3, normalFiniteDifference3)
	);

	return smoothstep(normal_threshold - normal_smoothing, normal_threshold + normal_smoothing, edgeNormal);
}

I don't think the change mentioned above for detectEdgesDepth is needed.

[pink-arcana]

Interesting! I opened the docs issue for CompositorEffects, and I was only able to discover the problem because the Spatial shader on my quadmesh WAS receiving the correct (converted) normals values, and my Compute shader wasn't. I just confirmed the Spatial shader seems to still be getting the correct normals values in 4.3-rc1.

However, when I open your MRP, I am able to reproduce the same artifacts.

Here's my Spatial shader. Results were the same with linear sampling.

shader_type spatial;

uniform sampler2D normal_roughness_texture : hint_normal_roughness_texture, repeat_disable, filter_nearest;

void vertex() {
	POSITION = vec4(VERTEX.xy, 1.0, 1.0);
}

void fragment() {
	vec3 screen_normal = texture(normal_roughness_texture, SCREEN_UV).xyz;
	screen_normal = screen_normal * 2.0 - 1.0;
	ALBEDO = screen_normal;
}

[clayjohn]

Very interesting catch!

We have code that automatically detects when the normal_roughness buffer is used and it automatically wraps the texture() call with normal_roughness_compatibility(). However, it looks like this is failing when the normal_roughness texture is passed to a function.

Here is the generated code:

fragment()

{
	float m_aspect=(float(read_viewport_size.y) / float(read_viewport_size.x));
	m_UVNeighbors m_n=m_getNeighbors(screen_uv, material.m_max_thickness, m_aspect);
	m_NeighborDepthSamples m_depth_samples=m_getLinearDepthSamples(m_n, depth_buffer, inv_projection_matrix);
	float m_min_d=m_getMinimumDepth(m_depth_samples);
	float m_thickness=clamp(m_remap(m_min_d, material.m_min_distance, material.m_max_distance, material.m_max_thickness, material.m_min_thickness), material.m_min_thickness, material.m_max_thickness);
	float m_fade_a=clamp(m_remap(m_min_d, material.m_min_distance, material.m_max_distance, 1.0, 0.0), 0.0, 1.0);
	m_n=m_getNeighbors(screen_uv, m_thickness, m_aspect);
	m_depth_samples=m_getLinearDepthSamples(m_n, depth_buffer, inv_projection_matrix);
	vec3 m_pixel_normal=normal_roughness_compatibility(texture(sampler2D(normal_roughness_buffer, SAMPLER_LINEAR_CLAMP), screen_uv)).xyz;
	float m_depthEdges=m_detectEdgesDepth(m_depth_samples, m_pixel_normal, view);
	float m_normEdges=min(m_detectEdgesNormal(m_n, normal_roughness_buffer, scene_data.inv_view_matrix[2].xyz), 1.0);
	albedo.rgb=material.m_outlineColor.rgb;
	alpha=((max(m_depthEdges, m_normEdges) * material.m_outlineColor.a) * m_fade_a);
}

And

float m_detectEdgesNormal(m_UVNeighbors m_uvs, texture2D m_normTex, vec3 m_camDirWorld)
	{
		vec3 m_n_u=texture(sampler2D(m_normTex, SAMPLER_LINEAR_CLAMP), m_uvs.up).xyz;
		vec3 m_n_d=texture(sampler2D(m_normTex, SAMPLER_LINEAR_CLAMP), m_uvs.down).xyz;
		vec3 m_n_l=texture(sampler2D(m_normTex, SAMPLER_LINEAR_CLAMP), m_uvs.left).xyz;
		vec3 m_n_r=texture(sampler2D(m_normTex, SAMPLER_LINEAR_CLAMP), m_uvs.right).xyz;
		vec3 m_n_tl=texture(sampler2D(m_normTex, SAMPLER_LINEAR_CLAMP), m_uvs.top_left).xyz;
		vec3 m_n_tr=texture(sampler2D(m_normTex, SAMPLER_LINEAR_CLAMP), m_uvs.top_right).xyz;
		vec3 m_n_bl=texture(sampler2D(m_normTex, SAMPLER_LINEAR_CLAMP), m_uvs.bottom_left).xyz;
		vec3 m_n_br=texture(sampler2D(m_normTex, SAMPLER_LINEAR_CLAMP), m_uvs.bottom_right).xyz;
		vec3 m_normalFiniteDifference0=(m_n_tr - m_n_bl);
		vec3 m_normalFiniteDifference1=(m_n_tl - m_n_br);
		vec3 m_normalFiniteDifference2=(m_n_l - m_n_r);
		vec3 m_normalFiniteDifference3=(m_n_u - m_n_d);
		float m_edgeNormal=sqrt((((dot(m_normalFiniteDifference0, m_normalFiniteDifference0) + dot(m_normalFiniteDifference1, m_normalFiniteDifference1)) + dot(m_normalFiniteDifference2, m_normalFiniteDifference2)) + dot(m_normalFiniteDifference3, m_normalFiniteDifference3)));
return smoothstep((material.m_normal_threshold - material.m_normal_smoothing), (material.m_normal_threshold + material.m_normal_smoothing), m_edgeNormal);	}

You can see the compatibility code is only added in the first case. Looking into a fix now

[clayjohn]

I have a patch that fixes this issue, but it introduces another issue that we have run into before. If we add the compatibility code to ensure that the normal is read correctly, it makes it so that function will always have the compatibility code. Therefore if you call the function with both the normal_roughness buffer and some other texture, it will fail for that other texture.

We ran into this issue for XR and ended up just banning passing the screen textures as arguments in custom functions when using XR. I think i can fix both cases by just banning using a screen texture as an argument in one place while using a different texture in another place

Edit: That was unexpectedly easy. PR incoming