Cull out-of-bounds triangles that were not clipped

GPU/Common/GPUStateUtils.cpp

@@ -130,6 +130,7 @@ bool IsColorTestTriviallyTrue() {
 	}
 }
 
+// TODO: Pack into 16-bit integer.
 const bool nonAlphaSrcFactors[16] = {
 	true,  // GE_SRCBLEND_DSTCOLOR,
 	true,  // GE_SRCBLEND_INVDSTCOLOR,

GPU/Common/ShaderCommon.h

@@ -73,6 +73,8 @@ enum : uint64_t {
 	DIRTY_UVSCALEOFFSET = 1ULL << 18,
 	DIRTY_DEPTHRANGE = 1ULL << 19,
 
+	DIRTY_GUARDBAND = 1ULL << 20,
+
 	DIRTY_WORLDMATRIX = 1ULL << 21,
 	DIRTY_VIEWMATRIX = 1ULL << 22,
 	DIRTY_TEXMATRIX = 1ULL << 23,

GPU/Common/ShaderUniforms.cpp

@@ -25,6 +25,25 @@ static void ConvertProjMatrixToD3D11(Matrix4x4 &in) {
 	in.translateAndScale(trans, scale);
 }
 
+void ComputeGuardband(float gb[4], float zmin) {
+	float vpWidth = fabsf(gstate_c.vpWidth);
+	float vpHeight = fabsf(gstate_c.vpHeight);
+	// Avoid bad values during initialization. Doubt these are really needed.
+	if (vpWidth == 0.0)
+		vpWidth = 480;
+	if (vpHeight == 0.0)
+		vpHeight = 272;
+
+	// We assume a symmetric guardband, even though it's not entirely correct to do so - but nearly everything does it
+	// this way and we have space for the NAN in the uniform.
+	// We also assume that everything behind the near clipping plane gets clipped and will thus not in reality
+	// exceed the guardband. This is a bit rough but should be ok.
+	gb[0] = (2048.0f / (vpWidth*0.5f));
+	gb[1] = (2048.0f / (vpHeight*0.5f));
+	gb[2] = zmin;
+	gb[3] = NAN;
+}
+
 void BaseUpdateUniforms(UB_VS_FS_Base *ub, uint64_t dirtyUniforms, bool flipViewport) {
 	if (dirtyUniforms & DIRTY_TEXENV) {
 		Uint8x3ToFloat4(ub->texEnvColor, gstate.texenvcolor);
@@ -58,7 +77,11 @@ void BaseUpdateUniforms(UB_VS_FS_Base *ub, uint64_t dirtyUniforms, bool flipView
 		ub->texClampOffset[0] = gstate_c.curTextureXOffset * invW;
 		ub->texClampOffset[1] = gstate_c.curTextureYOffset * invH;
 	}
-
+	if (dirtyUniforms & DIRTY_GUARDBAND) {
+		float gb[4];
+		ComputeGuardband(gb, 0.0f);
+		memcpy(ub->guardband, gb, sizeof(float) * 4);
+	}
 	if (dirtyUniforms & DIRTY_PROJMATRIX) {
 		Matrix4x4 flippedMatrix;
 		memcpy(&flippedMatrix, gstate.projMatrix, 16 * sizeof(float));
@@ -100,6 +123,7 @@ void BaseUpdateUniforms(UB_VS_FS_Base *ub, uint64_t dirtyUniforms, bool flipView
 		if (g_Config.iRenderingMode == FB_NON_BUFFERED_MODE && g_display_rotation != DisplayRotation::ROTATE_0) {
 			proj_through = proj_through * g_display_rot_matrix;
 		}
+		// proj_through.translateAndScale(Vec3(0, 0, 0), Vec3(1.0f / debugscale, 1.0f / debugscale, 0));
 		CopyMatrix4x4(ub->proj_through, proj_through.getReadPtr());
 	}
 

GPU/Common/ShaderUniforms.h

@@ -9,7 +9,7 @@
 enum : uint64_t {
 	DIRTY_BASE_UNIFORMS =
 	DIRTY_WORLDMATRIX | DIRTY_PROJTHROUGHMATRIX | DIRTY_VIEWMATRIX | DIRTY_TEXMATRIX | DIRTY_ALPHACOLORREF |
-	DIRTY_PROJMATRIX | DIRTY_FOGCOLOR | DIRTY_FOGCOEF | DIRTY_TEXENV | DIRTY_STENCILREPLACEVALUE |
+	DIRTY_PROJMATRIX | DIRTY_FOGCOLOR | DIRTY_FOGCOEF | DIRTY_TEXENV | DIRTY_STENCILREPLACEVALUE | DIRTY_GUARDBAND |
 	DIRTY_ALPHACOLORMASK | DIRTY_SHADERBLEND | DIRTY_UVSCALEOFFSET | DIRTY_TEXCLAMP | DIRTY_DEPTHRANGE | DIRTY_MATAMBIENTALPHA |
 	DIRTY_BEZIERSPLINE | DIRTY_DEPAL,
 	DIRTY_LIGHT_UNIFORMS =
@@ -32,6 +32,7 @@ struct UB_VS_FS_Base {
 	float matAmbient[4];
 	uint32_t spline_counts; uint32_t depal_mask_shift_off_fmt;  // 4 params packed into one.
 	int pad2; int pad3;
+	float guardband[4];
 	// Fragment data
 	float fogColor[4];
 	float texEnvColor[4];
@@ -58,6 +59,8 @@ R"(  mat4 proj_mtx;
   uint depal_mask_shift_off_fmt;
   int pad2;
   int pad3;
+  vec4 guardband;
+  // Fragment
   vec3 fogcolor;
   vec3 texenv;
   ivec4 alphacolorref;
@@ -84,6 +87,8 @@ R"(  float4x4 u_proj;
   uint u_depal_mask_shift_off_fmt;
   int pad2;
   int pad3;
+  float4 u_guardband;
+  // Fragment
   float3 u_fogcolor;
   float3 u_texenv;
   uint4 u_alphacolorref;
@@ -179,3 +184,4 @@ void BaseUpdateUniforms(UB_VS_FS_Base *ub, uint64_t dirtyUniforms, bool flipView
 void LightUpdateUniforms(UB_VS_Lights *ub, uint64_t dirtyUniforms);
 void BoneUpdateUniforms(UB_VS_Bones *ub, uint64_t dirtyUniforms);
 
+void ComputeGuardband(float gb[4], float zmin);

GPU/Directx9/ShaderManagerDX9.cpp

@@ -35,6 +35,7 @@
 #include "GPU/Math3D.h"
 #include "GPU/GPUState.h"
 #include "GPU/ge_constants.h"
+#include "GPU/Common/ShaderUniforms.h"
 #include "GPU/Directx9/ShaderManagerDX9.h"
 #include "GPU/Directx9/DrawEngineDX9.h"
 #include "GPU/Directx9/FramebufferDX9.h"
@@ -414,6 +415,11 @@ void ShaderManagerDX9::VSUpdateUniforms(u64 dirtyUniforms) {
 		}
 	}
 #endif
+	if (dirtyUniforms & DIRTY_GUARDBAND) {
+		float gb[4];
+		ComputeGuardband(gb, 0.0f);
+		VSSetFloatUniform4(CONST_VS_GUARDBAND, gb);
+	}
 
 	// Texturing
 	if (dirtyUniforms & DIRTY_UVSCALEOFFSET) {

GPU/Directx9/VertexShaderGeneratorDX9.cpp

@@ -55,6 +55,34 @@ enum DoLightComputation {
 	LIGHT_FULL,
 };
 
+// The PSP does not have a proper triangle clipper on the sides. It does have on for the front plane.
+// It has a guard band though and can rasterize rather large
+// triangles that go outside the viewport. However, there are limits, and it will drop triangles that are very
+// large. Some games appear to draw broken geometry, probably game bugs that were never discovered because the PSP
+// would drop the geometry, including Parappa The Rapper in an obscure case and Outrun. 
+// Additionally, some games clearly intentionally use this to cull triangles like the TOCA series.
+// Try to get rid of those triangles by setting the W of one of the vertices to NaN if they are discovered.
+
+// #define COLORGUARDBAND
+
+#ifdef COLORGUARDBAND
+// Coloring debug version
+static void WriteGuardBand(char *&p) {
+	WRITE(p, "  if (outPos.w >= 0.0) {\n");
+	WRITE(p, "		float3 projPos = outPos.xyz / outPos.w; \n");
+	WRITE(p, "		if (projPos.z >= 0.0 && projPos.z <= 1.0 && (abs(projPos.x) > u_guardband.x || projPos.y > u_guardband.y)) colorOverride.g = 0.0;\n");//outPos.w = u_guardband.w;\n");
+	WRITE(p, "  } else { colorOverride.b = 0.0; } \n");
+}
+#else
+// NOTE: We are skipping the bottom check. This fixes TOCA but I am dubious about it...
+static void WriteGuardBand(char *&p) {
+	WRITE(p, "  if (outPos.w >= 0.0) {\n");
+	WRITE(p, "		float3 projPos = outPos.xyz / outPos.w; \n");
+	WRITE(p, "		if (projPos.z >= 0.0 && projPos.z <= 1.0 && (abs(projPos.x) > u_guardband.x || abs(projPos.y) > u_guardband.y)) outPos.w = u_guardband.w;\n");
+	WRITE(p, "  }\n");
+}
+#endif
+
 void GenerateVertexShaderHLSL(const VShaderID &id, char *buffer, ShaderLanguage lang) {
 	char *p = buffer;
 	const u32 vertType = gstate.vertType;
@@ -116,6 +144,7 @@ void GenerateVertexShaderHLSL(const VShaderID &id, char *buffer, ShaderLanguage
 			WRITE(p, "float4x4 u_proj : register(c%i);\n", CONST_VS_PROJ);
 			// Add all the uniforms we'll need to transform properly.
 		}
+		WRITE(p, "float4 u_guardband : register(c%i);\n", CONST_VS_GUARDBAND);
 
 		if (enableFog) {
 			WRITE(p, "float2 u_fogcoef : register(c%i);\n", CONST_VS_FOGCOEF);
@@ -347,6 +376,9 @@ void GenerateVertexShaderHLSL(const VShaderID &id, char *buffer, ShaderLanguage
 
 	WRITE(p, "VS_OUT main(VS_IN In) {\n");
 	WRITE(p, "  VS_OUT Out;\n");  
+#ifdef COLORGUARDBAND
+	WRITE(p, "	float4 colorOverride = float4(1.0, 1.0, 1.0, 1.0);\n");
+#endif
 	if (!useHWTransform) {
 		// Simple pass-through of vertex data to fragment shader
 		if (doTexture) {
@@ -370,22 +402,22 @@ void GenerateVertexShaderHLSL(const VShaderID &id, char *buffer, ShaderLanguage
 		}
 		if (lang == HLSL_D3D11 || lang == HLSL_D3D11_LEVEL9) {
 			if (gstate.isModeThrough()) {
-				WRITE(p, "  Out.gl_Position = mul(u_proj_through, float4(In.position.xyz, 1.0));\n");
+				WRITE(p, "  float4 outPos = mul(u_proj_through, float4(In.position.xyz, 1.0));\n");
 			} else {
 				if (gstate_c.Supports(GPU_ROUND_DEPTH_TO_16BIT)) {
-					WRITE(p, "  Out.gl_Position = depthRoundZVP(mul(u_proj, float4(In.position.xyz, 1.0)));\n");
+					WRITE(p, "  float4 outPos = depthRoundZVP(mul(u_proj, float4(In.position.xyz, 1.0)));\n");
 				} else {
-					WRITE(p, "  Out.gl_Position = mul(u_proj, float4(In.position.xyz, 1.0));\n");
+					WRITE(p, "  float4 outPos = mul(u_proj, float4(In.position.xyz, 1.0));\n");
 				}
 			}
 		} else {
 			if (gstate.isModeThrough()) {
-				WRITE(p, "  Out.gl_Position = mul(float4(In.position.xyz, 1.0), u_proj_through);\n");
+				WRITE(p, "  float4 outPos = mul(float4(In.position.xyz, 1.0), u_proj_through);\n");
 			} else {
 				if (gstate_c.Supports(GPU_ROUND_DEPTH_TO_16BIT)) {
-					WRITE(p, "  Out.gl_Position = depthRoundZVP(mul(float4(In.position.xyz, 1.0), u_proj));\n");
+					WRITE(p, "  float4 outPos = depthRoundZVP(mul(float4(In.position.xyz, 1.0), u_proj));\n");
 				} else {
-					WRITE(p, "  Out.gl_Position = mul(float4(In.position.xyz, 1.0), u_proj);\n");
+					WRITE(p, "  float4 outPos = mul(float4(In.position.xyz, 1.0), u_proj);\n");
 				}
 			}
 		}
@@ -577,18 +609,20 @@ void GenerateVertexShaderHLSL(const VShaderID &id, char *buffer, ShaderLanguage
 		if (lang == HLSL_D3D11 || lang == HLSL_D3D11_LEVEL9) {
 			// Final view and projection transforms.
 			if (gstate_c.Supports(GPU_ROUND_DEPTH_TO_16BIT)) {
-				WRITE(p, "  Out.gl_Position = depthRoundZVP(mul(u_proj, viewPos));\n");
+				WRITE(p, "  float4 outPos = depthRoundZVP(mul(u_proj, viewPos));\n");
 			} else {
-				WRITE(p, "  Out.gl_Position = mul(u_proj, viewPos);\n");
+				WRITE(p, "  float4 outPos = mul(u_proj, viewPos);\n");
 			}
 		} else {
 			// Final view and projection transforms.
 			if (gstate_c.Supports(GPU_ROUND_DEPTH_TO_16BIT)) {
-				WRITE(p, "  Out.gl_Position = depthRoundZVP(mul(viewPos, u_proj));\n");
+				WRITE(p, "  float4 outPos = depthRoundZVP(mul(viewPos, u_proj));\n");
 			} else {
-				WRITE(p, "  Out.gl_Position = mul(viewPos, u_proj);\n");
+				WRITE(p, "  float4 outPos = mul(viewPos, u_proj);\n");
 			}
 		}
+		WriteGuardBand(p);
+		WRITE(p, "  Out.gl_Position = outPos;\n");
 
 		// TODO: Declare variables for dots for shade mapping if needed.
 
@@ -805,6 +839,9 @@ void GenerateVertexShaderHLSL(const VShaderID &id, char *buffer, ShaderLanguage
 		}
 	}
 
+#ifdef COLORGUARDBAND
+	WRITE(p, "  Out.v_color0 *= colorOverride;\n");
+#endif
 	WRITE(p, "  return Out;\n");
 	WRITE(p, "}\n");
 }

GPU/Directx9/VertexShaderGeneratorDX9.h

@@ -53,6 +53,7 @@ namespace DX9 {
 		CONST_VS_BONE6 = 71,
 		CONST_VS_BONE7 = 74,
 		CONST_VS_BONE8 = 77,
+		CONST_VS_GUARDBAND = 80,
 	};
 
 };

GPU/GLES/ShaderManagerGLES.cpp

@@ -43,6 +43,7 @@
 #include "GPU/ge_constants.h"
 #include "GPU/GLES/ShaderManagerGLES.h"
 #include "GPU/GLES/DrawEngineGLES.h"
+#include "GPU/Common/ShaderUniforms.h"
 #include "FramebufferManagerGLES.h"
 
 Shader::Shader(GLRenderManager *render, const char *code, const std::string &desc, uint32_t glShaderType, bool useHWTransform, uint32_t attrMask, uint64_t uniformMask)
@@ -134,6 +135,7 @@ LinkedShader::LinkedShader(GLRenderManager *render, VShaderID VSID, Shader *vs,
 	queries.push_back({ &u_uvscaleoffset, "u_uvscaleoffset" });
 	queries.push_back({ &u_texclamp, "u_texclamp" });
 	queries.push_back({ &u_texclampoff, "u_texclampoff" });
+	queries.push_back({ &u_guardband, "u_guardband" });
 
 	for (int i = 0; i < 4; i++) {
 		static const char * const lightPosNames[4] = { "u_lightpos0", "u_lightpos1", "u_lightpos2", "u_lightpos3", };
@@ -461,6 +463,13 @@ void LinkedShader::UpdateUniforms(u32 vertType, const ShaderID &vsid) {
 	if (dirty & DIRTY_TEXMATRIX) {
 		SetMatrix4x3(render_, &u_texmtx, gstate.tgenMatrix);
 	}
+
+	if (dirty & DIRTY_GUARDBAND) {
+		float gb[4];
+		ComputeGuardband(gb, 0.0f);
+		SetFloatUniform4(render_, &u_guardband, gb);
+	}
+
 	if ((dirty & DIRTY_DEPTHRANGE) && u_depthRange != -1) {
 		// Since depth is [-1, 1] mapping to [minz, maxz], this is easyish.
 		float vpZScale = gstate.getViewportZScale();

GPU/GLES/ShaderManagerGLES.h

@@ -101,6 +101,9 @@ class LinkedShader {
 	int u_texclamp;
 	int u_texclampoff;
 
+	// Clipping
+	int u_guardband;
+
 	// Lighting
 	int u_ambient;
 	int u_matambientalpha;

GPU/GLES/VertexShaderGeneratorGLES.cpp

@@ -68,6 +68,12 @@ enum DoLightComputation {
 	LIGHT_FULL,
 };
 
+static void WriteGuardBand(char *&p) {
+	WRITE(p, "  vec3 projPos = outPos.xyz / outPos.w; \n");
+	WRITE(p, "  if (outPos.w >= u_guardband.z) {\n");
+	WRITE(p, "		if (abs(projPos.x) > u_guardband.x || projPos.y < -u_guardband.y) outPos.w = u_guardband.w;\n");
+	WRITE(p, "  }\n");
+}
 
 // Depth range and viewport
 //
@@ -94,6 +100,7 @@ enum DoLightComputation {
 // TODO: Skip all this if we can actually get a 16-bit depth buffer along with stencil, which
 // is a bit of a rare configuration, although quite common on mobile.
 
+// NOTE: We are skipping the bottom check. This fixes TOCA but I am dubious about it...
 
 void GenerateVertexShader(const VShaderID &id, char *buffer, uint32_t *attrMask, uint64_t *uniformMask) {
 	char *p = buffer;
@@ -257,6 +264,9 @@ void GenerateVertexShader(const VShaderID &id, char *buffer, uint32_t *attrMask,
 		// Add all the uniforms we'll need to transform properly.
 	}
 
+	WRITE(p, "uniform vec4 u_guardband;\n");
+	*uniformMask |= DIRTY_GUARDBAND;
+
 	bool scaleUV = !throughmode && (uvGenMode == GE_TEXMAP_TEXTURE_COORDS || uvGenMode == GE_TEXMAP_UNKNOWN);
 
 	if (useHWTransform) {
@@ -473,15 +483,17 @@ void GenerateVertexShader(const VShaderID &id, char *buffer, uint32_t *attrMask,
 			WRITE(p, "  v_fogdepth = position.w;\n");
 		}
 		if (isModeThrough)	{
-			WRITE(p, "  gl_Position = u_proj_through * vec4(position.xyz, 1.0);\n");
+			WRITE(p, "  vec4 outPos = u_proj_through * vec4(position.xyz, 1.0);\n");
 		} else {
 			// The viewport is used in this case, so need to compensate for that.
 			if (gstate_c.Supports(GPU_ROUND_DEPTH_TO_16BIT)) {
-				WRITE(p, "  gl_Position = depthRoundZVP(u_proj * vec4(position.xyz, 1.0));\n");
+				WRITE(p, "  vec4 outPos = depthRoundZVP(u_proj * vec4(position.xyz, 1.0));\n");
 			} else {
-				WRITE(p, "  gl_Position = u_proj * vec4(position.xyz, 1.0);\n");
+				WRITE(p, "  vec4 outPos = u_proj * vec4(position.xyz, 1.0);\n");
 			}
 		}
+		WriteGuardBand(p);
+		WRITE(p, "  gl_Position = outPos;\n");
 	} else {
 		// Step 1: World Transform / Skinning
 		if (!enableBones) {
@@ -672,10 +684,12 @@ void GenerateVertexShader(const VShaderID &id, char *buffer, uint32_t *attrMask,
 
 		// Final view and projection transforms.
 		if (gstate_c.Supports(GPU_ROUND_DEPTH_TO_16BIT)) {
-			WRITE(p, "  gl_Position = depthRoundZVP(u_proj * viewPos);\n");
+			WRITE(p, "  vec4 outPos = depthRoundZVP(u_proj * viewPos);\n");
 		} else {
-			WRITE(p, "  gl_Position = u_proj * viewPos;\n");
+			WRITE(p, "  vec4 outPos = u_proj * viewPos;\n");
 		}
+		WriteGuardBand(p);
+		WRITE(p, "  gl_Position = outPos;\n");
 
 		// TODO: Declare variables for dots for shade mapping if needed.
 

GPU/GPUCommon.cpp

@@ -185,12 +185,12 @@ const CommonCommandTableEntry commonCommandTable[] = {
 	{ GE_CMD_ANTIALIASENABLE, FLAG_FLUSHBEFOREONCHANGE },
 
 	// Viewport.
-	{ GE_CMD_OFFSETX, FLAG_FLUSHBEFOREONCHANGE, DIRTY_VIEWPORTSCISSOR_STATE },
-	{ GE_CMD_OFFSETY, FLAG_FLUSHBEFOREONCHANGE, DIRTY_VIEWPORTSCISSOR_STATE },
-	{ GE_CMD_VIEWPORTXSCALE, FLAG_FLUSHBEFOREONCHANGE,  DIRTY_FRAMEBUF | DIRTY_TEXTURE_PARAMS | DIRTY_VIEWPORTSCISSOR_STATE },
-	{ GE_CMD_VIEWPORTYSCALE, FLAG_FLUSHBEFOREONCHANGE,  DIRTY_FRAMEBUF | DIRTY_TEXTURE_PARAMS | DIRTY_VIEWPORTSCISSOR_STATE },
-	{ GE_CMD_VIEWPORTXCENTER, FLAG_FLUSHBEFOREONCHANGE, DIRTY_FRAMEBUF | DIRTY_TEXTURE_PARAMS | DIRTY_VIEWPORTSCISSOR_STATE },
-	{ GE_CMD_VIEWPORTYCENTER, FLAG_FLUSHBEFOREONCHANGE, DIRTY_FRAMEBUF | DIRTY_TEXTURE_PARAMS | DIRTY_VIEWPORTSCISSOR_STATE },
+	{ GE_CMD_OFFSETX, FLAG_FLUSHBEFOREONCHANGE, DIRTY_VIEWPORTSCISSOR_STATE | DIRTY_GUARDBAND },
+	{ GE_CMD_OFFSETY, FLAG_FLUSHBEFOREONCHANGE, DIRTY_VIEWPORTSCISSOR_STATE | DIRTY_GUARDBAND },
+	{ GE_CMD_VIEWPORTXSCALE, FLAG_FLUSHBEFOREONCHANGE,  DIRTY_FRAMEBUF | DIRTY_TEXTURE_PARAMS | DIRTY_VIEWPORTSCISSOR_STATE | DIRTY_GUARDBAND },
+	{ GE_CMD_VIEWPORTYSCALE, FLAG_FLUSHBEFOREONCHANGE,  DIRTY_FRAMEBUF | DIRTY_TEXTURE_PARAMS | DIRTY_VIEWPORTSCISSOR_STATE | DIRTY_GUARDBAND },
+	{ GE_CMD_VIEWPORTXCENTER, FLAG_FLUSHBEFOREONCHANGE, DIRTY_FRAMEBUF | DIRTY_TEXTURE_PARAMS | DIRTY_VIEWPORTSCISSOR_STATE | DIRTY_GUARDBAND },
+	{ GE_CMD_VIEWPORTYCENTER, FLAG_FLUSHBEFOREONCHANGE, DIRTY_FRAMEBUF | DIRTY_TEXTURE_PARAMS | DIRTY_VIEWPORTSCISSOR_STATE | DIRTY_GUARDBAND },
 	{ GE_CMD_VIEWPORTZSCALE, FLAG_FLUSHBEFOREONCHANGE,  DIRTY_FRAMEBUF | DIRTY_TEXTURE_PARAMS | DIRTY_DEPTHRANGE | DIRTY_PROJMATRIX | DIRTY_VIEWPORTSCISSOR_STATE },
 	{ GE_CMD_VIEWPORTZCENTER, FLAG_FLUSHBEFOREONCHANGE, DIRTY_FRAMEBUF | DIRTY_TEXTURE_PARAMS | DIRTY_DEPTHRANGE | DIRTY_PROJMATRIX | DIRTY_VIEWPORTSCISSOR_STATE },
 	{ GE_CMD_CLIPENABLE, FLAG_FLUSHBEFOREONCHANGE, DIRTY_VIEWPORTSCISSOR_STATE | DIRTY_RASTER_STATE },