Mask away alpha when the stencil operation is KEEP

GPU/GLES/FragmentShaderGenerator.cpp

@@ -99,14 +99,7 @@ static bool IsAlphaTestTriviallyTrue() {
 	}
 }
 
-enum StencilValueType {
-	STENCIL_VALUE_UNKNOWN,
-	STENCIL_VALUE_UNIFORM,
-	STENCIL_VALUE_ZERO,
-	STENCIL_VALUE_ONE,
-};
-
-static StencilValueType ReplaceAlphaWithStencilType() {
+StencilValueType ReplaceAlphaWithStencilType() {
 	switch (gstate.FrameBufFormat()) {
 	case GE_FORMAT_565:
 		// There's never a stencil value.  Maybe the right alpha is 1?
@@ -127,9 +120,11 @@ static StencilValueType ReplaceAlphaWithStencilType() {
 		case GE_STENCILOP_INCR:
 			return STENCIL_VALUE_ONE;
 
-		case GE_STENCILOP_KEEP:
 		case GE_STENCILOP_INVERT:
 			return STENCIL_VALUE_UNKNOWN;
+
+		case GE_STENCILOP_KEEP:
+			return STENCIL_VALUE_KEEP;
 		}
 		break;
 
@@ -146,9 +141,10 @@ static StencilValueType ReplaceAlphaWithStencilType() {
 		// Decrementing always zeros, since there's only one bit.
 		case GE_STENCILOP_DECR:
 		case GE_STENCILOP_INCR:
-		case GE_STENCILOP_KEEP:
 		case GE_STENCILOP_INVERT:
 			return STENCIL_VALUE_UNKNOWN;
+		case GE_STENCILOP_KEEP:
+			return STENCIL_VALUE_KEEP;
 		}
 		break;
 	}
@@ -415,7 +411,7 @@ void GenerateFragmentShader(char *buffer) {
 		} else if (enableAlphaDoubling) {
 			WRITE(p, "  v.a = v.a * 2.0;\n");
 		}
-		
+
 		if (enableColorTest) {
 			GEComparison colorTestFunc = gstate.getColorTestFunction();
 			const char *colorTestFuncs[] = { "#", "#", " != ", " == " };	// never/always don't make sense
@@ -457,6 +453,10 @@ void GenerateFragmentShader(char *buffer) {
 			// if the stencil mode is KEEP for example.
 			WRITE(p, "  gl_FragColor.a = 0.0;\n");
 			break;
+
+		case STENCIL_VALUE_KEEP:
+			// Do nothing. We'll mask out the alpha using color mask.
+			break;
 		}
 	}
 #ifdef DEBUG_SHADER

GPU/GLES/FragmentShaderGenerator.h

@@ -44,3 +44,13 @@ struct FragmentShaderID {
 void ComputeFragmentShaderID(FragmentShaderID *id);
 void GenerateFragmentShader(char *buffer);
 
+enum StencilValueType {
+	STENCIL_VALUE_UNKNOWN,
+	STENCIL_VALUE_UNIFORM,
+	STENCIL_VALUE_ZERO,
+	STENCIL_VALUE_ONE,
+	STENCIL_VALUE_KEEP,
+};
+
+StencilValueType ReplaceAlphaWithStencilType();
+

GPU/GLES/StateMapping.cpp

@@ -33,6 +33,7 @@
 #include "GPU/GLES/ShaderManager.h"
 #include "GPU/GLES/TextureCache.h"
 #include "GPU/GLES/Framebuffer.h"
+#include "GPU/GLES/FragmentShaderGenerator.h"
 
 static const GLushort aLookup[11] = {
 	GL_DST_COLOR,
@@ -275,14 +276,13 @@ void TransformDrawEngine::ApplyDrawState(int prim) {
 		glstate.dither.disable();
 
 	if (gstate.isModeClear()) {
-
 #if !defined(USING_GLES2)
 		// Logic Ops
 		glstate.colorLogicOp.disable();
 #endif
-		// Culling 
+		// Culling
 		glstate.cullFace.disable();
-		
+
 		// Depth Test
 		glstate.depthTest.enable();
 		glstate.depthFunc.set(GL_ALWAYS);
@@ -343,6 +343,11 @@ void TransformDrawEngine::ApplyDrawState(int prim) {
 		// Let's not write to alpha if stencil isn't enabled.
 		if (!gstate.isStencilTestEnabled()) {
 			amask = false;
+		} else {
+			// If the stencil value is set to KEEP, we shouldn't write to the stencil/alpha channel.1
+			if (ReplaceAlphaWithStencilType() == STENCIL_VALUE_KEEP) {
+				amask = false;
+			}
 		}
 
 		glstate.colorMask.set(rmask, gmask, bmask, amask);
@@ -356,9 +361,9 @@ void TransformDrawEngine::ApplyDrawState(int prim) {
 			glstate.stencilOp.set(stencilOps[gstate.getStencilOpSFail()],  // stencil fail
 				stencilOps[gstate.getStencilOpZFail()],  // depth fail
 				stencilOps[gstate.getStencilOpZPass()]); // depth pass
-		} else 
+		} else {
 			glstate.stencilTest.disable();
-		
+		}
 	}
 
 	float renderWidthFactor, renderHeightFactor;
@@ -376,7 +381,7 @@ void TransformDrawEngine::ApplyDrawState(int prim) {
 		float pixelH = PSP_CoreParameter().pixelHeight;
 		CenterRect(&renderX, &renderY, &renderWidth, &renderHeight, 480, 272, pixelW, pixelH);
 	}
-	
+
 	renderWidthFactor = (float)renderWidth / framebufferManager_->GetTargetWidth();
 	renderHeightFactor = (float)renderHeight / framebufferManager_->GetTargetHeight();