GPU: Avoid stencil emulation if possible.

This reduces use of replaceAlpha, and reduces use of more complicated
blend states.  This simplifies fragment shaders a little.

GPU/Common/GPUStateUtils.cpp

@@ -40,6 +40,17 @@ bool CanUseHardwareTransform(int prim) {
 	return !gstate.isModeThrough() && prim != GE_PRIM_RECTANGLES;
 }
 
+bool IsStencilTestOutputDisabled() {
+	// The mask applies on all stencil ops.
+	if (gstate.isStencilTestEnabled() && (gstate.pmska & 0xFF) != 0xFF) {
+		if (gstate.FrameBufFormat() == GE_FORMAT_565) {
+			return true;
+		}
+		return gstate.getStencilOpZPass() == GE_STENCILOP_KEEP && gstate.getStencilOpZFail() == GE_STENCILOP_KEEP && gstate.getStencilOpSFail() == GE_STENCILOP_KEEP;
+	}
+	return true;
+}
+
 bool NeedsTestDiscard() {
 	// We assume this is called only when enabled and not trivially true (may also be for color testing.)
 	if (gstate.isStencilTestEnabled() && (gstate.pmska & 0xFF) != 0xFF)
@@ -169,7 +180,7 @@ const bool nonAlphaDestFactors[16] = {
 };
 
 ReplaceAlphaType ReplaceAlphaWithStencil(ReplaceBlendType replaceBlend) {
-	if (!gstate.isStencilTestEnabled() || gstate.isModeClear()) {
+	if (IsStencilTestOutputDisabled() || gstate.isModeClear()) {
 		return REPLACE_ALPHA_NO;
 	}
 
@@ -995,7 +1006,7 @@ void ConvertBlendState(GenericBlendState &blendState, bool allowShaderBlend) {
 
 	int constantAlpha = 255;
 	BlendFactor constantAlphaGL = BlendFactor::ONE;
-	if (gstate.isStencilTestEnabled() && replaceAlphaWithStencil == REPLACE_ALPHA_NO) {
+	if (!IsStencilTestOutputDisabled() && replaceAlphaWithStencil == REPLACE_ALPHA_NO) {
 		switch (ReplaceAlphaWithStencilType()) {
 		case STENCIL_VALUE_UNIFORM:
 			constantAlpha = gstate.getStencilTestRef();
@@ -1163,7 +1174,7 @@ void ConvertBlendState(GenericBlendState &blendState, bool allowShaderBlend) {
 			blendState.setFactors(glBlendFuncA, glBlendFuncB, BlendFactor::ONE, BlendFactor::ZERO);
 			break;
 		}
-	} else if (gstate.isStencilTestEnabled()) {
+	} else if (!IsStencilTestOutputDisabled()) {
 		switch (ReplaceAlphaWithStencilType()) {
 		case STENCIL_VALUE_KEEP:
 			blendState.setFactors(glBlendFuncA, glBlendFuncB, BlendFactor::ZERO, BlendFactor::ONE);

GPU/Common/GPUStateUtils.h

@@ -45,6 +45,7 @@ bool IsColorTestAgainstZero();
 bool IsColorTestTriviallyTrue();
 bool IsAlphaTestAgainstZero();
 bool NeedsTestDiscard();
+bool IsStencilTestOutputDisabled();
 
 StencilValueType ReplaceAlphaWithStencilType();
 ReplaceAlphaType ReplaceAlphaWithStencil(ReplaceBlendType replaceBlend);

GPU/D3D11/StateMappingD3D11.cpp

@@ -223,7 +223,7 @@ void DrawEngineD3D11::ApplyDrawState(int prim) {
 #endif
 
 			// Let's not write to alpha if stencil isn't enabled.
-			if (!gstate.isStencilTestEnabled()) {
+			if (IsStencilTestOutputDisabled()) {
 				amask = false;
 			} else {
 				// If the stencil type is set to KEEP, we shouldn't write to the stencil/alpha channel.

GPU/Directx9/StateMappingDX9.cpp

@@ -181,7 +181,7 @@ void DrawEngineDX9::ApplyDrawState(int prim) {
 #endif
 
 			// Let's not write to alpha if stencil isn't enabled.
-			if (!gstate.isStencilTestEnabled()) {
+			if (IsStencilTestOutputDisabled()) {
 				amask = false;
 			} else {
 				// If the stencil type is set to KEEP, we shouldn't write to the stencil/alpha channel.

GPU/GLES/StateMappingGLES.cpp

@@ -143,7 +143,7 @@ void DrawEngineGLES::ApplyDrawState(int prim) {
 	// amask is needed for both stencil and blend state so we keep it outside for now
 	bool amask = (gstate.pmska & 0xFF) < 128;
 	// Let's not write to alpha if stencil isn't enabled.
-	if (!gstate.isStencilTestEnabled()) {
+	if (IsStencilTestOutputDisabled()) {
 		amask = false;
 	} else {
 		// If the stencil type is set to KEEP, we shouldn't write to the stencil/alpha channel.

GPU/Vulkan/StateMappingVulkan.cpp

@@ -225,7 +225,7 @@ void DrawEngineVulkan::ConvertStateToVulkanKey(FramebufferManagerVulkan &fbManag
 #endif
 
 			// Let's not write to alpha if stencil isn't enabled.
-			if (!gstate.isStencilTestEnabled()) {
+			if (IsStencilTestOutputDisabled()) {
 				amask = false;
 			} else {
 				// If the stencil type is set to KEEP, we shouldn't write to the stencil/alpha channel.