From 9b46adb98501d321e43dc67761e38988da96e2f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20Rydg=C3=A5rd?= <hrydgard@gmail.com>
Date: Mon, 26 Sep 2022 22:51:16 +0200
Subject: [PATCH] Fix color test.

Fixes the new color test bug reported in #13324, though doesn't fix that
issue (didn't confirm it still is one).
---
 Common/GPU/Vulkan/VulkanFrameData.h    | 16 ++++++++--------
 GPU/Common/FragmentShaderGenerator.cpp |  7 +++++--
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/Common/GPU/Vulkan/VulkanFrameData.h b/Common/GPU/Vulkan/VulkanFrameData.h
index 9fdd865ebf..049f8d765c 100644
--- a/Common/GPU/Vulkan/VulkanFrameData.h
+++ b/Common/GPU/Vulkan/VulkanFrameData.h
@@ -48,16 +48,16 @@ struct FrameData {
 	std::condition_variable fenceCondVar;
 	bool readyForFence = true;
 
-	VkFence fence;
-	VkFence readbackFence;  // Strictly speaking we might only need one global of these.
+	VkFence fence = VK_NULL_HANDLE;
+	VkFence readbackFence = VK_NULL_HANDLE;  // Strictly speaking we might only need one global of these.
 
 	// These are on different threads so need separate pools.
-	VkCommandPool cmdPoolInit;  // Written to from main thread
-	VkCommandPool cmdPoolMain;  // Written to from render thread, which also submits
+	VkCommandPool cmdPoolInit = VK_NULL_HANDLE;  // Written to from main thread
+	VkCommandPool cmdPoolMain = VK_NULL_HANDLE;  // Written to from render thread, which also submits
 
-	VkCommandBuffer initCmd;
-	VkCommandBuffer mainCmd;
-	VkCommandBuffer presentCmd;
+	VkCommandBuffer initCmd = VK_NULL_HANDLE;
+	VkCommandBuffer mainCmd = VK_NULL_HANDLE;
+	VkCommandBuffer presentCmd = VK_NULL_HANDLE;
 
 	bool hasInitCommands = false;
 	bool hasMainCommands = false;
@@ -73,7 +73,7 @@ struct FrameData {
 
 	// Profiling.
 	QueueProfileContext profile;
-	bool profilingEnabled_;
+	bool profilingEnabled_ = false;
 
 	void Init(VulkanContext *vulkan, int index);
 	void Destroy(VulkanContext *vulkan);
diff --git a/GPU/Common/FragmentShaderGenerator.cpp b/GPU/Common/FragmentShaderGenerator.cpp
index 297cccfb4b..5cf923be46 100644
--- a/GPU/Common/FragmentShaderGenerator.cpp
+++ b/GPU/Common/FragmentShaderGenerator.cpp
@@ -184,6 +184,7 @@ bool GenerateFragmentShader(const FShaderID &id, char *buffer, const ShaderLangu
 		}
 		if (enableColorTest && !colorTestAgainstZero) {
 			WRITE(p, "uint roundAndScaleTo8x4(in highp vec3 x) { uvec3 u = uvec3(floor(x * 255.0 + 0.5)); return u.r | (u.g << 8) | (u.b << 16); }\n");
+			WRITE(p, "uint packFloatsTo8x4(in vec3 x) { uvec3 u = uvec3(x); return u.r | (u.g << 8) | (u.b << 16); }\n");
 		}
 
 		WRITE(p, "layout (location = 0, index = 0) out vec4 fragColor0;\n");
@@ -263,6 +264,7 @@ bool GenerateFragmentShader(const FShaderID &id, char *buffer, const ShaderLangu
 		if (enableColorTest) {
 			if (compat.shaderLanguage == HLSL_D3D11) {
 				WRITE(p, "uint roundAndScaleTo8x4(float3 x) { uvec3 u = (floor(x * 255.0f + 0.5f)); return u.r | (u.g << 8) | (u.b << 16); }\n");
+				WRITE(p, "uint packFloatsTo8x4(in vec3 x) { uvec3 u = uvec3(x); return u.r | (u.g << 8) | (u.b << 16); }\n");
 			} else {
 				WRITE(p, "vec3 roundAndScaleTo255v(float3 x) { return floor(x * 255.0f + 0.5f); }\n");
 			}
@@ -408,7 +410,8 @@ bool GenerateFragmentShader(const FShaderID &id, char *buffer, const ShaderLangu
 			}
 			if (enableColorTest && !colorTestAgainstZero) {
 				if (compat.bitwiseOps) {
-					WRITE(p, "uint roundAndScaleTo8x4(in vec3 x) { uvec3 u = uvec3(floor(x * 255.0 + 0.5)); return u.r | (u.g << 8) | (u.b << 16); }\n");
+					WRITE(p, "uint roundAndScaleTo8x4(in vec3 x) { uvec3 u = uvec3(floor(x * 255.99)); return u.r | (u.g << 8) | (u.b << 16); }\n");
+					WRITE(p, "uint packFloatsTo8x4(in vec3 x) { uvec3 u = uvec3(x); return u.r | (u.g << 8) | (u.b << 16); }\n");
 				} else if (gl_extensions.gpuVendor == GPU_VENDOR_IMGTEC) {
 					WRITE(p, "vec3 roundTo255thv(in vec3 x) { vec3 y = x + (0.5/255.0); return y - fract(y * 255.0) * (1.0 / 255.0); }\n");
 				} else {
@@ -943,7 +946,7 @@ bool GenerateFragmentShader(const FShaderID &id, char *buffer, const ShaderLangu
 					} else if (compat.bitwiseOps) {
 						WRITE(p, "  uint v_uint = roundAndScaleTo8x4(v.rgb);\n");
 						WRITE(p, "  uint v_masked = v_uint & u_alphacolormask;\n");
-						WRITE(p, "  uint colorTestRef = roundAndScaleTo8x4(u_alphacolorref.rgb) & u_alphacolormask;\n");
+						WRITE(p, "  uint colorTestRef = packFloatsTo8x4(u_alphacolorref.rgb) & u_alphacolormask;\n");
 						WRITE(p, "  if (v_masked %s colorTestRef) %s\n", test, discardStatement);
 					} else if (gl_extensions.gpuVendor == GPU_VENDOR_IMGTEC) {
 						WRITE(p, "  if (roundTo255thv(v.rgb) %s u_alphacolorref.rgb) %s\n", test, discardStatement);