Vulkan: Add a single background thread for pipeline creation

Add proper waits for compile-done

Don't rely on non-standard struct initialization of classes

Attempt to drain the pipeline compile queue before destroying the PipelineManager.

Vulkan: Bump the cache version for testing

Common/GPU/Vulkan/VulkanQueueRunner.cpp

@@ -1275,9 +1275,11 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c
 		{
 			VKRGraphicsPipeline *pipeline = c.graphics_pipeline.pipeline;
 			if (!pipeline->pipeline) {
-				// Late! Compile it.
+				// Stall processing, waiting for the compile queue to catch up.
-				if (!pipeline->Create(vulkan_))
+				std::unique_lock<std::mutex> lock(compileDoneMutex_);
-					break;
+				while (!pipeline->pipeline) {
+					compileDone_.wait(lock);
+				}
 			}
 			if (pipeline->pipeline != lastGraphicsPipeline) {
 				vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline);
@@ -1294,9 +1296,11 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c
 		{
 			VKRComputePipeline *pipeline = c.compute_pipeline.pipeline;
 			if (!pipeline->pipeline) {
-				// Late! Compile it.
+				// Stall processing, waiting for the compile queue to catch up.
-				if (!pipeline->Create(vulkan_))
+				std::unique_lock<std::mutex> lock(compileDoneMutex_);
-					break;
+				while (!pipeline->pipeline) {
+					compileDone_.wait(lock);
+				}
 			}
 			if (pipeline->pipeline != lastComputePipeline) {
 				vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline);

Common/GPU/Vulkan/VulkanQueueRunner.h

@@ -1,6 +1,9 @@
 #pragma once
 
 #include <cstdint>
+#include <mutex>
+#include <condition_variable>
+
 
 #include "Common/Data/Collections/Hashmaps.h"
 #include "Common/GPU/Vulkan/VulkanContext.h"
@@ -254,6 +257,15 @@ public:
 		hacksEnabled_ = hacks;
 	}
 
+	void NotifyCompileDone() {
+		compileDone_.notify_all();
+	}
+
+	void WaitForCompileNotification() {
+		std::unique_lock<std::mutex> lock(compileDoneMutex_);
+		compileDone_.wait(lock);
+	}
+
 private:
 	void InitBackbufferRenderPass();
 
@@ -302,4 +314,8 @@ private:
 
 	// TODO: Enable based on compat.ini.
 	uint32_t hacksEnabled_ = 0;
+
+	// Compile done notifications.
+	std::mutex compileDoneMutex_;
+	std::condition_variable compileDone_;
 };

Common/GPU/Vulkan/VulkanRenderManager.cpp

@@ -27,7 +27,9 @@ bool VKRGraphicsPipeline::Create(VulkanContext *vulkan) {
 		// Already failed to create this one.
 		return false;
 	}
+	VkPipeline pipeline;
 	VkResult result = vkCreateGraphicsPipelines(vulkan->GetDevice(), desc->pipelineCache, 1, &desc->pipe, nullptr, &pipeline);
+	this->pipeline = pipeline;
 	delete desc;
 	desc = nullptr;
 	if (result == VK_INCOMPLETE) {
@@ -49,7 +51,9 @@ bool VKRComputePipeline::Create(VulkanContext *vulkan) {
 		// Already failed to create this one.
 		return false;
 	}
+	VkPipeline pipeline;
 	VkResult result = vkCreateComputePipelines(vulkan->GetDevice(), desc->pipelineCache, 1, &desc->pipe, nullptr, &pipeline);
+	this->pipeline = pipeline;
 	delete desc;
 	desc = nullptr;
 	if (result != VK_SUCCESS) {
@@ -326,6 +330,8 @@ bool VulkanRenderManager::CreateBackbuffers() {
 		threadInitFrame_ = vulkan_->GetCurFrame();
 		INFO_LOG(G3D, "Starting Vulkan submission thread (threadInitFrame_ = %d)", vulkan_->GetCurFrame());
 		thread_ = std::thread(&VulkanRenderManager::ThreadFunc, this);
+		INFO_LOG(G3D, "Starting Vulkan compiler thread");
+		compileThread_ = std::thread(&VulkanRenderManager::CompileThreadFunc, this);
 	}
 	return true;
 }
@@ -349,6 +355,9 @@ void VulkanRenderManager::StopThread() {
 		}
 		thread_.join();
 		INFO_LOG(G3D, "Vulkan submission thread joined. Frame=%d", vulkan_->GetCurFrame());
+		compileCond_.notify_all();
+		compileThread_.join();
+		INFO_LOG(G3D, "Vulkan compiler thread joined.");
 
 		// Eat whatever has been queued up for this frame if anything.
 		Wipe();
@@ -415,6 +424,7 @@ VulkanRenderManager::~VulkanRenderManager() {
 	StopThread();
 	vulkan_->WaitUntilQueueIdle();
 
+	DrainCompileQueue();
 	VkDevice device = vulkan_->GetDevice();
 	vkDestroySemaphore(device, acquireSemaphore_, nullptr);
 	vkDestroySemaphore(device, renderingCompleteSemaphore_, nullptr);
@@ -430,6 +440,42 @@ VulkanRenderManager::~VulkanRenderManager() {
 	queueRunner_.DestroyDeviceObjects();
 }
 
+void VulkanRenderManager::CompileThreadFunc() {
+	SetCurrentThreadName("ShaderCompile");
+	while (true) {
+		std::vector<CompileQueueEntry> toCompile;
+		{
+			std::unique_lock<std::mutex> lock(compileMutex_);
+			if (compileQueue_.empty()) {
+				compileCond_.wait(lock);
+			}
+			toCompile = std::move(compileQueue_);
+			compileQueue_.clear();
+		}
+		if (!run_) {
+			break;
+		}
+		for (auto entry : toCompile) {
+			switch (entry.type) {
+			case CompileQueueEntry::Type::GRAPHICS:
+				entry.graphics->Create(vulkan_);
+				break;
+			case CompileQueueEntry::Type::COMPUTE:
+				entry.compute->Create(vulkan_);
+				break;
+			}
+		}
+		queueRunner_.NotifyCompileDone();
+	}
+}
+
+void VulkanRenderManager::DrainCompileQueue() {
+	std::unique_lock<std::mutex> lock(compileMutex_);
+	while (!compileQueue_.empty()) {
+		queueRunner_.WaitForCompileNotification();
+	}
+}
+
 void VulkanRenderManager::ThreadFunc() {
 	SetCurrentThreadName("RenderMan");
 	int threadFrame = threadInitFrame_;

Common/GPU/Vulkan/VulkanRenderManager.h

@@ -10,6 +10,7 @@
 #include <cstdint>
 #include <mutex>
 #include <thread>
+#include <queue>
 
 #include "Common/System/Display.h"
 #include "Common/GPU/Vulkan/VulkanContext.h"
@@ -136,25 +137,45 @@ struct VKRComputePipelineDesc {
 
 // Wrapped pipeline, which will later allow for background compilation while emulating the rest of the frame.
 struct VKRGraphicsPipeline {
+	VKRGraphicsPipeline() {
+		pipeline = VK_NULL_HANDLE;
+	}
 	VKRGraphicsPipelineDesc *desc = nullptr;  // While non-zero, is pending and pipeline isn't valid.
-	VkPipeline pipeline = VK_NULL_HANDLE;
+	std::atomic<VkPipeline> pipeline;
 
 	bool Create(VulkanContext *vulkan);
 };
 
 struct VKRComputePipeline {
+	VKRComputePipeline() {
+		pipeline = VK_NULL_HANDLE;
+	}
 	VKRComputePipelineDesc *desc = nullptr;
-	VkPipeline pipeline = VK_NULL_HANDLE;
+	std::atomic<VkPipeline> pipeline;
 
 	bool Create(VulkanContext *vulkan);
 };
 
+struct CompileQueueEntry {
+	CompileQueueEntry(VKRGraphicsPipeline *p) : type(Type::GRAPHICS), graphics(p) {}
+	CompileQueueEntry(VKRComputePipeline *p) : type(Type::COMPUTE), compute(p) {}
+	enum class Type {
+		GRAPHICS,
+		COMPUTE,
+	};
+	Type type;
+	VKRGraphicsPipeline *graphics = nullptr;
+	VKRComputePipeline *compute = nullptr;
+};
+
 class VulkanRenderManager {
 public:
 	VulkanRenderManager(VulkanContext *vulkan);
 	~VulkanRenderManager();
 
 	void ThreadFunc();
+	void CompileThreadFunc();
+	void DrainCompileQueue();
 
 	// Makes sure that the GPU has caught up enough that we can start writing buffers of this frame again.
 	void BeginFrame(bool enableProfiling);
@@ -197,6 +218,20 @@ public:
 	VKRGraphicsPipeline *CreateGraphicsPipeline(VKRGraphicsPipelineDesc *desc) {
 		VKRGraphicsPipeline *pipeline = new VKRGraphicsPipeline();
 		pipeline->desc = desc;
+		compileMutex_.lock();
+		compileQueue_.push_back(CompileQueueEntry(pipeline));
+		compileCond_.notify_one();
+		compileMutex_.unlock();
+		return pipeline;
+	}
+
+	VKRComputePipeline *CreateComputePipeline(VKRComputePipelineDesc *desc) {
+		VKRComputePipeline *pipeline = new VKRComputePipeline();
+		pipeline->desc = desc;
+		compileMutex_.lock();
+		compileQueue_.push_back(CompileQueueEntry(pipeline));
+		compileCond_.notify_one();
+		compileMutex_.unlock();
 		return pipeline;
 	}
 
@@ -487,6 +522,14 @@ private:
 	int threadInitFrame_ = 0;
 	VulkanQueueRunner queueRunner_;
 
+	// Shader compilation thread to compile while emulating the rest of the frame.
+	// Only one right now but we could use more.
+	std::thread compileThread_;
+	// Sync
+	std::condition_variable compileCond_;
+	std::mutex compileMutex_;
+	std::vector<CompileQueueEntry> compileQueue_;
+
 	// Swap chain management
 	struct SwapchainImageData {
 		VkImage image;

GPU/Vulkan/PipelineManagerVulkan.cpp

@@ -35,8 +35,12 @@ void PipelineManagerVulkan::Clear() {
 
 	pipelines_.Iterate([&](const VulkanPipelineKey &key, VulkanPipeline *value) {
 		if (value->pipeline) {
-			vulkan_->Delete().QueueDeletePipeline(value->pipeline->pipeline);
+			VkPipeline pipeline = value->pipeline->pipeline;
+			vulkan_->Delete().QueueDeletePipeline(pipeline);
 			delete value->pipeline;
+		} else {
+			// Something went wrong.
+			ERROR_LOG(G3D, "Null pipeline found in PipelineManagerVulkan::Clear - didn't wait for asyncs?");
 		}
 		delete value;
 	});