Switch Vulkan pipelines to use promises for synchronization

Slightly more expensive I guess but shouldn't be much of a bottleneck.

Common/GPU/Vulkan/VulkanQueueRunner.cpp

@@ -1182,18 +1182,11 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c
 
 		case VKRRenderCommand::BIND_GRAPHICS_PIPELINE:
 		{
-			VKRGraphicsPipeline *pipeline = c.graphics_pipeline.pipeline;
+			VkPipeline pipeline = c.graphics_pipeline.pipeline->BlockUntilReady();
-			if (pipeline->Pending()) {
+			if (pipeline != lastGraphicsPipeline && pipeline != VK_NULL_HANDLE) {
-				// Stall processing, waiting for the compile queue to catch up.
+				vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
-				std::unique_lock<std::mutex> lock(compileDoneMutex_);
-				while (!pipeline->pipeline) {
-					compileDone_.wait(lock);
-				}
-			}
-			if (pipeline->pipeline != lastGraphicsPipeline && pipeline->pipeline != VK_NULL_HANDLE) {
-				vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline);
 				pipelineLayout = c.pipeline.pipelineLayout;
-				lastGraphicsPipeline = pipeline->pipeline;
+				lastGraphicsPipeline = pipeline;
 				// Reset dynamic state so it gets refreshed with the new pipeline.
 				lastStencilWriteMask = -1;
 				lastStencilCompareMask = -1;
@@ -1204,18 +1197,11 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c
 
 		case VKRRenderCommand::BIND_COMPUTE_PIPELINE:
 		{
-			VKRComputePipeline *pipeline = c.compute_pipeline.pipeline;
+			VkPipeline pipeline = c.graphics_pipeline.pipeline->BlockUntilReady();
-			if (pipeline->Pending()) {
+			if (pipeline != lastComputePipeline && pipeline != VK_NULL_HANDLE) {
-				// Stall processing, waiting for the compile queue to catch up.
+				vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline);
-				std::unique_lock<std::mutex> lock(compileDoneMutex_);
-				while (!pipeline->pipeline) {
-					compileDone_.wait(lock);
-				}
-			}
-			if (pipeline->pipeline != lastComputePipeline && pipeline->pipeline != VK_NULL_HANDLE) {
-				vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline);
 				pipelineLayout = c.pipeline.pipelineLayout;
-				lastComputePipeline = pipeline->pipeline;
+				lastComputePipeline = pipeline;
 			}
 			break;
 		}
@@ -1335,7 +1321,6 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c
 		}
 		default:
 			ERROR_LOG(G3D, "Unimpl queue command");
-			;
 		}
 	}
 	vkCmdEndRenderPass(cmd);

Common/GPU/Vulkan/VulkanQueueRunner.h

@@ -4,7 +4,7 @@
 #include <mutex>
 #include <condition_variable>
 
-
+#include "Common/Thread/Promise.h"
 #include "Common/Data/Collections/Hashmaps.h"
 #include "Common/GPU/Vulkan/VulkanContext.h"
 #include "Common/GPU/Vulkan/VulkanBarrier.h"
@@ -55,11 +55,11 @@ struct VkRenderData {
 			VkPipelineLayout pipelineLayout;
 		} pipeline;
 		struct {
-			VKRGraphicsPipeline *pipeline;
+			Promise<VkPipeline> *pipeline;
 			VkPipelineLayout pipelineLayout;
 		} graphics_pipeline;
 		struct {
-			VKRComputePipeline *pipeline;
+			Promise<VkPipeline> *pipeline;
 			VkPipelineLayout pipelineLayout;
 		} compute_pipeline;
 		struct {

Common/GPU/Vulkan/VulkanRenderManager.cpp

@@ -5,6 +5,7 @@
 
 #include "Common/Log.h"
 #include "Common/StringUtils.h"
+#include "Common/TimeUtil.h"
 
 #include "Common/GPU/Vulkan/VulkanAlloc.h"
 #include "Common/GPU/Vulkan/VulkanContext.h"
@@ -57,23 +58,26 @@ bool VKRGraphicsPipeline::Create(VulkanContext *vulkan) {
 	desc->pipe.pStages = ss;
 	desc->pipe.stageCount = 2;
 
+	double start = time_now_d();
 	VkPipeline vkpipeline;
 	VkResult result = vkCreateGraphicsPipelines(vulkan->GetDevice(), desc->pipelineCache, 1, &desc->pipe, nullptr, &vkpipeline);
 
+	NOTICE_LOG(G3D, "Pipeline creation time: %0.2f ms", (time_now_d() - start) * 1000.0);
+
 	bool success = true;
 	if (result == VK_INCOMPLETE) {
 		// Bad (disallowed by spec) return value seen on Adreno in Burnout :(  Try to ignore?
 		// Would really like to log more here, we could probably attach more info to desc.
 		//
 		// At least create a null placeholder to avoid creating over and over if something is broken.
-		pipeline = VK_NULL_HANDLE;
+		pipeline->Post(VK_NULL_HANDLE);
 		success = false;
 	} else if (result != VK_SUCCESS) {
-		pipeline = VK_NULL_HANDLE;
+		pipeline->Post(VK_NULL_HANDLE);
 		ERROR_LOG(G3D, "Failed creating graphics pipeline! result='%s'", VulkanResultToString(result));
 		success = false;
 	} else {
-		pipeline = vkpipeline;
+		pipeline->Post(vkpipeline);
 	}
 
 	delete desc;
@@ -91,11 +95,11 @@ bool VKRComputePipeline::Create(VulkanContext *vulkan) {
 
 	bool success = true;
 	if (result != VK_SUCCESS) {
-		pipeline = VK_NULL_HANDLE;
+		pipeline->Post(VK_NULL_HANDLE);
 		ERROR_LOG(G3D, "Failed creating compute pipeline! result='%s'", VulkanResultToString(result));
 		success = false;
 	} else {
-		pipeline = vkpipeline;
+		pipeline->Post(vkpipeline);
 	}
 
 	delete desc;
@@ -478,6 +482,8 @@ void VulkanRenderManager::CompileThreadFunc() {
 			break;
 		}
 
+		NOTICE_LOG(G3D, "Compilation thread has %d pipelines to create", (int)toCompile.size());
+
 		// TODO: Here we can sort the pending pipelines by vertex and fragment shaders,
 		// and split up further.
 		// Those with the same pairs of shaders should be on the same thread.

Common/GPU/Vulkan/VulkanRenderManager.h

@@ -144,10 +144,12 @@ struct VKRComputePipelineDesc {
 // Wrapped pipeline, which will later allow for background compilation while emulating the rest of the frame.
 struct VKRGraphicsPipeline {
 	VKRGraphicsPipeline() {
-		pipeline = VK_NULL_HANDLE;
+		pipeline = Promise<VkPipeline>::CreateEmpty();
 	}
+
 	VKRGraphicsPipelineDesc *desc = nullptr;  // While non-zero, is pending and pipeline isn't valid.
-	std::atomic<VkPipeline> pipeline;
+
+	Promise<VkPipeline> *pipeline;
 
 	bool Create(VulkanContext *vulkan);
 	bool Pending() const {
@@ -160,7 +162,7 @@ struct VKRComputePipeline {
 		pipeline = VK_NULL_HANDLE;
 	}
 	VKRComputePipelineDesc *desc = nullptr;
-	std::atomic<VkPipeline> pipeline;
+	Promise<VkPipeline> *pipeline;
 
 	bool Create(VulkanContext *vulkan);
 	bool Pending() const {
@@ -261,7 +263,7 @@ public:
 		_dbg_assert_(curRenderStep_ && curRenderStep_->stepType == VKRStepType::RENDER);
 		_dbg_assert_(pipeline != nullptr);
 		VkRenderData data{ VKRRenderCommand::BIND_GRAPHICS_PIPELINE };
-		data.graphics_pipeline.pipeline = pipeline;
+		data.graphics_pipeline.pipeline = pipeline->pipeline;
 		data.graphics_pipeline.pipelineLayout = pipelineLayout;
 		curPipelineFlags_ |= flags;
 		curRenderStep_->commands.push_back(data);
@@ -271,7 +273,7 @@ public:
 		_dbg_assert_(curRenderStep_ && curRenderStep_->stepType == VKRStepType::RENDER);
 		_dbg_assert_(pipeline != nullptr);
 		VkRenderData data{ VKRRenderCommand::BIND_COMPUTE_PIPELINE };
-		data.compute_pipeline.pipeline = pipeline;
+		data.compute_pipeline.pipeline = pipeline->pipeline;
 		data.compute_pipeline.pipelineLayout = pipelineLayout;
 		curPipelineFlags_ |= flags;
 		curRenderStep_->commands.push_back(data);

Common/Thread/Promise.h

@@ -56,6 +56,13 @@ public:
 		return promise;
 	}
 
+	static Promise<T> *CreateEmpty() {
+		Mailbox<T> *mailbox = new Mailbox<T>();
+		Promise<T> *promise = new Promise<T>();
+		promise->rx_ = mailbox;
+		return promise;
+	}
+
 	~Promise() {
 		// A promise should have been fulfilled before it's destroyed.
 		_assert_(ready_);
@@ -90,6 +97,11 @@ public:
 		}
 	}
 
+	// For outside injection of data, when not using Spawn
+	void Post(T data) {
+		rx_->Send(data);
+	}
+
 private:
 	Promise() {}
 

GPU/Vulkan/PipelineManagerVulkan.cpp

@@ -36,7 +36,7 @@ void PipelineManagerVulkan::Clear() {
 
 	pipelines_.Iterate([&](const VulkanPipelineKey &key, VulkanPipeline *value) {
 		if (value->pipeline) {
-			VkPipeline pipeline = value->pipeline->pipeline;
+			VkPipeline pipeline = value->pipeline->pipeline->BlockUntilReady();
 			vulkan_->Delete().QueueDeletePipeline(pipeline);
 			vulkan_->Delete().QueueCallback([](void *p) {
 				VKRGraphicsPipeline *pipeline = (VKRGraphicsPipeline *)p;