diff --git a/Common/GPU/Vulkan/VulkanQueueRunner.cpp b/Common/GPU/Vulkan/VulkanQueueRunner.cpp index ecc81765db..3f75f6c6d0 100644 --- a/Common/GPU/Vulkan/VulkanQueueRunner.cpp +++ b/Common/GPU/Vulkan/VulkanQueueRunner.cpp @@ -1275,9 +1275,11 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c { VKRGraphicsPipeline *pipeline = c.graphics_pipeline.pipeline; if (!pipeline->pipeline) { - // Late! Compile it. - if (!pipeline->Create(vulkan_)) - break; + // Stall processing, waiting for the compile queue to catch up. + std::unique_lock lock(compileDoneMutex_); + while (!pipeline->pipeline) { + compileDone_.wait(lock); + } } if (pipeline->pipeline != lastGraphicsPipeline) { vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline->pipeline); @@ -1294,9 +1296,11 @@ void VulkanQueueRunner::PerformRenderPass(const VKRStep &step, VkCommandBuffer c { VKRComputePipeline *pipeline = c.compute_pipeline.pipeline; if (!pipeline->pipeline) { - // Late! Compile it. - if (!pipeline->Create(vulkan_)) - break; + // Stall processing, waiting for the compile queue to catch up. + std::unique_lock lock(compileDoneMutex_); + while (!pipeline->pipeline) { + compileDone_.wait(lock); + } } if (pipeline->pipeline != lastComputePipeline) { vkCmdBindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline->pipeline); diff --git a/Common/GPU/Vulkan/VulkanQueueRunner.h b/Common/GPU/Vulkan/VulkanQueueRunner.h index f582a3322d..dac2a2e1d1 100644 --- a/Common/GPU/Vulkan/VulkanQueueRunner.h +++ b/Common/GPU/Vulkan/VulkanQueueRunner.h @@ -1,6 +1,9 @@ #pragma once #include +#include +#include + #include "Common/Data/Collections/Hashmaps.h" #include "Common/GPU/Vulkan/VulkanContext.h" @@ -254,6 +257,15 @@ public: hacksEnabled_ = hacks; } + void NotifyCompileDone() { + compileDone_.notify_all(); + } + + void WaitForCompileNotification() { + std::unique_lock lock(compileDoneMutex_); + compileDone_.wait(lock); + } + private: void InitBackbufferRenderPass(); @@ -302,4 +314,8 @@ private: // TODO: Enable based on compat.ini. uint32_t hacksEnabled_ = 0; + + // Compile done notifications. + std::mutex compileDoneMutex_; + std::condition_variable compileDone_; }; diff --git a/Common/GPU/Vulkan/VulkanRenderManager.cpp b/Common/GPU/Vulkan/VulkanRenderManager.cpp index 475feaa2a0..952857354f 100644 --- a/Common/GPU/Vulkan/VulkanRenderManager.cpp +++ b/Common/GPU/Vulkan/VulkanRenderManager.cpp @@ -27,7 +27,9 @@ bool VKRGraphicsPipeline::Create(VulkanContext *vulkan) { // Already failed to create this one. return false; } + VkPipeline pipeline; VkResult result = vkCreateGraphicsPipelines(vulkan->GetDevice(), desc->pipelineCache, 1, &desc->pipe, nullptr, &pipeline); + this->pipeline = pipeline; delete desc; desc = nullptr; if (result == VK_INCOMPLETE) { @@ -49,7 +51,9 @@ bool VKRComputePipeline::Create(VulkanContext *vulkan) { // Already failed to create this one. return false; } + VkPipeline pipeline; VkResult result = vkCreateComputePipelines(vulkan->GetDevice(), desc->pipelineCache, 1, &desc->pipe, nullptr, &pipeline); + this->pipeline = pipeline; delete desc; desc = nullptr; if (result != VK_SUCCESS) { @@ -326,6 +330,8 @@ bool VulkanRenderManager::CreateBackbuffers() { threadInitFrame_ = vulkan_->GetCurFrame(); INFO_LOG(G3D, "Starting Vulkan submission thread (threadInitFrame_ = %d)", vulkan_->GetCurFrame()); thread_ = std::thread(&VulkanRenderManager::ThreadFunc, this); + INFO_LOG(G3D, "Starting Vulkan compiler thread"); + compileThread_ = std::thread(&VulkanRenderManager::CompileThreadFunc, this); } return true; } @@ -349,6 +355,9 @@ void VulkanRenderManager::StopThread() { } thread_.join(); INFO_LOG(G3D, "Vulkan submission thread joined. Frame=%d", vulkan_->GetCurFrame()); + compileCond_.notify_all(); + compileThread_.join(); + INFO_LOG(G3D, "Vulkan compiler thread joined."); // Eat whatever has been queued up for this frame if anything. Wipe(); @@ -415,6 +424,7 @@ VulkanRenderManager::~VulkanRenderManager() { StopThread(); vulkan_->WaitUntilQueueIdle(); + DrainCompileQueue(); VkDevice device = vulkan_->GetDevice(); vkDestroySemaphore(device, acquireSemaphore_, nullptr); vkDestroySemaphore(device, renderingCompleteSemaphore_, nullptr); @@ -430,6 +440,42 @@ VulkanRenderManager::~VulkanRenderManager() { queueRunner_.DestroyDeviceObjects(); } +void VulkanRenderManager::CompileThreadFunc() { + SetCurrentThreadName("ShaderCompile"); + while (true) { + std::vector toCompile; + { + std::unique_lock lock(compileMutex_); + if (compileQueue_.empty()) { + compileCond_.wait(lock); + } + toCompile = std::move(compileQueue_); + compileQueue_.clear(); + } + if (!run_) { + break; + } + for (auto entry : toCompile) { + switch (entry.type) { + case CompileQueueEntry::Type::GRAPHICS: + entry.graphics->Create(vulkan_); + break; + case CompileQueueEntry::Type::COMPUTE: + entry.compute->Create(vulkan_); + break; + } + } + queueRunner_.NotifyCompileDone(); + } +} + +void VulkanRenderManager::DrainCompileQueue() { + std::unique_lock lock(compileMutex_); + while (!compileQueue_.empty()) { + queueRunner_.WaitForCompileNotification(); + } +} + void VulkanRenderManager::ThreadFunc() { SetCurrentThreadName("RenderMan"); int threadFrame = threadInitFrame_; diff --git a/Common/GPU/Vulkan/VulkanRenderManager.h b/Common/GPU/Vulkan/VulkanRenderManager.h index 7b7a870233..179840edc1 100644 --- a/Common/GPU/Vulkan/VulkanRenderManager.h +++ b/Common/GPU/Vulkan/VulkanRenderManager.h @@ -10,6 +10,7 @@ #include #include #include +#include #include "Common/System/Display.h" #include "Common/GPU/Vulkan/VulkanContext.h" @@ -136,25 +137,45 @@ struct VKRComputePipelineDesc { // Wrapped pipeline, which will later allow for background compilation while emulating the rest of the frame. struct VKRGraphicsPipeline { + VKRGraphicsPipeline() { + pipeline = VK_NULL_HANDLE; + } VKRGraphicsPipelineDesc *desc = nullptr; // While non-zero, is pending and pipeline isn't valid. - VkPipeline pipeline = VK_NULL_HANDLE; + std::atomic pipeline; bool Create(VulkanContext *vulkan); }; struct VKRComputePipeline { + VKRComputePipeline() { + pipeline = VK_NULL_HANDLE; + } VKRComputePipelineDesc *desc = nullptr; - VkPipeline pipeline = VK_NULL_HANDLE; + std::atomic pipeline; bool Create(VulkanContext *vulkan); }; +struct CompileQueueEntry { + CompileQueueEntry(VKRGraphicsPipeline *p) : type(Type::GRAPHICS), graphics(p) {} + CompileQueueEntry(VKRComputePipeline *p) : type(Type::COMPUTE), compute(p) {} + enum class Type { + GRAPHICS, + COMPUTE, + }; + Type type; + VKRGraphicsPipeline *graphics = nullptr; + VKRComputePipeline *compute = nullptr; +}; + class VulkanRenderManager { public: VulkanRenderManager(VulkanContext *vulkan); ~VulkanRenderManager(); void ThreadFunc(); + void CompileThreadFunc(); + void DrainCompileQueue(); // Makes sure that the GPU has caught up enough that we can start writing buffers of this frame again. void BeginFrame(bool enableProfiling); @@ -197,6 +218,20 @@ public: VKRGraphicsPipeline *CreateGraphicsPipeline(VKRGraphicsPipelineDesc *desc) { VKRGraphicsPipeline *pipeline = new VKRGraphicsPipeline(); pipeline->desc = desc; + compileMutex_.lock(); + compileQueue_.push_back(CompileQueueEntry(pipeline)); + compileCond_.notify_one(); + compileMutex_.unlock(); + return pipeline; + } + + VKRComputePipeline *CreateComputePipeline(VKRComputePipelineDesc *desc) { + VKRComputePipeline *pipeline = new VKRComputePipeline(); + pipeline->desc = desc; + compileMutex_.lock(); + compileQueue_.push_back(CompileQueueEntry(pipeline)); + compileCond_.notify_one(); + compileMutex_.unlock(); return pipeline; } @@ -487,6 +522,14 @@ private: int threadInitFrame_ = 0; VulkanQueueRunner queueRunner_; + // Shader compilation thread to compile while emulating the rest of the frame. + // Only one right now but we could use more. + std::thread compileThread_; + // Sync + std::condition_variable compileCond_; + std::mutex compileMutex_; + std::vector compileQueue_; + // Swap chain management struct SwapchainImageData { VkImage image; diff --git a/GPU/Vulkan/PipelineManagerVulkan.cpp b/GPU/Vulkan/PipelineManagerVulkan.cpp index e2723ea028..27e34dd3c7 100644 --- a/GPU/Vulkan/PipelineManagerVulkan.cpp +++ b/GPU/Vulkan/PipelineManagerVulkan.cpp @@ -35,8 +35,12 @@ void PipelineManagerVulkan::Clear() { pipelines_.Iterate([&](const VulkanPipelineKey &key, VulkanPipeline *value) { if (value->pipeline) { - vulkan_->Delete().QueueDeletePipeline(value->pipeline->pipeline); + VkPipeline pipeline = value->pipeline->pipeline; + vulkan_->Delete().QueueDeletePipeline(pipeline); delete value->pipeline; + } else { + // Something went wrong. + ERROR_LOG(G3D, "Null pipeline found in PipelineManagerVulkan::Clear - didn't wait for asyncs?"); } delete value; });