Emulation/n64-emu
1
0
Fork 0
mirror of https://github.com/kmc-jp/n64-emu.git synced 2025-04-02 10:21:43 -04:00
Code Issues Projects Releases Packages Wiki Activity
n64-emu/third_party/parallel-rdp-standalone/parallel-rdp/rdp_renderer.cpp
Raiki Tamura 2be470e46b Add parallel-rdp-standalone
2023-08-06 14:03:29 +09:00

3578 lines
116 KiB
C++
Raw Permalink Blame History

/* Copyright (c) 2020 Themaister
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#define NOMINMAX
#include "rdp_renderer.hpp"
#include "rdp_device.hpp"
#include "logging.hpp"
#include "bitops.hpp"
#include "luts.hpp"
#include "timer.hpp"
#include <limits>
#include <stdlib.h>
#ifdef PARALLEL_RDP_SHADER_DIR
#include "global_managers.hpp"
#include "os_filesystem.hpp"
#else
#include "shaders/slangmosh.hpp"
#endif
namespace RDP
{
Renderer::Renderer(CommandProcessor &processor_)
: processor(processor_)
{
active_submissions = 0;
}
Renderer::~Renderer()
{
}
void Renderer::set_shader_bank(const ShaderBank *bank)
{
shader_bank = bank;
}
bool Renderer::init_renderer(const RendererOptions &options)
{
if (options.upscaling_factor == 0)
return false;
if (options.upscaling_factor == 1 && options.super_sampled_readback)
return false;
caps.max_width = options.upscaling_factor * Limits::MaxWidth;
caps.max_height = options.upscaling_factor * Limits::MaxHeight;
caps.max_tiles_x = options.upscaling_factor * ImplementationConstants::MaxTilesX;
caps.max_tiles_y = options.upscaling_factor * ImplementationConstants::MaxTilesY;
caps.max_num_tile_instances = options.upscaling_factor * options.upscaling_factor * Limits::MaxTileInstances;
#ifdef PARALLEL_RDP_SHADER_DIR
pipeline_worker.reset(new WorkerThread<Vulkan::DeferredPipelineCompile, PipelineExecutor>(
Granite::Global::create_thread_context(), { device }));
#else
pipeline_worker.reset(new WorkerThread<Vulkan::DeferredPipelineCompile, PipelineExecutor>({ device }));
#endif
#ifdef PARALLEL_RDP_SHADER_DIR
if (!GRANITE_FILESYSTEM()->get_backend("rdp"))
GRANITE_FILESYSTEM()->register_protocol("rdp", std::make_unique<Granite::OSFilesystem>(PARALLEL_RDP_SHADER_DIR));
device->get_shader_manager().add_include_directory("builtin://shaders/inc");
#endif
for (auto &buffer : buffer_instances)
buffer.init(*device);
if (const char *env = getenv("RDP_DEBUG"))
debug_channel = strtoul(env, nullptr, 0) != 0;
if (const char *env = getenv("RDP_DEBUG_X"))
filter_debug_channel_x = strtol(env, nullptr, 0);
if (const char *env = getenv("RDP_DEBUG_Y"))
filter_debug_channel_y = strtol(env, nullptr, 0);
{
Vulkan::BufferCreateInfo info = {};
info.size = Limits::MaxTMEMInstances * 0x1000;
info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
info.domain = Vulkan::BufferDomain::Device;
info.misc = Vulkan::BUFFER_MISC_ZERO_INITIALIZE_BIT;
tmem_instances = device->create_buffer(info);
device->set_name(*tmem_instances, "tmem-instances");
stream.tmem_upload_infos.reserve(Limits::MaxTMEMInstances);
}
{
Vulkan::BufferCreateInfo info = {};
info.size = options.upscaling_factor * Limits::MaxSpanSetups * sizeof(SpanSetup);
info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
info.domain = Vulkan::BufferDomain::Device;
info.misc = Vulkan::BUFFER_MISC_ZERO_INITIALIZE_BIT;
span_setups = device->create_buffer(info);
device->set_name(*span_setups, "span-setups");
}
init_blender_lut();
init_buffers(options);
if (options.upscaling_factor > 1 && !init_internal_upscaling_factor(options))
return false;
return init_caps();
}
void Renderer::set_device(Vulkan::Device *device_)
{
device = device_;
}
void Renderer::set_validation_interface(ValidationInterface *iface)
{
validation_iface = iface;
}
bool Renderer::init_caps()
{
auto &features = device->get_device_features();
if (const char *timestamp = getenv("PARALLEL_RDP_BENCH"))
{
caps.timestamp = strtol(timestamp, nullptr, 0);
LOGI("Enabling timestamps = %d\n", caps.timestamp);
}
if (const char *ubershader = getenv("PARALLEL_RDP_UBERSHADER"))
{
caps.ubershader = strtol(ubershader, nullptr, 0) > 0;
LOGI("Overriding ubershader = %d\n", int(caps.ubershader));
}
if (const char *force_sync = getenv("PARALLEL_RDP_FORCE_SYNC_SHADER"))
{
caps.force_sync = strtol(force_sync, nullptr, 0) > 0;
LOGI("Overriding force sync shader = %d\n", int(caps.force_sync));
}
bool allow_subgroup = true;
if (const char *subgroup = getenv("PARALLEL_RDP_SUBGROUP"))
{
allow_subgroup = strtol(subgroup, nullptr, 0) > 0;
LOGI("Allow subgroups = %d\n", int(allow_subgroup));
}
bool allow_small_types = true;
bool forces_small_types = false;
if (const char *small_type = getenv("PARALLEL_RDP_SMALL_TYPES"))
{
allow_small_types = strtol(small_type, nullptr, 0) > 0;
forces_small_types = true;
LOGI("Allow small types = %d.\n", int(allow_small_types));
}
if (!features.storage_16bit_features.storageBuffer16BitAccess)
{
LOGE("VK_KHR_16bit_storage for SSBOs is not supported! This is a minimum requirement for paraLLEl-RDP.\n");
return false;
}
if (!features.storage_8bit_features.storageBuffer8BitAccess)
{
LOGE("VK_KHR_8bit_storage for SSBOs is not supported! This is a minimum requirement for paraLLEl-RDP.\n");
return false;
}
// Driver workarounds here for 8/16-bit integer support.
if (features.supports_driver_properties && !forces_small_types)
{
if (features.driver_properties.driverID == VK_DRIVER_ID_AMD_PROPRIETARY_KHR)
{
LOGW("Current proprietary AMD driver is known to be buggy with 8/16-bit integer arithmetic, disabling support for time being.\n");
allow_small_types = false;
}
else if (features.driver_properties.driverID == VK_DRIVER_ID_AMD_OPEN_SOURCE_KHR ||
features.driver_properties.driverID == VK_DRIVER_ID_MESA_RADV_KHR)
{
LOGW("Current open-source AMD drivers are known to be slightly faster without 8/16-bit integer arithmetic.\n");
allow_small_types = false;
}
else if (features.driver_properties.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY_KHR)
{
LOGW("Current NVIDIA driver is known to be slightly faster without 8/16-bit integer arithmetic.\n");
allow_small_types = false;
}
else if (features.driver_properties.driverID == VK_DRIVER_ID_INTEL_PROPRIETARY_WINDOWS_KHR)
{
LOGW("Current proprietary Intel Windows driver is tested to perform much better without 8/16-bit integer support.\n");
allow_small_types = false;
}
// Intel ANV *must* use small integer arithmetic, or it doesn't pass test suite.
}
if (!allow_small_types)
{
caps.supports_small_integer_arithmetic = false;
}
else if (features.enabled_features.shaderInt16 && features.float16_int8_features.shaderInt8)
{
LOGI("Enabling 8 and 16-bit integer arithmetic support for more efficient shaders!\n");
caps.supports_small_integer_arithmetic = true;
}
else
{
LOGW("Device does not support 8 and 16-bit integer arithmetic support. Falling back to 32-bit arithmetic everywhere.\n");
caps.supports_small_integer_arithmetic = false;
}
uint32_t subgroup_size = features.subgroup_properties.subgroupSize;
const VkSubgroupFeatureFlags required =
VK_SUBGROUP_FEATURE_BALLOT_BIT |
VK_SUBGROUP_FEATURE_BASIC_BIT |
VK_SUBGROUP_FEATURE_VOTE_BIT |
VK_SUBGROUP_FEATURE_ARITHMETIC_BIT;
caps.subgroup_tile_binning =
allow_subgroup &&
(features.subgroup_properties.supportedOperations & required) == required &&
(features.subgroup_properties.supportedStages & VK_SHADER_STAGE_COMPUTE_BIT) != 0 &&
can_support_minimum_subgroup_size(32) && subgroup_size <= 64;
caps.subgroup_depth_blend =
caps.super_sample_readback &&
allow_subgroup &&
(features.subgroup_properties.supportedOperations & required) == required &&
(features.subgroup_properties.supportedStages & VK_SHADER_STAGE_COMPUTE_BIT) != 0;
return true;
}
int Renderer::resolve_shader_define(const char *name, const char *define) const
{
if (strcmp(define, "DEBUG_ENABLE") == 0)
return int(debug_channel);
else if (strcmp(define, "UBERSHADER") == 0)
return int(caps.ubershader);
else if (strcmp(define, "SMALL_TYPES") == 0)
return int(caps.supports_small_integer_arithmetic);
else if (strcmp(define, "SUBGROUP") == 0)
{
if (strcmp(name, "tile_binning_combined") == 0)
return int(caps.subgroup_tile_binning);
else if (strcmp(name, "depth_blend") == 0 || strcmp(name, "ubershader") == 0)
return int(caps.subgroup_depth_blend);
else
return 0;
}
else
return 0;
}
void Renderer::init_buffers(const RendererOptions &options)
{
Vulkan::BufferCreateInfo info = {};
info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
info.domain = Vulkan::BufferDomain::Device;
info.misc = Vulkan::BUFFER_MISC_ZERO_INITIALIZE_BIT;
static_assert((Limits::MaxPrimitives % 32) == 0, "MaxPrimitives must be divisble by 32.");
static_assert(Limits::MaxPrimitives <= (32 * 32), "MaxPrimitives must be less-or-equal than 1024.");
info.size = sizeof(uint32_t) *
(Limits::MaxPrimitives / 32) *
(caps.max_width / ImplementationConstants::TileWidth) *
(caps.max_height / ImplementationConstants::TileHeight);
tile_binning_buffer = device->create_buffer(info);
device->set_name(*tile_binning_buffer, "tile-binning-buffer");
info.size = sizeof(uint32_t) *
(caps.max_width / ImplementationConstants::TileWidth) *
(caps.max_height / ImplementationConstants::TileHeight);
tile_binning_buffer_coarse = device->create_buffer(info);
device->set_name(*tile_binning_buffer_coarse, "tile-binning-buffer-coarse");
if (!caps.ubershader)
{
info.size = sizeof(uint32_t) *
(Limits::MaxPrimitives / 32) *
(caps.max_width / ImplementationConstants::TileWidth) *
(caps.max_height / ImplementationConstants::TileHeight);
per_tile_offsets = device->create_buffer(info);
device->set_name(*per_tile_offsets, "per-tile-offsets");
info.size = sizeof(TileRasterWork) * Limits::MaxStaticRasterizationStates * caps.max_num_tile_instances;
tile_work_list = device->create_buffer(info);
device->set_name(*tile_work_list, "tile-work-list");
info.size = sizeof(uint32_t) *
caps.max_num_tile_instances *
ImplementationConstants::TileWidth *
ImplementationConstants::TileHeight;
per_tile_shaded_color = device->create_buffer(info);
device->set_name(*per_tile_shaded_color, "per-tile-shaded-color");
per_tile_shaded_depth = device->create_buffer(info);
device->set_name(*per_tile_shaded_depth, "per-tile-shaded-depth");
info.size = sizeof(uint8_t) *
caps.max_num_tile_instances *
ImplementationConstants::TileWidth *
ImplementationConstants::TileHeight;
per_tile_shaded_coverage = device->create_buffer(info);
per_tile_shaded_shaded_alpha = device->create_buffer(info);
device->set_name(*per_tile_shaded_coverage, "per-tile-shaded-coverage");
device->set_name(*per_tile_shaded_shaded_alpha, "per-tile-shaded-shaded-alpha");
}
}
void Renderer::init_blender_lut()
{
Vulkan::BufferCreateInfo info = {};
info.size = sizeof(blender_lut);
info.domain = Vulkan::BufferDomain::Device;
info.usage = VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT;
blender_divider_lut_buffer = device->create_buffer(info, blender_lut);
device->set_name(*blender_divider_lut_buffer, "blender-divider-lut-buffer");
Vulkan::BufferViewCreateInfo view = {};
view.buffer = blender_divider_lut_buffer.get();
view.format = VK_FORMAT_R8_UINT;
view.range = info.size;
blender_divider_buffer = device->create_buffer_view(view);
}
void Renderer::message(const std::string &tag, uint32_t code, uint32_t x, uint32_t y, uint32_t, uint32_t num_words,
const Vulkan::DebugChannelInterface::Word *words)
{
if (filter_debug_channel_x >= 0 && x != uint32_t(filter_debug_channel_x))
return;
if (filter_debug_channel_y >= 0 && y != uint32_t(filter_debug_channel_y))
return;
enum Code
{
ASSERT_EQUAL = 0,
ASSERT_NOT_EQUAL = 1,
ASSERT_LESS_THAN = 2,
ASSERT_LESS_THAN_EQUAL = 3,
GENERIC = 4,
HEX = 5
};
switch (Code(code))
{
case ASSERT_EQUAL:
LOGE("ASSERT TRIPPED FOR (%u, %u), line %d, %d == %d failed.\n",
x, y, words[0].s32, words[1].s32, words[2].s32);
break;
case ASSERT_NOT_EQUAL:
LOGE("ASSERT TRIPPED FOR (%u, %u), line %d, %d != %d failed.\n",
x, y, words[0].s32, words[1].s32, words[2].s32);
break;
case ASSERT_LESS_THAN:
LOGE("ASSERT TRIPPED FOR (%u, %u), line %d, %d < %d failed.\n",
x, y, words[0].s32, words[1].s32, words[2].s32);
break;
case ASSERT_LESS_THAN_EQUAL:
LOGE("ASSERT TRIPPED FOR (%u, %u), line %d, %d <= %d failed.\n",
x, y, words[0].s32, words[1].s32, words[2].s32);
break;
case GENERIC:
switch (num_words)
{
case 1:
LOGI("(%u, %u), line %d.\n", x, y, words[0].s32);
break;
case 2:
LOGI("(%u, %u), line %d: (%d).\n", x, y, words[0].s32, words[1].s32);
break;
case 3:
LOGI("(%u, %u), line %d: (%d, %d).\n", x, y, words[0].s32, words[1].s32, words[2].s32);
break;
case 4:
LOGI("(%u, %u), line %d: (%d, %d, %d).\n", x, y,
words[0].s32, words[1].s32, words[2].s32, words[3].s32);
break;
default:
LOGE("Unknown number of generic parameters: %u\n", num_words);
break;
}
break;
case HEX:
switch (num_words)
{
case 1:
LOGI("(%u, %u), line %d.\n", x, y, words[0].s32);
break;
case 2:
LOGI("(%u, %u), line %d: (0x%x).\n", x, y, words[0].s32, words[1].s32);
break;
case 3:
LOGI("(%u, %u), line %d: (0x%x, 0x%x).\n", x, y, words[0].s32, words[1].s32, words[2].s32);
break;
case 4:
LOGI("(%u, %u), line %d: (0x%x, 0x%x, 0x%x).\n", x, y,
words[0].s32, words[1].s32, words[2].s32, words[3].s32);
break;
default:
LOGE("Unknown number of generic parameters: %u\n", num_words);
break;
}
break;
default:
LOGE("Unexpected message code: %u\n", code);
break;
}
}
void Renderer::RenderBuffers::init(Vulkan::Device &device, Vulkan::BufferDomain domain,
RenderBuffers *borrow)
{
triangle_setup = create_buffer(device, domain,
sizeof(TriangleSetup) * Limits::MaxPrimitives,
borrow ? &borrow->triangle_setup : nullptr);
device.set_name(*triangle_setup.buffer, "triangle-setup");
attribute_setup = create_buffer(device, domain,
sizeof(AttributeSetup) * Limits::MaxPrimitives,
borrow ? &borrow->attribute_setup: nullptr);
device.set_name(*attribute_setup.buffer, "attribute-setup");
derived_setup = create_buffer(device, domain,
sizeof(DerivedSetup) * Limits::MaxPrimitives,
borrow ? &borrow->derived_setup : nullptr);
device.set_name(*derived_setup.buffer, "derived-setup");
scissor_setup = create_buffer(device, domain,
sizeof(ScissorState) * Limits::MaxPrimitives,
borrow ? &borrow->scissor_setup : nullptr);
device.set_name(*scissor_setup.buffer, "scissor-state");
static_raster_state = create_buffer(device, domain,
sizeof(StaticRasterizationState) * Limits::MaxStaticRasterizationStates,
borrow ? &borrow->static_raster_state : nullptr);
device.set_name(*static_raster_state.buffer, "static-raster-state");
depth_blend_state = create_buffer(device, domain,
sizeof(DepthBlendState) * Limits::MaxDepthBlendStates,
borrow ? &borrow->depth_blend_state : nullptr);
device.set_name(*depth_blend_state.buffer, "depth-blend-state");
tile_info_state = create_buffer(device, domain,
sizeof(TileInfo) * Limits::MaxTileInfoStates,
borrow ? &borrow->tile_info_state : nullptr);
device.set_name(*tile_info_state.buffer, "tile-info-state");
state_indices = create_buffer(device, domain,
sizeof(InstanceIndices) * Limits::MaxPrimitives,
borrow ? &borrow->state_indices : nullptr);
device.set_name(*state_indices.buffer, "state-indices");
span_info_offsets = create_buffer(device, domain,
sizeof(SpanInfoOffsets) * Limits::MaxPrimitives,
borrow ? &borrow->span_info_offsets : nullptr);
device.set_name(*span_info_offsets.buffer, "span-info-offsets");
span_info_jobs = create_buffer(device, domain,
sizeof(SpanInterpolationJob) * Limits::MaxSpanSetups,
borrow ? &borrow->span_info_jobs : nullptr);
device.set_name(*span_info_jobs.buffer, "span-info-jobs");
if (!borrow)
{
Vulkan::BufferViewCreateInfo info = {};
info.buffer = span_info_jobs.buffer.get();
info.format = VK_FORMAT_R16G16B16A16_UINT;
info.range = span_info_jobs.buffer->get_create_info().size;
span_info_jobs_view = device.create_buffer_view(info);
}
}
Renderer::MappedBuffer Renderer::RenderBuffers::create_buffer(
Vulkan::Device &device, Vulkan::BufferDomain domain, VkDeviceSize size,
Renderer::MappedBuffer *borrow)
{
Vulkan::BufferCreateInfo info = {};
info.domain = domain;
if (domain == Vulkan::BufferDomain::Device || domain == Vulkan::BufferDomain::LinkedDeviceHostPreferDevice)
{
info.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
VK_BUFFER_USAGE_TRANSFER_DST_BIT |
VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT;
}
else if (borrow && borrow->is_host)
{
return *borrow;
}
else
{
info.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
}
info.size = size;
Renderer::MappedBuffer buffer;
buffer.buffer = device.create_buffer(info);
buffer.is_host = device.map_host_buffer(*buffer.buffer, 0) != nullptr;
return buffer;
}
void Renderer::RenderBuffersUpdater::init(Vulkan::Device &device)
{
gpu.init(device, Vulkan::BufferDomain::LinkedDeviceHostPreferDevice, nullptr);
cpu.init(device, Vulkan::BufferDomain::Host, &gpu);
}
bool Renderer::init_internal_upscaling_factor(const RendererOptions &options)
{
unsigned factor = options.upscaling_factor;
if (!device || !rdram || !hidden_rdram)
{
LOGE("Renderer is not initialized.\n");
return false;
}
caps.upscaling = factor;
caps.super_sample_readback = options.super_sampled_readback;
caps.super_sample_readback_dither = options.super_sampled_readback_dither;
if (factor == 1)
{
upscaling_multisampled_hidden_rdram.reset();
upscaling_reference_rdram.reset();
upscaling_multisampled_rdram.reset();
return true;
}
Vulkan::BufferCreateInfo info;
info.domain = Vulkan::BufferDomain::Device;
info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
info.misc = Vulkan::BUFFER_MISC_ZERO_INITIALIZE_BIT;
info.size = rdram_size;
upscaling_reference_rdram = device->create_buffer(info);
device->set_name(*upscaling_reference_rdram, "reference-rdram");
info.size = rdram_size * factor * factor;
// If we're super-sampling we'll need to carry forward a u8 writemask per unscaled pixel.
// The resolve pass will conditionally write a resolved pixel to avoid potential race conditions.
// We allocate 2 bits per pixel (color / depth write).
// The SSAA resolve shader will convert this to a VRAM write mask if we also need to handle incoherent
// RDRAM.
if (caps.super_sample_readback)
info.size += 2 * Limits::MaxWidth * Limits::MaxHeight / 8;
upscaling_multisampled_rdram = device->create_buffer(info);
device->set_name(*upscaling_multisampled_rdram, "multisampled-rdram");
info.size = hidden_rdram->get_create_info().size * factor * factor;
upscaling_multisampled_hidden_rdram = device->create_buffer(info);
device->set_name(*upscaling_multisampled_hidden_rdram, "multisampled-hidden-rdram");
{
auto cmd = device->request_command_buffer();
cmd->fill_buffer(*upscaling_multisampled_hidden_rdram, 0x03030303);
cmd->barrier(VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_TRANSFER_WRITE_BIT,
VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT);
device->submit(cmd);
}
return true;
}
void Renderer::set_rdram(Vulkan::Buffer *buffer, uint8_t *host_rdram, size_t offset, size_t size, bool coherent)
{
rdram = buffer;
rdram_offset = offset;
rdram_size = size;
is_host_coherent = coherent;
device->set_name(*rdram, "rdram");
if (!is_host_coherent)
{
assert(rdram_offset == 0);
incoherent.host_rdram = host_rdram;
// If we're not host coherent (missing VK_EXT_external_memory_host),
// we need to create a staging RDRAM buffer which is used for the real RDRAM uploads.
// RDRAM may be uploaded in a masked way (if GPU has pending writes), or direct copy (if no pending writes are outstanding).
Vulkan::BufferCreateInfo info = {};
info.size = size;
info.domain = Vulkan::BufferDomain::Host;
info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
incoherent.staging_rdram = device->create_buffer(info);
device->set_name(*incoherent.staging_rdram, "staging-rdram");
const auto div_round_up = [](size_t a, size_t b) -> size_t { return (a + b - 1) / b; };
if (!rdram->get_allocation().is_host_allocation())
{
// If we cannot map RDRAM, we need a staging readback buffer.
Vulkan::BufferCreateInfo readback_info = {};
readback_info.domain = Vulkan::BufferDomain::CachedCoherentHostPreferCached;
readback_info.size = rdram_size * Limits::NumSyncStates;
readback_info.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT;
incoherent.staging_readback = device->create_buffer(readback_info);
device->set_name(*incoherent.staging_readback, "staging-readback");
incoherent.staging_readback_pages = div_round_up(readback_info.size, ImplementationConstants::IncoherentPageSize);
}
incoherent.page_to_direct_copy.clear();
incoherent.page_to_masked_copy.clear();
incoherent.page_to_pending_readback.clear();
auto packed_pages = div_round_up(size, ImplementationConstants::IncoherentPageSize * 32);
incoherent.num_pages = div_round_up(size, ImplementationConstants::IncoherentPageSize);
incoherent.page_to_direct_copy.resize(packed_pages);
incoherent.page_to_masked_copy.resize(packed_pages);
incoherent.page_to_pending_readback.resize(packed_pages);
incoherent.pending_writes_for_page.reset(new std::atomic_uint32_t[incoherent.num_pages]);
for (unsigned i = 0; i < incoherent.num_pages; i++)
incoherent.pending_writes_for_page[i].store(0);
}
else
{
incoherent = {};
}
}
void Renderer::set_hidden_rdram(Vulkan::Buffer *buffer)
{
hidden_rdram = buffer;
device->set_name(*hidden_rdram, "hidden-rdram");
}
void Renderer::set_tmem(Vulkan::Buffer *buffer)
{
tmem = buffer;
device->set_name(*tmem, "tmem");
}
void Renderer::flush_and_signal()
{
flush_queues();
submit_to_queue();
assert(!stream.cmd);
}
void Renderer::set_color_framebuffer(uint32_t addr, uint32_t width, FBFormat fmt)
{
if (fb.addr != addr || fb.width != width || fb.fmt != fmt)
flush_queues();
fb.addr = addr;
fb.width = width;
fb.fmt = fmt;
}
void Renderer::set_depth_framebuffer(uint32_t addr)
{
if (fb.depth_addr != addr)
flush_queues();
fb.depth_addr = addr;
}
void Renderer::set_scissor_state(const ScissorState &state)
{
stream.scissor_state = state;
}
void Renderer::set_static_rasterization_state(const StaticRasterizationState &state)
{
stream.static_raster_state = state;
}
void Renderer::set_depth_blend_state(const DepthBlendState &state)
{
stream.depth_blend_state = state;
}
void Renderer::draw_flat_primitive(TriangleSetup &setup)
{
draw_shaded_primitive(setup, {});
}
static int normalize_dzpix(int dz)
{
if (dz >= 0x8000)
return 0x8000;
else if (dz == 0)
return 1;
unsigned bit = 31 - leading_zeroes(dz);
return 1 << (bit + 1);
}
static uint16_t dz_compress(int dz)
{
int val = 0;
if (dz & 0xff00)
val |= 8;
if (dz & 0xf0f0)
val |= 4;
if (dz & 0xcccc)
val |= 2;
if (dz & 0xaaaa)
val |= 1;
return uint16_t(val);
}
static void encode_rgb(uint8_t *rgba, uint32_t color)
{
rgba[0] = uint8_t(color >> 24);
rgba[1] = uint8_t(color >> 16);
rgba[2] = uint8_t(color >> 8);
}
static void encode_alpha(uint8_t *rgba, uint32_t color)
{
rgba[3] = uint8_t(color);
}
void Renderer::build_combiner_constants(DerivedSetup &setup, unsigned cycle) const
{
auto &comb = stream.static_raster_state.combiner[cycle];
auto &output = setup.constants[cycle];
switch (comb.rgb.muladd)
{
case RGBMulAdd::Env:
encode_rgb(output.muladd, constants.env_color);
break;
case RGBMulAdd::Primitive:
encode_rgb(output.muladd, constants.primitive_color);
break;
default:
break;
}
switch (comb.rgb.mulsub)
{
case RGBMulSub::Env:
encode_rgb(output.mulsub, constants.env_color);
break;
case RGBMulSub::Primitive:
encode_rgb(output.mulsub, constants.primitive_color);
break;
case RGBMulSub::ConvertK4:
// Need to decode this specially since it's a 9-bit value.
encode_rgb(output.mulsub, uint32_t(constants.convert[4]) << 8);
break;
case RGBMulSub::KeyCenter:
output.mulsub[0] = constants.key_center[0];
output.mulsub[1] = constants.key_center[1];
output.mulsub[2] = constants.key_center[2];
break;
default:
break;
}
switch (comb.rgb.mul)
{
case RGBMul::Primitive:
encode_rgb(output.mul, constants.primitive_color);
break;
case RGBMul::Env:
encode_rgb(output.mul, constants.env_color);
break;
case RGBMul::PrimitiveAlpha:
encode_rgb(output.mul, 0x01010101 * ((constants.primitive_color) & 0xff));
break;
case RGBMul::EnvAlpha:
encode_rgb(output.mul, 0x01010101 * ((constants.env_color) & 0xff));
break;
case RGBMul::PrimLODFrac:
encode_rgb(output.mul, 0x01010101 * constants.prim_lod_frac);
break;
case RGBMul::ConvertK5:
// Need to decode this specially since it's a 9-bit value.
encode_rgb(output.mul, uint32_t(constants.convert[5]) << 8);
break;
case RGBMul::KeyScale:
output.mul[0] = constants.key_scale[0];
output.mul[1] = constants.key_scale[1];
output.mul[2] = constants.key_scale[2];
break;
default:
break;
}
switch (comb.rgb.add)
{
case RGBAdd::Primitive:
encode_rgb(output.add, constants.primitive_color);
break;
case RGBAdd::Env:
encode_rgb(output.add, constants.env_color);
break;
default:
break;
}
switch (comb.alpha.muladd)
{
case AlphaAddSub::PrimitiveAlpha:
encode_alpha(output.muladd, constants.primitive_color);
break;
case AlphaAddSub::EnvAlpha:
encode_alpha(output.muladd, constants.env_color);
break;
default:
break;
}
switch (comb.alpha.mulsub)
{
case AlphaAddSub::PrimitiveAlpha:
encode_alpha(output.mulsub, constants.primitive_color);
break;
case AlphaAddSub::EnvAlpha:
encode_alpha(output.mulsub, constants.env_color);
break;
default:
break;
}
switch (comb.alpha.mul)
{
case AlphaMul::PrimitiveAlpha:
encode_alpha(output.mul, constants.primitive_color);
break;
case AlphaMul::EnvAlpha:
encode_alpha(output.mul, constants.env_color);
break;
case AlphaMul::PrimLODFrac:
encode_alpha(output.mul, constants.prim_lod_frac);
break;
default:
break;
}
switch (comb.alpha.add)
{
case AlphaAddSub::PrimitiveAlpha:
encode_alpha(output.add, constants.primitive_color);
break;
case AlphaAddSub::EnvAlpha:
encode_alpha(output.add, constants.env_color);
break;
default:
break;
}
}
DerivedSetup Renderer::build_derived_attributes(const AttributeSetup &attr) const
{
DerivedSetup setup = {};
if (constants.use_prim_depth)
{
setup.dz = constants.prim_dz;
setup.dz_compressed = dz_compress(setup.dz);
}
else
{
int dzdx = attr.dzdx >> 16;
int dzdy = attr.dzdy >> 16;
int dzpix = (dzdx < 0 ? (~dzdx & 0x7fff) : dzdx) + (dzdy < 0 ? (~dzdy & 0x7fff) : dzdy);
dzpix = normalize_dzpix(dzpix);
setup.dz = dzpix;
setup.dz_compressed = dz_compress(dzpix);
}
build_combiner_constants(setup, 0);
build_combiner_constants(setup, 1);
setup.fog_color[0] = uint8_t(constants.fog_color >> 24);
setup.fog_color[1] = uint8_t(constants.fog_color >> 16);
setup.fog_color[2] = uint8_t(constants.fog_color >> 8);
setup.fog_color[3] = uint8_t(constants.fog_color >> 0);
setup.blend_color[0] = uint8_t(constants.blend_color >> 24);
setup.blend_color[1] = uint8_t(constants.blend_color >> 16);
setup.blend_color[2] = uint8_t(constants.blend_color >> 8);
setup.blend_color[3] = uint8_t(constants.blend_color >> 0);
setup.fill_color = constants.fill_color;
setup.min_lod = constants.min_level;
for (unsigned i = 0; i < 4; i++)
setup.convert_factors[i] = int16_t(constants.convert[i]);
return setup;
}
static constexpr unsigned SUBPIXELS_Y = 4;
static int32_t clamp_int32(int64_t v)
{
if (v < std::numeric_limits<int32_t>::min())
return std::numeric_limits<int32_t>::min();
else if (v > std::numeric_limits<int32_t>::max())
return std::numeric_limits<int32_t>::max();
else
return int32_t(v);
}
static std::pair<int32_t, int32_t> interpolate_x(const TriangleSetup &setup, int y, bool flip, int scaling)
{
// Interpolate in 64-bit so we are guaranteed to catch any overflow scenario.
int64_t yh_interpolation_base = setup.yh & ~(SUBPIXELS_Y - 1);
int64_t ym_interpolation_base = setup.ym;
yh_interpolation_base *= scaling;
ym_interpolation_base *= scaling;
int64_t xh = scaling * setup.xh + int64_t(y - yh_interpolation_base) * setup.dxhdy;
int64_t xm = scaling * setup.xm + int64_t(y - yh_interpolation_base) * setup.dxmdy;
int64_t xl = scaling * setup.xl + int64_t(y - ym_interpolation_base) * setup.dxldy;
if (y < scaling * setup.ym)
xl = xm;
int64_t xh_shifted = xh >> 15;
int64_t xl_shifted = xl >> 15;
int64_t xleft, xright;
if (flip)
{
xleft = xh_shifted;
xright = xl_shifted;
}
else
{
xleft = xl_shifted;
xright = xh_shifted;
}
return { clamp_int32(xleft), clamp_int32(xright) };
}
unsigned Renderer::compute_conservative_max_num_tiles(const TriangleSetup &setup) const
{
if (setup.yl <= setup.yh)
return 0;
int scaling = int(caps.upscaling);
int start_y = setup.yh & ~(SUBPIXELS_Y - 1);
int end_y = (setup.yl - 1) | (SUBPIXELS_Y - 1);
start_y = std::max(int(stream.scissor_state.ylo), start_y);
end_y = std::min(int(stream.scissor_state.yhi) - 1, end_y);
start_y *= scaling;
end_y *= scaling;
// Y is clipped out, exit early.
if (end_y < start_y)
return 0;
bool flip = (setup.flags & TRIANGLE_SETUP_FLIP_BIT) != 0;
auto upper = interpolate_x(setup, start_y, flip, scaling);
auto lower = interpolate_x(setup, end_y, flip, scaling);
auto mid = upper;
auto mid1 = upper;
int ym = scaling * setup.ym;
if (ym > start_y && ym < end_y)
{
mid = interpolate_x(setup, ym, flip, scaling);
mid1 = interpolate_x(setup, ym - 1, flip, scaling);
}
// Robustness, check if we overflow the X rasterizer precision.
// After shifting down, we should have 12 bits signed.
// If we detect any overflow here we need to assume X range is scissor rect.
// This really should never happen, but it's possible to write tests that intentionally trigger weird
// overflow behavior that needs to be specially handled.
// There might be freak scenarios where we cannot detect overflow since we're only sampling at 4 scanlines
// and we have ~32 overflows happening at once,
// So we need to interpolate in 64-bit to make this work.
auto start_x = std::min(std::min(upper.first, lower.first), std::min(mid.first, mid1.first));
auto end_x = std::max(std::max(upper.second, lower.second), std::max(mid.second, mid1.second));
auto max_range_x = std::max(std::abs(start_x), std::abs(end_x));
// Effective X range is [-2048, 2047], but just make it [-2047, 2047] to match binning shader.
// If we interpolate X to something outside that range,
// we must assume the entire X range will be covered.
if (max_range_x > 2047 * scaling)
{
start_x = 0;
end_x = 0x7fffffff;
}
start_x = std::max(start_x, scaling * (int(stream.scissor_state.xlo) >> 2));
end_x = std::min(end_x, scaling * ((int(stream.scissor_state.xhi) + 3) >> 2) - 1);
if (end_x < start_x)
return 0;
start_x /= ImplementationConstants::TileWidth;
end_x /= ImplementationConstants::TileWidth;
start_y /= (SUBPIXELS_Y * ImplementationConstants::TileHeight);
end_y /= (SUBPIXELS_Y * ImplementationConstants::TileHeight);
return (end_x - start_x + 1) * (end_y - start_y + 1);
}
static bool combiner_accesses_texel0(const CombinerInputs &inputs)
{
return inputs.rgb.muladd == RGBMulAdd::Texel0 ||
inputs.rgb.mulsub == RGBMulSub::Texel0 ||
inputs.rgb.mul == RGBMul::Texel0 ||
inputs.rgb.add == RGBAdd::Texel0 ||
inputs.rgb.mul == RGBMul::Texel0Alpha ||
inputs.alpha.muladd == AlphaAddSub::Texel0Alpha ||
inputs.alpha.mulsub == AlphaAddSub::Texel0Alpha ||
inputs.alpha.mul == AlphaMul::Texel0Alpha ||
inputs.alpha.add == AlphaAddSub::Texel0Alpha;
}
static bool combiner_accesses_lod_frac(const CombinerInputs &inputs)
{
return inputs.rgb.mul == RGBMul::LODFrac || inputs.alpha.mul == AlphaMul::LODFrac;
}
static bool combiner_accesses_texel1(const CombinerInputs &inputs)
{
return inputs.rgb.muladd == RGBMulAdd::Texel1 ||
inputs.rgb.mulsub == RGBMulSub::Texel1 ||
inputs.rgb.mul == RGBMul::Texel1 ||
inputs.rgb.add == RGBAdd::Texel1 ||
inputs.rgb.mul == RGBMul::Texel1Alpha ||
inputs.alpha.muladd == AlphaAddSub::Texel1Alpha ||
inputs.alpha.mulsub == AlphaAddSub::Texel1Alpha ||
inputs.alpha.mul == AlphaMul::Texel1Alpha ||
inputs.alpha.add == AlphaAddSub::Texel1Alpha;
}
static bool combiner_uses_texel0(const StaticRasterizationState &state)
{
// Texel0 can be safely used in cycle0 of CYCLE2 mode, or in cycle1 (only cycle) of CYCLE1 mode.
if ((state.flags & RASTERIZATION_MULTI_CYCLE_BIT) != 0)
{
// In second cycle, Texel0 and Texel1 swap around ...
return combiner_accesses_texel0(state.combiner[0]) ||
combiner_accesses_texel1(state.combiner[1]);
}
else
return combiner_accesses_texel0(state.combiner[1]);
}
static bool combiner_uses_texel1(const StaticRasterizationState &state)
{
// Texel1 can be safely used in cycle0 of CYCLE2 mode, and never in cycle1 mode.
// Texel0 can be safely accessed in cycle1, which is an alias due to pipelining.
if ((state.flags & RASTERIZATION_MULTI_CYCLE_BIT) != 0)
{
return combiner_accesses_texel1(state.combiner[0]) ||
combiner_accesses_texel0(state.combiner[1]);
}
else
return false;
}
static bool combiner_uses_pipelined_texel1(const StaticRasterizationState &state)
{
// If you access Texel1 in cycle1 mode, you end up reading the next pixel's color for whatever reason.
if ((state.flags & RASTERIZATION_MULTI_CYCLE_BIT) == 0)
return combiner_accesses_texel1(state.combiner[1]);
else
return false;
}
static bool combiner_uses_lod_frac(const StaticRasterizationState &state)
{
if ((state.flags & RASTERIZATION_MULTI_CYCLE_BIT) != 0)
return combiner_accesses_lod_frac(state.combiner[0]) || combiner_accesses_lod_frac(state.combiner[1]);
else
return false;
}
void Renderer::deduce_noise_state()
{
auto &state = stream.static_raster_state;
state.flags &= ~RASTERIZATION_NEED_NOISE_BIT;
// Figure out if we need to seed noise variable for this primitive.
if ((state.dither & 3) == 2 || ((state.dither >> 2) & 3) == 2)
{
state.flags |= RASTERIZATION_NEED_NOISE_BIT;
return;
}
if ((state.flags & (RASTERIZATION_COPY_BIT | RASTERIZATION_FILL_BIT)) != 0)
return;
if ((state.flags & RASTERIZATION_MULTI_CYCLE_BIT) != 0)
{
if (state.combiner[0].rgb.muladd == RGBMulAdd::Noise)
state.flags |= RASTERIZATION_NEED_NOISE_BIT;
}
else if (state.combiner[1].rgb.muladd == RGBMulAdd::Noise)
state.flags |= RASTERIZATION_NEED_NOISE_BIT;
if ((state.flags & (RASTERIZATION_ALPHA_TEST_BIT | RASTERIZATION_ALPHA_TEST_DITHER_BIT)) ==
(RASTERIZATION_ALPHA_TEST_BIT | RASTERIZATION_ALPHA_TEST_DITHER_BIT))
{
state.flags |= RASTERIZATION_NEED_NOISE_BIT;
}
}
static RGBMulAdd normalize_combiner(RGBMulAdd muladd)
{
switch (muladd)
{
case RGBMulAdd::Noise:
case RGBMulAdd::Texel0:
case RGBMulAdd::Texel1:
case RGBMulAdd::Combined:
case RGBMulAdd::One:
case RGBMulAdd::Shade:
return muladd;
default:
return RGBMulAdd::Zero;
}
}
static RGBMulSub normalize_combiner(RGBMulSub mulsub)
{
switch (mulsub)
{
case RGBMulSub::Combined:
case RGBMulSub::Texel0:
case RGBMulSub::Texel1:
case RGBMulSub::Shade:
case RGBMulSub::ConvertK4:
return mulsub;
default:
return RGBMulSub::Zero;
}
}
static RGBMul normalize_combiner(RGBMul mul)
{
switch (mul)
{
case RGBMul::Combined:
case RGBMul::CombinedAlpha:
case RGBMul::Texel0:
case RGBMul::Texel1:
case RGBMul::Texel0Alpha:
case RGBMul::Texel1Alpha:
case RGBMul::Shade:
case RGBMul::ShadeAlpha:
case RGBMul::LODFrac:
case RGBMul::ConvertK5:
return mul;
default:
return RGBMul::Zero;
}
}
static RGBAdd normalize_combiner(RGBAdd add)
{
switch (add)
{
case RGBAdd::Texel0:
case RGBAdd::Texel1:
case RGBAdd::Combined:
case RGBAdd::One:
case RGBAdd::Shade:
return add;
default:
return RGBAdd::Zero;
}
}
static AlphaAddSub normalize_combiner(AlphaAddSub addsub)
{
switch (addsub)
{
case AlphaAddSub::CombinedAlpha:
case AlphaAddSub::Texel0Alpha:
case AlphaAddSub::Texel1Alpha:
case AlphaAddSub::ShadeAlpha:
case AlphaAddSub::One:
return addsub;
default:
return AlphaAddSub::Zero;
}
}
static AlphaMul normalize_combiner(AlphaMul mul)
{
switch (mul)
{
case AlphaMul::LODFrac:
case AlphaMul::Texel0Alpha:
case AlphaMul::Texel1Alpha:
case AlphaMul::ShadeAlpha:
return mul;
default:
return AlphaMul::Zero;
}
}
static void normalize_combiner(CombinerInputsRGB &comb)
{
comb.muladd = normalize_combiner(comb.muladd);
comb.mulsub = normalize_combiner(comb.mulsub);
comb.mul = normalize_combiner(comb.mul);
comb.add = normalize_combiner(comb.add);
}
static void normalize_combiner(CombinerInputsAlpha &comb)
{
comb.muladd = normalize_combiner(comb.muladd);
comb.mulsub = normalize_combiner(comb.mulsub);
comb.mul = normalize_combiner(comb.mul);
comb.add = normalize_combiner(comb.add);
}
static void normalize_combiner(CombinerInputs &comb)
{
normalize_combiner(comb.rgb);
normalize_combiner(comb.alpha);
}
StaticRasterizationState Renderer::normalize_static_state(StaticRasterizationState state)
{
if ((state.flags & RASTERIZATION_FILL_BIT) != 0)
{
state = {};
state.flags = RASTERIZATION_FILL_BIT;
return state;
}
if ((state.flags & RASTERIZATION_COPY_BIT) != 0)
{
auto flags = state.flags &
(RASTERIZATION_COPY_BIT |
RASTERIZATION_TLUT_BIT |
RASTERIZATION_TLUT_TYPE_BIT |
RASTERIZATION_USES_TEXEL0_BIT |
RASTERIZATION_USE_STATIC_TEXTURE_SIZE_FORMAT_BIT |
RASTERIZATION_TEX_LOD_ENABLE_BIT |
RASTERIZATION_DETAIL_LOD_ENABLE_BIT |
RASTERIZATION_ALPHA_TEST_BIT);
auto fmt = state.texture_fmt;
auto siz = state.texture_size;
state = {};
state.flags = flags;
state.texture_fmt = fmt;
state.texture_size = siz;
return state;
}
if ((state.flags & (RASTERIZATION_MULTI_CYCLE_BIT | RASTERIZATION_USES_PIPELINED_TEXEL1_BIT)) == 0)
state.flags &= ~(RASTERIZATION_BILERP_1_BIT | RASTERIZATION_CONVERT_ONE_BIT);
normalize_combiner(state.combiner[0]);
normalize_combiner(state.combiner[1]);
return state;
}
void Renderer::deduce_static_texture_state(unsigned tile, unsigned max_lod_level)
{
auto &state = stream.static_raster_state;
state.flags &= ~RASTERIZATION_USE_STATIC_TEXTURE_SIZE_FORMAT_BIT;
state.texture_size = 0;
state.texture_fmt = 0;
if ((state.flags & RASTERIZATION_FILL_BIT) != 0)
return;
auto fmt = tiles[tile].meta.fmt;
auto siz = tiles[tile].meta.size;
if ((state.flags & RASTERIZATION_COPY_BIT) == 0)
{
// If all tiles we sample have the same fmt and size (common case), we can use a static variant.
bool uses_texel0 = combiner_uses_texel0(state);
bool uses_texel1 = combiner_uses_texel1(state);
bool uses_pipelined_texel1 = combiner_uses_pipelined_texel1(state);
bool uses_lod_frac = combiner_uses_lod_frac(state);
if (uses_texel1 && (state.flags & RASTERIZATION_CONVERT_ONE_BIT) != 0)
uses_texel0 = true;
state.flags &= ~(RASTERIZATION_USES_TEXEL0_BIT |
RASTERIZATION_USES_TEXEL1_BIT |
RASTERIZATION_USES_PIPELINED_TEXEL1_BIT |
RASTERIZATION_USES_LOD_BIT);
if (uses_texel0)
state.flags |= RASTERIZATION_USES_TEXEL0_BIT;
if (uses_texel1)
state.flags |= RASTERIZATION_USES_TEXEL1_BIT;
if (uses_pipelined_texel1)
state.flags |= RASTERIZATION_USES_PIPELINED_TEXEL1_BIT;
if (uses_lod_frac || (state.flags & RASTERIZATION_TEX_LOD_ENABLE_BIT) != 0)
state.flags |= RASTERIZATION_USES_LOD_BIT;
if (!uses_texel0 && !uses_texel1 && !uses_pipelined_texel1)
return;
bool use_lod = (state.flags & RASTERIZATION_TEX_LOD_ENABLE_BIT) != 0;
bool use_detail = (state.flags & RASTERIZATION_DETAIL_LOD_ENABLE_BIT) != 0;
bool uses_physical_texel1 = uses_texel1 &&
((state.flags & RASTERIZATION_CONVERT_ONE_BIT) == 0 ||
(state.flags & RASTERIZATION_BILERP_1_BIT) != 0);
if (!use_lod)
max_lod_level = uses_physical_texel1 ? 1 : 0;
if (use_detail)
max_lod_level++;
max_lod_level = std::min(max_lod_level, 7u);
for (unsigned i = 1; i <= max_lod_level; i++)
{
auto &t = tiles[(tile + i) & 7].meta;
if (t.fmt != fmt)
return;
if (t.size != siz)
return;
}
}
// We have a static format.
state.flags |= RASTERIZATION_USE_STATIC_TEXTURE_SIZE_FORMAT_BIT;
state.texture_fmt = uint32_t(fmt);
state.texture_size = uint32_t(siz);
}
void Renderer::fixup_triangle_setup(TriangleSetup &setup) const
{
// If YM is lower than the first sub-scanline we rasterize, we will never observe YM at all.
// To account for this, fixup here so that YM is out of range.
// No known content actually triggers this, but some public tests triggered it.
int start_y = setup.yh & ~(SUBPIXELS_Y - 1);
if (setup.ym < start_y)
setup.ym = std::numeric_limits<int16_t>::max();
if ((stream.static_raster_state.flags & RASTERIZATION_INTERLACE_FIELD_BIT) != 0)
{
setup.flags |= (stream.static_raster_state.flags & RASTERIZATION_INTERLACE_FIELD_BIT) ?
TRIANGLE_SETUP_INTERLACE_FIELD_BIT : 0;
setup.flags |= (stream.static_raster_state.flags & RASTERIZATION_INTERLACE_KEEP_ODD_BIT) ?
TRIANGLE_SETUP_INTERLACE_KEEP_ODD_BIT : 0;
}
// Span size is inclusive, not exclusive.
// Rasterization is based on X range directly.
if ((stream.static_raster_state.flags & (RASTERIZATION_COPY_BIT | RASTERIZATION_FILL_BIT)) != 0)
setup.flags |= TRIANGLE_SETUP_FILL_COPY_RASTER_BIT;
}
void Renderer::validate_draw_state() const
{
if ((stream.static_raster_state.flags & RASTERIZATION_FILL_BIT) != 0)
{
if (fb.fmt == FBFormat::I4)
{
validation_iface->report_rdp_crash(ValidationError::Fill4bpp,
"Attempted to use Fill mode on 4bpp surface.");
}
if ((stream.depth_blend_state.flags & DEPTH_BLEND_DEPTH_TEST_BIT) != 0)
{
validation_iface->report_rdp_crash(ValidationError::FillDepthTest,
"Attempted to use Fill mode with depth test.");
}
if ((stream.depth_blend_state.flags & DEPTH_BLEND_IMAGE_READ_ENABLE_BIT) != 0)
{
validation_iface->report_rdp_crash(ValidationError::FillImageReadEnable,
"Attempted to use Fill mode with image read enable.");
}
if ((stream.depth_blend_state.flags & DEPTH_BLEND_DEPTH_UPDATE_BIT) != 0 &&
!constants.use_prim_depth)
{
validation_iface->report_rdp_crash(ValidationError::FillDepthWrite,
"Attempted to use Fill mode with depth write enabled.");
}
}
else if ((stream.static_raster_state.flags & RASTERIZATION_COPY_BIT) != 0)
{
if (fb.fmt == FBFormat::RGBA8888)
{
validation_iface->report_rdp_crash(ValidationError::Copy32bpp,
"Attempted to use Copy mode on 32bpp surface.");
}
}
}
void Renderer::draw_shaded_primitive(TriangleSetup &setup, const AttributeSetup &attr)
{
if (validation_iface)
validate_draw_state();
fixup_triangle_setup(setup);
unsigned num_tiles = compute_conservative_max_num_tiles(setup);
#if 0
// Don't exit early, throws off seeding of noise channels.
if (!num_tiles)
return;
#endif
if (!caps.ubershader)
stream.max_shaded_tiles += num_tiles;
update_deduced_height(setup);
stream.span_info_offsets.add(allocate_span_jobs(setup));
stream.triangle_setup.add(setup);
if (constants.use_prim_depth)
{
auto tmp_attr = attr;
tmp_attr.z = constants.prim_depth;
tmp_attr.dzdx = 0;
tmp_attr.dzde = 0;
tmp_attr.dzdy = 0;
stream.attribute_setup.add(tmp_attr);
}
else
{
stream.attribute_setup.add(attr);
}
stream.derived_setup.add(build_derived_attributes(attr));
stream.scissor_setup.add(stream.scissor_state);
deduce_static_texture_state(setup.tile & 7, setup.tile >> 3);
deduce_noise_state();
InstanceIndices indices = {};
indices.static_index = stream.static_raster_state_cache.add(normalize_static_state(stream.static_raster_state));
indices.depth_blend_index = stream.depth_blend_state_cache.add(stream.depth_blend_state);
indices.tile_instance_index = uint8_t(stream.tmem_upload_infos.size());
for (unsigned i = 0; i < 8; i++)
indices.tile_indices[i] = stream.tile_info_state_cache.add(tiles[i]);
stream.state_indices.add(indices);
fb.color_write_pending = true;
if (stream.depth_blend_state.flags & DEPTH_BLEND_DEPTH_UPDATE_BIT)
fb.depth_write_pending = true;
pending_primitives++;
if (need_flush())
flush_queues();
}
SpanInfoOffsets Renderer::allocate_span_jobs(const TriangleSetup &setup)
{
int min_active_sub_scanline = std::max(int(setup.yh), int(stream.scissor_state.ylo));
int min_active_line = min_active_sub_scanline >> 2;
int max_active_sub_scanline = std::min(setup.yl - 1, int(stream.scissor_state.yhi) - 1);
int max_active_line = max_active_sub_scanline >> 2;
if (max_active_line < min_active_line)
return { 0, 0, -1, 0 };
// Need to poke into next scanline validation for certain workarounds.
int height = std::max(max_active_line - min_active_line + 2, 0);
height = std::min(height, 1024);
int num_jobs = (height + ImplementationConstants::DefaultWorkgroupSize - 1) / ImplementationConstants::DefaultWorkgroupSize;
SpanInfoOffsets offsets = {};
offsets.offset = uint32_t(stream.span_info_jobs.size()) * ImplementationConstants::DefaultWorkgroupSize;
offsets.ylo = min_active_line;
offsets.yhi = max_active_line;
for (int i = 0; i < num_jobs; i++)
{
SpanInterpolationJob interpolation_job = {};
interpolation_job.primitive_index = uint32_t(stream.triangle_setup.size());
interpolation_job.base_y = min_active_line + ImplementationConstants::DefaultWorkgroupSize * i;
interpolation_job.max_y = max_active_line + 1;
stream.span_info_jobs.add(interpolation_job);
}
return offsets;
}
void Renderer::update_deduced_height(const TriangleSetup &setup)
{
int max_active_sub_scanline = std::min(setup.yl - 1, int(stream.scissor_state.yhi) - 1);
int max_active_line = max_active_sub_scanline >> 2;
int height = std::max(max_active_line + 1, 0);
fb.deduced_height = std::max(fb.deduced_height, uint32_t(height));
}
bool Renderer::need_flush() const
{
bool cache_full =
stream.static_raster_state_cache.full() ||
stream.depth_blend_state_cache.full() ||
(stream.tile_info_state_cache.size() + 8 > Limits::MaxTileInfoStates);
bool triangle_full =
stream.triangle_setup.full();
bool span_info_full =
(stream.span_info_jobs.size() * ImplementationConstants::DefaultWorkgroupSize + Limits::MaxHeight > Limits::MaxSpanSetups);
bool max_shaded_tiles =
(stream.max_shaded_tiles + caps.max_tiles_x * caps.max_tiles_y > caps.max_num_tile_instances);
#ifdef VULKAN_DEBUG
if (cache_full)
LOGI("Cache is full.\n");
if (triangle_full)
LOGI("Triangle is full.\n");
if (span_info_full)
LOGI("Span info is full.\n");
if (max_shaded_tiles)
LOGI("Shaded tiles is full.\n");
#endif
return cache_full || triangle_full || span_info_full || max_shaded_tiles;
}
template <typename Cache>
void Renderer::RenderBuffersUpdater::upload(Vulkan::CommandBuffer &cmd, Vulkan::Device &device,
const MappedBuffer &gpu, const MappedBuffer &cpu, const Cache &cache,
bool &did_upload)
{
if (!cache.empty())
{
memcpy(device.map_host_buffer(*cpu.buffer, Vulkan::MEMORY_ACCESS_WRITE_BIT), cache.data(), cache.byte_size());
device.unmap_host_buffer(*cpu.buffer, Vulkan::MEMORY_ACCESS_WRITE_BIT);
if (gpu.buffer != cpu.buffer)
{
cmd.copy_buffer(*gpu.buffer, 0, *cpu.buffer, 0, cache.byte_size());
did_upload = true;
}
}
}
void Renderer::RenderBuffersUpdater::upload(Vulkan::Device &device, const Renderer::StreamCaches &caches,
Vulkan::CommandBuffer &cmd)
{
bool did_upload = false;
upload(cmd, device, gpu.triangle_setup, cpu.triangle_setup, caches.triangle_setup, did_upload);
upload(cmd, device, gpu.attribute_setup, cpu.attribute_setup, caches.attribute_setup, did_upload);
upload(cmd, device, gpu.derived_setup, cpu.derived_setup, caches.derived_setup, did_upload);
upload(cmd, device, gpu.scissor_setup, cpu.scissor_setup, caches.scissor_setup, did_upload);
upload(cmd, device, gpu.static_raster_state, cpu.static_raster_state, caches.static_raster_state_cache, did_upload);
upload(cmd, device, gpu.depth_blend_state, cpu.depth_blend_state, caches.depth_blend_state_cache, did_upload);
upload(cmd, device, gpu.tile_info_state, cpu.tile_info_state, caches.tile_info_state_cache, did_upload);
upload(cmd, device, gpu.state_indices, cpu.state_indices, caches.state_indices, did_upload);
upload(cmd, device, gpu.span_info_offsets, cpu.span_info_offsets, caches.span_info_offsets, did_upload);
upload(cmd, device, gpu.span_info_jobs, cpu.span_info_jobs, caches.span_info_jobs, did_upload);
if (did_upload)
{
cmd.barrier(VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_TRANSFER_WRITE_BIT,
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_READ_BIT);
}
}
void Renderer::update_tmem_instances(Vulkan::CommandBuffer &cmd)
{
cmd.begin_region("tmem-update");
cmd.set_storage_buffer(0, 0, *rdram, rdram_offset, rdram_size);
cmd.set_storage_buffer(0, 1, *tmem);
cmd.set_storage_buffer(0, 2, *tmem_instances);
memcpy(cmd.allocate_typed_constant_data<UploadInfo>(1, 0, stream.tmem_upload_infos.size()),
stream.tmem_upload_infos.data(),
stream.tmem_upload_infos.size() * sizeof(UploadInfo));
auto count = uint32_t(stream.tmem_upload_infos.size());
#ifdef PARALLEL_RDP_SHADER_DIR
cmd.set_program("rdp://tmem_update.comp", {{ "DEBUG_ENABLE", debug_channel ? 1 : 0 }});
#else
cmd.set_program(shader_bank->tmem_update);
#endif
cmd.push_constants(&count, 0, sizeof(count));
cmd.set_specialization_constant_mask(1);
cmd.set_specialization_constant(0, ImplementationConstants::DefaultWorkgroupSize);
Vulkan::QueryPoolHandle start_ts, end_ts;
if (caps.timestamp >= 2)
start_ts = cmd.write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
cmd.dispatch(2048 / ImplementationConstants::DefaultWorkgroupSize, 1, 1);
if (caps.timestamp >= 2)
{
end_ts = cmd.write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
device->register_time_interval("RDP GPU", std::move(start_ts), std::move(end_ts),
"tmem-update");
}
cmd.end_region();
}
void Renderer::submit_span_setup_jobs(Vulkan::CommandBuffer &cmd, bool upscale)
{
cmd.begin_region("span-setup");
auto &instance = buffer_instances[buffer_instance];
cmd.set_storage_buffer(0, 0, *instance.gpu.triangle_setup.buffer);
cmd.set_storage_buffer(0, 1, *instance.gpu.attribute_setup.buffer);
cmd.set_storage_buffer(0, 2, *instance.gpu.scissor_setup.buffer);
cmd.set_storage_buffer(0, 3, *span_setups);
#ifdef PARALLEL_RDP_SHADER_DIR
cmd.set_program("rdp://span_setup.comp", {{ "DEBUG_ENABLE", debug_channel ? 1 : 0 }});
#else
cmd.set_program(shader_bank->span_setup);
#endif
cmd.set_buffer_view(1, 0, *instance.gpu.span_info_jobs_view);
cmd.set_specialization_constant_mask(3);
cmd.set_specialization_constant(0, (upscale ? caps.upscaling : 1) * ImplementationConstants::DefaultWorkgroupSize);
cmd.set_specialization_constant(1, upscale ? trailing_zeroes(caps.upscaling) : 0u);
Vulkan::QueryPoolHandle begin_ts, end_ts;
if (caps.timestamp >= 2)
begin_ts = cmd.write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
cmd.dispatch(stream.span_info_jobs.size(), 1, 1);
if (caps.timestamp >= 2)
{
end_ts = cmd.write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
device->register_time_interval("RDP GPU", std::move(begin_ts), std::move(end_ts), "span-info-jobs");
}
cmd.end_region();
}
void Renderer::clear_indirect_buffer(Vulkan::CommandBuffer &cmd)
{
cmd.begin_region("clear-indirect-buffer");
#ifdef PARALLEL_RDP_SHADER_DIR
cmd.set_program("rdp://clear_indirect_buffer.comp");
#else
cmd.set_program(shader_bank->clear_indirect_buffer);
#endif
cmd.set_storage_buffer(0, 0, *indirect_dispatch_buffer);
static_assert((Limits::MaxStaticRasterizationStates % ImplementationConstants::DefaultWorkgroupSize) == 0, "MaxStaticRasterizationStates does not align.");
cmd.set_specialization_constant_mask(1);
cmd.set_specialization_constant(0, ImplementationConstants::DefaultWorkgroupSize);
cmd.dispatch(Limits::MaxStaticRasterizationStates / ImplementationConstants::DefaultWorkgroupSize, 1, 1);
cmd.end_region();
}
void Renderer::submit_rasterization(Vulkan::CommandBuffer &cmd, Vulkan::Buffer &tmem, bool upscaling)
{
cmd.begin_region("rasterization");
auto &instance = buffer_instances[buffer_instance];
cmd.set_storage_buffer(0, 0, *instance.gpu.triangle_setup.buffer);
cmd.set_storage_buffer(0, 1, *instance.gpu.attribute_setup.buffer);
cmd.set_storage_buffer(0, 2, *instance.gpu.derived_setup.buffer);
cmd.set_storage_buffer(0, 3, *instance.gpu.static_raster_state.buffer);
cmd.set_storage_buffer(0, 4, *instance.gpu.state_indices.buffer);
cmd.set_storage_buffer(0, 5, *instance.gpu.span_info_offsets.buffer);
cmd.set_storage_buffer(0, 6, *span_setups);
cmd.set_storage_buffer(0, 7, tmem);
cmd.set_storage_buffer(0, 8, *instance.gpu.tile_info_state.buffer);
cmd.set_storage_buffer(0, 9, *per_tile_shaded_color);
cmd.set_storage_buffer(0, 10, *per_tile_shaded_depth);
cmd.set_storage_buffer(0, 11, *per_tile_shaded_shaded_alpha);
cmd.set_storage_buffer(0, 12, *per_tile_shaded_coverage);
auto *global_fb_info = cmd.allocate_typed_constant_data<GlobalFBInfo>(2, 0, 1);
switch (fb.fmt)
{
case FBFormat::I4:
global_fb_info->fb_size = 0;
global_fb_info->dx_mask = 0;
global_fb_info->dx_shift = 0;
break;
case FBFormat::I8:
global_fb_info->fb_size = 1;
global_fb_info->dx_mask = ~7u;
global_fb_info->dx_shift = 3;
break;
case FBFormat::RGBA5551:
case FBFormat::IA88:
global_fb_info->fb_size = 2;
global_fb_info->dx_mask = ~3u;
global_fb_info->dx_shift = 2;
break;
case FBFormat::RGBA8888:
global_fb_info->fb_size = 4;
global_fb_info->dx_shift = ~1u;
global_fb_info->dx_shift = 1;
break;
}
global_fb_info->base_primitive_index = base_primitive_index;
#ifdef PARALLEL_RDP_SHADER_DIR
cmd.set_program("rdp://rasterizer.comp", {
{ "DEBUG_ENABLE", debug_channel ? 1 : 0 },
{ "SMALL_TYPES", caps.supports_small_integer_arithmetic ? 1 : 0 },
});
#else
cmd.set_program(shader_bank->rasterizer);
#endif
cmd.set_specialization_constant(0, ImplementationConstants::TileWidth);
cmd.set_specialization_constant(1, ImplementationConstants::TileHeight);
Vulkan::QueryPoolHandle start_ts, end_ts;
if (caps.timestamp >= 2)
start_ts = cmd.write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
uint32_t scale_log2_bit = (upscaling ? trailing_zeroes(caps.upscaling) : 0u) << RASTERIZATION_UPSCALING_LOG2_BIT_OFFSET;
for (size_t i = 0; i < stream.static_raster_state_cache.size(); i++)
{
cmd.set_storage_buffer(1, 0, *tile_work_list,
i * sizeof(TileRasterWork) * caps.max_num_tile_instances,
sizeof(TileRasterWork) * caps.max_num_tile_instances);
auto &state = stream.static_raster_state_cache.data()[i];
cmd.set_specialization_constant(2, state.flags | RASTERIZATION_USE_SPECIALIZATION_CONSTANT_BIT | scale_log2_bit);
cmd.set_specialization_constant(3, state.combiner[0].rgb);
cmd.set_specialization_constant(4, state.combiner[0].alpha);
cmd.set_specialization_constant(5, state.combiner[1].rgb);
cmd.set_specialization_constant(6, state.combiner[1].alpha);
cmd.set_specialization_constant(7, state.dither |
(state.texture_size << 8u) |
(state.texture_fmt << 16u));
cmd.set_specialization_constant_mask(0xff);
if (!caps.force_sync && !cmd.flush_pipeline_state_without_blocking())
{
Vulkan::DeferredPipelineCompile compile;
cmd.extract_pipeline_state(compile);
if (pending_async_pipelines.count(compile.hash) == 0)
{
pending_async_pipelines.insert(compile.hash);
pipeline_worker->push(std::move(compile));
}
cmd.set_specialization_constant_mask(7);
cmd.set_specialization_constant(2, scale_log2_bit);
}
cmd.dispatch_indirect(*indirect_dispatch_buffer, 4 * sizeof(uint32_t) * i);
}
if (caps.timestamp >= 2)
{
end_ts = cmd.write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
device->register_time_interval("RDP GPU", std::move(start_ts), std::move(end_ts), "shading");
}
cmd.end_region();
}
void Renderer::submit_tile_binning_combined(Vulkan::CommandBuffer &cmd, bool upscale)
{
cmd.begin_region("tile-binning-combined");
auto &instance = buffer_instances[buffer_instance];
cmd.set_storage_buffer(0, 0, *instance.gpu.triangle_setup.buffer);
cmd.set_storage_buffer(0, 1, *instance.gpu.scissor_setup.buffer);
cmd.set_storage_buffer(0, 2, *instance.gpu.state_indices.buffer);
cmd.set_storage_buffer(0, 3, *tile_binning_buffer);
cmd.set_storage_buffer(0, 4, *tile_binning_buffer_coarse);
if (!caps.ubershader)
{
cmd.set_storage_buffer(0, 5, *per_tile_offsets);
cmd.set_storage_buffer(0, 6, *indirect_dispatch_buffer);
cmd.set_storage_buffer(0, 7, *tile_work_list);
}
cmd.set_specialization_constant_mask(0x7f);
cmd.set_specialization_constant(1, ImplementationConstants::TileWidth);
cmd.set_specialization_constant(2, ImplementationConstants::TileHeight);
cmd.set_specialization_constant(3, Limits::MaxPrimitives);
cmd.set_specialization_constant(4, upscale ? caps.max_width : Limits::MaxWidth);
cmd.set_specialization_constant(5, caps.max_num_tile_instances);
cmd.set_specialization_constant(6, upscale ? caps.upscaling : 1u);
struct PushData
{
uint32_t width, height;
uint32_t num_primitives;
} push = {};
push.width = fb.width;
push.height = fb.deduced_height;
if (upscale)
{
push.width *= caps.upscaling;
push.height *= caps.upscaling;
}
push.num_primitives = uint32_t(stream.triangle_setup.size());
unsigned num_primitives_32 = (push.num_primitives + 31) / 32;
cmd.push_constants(&push, 0, sizeof(push));
auto &features = device->get_device_features();
uint32_t subgroup_size = features.subgroup_properties.subgroupSize;
Vulkan::QueryPoolHandle start_ts, end_ts;
if (caps.timestamp >= 2)
start_ts = cmd.write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
if (caps.subgroup_tile_binning)
{
#ifdef PARALLEL_RDP_SHADER_DIR
cmd.set_program("rdp://tile_binning_combined.comp", {
{ "DEBUG_ENABLE", debug_channel ? 1 : 0 },
{ "SUBGROUP", 1 },
{ "UBERSHADER", int(caps.ubershader) },
{ "SMALL_TYPES", caps.supports_small_integer_arithmetic ? 1 : 0 },
});
#else
cmd.set_program(shader_bank->tile_binning_combined);
#endif
if (supports_subgroup_size_control(32, subgroup_size))
{
cmd.enable_subgroup_size_control(true);
cmd.set_subgroup_size_log2(true, 5, trailing_zeroes(subgroup_size));
}
}
else
{
#ifdef PARALLEL_RDP_SHADER_DIR
cmd.set_program("rdp://tile_binning_combined.comp", {
{ "DEBUG_ENABLE", debug_channel ? 1 : 0 },
{ "SUBGROUP", 0 },
{ "UBERSHADER", int(caps.ubershader) },
{ "SMALL_TYPES", caps.supports_small_integer_arithmetic ? 1 : 0 },
});
#else
cmd.set_program(shader_bank->tile_binning_combined);
#endif
subgroup_size = 32;
}
cmd.set_specialization_constant(0, subgroup_size);
unsigned meta_tiles_x = 8;
unsigned meta_tiles_y = subgroup_size / meta_tiles_x;
unsigned num_tiles_x = (push.width + ImplementationConstants::TileWidth - 1) / ImplementationConstants::TileWidth;
unsigned num_tiles_y = (push.height + ImplementationConstants::TileHeight - 1) / ImplementationConstants::TileHeight;
unsigned num_meta_tiles_x = (num_tiles_x + meta_tiles_x - 1) / meta_tiles_x;
unsigned num_meta_tiles_y = (num_tiles_y + meta_tiles_y - 1) / meta_tiles_y;
cmd.dispatch(num_primitives_32, num_meta_tiles_x, num_meta_tiles_y);
if (caps.timestamp >= 2)
{
end_ts = cmd.write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
device->register_time_interval("RDP GPU", std::move(start_ts), std::move(end_ts), "tile-binning");
}
cmd.enable_subgroup_size_control(false);
cmd.end_region();
}
void Renderer::submit_update_upscaled_domain_external(Vulkan::CommandBuffer &cmd,
unsigned addr, unsigned length, unsigned pixel_size_log2)
{
submit_update_upscaled_domain(cmd, ResolveStage::Pre, addr, addr, length, 1, pixel_size_log2);
}
void Renderer::submit_update_upscaled_domain(Vulkan::CommandBuffer &cmd, ResolveStage stage,
unsigned addr, unsigned depth_addr,
unsigned width, unsigned height,
unsigned pixel_size_log2)
{
#ifdef PARALLEL_RDP_SHADER_DIR
if (stage == ResolveStage::Pre)
cmd.set_program("rdp://update_upscaled_domain_pre.comp");
else if (stage == ResolveStage::Post)
cmd.set_program("rdp://update_upscaled_domain_post.comp");
else
cmd.set_program("rdp://update_upscaled_domain_resolve.comp");
#else
if (stage == ResolveStage::Pre)
cmd.set_program(shader_bank->update_upscaled_domain_pre);
else if (stage == ResolveStage::Post)
cmd.set_program(shader_bank->update_upscaled_domain_post);
else
cmd.set_program(shader_bank->update_upscaled_domain_resolve);
#endif
unsigned num_pixels = width * height;
if (stage != ResolveStage::SSAAResolve)
{
// Ensure that we always process entire words, thus we avoid having to do weird swizzles,
// and memory access patterns are linear with gl_GlobalInvocationID.x.
addr &= ~3u;
depth_addr &= ~3u;
unsigned align_pixels = 4u >> pixel_size_log2;
num_pixels = (num_pixels + align_pixels - 1u) & ~(align_pixels - 1u);
}
cmd.set_storage_buffer(0, 0, *rdram, rdram_offset,
(stage == ResolveStage::SSAAResolve && !is_host_coherent ? 2 : 1) * rdram_size);
cmd.set_storage_buffer(0, 1, *hidden_rdram);
cmd.set_storage_buffer(0, 2, *upscaling_reference_rdram);
cmd.set_storage_buffer(0, 3, *upscaling_multisampled_rdram);
cmd.set_storage_buffer(0, 4, *upscaling_multisampled_hidden_rdram);
cmd.set_specialization_constant_mask(0x7f);
cmd.set_specialization_constant(0, uint32_t(rdram_size));
cmd.set_specialization_constant(1, pixel_size_log2);
cmd.set_specialization_constant(2, int(addr == depth_addr));
cmd.set_specialization_constant(3, ImplementationConstants::DefaultWorkgroupSize);
cmd.set_specialization_constant(4, caps.upscaling * caps.upscaling);
if (stage == ResolveStage::SSAAResolve)
{
cmd.set_specialization_constant(5, uint32_t(caps.super_sample_readback_dither));
cmd.set_specialization_constant(6, uint32_t(!is_host_coherent));
}
uint32_t num_workgroups_x, num_workgroups_y;
if (stage == ResolveStage::SSAAResolve)
{
num_workgroups_x =
(width + ImplementationConstants::DefaultWorkgroupSize - 1) /
ImplementationConstants::DefaultWorkgroupSize;
num_workgroups_y = height;
}
else
{
num_workgroups_x =
(num_pixels + ImplementationConstants::DefaultWorkgroupSize - 1) /
ImplementationConstants::DefaultWorkgroupSize;
num_workgroups_y = 1;
}
struct Push
{
uint32_t pixels;
uint32_t fb_addr, fb_depth_addr;
uint32_t width, height;
} push = {};
push.pixels = num_pixels;
push.fb_addr = addr >> pixel_size_log2;
push.fb_depth_addr = depth_addr >> 1;
push.width = width;
push.height = height;
cmd.push_constants(&push, 0, sizeof(push));
Vulkan::QueryPoolHandle start_ts, end_ts;
if (caps.timestamp >= 2 && stage == ResolveStage::SSAAResolve)
start_ts = cmd.write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
cmd.dispatch(num_workgroups_x, num_workgroups_y, 1);
if (caps.timestamp >= 2 && stage == ResolveStage::SSAAResolve)
{
end_ts = cmd.write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
device->register_time_interval("RDP GPU", std::move(start_ts), std::move(end_ts), "ssaa-resolve");
}
}
void Renderer::submit_clear_super_sample_write_mask(Vulkan::CommandBuffer &cmd, unsigned width, unsigned height)
{
// Pack 4x4 block of pixel's writemasks in one u32. Allows for one depth_blend workgroup to target a single u32
// in all cases, which is nice :)
unsigned blocks_x = (width + 3) / 4;
unsigned blocks_y = (height + 3) / 4;
unsigned num_words = blocks_x * blocks_y;
unsigned num_workgroups = (num_words + ImplementationConstants::DefaultWorkgroupSize - 1) /
ImplementationConstants::DefaultWorkgroupSize;
#ifdef PARALLEL_RDP_SHADER_DIR
cmd.set_program("rdp://clear_super_sampled_write_mask.comp");
#else
cmd.set_program(shader_bank->clear_super_sampled_write_mask);
#endif
cmd.set_storage_buffer(0, 0, *upscaling_multisampled_rdram, rdram_size * caps.upscaling * caps.upscaling,
2 * Limits::MaxWidth * Limits::MaxHeight / 8);
cmd.set_specialization_constant_mask(1);
cmd.set_specialization_constant(0, ImplementationConstants::DefaultWorkgroupSize);
cmd.dispatch(num_workgroups, 1, 1);
cmd.set_specialization_constant_mask(0);
}
void Renderer::submit_update_upscaled_domain(Vulkan::CommandBuffer &cmd, ResolveStage stage)
{
unsigned pixel_size_log2;
switch (fb.fmt)
{
case FBFormat::RGBA8888:
pixel_size_log2 = 2;
break;
case FBFormat::RGBA5551:
case FBFormat::IA88:
pixel_size_log2 = 1;
break;
default:
pixel_size_log2 = 0;
break;
}
submit_update_upscaled_domain(cmd, stage, fb.addr, fb.depth_addr,
fb.width, fb.deduced_height, pixel_size_log2);
}
void Renderer::submit_depth_blend(Vulkan::CommandBuffer &cmd, Vulkan::Buffer &tmem, bool upscaled, bool force_write_mask)
{
cmd.begin_region("render-pass");
auto &instance = buffer_instances[buffer_instance];
cmd.set_specialization_constant_mask(0xff);
cmd.set_specialization_constant(0, uint32_t(rdram_size));
cmd.set_specialization_constant(1, uint32_t(fb.fmt));
cmd.set_specialization_constant(2, int(fb.addr == fb.depth_addr));
cmd.set_specialization_constant(3, ImplementationConstants::TileWidth);
cmd.set_specialization_constant(4, ImplementationConstants::TileHeight);
cmd.set_specialization_constant(5, Limits::MaxPrimitives);
cmd.set_specialization_constant(6, upscaled ? caps.max_width : Limits::MaxWidth);
cmd.set_specialization_constant(7, uint32_t(force_write_mask || (!is_host_coherent && !upscaled)) |
((upscaled ? trailing_zeroes(caps.upscaling) : 0u) << 1u));
if (upscaled)
cmd.set_storage_buffer(0, 0, *upscaling_multisampled_rdram);
else
cmd.set_storage_buffer(0, 0, *rdram, rdram_offset, rdram_size * (is_host_coherent ? 1 : 2));
cmd.set_storage_buffer(0, 1, upscaled ? *upscaling_multisampled_hidden_rdram : *hidden_rdram);
cmd.set_storage_buffer(0, 2, tmem);
if (!caps.ubershader)
{
cmd.set_storage_buffer(0, 3, *per_tile_shaded_color);
cmd.set_storage_buffer(0, 4, *per_tile_shaded_depth);
cmd.set_storage_buffer(0, 5, *per_tile_shaded_shaded_alpha);
cmd.set_storage_buffer(0, 6, *per_tile_shaded_coverage);
cmd.set_storage_buffer(0, 7, *per_tile_offsets);
}
cmd.set_storage_buffer(1, 0, *instance.gpu.triangle_setup.buffer);
cmd.set_storage_buffer(1, 1, *instance.gpu.attribute_setup.buffer);
cmd.set_storage_buffer(1, 2, *instance.gpu.derived_setup.buffer);
cmd.set_storage_buffer(1, 3, *instance.gpu.scissor_setup.buffer);
cmd.set_storage_buffer(1, 4, *instance.gpu.static_raster_state.buffer);
cmd.set_storage_buffer(1, 5, *instance.gpu.depth_blend_state.buffer);
cmd.set_storage_buffer(1, 6, *instance.gpu.state_indices.buffer);
cmd.set_storage_buffer(1, 7, *instance.gpu.tile_info_state.buffer);
cmd.set_storage_buffer(1, 8, *span_setups);
cmd.set_storage_buffer(1, 9, *instance.gpu.span_info_offsets.buffer);
cmd.set_buffer_view(1, 10, *blender_divider_buffer);
cmd.set_storage_buffer(1, 11, *tile_binning_buffer);
cmd.set_storage_buffer(1, 12, *tile_binning_buffer_coarse);
auto *global_fb_info = cmd.allocate_typed_constant_data<GlobalFBInfo>(2, 0, 1);
GlobalState push = {};
push.fb_width = fb.width;
push.fb_height = fb.deduced_height;
if (upscaled)
{
push.fb_width *= caps.upscaling;
push.fb_height *= caps.upscaling;
}
switch (fb.fmt)
{
case FBFormat::I4:
push.addr_index = fb.addr;
global_fb_info->fb_size = 0;
global_fb_info->dx_mask = 0;
global_fb_info->dx_shift = 0;
break;
case FBFormat::I8:
push.addr_index = fb.addr;
global_fb_info->fb_size = 1;
global_fb_info->dx_mask = ~7u;
global_fb_info->dx_shift = 3;
break;
case FBFormat::RGBA5551:
case FBFormat::IA88:
push.addr_index = fb.addr >> 1u;
global_fb_info->fb_size = 2;
global_fb_info->dx_mask = ~3u;
global_fb_info->dx_shift = 2;
break;
case FBFormat::RGBA8888:
push.addr_index = fb.addr >> 2u;
global_fb_info->fb_size = 4;
global_fb_info->dx_mask = ~1u;
global_fb_info->dx_shift = 1;
break;
}
global_fb_info->base_primitive_index = base_primitive_index;
push.depth_addr_index = fb.depth_addr >> 1;
unsigned num_primitives_32 = (stream.triangle_setup.size() + 31) / 32;
push.group_mask = (1u << num_primitives_32) - 1;
cmd.push_constants(&push, 0, sizeof(push));
if (caps.ubershader)
{
#ifdef PARALLEL_RDP_SHADER_DIR
cmd.set_program("rdp://ubershader.comp", {
{ "DEBUG_ENABLE", debug_channel ? 1 : 0 },
{ "SMALL_TYPES", caps.supports_small_integer_arithmetic ? 1 : 0 },
{ "SUBGROUP", caps.subgroup_depth_blend ? 1 : 0 },
});
#else
cmd.set_program(shader_bank->ubershader);
#endif
}
else
{
#ifdef PARALLEL_RDP_SHADER_DIR
cmd.set_program("rdp://depth_blend.comp", {
{ "DEBUG_ENABLE", debug_channel ? 1 : 0 },
{ "SMALL_TYPES", caps.supports_small_integer_arithmetic ? 1 : 0 },
{ "SUBGROUP", caps.subgroup_depth_blend ? 1 : 0 },
});
#else
cmd.set_program(shader_bank->depth_blend);
#endif
}
Vulkan::QueryPoolHandle start_ts, end_ts;
if (caps.timestamp >= 2)
start_ts = cmd.write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
cmd.dispatch((push.fb_width + 7) / 8, (push.fb_height + 7) / 8, 1);
if (caps.timestamp >= 2)
{
end_ts = cmd.write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
device->register_time_interval("RDP GPU", std::move(start_ts), std::move(end_ts), "depth-blending");
}
cmd.end_region();
}
void Renderer::submit_render_pass(Vulkan::CommandBuffer &cmd)
{
bool need_render_pass = fb.width != 0 && fb.deduced_height != 0 && !stream.span_info_jobs.empty();
bool need_tmem_upload = !stream.tmem_upload_infos.empty();
bool need_submit = need_render_pass || need_tmem_upload;
if (!need_submit)
return;
Vulkan::QueryPoolHandle render_pass_start, render_pass_end;
if (caps.timestamp >= 1)
render_pass_start = cmd.write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
if (debug_channel)
cmd.begin_debug_channel(this, "Debug", 16 * 1024 * 1024);
// Here we run 3 dispatches in parallel. Span setup and TMEM instances are low occupancy kind of jobs, but the binning
// pass should dominate here unless the workload is trivial.
if (need_render_pass)
{
submit_span_setup_jobs(cmd, false);
submit_tile_binning_combined(cmd, false);
if (caps.upscaling > 1)
submit_update_upscaled_domain(cmd, ResolveStage::Pre);
}
if (need_tmem_upload)
update_tmem_instances(cmd);
cmd.barrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | (!caps.ubershader ? VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT : 0),
VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT |
(!caps.ubershader ? VK_ACCESS_INDIRECT_COMMAND_READ_BIT : 0));
if (need_render_pass && !caps.ubershader)
{
submit_rasterization(cmd, need_tmem_upload ? *tmem_instances : *tmem, false);
cmd.barrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_READ_BIT);
}
if (need_render_pass)
submit_depth_blend(cmd, need_tmem_upload ? *tmem_instances : *tmem, false, false);
if (!caps.ubershader)
clear_indirect_buffer(cmd);
if (render_pass_is_upscaled())
{
cmd.barrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT);
// TODO: Could probably do this reference update in the render pass itself,
// just write output to two buffers ... This is more composable for now.
submit_update_upscaled_domain(cmd, ResolveStage::Post);
}
if (caps.timestamp >= 1)
{
render_pass_end = cmd.write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
device->register_time_interval("RDP GPU", std::move(render_pass_start), std::move(render_pass_end), "render-pass");
}
}
void Renderer::submit_render_pass_upscaled(Vulkan::CommandBuffer &cmd)
{
cmd.begin_region("render-pass-upscaled");
Vulkan::QueryPoolHandle start_ts, end_ts;
if (caps.timestamp >= 1)
start_ts = cmd.write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
bool need_tmem_upload = !stream.tmem_upload_infos.empty();
submit_span_setup_jobs(cmd, true);
submit_tile_binning_combined(cmd, true);
if (caps.super_sample_readback)
{
submit_update_upscaled_domain(cmd, ResolveStage::Pre);
submit_clear_super_sample_write_mask(cmd, fb.width, fb.deduced_height);
if (need_tmem_upload)
update_tmem_instances(cmd);
}
cmd.barrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT |
(!caps.ubershader ? VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT : 0),
VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT |
(!caps.ubershader ? VK_ACCESS_INDIRECT_COMMAND_READ_BIT : 0));
if (!caps.ubershader)
{
submit_rasterization(cmd, need_tmem_upload ? *tmem_instances : *tmem, true);
cmd.barrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
VK_ACCESS_2_SHADER_STORAGE_READ_BIT);
}
submit_depth_blend(cmd, need_tmem_upload ? *tmem_instances : *tmem, true, caps.super_sample_readback);
if (!caps.ubershader)
clear_indirect_buffer(cmd);
if (caps.super_sample_readback)
{
cmd.begin_region("ssaa-resolve");
cmd.barrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT);
submit_update_upscaled_domain(cmd, ResolveStage::SSAAResolve);
cmd.end_region();
}
if (caps.timestamp >= 1)
{
end_ts = cmd.write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
device->register_time_interval("RDP GPU", std::move(start_ts), std::move(end_ts), "render-pass-upscaled");
}
cmd.end_region();
}
void Renderer::submit_render_pass_end(Vulkan::CommandBuffer &cmd)
{
base_primitive_index += uint32_t(stream.triangle_setup.size());
cmd.barrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
VK_ACCESS_2_SHADER_STORAGE_READ_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT);
}
void Renderer::maintain_queues()
{
// Some conditions dictate if we should flush a render pass.
// These heuristics ensures we don't wait too long to flush render passes,
// and also ensure that we don't spam submissions too often, causing massive bubbles on GPU.
// If we get a lot of small render passes in a row, it makes sense to batch them up, e.g. 8 at a time.
// If we get 2 full render passes of ~256 primitives, that's also a good indication we should flush since we're getting spammed.
// If we have no pending submissions, the GPU is idle and there is no reason not to submit.
// If we haven't submitted anything in a while (1.0 ms), it's probably fine to submit again.
if (pending_render_passes >= ImplementationConstants::MaxPendingRenderPassesBeforeFlush ||
(caps.super_sample_readback && pending_render_passes_upscaled >= ImplementationConstants::MaxPendingRenderPassesBeforeFlush) ||
pending_primitives >= Limits::MaxPrimitives ||
pending_primitives_upscaled >= Limits::MaxPrimitives ||
active_submissions.load(std::memory_order_relaxed) == 0 ||
int64_t(Util::get_current_time_nsecs() - last_submit_ns) > 1000000)
{
submit_to_queue();
}
}
void Renderer::lock_command_processing()
{
idle_lock.lock();
}
void Renderer::unlock_command_processing()
{
idle_lock.unlock();
}
void Renderer::maintain_queues_idle()
{
std::lock_guard<std::mutex> holder{idle_lock};
if (pending_primitives >= ImplementationConstants::MinimumPrimitivesForIdleFlush ||
pending_render_passes >= ImplementationConstants::MinimumRenderPassesForIdleFlush)
{
flush_queues();
submit_to_queue();
}
}
void Renderer::enqueue_fence_wait(Vulkan::Fence fence)
{
CoherencyOperation op;
op.fence = std::move(fence);
op.unlock_cookie = &active_submissions;
active_submissions.fetch_add(1, std::memory_order_relaxed);
processor.enqueue_coherency_operation(std::move(op));
last_submit_ns = Util::get_current_time_nsecs();
}
void Renderer::submit_to_queue()
{
bool pending_host_visible_render_passes =
(caps.super_sample_readback ? pending_render_passes_upscaled : pending_render_passes) != 0;
bool pending_upscaled_passes = pending_render_passes_upscaled != 0;
pending_render_passes = 0;
pending_render_passes_upscaled = 0;
pending_primitives = 0;
pending_primitives_upscaled = 0;
if (!stream.cmd)
{
if (pending_host_visible_render_passes)
{
Vulkan::Fence fence;
device->submit_empty(Vulkan::CommandBuffer::Type::AsyncCompute, &fence);
enqueue_fence_wait(fence);
}
return;
}
bool need_host_barrier = is_host_coherent || !incoherent.staging_readback;
// If we maintain queues in-between doing 1x render pass and upscaled render pass,
// we haven't flushed memory yet.
bool need_memory_flush = pending_host_visible_render_passes && !pending_upscaled_passes;
stream.cmd->barrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
need_memory_flush ? VK_ACCESS_MEMORY_WRITE_BIT : 0,
(need_host_barrier ? VK_PIPELINE_STAGE_2_HOST_BIT : VK_PIPELINE_STAGE_2_COPY_BIT),
(need_host_barrier ? VK_ACCESS_HOST_READ_BIT : VK_ACCESS_TRANSFER_READ_BIT));
Vulkan::Fence fence;
if (is_host_coherent)
{
device->submit(stream.cmd, &fence);
if (pending_host_visible_render_passes)
enqueue_fence_wait(fence);
}
else
{
CoherencyOperation op;
if (pending_host_visible_render_passes)
resolve_coherency_gpu_to_host(op, *stream.cmd);
device->submit(stream.cmd, &fence);
if (pending_host_visible_render_passes)
{
enqueue_fence_wait(fence);
op.fence = fence;
if (!op.copies.empty())
processor.enqueue_coherency_operation(std::move(op));
}
}
Util::for_each_bit(sync_indices_needs_flush, [&](unsigned bit) {
auto &sync = internal_sync[bit];
sync.fence = fence;
});
sync_indices_needs_flush = 0;
stream.cmd.reset();
}
void Renderer::reset_context()
{
stream.scissor_setup.reset();
stream.static_raster_state_cache.reset();
stream.depth_blend_state_cache.reset();
stream.tile_info_state_cache.reset();
stream.triangle_setup.reset();
stream.attribute_setup.reset();
stream.derived_setup.reset();
stream.state_indices.reset();
stream.span_info_offsets.reset();
stream.span_info_jobs.reset();
stream.max_shaded_tiles = 0;
fb.deduced_height = 0;
fb.color_write_pending = false;
fb.depth_write_pending = false;
stream.tmem_upload_infos.clear();
}
void Renderer::begin_new_context()
{
buffer_instance = (buffer_instance + 1) % Limits::NumSyncStates;
reset_context();
}
uint32_t Renderer::get_byte_size_for_bound_color_framebuffer() const
{
unsigned pixel_count = fb.width * fb.deduced_height;
unsigned byte_count;
switch (fb.fmt)
{
case FBFormat::RGBA8888:
byte_count = pixel_count * 4;
break;
case FBFormat::RGBA5551:
case FBFormat::IA88:
byte_count = pixel_count * 2;
break;
default:
byte_count = pixel_count;
break;
}
return byte_count;
}
uint32_t Renderer::get_byte_size_for_bound_depth_framebuffer() const
{
return fb.width * fb.deduced_height * 2;
}
void Renderer::mark_pages_for_gpu_read(uint32_t base_addr, uint32_t byte_count)
{
if (byte_count == 0)
return;
uint32_t start_page = base_addr / ImplementationConstants::IncoherentPageSize;
uint32_t end_page = (base_addr + byte_count - 1) / ImplementationConstants::IncoherentPageSize + 1;
start_page &= incoherent.num_pages - 1;
end_page &= incoherent.num_pages - 1;
uint32_t page = start_page;
while (page != end_page)
{
bool pending_writes = (incoherent.page_to_pending_readback[page / 32] & (1u << (page & 31))) != 0 ||
incoherent.pending_writes_for_page[page].load(std::memory_order_relaxed) != 0;
// We'll do an acquire memory barrier later before we start memcpy-ing from host memory.
if (pending_writes)
incoherent.page_to_masked_copy[page / 32] |= 1u << (page & 31);
else
incoherent.page_to_direct_copy[page / 32] |= 1u << (page & 31);
page = (page + 1) & (incoherent.num_pages - 1);
}
}
void Renderer::lock_pages_for_gpu_write(uint32_t base_addr, uint32_t byte_count)
{
if (byte_count == 0)
return;
uint32_t start_page = base_addr / ImplementationConstants::IncoherentPageSize;
uint32_t end_page = (base_addr + byte_count - 1) / ImplementationConstants::IncoherentPageSize + 1;
for (uint32_t page = start_page; page < end_page; page++)
{
uint32_t wrapped_page = page & (incoherent.num_pages - 1);
incoherent.page_to_pending_readback[wrapped_page / 32] |= 1u << (wrapped_page & 31);
}
}
void Renderer::resolve_coherency_gpu_to_host(CoherencyOperation &op, Vulkan::CommandBuffer &cmd)
{
cmd.begin_region("resolve-coherency-gpu-to-host");
if (!incoherent.staging_readback)
{
// iGPU path.
op.src = rdram;
op.dst = incoherent.host_rdram;
op.timeline_value = 0;
for (auto &readback : incoherent.page_to_pending_readback)
{
uint32_t base_index = 32 * uint32_t(&readback - incoherent.page_to_pending_readback.data());
Util::for_each_bit_range(readback, [&](unsigned index, unsigned count) {
index += base_index;
for (unsigned i = 0; i < count; i++)
incoherent.pending_writes_for_page[index + i].fetch_add(1, std::memory_order_relaxed);
CoherencyCopy coherent_copy = {};
coherent_copy.counter_base = &incoherent.pending_writes_for_page[index];
coherent_copy.counters = count;
coherent_copy.src_offset = index * ImplementationConstants::IncoherentPageSize;
coherent_copy.mask_offset = coherent_copy.src_offset + rdram_size;
coherent_copy.dst_offset = index * ImplementationConstants::IncoherentPageSize;
coherent_copy.size = ImplementationConstants::IncoherentPageSize * count;
op.copies.push_back(coherent_copy);
});
readback = 0;
}
}
else
{
// Discrete GPU path.
Util::SmallVector<VkBufferCopy, 1024> copies;
op.src = incoherent.staging_readback.get();
op.dst = incoherent.host_rdram;
op.timeline_value = 0;
for (auto &readback : incoherent.page_to_pending_readback)
{
uint32_t base_index = 32 * uint32_t(&readback - incoherent.page_to_pending_readback.data());
Util::for_each_bit_range(readback, [&](unsigned index, unsigned count) {
index += base_index;
for (unsigned i = 0; i < count; i++)
incoherent.pending_writes_for_page[index + i].fetch_add(1, std::memory_order_relaxed);
VkBufferCopy copy = {};
copy.srcOffset = index * ImplementationConstants::IncoherentPageSize;
unsigned dst_page_index = incoherent.staging_readback_index;
copy.dstOffset = dst_page_index * ImplementationConstants::IncoherentPageSize;
incoherent.staging_readback_index += count;
incoherent.staging_readback_index &= (incoherent.staging_readback_pages - 1);
// Unclean wraparound check.
if (incoherent.staging_readback_index != 0 && incoherent.staging_readback_index < dst_page_index)
{
copy.dstOffset = 0;
incoherent.staging_readback_index = count;
}
copy.size = ImplementationConstants::IncoherentPageSize * count;
copies.push_back(copy);
CoherencyCopy coherent_copy = {};
coherent_copy.counter_base = &incoherent.pending_writes_for_page[index];
coherent_copy.counters = count;
coherent_copy.src_offset = copy.dstOffset;
coherent_copy.dst_offset = index * ImplementationConstants::IncoherentPageSize;
coherent_copy.size = ImplementationConstants::IncoherentPageSize * count;
VkBufferCopy mask_copy = {};
mask_copy.srcOffset = index * ImplementationConstants::IncoherentPageSize + rdram_size;
dst_page_index = incoherent.staging_readback_index;
mask_copy.dstOffset = dst_page_index * ImplementationConstants::IncoherentPageSize;
incoherent.staging_readback_index += count;
incoherent.staging_readback_index &= (incoherent.staging_readback_pages - 1);
// Unclean wraparound check.
if (incoherent.staging_readback_index != 0 && incoherent.staging_readback_index < dst_page_index)
{
mask_copy.dstOffset = 0;
incoherent.staging_readback_index = count;
}
mask_copy.size = ImplementationConstants::IncoherentPageSize * count;
copies.push_back(mask_copy);
coherent_copy.mask_offset = mask_copy.dstOffset;
op.copies.push_back(coherent_copy);
});
readback = 0;
}
if (!copies.empty())
{
//#define COHERENCY_READBACK_TIMESTAMPS
#ifdef COHERENCY_READBACK_TIMESTAMPS
Vulkan::QueryPoolHandle start_ts, end_ts;
start_ts = cmd.write_timestamp(VK_PIPELINE_STAGE_2_COPY_BIT);
#endif
cmd.copy_buffer(*incoherent.staging_readback, *rdram, copies.data(), copies.size());
#ifdef COHERENCY_READBACK_TIMESTAMPS
end_ts = cmd.write_timestamp(VK_PIPELINE_STAGE_2_COPY_BIT);
device->register_time_interval(std::move(start_ts), std::move(end_ts), "coherency-readback");
#endif
cmd.barrier(VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_TRANSFER_WRITE_BIT,
VK_PIPELINE_STAGE_HOST_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
VK_ACCESS_HOST_READ_BIT);
}
}
cmd.end_region();
}
void Renderer::resolve_coherency_external(unsigned offset, unsigned length)
{
mark_pages_for_gpu_read(offset, length);
ensure_command_buffer();
resolve_coherency_host_to_gpu(*stream.cmd);
device->submit(stream.cmd);
stream.cmd.reset();
}
unsigned Renderer::get_scaling_factor() const
{
return caps.upscaling;
}
const Vulkan::Buffer *Renderer::get_upscaled_rdram_buffer() const
{
return upscaling_multisampled_rdram.get();
}
const Vulkan::Buffer *Renderer::get_upscaled_hidden_rdram_buffer() const
{
return upscaling_multisampled_hidden_rdram.get();
}
void Renderer::resolve_coherency_host_to_gpu(Vulkan::CommandBuffer &cmd)
{
// Now, ensure that the GPU sees a coherent view of the CPU memory writes up until now.
// Writes made by the GPU which are not known to be resolved on the timeline waiter thread will always
// "win" over writes made by CPU, since CPU is not allowed to meaningfully overwrite data which the GPU
// is going to touch.
cmd.begin_region("resolve-coherency-host-to-gpu");
Vulkan::QueryPoolHandle start_ts, end_ts;
if (caps.timestamp)
start_ts = device->write_calibrated_timestamp();
std::atomic_thread_fence(std::memory_order_acquire);
Util::SmallVector<VkBufferCopy, 1024> buffer_copies;
Util::SmallVector<uint32_t, 1024> masked_page_copies;
Util::SmallVector<uint32_t, 1024> to_clear_write_mask;
// If we're able to map RDRAM directly, we can just memcpy straight into RDRAM if we have an unmasked copy.
// Important for iGPU.
if (rdram->get_allocation().is_host_allocation())
{
for (auto &direct : incoherent.page_to_direct_copy)
{
uint32_t base_index = 32 * (&direct - incoherent.page_to_direct_copy.data());
Util::for_each_bit_range(direct, [&](unsigned index, unsigned count) {
index += base_index;
auto *mapped_rdram = device->map_host_buffer(*rdram, Vulkan::MEMORY_ACCESS_WRITE_BIT,
ImplementationConstants::IncoherentPageSize * index,
ImplementationConstants::IncoherentPageSize * count);
memcpy(mapped_rdram,
incoherent.host_rdram + ImplementationConstants::IncoherentPageSize * index,
ImplementationConstants::IncoherentPageSize * count);
device->unmap_host_buffer(*rdram, Vulkan::MEMORY_ACCESS_WRITE_BIT,
ImplementationConstants::IncoherentPageSize * index,
ImplementationConstants::IncoherentPageSize * count);
mapped_rdram = device->map_host_buffer(*rdram, Vulkan::MEMORY_ACCESS_WRITE_BIT,
ImplementationConstants::IncoherentPageSize * index + rdram_size,
ImplementationConstants::IncoherentPageSize * count);
memset(mapped_rdram, 0, ImplementationConstants::IncoherentPageSize * count);
device->unmap_host_buffer(*rdram, Vulkan::MEMORY_ACCESS_WRITE_BIT,
ImplementationConstants::IncoherentPageSize * index + rdram_size,
ImplementationConstants::IncoherentPageSize * count);
});
direct = 0;
}
auto *mapped_staging = static_cast<uint8_t *>(device->map_host_buffer(*incoherent.staging_rdram,
Vulkan::MEMORY_ACCESS_WRITE_BIT));
for (auto &indirect : incoherent.page_to_masked_copy)
{
uint32_t base_index = 32 * (&indirect - incoherent.page_to_masked_copy.data());
Util::for_each_bit(indirect, [&](unsigned index) {
index += base_index;
masked_page_copies.push_back(index);
memcpy(mapped_staging + ImplementationConstants::IncoherentPageSize * index,
incoherent.host_rdram + ImplementationConstants::IncoherentPageSize * index,
ImplementationConstants::IncoherentPageSize);
});
indirect = 0;
}
device->unmap_host_buffer(*incoherent.staging_rdram, Vulkan::MEMORY_ACCESS_WRITE_BIT);
}
else
{
auto *mapped_rdram = static_cast<uint8_t *>(device->map_host_buffer(*incoherent.staging_rdram, Vulkan::MEMORY_ACCESS_WRITE_BIT));
size_t num_packed_pages = incoherent.page_to_masked_copy.size();
for (size_t i = 0; i < num_packed_pages; i++)
{
uint32_t base_index = 32 * i;
uint32_t tmp = incoherent.page_to_masked_copy[i] | incoherent.page_to_direct_copy[i];
Util::for_each_bit(tmp, [&](unsigned index) {
unsigned bit = index;
index += base_index;
if ((1u << bit) & incoherent.page_to_masked_copy[i])
masked_page_copies.push_back(index);
else
{
VkBufferCopy copy = {};
copy.size = ImplementationConstants::IncoherentPageSize;
copy.dstOffset = copy.srcOffset = index * ImplementationConstants::IncoherentPageSize;
buffer_copies.push_back(copy);
to_clear_write_mask.push_back(index);
}
memcpy(mapped_rdram + ImplementationConstants::IncoherentPageSize * index,
incoherent.host_rdram + ImplementationConstants::IncoherentPageSize * index,
ImplementationConstants::IncoherentPageSize);
});
incoherent.page_to_masked_copy[i] = 0;
incoherent.page_to_direct_copy[i] = 0;
}
device->unmap_host_buffer(*incoherent.staging_rdram, Vulkan::MEMORY_ACCESS_WRITE_BIT);
}
if (!masked_page_copies.empty())
{
#ifdef PARALLEL_RDP_SHADER_DIR
cmd.set_program("rdp://masked_rdram_resolve.comp");
#else
cmd.set_program(shader_bank->masked_rdram_resolve);
#endif
cmd.set_specialization_constant_mask(3);
cmd.set_specialization_constant(0, ImplementationConstants::IncoherentPageSize / 4);
cmd.set_specialization_constant(1, ImplementationConstants::IncoherentPageSize / 4);
cmd.set_storage_buffer(0, 0, *rdram, rdram_offset, rdram_size);
cmd.set_storage_buffer(0, 1, *incoherent.staging_rdram);
cmd.set_storage_buffer(0, 2, *rdram, rdram_offset + rdram_size, rdram_size);
//#define COHERENCY_MASK_TIMESTAMPS
#ifdef COHERENCY_MASK_TIMESTAMPS
Vulkan::QueryPoolHandle start_ts, end_ts;
start_ts = cmd->write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
#endif
for (size_t i = 0; i < masked_page_copies.size(); i += 4096)
{
size_t to_copy = std::min(masked_page_copies.size() - i, size_t(4096));
memcpy(cmd.allocate_typed_constant_data<uint32_t>(1, 0, to_copy),
masked_page_copies.data() + i,
to_copy * sizeof(uint32_t));
cmd.dispatch(to_copy, 1, 1);
}
#ifdef COHERENCY_MASK_TIMESTAMPS
end_ts = cmd->write_timestamp(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
device->register_time_interval(std::move(start_ts), std::move(end_ts), "coherent-mask-copy");
#endif
}
// Could use FillBuffer here, but would need to use TRANSFER stage, and introduce more barriers than needed.
if (!to_clear_write_mask.empty())
{
#ifdef PARALLEL_RDP_SHADER_DIR
cmd.set_program("rdp://clear_write_mask.comp");
#else
cmd.set_program(shader_bank->clear_write_mask);
#endif
cmd.set_specialization_constant_mask(3);
cmd.set_specialization_constant(0, ImplementationConstants::IncoherentPageSize / 4);
cmd.set_specialization_constant(1, ImplementationConstants::IncoherentPageSize / 4);
cmd.set_storage_buffer(0, 0, *rdram, rdram_offset + rdram_size, rdram_size);
for (size_t i = 0; i < to_clear_write_mask.size(); i += 4096)
{
size_t to_copy = std::min(to_clear_write_mask.size() - i, size_t(4096));
memcpy(cmd.allocate_typed_constant_data<uint32_t>(1, 0, to_copy),
to_clear_write_mask.data() + i,
to_copy * sizeof(uint32_t));
cmd.dispatch(to_copy, 1, 1);
}
}
if (!to_clear_write_mask.empty() || !masked_page_copies.empty())
{
cmd.barrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_READ_BIT);
}
// If we cannot map the device memory, copy. We're latency sensitive, so don't use DMA queue.
if (!buffer_copies.empty())
{
cmd.barrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0,
VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_TRANSFER_WRITE_BIT);
//#define COHERENCY_COPY_TIMESTAMPS
#ifdef COHERENCY_COPY_TIMESTAMPS
Vulkan::QueryPoolHandle start_ts, end_ts;
start_ts = cmd->write_timestamp(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT);
#endif
cmd.copy_buffer(*rdram, *incoherent.staging_rdram, buffer_copies.data(), buffer_copies.size());
#ifdef COHERENCY_COPY_TIMESTAMPS
end_ts = cmd->write_timestamp(VK_PIPELINE_STAGE_2_COPY_BIT);
device->register_time_interval(std::move(start_ts), std::move(end_ts), "coherent-copy");
#endif
cmd.barrier(VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_TRANSFER_WRITE_BIT,
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_READ_BIT);
}
if (caps.timestamp)
{
end_ts = device->write_calibrated_timestamp();
device->register_time_interval("RDP CPU", std::move(start_ts), std::move(end_ts), "coherency-host-to-gpu");
}
cmd.end_region();
}
void Renderer::flush_queues()
{
if (stream.tmem_upload_infos.empty() && stream.span_info_jobs.empty())
{
base_primitive_index += stream.triangle_setup.size();
reset_context();
return;
}
if (!is_host_coherent)
{
mark_pages_for_gpu_read(fb.addr, get_byte_size_for_bound_color_framebuffer());
mark_pages_for_gpu_read(fb.depth_addr, get_byte_size_for_bound_depth_framebuffer());
// We're going to write to these pages, so lock them down.
lock_pages_for_gpu_write(fb.addr, get_byte_size_for_bound_color_framebuffer());
lock_pages_for_gpu_write(fb.depth_addr, get_byte_size_for_bound_depth_framebuffer());
}
auto &instance = buffer_instances[buffer_instance];
auto &sync = internal_sync[buffer_instance];
if (sync_indices_needs_flush & (1u << buffer_instance))
submit_to_queue();
sync_indices_needs_flush |= 1u << buffer_instance;
if (sync.fence)
{
Vulkan::QueryPoolHandle start_ts, end_ts;
if (caps.timestamp)
start_ts = device->write_calibrated_timestamp();
sync.fence->wait();
if (caps.timestamp)
{
end_ts = device->write_calibrated_timestamp();
device->register_time_interval("RDP CPU", std::move(start_ts), std::move(end_ts), "render-pass-fence");
}
sync.fence.reset();
}
ensure_command_buffer();
if (!is_host_coherent)
resolve_coherency_host_to_gpu(*stream.cmd);
instance.upload(*device, stream, *stream.cmd);
// If we have super-sampled readback, then this is meaningless.
bool has_single_sampled_render_pass = !caps.super_sample_readback;
if (has_single_sampled_render_pass)
{
stream.cmd->begin_region("render-pass-1x");
submit_render_pass(*stream.cmd);
stream.cmd->end_region();
pending_render_passes++;
}
if (render_pass_is_upscaled())
{
if (has_single_sampled_render_pass)
{
maintain_queues();
ensure_command_buffer();
// We're going to keep reading the same data structures, so make sure
// we signal fence after upscaled render pass is submitted.
sync_indices_needs_flush |= 1u << buffer_instance;
}
submit_render_pass_upscaled(*stream.cmd);
pending_render_passes_upscaled++;
pending_primitives_upscaled += uint32_t(stream.triangle_setup.size());
}
submit_render_pass_end(*stream.cmd);
begin_new_context();
maintain_queues();
}
bool Renderer::render_pass_is_upscaled() const
{
if (caps.super_sample_readback)
return true;
bool need_render_pass = fb.width != 0 && fb.deduced_height != 0 && !stream.span_info_jobs.empty();
return need_render_pass && should_render_upscaled();
}
bool Renderer::should_render_upscaled() const
{
if (caps.upscaling > 1)
{
// A heuristic. There is no point to render upscaled for purely off-screen passes.
// We should ideally only upscale the final pass which hits screen.
// From a heuristic point-of-view we expect only 16-bit/32-bit frame buffers to be relevant,
// and only frame buffers with at least 256 pixels.
return (fb.fmt == FBFormat::RGBA5551 || fb.fmt == FBFormat::RGBA8888) && fb.width >= 256;
}
else
return false;
}
void Renderer::ensure_command_buffer()
{
if (!stream.cmd)
stream.cmd = device->request_command_buffer(Vulkan::CommandBuffer::Type::AsyncCompute);
if (!caps.ubershader && !indirect_dispatch_buffer)
{
Vulkan::BufferCreateInfo indirect_info = {};
indirect_info.size = 4 * sizeof(uint32_t) * Limits::MaxStaticRasterizationStates;
indirect_info.domain = Vulkan::BufferDomain::Device;
indirect_info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT;
indirect_dispatch_buffer = device->create_buffer(indirect_info);
device->set_name(*indirect_dispatch_buffer, "indirect-dispatch-buffer");
clear_indirect_buffer(*stream.cmd);
stream.cmd->barrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT,
VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT | VK_ACCESS_2_SHADER_STORAGE_READ_BIT);
}
}
void Renderer::set_tile(uint32_t tile, const TileMeta &meta)
{
tiles[tile].meta = meta;
}
void Renderer::set_tile_size(uint32_t tile, uint32_t slo, uint32_t shi, uint32_t tlo, uint32_t thi)
{
tiles[tile].size.slo = slo;
tiles[tile].size.shi = shi;
tiles[tile].size.tlo = tlo;
tiles[tile].size.thi = thi;
}
void Renderer::notify_idle_command_thread()
{
maintain_queues_idle();
}
bool Renderer::tmem_upload_needs_flush(uint32_t addr) const
{
// Not perfect, since TMEM upload could slice into framebuffer,
// but I doubt this will be an issue (famous last words ...)
if (fb.color_write_pending)
{
uint32_t offset = (addr - fb.addr) & (rdram_size - 1);
uint32_t pending_pixels = fb.deduced_height * fb.width;
switch (fb.fmt)
{
case FBFormat::RGBA5551:
case FBFormat::I8:
offset >>= 1;
break;
case FBFormat::RGBA8888:
offset >>= 2;
break;
default:
break;
}
if (offset < pending_pixels)
{
//LOGI("Flushing render pass due to coherent TMEM fetch from color buffer.\n");
return true;
}
}
if (fb.depth_write_pending)
{
uint32_t offset = (addr - fb.depth_addr) & (rdram_size - 1);
uint32_t pending_pixels = fb.deduced_height * fb.width;
offset >>= 1;
if (offset < pending_pixels)
{
//LOGI("Flushing render pass due to coherent TMEM fetch from depth buffer.\n");
return true;
}
}
return false;
}
void Renderer::load_tile(uint32_t tile, const LoadTileInfo &info)
{
if (validation_iface && info.mode == UploadMode::TLUT)
{
if ((info.thi >> 2) > (info.tlo >> 2))
{
validation_iface->report_rdp_crash(ValidationError::InvalidMultilineLoadTlut,
"Attempting to load multiple lines in TLUT.");
}
}
if (tmem_upload_needs_flush(info.tex_addr))
flush_queues();
// Detect noop cases.
if (info.mode != UploadMode::Block)
{
if ((info.thi >> 2) < (info.tlo >> 2))
return;
unsigned pixel_count = (((info.shi >> 2) - (info.slo >> 2)) + 1) & 0xfff;
if (!pixel_count)
return;
}
else
{
unsigned pixel_count = ((info.shi - info.slo) + 1) & 0xfff;
if (!pixel_count || pixel_count > 2048)
return;
}
if (!is_host_coherent)
{
unsigned pixel_count;
unsigned offset_pixels;
unsigned base_addr = info.tex_addr;
if (info.mode == UploadMode::Block)
{
pixel_count = (info.shi - info.slo + 1) & 0xfff;
offset_pixels = info.slo + info.tex_width * info.tlo;
}
else
{
unsigned max_x = ((info.shi >> 2) - (info.slo >> 2)) & 0xfff;
unsigned max_y = (info.thi >> 2) - (info.tlo >> 2);
pixel_count = max_y * info.tex_width + max_x + 1;
offset_pixels = (info.slo >> 2) + info.tex_width * (info.tlo >> 2);
}
unsigned byte_size = pixel_count << (unsigned(info.size) - 1);
byte_size = (byte_size + 7) & ~7;
base_addr += offset_pixels << (unsigned(info.size) - 1);
mark_pages_for_gpu_read(base_addr, byte_size);
}
if (info.mode == UploadMode::Tile)
{
auto &meta = tiles[tile].meta;
unsigned pixels_coverered_per_line = (((info.shi >> 2) - (info.slo >> 2)) + 1) & 0xfff;
// Technically, 32-bpp TMEM upload and YUV upload will work like 16bpp, just split into two halves, but that also means
// we get 2kB wraparound instead of 4kB wraparound, so this works out just fine for our purposes.
unsigned quad_words_covered_per_line = ((pixels_coverered_per_line << unsigned(meta.size)) + 15) >> 4;
// Deal with mismatch in state, there is no reasonable scenarios where this should even matter, but you never know ...
if (unsigned(meta.size) > unsigned(info.size))
quad_words_covered_per_line <<= unsigned(meta.size) - unsigned(info.size);
else if (unsigned(meta.size) < unsigned(info.size))
quad_words_covered_per_line >>= unsigned(info.size) - unsigned(meta.size);
// Compute a conservative estimate for how many bytes we're going to splat down into TMEM.
unsigned bytes_covered_per_line = std::max<unsigned>(quad_words_covered_per_line * 8, meta.stride);
unsigned max_bytes_per_line = 0x1000;
// We need to write lower and upper halves at once,
// so we need to wrap around at 2k boundary.
if (meta.fmt == TextureFormat::YUV)
max_bytes_per_line /= 2;
unsigned num_lines = ((info.thi >> 2) - (info.tlo >> 2)) + 1;
unsigned total_bytes_covered = bytes_covered_per_line * num_lines;
if (total_bytes_covered > max_bytes_per_line)
{
// Welp, for whatever reason, the game wants to write more than 4k of texture data to TMEM in one go.
// We can only handle 4kB in one go due to wrap-around effects,
// so split up the upload in multiple chunks.
unsigned max_lines_per_iteration = max_bytes_per_line / bytes_covered_per_line;
// Align T-state.
max_lines_per_iteration &= ~1u;
if (max_lines_per_iteration == 0)
{
LOGE("Pure insanity where content is attempting to load more than 2kB of TMEM data in one single line ...\n");
// Could be supported if we start splitting up horizonal direction as well, but seriously ...
return;
}
for (unsigned line = 0; line < num_lines; line += max_lines_per_iteration)
{
unsigned to_copy_lines = std::min(num_lines - line, max_lines_per_iteration);
LoadTileInfo tmp_info = info;
tmp_info.tlo = info.tlo + (line << 2);
tmp_info.thi = tmp_info.tlo + ((to_copy_lines - 1) << 2);
load_tile_iteration(tile, tmp_info, line * meta.stride);
}
auto &size = tiles[tile].size;
size.slo = info.slo;
size.shi = info.shi;
size.tlo = info.tlo;
size.thi = info.thi;
}
else
load_tile_iteration(tile, info, 0);
}
else
load_tile_iteration(tile, info, 0);
}
void Renderer::load_tile_iteration(uint32_t tile, const LoadTileInfo &info, uint32_t tmem_offset)
{
auto &size = tiles[tile].size;
auto &meta = tiles[tile].meta;
size.slo = info.slo;
size.shi = info.shi;
size.tlo = info.tlo;
size.thi = info.thi;
if (meta.fmt == TextureFormat::YUV && ((meta.size != TextureSize::Bpp16) || (info.size != TextureSize::Bpp16)))
{
LOGE("Only 16bpp is supported for YUV uploads.\n");
return;
}
// This case does not appear to be supported.
if (info.size == TextureSize::Bpp4)
{
LOGE("4-bit VRAM pointer crashes the RDP.\n");
if (validation_iface)
validation_iface->report_rdp_crash(ValidationError::LoadTile4bpp, "4-bit VRAM pointer crashes the RDP.");
return;
}
if (meta.size == TextureSize::Bpp32 && meta.fmt != TextureFormat::RGBA)
{
LOGE("32bpp tile uploads must using RGBA texture format, unsupported otherwise.\n");
return;
}
if (info.mode == UploadMode::TLUT && meta.size == TextureSize::Bpp32)
{
LOGE("TLUT uploads with 32bpp tiles are unsupported.\n");
return;
}
if (info.mode != UploadMode::TLUT)
{
if (info.size == TextureSize::Bpp32 && meta.size == TextureSize::Bpp8)
{
LOGE("FIXME: Loading tile with Texture 32-bit and Tile 8-bit. This creates insane results, unsupported.\n");
return;
}
else if (info.size == TextureSize::Bpp16 && meta.size == TextureSize::Bpp4)
{
LOGE("FIXME: Loading tile with Texture 16-bit and Tile 4-bit. This creates insane results, unsupported.\n");
return;
}
else if (info.size == TextureSize::Bpp32 && meta.size == TextureSize::Bpp4)
{
LOGE("FIXME: Loading tile with Texture 32-bit and Tile 4-bit. This creates insane results, unsupported.\n");
return;
}
}
UploadInfo upload = {};
upload.tmem_stride_words = meta.stride >> 1;
uint32_t upload_x = 0;
uint32_t upload_y = 0;
auto upload_mode = info.mode;
if (upload_mode == UploadMode::Block)
{
upload_x = info.slo;
upload_y = info.tlo;
// LoadBlock is kinda awkward. Rather than specifying width and height, we get width and dTdx.
// dTdx will increment and generate a T coordinate based on S coordinate (T = (S_64bpp_word * dTdx) >> 11).
// The stride is added on top of this, so effective stride is stride(T) + stride(tile).
// Usually it makes sense for stride(tile) to be 0, but it doesn't have to be ...
// The only reasonable solution is to try to decompose this mess into a normal width/height/stride.
// In the general dTdx case, we don't have to deduce a stable value for stride.
// If dTdx is very weird, we might get variable stride, which is near-impossible to deal with.
// However, it makes zero sense for content to actually rely on this behavior.
// Even if there are inaccuracies in the fraction, we always floor it to get T, and thus we'll have to run
// for quite some time to observe the fractional error accumulate.
unsigned pixel_count = (info.shi - info.slo + 1) & 0xfff;
unsigned dt = info.thi;
unsigned max_tmem_iteration = (pixel_count - 1) >> (4u - unsigned(info.size));
unsigned max_t = (max_tmem_iteration * dt) >> 11;
if (max_t != 0)
{
// dT is an inverse which is not necessarily accurate, we can end up with an uneven amount of
// texels per "line". If we have stride == 0, this is fairly easy to deal with,
// but for the case where stride != 0, it is very difficult to implement it correctly.
// We will need to solve this kind of equation for X:
// TMEM word = floor((x * dt) / 2048) * stride + x
// This equation has no solutions for cases where we stride over TMEM words.
// The only way I can think of is to test all candidates for the floor() expression, and see if that is a valid solution.
// We can find an conservative estimate for floor() by:
// t_min = TMEM word / (max_num_64bpp_elements + stride)
// t_max = TMEM word / (min_num_64bpp_elements + stride)
unsigned max_num_64bpp_elements_before_wrap = ((1u << 11u) + dt - 1u) / dt;
unsigned min_num_64bpp_elements_before_wrap = (1u << 11u) / dt;
bool uneven_dt = max_num_64bpp_elements_before_wrap != min_num_64bpp_elements_before_wrap;
if (uneven_dt)
{
// If we never get rounding errors, we can handwave this issue away and pretend that min == max iterations.
// This is by far the common case.
// Each overflow into next T adds a certain amount of error.
unsigned overflow_amt = dt * max_num_64bpp_elements_before_wrap - (1 << 11);
// Multiply this by maximum value of T we can observe, and we have a conservative estimate for our T error.
overflow_amt *= max_t;
// If this error is less than 1 step of dt, we can be certain that we will get max_num iterations every time,
// and we can ignore the worst edge cases.
if (overflow_amt < dt)
{
min_num_64bpp_elements_before_wrap = max_num_64bpp_elements_before_wrap;
uneven_dt = false;
}
}
// Add more precision bits to DXT. We might have to shift it down if we have a meta.size fixup down below.
// Also makes the right shift nicer (16 vs 11).
upload.dxt = dt << 5;
if (meta.size == TextureSize::Bpp32 || meta.fmt == TextureFormat::YUV)
{
// We iterate twice for Bpp32 and YUV to complete a 64bpp word.
upload.tmem_stride_words <<= 1;
// Pure, utter insanity, but no content should *ever* hit this ...
if (uneven_dt && meta.size != info.size)
{
LOGE("Got uneven_dt, and texture size != tile size.\n");
return;
}
}
// If TMEM and VRAM bpp misalign, we need to fixup this since we step too fast or slow.
if (unsigned(meta.size) > unsigned(info.size))
{
unsigned shamt = unsigned(meta.size) - unsigned(info.size);
max_num_64bpp_elements_before_wrap <<= shamt;
min_num_64bpp_elements_before_wrap <<= shamt;
// Need to step slower so we can handle the added striding.
upload.dxt >>= shamt;
}
else if (unsigned(info.size) > unsigned(meta.size))
{
// Here we step multiple times over the same pixel, but potentially with different T state,
// since dTdx applies between the iterations.
// Horrible, horrible mess ...
LOGE("LoadBlock: VRAM bpp size is larger than tile bpp. This is unsupported.\n");
return;
}
unsigned max_line_stride_64bpp = max_num_64bpp_elements_before_wrap + (upload.tmem_stride_words >> 2);
unsigned min_line_stride_64bpp = min_num_64bpp_elements_before_wrap + (upload.tmem_stride_words >> 2);
// Multiplying 64bpp TMEM word by these gives us lower and upper bounds for T.
// These serve as candidate expressions for floor().
float min_t_mod = 1.0f / float(max_line_stride_64bpp);
float max_t_mod = 1.0f / float(min_line_stride_64bpp);
upload.min_t_mod = min_t_mod;
upload.max_t_mod = max_t_mod;
upload.width = pixel_count;
upload.height = 1;
upload.tmem_stride_words >>= 2; // Stride in 64bpp instead of 16bpp.
}
else
{
// We never trigger a case where T is non-zero, so this is equivalent to a Tile upload.
upload.width = pixel_count;
upload.height = 1;
upload.tmem_stride_words = 0;
upload_mode = UploadMode::Tile;
}
}
else
{
upload_x = info.slo >> 2;
upload_y = info.tlo >> 2;
upload.width = (((info.shi >> 2) - (info.slo >> 2)) + 1) & 0xfff;
upload.height = ((info.thi >> 2) - (info.tlo >> 2)) + 1;
}
if (!upload.width)
return;
switch (info.size)
{
case TextureSize::Bpp8:
upload.vram_effective_width = (upload.width + 7) & ~7;
break;
case TextureSize::Bpp16:
// In 16-bit VRAM pointer with TLUT, we iterate one texel at a time, not 4.
if (upload_mode == UploadMode::TLUT)
upload.vram_effective_width = upload.width;
else
upload.vram_effective_width = (upload.width + 3) & ~3;
break;
case TextureSize::Bpp32:
upload.vram_effective_width = (upload.width + 1) & ~1;
break;
default:
break;
}
// Uploads happen in chunks of 8 bytes in groups of 4x16-bits.
switch (meta.size)
{
case TextureSize::Bpp4:
upload.width = (upload.width + 15) & ~15;
upload.width >>= 2;
break;
case TextureSize::Bpp8:
upload.width = (upload.width + 7) & ~7;
upload.width >>= 1;
break;
case TextureSize::Bpp16:
upload.width = (upload.width + 3) & ~3;
// Consider YUV uploads to be 32bpp since that's kinda what they are.
if (meta.fmt == TextureFormat::YUV)
upload.width >>= 1;
break;
case TextureSize::Bpp32:
upload.width = (upload.width + 1) & ~1;
break;
default:
LOGE("Unimplemented!\n");
break;
}
if (upload.height > 1 && upload_mode == UploadMode::TLUT)
{
LOGE("Load TLUT with height > 1 is not supported.\n");
return;
}
upload.vram_addr = info.tex_addr + ((info.tex_width * upload_y + upload_x) << (unsigned(info.size) - 1));
upload.vram_width = upload_mode == UploadMode::Block ? upload.vram_effective_width : info.tex_width;
upload.vram_size = int32_t(info.size);
upload.tmem_offset = (meta.offset + tmem_offset) & 0xfff;
upload.tmem_size = int32_t(meta.size);
upload.tmem_fmt = int32_t(meta.fmt);
upload.mode = int32_t(upload_mode);
upload.inv_tmem_stride_words = 1.0f / float(upload.tmem_stride_words);
stream.tmem_upload_infos.push_back(upload);
if (stream.tmem_upload_infos.size() + 1 >= Limits::MaxTMEMInstances)
flush_queues();
}
void Renderer::set_blend_color(uint32_t color)
{
constants.blend_color = color;
}
void Renderer::set_fog_color(uint32_t color)
{
constants.fog_color = color;
}
void Renderer::set_env_color(uint32_t color)
{
constants.env_color = color;
}
void Renderer::set_fill_color(uint32_t color)
{
constants.fill_color = color;
}
void Renderer::set_primitive_depth(uint16_t prim_depth, uint16_t prim_dz)
{
constants.prim_depth = int32_t(prim_depth & 0x7fff) << 16;
constants.prim_dz = prim_dz;
}
void Renderer::set_enable_primitive_depth(bool enable)
{
constants.use_prim_depth = enable;
}
void Renderer::set_convert(uint16_t k0, uint16_t k1, uint16_t k2, uint16_t k3, uint16_t k4, uint16_t k5)
{
constants.convert[0] = 2 * sext<9>(k0) + 1;
constants.convert[1] = 2 * sext<9>(k1) + 1;
constants.convert[2] = 2 * sext<9>(k2) + 1;
constants.convert[3] = 2 * sext<9>(k3) + 1;
constants.convert[4] = k4;
constants.convert[5] = k5;
}
void Renderer::set_color_key(unsigned component, uint32_t width, uint32_t center, uint32_t scale)
{
constants.key_width[component] = width;
constants.key_center[component] = center;
constants.key_scale[component] = scale;
}
void Renderer::set_primitive_color(uint8_t min_level, uint8_t prim_lod_frac, uint32_t color)
{
constants.primitive_color = color;
constants.min_level = min_level;
constants.prim_lod_frac = prim_lod_frac;
}
bool Renderer::can_support_minimum_subgroup_size(unsigned size) const
{
return supports_subgroup_size_control(size, device->get_device_features().subgroup_properties.subgroupSize);
}
bool Renderer::supports_subgroup_size_control(uint32_t minimum_size, uint32_t maximum_size) const
{
auto &features = device->get_device_features();
if (!features.subgroup_size_control_features.computeFullSubgroups)
return false;
bool use_varying = minimum_size <= features.subgroup_size_control_properties.minSubgroupSize &&
maximum_size >= features.subgroup_size_control_properties.maxSubgroupSize;
if (!use_varying)
{
bool outside_range = minimum_size > features.subgroup_size_control_properties.maxSubgroupSize ||
maximum_size < features.subgroup_size_control_properties.minSubgroupSize;
if (outside_range)
return false;
if ((features.subgroup_size_control_properties.requiredSubgroupSizeStages & VK_SHADER_STAGE_COMPUTE_BIT) == 0)
return false;
}
return true;
}
void Renderer::PipelineExecutor::perform_work(const Vulkan::DeferredPipelineCompile &compile) const
{
auto start_ts = device->write_calibrated_timestamp();
Vulkan::CommandBuffer::build_compute_pipeline(device, compile, Vulkan::CommandBuffer::CompileMode::AsyncThread);
auto end_ts = device->write_calibrated_timestamp();
device->register_time_interval("RDP Pipeline", std::move(start_ts), std::move(end_ts),
"pipeline-compilation");
}
bool Renderer::PipelineExecutor::is_sentinel(const Vulkan::DeferredPipelineCompile &compile) const
{
return compile.hash == 0;
}
void Renderer::PipelineExecutor::notify_work_locked(const Vulkan::DeferredPipelineCompile &) const
{
}
}
Reference in a new issue View git blame Copy permalink