Merge pull request #17565 from hrydgard/breakout-vcache-vulkan

Vulkan: Breakout the vertex cache logic from DoFlush()
This commit is contained in:
Henrik Rydgård 2023-06-13 09:56:52 +02:00 committed by GitHub
commit 22632b82bd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 195 additions and 178 deletions

View file

@ -135,6 +135,8 @@ public:
return blocks_[curBlockIndex_].writePtr;
}
// NOTE: If you can avoid this by writing the data directly into memory returned from Allocate,
// do so. Savings from avoiding memcpy can be significant.
VkDeviceSize Push(const void *data, VkDeviceSize numBytes, int alignment, VkBuffer *vkbuf) {
uint32_t bindOffset;
uint8_t *ptr = Allocate(numBytes, alignment, vkbuf, &bindOffset);

View file

@ -74,17 +74,18 @@ VertexDecoder *DrawEngineCommon::GetVertexDecoder(u32 vtype) {
int DrawEngineCommon::ComputeNumVertsToDecode() const {
int vertsToDecode = 0;
int numDrawCalls = numDrawCalls_;
if (drawCalls_[0].indexType == GE_VTYPE_IDX_NONE >> GE_VTYPE_IDX_SHIFT) {
for (int i = 0; i < numDrawCalls_; i++) {
for (int i = 0; i < numDrawCalls; i++) {
const DeferredDrawCall &dc = drawCalls_[i];
vertsToDecode += dc.vertexCount;
}
} else {
// TODO: Share this computation with DecodeVertsStep?
for (int i = 0; i < numDrawCalls_; i++) {
for (int i = 0; i < numDrawCalls; i++) {
const DeferredDrawCall &dc = drawCalls_[i];
int lastMatch = i;
const int total = numDrawCalls_;
const int total = numDrawCalls;
int indexLowerBound = dc.indexLowerBound;
int indexUpperBound = dc.indexUpperBound;
for (int j = i + 1; j < total; ++j) {
@ -642,7 +643,7 @@ void DrawEngineCommon::DecodeVertsStep(u8 *dest, int &i, int &decodedVerts, cons
for (int j = i + 1; j < total; ++j) {
if (drawCalls_[j].verts != dc.verts)
break;
// TODO: What if UV scale/offset changes between drawcalls here?
indexLowerBound = std::min(indexLowerBound, (int)drawCalls_[j].indexLowerBound);
indexUpperBound = std::max(indexUpperBound, (int)drawCalls_[j].indexUpperBound);
lastMatch = j;
@ -779,16 +780,6 @@ uint64_t DrawEngineCommon::ComputeHash() {
return fullhash;
}
// Cheap bit scrambler from https://nullprogram.com/blog/2018/07/31/
inline uint32_t lowbias32_r(uint32_t x) {
x ^= x >> 16;
x *= 0x43021123U;
x ^= x >> 15 ^ x >> 30;
x *= 0x1d69e2a5U;
x ^= x >> 16;
return x;
}
// vertTypeID is the vertex type but with the UVGen mode smashed into the top bits.
void DrawEngineCommon::SubmitPrim(const void *verts, const void *inds, GEPrimitiveType prim, int vertexCount, u32 vertTypeID, int cullMode, int *bytesRead) {
if (!indexGen.PrimCompatible(prevPrim_, prim) || numDrawCalls_ >= MAX_DEFERRED_DRAW_CALLS || vertexCountInDrawCalls_ + vertexCount > VERTEX_BUFFER_MAX) {
@ -818,15 +809,6 @@ void DrawEngineCommon::SubmitPrim(const void *verts, const void *inds, GEPrimiti
if ((vertexCount < 2 && prim > 0) || (vertexCount < 3 && prim > GE_PRIM_LINE_STRIP && prim != GE_PRIM_RECTANGLES))
return;
if (g_Config.bVertexCache) {
u32 dhash = dcid_;
dhash = __rotl(dhash ^ (u32)(uintptr_t)verts, 13);
dhash = __rotl(dhash ^ (u32)(uintptr_t)inds, 19);
dhash = __rotl(dhash ^ (u32)vertTypeID, 7);
dhash = __rotl(dhash ^ (u32)vertexCount, 11);
dcid_ = lowbias32_r(dhash ^ (u32)prim);
}
DeferredDrawCall &dc = drawCalls_[numDrawCalls_];
dc.verts = verts;
dc.inds = inds;
@ -870,6 +852,29 @@ bool DrawEngineCommon::CanUseHardwareTessellation(GEPatchPrimType prim) {
return false;
}
// Cheap bit scrambler from https://nullprogram.com/blog/2018/07/31/
inline uint32_t lowbias32_r(uint32_t x) {
x ^= x >> 16;
x *= 0x43021123U;
x ^= x >> 15 ^ x >> 30;
x *= 0x1d69e2a5U;
x ^= x >> 16;
return x;
}
uint32_t DrawEngineCommon::ComputeDrawcallsHash() const {
uint32_t dcid = 0;
for (int i = 0; i < numDrawCalls_; i++) {
u32 dhash = dcid;
dhash = __rotl(dhash ^ (u32)(uintptr_t)drawCalls_[i].verts, 13);
dhash = __rotl(dhash ^ (u32)(uintptr_t)drawCalls_[i].inds, 19);
dhash = __rotl(dhash ^ (u32)drawCalls_[i].indexType, 7);
dhash = __rotl(dhash ^ (u32)drawCalls_[i].vertexCount, 11);
dcid = lowbias32_r(dhash ^ (u32)drawCalls_[i].prim);
}
return dcid;
}
void TessellationDataTransfer::CopyControlPoints(float *pos, float *tex, float *col, int posStride, int texStride, int colStride, const SimpleVertex *const *points, int size, u32 vertType) {
bool hasColor = (vertType & GE_VTYPE_COL_MASK) != 0;
bool hasTexCoord = (vertType & GE_VTYPE_TC_MASK) != 0;

View file

@ -175,6 +175,8 @@ protected:
}
}
uint32_t ComputeDrawcallsHash() const;
bool useHWTransform_ = false;
bool useHWTessellation_ = false;
// Used to prevent unnecessary flushing in softgpu.
@ -218,7 +220,6 @@ protected:
int decimationCounter_ = 0;
int decodeCounter_ = 0;
u32 dcid_ = 0;
// Vertex collector state
IndexGenerator indexGen;

View file

@ -363,12 +363,13 @@ void DrawEngineD3D11::DoFlush() {
useCache = false;
if (useCache) {
u32 id = dcid_ ^ gstate.getUVGenMode(); // This can have an effect on which UV decoder we need to use! And hence what the decoded data will look like. See #9263
// getUVGenMode can have an effect on which UV decoder we need to use! And hence what the decoded data will look like. See #9263
u32 dcid = (u32)XXH3_64bits(&drawCalls_, sizeof(DeferredDrawCall) * numDrawCalls_) ^ gstate.getUVGenMode();
VertexArrayInfoD3D11 *vai = vai_.Get(id);
VertexArrayInfoD3D11 *vai = vai_.Get(dcid);
if (!vai) {
vai = new VertexArrayInfoD3D11();
vai_.Insert(id, vai);
vai_.Insert(dcid, vai);
}
switch (vai->status) {
@ -724,7 +725,6 @@ rotateVBO:
numDrawCalls_ = 0;
vertexCountInDrawCalls_ = 0;
decodeCounter_ = 0;
dcid_ = 0;
gstate_c.vertexFullAlpha = true;
framebufferManager_->SetColorUpdated(gstate_c.skipDrawReason);

View file

@ -345,11 +345,12 @@ void DrawEngineDX9::DoFlush() {
useCache = false;
if (useCache) {
u32 id = dcid_ ^ gstate.getUVGenMode(); // This can have an effect on which UV decoder we need to use! And hence what the decoded data will look like. See #9263
VertexArrayInfoDX9 *vai = vai_.Get(id);
// getUVGenMode can have an effect on which UV decoder we need to use! And hence what the decoded data will look like. See #9263
u32 dcid = (u32)XXH3_64bits(&drawCalls_, sizeof(DeferredDrawCall) * numDrawCalls_) ^ gstate.getUVGenMode();
VertexArrayInfoDX9 *vai = vai_.Get(dcid);
if (!vai) {
vai = new VertexArrayInfoDX9();
vai_.Insert(id, vai);
vai_.Insert(dcid, vai);
}
switch (vai->status) {
@ -666,7 +667,6 @@ rotateVBO:
numDrawCalls_ = 0;
vertexCountInDrawCalls_ = 0;
decodeCounter_ = 0;
dcid_ = 0;
gstate_c.vertexFullAlpha = true;
framebufferManager_->SetColorUpdated(gstate_c.skipDrawReason);

View file

@ -248,7 +248,6 @@ void DrawEngineGLES::DoFlush() {
numDrawCalls_ = 0;
vertexCountInDrawCalls_ = 0;
decodeCounter_ = 0;
dcid_ = 0;
return;
}
@ -483,7 +482,6 @@ bail:
numDrawCalls_ = 0;
vertexCountInDrawCalls_ = 0;
decodeCounter_ = 0;
dcid_ = 0;
gstate_c.vertexFullAlpha = true;
framebufferManager_->SetColorUpdated(gstate_c.skipDrawReason);

View file

@ -552,6 +552,153 @@ void DrawEngineVulkan::Invalidate(InvalidationCallbackFlags flags) {
}
}
bool DrawEngineVulkan::VertexCacheLookup(int &vertexCount, GEPrimitiveType &prim, VkBuffer &vbuf, uint32_t &vbOffset, VkBuffer &ibuf, uint32_t &ibOffset, bool &useElements, bool forceIndexed) {
// getUVGenMode can have an effect on which UV decoder we need to use! And hence what the decoded data will look like. See #9263
// u32 dcid = (u32)XXH3_64bits(&drawCalls_, sizeof(DeferredDrawCall) * numDrawCalls_) ^ gstate.getUVGenMode();
u32 dcid = ComputeDrawcallsHash() ^ gstate.getUVGenMode();
PROFILE_THIS_SCOPE("vcache");
VertexArrayInfoVulkan *vai = vai_.Get(dcid);
if (!vai) {
vai = new VertexArrayInfoVulkan();
vai_.Insert(dcid, vai);
}
switch (vai->status) {
case VertexArrayInfoVulkan::VAI_NEW:
{
// Haven't seen this one before. We don't actually upload the vertex data yet.
uint64_t dataHash = ComputeHash();
vai->hash = dataHash;
vai->minihash = ComputeMiniHash();
vai->status = VertexArrayInfoVulkan::VAI_HASHING;
vai->drawsUntilNextFullHash = 0;
DecodeVertsToPushPool(pushVertex_, &vbOffset, &vbuf); // writes to indexGen
vai->numVerts = indexGen.VertexCount();
vai->prim = indexGen.Prim();
vai->maxIndex = indexGen.MaxIndex();
vai->flags = gstate_c.vertexFullAlpha ? VAIVULKAN_FLAG_VERTEXFULLALPHA : 0;
return true;
}
// Hashing - still gaining confidence about the buffer.
// But if we get this far it's likely to be worth uploading the data.
case VertexArrayInfoVulkan::VAI_HASHING:
{
PROFILE_THIS_SCOPE("vcachehash");
vai->numDraws++;
if (vai->lastFrame != gpuStats.numFlips) {
vai->numFrames++;
}
if (vai->drawsUntilNextFullHash == 0) {
// Let's try to skip a full hash if mini would fail.
const u32 newMiniHash = ComputeMiniHash();
uint64_t newHash = vai->hash;
if (newMiniHash == vai->minihash) {
newHash = ComputeHash();
}
if (newMiniHash != vai->minihash || newHash != vai->hash) {
MarkUnreliable(vai);
DecodeVertsToPushPool(pushVertex_, &vbOffset, &vbuf);
return true;
}
if (vai->numVerts > 64) {
// exponential backoff up to 16 draws, then every 24
vai->drawsUntilNextFullHash = std::min(24, vai->numFrames);
} else {
// Lower numbers seem much more likely to change.
vai->drawsUntilNextFullHash = 0;
}
// TODO: tweak
//if (vai->numFrames > 1000) {
// vai->status = VertexArrayInfo::VAI_RELIABLE;
//}
} else {
vai->drawsUntilNextFullHash--;
u32 newMiniHash = ComputeMiniHash();
if (newMiniHash != vai->minihash) {
MarkUnreliable(vai);
DecodeVertsToPushPool(pushVertex_, &vbOffset, &vbuf);
return true;
}
}
if (!vai->vb) {
// Directly push to the vertex cache.
DecodeVertsToPushBuffer(vertexCache_, &vai->vbOffset, &vai->vb);
_dbg_assert_msg_(gstate_c.vertBounds.minV >= gstate_c.vertBounds.maxV, "Should not have checked UVs when caching.");
vai->numVerts = indexGen.VertexCount();
vai->maxIndex = indexGen.MaxIndex();
vai->flags = gstate_c.vertexFullAlpha ? VAIVULKAN_FLAG_VERTEXFULLALPHA : 0;
if (forceIndexed) {
vai->prim = indexGen.GeneralPrim();
useElements = true;
} else {
vai->prim = indexGen.Prim();
useElements = !indexGen.SeenOnlyPurePrims();
if (!useElements && indexGen.PureCount()) {
vai->numVerts = indexGen.PureCount();
}
}
if (useElements) {
u32 size = sizeof(uint16_t) * indexGen.VertexCount();
void *dest = vertexCache_->Allocate(size, 4, &vai->ib, &vai->ibOffset);
memcpy(dest, decIndex_, size);
} else {
vai->ib = VK_NULL_HANDLE;
vai->ibOffset = 0;
}
} else {
gpuStats.numCachedDrawCalls++;
useElements = vai->ib ? true : false;
gpuStats.numCachedVertsDrawn += vai->numVerts;
gstate_c.vertexFullAlpha = vai->flags & VAIVULKAN_FLAG_VERTEXFULLALPHA;
}
vbuf = vai->vb;
ibuf = vai->ib;
vbOffset = vai->vbOffset;
ibOffset = vai->ibOffset;
vertexCount = vai->numVerts;
prim = static_cast<GEPrimitiveType>(vai->prim);
break;
}
// Reliable - we don't even bother hashing anymore. Right now we don't go here until after a very long time.
case VertexArrayInfoVulkan::VAI_RELIABLE:
{
vai->numDraws++;
if (vai->lastFrame != gpuStats.numFlips) {
vai->numFrames++;
}
gpuStats.numCachedDrawCalls++;
gpuStats.numCachedVertsDrawn += vai->numVerts;
vbuf = vai->vb;
ibuf = vai->ib;
vbOffset = vai->vbOffset;
ibOffset = vai->ibOffset;
vertexCount = vai->numVerts;
prim = static_cast<GEPrimitiveType>(vai->prim);
gstate_c.vertexFullAlpha = vai->flags & VAIVULKAN_FLAG_VERTEXFULLALPHA;
break;
}
case VertexArrayInfoVulkan::VAI_UNRELIABLE:
{
vai->numDraws++;
if (vai->lastFrame != gpuStats.numFlips) {
vai->numFrames++;
}
DecodeVertsToPushPool(pushVertex_, &vbOffset, &vbuf);
return true;
}
default:
break;
}
return false;
}
// The inline wrapper in the header checks for numDrawCalls_ == 0
void DrawEngineVulkan::DoFlush() {
VulkanRenderManager *renderManager = (VulkanRenderManager *)draw_->GetNativeObject(Draw::NativeObject::RENDER_MANAGER);
@ -595,148 +742,9 @@ void DrawEngineVulkan::DoFlush() {
if (decOptions_.applySkinInDecode && (lastVType_ & GE_VTYPE_WEIGHT_MASK)) {
useCache = false;
}
bool useIndexGen = true;
if (useCache) {
PROFILE_THIS_SCOPE("vcache");
u32 id = dcid_ ^ gstate.getUVGenMode(); // This can have an effect on which UV decoder we need to use! And hence what the decoded data will look like. See #9263
VertexArrayInfoVulkan *vai = vai_.Get(id);
if (!vai) {
vai = new VertexArrayInfoVulkan();
vai_.Insert(id, vai);
}
switch (vai->status) {
case VertexArrayInfoVulkan::VAI_NEW:
{
// Haven't seen this one before. We don't actually upload the vertex data yet.
uint64_t dataHash = ComputeHash();
vai->hash = dataHash;
vai->minihash = ComputeMiniHash();
vai->status = VertexArrayInfoVulkan::VAI_HASHING;
vai->drawsUntilNextFullHash = 0;
DecodeVertsToPushPool(pushVertex_, &vbOffset, &vbuf); // writes to indexGen
vai->numVerts = indexGen.VertexCount();
vai->prim = indexGen.Prim();
vai->maxIndex = indexGen.MaxIndex();
vai->flags = gstate_c.vertexFullAlpha ? VAIVULKAN_FLAG_VERTEXFULLALPHA : 0;
goto rotateVBO;
}
// Hashing - still gaining confidence about the buffer.
// But if we get this far it's likely to be worth uploading the data.
case VertexArrayInfoVulkan::VAI_HASHING:
{
PROFILE_THIS_SCOPE("vcachehash");
vai->numDraws++;
if (vai->lastFrame != gpuStats.numFlips) {
vai->numFrames++;
}
if (vai->drawsUntilNextFullHash == 0) {
// Let's try to skip a full hash if mini would fail.
const u32 newMiniHash = ComputeMiniHash();
uint64_t newHash = vai->hash;
if (newMiniHash == vai->minihash) {
newHash = ComputeHash();
}
if (newMiniHash != vai->minihash || newHash != vai->hash) {
MarkUnreliable(vai);
DecodeVertsToPushPool(pushVertex_, &vbOffset, &vbuf);
goto rotateVBO;
}
if (vai->numVerts > 64) {
// exponential backoff up to 16 draws, then every 24
vai->drawsUntilNextFullHash = std::min(24, vai->numFrames);
} else {
// Lower numbers seem much more likely to change.
vai->drawsUntilNextFullHash = 0;
}
// TODO: tweak
//if (vai->numFrames > 1000) {
// vai->status = VertexArrayInfo::VAI_RELIABLE;
//}
} else {
vai->drawsUntilNextFullHash--;
u32 newMiniHash = ComputeMiniHash();
if (newMiniHash != vai->minihash) {
MarkUnreliable(vai);
DecodeVertsToPushPool(pushVertex_, &vbOffset, &vbuf);
goto rotateVBO;
}
}
if (!vai->vb) {
// Directly push to the vertex cache.
DecodeVertsToPushBuffer(vertexCache_, &vai->vbOffset, &vai->vb);
_dbg_assert_msg_(gstate_c.vertBounds.minV >= gstate_c.vertBounds.maxV, "Should not have checked UVs when caching.");
vai->numVerts = indexGen.VertexCount();
vai->maxIndex = indexGen.MaxIndex();
vai->flags = gstate_c.vertexFullAlpha ? VAIVULKAN_FLAG_VERTEXFULLALPHA : 0;
if (forceIndexed) {
vai->prim = indexGen.GeneralPrim();
useElements = true;
} else {
vai->prim = indexGen.Prim();
useElements = !indexGen.SeenOnlyPurePrims();
if (!useElements && indexGen.PureCount()) {
vai->numVerts = indexGen.PureCount();
}
}
if (useElements) {
u32 size = sizeof(uint16_t) * indexGen.VertexCount();
void *dest = vertexCache_->Allocate(size, 4, &vai->ib, &vai->ibOffset);
memcpy(dest, decIndex_, size);
} else {
vai->ib = VK_NULL_HANDLE;
vai->ibOffset = 0;
}
} else {
gpuStats.numCachedDrawCalls++;
useElements = vai->ib ? true : false;
gpuStats.numCachedVertsDrawn += vai->numVerts;
gstate_c.vertexFullAlpha = vai->flags & VAIVULKAN_FLAG_VERTEXFULLALPHA;
}
vbuf = vai->vb;
ibuf = vai->ib;
vbOffset = vai->vbOffset;
ibOffset = vai->ibOffset;
vertexCount = vai->numVerts;
prim = static_cast<GEPrimitiveType>(vai->prim);
break;
}
// Reliable - we don't even bother hashing anymore. Right now we don't go here until after a very long time.
case VertexArrayInfoVulkan::VAI_RELIABLE:
{
vai->numDraws++;
if (vai->lastFrame != gpuStats.numFlips) {
vai->numFrames++;
}
gpuStats.numCachedDrawCalls++;
gpuStats.numCachedVertsDrawn += vai->numVerts;
vbuf = vai->vb;
ibuf = vai->ib;
vbOffset = vai->vbOffset;
ibOffset = vai->ibOffset;
vertexCount = vai->numVerts;
prim = static_cast<GEPrimitiveType>(vai->prim);
gstate_c.vertexFullAlpha = vai->flags & VAIVULKAN_FLAG_VERTEXFULLALPHA;
break;
}
case VertexArrayInfoVulkan::VAI_UNRELIABLE:
{
vai->numDraws++;
if (vai->lastFrame != gpuStats.numFlips) {
vai->numFrames++;
}
DecodeVertsToPushPool(pushVertex_, &vbOffset, &vbuf);
goto rotateVBO;
}
default:
break;
}
useIndexGen = VertexCacheLookup(vertexCount, prim, vbuf, vbOffset, ibuf, ibOffset, useElements, forceIndexed);
} else {
if (decOptions_.applySkinInDecode && (lastVType_ & GE_VTYPE_WEIGHT_MASK)) {
// If software skinning, we've already predecoded into "decoded". So push that content.
@ -747,10 +755,10 @@ void DrawEngineVulkan::DoFlush() {
// Decode directly into the pushbuffer
DecodeVertsToPushPool(pushVertex_, &vbOffset, &vbuf);
}
rotateVBO:
gpuStats.numUncachedVertsDrawn += indexGen.VertexCount();
}
if (useIndexGen) {
vertexCount = indexGen.VertexCount();
if (forceIndexed) {
useElements = true;
@ -823,6 +831,8 @@ void DrawEngineVulkan::DoFlush() {
VkDescriptorSet ds = GetOrCreateDescriptorSet(imageView, sampler, baseBuf, lightBuf, boneBuf, tess);
// TODO: Can we avoid binding all three when not needed? Same below for hardware transform.
// Think this will require different descriptor set layouts.
const uint32_t dynamicUBOOffsets[3] = {
baseUBOOffset, lightUBOOffset, boneUBOOffset,
};
@ -1011,7 +1021,6 @@ void DrawEngineVulkan::DoFlush() {
numDrawCalls_ = 0;
vertexCountInDrawCalls_ = 0;
decodeCounter_ = 0;
dcid_ = 0;
gstate_c.vertexFullAlpha = true;
framebufferManager_->SetColorUpdated(gstate_c.skipDrawReason);

View file

@ -222,6 +222,8 @@ private:
void DestroyDeviceObjects();
bool VertexCacheLookup(int &vertexCount, GEPrimitiveType &prim, VkBuffer &vbuf, uint32_t &vbOffset, VkBuffer &ibuf, uint32_t &ibOffset, bool &useElements, bool forceIndexed);
void DecodeVertsToPushPool(VulkanPushPool *push, uint32_t *bindOffset, VkBuffer *vkbuf);
void DecodeVertsToPushBuffer(VulkanPushBuffer *push, uint32_t *bindOffset, VkBuffer *vkbuf);