diff --git a/GPU/Common/TextureCacheCommon.cpp b/GPU/Common/TextureCacheCommon.cpp
index 578e785704..966bfa8117 100644
--- a/GPU/Common/TextureCacheCommon.cpp
+++ b/GPU/Common/TextureCacheCommon.cpp
@@ -2143,7 +2143,7 @@ bool TextureCacheCommon::PrepareBuildTexture(BuildTexturePlan &plan, TexCacheEnt
 	return true;
 }
 
-uint8_t *TextureCacheCommon::LoadTextureLevel(TexCacheEntry &entry, uint8_t *data, int stride, ReplacedTexture &replaced, int srcLevel, int scaleFactor, Draw::DataFormat dstFmt) {
+void TextureCacheCommon::LoadTextureLevel(TexCacheEntry &entry, uint8_t *data, int stride, ReplacedTexture &replaced, int srcLevel, int scaleFactor, Draw::DataFormat dstFmt) {
 	int w = gstate.getTextureWidth(srcLevel);
 	int h = gstate.getTextureHeight(srcLevel);
 
@@ -2205,6 +2205,4 @@ uint8_t *TextureCacheCommon::LoadTextureLevel(TexCacheEntry &entry, uint8_t *dat
 			replacer_.NotifyTextureDecoded(replacedInfo, pixelData, decPitch, srcLevel, w, h);
 		}
 	}
-
-	return data;
 }
diff --git a/GPU/Common/TextureCacheCommon.h b/GPU/Common/TextureCacheCommon.h
index fda76db41b..85187af204 100644
--- a/GPU/Common/TextureCacheCommon.h
+++ b/GPU/Common/TextureCacheCommon.h
@@ -330,7 +330,7 @@ protected:
 	ReplacedTexture &FindReplacement(TexCacheEntry *entry, int &w, int &h);
 
 	// Return value is mapData normally, but could be another buffer allocated with AllocateAlignedMemory.
-	uint8_t *LoadTextureLevel(TexCacheEntry &entry, uint8_t *mapData, int mapRowPitch, ReplacedTexture &replaced, int srcLevel, int scaleFactor, Draw::DataFormat dstFmt);
+	void LoadTextureLevel(TexCacheEntry &entry, uint8_t *mapData, int mapRowPitch, ReplacedTexture &replaced, int srcLevel, int scaleFactor, Draw::DataFormat dstFmt);
 
 	template <typename T>
 	inline const T *GetCurrentClut() {
diff --git a/GPU/D3D11/TextureCacheD3D11.cpp b/GPU/D3D11/TextureCacheD3D11.cpp
index 85b27b70d8..e48f523e1f 100644
--- a/GPU/D3D11/TextureCacheD3D11.cpp
+++ b/GPU/D3D11/TextureCacheD3D11.cpp
@@ -493,28 +493,29 @@ void TextureCacheD3D11::BuildTexture(TexCacheEntry *const entry) {
 	Draw::DataFormat texFmt = FromD3D11Format(dstFmt);
 
 	for (int i = 0; i < levels; i++) {
-		u8 *data = nullptr;
-		int stride = 0;
-		int bpp = dstFmt == DXGI_FORMAT_B8G8R8A8_UNORM ? 4 : 2;
-
 		int srcLevel = (i == 0) ? plan.baseLevelSrc : i;
 
 		int w = gstate.getTextureWidth(srcLevel);
 		int h = gstate.getTextureHeight(srcLevel);
 
+		u8 *data = nullptr;
+		int stride = 0;
+
 		// For UpdateSubresource, we can't decode directly into the texture so we allocate a buffer :(
 		// NOTE: Could reuse it between levels or textures!
 		if (plan.replaced->GetSize(srcLevel, w, h)) {
-			data = (u8 *)AllocateAlignedMemory(w * h * 4, 16);
-			stride = w * 4;
+			int bpp = (int)Draw::DataFormatSizeInBytes(plan.replaced->Format(srcLevel));
+			stride = w * bpp;
+			data = (u8 *)AllocateAlignedMemory(stride * h, 16);
 		} else {
 			if (plan.scaleFactor > 1) {
 				data = (u8 *)AllocateAlignedMemory(4 * (w * plan.scaleFactor) * (h * plan.scaleFactor), 16);
 				stride = w * plan.scaleFactor * 4;
 			} else {
+				int bpp = dstFmt == DXGI_FORMAT_B8G8R8A8_UNORM ? 4 : 2;
+
 				stride = std::max(w * bpp, 16);
-				size_t bufSize = sizeof(u32) * (stride / bpp) * h;
-				data = (u8 *)AllocateAlignedMemory(bufSize, 16);
+				data = (u8 *)AllocateAlignedMemory(stride * h, 16);
 			}
 		}
 
@@ -523,7 +524,7 @@ void TextureCacheD3D11::BuildTexture(TexCacheEntry *const entry) {
 			return;
 		}
 
-		data = LoadTextureLevel(*entry, data, stride, *plan.replaced, srcLevel, plan.scaleFactor, texFmt);
+		LoadTextureLevel(*entry, data, stride, *plan.replaced, srcLevel, plan.scaleFactor, texFmt);
 
 		ID3D11Texture2D *texture = DxTex(entry);
 		context_->UpdateSubresource(texture, i, nullptr, data, stride, 0);
diff --git a/GPU/Directx9/TextureCacheDX9.cpp b/GPU/Directx9/TextureCacheDX9.cpp
index 40d6f3320f..08f1e377f0 100644
--- a/GPU/Directx9/TextureCacheDX9.cpp
+++ b/GPU/Directx9/TextureCacheDX9.cpp
@@ -460,7 +460,7 @@ void TextureCacheDX9::BuildTexture(TexCacheEntry *const entry) {
 		uint8_t *data = (uint8_t *)rect.pBits;
 		int stride = rect.Pitch;
 
-		data = LoadTextureLevel(*entry, data, stride, *plan.replaced, (i == 0) ? plan.baseLevelSrc : i, plan.scaleFactor, texFmt);
+		LoadTextureLevel(*entry, data, stride, *plan.replaced, (i == 0) ? plan.baseLevelSrc : i, plan.scaleFactor, texFmt);
 
 		texture->UnlockRect(dstLevel);
 	}