diff --git a/GPU/Vulkan/TextureCacheVulkan.cpp b/GPU/Vulkan/TextureCacheVulkan.cpp
index 061eb2373f..43ca25d835 100644
--- a/GPU/Vulkan/TextureCacheVulkan.cpp
+++ b/GPU/Vulkan/TextureCacheVulkan.cpp
@@ -1389,6 +1389,8 @@ void TextureCacheVulkan::SetTexture(VulkanPushBuffer *uploadBuffer) {
 		maxLevel = 0;
 	}
 
+	VkFormat actualFmt = scaleFactor > 1 ? VULKAN_8888_FORMAT : dstFmt;
+
 	if (replaceImages) {
 		if (!entry->vkTex) {
 			Crash();
@@ -1399,7 +1401,7 @@ void TextureCacheVulkan::SetTexture(VulkanPushBuffer *uploadBuffer) {
 		VulkanTexture *image = entry->vkTex->texture_;
 
 		const VkComponentMapping *mapping;
-		switch (dstFmt) {
+		switch (actualFmt) {
 		case VULKAN_4444_FORMAT:
 			mapping = &VULKAN_4444_SWIZZLE;
 			break;
@@ -1417,7 +1419,7 @@ void TextureCacheVulkan::SetTexture(VulkanPushBuffer *uploadBuffer) {
 			break;
 		}
 
-		image->CreateDirect(w * scaleFactor, h * scaleFactor, maxLevel + 1, dstFmt, VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT, mapping);
+		image->CreateDirect(w * scaleFactor, h * scaleFactor, maxLevel + 1, actualFmt, VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT, mapping);
 	}
 	lastBoundTexture = entry->vkTex;
 
@@ -1425,7 +1427,7 @@ void TextureCacheVulkan::SetTexture(VulkanPushBuffer *uploadBuffer) {
 	for (int i = 0; i <= maxLevel; i++) {
 		int mipWidth = gstate.getTextureWidth(i) * scaleFactor;
 		int mipHeight = gstate.getTextureHeight(i) * scaleFactor;
-		int bpp = dstFmt == VULKAN_8888_FORMAT ? 4 : 2;
+		int bpp = actualFmt == VULKAN_8888_FORMAT ? 4 : 2;
 		int stride = (mipWidth * bpp + 15) & ~15;
 		int size = stride * mipHeight;
 		uint32_t bufferOffset;
@@ -1697,7 +1699,8 @@ void TextureCacheVulkan::LoadTextureLevel(TexCacheEntry &entry, uint8_t *writePt
 		if (scaleFactor > 1) {
 			tmpTexBufRearrange.resize(std::max(bufw, w) * h);
 			pixelData = tmpTexBufRearrange.data();
-			decPitch = bufw * bpp;
+			// We want to end up with a neatly packed texture for scaling.
+			decPitch = w * bpp;
 		}
 
 		void *finalBuf = DecodeTextureLevel((u8 *)pixelData, decPitch, tfmt, clutformat, texaddr, level, dstFmt, scaleFactor, bufw);
@@ -1705,18 +1708,35 @@ void TextureCacheVulkan::LoadTextureLevel(TexCacheEntry &entry, uint8_t *writePt
 			return;
 		}
 		if (finalBuf != pixelData) {
+			// Since we didn't write to pixelData, we must've ignored decPitch too.
+			// TODO: Can remove once all formats are decoded directly and respect decPitch.
 			decPitch = bufw * bpp;
 		}
+		pixelData = (u32 *)finalBuf;
 		rowBytes = w * bpp;
 		gpuStats.numTexturesDecoded++;
 
-		pixelData = (u32 *)finalBuf;
 		if (scaleFactor > 1) {
 			u32 fmt = dstFmt;
+			// We need to repack it sometimes.
+			// TODO: Can remove once all formats are decoded directly and respect decPitch.
+			if (decPitch != w * bpp) {
+				tmpTexBufRearrange.resize(w * h);
+				const u8 *src = (u8 *)pixelData;
+				u8 *dst = (u8 *)tmpTexBufRearrange.data();
+				for (int y = 0; y < h; y++) {
+					memcpy(dst + rowBytes * y, src + decPitch * y, rowBytes);
+				}
+				pixelData = (u32 *)dst;
+			}
+
 			scaler.Scale(pixelData, fmt, w, h, scaleFactor);
 			dstFmt = (VkFormat)fmt;
-			decPitch *= scaleFactor;
-			rowBytes *= scaleFactor;
+
+			// We always end up at 8888.  Other parts check for this.
+			assert(dstFmt == VULKAN_8888_FORMAT);
+			decPitch = w * sizeof(u32);
+			rowBytes = w * sizeof(u32);
 		}
 
 		if ((entry.status & TexCacheEntry::STATUS_CHANGE_FREQUENT) == 0) {
diff --git a/GPU/Vulkan/TextureScalerVulkan.cpp b/GPU/Vulkan/TextureScalerVulkan.cpp
index 267538846c..278607349d 100644
--- a/GPU/Vulkan/TextureScalerVulkan.cpp
+++ b/GPU/Vulkan/TextureScalerVulkan.cpp
@@ -21,38 +21,45 @@
 #endif
 
 #include <algorithm>
-#include "gfx/gl_common.h"
 
-#include "GPU/Common/TextureScalerCommon.h"
-#include "GPU/Vulkan/TextureScalerVulkan.h"
+#include "Common/Vulkan/VulkanContext.h"
 #include "Common/ColorConv.h"
 #include "Common/Log.h"
 #include "Common/ThreadPools.h"
+#include "GPU/Common/TextureScalerCommon.h"
+#include "GPU/Vulkan/TextureScalerVulkan.h"
+
+// TODO: Share in TextureCacheVulkan.h?
+// Note: some drivers prefer B4G4R4A4_UNORM_PACK16 over R4G4B4A4_UNORM_PACK16.
+#define VULKAN_4444_FORMAT VK_FORMAT_B4G4R4A4_UNORM_PACK16
+#define VULKAN_1555_FORMAT VK_FORMAT_A1R5G5B5_UNORM_PACK16
+#define VULKAN_565_FORMAT  VK_FORMAT_B5G6R5_UNORM_PACK16
+#define VULKAN_8888_FORMAT VK_FORMAT_R8G8B8A8_UNORM
 
 int TextureScalerVulkan::BytesPerPixel(u32 format) {
-	return (format == GL_UNSIGNED_BYTE) ? 4 : 2;
+	return (format == VULKAN_8888_FORMAT) ? 4 : 2;
 }
 
 u32 TextureScalerVulkan::Get8888Format() {
-	return GL_UNSIGNED_BYTE;
+	return VULKAN_8888_FORMAT;
 }
 
 void TextureScalerVulkan::ConvertTo8888(u32 format, u32* source, u32* &dest, int width, int height) {
 	switch (format) {
-	case GL_UNSIGNED_BYTE:
+	case VULKAN_8888_FORMAT:
 		dest = source; // already fine
 		break;
 
-	case GL_UNSIGNED_SHORT_4_4_4_4:
-		GlobalThreadPool::Loop(std::bind(&convert4444_gl, (u16*)source, dest, width, placeholder::_1, placeholder::_2), 0, height);
+	case VULKAN_4444_FORMAT:
+		GlobalThreadPool::Loop(std::bind(&convert4444_dx9, (u16*)source, dest, width, placeholder::_1, placeholder::_2), 0, height);
 		break;
 
-	case GL_UNSIGNED_SHORT_5_6_5:
-		GlobalThreadPool::Loop(std::bind(&convert565_gl, (u16*)source, dest, width, placeholder::_1, placeholder::_2), 0, height);
+	case VULKAN_565_FORMAT:
+		GlobalThreadPool::Loop(std::bind(&convert565_dx9, (u16*)source, dest, width, placeholder::_1, placeholder::_2), 0, height);
 		break;
 
-	case GL_UNSIGNED_SHORT_5_5_5_1:
-		GlobalThreadPool::Loop(std::bind(&convert5551_gl, (u16*)source, dest, width, placeholder::_1, placeholder::_2), 0, height);
+	case VULKAN_1555_FORMAT:
+		GlobalThreadPool::Loop(std::bind(&convert5551_dx9, (u16*)source, dest, width, placeholder::_1, placeholder::_2), 0, height);
 		break;
 
 	default: