diff --git a/Core/TextureReplacer.cpp b/Core/TextureReplacer.cpp
index bb854b3d92..b0a473f20e 100644
--- a/Core/TextureReplacer.cpp
+++ b/Core/TextureReplacer.cpp
@@ -441,6 +441,12 @@ void TextureReplacer::PopulateReplacement(ReplacedTexture *result, u64 cachekey,
 			break;
 	}
 
+	// Populate the level data pointers for each level.
+	result->levelData_.resize(result->levels_.size());
+	for (size_t i = 0; i < result->levels_.size(); ++i) {
+		result->levelData_[i] = &levelCache_[result->levels_[i]];
+	}
+
 	result->prepareDone_ = true;
 }
 
@@ -836,8 +842,11 @@ bool ReplacedTexture::IsReady(double budget) {
 	}
 
 	// Loaded already, or not yet on a thread?
-	if (initDone_ && !levelData_.empty())
+	if (initDone_ && !levelData_.empty()) {
+		for (auto &l : levelData_)
+			l->lastUsed = lastUsed_;
 		return true;
+	}
 	// Let's not even start a new texture if we're already behind.
 	if (budget < 0.0)
 		return false;
@@ -871,7 +880,6 @@ void ReplacedTexture::Prepare() {
 		return;
 	}
 
-	levelData_.resize(levels_.size());
 	for (int i = 0; i < (int)levels_.size(); ++i) {
 		if (cancelPrepare_)
 			break;
@@ -885,9 +893,14 @@ void ReplacedTexture::Prepare() {
 
 void ReplacedTexture::PrepareData(int level) {
 	_assert_msg_((size_t)level < levels_.size(), "Invalid miplevel");
+	_assert_msg_(levelData_[level] != nullptr, "Level cache not set for miplevel");
 
 	const ReplacedTextureLevel &info = levels_[level];
-	std::vector<uint8_t> &out = levelData_[level];
+	std::vector<uint8_t> &out = levelData_[level]->data;
+
+	// Already populated from cache.
+	if (!out.empty())
+		return;
 
 	FILE *fp = File::OpenCFile(info.file, "rb");
 	if (!fp) {
@@ -987,14 +1000,19 @@ size_t ReplacedTexture::PurgeIfOlder(double t) {
 		return 0;
 
 	if (lastUsed_ < t) {
-		levelData_.clear();
-		initDone_ = false;
+		for (auto &l : levelData_) {
+			if (l->lastUsed < t) {
+				l->data.clear();
+				// This means we have to reload.  If we never purge any, there's no need.
+				initDone_ = false;
+			}
+		}
 		return 0;
 	}
 
 	size_t s = 0;
 	for (auto &l : levelData_) {
-		s += l.size();
+		s += l->data.size();
 	}
 	return s;
 }
@@ -1018,8 +1036,10 @@ bool ReplacedTexture::Load(int level, void *out, int rowPitch) {
 	if (levelData_.empty())
 		return false;
 
+	_assert_msg_(levelData_[level] != nullptr, "Level cache not set for miplevel");
+
 	const ReplacedTextureLevel &info = levels_[level];
-	const std::vector<uint8_t> &data = levelData_[level];
+	const std::vector<uint8_t> &data = levelData_[level]->data;
 
 	if (data.empty())
 		return false;
diff --git a/Core/TextureReplacer.h b/Core/TextureReplacer.h
index 62aceee7fa..347e987f16 100644
--- a/Core/TextureReplacer.h
+++ b/Core/TextureReplacer.h
@@ -17,11 +17,13 @@
 
 #pragma once
 
+#include "ppsspp_config.h"
 #include <mutex>
 #include <string>
 #include <unordered_map>
 #include <vector>
 
+#include "Common/CommonFuncs.h"
 #include "Common/CommonTypes.h"
 #include "Common/MemoryUtil.h"
 #include "Common/File/Path.h"
@@ -54,6 +56,33 @@ struct ReplacedTextureLevel {
 	int h;
 	Draw::DataFormat fmt;  // NOTE: Right now, the only supported format is Draw::DataFormat::R8G8B8A8_UNORM.
 	Path file;
+
+	bool operator ==(const ReplacedTextureLevel &other) const {
+		if (w != other.w || h != other.h || fmt != other.fmt)
+			return false;
+		return file == other.file;
+	}
+};
+
+namespace std {
+	template <>
+	struct hash<ReplacedTextureLevel> {
+		std::size_t operator()(const ReplacedTextureLevel &k) const {
+#if PPSSPP_ARCH(64BIT)
+			uint64_t v = (uint64_t)k.w | ((uint64_t)k.h << 32);
+			v = __rotl64(v ^ (uint64_t)k.fmt, 13);
+#else
+			uint32_t v = k.w ^ (uint32_t)k.fmt;
+			v = __rotl(__rotl(v, 13) ^ k.h, 13);
+#endif
+			return v ^ hash<string>()(k.file.ToString());
+		}
+	};
+}
+
+struct ReplacedLevelCache {
+	std::vector<uint8_t> data;
+	double lastUsed = 0.0;
 };
 
 struct ReplacementCacheKey {
@@ -170,7 +199,7 @@ protected:
 	size_t PurgeIfOlder(double t);
 
 	std::vector<ReplacedTextureLevel> levels_;
-	std::vector<std::vector<uint8_t>> levelData_;
+	std::vector<ReplacedLevelCache *> levelData_;
 	ReplacedTextureAlpha alphaStatus_ = ReplacedTextureAlpha::UNKNOWN;
 	double lastUsed_ = 0.0;
 	LimitedWaitable *threadWaitable_ = nullptr;
@@ -263,4 +292,5 @@ protected:
 	ReplacedTexture none_;
 	std::unordered_map<ReplacementCacheKey, ReplacedTexture> cache_;
 	std::unordered_map<ReplacementCacheKey, std::pair<ReplacedTextureLevel, double>> savedCache_;
+	std::unordered_map<ReplacedTextureLevel, ReplacedLevelCache> levelCache_;
 };