Implement simple file caching.

Common/Buffer.cpp

@@ -115,7 +115,7 @@ void Buffer::Printf(const char *fmt, ...) {
 	memcpy(ptr, buffer, retval);
 }
 
-bool Buffer::FlushToFile(const Path &filename) {
+bool Buffer::FlushToFile(const Path &filename, bool clear) {
 	FILE *f = File::OpenCFile(filename, "wb");
 	if (!f)
 		return false;
@@ -124,7 +124,9 @@ bool Buffer::FlushToFile(const Path &filename) {
 		data_.iterate_blocks([=](const char *blockData, size_t blockSize) {
 			return fwrite(blockData, 1, blockSize, f) == blockSize;
 		});
-		data_.clear();
+		if (clear) {
+			data_.clear();
+		}
 	}
 	fclose(f);
 	return true;

Common/Buffer.h

@@ -66,9 +66,9 @@ public:
 	// Simple I/O.
 
 	// Writes the entire buffer to the file descriptor. Also resets the
-	// size to zero. On failure, data remains in buffer and nothing is
+	// size to zero, if the clear flag is true. On failure, data remains in buffer and nothing is
 	// written.
-	bool FlushToFile(const Path &filename);
+	bool FlushToFile(const Path &filename, bool clear);
 
 	// Utilities. Try to avoid checking for size.
 	size_t size() const { return data_.size(); }

Common/Net/HTTPClient.cpp

@@ -494,7 +494,9 @@ HTTPRequest::HTTPRequest(RequestMethod method, std::string_view url, std::string
 HTTPRequest::~HTTPRequest() {
 	g_OSD.RemoveProgressBar(url_, !failed_, 0.5f);
 
-	_assert_msg_(joined_, "Download destructed without join");
+	if (thread_.joinable()) {
+		_dbg_assert_msg_(false, "Download destructed without join");
+	}
 }
 
 void HTTPRequest::Start() {
@@ -502,11 +504,11 @@ void HTTPRequest::Start() {
 }
 
 void HTTPRequest::Join() {
-	if (joined_) {
+	if (!thread_.joinable()) {
 		ERROR_LOG(Log::HTTP, "Already joined thread!");
+		_dbg_assert_(false);
 	}
 	thread_.join();
-	joined_ = true;
 }
 
 void HTTPRequest::SetFailed(int code) {
@@ -599,7 +601,8 @@ void HTTPRequest::Do() {
 
 		if (resultCode == 200) {
 			INFO_LOG(Log::HTTP, "Completed requesting %s (storing result to %s)", url_.c_str(), outfile_.empty() ? "memory" : outfile_.c_str());
-			if (!outfile_.empty() && !buffer_.FlushToFile(outfile_)) {
+			bool clear = !(flags_ & RequestFlags::KeepInMemory);
+			if (!outfile_.empty() && !buffer_.FlushToFile(outfile_, clear)) {
 				ERROR_LOG(Log::HTTP, "Failed writing download to '%s'", outfile_.c_str());
 			}
 		} else {

Common/Net/HTTPClient.h

@@ -35,7 +35,6 @@ protected:
 
 private:
 	uintptr_t sock_ = -1;
-
 };
 
 }	// namespace net
@@ -116,10 +115,11 @@ private:
 	std::string postMime_;
 	bool completed_ = false;
 	bool failed_ = false;
-	bool joined_ = false;
 };
 
 // Fake request for cache hits.
+// The download manager uses this when caching was requested, and a new-enough file was present in the cache directory.
+// This is simply a finished request, that can still be queried like a normal one so users don't know it came from the cache.
 class CachedRequest : public Request {
 public:
 	CachedRequest(RequestMethod method, std::string_view url, std::string_view name, bool *cancelled, RequestFlags flags, std::string_view responseData)

Common/Net/HTTPNaettRequest.cpp

@@ -107,7 +107,8 @@ bool HTTPSRequest::Done() {
 		failed_ = true;
 		progress_.Update(bodyLength, bodyLength, true);
 	} else if (resultCode_ == 200) {
-		if (!outfile_.empty() && !buffer_.FlushToFile(outfile_)) {
+		bool clear = !(flags_ & RequestFlags::KeepInMemory);
+		if (!outfile_.empty() && !buffer_.FlushToFile(outfile_, clear)) {
 			ERROR_LOG(Log::IO, "Failed writing download to '%s'", outfile_.c_str());
 		}
 		progress_.Update(bodyLength, bodyLength, true);

Common/Net/HTTPRequest.cpp

@@ -2,6 +2,7 @@
 #include "Common/Net/HTTPClient.h"
 #include "Common/Net/HTTPNaettRequest.h"
 #include "Common/TimeUtil.h"
+#include "Common/File/FileUtil.h"
 #include "Common/StringUtils.h"
 #include "Common/Log.h"
 #include "Common/System/OSD.h"
@@ -39,25 +40,66 @@ static bool IsHttpsUrl(std::string_view url) {
 	return startsWith(url, "https:");
 }
 
+Path UrlToCachePath(const Path &cacheDir, std::string_view url) {
+	std::string fn = "DLCACHE_";
+	for (auto c : url) {
+		if (isalnum(c) || c == '.' || c == '-' || c == '_') {
+			fn.push_back(tolower(c));
+		} else {
+			fn.push_back('_');
+		}
+	}
+	return cacheDir / fn;
+}
+
 std::shared_ptr<Request> CreateRequest(RequestMethod method, std::string_view url, std::string_view postdata, std::string_view postMime, const Path &outfile, RequestFlags flags, std::string_view name) {
 	if (IsHttpsUrl(url) && System_GetPropertyBool(SYSPROP_SUPPORTS_HTTPS)) {
 #ifndef HTTPS_NOT_AVAILABLE
-		return std::shared_ptr<Request>(new HTTPSRequest(method, url, postdata, postMime, outfile, flags, name));
+		return std::make_shared<HTTPSRequest>(method, url, postdata, postMime, outfile, flags, name);
 #else
 		return std::shared_ptr<Request>();
 #endif
 	} else {
-		return std::shared_ptr<Request>(new HTTPRequest(method, url, postdata, postMime, outfile, flags, name));
+		return std::make_shared<HTTPRequest>(method, url, postdata, postMime, outfile, flags, name);
 	}
 }
 
 std::shared_ptr<Request> RequestManager::StartDownload(std::string_view url, const Path &outfile, RequestFlags flags, const char *acceptMime) {
 	std::shared_ptr<Request> dl = CreateRequest(RequestMethod::GET, url, "", "", outfile, flags, "");
 
+	if (!cacheDir_.empty() && (flags & RequestFlags::Cached24H)) {
+		_dbg_assert_(outfile.empty());  // It's automatically replaced below
+
+		// Come up with a cache file path.
+		Path cacheFile = UrlToCachePath(cacheDir_, url);
+
+		// TODO: This should be done on the thread, maybe. But let's keep it simple for now.
+		time_t cacheFileTime;
+		if (File::GetModifTimeT(cacheFile, &cacheFileTime)) {
+			time_t now = (time_t)time_now_unix_utc();
+			if (cacheFileTime > now - 24 * 60 * 60) {
+				// The file is new enough. Let's construct a fake, already finished download so we don't need
+				// to modify the calling code.
+				std::string contents;
+				if (File::ReadBinaryFileToString(cacheFile, &contents)) {
+					// All is well, but we've indented a bit much here.
+					dl.reset(new CachedRequest(RequestMethod::GET, url, "", nullptr, flags, contents));
+					newDownloads_.push_back(dl);
+					return dl;
+				}
+			}
+		}
+
+		// OK, didn't get it from cache, so let's continue with the download, putting it in the cache.
+		dl->OverrideOutFile(cacheFile);
+		dl->AddFlag(RequestFlags::KeepInMemory);
+	}
+
 	if (!userAgent_.empty())
 		dl->SetUserAgent(userAgent_);
 	if (acceptMime)
 		dl->SetAccept(acceptMime);
+
 	newDownloads_.push_back(dl);
 	dl->Start();
 	return dl;

Common/Net/HTTPRequest.h

@@ -18,7 +18,8 @@ enum class RequestFlags {
 	Default = 0,
 	ProgressBar = 1,
 	ProgressBarDelayed = 2,
-	// Cached = 4 etc
+	Cached24H = 4,
+	KeepInMemory = 8,
 };
 ENUM_CLASS_BITOPS(RequestFlags);
 
@@ -59,6 +60,12 @@ public:
 	std::string url() const { return url_; }
 
 	const Path &OutFile() const { return outfile_; }
+	void OverrideOutFile(const Path &path) {
+		outfile_ = path;
+	}
+	void AddFlag(RequestFlags flag) {
+		flags_ |= flag;
+	}
 
 	void Cancel() { cancelled_ = true; }
 	bool IsCancelled() const { return cancelled_; }
@@ -97,6 +104,7 @@ public:
 		CancelAll();
 	}
 
+	// NOTE: This is the only version that supports the cache flag (for now).
 	std::shared_ptr<Request> StartDownload(std::string_view url, const Path &outfile, RequestFlags flags, const char *acceptMime = nullptr);
 
 	std::shared_ptr<Request> StartDownloadWithCallback(
@@ -123,6 +131,10 @@ public:
 		userAgent_ = userAgent;
 	}
 
+	void SetCacheDir(const Path &path) {
+		cacheDir_ = path;
+	}
+
 private:
 	std::vector<std::shared_ptr<Request>> downloads_;
 	// These get copied to downloads_ in Update(). It's so that callbacks can add new downloads
@@ -130,6 +142,7 @@ private:
 	std::vector<std::shared_ptr<Request>> newDownloads_;
 
 	std::string userAgent_;
+	Path cacheDir_;
 };
 
 inline const char *RequestMethodToString(RequestMethod method) {

Core/HLE/sceNet.cpp

@@ -328,7 +328,7 @@ void StartInfraJsonDownload() {
 		INFO_LOG(Log::sceNet, "json is already being downloaded");
 	}
 	const char *acceptMime = "application/json, text/*; q=0.9, */*; q=0.8";
-	g_infraDL = g_DownloadManager.StartDownload("http://metadata.ppsspp.org/infra-dns.json", Path(), http::RequestFlags::Default, acceptMime);
+	g_infraDL = g_DownloadManager.StartDownload("http://metadata.ppsspp.org/infra-dns.json", Path(), http::RequestFlags::Cached24H, acceptMime);
 }
 
 bool PollInfraJsonDownload(std::string *jsonOutput) {
@@ -359,9 +359,13 @@ bool PollInfraJsonDownload(std::string *jsonOutput) {
 	}
 
 	// OK, we actually got data. Load it!
-	INFO_LOG(Log::sceNet, "json downloaded");
 	g_infraDL->buffer().TakeAll(jsonOutput);
 
+	if (jsonOutput->empty()) {
+		_dbg_assert_msg_(false, "Json output is empty!");
+		ERROR_LOG(Log::sceNet, "JSON output is empty! Something went wrong.");
+	}
+
 	LoadAutoDNS(*jsonOutput);
 	return true;
 }

UI/NativeApp.cpp

@@ -724,6 +724,8 @@ void NativeInit(int argc, const char *argv[], const char *savegame_dir, const ch
 		}
 	}
 
+	g_DownloadManager.SetCacheDir(GetSysDirectory(DIRECTORY_APP_CACHE));
+
 	DEBUG_LOG(Log::System, "ScreenManager!");
 	g_screenManager = new ScreenManager();
 	if (g_Config.memStickDirectory.empty()) {