Merge pull request #19913 from hrydgard/download-infra-json

Keep infra-dns.json updated from metadata.ppsspp.org

Common/Buffer.cpp

@@ -115,7 +115,7 @@ void Buffer::Printf(const char *fmt, ...) {
 	memcpy(ptr, buffer, retval);
 }
 
-bool Buffer::FlushToFile(const Path &filename) {
+bool Buffer::FlushToFile(const Path &filename, bool clear) {
 	FILE *f = File::OpenCFile(filename, "wb");
 	if (!f)
 		return false;
@@ -124,7 +124,9 @@ bool Buffer::FlushToFile(const Path &filename) {
 		data_.iterate_blocks([=](const char *blockData, size_t blockSize) {
 			return fwrite(blockData, 1, blockSize, f) == blockSize;
 		});
-		data_.clear();
+		if (clear) {
+			data_.clear();
+		}
 	}
 	fclose(f);
 	return true;

Common/Buffer.h

@@ -66,9 +66,9 @@ class Buffer {
 	// Simple I/O.
 
 	// Writes the entire buffer to the file descriptor. Also resets the
-	// size to zero. On failure, data remains in buffer and nothing is
+	// size to zero, if the clear flag is true. On failure, data remains in buffer and nothing is
 	// written.
-	bool FlushToFile(const Path &filename);
+	bool FlushToFile(const Path &filename, bool clear);
 
 	// Utilities. Try to avoid checking for size.
 	size_t size() const { return data_.size(); }

Common/Net/HTTPClient.cpp

@@ -486,27 +486,29 @@ int Client::ReadResponseEntity(net::Buffer *readbuf, const std::vector<std::stri
 	return 0;
 }
 
-HTTPRequest::HTTPRequest(RequestMethod method, std::string_view url, std::string_view postData, std::string_view postMime, const Path &outfile, RequestFlags progressBarMode, std::string_view name)
-	: Request(method, url, name, &cancelled_, progressBarMode), postData_(postData), postMime_(postMime) {
+HTTPRequest::HTTPRequest(RequestMethod method, std::string_view url, std::string_view postData, std::string_view postMime, const Path &outfile, RequestFlags flags, std::string_view name)
+	: Request(method, url, name, &cancelled_, flags), postData_(postData), postMime_(postMime) {
 	outfile_ = outfile;
 }
 
 HTTPRequest::~HTTPRequest() {
 	g_OSD.RemoveProgressBar(url_, !failed_, 0.5f);
 
-	_assert_msg_(joined_, "Download destructed without join");
+	if (thread_.joinable()) {
+		_dbg_assert_msg_(false, "Download destructed without join");
+	}
 }
 
 void HTTPRequest::Start() {
 	thread_ = std::thread([this] { Do(); });
 }
 
 void HTTPRequest::Join() {
-	if (joined_) {
+	if (!thread_.joinable()) {
 		ERROR_LOG(Log::HTTP, "Already joined thread!");
+		_dbg_assert_(false);
 	}
 	thread_.join();
-	joined_ = true;
 }
 
 void HTTPRequest::SetFailed(int code) {
@@ -599,7 +601,8 @@ void HTTPRequest::Do() {
 
 		if (resultCode == 200) {
 			INFO_LOG(Log::HTTP, "Completed requesting %s (storing result to %s)", url_.c_str(), outfile_.empty() ? "memory" : outfile_.c_str());
-			if (!outfile_.empty() && !buffer_.FlushToFile(outfile_)) {
+			bool clear = !(flags_ & RequestFlags::KeepInMemory);
+			if (!outfile_.empty() && !buffer_.FlushToFile(outfile_, clear)) {
 				ERROR_LOG(Log::HTTP, "Failed writing download to '%s'", outfile_.c_str());
 			}
 		} else {

Common/Net/HTTPClient.h

@@ -35,7 +35,6 @@ class Connection {
 
 private:
 	uintptr_t sock_ = -1;
-
 };
 
 }	// namespace net
@@ -96,7 +95,7 @@ class Client : public net::Connection {
 // Really an asynchronous request.
 class HTTPRequest : public Request {
 public:
-	HTTPRequest(RequestMethod method, std::string_view url, std::string_view postData, std::string_view postMime, const Path &outfile, RequestFlags progressBarMode = RequestFlags::ProgressBar | RequestFlags::ProgressBarDelayed, std::string_view name = "");
+	HTTPRequest(RequestMethod method, std::string_view url, std::string_view postData, std::string_view postMime, const Path &outfile, RequestFlags flags = RequestFlags::ProgressBar | RequestFlags::ProgressBarDelayed, std::string_view name = "");
 	~HTTPRequest();
 
 	void Start() override;
@@ -116,7 +115,22 @@ class HTTPRequest : public Request {
 	std::string postMime_;
 	bool completed_ = false;
 	bool failed_ = false;
-	bool joined_ = false;
+};
+
+// Fake request for cache hits.
+// The download manager uses this when caching was requested, and a new-enough file was present in the cache directory.
+// This is simply a finished request, that can still be queried like a normal one so users don't know it came from the cache.
+class CachedRequest : public Request {
+public:
+	CachedRequest(RequestMethod method, std::string_view url, std::string_view name, bool *cancelled, RequestFlags flags, std::string_view responseData)
+		: Request(method, url, name, cancelled, flags)
+	{
+		buffer_.Append(responseData);
+	}
+	void Start() override {}
+	void Join() override {}
+	bool Done() override { return true; }
+	bool Failed() const override { return false; }
 };
 
 }  // namespace http

Common/Net/HTTPNaettRequest.cpp

@@ -12,8 +12,8 @@
 
 namespace http {
 
-HTTPSRequest::HTTPSRequest(RequestMethod method, std::string_view url, std::string_view postData, std::string_view postMime, const Path &outfile, RequestFlags progressBarMode, std::string_view name)
-	: Request(method, url, name, &cancelled_, progressBarMode), method_(method), postData_(postData), postMime_(postMime) {
+HTTPSRequest::HTTPSRequest(RequestMethod method, std::string_view url, std::string_view postData, std::string_view postMime, const Path &outfile, RequestFlags flags, std::string_view name)
+	: Request(method, url, name, &cancelled_, flags), method_(method), postData_(postData), postMime_(postMime) {
 	outfile_ = outfile;
 }
 
@@ -107,7 +107,8 @@ bool HTTPSRequest::Done() {
 		failed_ = true;
 		progress_.Update(bodyLength, bodyLength, true);
 	} else if (resultCode_ == 200) {
-		if (!outfile_.empty() && !buffer_.FlushToFile(outfile_)) {
+		bool clear = !(flags_ & RequestFlags::KeepInMemory);
+		if (!outfile_.empty() && !buffer_.FlushToFile(outfile_, clear)) {
 			ERROR_LOG(Log::IO, "Failed writing download to '%s'", outfile_.c_str());
 		}
 		progress_.Update(bodyLength, bodyLength, true);

Common/Net/HTTPNaettRequest.h

@@ -14,7 +14,7 @@ namespace http {
 // Really an asynchronous request.
 class HTTPSRequest : public Request {
 public:
-	HTTPSRequest(RequestMethod method, std::string_view url, std::string_view postData, std::string_view postMime, const Path &outfile, RequestFlags progressBarMode = RequestFlags::ProgressBar | RequestFlags::ProgressBarDelayed, std::string_view name = "");
+	HTTPSRequest(RequestMethod method, std::string_view url, std::string_view postData, std::string_view postMime, const Path &outfile, RequestFlags flags = RequestFlags::ProgressBar | RequestFlags::ProgressBarDelayed, std::string_view name = "");
 	~HTTPSRequest();
 
 	void Start() override;

Common/Net/HTTPRequest.cpp

@@ -2,15 +2,16 @@
 #include "Common/Net/HTTPClient.h"
 #include "Common/Net/HTTPNaettRequest.h"
 #include "Common/TimeUtil.h"
+#include "Common/File/FileUtil.h"
 #include "Common/StringUtils.h"
 #include "Common/Log.h"
 #include "Common/System/OSD.h"
 #include "Common/System/System.h"
 
 namespace http {
 
-Request::Request(RequestMethod method, std::string_view url, std::string_view name, bool *cancelled, RequestFlags mode)
-	: method_(method), url_(url), name_(name), progress_(cancelled), flags_(mode) {
+Request::Request(RequestMethod method, std::string_view url, std::string_view name, bool *cancelled, RequestFlags flags)
+	: method_(method), url_(url), name_(name), progress_(cancelled), flags_(flags) {
 	INFO_LOG(Log::HTTP, "HTTP %s request: %.*s (%.*s)", RequestMethodToString(method), (int)url.size(), url.data(), (int)name.size(), name.data());
 
 	progress_.callback = [=](int64_t bytes, int64_t contentLength, bool done) {
@@ -39,25 +40,73 @@ static bool IsHttpsUrl(std::string_view url) {
 	return startsWith(url, "https:");
 }
 
+Path UrlToCachePath(const Path &cacheDir, std::string_view url) {
+	std::string fn = "DLCACHE_";
+	for (auto c : url) {
+		if (isalnum(c) || c == '.' || c == '-' || c == '_') {
+			fn.push_back(tolower(c));
+		} else {
+			fn.push_back('_');
+		}
+	}
+	return cacheDir / fn;
+}
+
+Path RequestManager::UrlToCachePath(const std::string_view url) {
+	if (cacheDir_.empty()) {
+		return Path();
+	}
+	return http::UrlToCachePath(cacheDir_, url);
+}
+
 std::shared_ptr<Request> CreateRequest(RequestMethod method, std::string_view url, std::string_view postdata, std::string_view postMime, const Path &outfile, RequestFlags flags, std::string_view name) {
 	if (IsHttpsUrl(url) && System_GetPropertyBool(SYSPROP_SUPPORTS_HTTPS)) {
 #ifndef HTTPS_NOT_AVAILABLE
-		return std::shared_ptr<Request>(new HTTPSRequest(method, url, postdata, postMime, outfile, flags, name));
+		return std::make_shared<HTTPSRequest>(method, url, postdata, postMime, outfile, flags, name);
 #else
 		return std::shared_ptr<Request>();
 #endif
 	} else {
-		return std::shared_ptr<Request>(new HTTPRequest(method, url, postdata, postMime, outfile, flags, name));
+		return std::make_shared<HTTPRequest>(method, url, postdata, postMime, outfile, flags, name);
 	}
 }
 
 std::shared_ptr<Request> RequestManager::StartDownload(std::string_view url, const Path &outfile, RequestFlags flags, const char *acceptMime) {
 	std::shared_ptr<Request> dl = CreateRequest(RequestMethod::GET, url, "", "", outfile, flags, "");
 
+	if (!cacheDir_.empty() && (flags & RequestFlags::Cached24H)) {
+		_dbg_assert_(outfile.empty());  // It's automatically replaced below
+
+		// Come up with a cache file path.
+		Path cacheFile = UrlToCachePath(url);
+
+		// TODO: This should be done on the thread, maybe. But let's keep it simple for now.
+		time_t cacheFileTime;
+		if (File::GetModifTimeT(cacheFile, &cacheFileTime)) {
+			time_t now = (time_t)time_now_unix_utc();
+			if (cacheFileTime > now - 24 * 60 * 60) {
+				// The file is new enough. Let's construct a fake, already finished download so we don't need
+				// to modify the calling code.
+				std::string contents;
+				if (File::ReadBinaryFileToString(cacheFile, &contents)) {
+					// All is well, but we've indented a bit much here.
+					dl.reset(new CachedRequest(RequestMethod::GET, url, "", nullptr, flags, contents));
+					newDownloads_.push_back(dl);
+					return dl;
+				}
+			}
+		}
+
+		// OK, didn't get it from cache, so let's continue with the download, putting it in the cache.
+		dl->OverrideOutFile(cacheFile);
+		dl->AddFlag(RequestFlags::KeepInMemory);
+	}
+
 	if (!userAgent_.empty())
 		dl->SetUserAgent(userAgent_);
 	if (acceptMime)
 		dl->SetAccept(acceptMime);
+
 	newDownloads_.push_back(dl);
 	dl->Start();
 	return dl;

Common/Net/HTTPRequest.h

@@ -18,7 +18,8 @@ enum class RequestFlags {
 	Default = 0,
 	ProgressBar = 1,
 	ProgressBarDelayed = 2,
-	// Cached = 4 etc
+	Cached24H = 4,
+	KeepInMemory = 8,
 };
 ENUM_CLASS_BITOPS(RequestFlags);
 
@@ -59,6 +60,12 @@ class Request {
 	std::string url() const { return url_; }
 
 	const Path &OutFile() const { return outfile_; }
+	void OverrideOutFile(const Path &path) {
+		outfile_ = path;
+	}
+	void AddFlag(RequestFlags flag) {
+		flags_ |= flag;
+	}
 
 	void Cancel() { cancelled_ = true; }
 	bool IsCancelled() const { return cancelled_; }
@@ -97,6 +104,7 @@ class RequestManager {
 		CancelAll();
 	}
 
+	// NOTE: This is the only version that supports the cache flag (for now).
 	std::shared_ptr<Request> StartDownload(std::string_view url, const Path &outfile, RequestFlags flags, const char *acceptMime = nullptr);
 
 	std::shared_ptr<Request> StartDownloadWithCallback(
@@ -119,17 +127,24 @@ class RequestManager {
 	void Update();
 	void CancelAll();
 
-	void SetUserAgent(const std::string &userAgent) {
+	void SetUserAgent(std::string_view userAgent) {
 		userAgent_ = userAgent;
 	}
 
+	void SetCacheDir(const Path &path) {
+		cacheDir_ = path;
+	}
+
+	Path UrlToCachePath(const std::string_view url);
+
 private:
 	std::vector<std::shared_ptr<Request>> downloads_;
 	// These get copied to downloads_ in Update(). It's so that callbacks can add new downloads
 	// while running.
 	std::vector<std::shared_ptr<Request>> newDownloads_;
 
 	std::string userAgent_;
+	Path cacheDir_;
 };
 
 inline const char *RequestMethodToString(RequestMethod method) {

Core/Dialog/PSPNetconfDialog.cpp

@@ -58,6 +58,9 @@ int PSPNetconfDialog::Init(u32 paramAddr) {
 	if (ReadStatus() != SCE_UTILITY_STATUS_NONE)
 		return SCE_ERROR_UTILITY_INVALID_STATUS;
 
+	// Kick off a request to the infra-dns.json since we'll need it later.
+	StartInfraJsonDownload();
+
 	requestAddr = paramAddr;
 	int size = Memory::Read_U32(paramAddr);
 	memset(&request, 0, sizeof(request));
@@ -106,6 +109,17 @@ int PSPNetconfDialog::Update(int animSpeed) {
 	auto di = GetI18NCategory(I18NCat::DIALOG);
 	u64 now = (u64)(time_now_d() * 1000000.0);
 
+	std::string json;
+	if (!jsonReady_ && PollInfraJsonDownload(&json)) {
+		if (!json.empty()) {
+			INFO_LOG(Log::sceNet, "Got and processed the json.");
+		} else {
+			// TODO: Show a notice?
+			WARN_LOG(Log::sceNet, "Failed to get json file. Autoconfig will not work.");
+		}
+		jsonReady_ = true;
+	}
+
 	// It seems JPCSP doesn't check for NETCONF_STATUS_APNET
 	if (request.netAction == NETCONF_CONNECT_APNET || request.netAction == NETCONF_STATUS_APNET || request.netAction == NETCONF_CONNECT_APNET_LAST) {
 		int state = NetApctl_GetState();
@@ -161,15 +175,14 @@ int PSPNetconfDialog::Update(int animSpeed) {
 			}
 			DisplayButtons(DS_BUTTON_CANCEL, di->T("Cancel"));
 
-			// The Netconf dialog stays visible until the network reaches the state PSP_NET_APCTL_STATE_GOT_IP.			
-			if (state == PSP_NET_APCTL_STATE_GOT_IP) {
+			// The Netconf dialog stays visible until the network reaches the state PSP_NET_APCTL_STATE_GOT_IP,
+			// *AND* we have the json.
+			if (state == PSP_NET_APCTL_STATE_GOT_IP && jsonReady_) {
 				if (pendingStatus != SCE_UTILITY_STATUS_FINISHED) {
 					StartFade(false);
 					ChangeStatus(SCE_UTILITY_STATUS_FINISHED, NET_SHUTDOWN_DELAY_US);
 				}
-			}
-
-			else if (state == PSP_NET_APCTL_STATE_JOINING) {
+			} else if (state == PSP_NET_APCTL_STATE_JOINING) {
 				// Switch to the next message
 				StartFade(true);
 			}

Core/Dialog/PSPNetconfDialog.h

@@ -61,4 +61,6 @@ class PSPNetconfDialog : public PSPDialog {
 	u32 scanInfosAddr = 0;
 	int scanStep = 0;
 	u64 startTime = 0;
+
+	bool jsonReady_ = false;
 };

Core/HLE/sceIo.cpp

@@ -2791,8 +2791,7 @@ int __IoIoctl(u32 id, u32 cmd, u32 indataPtr, u32 inlen, u32 outdataPtr, u32 out
 	return hleLogDebug(Log::sceIo, 0);
 }
 
-u32 sceIoIoctl(u32 id, u32 cmd, u32 indataPtr, u32 inlen, u32 outdataPtr, u32 outlen)
-{
+u32 sceIoIoctl(u32 id, u32 cmd, u32 indataPtr, u32 inlen, u32 outdataPtr, u32 outlen) {
 	int usec = 0;
 	int result = __IoIoctl(id, cmd, indataPtr, inlen, outdataPtr, outlen, usec);
 	if (usec != 0) {