Add an internal comparison algorithm to headless.

This is slightly smarter than the test.py version, and should be
completely compatible.  It doesn't handle timeouts though.

headless/Compare.cpp

@@ -15,22 +15,182 @@
 // Official git repository and contact information can be found at
 // https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
 
-#include "Compare.h"
-#include "FileUtil.h"
+#include "headless/Compare.h"
+#include "Common/FileUtil.h"
+#include "Core/Host.h"
 
-#include <math.h>
+#include <cmath>
+#include <cstdarg>
+#include <iostream>
+#include <fstream>
 
-bool CompareOutput(const std::string bootFilename)
+bool teamCityMode = false;
+std::string teamCityName = "";
+
+void TeamCityPrint(const char *fmt, ...)
 {
-	std::string expect_filename = bootFilename.substr(bootFilename.length() - 4) + ".expected";
-	if (File::Exists(expect_filename))
-	{
-		// TODO: Do the compare here
+	if (!teamCityMode)
+		return;
+
+	const int TEMP_BUFFER_SIZE = 32768;
+	char temp[TEMP_BUFFER_SIZE];
+
+	va_list args;
+	va_start(args, fmt);
+	vsnprintf(temp, TEMP_BUFFER_SIZE - 1, fmt, args);
+	temp[TEMP_BUFFER_SIZE - 1] = '\0';
+	va_end(args);
+
+	printf("%s", temp);
+}
+
+struct BufferedLineReader {
+	const static int MAX_BUFFER = 5;
+	const static int TEMP_BUFFER_SIZE = 32768;
+
+	BufferedLineReader(const std::string &data) : valid_(0), data_(data), pos_(0) {
+	}
+
+	void Fill() {
+		while (valid_ < MAX_BUFFER && HasLines()) {
+			buffer_[valid_++] = ReadLine();
+		}
+	}
+
+	const std::string Peek(int pos) {
+		if (pos >= valid_) {
+			Fill();
+		}
+		if (pos >= valid_) {
+			return "";
+		}
+		return buffer_[pos];
+	}
+
+	void Skip(int count) {
+		if (count > valid_) {
+			count = valid_;
+		}
+		valid_ -= count;
+		for (int i = 0; i < valid_; ++i) {
+			buffer_[i] = buffer_[i + count];
+		}
+		Fill();
+	}
+
+	const std::string Consume() {
+		const std::string result = Peek(0);
+		Skip(1);
+		return result;
+	}
+
+	virtual bool HasLines() {
+		return pos_ != data_.npos;
+	}
+
+	bool Compare(BufferedLineReader &other) {
+		if (Peek(0) != other.Peek(0)) {
+			return false;
+		}
+
+		Skip(1);
+		other.Skip(1);
 		return true;
 	}
+
+protected:
+	BufferedLineReader() : valid_(0) {
+	}
+
+	virtual std::string ReadLine() {
+		size_t next = data_.find('\n', pos_);
+		if (next == data_.npos) {
+			std::string result = data_.substr(pos_);
+			pos_ = next;
+			return result;
+		} else {
+			std::string result = data_.substr(pos_, next - pos_);
+			pos_ = next + 1;
+			return result;
+		}
+	}
+
+	int valid_;
+	std::string buffer_[MAX_BUFFER];
+	const std::string data_;
+	size_t pos_;
+};
+
+struct BufferedLineReaderFile : public BufferedLineReader {
+	BufferedLineReaderFile(std::ifstream &in) : BufferedLineReader(), in_(in) {
+	}
+
+	virtual bool HasLines() {
+		return !in_.eof();
+	}
+
+protected:
+	virtual std::string ReadLine() {
+		char temp[TEMP_BUFFER_SIZE];
+		in_.getline(temp, TEMP_BUFFER_SIZE);
+		return temp;
+	}
+
+	std::ifstream &in_;
+};
+
+bool CompareOutput(const std::string &bootFilename, const std::string &output)
+{
+	std::string expect_filename = bootFilename.substr(0, bootFilename.length() - 4) + ".expected";
+	std::ifstream in;
+	in.open(expect_filename.c_str(), std::ios::in);
+	if (!in.fail())
+	{
+		BufferedLineReaderFile expected(in);
+		BufferedLineReader actual(output);
+
+		bool failed = false;
+		while (expected.HasLines())
+		{
+			if (expected.Compare(actual))
+				continue;
+
+			if (!failed)
+			{
+				TeamCityPrint("##teamcity[testFailed name='%s' message='Output different from expected file']\n", teamCityName.c_str());
+				failed = true;
+			}
+
+			// This is a really dirt simple comparing algorithm.
+
+			// Perhaps it was an extra line?
+			if (expected.Peek(0) == actual.Peek(1))
+				printf("+ %s\n", actual.Consume().c_str());
+			// A single missing line?
+			else if (expected.Peek(1) == actual.Peek(0))
+				printf("- %s\n", expected.Consume().c_str());
+			else
+			{
+				printf("O %s\n", actual.Consume().c_str());
+				printf("E %s\n", expected.Consume().c_str());
+			}
+		}
+
+		while (actual.HasLines())
+		{
+			// If it's a blank line, this will pass.
+			if (actual.Compare(expected))
+				continue;
+
+			printf("+ %s\n", actual.Consume().c_str());
+		}
+
+		return failed;
+	}
 	else
 	{
-		fprintf(stderr, "Expectation file %s not found", expect_filename.c_str());
+		fprintf(stderr, "Expectation file %s not found\n", expect_filename.c_str());
+		TeamCityPrint("##teamcity[testIgnored name='%s' message='Expects file missing']\n", teamCityName.c_str());
 		return false;
 	}
 }

headless/Compare.h

@@ -19,5 +19,9 @@
 
 #include "Globals.h"
 
-bool CompareOutput(std::string bootFilename);
+extern bool teamCityMode;
+extern std::string teamCityName;
+void TeamCityPrint(const char *fmt, ...);
+
+bool CompareOutput(const std::string &bootFilename, const std::string &output);
 double CompareScreenshot(const u8 *pixels, int w, int h, int stride, const std::string screenshotFilename, std::string &error);

headless/Headless.cpp

@@ -102,6 +102,27 @@ static HeadlessHost * getHost(GPUCore gpuCore) {
 	}
 }
 
+static std::string ChopFront(std::string s, std::string front)
+{
+	if (s.size() >= front.size())
+	{
+		if (s.substr(0, front.size()) == front)
+			return s.substr(front.size());
+	}
+	return s;
+}
+
+static std::string ChopEnd(std::string s, std::string end)
+{
+	if (s.size() >= end.size())
+	{
+		size_t endpos = s.size() - end.size();
+		if (s.substr(endpos) == end)
+			return s.substr(0, endpos);
+	}
+	return s;
+}
+
 int main(int argc, const char* argv[])
 {
 	bool fullLog = false;
@@ -154,6 +175,8 @@ int main(int argc, const char* argv[])
 			gpuCore = GPU_GLES;
 		else if (!strncmp(argv[i], "--screenshot=", strlen("--screenshot=")) && strlen(argv[i]) > strlen("--screenshot="))
 			screenshotFilename = argv[i] + strlen("--screenshot=");
+		else if (!strcmp(argv[i], "--teamcity"))
+			teamCityMode = true;
 		else if (bootFilename == 0)
 			bootFilename = argv[i];
 		else
@@ -199,6 +222,8 @@ int main(int argc, const char* argv[])
 		logman->AddListener(type, printfLogger);
 	}
 
+	std::string output;
+
 	CoreParameter coreParameter;
 	coreParameter.cpuCore = useJit ? CPU_JIT : CPU_INTERPRETER;
 	coreParameter.gpuCore = glWorking ? gpuCore : GPU_NULL;
@@ -207,7 +232,9 @@ int main(int argc, const char* argv[])
 	coreParameter.mountIso = mountIso ? mountIso : "";
 	coreParameter.startPaused = false;
 	coreParameter.enableDebugging = false;
-	coreParameter.printfEmuLog = true;
+	coreParameter.printfEmuLog = !autoCompare;
+	if (autoCompare)
+		coreParameter.collectEmuLog = &output;
 	coreParameter.headLess = true;
 	coreParameter.renderWidth = 480;
 	coreParameter.renderHeight = 272;
@@ -249,12 +276,20 @@ int main(int argc, const char* argv[])
 	g_Config.flashDirectory = g_Config.memCardDirectory+"/flash/";
 #endif
 
+	if (teamCityMode) {
+		// Kinda ugly, trying to guesstimate the test name from filename...
+		teamCityName = ChopEnd(ChopFront(ChopFront(bootFilename, "tests/"), "pspautotests/tests/"), ".prx");
+	}
+
 	if (!PSP_Init(coreParameter, &error_string)) {
 		fprintf(stderr, "Failed to start %s. Error: %s\n", coreParameter.fileToStart.c_str(), error_string.c_str());
 		printf("TESTERROR\n");
+		TeamCityPrint("##teamcity[testIgnored name='%s' message='PRX/ELF missing']\n", teamCityName.c_str());
 		return 1;
 	}
 
+	TeamCityPrint("##teamcity[testStarted name='%s' captureStandardOutput='true']\n", teamCityName.c_str());
+
 	host->BootDone();
 
 	if (screenshotFilename != 0)
@@ -277,12 +312,15 @@ int main(int argc, const char* argv[])
 	PSP_Shutdown();
 
 	headlessHost->FlushDebugOutput();
+
 	delete host;
 	host = NULL;
 	headlessHost = NULL;
 
 	if (autoCompare)
-		CompareOutput(bootFilename);
+		CompareOutput(bootFilename, output);
+
+	TeamCityPrint("##teamcity[testFinished name='%s']\n", teamCityName.c_str());
 
 	return 0;
 }

headless/WindowsHeadlessHost.cpp

@@ -22,6 +22,9 @@
 #include "Common/CommonWindows.h"
 #include <io.h>
 
+#include "Core/CoreParameter.h"
+#include "Core/System.h"
+
 #include "base/logging.h"
 #include "gfx_es2/gl_state.h"
 #include "gfx/gl_common.h"
@@ -87,6 +90,16 @@ void WindowsHeadlessHost::SendDebugOutput(const std::string &output)
 	OutputDebugStringUTF8(output.c_str());
 }
 
+void WindowsHeadlessHost::SendOrCollectDebugOutput(const std::string &data)
+{
+	if (PSP_CoreParameter().printfEmuLog)
+		SendDebugOutput(data);
+	else if (PSP_CoreParameter().collectEmuLog)
+		*PSP_CoreParameter().collectEmuLog += data;
+	else
+		DEBUG_LOG(COMMON, "%s", data.c_str());
+}
+
 void WindowsHeadlessHost::SendDebugScreenshot(const u8 *pixbuf, u32 w, u32 h)
 {
 	// We ignore the current framebuffer parameters and just grab the full screen.
@@ -101,14 +114,17 @@ void WindowsHeadlessHost::SendDebugScreenshot(const u8 *pixbuf, u32 w, u32 h)
 	std::string error;
 	double errors = CompareScreenshot(pixels, FRAME_WIDTH, FRAME_HEIGHT, FRAME_WIDTH, comparisonScreenshot, error);
 	if (errors < 0)
-		SendDebugOutput(error);
+		SendOrCollectDebugOutput(error);
 
 	if (errors > 0)
 	{
 		char temp[256];
 		sprintf_s(temp, "Screenshot error: %f%%\n", errors * 100.0f);
-		SendDebugOutput(temp);
+		SendOrCollectDebugOutput(temp);
+	}
 
+	if (errors > 0 && !teamCityMode)
+	{
 		// Lazy, just read in the original header to output the failed screenshot.
 		u8 header[14 + 40] = {0};
 		FILE *bmp = fopen(comparisonScreenshot.c_str(), "rb");
@@ -125,7 +141,7 @@ void WindowsHeadlessHost::SendDebugScreenshot(const u8 *pixbuf, u32 w, u32 h)
 			fwrite(pixels, sizeof(u32), FRAME_WIDTH * FRAME_HEIGHT, saved);
 			fclose(saved);
 
-			SendDebugOutput("Actual output written to: __testfailure.bmp\n");
+			SendOrCollectDebugOutput("Actual output written to: __testfailure.bmp\n");
 		}
 	}
 

headless/WindowsHeadlessHost.h

@@ -40,6 +40,7 @@ public:
 private:
 	bool ResizeGL();
 	void LoadNativeAssets();
+	void SendOrCollectDebugOutput(const std::string &output);
 
 	HWND hWnd;
 	HDC hDC;