diff --git a/headless/Compare.cpp b/headless/Compare.cpp index 02252d9833..b87c03dc93 100644 --- a/headless/Compare.cpp +++ b/headless/Compare.cpp @@ -15,22 +15,182 @@ // Official git repository and contact information can be found at // https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/. -#include "Compare.h" -#include "FileUtil.h" +#include "headless/Compare.h" +#include "Common/FileUtil.h" +#include "Core/Host.h" -#include +#include +#include +#include +#include -bool CompareOutput(const std::string bootFilename) +bool teamCityMode = false; +std::string teamCityName = ""; + +void TeamCityPrint(const char *fmt, ...) { - std::string expect_filename = bootFilename.substr(bootFilename.length() - 4) + ".expected"; - if (File::Exists(expect_filename)) - { - // TODO: Do the compare here + if (!teamCityMode) + return; + + const int TEMP_BUFFER_SIZE = 32768; + char temp[TEMP_BUFFER_SIZE]; + + va_list args; + va_start(args, fmt); + vsnprintf(temp, TEMP_BUFFER_SIZE - 1, fmt, args); + temp[TEMP_BUFFER_SIZE - 1] = '\0'; + va_end(args); + + printf("%s", temp); +} + +struct BufferedLineReader { + const static int MAX_BUFFER = 5; + const static int TEMP_BUFFER_SIZE = 32768; + + BufferedLineReader(const std::string &data) : valid_(0), data_(data), pos_(0) { + } + + void Fill() { + while (valid_ < MAX_BUFFER && HasLines()) { + buffer_[valid_++] = ReadLine(); + } + } + + const std::string Peek(int pos) { + if (pos >= valid_) { + Fill(); + } + if (pos >= valid_) { + return ""; + } + return buffer_[pos]; + } + + void Skip(int count) { + if (count > valid_) { + count = valid_; + } + valid_ -= count; + for (int i = 0; i < valid_; ++i) { + buffer_[i] = buffer_[i + count]; + } + Fill(); + } + + const std::string Consume() { + const std::string result = Peek(0); + Skip(1); + return result; + } + + virtual bool HasLines() { + return pos_ != data_.npos; + } + + bool Compare(BufferedLineReader &other) { + if (Peek(0) != other.Peek(0)) { + return false; + } + + Skip(1); + other.Skip(1); return true; } + +protected: + BufferedLineReader() : valid_(0) { + } + + virtual std::string ReadLine() { + size_t next = data_.find('\n', pos_); + if (next == data_.npos) { + std::string result = data_.substr(pos_); + pos_ = next; + return result; + } else { + std::string result = data_.substr(pos_, next - pos_); + pos_ = next + 1; + return result; + } + } + + int valid_; + std::string buffer_[MAX_BUFFER]; + const std::string data_; + size_t pos_; +}; + +struct BufferedLineReaderFile : public BufferedLineReader { + BufferedLineReaderFile(std::ifstream &in) : BufferedLineReader(), in_(in) { + } + + virtual bool HasLines() { + return !in_.eof(); + } + +protected: + virtual std::string ReadLine() { + char temp[TEMP_BUFFER_SIZE]; + in_.getline(temp, TEMP_BUFFER_SIZE); + return temp; + } + + std::ifstream &in_; +}; + +bool CompareOutput(const std::string &bootFilename, const std::string &output) +{ + std::string expect_filename = bootFilename.substr(0, bootFilename.length() - 4) + ".expected"; + std::ifstream in; + in.open(expect_filename.c_str(), std::ios::in); + if (!in.fail()) + { + BufferedLineReaderFile expected(in); + BufferedLineReader actual(output); + + bool failed = false; + while (expected.HasLines()) + { + if (expected.Compare(actual)) + continue; + + if (!failed) + { + TeamCityPrint("##teamcity[testFailed name='%s' message='Output different from expected file']\n", teamCityName.c_str()); + failed = true; + } + + // This is a really dirt simple comparing algorithm. + + // Perhaps it was an extra line? + if (expected.Peek(0) == actual.Peek(1)) + printf("+ %s\n", actual.Consume().c_str()); + // A single missing line? + else if (expected.Peek(1) == actual.Peek(0)) + printf("- %s\n", expected.Consume().c_str()); + else + { + printf("O %s\n", actual.Consume().c_str()); + printf("E %s\n", expected.Consume().c_str()); + } + } + + while (actual.HasLines()) + { + // If it's a blank line, this will pass. + if (actual.Compare(expected)) + continue; + + printf("+ %s\n", actual.Consume().c_str()); + } + + return failed; + } else { - fprintf(stderr, "Expectation file %s not found", expect_filename.c_str()); + fprintf(stderr, "Expectation file %s not found\n", expect_filename.c_str()); + TeamCityPrint("##teamcity[testIgnored name='%s' message='Expects file missing']\n", teamCityName.c_str()); return false; } } diff --git a/headless/Compare.h b/headless/Compare.h index f51f6c1be2..2152e13b70 100644 --- a/headless/Compare.h +++ b/headless/Compare.h @@ -19,5 +19,9 @@ #include "Globals.h" -bool CompareOutput(std::string bootFilename); +extern bool teamCityMode; +extern std::string teamCityName; +void TeamCityPrint(const char *fmt, ...); + +bool CompareOutput(const std::string &bootFilename, const std::string &output); double CompareScreenshot(const u8 *pixels, int w, int h, int stride, const std::string screenshotFilename, std::string &error); \ No newline at end of file diff --git a/headless/Headless.cpp b/headless/Headless.cpp index 3990979bdf..4c425a5baa 100644 --- a/headless/Headless.cpp +++ b/headless/Headless.cpp @@ -102,6 +102,27 @@ static HeadlessHost * getHost(GPUCore gpuCore) { } } +static std::string ChopFront(std::string s, std::string front) +{ + if (s.size() >= front.size()) + { + if (s.substr(0, front.size()) == front) + return s.substr(front.size()); + } + return s; +} + +static std::string ChopEnd(std::string s, std::string end) +{ + if (s.size() >= end.size()) + { + size_t endpos = s.size() - end.size(); + if (s.substr(endpos) == end) + return s.substr(0, endpos); + } + return s; +} + int main(int argc, const char* argv[]) { bool fullLog = false; @@ -154,6 +175,8 @@ int main(int argc, const char* argv[]) gpuCore = GPU_GLES; else if (!strncmp(argv[i], "--screenshot=", strlen("--screenshot=")) && strlen(argv[i]) > strlen("--screenshot=")) screenshotFilename = argv[i] + strlen("--screenshot="); + else if (!strcmp(argv[i], "--teamcity")) + teamCityMode = true; else if (bootFilename == 0) bootFilename = argv[i]; else @@ -199,6 +222,8 @@ int main(int argc, const char* argv[]) logman->AddListener(type, printfLogger); } + std::string output; + CoreParameter coreParameter; coreParameter.cpuCore = useJit ? CPU_JIT : CPU_INTERPRETER; coreParameter.gpuCore = glWorking ? gpuCore : GPU_NULL; @@ -207,7 +232,9 @@ int main(int argc, const char* argv[]) coreParameter.mountIso = mountIso ? mountIso : ""; coreParameter.startPaused = false; coreParameter.enableDebugging = false; - coreParameter.printfEmuLog = true; + coreParameter.printfEmuLog = !autoCompare; + if (autoCompare) + coreParameter.collectEmuLog = &output; coreParameter.headLess = true; coreParameter.renderWidth = 480; coreParameter.renderHeight = 272; @@ -249,12 +276,20 @@ int main(int argc, const char* argv[]) g_Config.flashDirectory = g_Config.memCardDirectory+"/flash/"; #endif + if (teamCityMode) { + // Kinda ugly, trying to guesstimate the test name from filename... + teamCityName = ChopEnd(ChopFront(ChopFront(bootFilename, "tests/"), "pspautotests/tests/"), ".prx"); + } + if (!PSP_Init(coreParameter, &error_string)) { fprintf(stderr, "Failed to start %s. Error: %s\n", coreParameter.fileToStart.c_str(), error_string.c_str()); printf("TESTERROR\n"); + TeamCityPrint("##teamcity[testIgnored name='%s' message='PRX/ELF missing']\n", teamCityName.c_str()); return 1; } + TeamCityPrint("##teamcity[testStarted name='%s' captureStandardOutput='true']\n", teamCityName.c_str()); + host->BootDone(); if (screenshotFilename != 0) @@ -277,12 +312,15 @@ int main(int argc, const char* argv[]) PSP_Shutdown(); headlessHost->FlushDebugOutput(); + delete host; host = NULL; headlessHost = NULL; if (autoCompare) - CompareOutput(bootFilename); + CompareOutput(bootFilename, output); + + TeamCityPrint("##teamcity[testFinished name='%s']\n", teamCityName.c_str()); return 0; } diff --git a/headless/WindowsHeadlessHost.cpp b/headless/WindowsHeadlessHost.cpp index 467d641a81..0b48c9e579 100644 --- a/headless/WindowsHeadlessHost.cpp +++ b/headless/WindowsHeadlessHost.cpp @@ -22,6 +22,9 @@ #include "Common/CommonWindows.h" #include +#include "Core/CoreParameter.h" +#include "Core/System.h" + #include "base/logging.h" #include "gfx_es2/gl_state.h" #include "gfx/gl_common.h" @@ -87,6 +90,16 @@ void WindowsHeadlessHost::SendDebugOutput(const std::string &output) OutputDebugStringUTF8(output.c_str()); } +void WindowsHeadlessHost::SendOrCollectDebugOutput(const std::string &data) +{ + if (PSP_CoreParameter().printfEmuLog) + SendDebugOutput(data); + else if (PSP_CoreParameter().collectEmuLog) + *PSP_CoreParameter().collectEmuLog += data; + else + DEBUG_LOG(COMMON, "%s", data.c_str()); +} + void WindowsHeadlessHost::SendDebugScreenshot(const u8 *pixbuf, u32 w, u32 h) { // We ignore the current framebuffer parameters and just grab the full screen. @@ -101,14 +114,17 @@ void WindowsHeadlessHost::SendDebugScreenshot(const u8 *pixbuf, u32 w, u32 h) std::string error; double errors = CompareScreenshot(pixels, FRAME_WIDTH, FRAME_HEIGHT, FRAME_WIDTH, comparisonScreenshot, error); if (errors < 0) - SendDebugOutput(error); + SendOrCollectDebugOutput(error); if (errors > 0) { char temp[256]; sprintf_s(temp, "Screenshot error: %f%%\n", errors * 100.0f); - SendDebugOutput(temp); + SendOrCollectDebugOutput(temp); + } + if (errors > 0 && !teamCityMode) + { // Lazy, just read in the original header to output the failed screenshot. u8 header[14 + 40] = {0}; FILE *bmp = fopen(comparisonScreenshot.c_str(), "rb"); @@ -125,7 +141,7 @@ void WindowsHeadlessHost::SendDebugScreenshot(const u8 *pixbuf, u32 w, u32 h) fwrite(pixels, sizeof(u32), FRAME_WIDTH * FRAME_HEIGHT, saved); fclose(saved); - SendDebugOutput("Actual output written to: __testfailure.bmp\n"); + SendOrCollectDebugOutput("Actual output written to: __testfailure.bmp\n"); } } diff --git a/headless/WindowsHeadlessHost.h b/headless/WindowsHeadlessHost.h index e03010a5ba..f4a3db97c2 100644 --- a/headless/WindowsHeadlessHost.h +++ b/headless/WindowsHeadlessHost.h @@ -40,6 +40,7 @@ public: private: bool ResizeGL(); void LoadNativeAssets(); + void SendOrCollectDebugOutput(const std::string &output); HWND hWnd; HDC hDC;