mirror of
https://github.com/hrydgard/ppsspp.git
synced 2025-04-02 11:01:50 -04:00
Add an internal comparison algorithm to headless.
This is slightly smarter than the test.py version, and should be completely compatible. It doesn't handle timeouts though.
This commit is contained in:
parent
35cd59ae87
commit
57b636816b
5 changed files with 234 additions and 15 deletions
|
@ -15,22 +15,182 @@
|
|||
// Official git repository and contact information can be found at
|
||||
// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
|
||||
|
||||
#include "Compare.h"
|
||||
#include "FileUtil.h"
|
||||
#include "headless/Compare.h"
|
||||
#include "Common/FileUtil.h"
|
||||
#include "Core/Host.h"
|
||||
|
||||
#include <math.h>
|
||||
#include <cmath>
|
||||
#include <cstdarg>
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
|
||||
bool CompareOutput(const std::string bootFilename)
|
||||
bool teamCityMode = false;
|
||||
std::string teamCityName = "";
|
||||
|
||||
void TeamCityPrint(const char *fmt, ...)
|
||||
{
|
||||
std::string expect_filename = bootFilename.substr(bootFilename.length() - 4) + ".expected";
|
||||
if (File::Exists(expect_filename))
|
||||
{
|
||||
// TODO: Do the compare here
|
||||
if (!teamCityMode)
|
||||
return;
|
||||
|
||||
const int TEMP_BUFFER_SIZE = 32768;
|
||||
char temp[TEMP_BUFFER_SIZE];
|
||||
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
vsnprintf(temp, TEMP_BUFFER_SIZE - 1, fmt, args);
|
||||
temp[TEMP_BUFFER_SIZE - 1] = '\0';
|
||||
va_end(args);
|
||||
|
||||
printf("%s", temp);
|
||||
}
|
||||
|
||||
struct BufferedLineReader {
|
||||
const static int MAX_BUFFER = 5;
|
||||
const static int TEMP_BUFFER_SIZE = 32768;
|
||||
|
||||
BufferedLineReader(const std::string &data) : valid_(0), data_(data), pos_(0) {
|
||||
}
|
||||
|
||||
void Fill() {
|
||||
while (valid_ < MAX_BUFFER && HasLines()) {
|
||||
buffer_[valid_++] = ReadLine();
|
||||
}
|
||||
}
|
||||
|
||||
const std::string Peek(int pos) {
|
||||
if (pos >= valid_) {
|
||||
Fill();
|
||||
}
|
||||
if (pos >= valid_) {
|
||||
return "";
|
||||
}
|
||||
return buffer_[pos];
|
||||
}
|
||||
|
||||
void Skip(int count) {
|
||||
if (count > valid_) {
|
||||
count = valid_;
|
||||
}
|
||||
valid_ -= count;
|
||||
for (int i = 0; i < valid_; ++i) {
|
||||
buffer_[i] = buffer_[i + count];
|
||||
}
|
||||
Fill();
|
||||
}
|
||||
|
||||
const std::string Consume() {
|
||||
const std::string result = Peek(0);
|
||||
Skip(1);
|
||||
return result;
|
||||
}
|
||||
|
||||
virtual bool HasLines() {
|
||||
return pos_ != data_.npos;
|
||||
}
|
||||
|
||||
bool Compare(BufferedLineReader &other) {
|
||||
if (Peek(0) != other.Peek(0)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Skip(1);
|
||||
other.Skip(1);
|
||||
return true;
|
||||
}
|
||||
|
||||
protected:
|
||||
BufferedLineReader() : valid_(0) {
|
||||
}
|
||||
|
||||
virtual std::string ReadLine() {
|
||||
size_t next = data_.find('\n', pos_);
|
||||
if (next == data_.npos) {
|
||||
std::string result = data_.substr(pos_);
|
||||
pos_ = next;
|
||||
return result;
|
||||
} else {
|
||||
std::string result = data_.substr(pos_, next - pos_);
|
||||
pos_ = next + 1;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
int valid_;
|
||||
std::string buffer_[MAX_BUFFER];
|
||||
const std::string data_;
|
||||
size_t pos_;
|
||||
};
|
||||
|
||||
struct BufferedLineReaderFile : public BufferedLineReader {
|
||||
BufferedLineReaderFile(std::ifstream &in) : BufferedLineReader(), in_(in) {
|
||||
}
|
||||
|
||||
virtual bool HasLines() {
|
||||
return !in_.eof();
|
||||
}
|
||||
|
||||
protected:
|
||||
virtual std::string ReadLine() {
|
||||
char temp[TEMP_BUFFER_SIZE];
|
||||
in_.getline(temp, TEMP_BUFFER_SIZE);
|
||||
return temp;
|
||||
}
|
||||
|
||||
std::ifstream &in_;
|
||||
};
|
||||
|
||||
bool CompareOutput(const std::string &bootFilename, const std::string &output)
|
||||
{
|
||||
std::string expect_filename = bootFilename.substr(0, bootFilename.length() - 4) + ".expected";
|
||||
std::ifstream in;
|
||||
in.open(expect_filename.c_str(), std::ios::in);
|
||||
if (!in.fail())
|
||||
{
|
||||
BufferedLineReaderFile expected(in);
|
||||
BufferedLineReader actual(output);
|
||||
|
||||
bool failed = false;
|
||||
while (expected.HasLines())
|
||||
{
|
||||
if (expected.Compare(actual))
|
||||
continue;
|
||||
|
||||
if (!failed)
|
||||
{
|
||||
TeamCityPrint("##teamcity[testFailed name='%s' message='Output different from expected file']\n", teamCityName.c_str());
|
||||
failed = true;
|
||||
}
|
||||
|
||||
// This is a really dirt simple comparing algorithm.
|
||||
|
||||
// Perhaps it was an extra line?
|
||||
if (expected.Peek(0) == actual.Peek(1))
|
||||
printf("+ %s\n", actual.Consume().c_str());
|
||||
// A single missing line?
|
||||
else if (expected.Peek(1) == actual.Peek(0))
|
||||
printf("- %s\n", expected.Consume().c_str());
|
||||
else
|
||||
{
|
||||
printf("O %s\n", actual.Consume().c_str());
|
||||
printf("E %s\n", expected.Consume().c_str());
|
||||
}
|
||||
}
|
||||
|
||||
while (actual.HasLines())
|
||||
{
|
||||
// If it's a blank line, this will pass.
|
||||
if (actual.Compare(expected))
|
||||
continue;
|
||||
|
||||
printf("+ %s\n", actual.Consume().c_str());
|
||||
}
|
||||
|
||||
return failed;
|
||||
}
|
||||
else
|
||||
{
|
||||
fprintf(stderr, "Expectation file %s not found", expect_filename.c_str());
|
||||
fprintf(stderr, "Expectation file %s not found\n", expect_filename.c_str());
|
||||
TeamCityPrint("##teamcity[testIgnored name='%s' message='Expects file missing']\n", teamCityName.c_str());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -19,5 +19,9 @@
|
|||
|
||||
#include "Globals.h"
|
||||
|
||||
bool CompareOutput(std::string bootFilename);
|
||||
extern bool teamCityMode;
|
||||
extern std::string teamCityName;
|
||||
void TeamCityPrint(const char *fmt, ...);
|
||||
|
||||
bool CompareOutput(const std::string &bootFilename, const std::string &output);
|
||||
double CompareScreenshot(const u8 *pixels, int w, int h, int stride, const std::string screenshotFilename, std::string &error);
|
|
@ -102,6 +102,27 @@ static HeadlessHost * getHost(GPUCore gpuCore) {
|
|||
}
|
||||
}
|
||||
|
||||
static std::string ChopFront(std::string s, std::string front)
|
||||
{
|
||||
if (s.size() >= front.size())
|
||||
{
|
||||
if (s.substr(0, front.size()) == front)
|
||||
return s.substr(front.size());
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
static std::string ChopEnd(std::string s, std::string end)
|
||||
{
|
||||
if (s.size() >= end.size())
|
||||
{
|
||||
size_t endpos = s.size() - end.size();
|
||||
if (s.substr(endpos) == end)
|
||||
return s.substr(0, endpos);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
int main(int argc, const char* argv[])
|
||||
{
|
||||
bool fullLog = false;
|
||||
|
@ -154,6 +175,8 @@ int main(int argc, const char* argv[])
|
|||
gpuCore = GPU_GLES;
|
||||
else if (!strncmp(argv[i], "--screenshot=", strlen("--screenshot=")) && strlen(argv[i]) > strlen("--screenshot="))
|
||||
screenshotFilename = argv[i] + strlen("--screenshot=");
|
||||
else if (!strcmp(argv[i], "--teamcity"))
|
||||
teamCityMode = true;
|
||||
else if (bootFilename == 0)
|
||||
bootFilename = argv[i];
|
||||
else
|
||||
|
@ -199,6 +222,8 @@ int main(int argc, const char* argv[])
|
|||
logman->AddListener(type, printfLogger);
|
||||
}
|
||||
|
||||
std::string output;
|
||||
|
||||
CoreParameter coreParameter;
|
||||
coreParameter.cpuCore = useJit ? CPU_JIT : CPU_INTERPRETER;
|
||||
coreParameter.gpuCore = glWorking ? gpuCore : GPU_NULL;
|
||||
|
@ -207,7 +232,9 @@ int main(int argc, const char* argv[])
|
|||
coreParameter.mountIso = mountIso ? mountIso : "";
|
||||
coreParameter.startPaused = false;
|
||||
coreParameter.enableDebugging = false;
|
||||
coreParameter.printfEmuLog = true;
|
||||
coreParameter.printfEmuLog = !autoCompare;
|
||||
if (autoCompare)
|
||||
coreParameter.collectEmuLog = &output;
|
||||
coreParameter.headLess = true;
|
||||
coreParameter.renderWidth = 480;
|
||||
coreParameter.renderHeight = 272;
|
||||
|
@ -249,12 +276,20 @@ int main(int argc, const char* argv[])
|
|||
g_Config.flashDirectory = g_Config.memCardDirectory+"/flash/";
|
||||
#endif
|
||||
|
||||
if (teamCityMode) {
|
||||
// Kinda ugly, trying to guesstimate the test name from filename...
|
||||
teamCityName = ChopEnd(ChopFront(ChopFront(bootFilename, "tests/"), "pspautotests/tests/"), ".prx");
|
||||
}
|
||||
|
||||
if (!PSP_Init(coreParameter, &error_string)) {
|
||||
fprintf(stderr, "Failed to start %s. Error: %s\n", coreParameter.fileToStart.c_str(), error_string.c_str());
|
||||
printf("TESTERROR\n");
|
||||
TeamCityPrint("##teamcity[testIgnored name='%s' message='PRX/ELF missing']\n", teamCityName.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
TeamCityPrint("##teamcity[testStarted name='%s' captureStandardOutput='true']\n", teamCityName.c_str());
|
||||
|
||||
host->BootDone();
|
||||
|
||||
if (screenshotFilename != 0)
|
||||
|
@ -277,12 +312,15 @@ int main(int argc, const char* argv[])
|
|||
PSP_Shutdown();
|
||||
|
||||
headlessHost->FlushDebugOutput();
|
||||
|
||||
delete host;
|
||||
host = NULL;
|
||||
headlessHost = NULL;
|
||||
|
||||
if (autoCompare)
|
||||
CompareOutput(bootFilename);
|
||||
CompareOutput(bootFilename, output);
|
||||
|
||||
TeamCityPrint("##teamcity[testFinished name='%s']\n", teamCityName.c_str());
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
@ -22,6 +22,9 @@
|
|||
#include "Common/CommonWindows.h"
|
||||
#include <io.h>
|
||||
|
||||
#include "Core/CoreParameter.h"
|
||||
#include "Core/System.h"
|
||||
|
||||
#include "base/logging.h"
|
||||
#include "gfx_es2/gl_state.h"
|
||||
#include "gfx/gl_common.h"
|
||||
|
@ -87,6 +90,16 @@ void WindowsHeadlessHost::SendDebugOutput(const std::string &output)
|
|||
OutputDebugStringUTF8(output.c_str());
|
||||
}
|
||||
|
||||
void WindowsHeadlessHost::SendOrCollectDebugOutput(const std::string &data)
|
||||
{
|
||||
if (PSP_CoreParameter().printfEmuLog)
|
||||
SendDebugOutput(data);
|
||||
else if (PSP_CoreParameter().collectEmuLog)
|
||||
*PSP_CoreParameter().collectEmuLog += data;
|
||||
else
|
||||
DEBUG_LOG(COMMON, "%s", data.c_str());
|
||||
}
|
||||
|
||||
void WindowsHeadlessHost::SendDebugScreenshot(const u8 *pixbuf, u32 w, u32 h)
|
||||
{
|
||||
// We ignore the current framebuffer parameters and just grab the full screen.
|
||||
|
@ -101,14 +114,17 @@ void WindowsHeadlessHost::SendDebugScreenshot(const u8 *pixbuf, u32 w, u32 h)
|
|||
std::string error;
|
||||
double errors = CompareScreenshot(pixels, FRAME_WIDTH, FRAME_HEIGHT, FRAME_WIDTH, comparisonScreenshot, error);
|
||||
if (errors < 0)
|
||||
SendDebugOutput(error);
|
||||
SendOrCollectDebugOutput(error);
|
||||
|
||||
if (errors > 0)
|
||||
{
|
||||
char temp[256];
|
||||
sprintf_s(temp, "Screenshot error: %f%%\n", errors * 100.0f);
|
||||
SendDebugOutput(temp);
|
||||
SendOrCollectDebugOutput(temp);
|
||||
}
|
||||
|
||||
if (errors > 0 && !teamCityMode)
|
||||
{
|
||||
// Lazy, just read in the original header to output the failed screenshot.
|
||||
u8 header[14 + 40] = {0};
|
||||
FILE *bmp = fopen(comparisonScreenshot.c_str(), "rb");
|
||||
|
@ -125,7 +141,7 @@ void WindowsHeadlessHost::SendDebugScreenshot(const u8 *pixbuf, u32 w, u32 h)
|
|||
fwrite(pixels, sizeof(u32), FRAME_WIDTH * FRAME_HEIGHT, saved);
|
||||
fclose(saved);
|
||||
|
||||
SendDebugOutput("Actual output written to: __testfailure.bmp\n");
|
||||
SendOrCollectDebugOutput("Actual output written to: __testfailure.bmp\n");
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -40,6 +40,7 @@ public:
|
|||
private:
|
||||
bool ResizeGL();
|
||||
void LoadNativeAssets();
|
||||
void SendOrCollectDebugOutput(const std::string &output);
|
||||
|
||||
HWND hWnd;
|
||||
HDC hDC;
|
||||
|
|
Loading…
Add table
Reference in a new issue