Add an internal comparison algorithm to headless.

This is slightly smarter than the test.py version, and should be
completely compatible.  It doesn't handle timeouts though.
This commit is contained in:
Unknown W. Brackets 2013-09-16 08:15:59 -07:00
parent 35cd59ae87
commit 57b636816b
5 changed files with 234 additions and 15 deletions

View file

@ -15,22 +15,182 @@
// Official git repository and contact information can be found at
// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
#include "Compare.h"
#include "FileUtil.h"
#include "headless/Compare.h"
#include "Common/FileUtil.h"
#include "Core/Host.h"
#include <math.h>
#include <cmath>
#include <cstdarg>
#include <iostream>
#include <fstream>
bool CompareOutput(const std::string bootFilename)
bool teamCityMode = false;
std::string teamCityName = "";
void TeamCityPrint(const char *fmt, ...)
{
std::string expect_filename = bootFilename.substr(bootFilename.length() - 4) + ".expected";
if (File::Exists(expect_filename))
{
// TODO: Do the compare here
if (!teamCityMode)
return;
const int TEMP_BUFFER_SIZE = 32768;
char temp[TEMP_BUFFER_SIZE];
va_list args;
va_start(args, fmt);
vsnprintf(temp, TEMP_BUFFER_SIZE - 1, fmt, args);
temp[TEMP_BUFFER_SIZE - 1] = '\0';
va_end(args);
printf("%s", temp);
}
struct BufferedLineReader {
const static int MAX_BUFFER = 5;
const static int TEMP_BUFFER_SIZE = 32768;
BufferedLineReader(const std::string &data) : valid_(0), data_(data), pos_(0) {
}
void Fill() {
while (valid_ < MAX_BUFFER && HasLines()) {
buffer_[valid_++] = ReadLine();
}
}
const std::string Peek(int pos) {
if (pos >= valid_) {
Fill();
}
if (pos >= valid_) {
return "";
}
return buffer_[pos];
}
void Skip(int count) {
if (count > valid_) {
count = valid_;
}
valid_ -= count;
for (int i = 0; i < valid_; ++i) {
buffer_[i] = buffer_[i + count];
}
Fill();
}
const std::string Consume() {
const std::string result = Peek(0);
Skip(1);
return result;
}
virtual bool HasLines() {
return pos_ != data_.npos;
}
bool Compare(BufferedLineReader &other) {
if (Peek(0) != other.Peek(0)) {
return false;
}
Skip(1);
other.Skip(1);
return true;
}
protected:
BufferedLineReader() : valid_(0) {
}
virtual std::string ReadLine() {
size_t next = data_.find('\n', pos_);
if (next == data_.npos) {
std::string result = data_.substr(pos_);
pos_ = next;
return result;
} else {
std::string result = data_.substr(pos_, next - pos_);
pos_ = next + 1;
return result;
}
}
int valid_;
std::string buffer_[MAX_BUFFER];
const std::string data_;
size_t pos_;
};
struct BufferedLineReaderFile : public BufferedLineReader {
BufferedLineReaderFile(std::ifstream &in) : BufferedLineReader(), in_(in) {
}
virtual bool HasLines() {
return !in_.eof();
}
protected:
virtual std::string ReadLine() {
char temp[TEMP_BUFFER_SIZE];
in_.getline(temp, TEMP_BUFFER_SIZE);
return temp;
}
std::ifstream &in_;
};
bool CompareOutput(const std::string &bootFilename, const std::string &output)
{
std::string expect_filename = bootFilename.substr(0, bootFilename.length() - 4) + ".expected";
std::ifstream in;
in.open(expect_filename.c_str(), std::ios::in);
if (!in.fail())
{
BufferedLineReaderFile expected(in);
BufferedLineReader actual(output);
bool failed = false;
while (expected.HasLines())
{
if (expected.Compare(actual))
continue;
if (!failed)
{
TeamCityPrint("##teamcity[testFailed name='%s' message='Output different from expected file']\n", teamCityName.c_str());
failed = true;
}
// This is a really dirt simple comparing algorithm.
// Perhaps it was an extra line?
if (expected.Peek(0) == actual.Peek(1))
printf("+ %s\n", actual.Consume().c_str());
// A single missing line?
else if (expected.Peek(1) == actual.Peek(0))
printf("- %s\n", expected.Consume().c_str());
else
{
printf("O %s\n", actual.Consume().c_str());
printf("E %s\n", expected.Consume().c_str());
}
}
while (actual.HasLines())
{
// If it's a blank line, this will pass.
if (actual.Compare(expected))
continue;
printf("+ %s\n", actual.Consume().c_str());
}
return failed;
}
else
{
fprintf(stderr, "Expectation file %s not found", expect_filename.c_str());
fprintf(stderr, "Expectation file %s not found\n", expect_filename.c_str());
TeamCityPrint("##teamcity[testIgnored name='%s' message='Expects file missing']\n", teamCityName.c_str());
return false;
}
}

View file

@ -19,5 +19,9 @@
#include "Globals.h"
bool CompareOutput(std::string bootFilename);
extern bool teamCityMode;
extern std::string teamCityName;
void TeamCityPrint(const char *fmt, ...);
bool CompareOutput(const std::string &bootFilename, const std::string &output);
double CompareScreenshot(const u8 *pixels, int w, int h, int stride, const std::string screenshotFilename, std::string &error);

View file

@ -102,6 +102,27 @@ static HeadlessHost * getHost(GPUCore gpuCore) {
}
}
static std::string ChopFront(std::string s, std::string front)
{
if (s.size() >= front.size())
{
if (s.substr(0, front.size()) == front)
return s.substr(front.size());
}
return s;
}
static std::string ChopEnd(std::string s, std::string end)
{
if (s.size() >= end.size())
{
size_t endpos = s.size() - end.size();
if (s.substr(endpos) == end)
return s.substr(0, endpos);
}
return s;
}
int main(int argc, const char* argv[])
{
bool fullLog = false;
@ -154,6 +175,8 @@ int main(int argc, const char* argv[])
gpuCore = GPU_GLES;
else if (!strncmp(argv[i], "--screenshot=", strlen("--screenshot=")) && strlen(argv[i]) > strlen("--screenshot="))
screenshotFilename = argv[i] + strlen("--screenshot=");
else if (!strcmp(argv[i], "--teamcity"))
teamCityMode = true;
else if (bootFilename == 0)
bootFilename = argv[i];
else
@ -199,6 +222,8 @@ int main(int argc, const char* argv[])
logman->AddListener(type, printfLogger);
}
std::string output;
CoreParameter coreParameter;
coreParameter.cpuCore = useJit ? CPU_JIT : CPU_INTERPRETER;
coreParameter.gpuCore = glWorking ? gpuCore : GPU_NULL;
@ -207,7 +232,9 @@ int main(int argc, const char* argv[])
coreParameter.mountIso = mountIso ? mountIso : "";
coreParameter.startPaused = false;
coreParameter.enableDebugging = false;
coreParameter.printfEmuLog = true;
coreParameter.printfEmuLog = !autoCompare;
if (autoCompare)
coreParameter.collectEmuLog = &output;
coreParameter.headLess = true;
coreParameter.renderWidth = 480;
coreParameter.renderHeight = 272;
@ -249,12 +276,20 @@ int main(int argc, const char* argv[])
g_Config.flashDirectory = g_Config.memCardDirectory+"/flash/";
#endif
if (teamCityMode) {
// Kinda ugly, trying to guesstimate the test name from filename...
teamCityName = ChopEnd(ChopFront(ChopFront(bootFilename, "tests/"), "pspautotests/tests/"), ".prx");
}
if (!PSP_Init(coreParameter, &error_string)) {
fprintf(stderr, "Failed to start %s. Error: %s\n", coreParameter.fileToStart.c_str(), error_string.c_str());
printf("TESTERROR\n");
TeamCityPrint("##teamcity[testIgnored name='%s' message='PRX/ELF missing']\n", teamCityName.c_str());
return 1;
}
TeamCityPrint("##teamcity[testStarted name='%s' captureStandardOutput='true']\n", teamCityName.c_str());
host->BootDone();
if (screenshotFilename != 0)
@ -277,12 +312,15 @@ int main(int argc, const char* argv[])
PSP_Shutdown();
headlessHost->FlushDebugOutput();
delete host;
host = NULL;
headlessHost = NULL;
if (autoCompare)
CompareOutput(bootFilename);
CompareOutput(bootFilename, output);
TeamCityPrint("##teamcity[testFinished name='%s']\n", teamCityName.c_str());
return 0;
}

View file

@ -22,6 +22,9 @@
#include "Common/CommonWindows.h"
#include <io.h>
#include "Core/CoreParameter.h"
#include "Core/System.h"
#include "base/logging.h"
#include "gfx_es2/gl_state.h"
#include "gfx/gl_common.h"
@ -87,6 +90,16 @@ void WindowsHeadlessHost::SendDebugOutput(const std::string &output)
OutputDebugStringUTF8(output.c_str());
}
void WindowsHeadlessHost::SendOrCollectDebugOutput(const std::string &data)
{
if (PSP_CoreParameter().printfEmuLog)
SendDebugOutput(data);
else if (PSP_CoreParameter().collectEmuLog)
*PSP_CoreParameter().collectEmuLog += data;
else
DEBUG_LOG(COMMON, "%s", data.c_str());
}
void WindowsHeadlessHost::SendDebugScreenshot(const u8 *pixbuf, u32 w, u32 h)
{
// We ignore the current framebuffer parameters and just grab the full screen.
@ -101,14 +114,17 @@ void WindowsHeadlessHost::SendDebugScreenshot(const u8 *pixbuf, u32 w, u32 h)
std::string error;
double errors = CompareScreenshot(pixels, FRAME_WIDTH, FRAME_HEIGHT, FRAME_WIDTH, comparisonScreenshot, error);
if (errors < 0)
SendDebugOutput(error);
SendOrCollectDebugOutput(error);
if (errors > 0)
{
char temp[256];
sprintf_s(temp, "Screenshot error: %f%%\n", errors * 100.0f);
SendDebugOutput(temp);
SendOrCollectDebugOutput(temp);
}
if (errors > 0 && !teamCityMode)
{
// Lazy, just read in the original header to output the failed screenshot.
u8 header[14 + 40] = {0};
FILE *bmp = fopen(comparisonScreenshot.c_str(), "rb");
@ -125,7 +141,7 @@ void WindowsHeadlessHost::SendDebugScreenshot(const u8 *pixbuf, u32 w, u32 h)
fwrite(pixels, sizeof(u32), FRAME_WIDTH * FRAME_HEIGHT, saved);
fclose(saved);
SendDebugOutput("Actual output written to: __testfailure.bmp\n");
SendOrCollectDebugOutput("Actual output written to: __testfailure.bmp\n");
}
}

View file

@ -40,6 +40,7 @@ public:
private:
bool ResizeGL();
void LoadNativeAssets();
void SendOrCollectDebugOutput(const std::string &output);
HWND hWnd;
HDC hDC;