Emulation/Gopher2600
1
0
Fork 0
mirror of https://github.com/JetSetIlly/Gopher2600.git synced 2025-04-02 11:02:17 -04:00
Code Issues Projects Releases Packages Wiki Activity

o regression

Browse source 
- tidied up regression database operation
    - removed UPDATE option (for now). it's probably never needed but we
    can add it again if it's ever needed
This commit is contained in:
steve 2019-05-01 02:52:38 +01:00
parent 218c3d7823
commit 609ab331a2
7 changed files with 375 additions and 180 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
errors/categories.go
View file

@ -20,10 +20,8 @@ const (
ScriptEnd
// Regression
RegressionEntryExists
RegressionEntryCollision
RegressionEntryDoesNotExist
RegressionEntryFail
RegressionDBError
RegressionFail
// CPU
UnimplementedInstruction

6
errors/messages.go
View file

@ -19,10 +19,8 @@ var messages = map[Errno]string{
ScriptRecordingError: "error when recording script (%s)",
// Regression
RegressionEntryExists: "entry exists (%s)",
RegressionEntryCollision: "ROM hash collision (%s AND %s)",
RegressionEntryDoesNotExist: "entry missing (%s)",
RegressionEntryFail: "screen digest mismatch (%s)",
RegressionDBError: "database error: (%s)",
RegressionFail: "screen digest mismatch (%s)",
// CPU
UnimplementedInstruction: "unimplemented instruction (%0#x) at (%#04x)",

45
gopher2600.go
View file

@ -235,9 +235,9 @@ func main() {
default:
modeArgPos-- // undo modeArgPos adjustment
fallthrough
case "RUN":
verbose := modeFlags.Bool("verbose", false, "display details of each test")
failOnError := modeFlags.Bool("fail", false, "fail on error: boolean")
modeFlagsParse()
var output io.Writer
@ -247,7 +247,7 @@ func main() {
switch len(modeFlags.Args()) {
case 0:
succeed, fail, err := regression.RegressRunTests(output, *failOnError)
succeed, fail, err := regression.RegressRunTests(output)
if err != nil {
fmt.Printf("* error during regression tests: %s\n", err)
os.Exit(2)
@ -258,6 +258,15 @@ func main() {
os.Exit(2)
}
case "LIST":
var output io.Writer
output = os.Stdout
err := regression.RegressList(output)
if err != nil {
fmt.Printf("* error during regression listing: %s\n", err)
os.Exit(2)
}
case "DELETE":
modeFlagsParse()
@ -266,7 +275,7 @@ func main() {
fmt.Println("* 2600 cartridge required")
os.Exit(2)
case 1:
err := regression.RegressDeleteCartridge(modeFlags.Arg(0))
err := regression.RegressDelete(modeFlags.Arg(0))
if err != nil {
fmt.Printf("* error deleting regression entry: %s\n", err)
os.Exit(2)
@ -284,10 +293,16 @@ func main() {
switch len(modeFlags.Args()) {
case 0:
fmt.Println("* 2600 cartridge required")
fmt.Println("* 2600 cartridge or playback file required")
os.Exit(2)
case 1:
err := regression.RegressAddCartridge(modeFlags.Arg(0), *tvType, *numFrames)
// TODO: adding different record types
newRecord := &regression.FrameRecord{
CartridgeFile: modeFlags.Arg(0),
TVtype: *tvType,
NumFrames: *numFrames}
err := regression.RegressAdd(newRecord)
if err != nil {
fmt.Printf("* error adding regression test: %s\n", err)
os.Exit(2)
@ -297,26 +312,6 @@ func main() {
fmt.Printf("* too many arguments for %s mode\n", mode)
os.Exit(2)
}
case "UPDATE":
tvType := modeFlags.String("tv", "NTSC", "television specification: NTSC, PAL")
numFrames := modeFlags.Int("frames", 10, "number of frames to run")
modeFlagsParse()
switch len(modeFlags.Args()) {
case 0:
fmt.Println("* 2600 cartridge required")
os.Exit(2)
case 1:
err := regression.RegressUpdateCartridge(modeFlags.Arg(0), *tvType, *numFrames)
if err != nil {
fmt.Printf("* error updating regression test: %s\n", err)
os.Exit(2)
}
fmt.Printf("! updated %s in regression database\n", path.Base(modeFlags.Arg(0)))
default:
fmt.Printf("* too many arguments for %s mode\n", mode)
os.Exit(2)
}
}
}
}

237
regression/database.go
View file

@ -1,32 +1,65 @@
package regression
import (
"encoding/csv"
"fmt"
"gopher2600/errors"
"io"
"io/ioutil"
"os"
"sort"
"strconv"
"strings"
)
const regressionDBFile = ".gopher2600/regressionDB"
const fieldSep = ","
const recordSep = "\n"
type regressionEntry struct {
cartridgePath string
tvMode string
numOFrames int
screenDigest string
}
const numFields = 4
func (entry regressionEntry) String() string {
return fmt.Sprintf("%s [%s] frames=%d", entry.cartridgePath, entry.tvMode, entry.numOFrames)
}
// arbitrary number of records
const maxRecords = 1000
type regressionDB struct {
dbfile *os.File
entries map[string]regressionEntry
records map[int]record
// sorted list of keys. used for:
// - displaying records in correct order in listRecords()
// - saving in correct order in endSession()
keys []int
}
type record interface {
// getID returns the string that is used to identify the record type in the
// database
getID() string
// String implements the Stringer interface
String() string
// setKey sets the key value for the record
setKey(int)
// getKey returns the key assigned to the record
getKey() int
// getCSV returns the comma separated string representing the record.
// without record separator. the first two fields should be the result of
// csvRecordLeader()
getCSV() string
// Run performs the regression test for the record type
regress(newRecord bool) (bool, error)
}
const (
leaderFieldKey int = iota
leaderFieldID
numLeaderFields
)
// csvRecordLeader returns the first two fields required for every record type
func csvLeader(rec record) string {
return fmt.Sprintf("%03d%s%s", rec.getKey(), fieldSep, rec.getID())
}
func startSession() (*regressionDB, error) {
@ -39,7 +72,7 @@ func startSession() (*regressionDB, error) {
return nil, err
}
err = db.readEntries()
err = db.readRecords()
if err != nil {
return nil, err
}
@ -48,10 +81,8 @@ func startSession() (*regressionDB, error) {
}
func (db *regressionDB) endSession(commitChanges bool) error {
// write entries to regression database
// write records to regression database
if commitChanges {
csvw := csv.NewWriter(db.dbfile)
err := db.dbfile.Truncate(0)
if err != nil {
return err
@ -59,24 +90,9 @@ func (db *regressionDB) endSession(commitChanges bool) error {
db.dbfile.Seek(0, os.SEEK_SET)
for _, entry := range db.entries {
rec := make([]string, numFields)
rec[0] = entry.cartridgePath
rec[1] = entry.tvMode
rec[2] = strconv.Itoa(entry.numOFrames)
rec[3] = entry.screenDigest
err := csvw.Write(rec)
if err != nil {
return err
}
}
// make sure everything's been written
csvw.Flush()
err = csvw.Error()
if err != nil {
return err
for _, key := range db.keys {
db.dbfile.WriteString(db.records[key].getCSV())
db.dbfile.WriteString(recordSep)
}
}
@ -91,78 +107,115 @@ func (db *regressionDB) endSession(commitChanges bool) error {
return nil
}
func (db *regressionDB) readEntries() error {
// readEntries clobbers the contents of db.entries
db.entries = make(map[string]regressionEntry, len(db.entries))
// treat the file as a CSV file
csvr := csv.NewReader(db.dbfile)
csvr.Comment = rune('#')
csvr.TrimLeadingSpace = true
csvr.ReuseRecord = true
csvr.FieldsPerRecord = numFields
func (db *regressionDB) readRecords() error {
// readrecords clobbers the contents of db.entrie
db.records = make(map[int]record, len(db.records))
// make sure we're at the beginning of the file
db.dbfile.Seek(0, os.SEEK_SET)
for {
buffer, err := ioutil.ReadAll(db.dbfile)
if err != nil {
return errors.NewFormattedError(errors.RegressionDBError, err)
}
// split records
lines := strings.Split(string(buffer), recordSep)
for i := 0; i < len(lines); i++ {
lines[i] = strings.TrimSpace(lines[i])
if len(lines[i]) == 0 {
continue
}
// loop through file until EOF is reached
rec, err := csvr.Read()
if err == io.EOF {
fields := strings.SplitN(lines[i], fieldSep, numLeaderFields+1)
key, err := strconv.Atoi(fields[leaderFieldKey])
if err != nil {
msg := fmt.Sprintf("invalid key [%s] at line %d", fields[leaderFieldKey], i+1)
return errors.NewFormattedError(errors.RegressionDBError, msg)
}
if _, ok := db.records[key]; ok {
msg := fmt.Sprintf("duplicate key [%v] at line %d", key, i+1)
return errors.NewFormattedError(errors.RegressionDBError, msg)
}
var rec record
switch fields[leaderFieldID] {
case "frame":
rec, err = newFrameRecord(key, fields[numLeaderFields])
if err != nil {
return err
}
default:
msg := fmt.Sprintf("unrecognised record type [%s]", fields[leaderFieldID])
return errors.NewFormattedError(errors.RegressionDBError, msg)
}
db.records[key] = rec
// add key to list
db.keys = append(db.keys, key)
}
// sort key list
sort.Ints(db.keys)
return nil
}
func (db regressionDB) listRecords(output io.Writer) {
for k := range db.keys {
output.Write([]byte(fmt.Sprintf("%03d [%s] ", db.keys[k], db.records[db.keys[k]].getID())))
output.Write([]byte(db.records[db.keys[k]].String()))
output.Write([]byte("\n"))
}
}
// addRecord adds a cartridge to the regression db
func (db *regressionDB) addRecord(rec record) error {
var key int
// find spare key
for key = 0; key < maxRecords; key++ {
if _, ok := db.records[key]; !ok {
break
}
if err != nil {
return err
}
numOfFrames, err := strconv.Atoi(rec[2])
if err != nil {
return err
}
// add entry to database
entry := regressionEntry{
cartridgePath: rec[0],
tvMode: rec[1],
numOFrames: numOfFrames,
screenDigest: rec[3]}
db.entries[entry.cartridgePath] = entry
}
if key == maxRecords {
msg := fmt.Sprintf("%d record maximum exceeded", maxRecords)
return errors.NewFormattedError(errors.RegressionDBError, msg)
}
rec.setKey(key)
db.records[key] = rec
// add key to list and resort
db.keys = append(db.keys, key)
sort.Ints(db.keys)
return nil
}
// RegressAddCartridge adds a cartridge to the regression db
func addCartridge(cartridgeFile string, tvMode string, numOfFrames int, allowUpdate bool) error {
db, err := startSession()
if err != nil {
return err
}
defer db.endSession(true)
// run cartdrige and get digest
digest, err := run(cartridgeFile, tvMode, numOfFrames)
if err != nil {
return err
func (db *regressionDB) delRecord(key int) error {
if _, ok := db.records[key]; ok == false {
msg := fmt.Sprintf("key not found [%d]", key)
return errors.NewFormattedError(errors.RegressionDBError, msg)
}
entry := regressionEntry{
cartridgePath: cartridgeFile,
tvMode: tvMode,
numOFrames: numOfFrames,
screenDigest: digest}
delete(db.records, key)
if allowUpdate == false {
if existEntry, ok := db.entries[entry.cartridgePath]; ok {
if existEntry.cartridgePath == entry.cartridgePath {
return errors.NewFormattedError(errors.RegressionEntryExists, entry)
}
return errors.NewFormattedError(errors.RegressionEntryCollision, entry.cartridgePath, existEntry.cartridgePath)
// find key in list and delete
for i := 0; i < len(db.keys); i++ {
if db.keys[i] == key {
db.keys = append(db.keys[:i], db.keys[i+1:]...)
break // for loop
}
}
db.entries[entry.cartridgePath] = entry
return nil
}

102
regression/frame.go Normal file
View file

@ -0,0 +1,102 @@
package regression
import (
"fmt"
"gopher2600/errors"
"gopher2600/hardware"
"gopher2600/television/renderers"
"strconv"
"strings"
)
const (
frameFieldCartName int = iota
frameFieldTVtype
frameFieldNumFrames
frameFieldDigest
numFrameFields
)
// FrameRecord is the simplest regression database record type
type FrameRecord struct {
key int
CartridgeFile string
TVtype string
NumFrames int
screenDigest string
}
func (rec FrameRecord) getID() string {
return "frame"
}
func newFrameRecord(key int, csv string) (*FrameRecord, error) {
rec := &FrameRecord{key: key}
// loop through file until EOF is reached
fields := strings.Split(csv, ",")
rec.screenDigest = fields[frameFieldDigest]
rec.CartridgeFile = fields[frameFieldCartName]
rec.TVtype = fields[frameFieldTVtype]
var err error
rec.NumFrames, err = strconv.Atoi(fields[frameFieldNumFrames])
if err != nil {
msg := fmt.Sprintf("invalid numFrames field [%s]", fields[frameFieldNumFrames])
return nil, errors.NewFormattedError(errors.RegressionDBError, msg)
}
return rec, nil
}
func (rec *FrameRecord) setKey(key int) {
rec.key = key
}
func (rec FrameRecord) getKey() int {
return rec.key
}
func (rec *FrameRecord) getCSV() string {
return fmt.Sprintf("%s%s%s%s%s%s%d%s%s",
csvLeader(rec), fieldSep,
rec.CartridgeFile, fieldSep,
rec.TVtype, fieldSep,
rec.NumFrames, fieldSep,
rec.screenDigest,
)
}
func (rec FrameRecord) String() string {
return fmt.Sprintf("%s [%s] frames=%d", rec.CartridgeFile, rec.TVtype, rec.NumFrames)
}
func (rec *FrameRecord) regress(newRecord bool) (bool, error) {
tv, err := renderers.NewDigestTV(rec.TVtype, nil)
if err != nil {
return false, fmt.Errorf("error preparing television: %s", err)
}
vcs, err := hardware.NewVCS(tv)
if err != nil {
return false, fmt.Errorf("error preparing VCS: %s", err)
}
err = vcs.AttachCartridge(rec.CartridgeFile)
if err != nil {
return false, err
}
err = vcs.RunForFrameCount(rec.NumFrames)
if err != nil {
return false, err
}
if newRecord {
rec.screenDigest = tv.String()
return true, nil
}
return tv.String() == rec.screenDigest, nil
}

56
regression/playback.go Normal file
View file

@ -0,0 +1,56 @@
package regression
import (
"fmt"
"strings"
)
const (
playbackFieldScript int = iota
numPlaybackFields
)
// PlaybackRecord i
type PlaybackRecord struct {
key int
Script string
}
func (rec PlaybackRecord) getID() string {
return "playback"
}
func newPlaybackRecord(key int, csv string) (*PlaybackRecord, error) {
// loop through file until EOF is reached
fields := strings.Split(csv, ",")
rec := &PlaybackRecord{
key: key,
Script: fields[playbackFieldScript],
}
return rec, nil
}
func (rec *PlaybackRecord) setKey(key int) {
rec.key = key
}
func (rec PlaybackRecord) getKey() int {
return rec.key
}
func (rec *PlaybackRecord) getCSV() string {
return fmt.Sprintf("%s%s%s",
csvLeader(rec), fieldSep,
rec.Script,
)
}
func (rec PlaybackRecord) String() string {
return rec.Script
}
func (rec *PlaybackRecord) regress(newRecord bool) (bool, error) {
return true, nil
}

103
regression/regression.go
View file

@ -3,40 +3,60 @@ package regression
import (
"fmt"
"gopher2600/errors"
"gopher2600/hardware"
"gopher2600/television/renderers"
"io"
"strconv"
)
// RegressDeleteCartridge removes a cartridge from the regression db
func RegressDeleteCartridge(cartridgeFile string) error {
// RegressList displays all entries in the database
func RegressList(output io.Writer) error {
db, err := startSession()
if err != nil {
return err
}
defer db.endSession(false)
db.listRecords(output)
return nil
}
// RegressDelete removes a cartridge from the regression db
func RegressDelete(key string) error {
v, err := strconv.Atoi(key)
if err != nil {
msg := fmt.Sprintf("invalid key [%s]", key)
return errors.NewFormattedError(errors.RegressionDBError, msg)
}
// TODO: display record and ask for confirmatio
db, err := startSession()
if err != nil {
return err
}
defer db.endSession(true)
if _, ok := db.entries[cartridgeFile]; ok == false {
return errors.NewFormattedError(errors.RegressionEntryDoesNotExist, cartridgeFile)
return db.delRecord(v)
}
// RegressAdd adds a cartridge or run-recording to the regression db
func RegressAdd(rec record) error {
_, err := rec.regress(true)
if err != nil {
return err
}
delete(db.entries, cartridgeFile)
db, err := startSession()
if err != nil {
return err
}
defer db.endSession(true)
return nil
}
// RegressAddCartridge adds a cartridge to the regression db
func RegressAddCartridge(cartridgeFile string, tvMode string, numOfFrames int) error {
return addCartridge(cartridgeFile, tvMode, numOfFrames, false)
}
// RegressUpdateCartridge updates a entry (or adds it if it doesn't exist)
func RegressUpdateCartridge(cartridgeFile string, tvMode string, numOfFrames int) error {
return addCartridge(cartridgeFile, tvMode, numOfFrames, true)
return db.addRecord(rec)
}
// RegressRunTests runs all the tests in the regression database
func RegressRunTests(output io.Writer, failOnError bool) (int, int, error) {
func RegressRunTests(output io.Writer) (int, int, error) {
db, err := startSession()
if err != nil {
return -1, -1, err
@ -45,54 +65,27 @@ func RegressRunTests(output io.Writer, failOnError bool) (int, int, error) {
numSucceed := 0
numFail := 0
for _, entry := range db.entries {
digest, err := run(entry.cartridgePath, entry.tvMode, entry.numOFrames)
for _, key := range db.keys {
rec := db.records[key]
if err != nil || entry.screenDigest != digest {
if err == nil {
err = errors.NewFormattedError(errors.RegressionEntryFail, entry)
}
ok, err := rec.regress(false)
if err != nil {
return numSucceed, numFail, errors.NewFormattedError(errors.RegressionFail, rec.String())
}
if !ok {
numFail++
if failOnError {
return numSucceed, numFail, err
}
if output != nil {
output.Write([]byte(fmt.Sprintf("fail: %s\n", err)))
output.Write([]byte(fmt.Sprintf("fail: %s\n", rec)))
}
} else {
numSucceed++
if output != nil {
output.Write([]byte(fmt.Sprintf("succeed: %s\n", entry)))
output.Write([]byte(fmt.Sprintf("succeed: %s\n", rec)))
}
}
}
return numSucceed, numFail, nil
}
func run(cartridgeFile string, tvMode string, numOfFrames int) (string, error) {
tv, err := renderers.NewDigestTV(tvMode, nil)
if err != nil {
return "", fmt.Errorf("error preparing television: %s", err)
}
vcs, err := hardware.NewVCS(tv)
if err != nil {
return "", fmt.Errorf("error preparing VCS: %s", err)
}
err = vcs.AttachCartridge(cartridgeFile)
if err != nil {
return "", err
}
err = vcs.RunForFrameCount(numOfFrames)
if err != nil {
return "", err
}
// output current digest
return fmt.Sprintf("%s", tv), nil
}