From 23bd5917524f4a728e82f770ba0c06408e53485f Mon Sep 17 00:00:00 2001 From: steve Date: Tue, 1 Jan 2019 21:32:09 +0000 Subject: [PATCH] o gopher2600 - tidy up of command line parsing o regression testing - added regression package - implemented simple database to facilitate regression tests - uses DigestTV --- debugger/debugger.go | 2 + errors/categories.go | 6 + errors/messages.go | 6 + gopher2600.go | 315 ++++++++++++------ .../cpu/definitions/csv/instructions_gen.go | 2 +- hardware/vcs.go | 22 ++ regression/regression.go | 286 ++++++++++++++++ 7 files changed, 528 insertions(+), 111 deletions(-) create mode 100644 regression/regression.go diff --git a/debugger/debugger.go b/debugger/debugger.go index ea0ab974..ede428f6 100644 --- a/debugger/debugger.go +++ b/debugger/debugger.go @@ -55,6 +55,8 @@ type Debugger struct { // single-fire step traps. these are used for the STEP command, allowing // things like "STEP FRAME". + // -- note that the hardware.VCS type has the StepFrames() function, we're + // not using that here because this solution is more general and flexible stepTraps *traps // commandOnHalt says whether an sequence of commands should run automatically diff --git a/errors/categories.go b/errors/categories.go index cd1d6478..d8c17b69 100644 --- a/errors/categories.go +++ b/errors/categories.go @@ -12,6 +12,12 @@ const ( ScriptFileError InvalidTarget + // Regression + RegressionEntryExists + RegressionEntryCollision + RegressionEntryDoesNotExist + RegressionEntryFail + // CPU UnimplementedInstruction NullInstruction diff --git a/errors/messages.go b/errors/messages.go index 9fae19f4..3b85bc64 100644 --- a/errors/messages.go +++ b/errors/messages.go @@ -10,6 +10,12 @@ var messages = map[Errno]string{ ScriptFileCannotOpen: "cannot open script file (%s)", InvalidTarget: "invalid target (%s)", + // Regression + RegressionEntryExists: "entry exists (%s)", + RegressionEntryCollision: "ROM hash collision (%s AND %s)", + RegressionEntryDoesNotExist: "entry missing (%s)", + RegressionEntryFail: "screen digest mismatch (%s)", + // CPU UnimplementedInstruction: "unimplemented instruction (%0#x) at (%#04x)", NullInstruction: "unimplemented instruction (0xff)", diff --git a/gopher2600.go b/gopher2600.go index bb654307..ce07a065 100644 --- a/gopher2600.go +++ b/gopher2600.go @@ -9,10 +9,12 @@ import ( "gopher2600/disassembly" "gopher2600/errors" "gopher2600/hardware" + "gopher2600/regression" "gopher2600/television" - "gopher2600/television/digesttv" "gopher2600/television/sdltv" + "io" "os" + "path" "runtime" "runtime/pprof" "strings" @@ -23,142 +25,236 @@ import ( const initScript = ".gopher2600/debuggerInit" func main() { - mode := flag.String("mode", "DEBUG", "emulation mode: DEBUG, DISASM, RUN, PLAY, FPS, TVFPS, IMAGEGEN, REGRESS") - termType := flag.String("term", "COLOR", "terminal type to use in debug mode: COLOR, PLAIN") - flag.Parse() + progName := path.Base(os.Args[0]) - cartridgeFile := "" - if len(flag.Args()) == 1 { - cartridgeFile = flag.Args()[0] - } else if len(flag.Args()) > 1 { - fmt.Println("* too many arguments") - os.Exit(10) + progFlags := flag.NewFlagSet(progName, flag.ExitOnError) + progFlags.Parse(os.Args[1:]) + + if len(progFlags.Args()) == 0 { + fmt.Println("* mode or cartridge required") + os.Exit(2) } - switch strings.ToUpper(*mode) { + mode := strings.ToUpper(progFlags.Arg(0)) + modeArgPos := 1 + modeFlags := flag.NewFlagSet(fmt.Sprintf("%s %s", progName, mode), flag.ExitOnError) + modeFlagsParse := func() { + if len(progFlags.Args()) >= modeArgPos { + modeFlags.Parse(progFlags.Args()[modeArgPos:]) + } + } + + switch mode { + default: + // RUN is the default mode + modeArgPos = 0 + fallthrough + + case "RUN": + tvMode := modeFlags.String("tv", "NTSC", "television specification: NTSC, PAL") + scaling := modeFlags.Float64("scale", 3.0, "television scaling") + modeFlagsParse() + + switch len(modeFlags.Args()) { + case 0: + fmt.Println("* 2600 cartridge required") + os.Exit(2) + case 1: + err := run(modeFlags.Arg(0), *tvMode, float32(*scaling)) + if err != nil { + fmt.Printf("* error running emulator: %s\n", err) + os.Exit(2) + } + default: + fmt.Printf("* too many arguments for %s mode\n", mode) + os.Exit(2) + } + case "DEBUG": + termType := modeFlags.String("term", "COLOR", "terminal type to use in debug mode: COLOR, PLAIN") + modeFlagsParse() + dbg, err := debugger.NewDebugger() if err != nil { fmt.Printf("* error starting debugger: %s\n", err) - os.Exit(10) + os.Exit(2) } // start debugger with choice of interface and cartridge var term ui.UserInterface switch strings.ToUpper(*termType) { - case "COLOR": - term = new(colorterm.ColorTerminal) default: fmt.Printf("! unknown terminal type (%s) defaulting to plain\n", *termType) fallthrough case "PLAIN": term = nil + case "COLOR": + term = new(colorterm.ColorTerminal) } - err = dbg.Start(term, cartridgeFile, initScript) - if err != nil { - fmt.Printf("* error running debugger: %s\n", err) - os.Exit(10) + switch len(modeFlags.Args()) { + case 0: + // it's okay if DEBUG mode is started with no cartridges + fallthrough + case 1: + err := dbg.Start(term, modeFlags.Arg(0), initScript) + if err != nil { + fmt.Printf("* error running debugger: %s\n", err) + os.Exit(2) + } + default: + fmt.Printf("* too many arguments for %s mode\n", mode) + os.Exit(2) } case "DISASM": - dsm, err := disassembly.NewDisassembly(cartridgeFile) - if err != nil { - switch err.(type) { - case errors.GopherError: - // print what disassembly output we do have - if dsm != nil { - dsm.Dump(os.Stdout) + modeFlagsParse() + + switch len(modeFlags.Args()) { + case 0: + fmt.Println("* 2600 cartridge required") + os.Exit(2) + case 1: + dsm, err := disassembly.NewDisassembly(modeFlags.Arg(0)) + if err != nil { + switch err.(type) { + case errors.GopherError: + // print what disassembly output we do have + if dsm != nil { + dsm.Dump(os.Stdout) + } } + fmt.Printf("* error during disassembly: %s\n", err) + os.Exit(2) } - fmt.Printf("* error during disassembly: %s\n", err) - os.Exit(10) + dsm.Dump(os.Stdout) + default: + fmt.Printf("* too many arguments for %s mode\n", mode) + os.Exit(2) } - dsm.Dump(os.Stdout) case "FPS": - err := fps(cartridgeFile, true) - if err != nil { - fmt.Printf("* error starting FPS profiler: %s\n", err) - os.Exit(10) - } + display := modeFlags.Bool("display", false, "display TV output: boolean") + tvMode := modeFlags.String("tv", "NTSC", "television specification: NTSC, PAL") + scaling := modeFlags.Float64("scale", 3.0, "television scaling") + frames := modeFlags.Int("frames", 100, "number of frames to run") + modeFlagsParse() - case "TVFPS": - err := fps(cartridgeFile, false) - if err != nil { - fmt.Printf("* error starting TVFPS profiler: %s\n", err) - os.Exit(10) + switch len(modeFlags.Args()) { + case 0: + fmt.Println("* 2600 cartridge required") + os.Exit(2) + case 1: + err := fps(modeFlags.Arg(0), *display, *tvMode, float32(*scaling), *frames) + if err != nil { + fmt.Printf("* error starting fps profiler: %s\n", err) + os.Exit(2) + } + default: + fmt.Printf("* too many arguments for %s mode\n", mode) + os.Exit(2) } case "REGRESS": - err := regress(cartridgeFile, 3) - if err != nil { - fmt.Printf("* error running TEST: %s\n", err) - os.Exit(10) - } + subMode := strings.ToUpper(progFlags.Arg(1)) + modeArgPos++ + switch subMode { + default: + modeArgPos-- // undo modeArgPos adjustment + fallthrough + case "RUN": + verbose := modeFlags.Bool("verbose", false, "display details of each test") + failOnError := modeFlags.Bool("fail", false, "fail on error: boolean") + modeFlagsParse() - case "PLAY": - // PLAY is a synonym for RUN - fallthrough + var output io.Writer + if *verbose == true { + output = os.Stdout + } - case "RUN": - err := run(cartridgeFile) - if err != nil { - fmt.Printf("* error running emulator: %s\n", err) - os.Exit(10) + switch len(modeFlags.Args()) { + case 0: + succeed, fail, err := regression.RegressRunTests(output, *failOnError) + if err != nil { + fmt.Printf("* error during regression tests: %s\n", err) + os.Exit(2) + } + fmt.Printf("regression tests: %d succeed, %d fail\n", succeed, fail) + default: + fmt.Printf("* too many arguments for %s mode\n", mode) + os.Exit(2) + } + + case "DELETE": + modeFlagsParse() + + switch len(modeFlags.Args()) { + case 0: + fmt.Println("* 2600 cartridge required") + os.Exit(2) + case 1: + err := regression.RegressDeleteCartridge(modeFlags.Arg(0)) + if err != nil { + fmt.Printf("* error deleting regression entry: %s\n", err) + os.Exit(2) + } + fmt.Printf("! deleted %s from regression database\n", path.Base(modeFlags.Arg(0))) + default: + fmt.Printf("* too many arguments for %s mode\n", mode) + os.Exit(2) + } + + case "ADD": + tvMode := modeFlags.String("tv", "NTSC", "television specification: NTSC, PAL") + numFrames := modeFlags.Int("frames", 10, "number of frames to run") + modeFlagsParse() + + switch len(modeFlags.Args()) { + case 0: + fmt.Println("* 2600 cartridge required") + os.Exit(2) + case 1: + err := regression.RegressAddCartridge(modeFlags.Arg(0), *tvMode, *numFrames) + if err != nil { + fmt.Printf("* error adding regression test: %s\n", err) + os.Exit(2) + } + fmt.Printf("! added %s to regression database\n", path.Base(modeFlags.Arg(0))) + default: + fmt.Printf("* too many arguments for %s mode\n", mode) + os.Exit(2) + } + case "UPDATE": + tvMode := modeFlags.String("tv", "NTSC", "television specification: NTSC, PAL") + numFrames := modeFlags.Int("frames", 10, "number of frames to run") + modeFlagsParse() + + switch len(modeFlags.Args()) { + case 0: + fmt.Println("* 2600 cartridge required") + os.Exit(2) + case 1: + err := regression.RegressUpdateCartridge(modeFlags.Arg(0), *tvMode, *numFrames) + if err != nil { + fmt.Printf("* error updating regression test: %s\n", err) + os.Exit(2) + } + fmt.Printf("! updated %s in regression database\n", path.Base(modeFlags.Arg(0))) + default: + fmt.Printf("* too many arguments for %s mode\n", mode) + os.Exit(2) + } } - default: - fmt.Printf("* unknown mode: %s\n", strings.ToUpper(*mode)) - os.Exit(10) } } -func regress(cartridgeFile string, numOfFrames int) error { - tv, err := digesttv.NewDigestTV("NTSC") - if err != nil { - return fmt.Errorf("error preparing television: %s", err) - } - - vcs, err := hardware.NewVCS(tv) - if err != nil { - return fmt.Errorf("error preparing VCS: %s", err) - } - - err = vcs.AttachCartridge(cartridgeFile) - if err != nil { - return err - } - - const cyclesPerFrame = 19912 - - // run emulation for a while - cycles := cyclesPerFrame * numOfFrames - for cycles > 0 { - stepCycles, _, err := vcs.Step(hardware.StubVideoCycleCallback) - if err != nil { - return err - } - cycles -= stepCycles - } - - // output current digest - fmt.Println(tv) - - return nil -} - -func fps(cartridgeFile string, justTheVCS bool) error { +func fps(cartridgeFile string, display bool, tvMode string, scaling float32, numOfFrames int) error { var tv television.Television var err error - if justTheVCS { - tv = new(television.DummyTV) - if tv == nil { - return fmt.Errorf("error preparing television") - } - } else { - tv, err = sdltv.NewSDLTV("NTSC", 3.0) + if display { + tv, err = sdltv.NewSDLTV(tvMode, scaling) if err != nil { return fmt.Errorf("error preparing television: %s", err) } @@ -167,6 +263,11 @@ func fps(cartridgeFile string, justTheVCS bool) error { if err != nil { return fmt.Errorf("error preparing television: %s", err) } + } else { + tv, err = television.NewHeadlessTV("NTSC") + if err != nil { + return fmt.Errorf("error preparing television: %s", err) + } } vcs, err := hardware.NewVCS(tv) @@ -179,9 +280,6 @@ func fps(cartridgeFile string, justTheVCS bool) error { return err } - const cyclesPerFrame = 19912 - const numOfFrames = 180 - // start cpu profile f, err := os.Create("cpu.profile") if err != nil { @@ -193,19 +291,16 @@ func fps(cartridgeFile string, justTheVCS bool) error { } defer pprof.StopCPUProfile() - // run emulation for a while - cycles := cyclesPerFrame * numOfFrames + // run for numOfFrames and calculate frames-per-second startTime := time.Now() - for cycles > 0 { - stepCycles, _, err := vcs.Step(hardware.StubVideoCycleCallback) - if err != nil { - return err - } - cycles -= stepCycles + err = vcs.RunFrames(numOfFrames) + if err != nil { + return err } + fps := float64(numOfFrames) / time.Since(startTime).Seconds() // display estimated fps - fmt.Printf("%f fps\n", float64(numOfFrames)/time.Since(startTime).Seconds()) + fmt.Printf("%f fps\n", fps) // write memory profile f, err = os.Create("mem.profile") @@ -222,8 +317,8 @@ func fps(cartridgeFile string, justTheVCS bool) error { return nil } -func run(cartridgeFile string) error { - tv, err := sdltv.NewSDLTV("NTSC", 3.0) +func run(cartridgeFile, tvMode string, scaling float32) error { + tv, err := sdltv.NewSDLTV(tvMode, scaling) if err != nil { return fmt.Errorf("error preparing television: %s", err) } diff --git a/hardware/cpu/definitions/csv/instructions_gen.go b/hardware/cpu/definitions/csv/instructions_gen.go index 0907510b..f52e68c9 100644 --- a/hardware/cpu/definitions/csv/instructions_gen.go +++ b/hardware/cpu/definitions/csv/instructions_gen.go @@ -67,7 +67,7 @@ func parseCSV() (map[uint8]definitions.InstructionDefinition, error) { return nil, fmt.Errorf("wrong number of fields in instruction definition (%s)", rec) } - // trim trailing comment from last record + // trim trailing comment from record rec[len(rec)-1] = strings.Split(rec[len(rec)-1], "#")[0] // manually trim trailing space from all fields in the record diff --git a/hardware/vcs.go b/hardware/vcs.go index fd856827..a19621a9 100644 --- a/hardware/vcs.go +++ b/hardware/vcs.go @@ -216,3 +216,25 @@ func (vcs *VCS) Reset() error { return nil } + +// RunFrames sets emulator running for the specified number of frames +// - not used by the debugger because traps and steptraps are more flexible +// - useful for fps and regression tests +func (vcs *VCS) RunFrames(numFrames int) error { + frm, err := vcs.TV.RequestTVState(television.ReqFramenum) + if err != nil { + return err + } + + targetFrame := frm.Value().(int) + numFrames + + for frm.Value().(int) != targetFrame { + _, _, err = vcs.Step(func(*result.Instruction) error { return nil }) + frm, err = vcs.TV.RequestTVState(television.ReqFramenum) + if err != nil { + return err + } + } + + return nil +} diff --git a/regression/regression.go b/regression/regression.go new file mode 100644 index 00000000..4122114d --- /dev/null +++ b/regression/regression.go @@ -0,0 +1,286 @@ +package regression + +import ( + "crypto/sha1" + "encoding/csv" + "fmt" + "gopher2600/errors" + "gopher2600/hardware" + "gopher2600/television/digesttv" + "io" + "os" + "path" + "strconv" +) + +const regressionDBFile = ".gopher2600/regressionDB" + +func keyify(cartridgeFile string) (string, error) { + f, err := os.Open(cartridgeFile) + if err != nil { + return "", err + } + defer f.Close() + + key := sha1.New() + if _, err := io.Copy(key, f); err != nil { + return "", err + } + + return fmt.Sprintf("%x", key.Sum(nil)), nil +} + +type regressionEntry struct { + key string + cartridgeFile string + tvMode string + numOFrames int + digest string +} + +func (entry regressionEntry) String() string { + return fmt.Sprintf("%s [%s] frames=%d", path.Base(entry.cartridgeFile), entry.tvMode, entry.numOFrames) +} + +type regressionDB struct { + dbfile *os.File + entries map[string]regressionEntry +} + +func (db *regressionDB) endSession() error { + // write entries to regression database + csvw := csv.NewWriter(db.dbfile) + + err := db.dbfile.Truncate(0) + if err != nil { + return err + } + + db.dbfile.Seek(0, os.SEEK_SET) + + for _, entry := range db.entries { + rec := make([]string, 5) + rec[0] = entry.key + rec[1] = entry.cartridgeFile + rec[2] = entry.tvMode + rec[3] = strconv.Itoa(entry.numOFrames) + rec[4] = entry.digest + + err := csvw.Write(rec) + if err != nil { + return err + } + } + + // make sure everything's been written + csvw.Flush() + err = csvw.Error() + if err != nil { + return err + } + + // end session by closing file + if db.dbfile != nil { + if err := db.dbfile.Close(); err != nil { + return err + } + db.dbfile = nil + } + + return nil +} + +func (db *regressionDB) readEntries() error { + // readEntries clobbers the contents of db.entries + db.entries = make(map[string]regressionEntry, len(db.entries)) + + // treat the file as a CSV file + csvr := csv.NewReader(db.dbfile) + csvr.Comment = rune('#') + csvr.TrimLeadingSpace = true + csvr.ReuseRecord = true + csvr.FieldsPerRecord = 5 + + db.dbfile.Seek(0, os.SEEK_SET) + + for { + // loop through file until EOF is reached + rec, err := csvr.Read() + if err == io.EOF { + break + } + if err != nil { + return err + } + + numOfFrames, err := strconv.Atoi(rec[3]) + if err != nil { + return err + } + + // add entry to database + entry := regressionEntry{ + key: rec[0], + cartridgeFile: rec[1], + tvMode: rec[2], + numOFrames: numOfFrames, + digest: rec[4]} + + db.entries[entry.key] = entry + } + + return nil +} + +func startSession() (*regressionDB, error) { + var err error + + db := ®ressionDB{} + + db.dbfile, err = os.OpenFile(regressionDBFile, os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + return nil, err + } + + err = db.readEntries() + if err != nil { + return nil, err + } + + return db, nil +} + +func run(cartridgeFile string, tvMode string, numOfFrames int) (string, error) { + tv, err := digesttv.NewDigestTV(tvMode) + if err != nil { + return "", fmt.Errorf("error preparing television: %s", err) + } + + vcs, err := hardware.NewVCS(tv) + if err != nil { + return "", fmt.Errorf("error preparing VCS: %s", err) + } + + err = vcs.AttachCartridge(cartridgeFile) + if err != nil { + return "", err + } + + err = vcs.RunFrames(numOfFrames) + if err != nil { + return "", err + } + + // output current digest + return fmt.Sprintf("%s", tv), nil +} + +// RegressAddCartridge adds a cartridge to the regression db +func addCartridge(cartridgeFile string, tvMode string, numOfFrames int, allowUpdate bool) error { + db, err := startSession() + if err != nil { + return err + } + defer db.endSession() + + // run cartdrige and get digest + digest, err := run(cartridgeFile, tvMode, numOfFrames) + if err != nil { + return err + } + + // add new entry to database + key, err := keyify(cartridgeFile) + if err != nil { + return err + } + entry := regressionEntry{ + key: key, + cartridgeFile: cartridgeFile, + tvMode: tvMode, + numOFrames: numOfFrames, + digest: digest} + + if allowUpdate == false { + if existEntry, ok := db.entries[entry.key]; ok { + if existEntry.cartridgeFile == entry.cartridgeFile { + return errors.NewGopherError(errors.RegressionEntryExists, entry) + } + + return errors.NewGopherError(errors.RegressionEntryCollision, path.Base(entry.cartridgeFile), path.Base(existEntry.cartridgeFile)) + } + } + + db.entries[entry.key] = entry + + return nil +} + +// RegressDeleteCartridge removes a cartridge from the regression db +func RegressDeleteCartridge(cartridgeFile string) error { + db, err := startSession() + if err != nil { + return err + } + defer db.endSession() + + key, err := keyify(cartridgeFile) + if err != nil { + return err + } + + if _, ok := db.entries[key]; ok == false { + return errors.NewGopherError(errors.RegressionEntryDoesNotExist, path.Base(cartridgeFile)) + } + + delete(db.entries, key) + + return nil +} + +// RegressAddCartridge adds a cartridge to the regression db +func RegressAddCartridge(cartridgeFile string, tvMode string, numOfFrames int) error { + return addCartridge(cartridgeFile, tvMode, numOfFrames, false) +} + +// RegressUpdateCartridge updates a entry (or adds it if it doesn't exist) +func RegressUpdateCartridge(cartridgeFile string, tvMode string, numOfFrames int) error { + return addCartridge(cartridgeFile, tvMode, numOfFrames, true) +} + +// RegressRunTests runs the +func RegressRunTests(output io.Writer, failOnError bool) (int, int, error) { + db, err := startSession() + if err != nil { + return -1, -1, err + } + defer db.endSession() + + numSucceed := 0 + numFail := 0 + for _, entry := range db.entries { + digest, err := run(entry.cartridgeFile, entry.tvMode, entry.numOFrames) + + if err != nil || entry.digest != digest { + if err == nil { + err = errors.NewGopherError(errors.RegressionEntryFail, entry) + } + + numFail++ + if failOnError { + return numSucceed, numFail, err + } + if output != nil { + output.Write([]byte(fmt.Sprintf("fail: %s\n", err))) + } + + } else { + numSucceed++ + if output != nil { + output.Write([]byte(fmt.Sprintf("succeed: %s\n", entry))) + } + } + } + + return numSucceed, numFail, nil +}