o gopher2600

- tidy up of command line parsing

o regression testing
    - added regression package
    - implemented simple database to facilitate regression tests
    - uses DigestTV

debugger/debugger.go

@@ -55,6 +55,8 @@ type Debugger struct {
 
 	// single-fire step traps. these are used for the STEP command, allowing
 	// things like "STEP FRAME".
+	// -- note that the hardware.VCS type has the StepFrames() function, we're
+	// not using that here because this solution is more general and flexible
 	stepTraps *traps
 
 	// commandOnHalt says whether an sequence of commands should run automatically

errors/categories.go

@@ -12,6 +12,12 @@ const (
 	ScriptFileError
 	InvalidTarget
 
+	// Regression
+	RegressionEntryExists
+	RegressionEntryCollision
+	RegressionEntryDoesNotExist
+	RegressionEntryFail
+
 	// CPU
 	UnimplementedInstruction
 	NullInstruction

errors/messages.go

@@ -10,6 +10,12 @@ var messages = map[Errno]string{
 	ScriptFileCannotOpen:  "cannot open script file (%s)",
 	InvalidTarget:         "invalid target (%s)",
 
+	// Regression
+	RegressionEntryExists:       "entry exists (%s)",
+	RegressionEntryCollision:    "ROM hash collision (%s AND %s)",
+	RegressionEntryDoesNotExist: "entry missing (%s)",
+	RegressionEntryFail:         "screen digest mismatch (%s)",
+
 	// CPU
 	UnimplementedInstruction:       "unimplemented instruction (%0#x) at (%#04x)",
 	NullInstruction:                "unimplemented instruction (0xff)",

gopher2600.go

@@ -9,10 +9,12 @@ import (
 	"gopher2600/disassembly"
 	"gopher2600/errors"
 	"gopher2600/hardware"
+	"gopher2600/regression"
 	"gopher2600/television"
-	"gopher2600/television/digesttv"
 	"gopher2600/television/sdltv"
+	"io"
 	"os"
+	"path"
 	"runtime"
 	"runtime/pprof"
 	"strings"
@@ -23,142 +25,236 @@ import (
 const initScript = ".gopher2600/debuggerInit"
 
 func main() {
-	mode := flag.String("mode", "DEBUG", "emulation mode: DEBUG, DISASM, RUN, PLAY, FPS, TVFPS, IMAGEGEN, REGRESS")
-	termType := flag.String("term", "COLOR", "terminal type to use in debug mode: COLOR, PLAIN")
-	flag.Parse()
+	progName := path.Base(os.Args[0])
 
-	cartridgeFile := ""
-	if len(flag.Args()) == 1 {
-		cartridgeFile = flag.Args()[0]
-	} else if len(flag.Args()) > 1 {
-		fmt.Println("* too many arguments")
-		os.Exit(10)
+	progFlags := flag.NewFlagSet(progName, flag.ExitOnError)
+	progFlags.Parse(os.Args[1:])
+
+	if len(progFlags.Args()) == 0 {
+		fmt.Println("* mode or cartridge required")
+		os.Exit(2)
 	}
 
-	switch strings.ToUpper(*mode) {
+	mode := strings.ToUpper(progFlags.Arg(0))
+	modeArgPos := 1
+	modeFlags := flag.NewFlagSet(fmt.Sprintf("%s %s", progName, mode), flag.ExitOnError)
+	modeFlagsParse := func() {
+		if len(progFlags.Args()) >= modeArgPos {
+			modeFlags.Parse(progFlags.Args()[modeArgPos:])
+		}
+	}
+
+	switch mode {
+	default:
+		// RUN is the default mode
+		modeArgPos = 0
+		fallthrough
+
+	case "RUN":
+		tvMode := modeFlags.String("tv", "NTSC", "television specification: NTSC, PAL")
+		scaling := modeFlags.Float64("scale", 3.0, "television scaling")
+		modeFlagsParse()
+
+		switch len(modeFlags.Args()) {
+		case 0:
+			fmt.Println("* 2600 cartridge required")
+			os.Exit(2)
+		case 1:
+			err := run(modeFlags.Arg(0), *tvMode, float32(*scaling))
+			if err != nil {
+				fmt.Printf("* error running emulator: %s\n", err)
+				os.Exit(2)
+			}
+		default:
+			fmt.Printf("* too many arguments for %s mode\n", mode)
+			os.Exit(2)
+		}
+
 	case "DEBUG":
+		termType := modeFlags.String("term", "COLOR", "terminal type to use in debug mode: COLOR, PLAIN")
+		modeFlagsParse()
+
 		dbg, err := debugger.NewDebugger()
 		if err != nil {
 			fmt.Printf("* error starting debugger: %s\n", err)
-			os.Exit(10)
+			os.Exit(2)
 		}
 
 		// start debugger with choice of interface and cartridge
 		var term ui.UserInterface
 
 		switch strings.ToUpper(*termType) {
-		case "COLOR":
-			term = new(colorterm.ColorTerminal)
 		default:
 			fmt.Printf("! unknown terminal type (%s) defaulting to plain\n", *termType)
 			fallthrough
 		case "PLAIN":
 			term = nil
+		case "COLOR":
+			term = new(colorterm.ColorTerminal)
 		}
 
-		err = dbg.Start(term, cartridgeFile, initScript)
-		if err != nil {
-			fmt.Printf("* error running debugger: %s\n", err)
-			os.Exit(10)
+		switch len(modeFlags.Args()) {
+		case 0:
+			// it's okay if DEBUG mode is started with no cartridges
+			fallthrough
+		case 1:
+			err := dbg.Start(term, modeFlags.Arg(0), initScript)
+			if err != nil {
+				fmt.Printf("* error running debugger: %s\n", err)
+				os.Exit(2)
+			}
+		default:
+			fmt.Printf("* too many arguments for %s mode\n", mode)
+			os.Exit(2)
 		}
 
 	case "DISASM":
-		dsm, err := disassembly.NewDisassembly(cartridgeFile)
-		if err != nil {
-			switch err.(type) {
-			case errors.GopherError:
-				// print what disassembly output we do have
-				if dsm != nil {
-					dsm.Dump(os.Stdout)
+		modeFlagsParse()
+
+		switch len(modeFlags.Args()) {
+		case 0:
+			fmt.Println("* 2600 cartridge required")
+			os.Exit(2)
+		case 1:
+			dsm, err := disassembly.NewDisassembly(modeFlags.Arg(0))
+			if err != nil {
+				switch err.(type) {
+				case errors.GopherError:
+					// print what disassembly output we do have
+					if dsm != nil {
+						dsm.Dump(os.Stdout)
+					}
 				}
+				fmt.Printf("* error during disassembly: %s\n", err)
+				os.Exit(2)
 			}
-			fmt.Printf("* error during disassembly: %s\n", err)
-			os.Exit(10)
+			dsm.Dump(os.Stdout)
+		default:
+			fmt.Printf("* too many arguments for %s mode\n", mode)
+			os.Exit(2)
 		}
-		dsm.Dump(os.Stdout)
 
 	case "FPS":
-		err := fps(cartridgeFile, true)
-		if err != nil {
-			fmt.Printf("* error starting FPS profiler: %s\n", err)
-			os.Exit(10)
-		}
+		display := modeFlags.Bool("display", false, "display TV output: boolean")
+		tvMode := modeFlags.String("tv", "NTSC", "television specification: NTSC, PAL")
+		scaling := modeFlags.Float64("scale", 3.0, "television scaling")
+		frames := modeFlags.Int("frames", 100, "number of frames to run")
+		modeFlagsParse()
 
-	case "TVFPS":
-		err := fps(cartridgeFile, false)
-		if err != nil {
-			fmt.Printf("* error starting TVFPS profiler: %s\n", err)
-			os.Exit(10)
+		switch len(modeFlags.Args()) {
+		case 0:
+			fmt.Println("* 2600 cartridge required")
+			os.Exit(2)
+		case 1:
+			err := fps(modeFlags.Arg(0), *display, *tvMode, float32(*scaling), *frames)
+			if err != nil {
+				fmt.Printf("* error starting fps profiler: %s\n", err)
+				os.Exit(2)
+			}
+		default:
+			fmt.Printf("* too many arguments for %s mode\n", mode)
+			os.Exit(2)
 		}
 
 	case "REGRESS":
-		err := regress(cartridgeFile, 3)
-		if err != nil {
-			fmt.Printf("* error running TEST: %s\n", err)
-			os.Exit(10)
-		}
+		subMode := strings.ToUpper(progFlags.Arg(1))
+		modeArgPos++
+		switch subMode {
+		default:
+			modeArgPos-- // undo modeArgPos adjustment
+			fallthrough
+		case "RUN":
+			verbose := modeFlags.Bool("verbose", false, "display details of each test")
+			failOnError := modeFlags.Bool("fail", false, "fail on error: boolean")
+			modeFlagsParse()
 
-	case "PLAY":
-		// PLAY is a synonym for RUN
-		fallthrough
+			var output io.Writer
+			if *verbose == true {
+				output = os.Stdout
+			}
 
-	case "RUN":
-		err := run(cartridgeFile)
-		if err != nil {
-			fmt.Printf("* error running emulator: %s\n", err)
-			os.Exit(10)
+			switch len(modeFlags.Args()) {
+			case 0:
+				succeed, fail, err := regression.RegressRunTests(output, *failOnError)
+				if err != nil {
+					fmt.Printf("* error during regression tests: %s\n", err)
+					os.Exit(2)
+				}
+				fmt.Printf("regression tests: %d succeed, %d fail\n", succeed, fail)
+			default:
+				fmt.Printf("* too many arguments for %s mode\n", mode)
+				os.Exit(2)
+			}
+
+		case "DELETE":
+			modeFlagsParse()
+
+			switch len(modeFlags.Args()) {
+			case 0:
+				fmt.Println("* 2600 cartridge required")
+				os.Exit(2)
+			case 1:
+				err := regression.RegressDeleteCartridge(modeFlags.Arg(0))
+				if err != nil {
+					fmt.Printf("* error deleting regression entry: %s\n", err)
+					os.Exit(2)
+				}
+				fmt.Printf("! deleted %s from regression database\n", path.Base(modeFlags.Arg(0)))
+			default:
+				fmt.Printf("* too many arguments for %s mode\n", mode)
+				os.Exit(2)
+			}
+
+		case "ADD":
+			tvMode := modeFlags.String("tv", "NTSC", "television specification: NTSC, PAL")
+			numFrames := modeFlags.Int("frames", 10, "number of frames to run")
+			modeFlagsParse()
+
+			switch len(modeFlags.Args()) {
+			case 0:
+				fmt.Println("* 2600 cartridge required")
+				os.Exit(2)
+			case 1:
+				err := regression.RegressAddCartridge(modeFlags.Arg(0), *tvMode, *numFrames)
+				if err != nil {
+					fmt.Printf("* error adding regression test: %s\n", err)
+					os.Exit(2)
+				}
+				fmt.Printf("! added %s to regression database\n", path.Base(modeFlags.Arg(0)))
+			default:
+				fmt.Printf("* too many arguments for %s mode\n", mode)
+				os.Exit(2)
+			}
+		case "UPDATE":
+			tvMode := modeFlags.String("tv", "NTSC", "television specification: NTSC, PAL")
+			numFrames := modeFlags.Int("frames", 10, "number of frames to run")
+			modeFlagsParse()
+
+			switch len(modeFlags.Args()) {
+			case 0:
+				fmt.Println("* 2600 cartridge required")
+				os.Exit(2)
+			case 1:
+				err := regression.RegressUpdateCartridge(modeFlags.Arg(0), *tvMode, *numFrames)
+				if err != nil {
+					fmt.Printf("* error updating regression test: %s\n", err)
+					os.Exit(2)
+				}
+				fmt.Printf("! updated %s in regression database\n", path.Base(modeFlags.Arg(0)))
+			default:
+				fmt.Printf("* too many arguments for %s mode\n", mode)
+				os.Exit(2)
+			}
 		}
-	default:
-		fmt.Printf("* unknown mode: %s\n", strings.ToUpper(*mode))
-		os.Exit(10)
 	}
 }
 
-func regress(cartridgeFile string, numOfFrames int) error {
-	tv, err := digesttv.NewDigestTV("NTSC")
-	if err != nil {
-		return fmt.Errorf("error preparing television: %s", err)
-	}
-
-	vcs, err := hardware.NewVCS(tv)
-	if err != nil {
-		return fmt.Errorf("error preparing VCS: %s", err)
-	}
-
-	err = vcs.AttachCartridge(cartridgeFile)
-	if err != nil {
-		return err
-	}
-
-	const cyclesPerFrame = 19912
-
-	// run emulation for a while
-	cycles := cyclesPerFrame * numOfFrames
-	for cycles > 0 {
-		stepCycles, _, err := vcs.Step(hardware.StubVideoCycleCallback)
-		if err != nil {
-			return err
-		}
-		cycles -= stepCycles
-	}
-
-	// output current digest
-	fmt.Println(tv)
-
-	return nil
-}
-
-func fps(cartridgeFile string, justTheVCS bool) error {
+func fps(cartridgeFile string, display bool, tvMode string, scaling float32, numOfFrames int) error {
 	var tv television.Television
 	var err error
 
-	if justTheVCS {
-		tv = new(television.DummyTV)
-		if tv == nil {
-			return fmt.Errorf("error preparing television")
-		}
-	} else {
-		tv, err = sdltv.NewSDLTV("NTSC", 3.0)
+	if display {
+		tv, err = sdltv.NewSDLTV(tvMode, scaling)
 		if err != nil {
 			return fmt.Errorf("error preparing television: %s", err)
 		}
@@ -167,6 +263,11 @@ func fps(cartridgeFile string, justTheVCS bool) error {
 		if err != nil {
 			return fmt.Errorf("error preparing television: %s", err)
 		}
+	} else {
+		tv, err = television.NewHeadlessTV("NTSC")
+		if err != nil {
+			return fmt.Errorf("error preparing television: %s", err)
+		}
 	}
 
 	vcs, err := hardware.NewVCS(tv)
@@ -179,9 +280,6 @@ func fps(cartridgeFile string, justTheVCS bool) error {
 		return err
 	}
 
-	const cyclesPerFrame = 19912
-	const numOfFrames = 180
-
 	// start cpu profile
 	f, err := os.Create("cpu.profile")
 	if err != nil {
@@ -193,19 +291,16 @@ func fps(cartridgeFile string, justTheVCS bool) error {
 	}
 	defer pprof.StopCPUProfile()
 
-	// run emulation for a while
-	cycles := cyclesPerFrame * numOfFrames
+	// run for numOfFrames and calculate frames-per-second
 	startTime := time.Now()
-	for cycles > 0 {
-		stepCycles, _, err := vcs.Step(hardware.StubVideoCycleCallback)
-		if err != nil {
-			return err
-		}
-		cycles -= stepCycles
+	err = vcs.RunFrames(numOfFrames)
+	if err != nil {
+		return err
 	}
+	fps := float64(numOfFrames) / time.Since(startTime).Seconds()
 
 	// display estimated fps
-	fmt.Printf("%f fps\n", float64(numOfFrames)/time.Since(startTime).Seconds())
+	fmt.Printf("%f fps\n", fps)
 
 	// write memory profile
 	f, err = os.Create("mem.profile")
@@ -222,8 +317,8 @@ func fps(cartridgeFile string, justTheVCS bool) error {
 	return nil
 }
 
-func run(cartridgeFile string) error {
-	tv, err := sdltv.NewSDLTV("NTSC", 3.0)
+func run(cartridgeFile, tvMode string, scaling float32) error {
+	tv, err := sdltv.NewSDLTV(tvMode, scaling)
 	if err != nil {
 		return fmt.Errorf("error preparing television: %s", err)
 	}

hardware/cpu/definitions/csv/instructions_gen.go

@@ -67,7 +67,7 @@ func parseCSV() (map[uint8]definitions.InstructionDefinition, error) {
 			return nil, fmt.Errorf("wrong number of fields in instruction definition (%s)", rec)
 		}
 
-		// trim trailing comment from last record
+		// trim trailing comment from record
 		rec[len(rec)-1] = strings.Split(rec[len(rec)-1], "#")[0]
 
 		// manually trim trailing space from all fields in the record

hardware/vcs.go

@@ -216,3 +216,25 @@ func (vcs *VCS) Reset() error {
 
 	return nil
 }
+
+// RunFrames sets emulator running for the specified number of frames
+// - not used by the debugger because traps and steptraps are more flexible
+// - useful for fps and regression tests
+func (vcs *VCS) RunFrames(numFrames int) error {
+	frm, err := vcs.TV.RequestTVState(television.ReqFramenum)
+	if err != nil {
+		return err
+	}
+
+	targetFrame := frm.Value().(int) + numFrames
+
+	for frm.Value().(int) != targetFrame {
+		_, _, err = vcs.Step(func(*result.Instruction) error { return nil })
+		frm, err = vcs.TV.RequestTVState(television.ReqFramenum)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

regression/regression.go

@@ -0,0 +1,286 @@
+package regression
+
+import (
+	"crypto/sha1"
+	"encoding/csv"
+	"fmt"
+	"gopher2600/errors"
+	"gopher2600/hardware"
+	"gopher2600/television/digesttv"
+	"io"
+	"os"
+	"path"
+	"strconv"
+)
+
+const regressionDBFile = ".gopher2600/regressionDB"
+
+func keyify(cartridgeFile string) (string, error) {
+	f, err := os.Open(cartridgeFile)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	key := sha1.New()
+	if _, err := io.Copy(key, f); err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%x", key.Sum(nil)), nil
+}
+
+type regressionEntry struct {
+	key           string
+	cartridgeFile string
+	tvMode        string
+	numOFrames    int
+	digest        string
+}
+
+func (entry regressionEntry) String() string {
+	return fmt.Sprintf("%s [%s] frames=%d", path.Base(entry.cartridgeFile), entry.tvMode, entry.numOFrames)
+}
+
+type regressionDB struct {
+	dbfile  *os.File
+	entries map[string]regressionEntry
+}
+
+func (db *regressionDB) endSession() error {
+	// write entries to regression database
+	csvw := csv.NewWriter(db.dbfile)
+
+	err := db.dbfile.Truncate(0)
+	if err != nil {
+		return err
+	}
+
+	db.dbfile.Seek(0, os.SEEK_SET)
+
+	for _, entry := range db.entries {
+		rec := make([]string, 5)
+		rec[0] = entry.key
+		rec[1] = entry.cartridgeFile
+		rec[2] = entry.tvMode
+		rec[3] = strconv.Itoa(entry.numOFrames)
+		rec[4] = entry.digest
+
+		err := csvw.Write(rec)
+		if err != nil {
+			return err
+		}
+	}
+
+	// make sure everything's been written
+	csvw.Flush()
+	err = csvw.Error()
+	if err != nil {
+		return err
+	}
+
+	// end session by closing file
+	if db.dbfile != nil {
+		if err := db.dbfile.Close(); err != nil {
+			return err
+		}
+		db.dbfile = nil
+	}
+
+	return nil
+}
+
+func (db *regressionDB) readEntries() error {
+	// readEntries clobbers the contents of db.entries
+	db.entries = make(map[string]regressionEntry, len(db.entries))
+
+	// treat the file as a CSV file
+	csvr := csv.NewReader(db.dbfile)
+	csvr.Comment = rune('#')
+	csvr.TrimLeadingSpace = true
+	csvr.ReuseRecord = true
+	csvr.FieldsPerRecord = 5
+
+	db.dbfile.Seek(0, os.SEEK_SET)
+
+	for {
+		// loop through file until EOF is reached
+		rec, err := csvr.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		numOfFrames, err := strconv.Atoi(rec[3])
+		if err != nil {
+			return err
+		}
+
+		// add entry to database
+		entry := regressionEntry{
+			key:           rec[0],
+			cartridgeFile: rec[1],
+			tvMode:        rec[2],
+			numOFrames:    numOfFrames,
+			digest:        rec[4]}
+
+		db.entries[entry.key] = entry
+	}
+
+	return nil
+}
+
+func startSession() (*regressionDB, error) {
+	var err error
+
+	db := &regressionDB{}
+
+	db.dbfile, err = os.OpenFile(regressionDBFile, os.O_RDWR|os.O_CREATE, 0600)
+	if err != nil {
+		return nil, err
+	}
+
+	err = db.readEntries()
+	if err != nil {
+		return nil, err
+	}
+
+	return db, nil
+}
+
+func run(cartridgeFile string, tvMode string, numOfFrames int) (string, error) {
+	tv, err := digesttv.NewDigestTV(tvMode)
+	if err != nil {
+		return "", fmt.Errorf("error preparing television: %s", err)
+	}
+
+	vcs, err := hardware.NewVCS(tv)
+	if err != nil {
+		return "", fmt.Errorf("error preparing VCS: %s", err)
+	}
+
+	err = vcs.AttachCartridge(cartridgeFile)
+	if err != nil {
+		return "", err
+	}
+
+	err = vcs.RunFrames(numOfFrames)
+	if err != nil {
+		return "", err
+	}
+
+	// output current digest
+	return fmt.Sprintf("%s", tv), nil
+}
+
+// RegressAddCartridge adds a cartridge to the regression db
+func addCartridge(cartridgeFile string, tvMode string, numOfFrames int, allowUpdate bool) error {
+	db, err := startSession()
+	if err != nil {
+		return err
+	}
+	defer db.endSession()
+
+	// run cartdrige and get digest
+	digest, err := run(cartridgeFile, tvMode, numOfFrames)
+	if err != nil {
+		return err
+	}
+
+	// add new entry to database
+	key, err := keyify(cartridgeFile)
+	if err != nil {
+		return err
+	}
+	entry := regressionEntry{
+		key:           key,
+		cartridgeFile: cartridgeFile,
+		tvMode:        tvMode,
+		numOFrames:    numOfFrames,
+		digest:        digest}
+
+	if allowUpdate == false {
+		if existEntry, ok := db.entries[entry.key]; ok {
+			if existEntry.cartridgeFile == entry.cartridgeFile {
+				return errors.NewGopherError(errors.RegressionEntryExists, entry)
+			}
+
+			return errors.NewGopherError(errors.RegressionEntryCollision, path.Base(entry.cartridgeFile), path.Base(existEntry.cartridgeFile))
+		}
+	}
+
+	db.entries[entry.key] = entry
+
+	return nil
+}
+
+// RegressDeleteCartridge removes a cartridge from the regression db
+func RegressDeleteCartridge(cartridgeFile string) error {
+	db, err := startSession()
+	if err != nil {
+		return err
+	}
+	defer db.endSession()
+
+	key, err := keyify(cartridgeFile)
+	if err != nil {
+		return err
+	}
+
+	if _, ok := db.entries[key]; ok == false {
+		return errors.NewGopherError(errors.RegressionEntryDoesNotExist, path.Base(cartridgeFile))
+	}
+
+	delete(db.entries, key)
+
+	return nil
+}
+
+// RegressAddCartridge adds a cartridge to the regression db
+func RegressAddCartridge(cartridgeFile string, tvMode string, numOfFrames int) error {
+	return addCartridge(cartridgeFile, tvMode, numOfFrames, false)
+}
+
+// RegressUpdateCartridge updates a entry (or adds it if it doesn't exist)
+func RegressUpdateCartridge(cartridgeFile string, tvMode string, numOfFrames int) error {
+	return addCartridge(cartridgeFile, tvMode, numOfFrames, true)
+}
+
+// RegressRunTests runs the
+func RegressRunTests(output io.Writer, failOnError bool) (int, int, error) {
+	db, err := startSession()
+	if err != nil {
+		return -1, -1, err
+	}
+	defer db.endSession()
+
+	numSucceed := 0
+	numFail := 0
+	for _, entry := range db.entries {
+		digest, err := run(entry.cartridgeFile, entry.tvMode, entry.numOFrames)
+
+		if err != nil || entry.digest != digest {
+			if err == nil {
+				err = errors.NewGopherError(errors.RegressionEntryFail, entry)
+			}
+
+			numFail++
+			if failOnError {
+				return numSucceed, numFail, err
+			}
+			if output != nil {
+				output.Write([]byte(fmt.Sprintf("fail: %s\n", err)))
+			}
+
+		} else {
+			numSucceed++
+			if output != nil {
+				output.Write([]byte(fmt.Sprintf("succeed: %s\n", entry)))
+			}
+		}
+	}
+
+	return numSucceed, numFail, nil
+}