New experimental simulation runner

.circleci/config.yml

@@ -196,7 +196,8 @@ jobs:
           command: |
             export PATH="$GOBIN:$PATH"
             export GO111MODULE=on
-            cmd/gaia/contrib/sim/multisim.sh 500 50 TestFullGaiaSimulation
+            make runsim
+            runsim 500 50 TestFullGaiaSimulation
 
   test_sim_gaia_multi_seed:
     <<: *linux_defaults
@@ -214,7 +215,8 @@ jobs:
           command: |
             export PATH="$GOBIN:$PATH"
             export GO111MODULE=on
-            cmd/gaia/contrib/sim/multisim.sh 50 10 TestFullGaiaSimulation
+            make runsim
+            runsim 50 10 TestFullGaiaSimulation
 
   test_cover:
     <<: *linux_defaults

Makefile

@@ -165,29 +165,34 @@ test_sim_gaia_fast:
 	@echo "Running quick Gaia simulation. This may take several minutes..."
 	@go test -mod=readonly ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=100 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -SimulationPeriod=5 -v -timeout 24h
 
-test_sim_gaia_import_export:
+test_sim_gaia_import_export: runsim
 	@echo "Running Gaia import/export simulation. This may take several minutes..."
-	@bash cmd/gaia/contrib/sim/multisim.sh 50 5 TestGaiaImportExport
+	$(GOBIN)/runsim 50 5 TestGaiaImportExport
 
-test_sim_gaia_simulation_after_import:
+test_sim_gaia_simulation_after_import: runsim
 	@echo "Running Gaia simulation-after-import. This may take several minutes..."
-	@bash cmd/gaia/contrib/sim/multisim.sh 50 5 TestGaiaSimulationAfterImport
+	$(GOBIN)/runsim 50 5 TestGaiaSimulationAfterImport
 
-test_sim_gaia_custom_genesis_multi_seed:
+test_sim_gaia_custom_genesis_multi_seed: runsim
 	@echo "Running multi-seed custom genesis simulation..."
 	@echo "By default, ${HOME}/.gaiad/config/genesis.json will be used."
-	@bash cmd/gaia/contrib/sim/multisim.sh 400 5 TestFullGaiaSimulation ${HOME}/.gaiad/config/genesis.json
+	$(GOBIN)/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation
 
-test_sim_gaia_multi_seed:
+test_sim_gaia_multi_seed: runsim
 	@echo "Running multi-seed Gaia simulation. This may take awhile!"
-	@bash cmd/gaia/contrib/sim/multisim.sh 400 5 TestFullGaiaSimulation
+	$(GOBIN)/runsim 400 5 TestFullGaiaSimulation
 
 test_sim_benchmark_invariants:
 	@echo "Running simulation invariant benchmarks..."
 	@go test -mod=readonly ./cmd/gaia/app -benchmem -bench=BenchmarkInvariants -run=^$ \
 	-SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationBlockSize=200 \
 	-SimulationCommit=true -SimulationSeed=57 -v -timeout 24h
 
+# Don't move it into tools - this will be gone once gaia has moved into the new repo
+runsim: $(GOBIN)/runsim
+$(GOBIN)/runsim: cmd/gaia/contrib/runsim/main.go
+	go install github.com/cosmos/cosmos-sdk/cmd/gaia/contrib/runsim
+
 SIM_NUM_BLOCKS ?= 500
 SIM_BLOCK_SIZE ?= 200
 SIM_COMMIT ?= true

cmd/gaia/contrib/runsim/main.go

@@ -0,0 +1,228 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+)
+
+var (
+	// default seeds
+	seeds = []int{
+		1, 2, 4, 7, 32, 123, 124, 582, 1893, 2989,
+		3012, 4728, 37827, 981928, 87821, 891823782,
+		989182, 89182391, 11, 22, 44, 77, 99, 2020,
+		3232, 123123, 124124, 582582, 18931893,
+		29892989, 30123012, 47284728, 37827,
+	}
+
+	// goroutine-safe process map
+	procs map[int]*os.Process
+	mutex *sync.Mutex
+
+	// results channel
+	results chan bool
+
+	// command line arguments and options
+	jobs     int
+	blocks   string
+	period   string
+	testname string
+	genesis  string
+
+	// logs temporary directory
+	tempdir string
+)
+
+func init() {
+	log.SetPrefix("")
+	log.SetFlags(0)
+
+	procs = map[int]*os.Process{}
+	mutex = &sync.Mutex{}
+	flag.IntVar(&jobs, "j", 10, "Number of parallel processes")
+	flag.StringVar(&genesis, "g", "", "Genesis file")
+
+	flag.Usage = func() {
+		fmt.Fprintf(flag.CommandLine.Output(),
+			`Usage: %s [-j maxprocs] [-g genesis.json] [blocks] [period] [testname]
+Run simulations in parallel
+
+`, filepath.Base(os.Args[0]))
+		flag.PrintDefaults()
+	}
+}
+
+func main() {
+	var err error
+
+	flag.Parse()
+	if flag.NArg() != 3 {
+		log.Fatal("wrong number of arguments")
+	}
+
+	// prepare input channel
+	queue := make(chan int, len(seeds))
+	for _, seed := range seeds {
+		queue <- seed
+	}
+	close(queue)
+
+	// jobs cannot be > len(seeds)
+	if jobs > len(seeds) {
+		jobs = len(seeds)
+	}
+	results = make(chan bool, len(seeds))
+
+	// setup signal handling
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+
+	go func() {
+		_ = <-sigs
+		fmt.Println()
+
+		// drain the queue
+		log.Printf("Draining seeds queue...")
+		for seed := range queue {
+			log.Printf("%d", seed)
+		}
+		log.Printf("Kill all remaining processes...")
+		killAllProcs()
+		os.Exit(1)
+	}()
+
+	// initialise common test parameters
+	blocks = flag.Arg(0)
+	period = flag.Arg(1)
+	testname = flag.Arg(2)
+	tempdir, err = ioutil.TempDir("", "")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// set up worker pool
+	wg := sync.WaitGroup{}
+	for workerId := 0; workerId < jobs; workerId++ {
+		wg.Add(1)
+
+		go func(workerId int) {
+			defer wg.Done()
+			worker(workerId, queue)
+		}(workerId)
+	}
+
+	// idiomatic hack required to use wg.Wait() with select
+	waitCh := make(chan struct{})
+	go func() {
+		defer close(waitCh)
+		wg.Wait()
+	}()
+
+wait:
+	for {
+		select {
+		case <-waitCh:
+			break wait
+		case <-time.After(1 * time.Minute):
+			fmt.Println(".")
+		}
+	}
+
+	// analyse results and exit with 1 on first error
+	close(results)
+	for rc := range results {
+		if !rc {
+			os.Exit(1)
+		}
+	}
+
+	os.Exit(0)
+}
+
+func buildCommand(testname, blocks, period, genesis string, seed int) string {
+	return fmt.Sprintf("go test github.com/cosmos/cosmos-sdk/cmd/gaia/app -run %s -SimulationEnabled=true "+
+		"-SimulationNumBlocks=%s -SimulationGenesis=%s "+
+		"-SimulationVerbose=true -SimulationCommit=true -SimulationSeed=%d -SimulationPeriod=%s -v -timeout 24h",
+		testname, blocks, genesis, seed, period)
+}
+
+func makeCmd(cmdStr string) *exec.Cmd {
+	cmdSlice := strings.Split(cmdStr, " ")
+	return exec.Command(cmdSlice[0], cmdSlice[1:]...)
+}
+
+func makeFilename(seed int) string {
+	return fmt.Sprintf("gaia-simulation-seed-%d-date-%s", seed, time.Now().Format("01-02-2006_15:04:05.000000000"))
+}
+
+func worker(id int, seeds <-chan int) {
+	log.Printf("[W%d] Worker is up and running", id)
+	for seed := range seeds {
+		if err := spawnProc(id, seed); err != nil {
+			results <- false
+			log.Printf("[W%d] Seed %d: FAILED", id, seed)
+			log.Printf("To reproduce run: %s", buildCommand(testname, blocks, period, genesis, seed))
+		} else {
+			log.Printf("[W%d] Seed %d: OK", id, seed)
+		}
+	}
+	log.Printf("[W%d] no seeds left, shutting down", id)
+}
+
+func spawnProc(workerId int, seed int) error {
+	stderrFile, _ := os.Create(filepath.Join(tempdir, makeFilename(seed)+".stderr"))
+	stdoutFile, _ := os.Create(filepath.Join(tempdir, makeFilename(seed)+".stdout"))
+	s := buildCommand(testname, blocks, period, genesis, seed)
+	cmd := makeCmd(s)
+	cmd.Stdout = stdoutFile
+	cmd.Stderr = stderrFile
+	err := cmd.Start()
+	if err != nil {
+		log.Printf("couldn't start %q", s)
+		return err
+	}
+	log.Printf("[W%d] Spawned simulation with pid %d [seed=%d stdout=%s stderr=%s]",
+		workerId, cmd.Process.Pid, seed, stdoutFile.Name(), stderrFile.Name())
+	pushProcess(cmd.Process)
+	defer popProcess(cmd.Process)
+	return cmd.Wait()
+}
+
+func pushProcess(proc *os.Process) {
+	mutex.Lock()
+	defer mutex.Unlock()
+	procs[proc.Pid] = proc
+}
+
+func popProcess(proc *os.Process) {
+	mutex.Lock()
+	defer mutex.Unlock()
+	if _, ok := procs[proc.Pid]; ok {
+		delete(procs, proc.Pid)
+	}
+}
+
+func killAllProcs() {
+	mutex.Lock()
+	defer mutex.Unlock()
+	for _, proc := range procs {
+		checkSignal(proc, syscall.SIGTERM)
+		checkSignal(proc, syscall.SIGKILL)
+	}
+}
+
+func checkSignal(proc *os.Process, signal syscall.Signal) {
+	if err := proc.Signal(signal); err != nil {
+		log.Printf("Failed to send %s to PID %d", signal, proc.Pid)
+	}
+}

cmd/gaia/sims.mk

@@ -3,6 +3,10 @@
 ########################################
 ### Simulations
 
+runsim: $(GOBIN)/runsim
+$(GOBIN)/runsim: contrib/runsim/main.go
+	go install github.com/cosmos/cosmos-sdk/cmd/gaia/contrib/runsim
+
 sim-gaia-nondeterminism:
 	@echo "Running nondeterminism test..."
 	@go test -mod=readonly ./cmd/gaia/app -run TestAppStateDeterminism -SimulationEnabled=true -v -timeout 10m
@@ -17,22 +21,22 @@ sim-gaia-fast:
 	@echo "Running quick Gaia simulation. This may take several minutes..."
 	@go test -mod=readonly github.com/cosmos/cosmos-sdk/cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=100 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -SimulationPeriod=5 -v -timeout 24h
 
-sim-gaia-import-export:
+sim-gaia-import-export: runsim
 	@echo "Running Gaia import/export simulation. This may take several minutes..."
-	@bash contrib/sim/multisim.sh 50 5 TestGaiaImportExport
+	$(GOBIN)/runsim 50 5 TestGaiaImportExport
 
-sim-gaia-simulation-after-import:
+sim-gaia-simulation-after-import: runsim
 	@echo "Running Gaia simulation-after-import. This may take several minutes..."
-	@bash contrib/sim/multisim.sh 50 5 TestGaiaSimulationAfterImport
+	$(GOBIN)/runsim 50 5 TestGaiaSimulationAfterImport
 
-sim-gaia-custom-genesis-multi-seed:
+sim-gaia-custom-genesis-multi-seed: runsim
 	@echo "Running multi-seed custom genesis simulation..."
 	@echo "By default, ${HOME}/.gaiad/config/genesis.json will be used."
-	@bash contrib/sim/multisim.sh 400 5 TestFullGaiaSimulation ${HOME}/.gaiad/config/genesis.json
+	$(GOBIN)/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation
 
-sim-gaia-multi-seed:
+sim-gaia-multi-seed: runsim
 	@echo "Running multi-seed Gaia simulation. This may take awhile!"
-	@bash contrib/sim/multisim.sh 400 5 TestFullGaiaSimulation
+	$(GOBIN)/runsim 400 5 TestFullGaiaSimulation
 
 sim-benchmark-invariants:
 	@echo "Running simulation invariant benchmarks..."