From 4bf9d2b3ec6c22033ef2e0d63dc55432f996f567 Mon Sep 17 00:00:00 2001 From: Alessio Treglia Date: Tue, 23 Apr 2019 14:01:59 +0100 Subject: [PATCH] Merge PR #4161: New experimental simulation runner --- .circleci/config.yml | 6 +- Makefile | 21 +-- cmd/gaia/contrib/runsim/main.go | 228 ++++++++++++++++++++++++++++++++ cmd/gaia/sims.mk | 20 +-- 4 files changed, 257 insertions(+), 18 deletions(-) create mode 100644 cmd/gaia/contrib/runsim/main.go diff --git a/.circleci/config.yml b/.circleci/config.yml index 68c0f794b3fc..e4afa15f08ca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -196,7 +196,8 @@ jobs: command: | export PATH="$GOBIN:$PATH" export GO111MODULE=on - cmd/gaia/contrib/sim/multisim.sh 500 50 TestFullGaiaSimulation + make runsim + runsim 500 50 TestFullGaiaSimulation test_sim_gaia_multi_seed: <<: *linux_defaults @@ -214,7 +215,8 @@ jobs: command: | export PATH="$GOBIN:$PATH" export GO111MODULE=on - cmd/gaia/contrib/sim/multisim.sh 50 10 TestFullGaiaSimulation + make runsim + runsim 50 10 TestFullGaiaSimulation test_cover: <<: *linux_defaults diff --git a/Makefile b/Makefile index 6d5df8a78c92..ff5f742e6f69 100644 --- a/Makefile +++ b/Makefile @@ -165,22 +165,22 @@ test_sim_gaia_fast: @echo "Running quick Gaia simulation. This may take several minutes..." @go test -mod=readonly ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=100 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -SimulationPeriod=5 -v -timeout 24h -test_sim_gaia_import_export: +test_sim_gaia_import_export: runsim @echo "Running Gaia import/export simulation. This may take several minutes..." - @bash cmd/gaia/contrib/sim/multisim.sh 50 5 TestGaiaImportExport + $(GOBIN)/runsim 50 5 TestGaiaImportExport -test_sim_gaia_simulation_after_import: +test_sim_gaia_simulation_after_import: runsim @echo "Running Gaia simulation-after-import. This may take several minutes..." - @bash cmd/gaia/contrib/sim/multisim.sh 50 5 TestGaiaSimulationAfterImport + $(GOBIN)/runsim 50 5 TestGaiaSimulationAfterImport -test_sim_gaia_custom_genesis_multi_seed: +test_sim_gaia_custom_genesis_multi_seed: runsim @echo "Running multi-seed custom genesis simulation..." @echo "By default, ${HOME}/.gaiad/config/genesis.json will be used." - @bash cmd/gaia/contrib/sim/multisim.sh 400 5 TestFullGaiaSimulation ${HOME}/.gaiad/config/genesis.json + $(GOBIN)/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation -test_sim_gaia_multi_seed: +test_sim_gaia_multi_seed: runsim @echo "Running multi-seed Gaia simulation. This may take awhile!" - @bash cmd/gaia/contrib/sim/multisim.sh 400 5 TestFullGaiaSimulation + $(GOBIN)/runsim 400 5 TestFullGaiaSimulation test_sim_benchmark_invariants: @echo "Running simulation invariant benchmarks..." @@ -188,6 +188,11 @@ test_sim_benchmark_invariants: -SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationBlockSize=200 \ -SimulationCommit=true -SimulationSeed=57 -v -timeout 24h +# Don't move it into tools - this will be gone once gaia has moved into the new repo +runsim: $(GOBIN)/runsim +$(GOBIN)/runsim: cmd/gaia/contrib/runsim/main.go + go install github.com/cosmos/cosmos-sdk/cmd/gaia/contrib/runsim + SIM_NUM_BLOCKS ?= 500 SIM_BLOCK_SIZE ?= 200 SIM_COMMIT ?= true diff --git a/cmd/gaia/contrib/runsim/main.go b/cmd/gaia/contrib/runsim/main.go new file mode 100644 index 000000000000..2d13719f9a25 --- /dev/null +++ b/cmd/gaia/contrib/runsim/main.go @@ -0,0 +1,228 @@ +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strings" + "sync" + "syscall" + "time" +) + +var ( + // default seeds + seeds = []int{ + 1, 2, 4, 7, 32, 123, 124, 582, 1893, 2989, + 3012, 4728, 37827, 981928, 87821, 891823782, + 989182, 89182391, 11, 22, 44, 77, 99, 2020, + 3232, 123123, 124124, 582582, 18931893, + 29892989, 30123012, 47284728, 37827, + } + + // goroutine-safe process map + procs map[int]*os.Process + mutex *sync.Mutex + + // results channel + results chan bool + + // command line arguments and options + jobs int + blocks string + period string + testname string + genesis string + + // logs temporary directory + tempdir string +) + +func init() { + log.SetPrefix("") + log.SetFlags(0) + + procs = map[int]*os.Process{} + mutex = &sync.Mutex{} + flag.IntVar(&jobs, "j", 10, "Number of parallel processes") + flag.StringVar(&genesis, "g", "", "Genesis file") + + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), + `Usage: %s [-j maxprocs] [-g genesis.json] [blocks] [period] [testname] +Run simulations in parallel + +`, filepath.Base(os.Args[0])) + flag.PrintDefaults() + } +} + +func main() { + var err error + + flag.Parse() + if flag.NArg() != 3 { + log.Fatal("wrong number of arguments") + } + + // prepare input channel + queue := make(chan int, len(seeds)) + for _, seed := range seeds { + queue <- seed + } + close(queue) + + // jobs cannot be > len(seeds) + if jobs > len(seeds) { + jobs = len(seeds) + } + results = make(chan bool, len(seeds)) + + // setup signal handling + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + + go func() { + _ = <-sigs + fmt.Println() + + // drain the queue + log.Printf("Draining seeds queue...") + for seed := range queue { + log.Printf("%d", seed) + } + log.Printf("Kill all remaining processes...") + killAllProcs() + os.Exit(1) + }() + + // initialise common test parameters + blocks = flag.Arg(0) + period = flag.Arg(1) + testname = flag.Arg(2) + tempdir, err = ioutil.TempDir("", "") + if err != nil { + log.Fatal(err) + } + + // set up worker pool + wg := sync.WaitGroup{} + for workerId := 0; workerId < jobs; workerId++ { + wg.Add(1) + + go func(workerId int) { + defer wg.Done() + worker(workerId, queue) + }(workerId) + } + + // idiomatic hack required to use wg.Wait() with select + waitCh := make(chan struct{}) + go func() { + defer close(waitCh) + wg.Wait() + }() + +wait: + for { + select { + case <-waitCh: + break wait + case <-time.After(1 * time.Minute): + fmt.Println(".") + } + } + + // analyse results and exit with 1 on first error + close(results) + for rc := range results { + if !rc { + os.Exit(1) + } + } + + os.Exit(0) +} + +func buildCommand(testname, blocks, period, genesis string, seed int) string { + return fmt.Sprintf("go test github.com/cosmos/cosmos-sdk/cmd/gaia/app -run %s -SimulationEnabled=true "+ + "-SimulationNumBlocks=%s -SimulationGenesis=%s "+ + "-SimulationVerbose=true -SimulationCommit=true -SimulationSeed=%d -SimulationPeriod=%s -v -timeout 24h", + testname, blocks, genesis, seed, period) +} + +func makeCmd(cmdStr string) *exec.Cmd { + cmdSlice := strings.Split(cmdStr, " ") + return exec.Command(cmdSlice[0], cmdSlice[1:]...) +} + +func makeFilename(seed int) string { + return fmt.Sprintf("gaia-simulation-seed-%d-date-%s", seed, time.Now().Format("01-02-2006_15:04:05.000000000")) +} + +func worker(id int, seeds <-chan int) { + log.Printf("[W%d] Worker is up and running", id) + for seed := range seeds { + if err := spawnProc(id, seed); err != nil { + results <- false + log.Printf("[W%d] Seed %d: FAILED", id, seed) + log.Printf("To reproduce run: %s", buildCommand(testname, blocks, period, genesis, seed)) + } else { + log.Printf("[W%d] Seed %d: OK", id, seed) + } + } + log.Printf("[W%d] no seeds left, shutting down", id) +} + +func spawnProc(workerId int, seed int) error { + stderrFile, _ := os.Create(filepath.Join(tempdir, makeFilename(seed)+".stderr")) + stdoutFile, _ := os.Create(filepath.Join(tempdir, makeFilename(seed)+".stdout")) + s := buildCommand(testname, blocks, period, genesis, seed) + cmd := makeCmd(s) + cmd.Stdout = stdoutFile + cmd.Stderr = stderrFile + err := cmd.Start() + if err != nil { + log.Printf("couldn't start %q", s) + return err + } + log.Printf("[W%d] Spawned simulation with pid %d [seed=%d stdout=%s stderr=%s]", + workerId, cmd.Process.Pid, seed, stdoutFile.Name(), stderrFile.Name()) + pushProcess(cmd.Process) + defer popProcess(cmd.Process) + return cmd.Wait() +} + +func pushProcess(proc *os.Process) { + mutex.Lock() + defer mutex.Unlock() + procs[proc.Pid] = proc +} + +func popProcess(proc *os.Process) { + mutex.Lock() + defer mutex.Unlock() + if _, ok := procs[proc.Pid]; ok { + delete(procs, proc.Pid) + } +} + +func killAllProcs() { + mutex.Lock() + defer mutex.Unlock() + for _, proc := range procs { + checkSignal(proc, syscall.SIGTERM) + checkSignal(proc, syscall.SIGKILL) + } +} + +func checkSignal(proc *os.Process, signal syscall.Signal) { + if err := proc.Signal(signal); err != nil { + log.Printf("Failed to send %s to PID %d", signal, proc.Pid) + } +} diff --git a/cmd/gaia/sims.mk b/cmd/gaia/sims.mk index 0e731299e575..c6776594fc8e 100644 --- a/cmd/gaia/sims.mk +++ b/cmd/gaia/sims.mk @@ -3,6 +3,10 @@ ######################################## ### Simulations +runsim: $(GOBIN)/runsim +$(GOBIN)/runsim: contrib/runsim/main.go + go install github.com/cosmos/cosmos-sdk/cmd/gaia/contrib/runsim + sim-gaia-nondeterminism: @echo "Running nondeterminism test..." @go test -mod=readonly ./cmd/gaia/app -run TestAppStateDeterminism -SimulationEnabled=true -v -timeout 10m @@ -17,22 +21,22 @@ sim-gaia-fast: @echo "Running quick Gaia simulation. This may take several minutes..." @go test -mod=readonly github.com/cosmos/cosmos-sdk/cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=100 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=99 -SimulationPeriod=5 -v -timeout 24h -sim-gaia-import-export: +sim-gaia-import-export: runsim @echo "Running Gaia import/export simulation. This may take several minutes..." - @bash contrib/sim/multisim.sh 50 5 TestGaiaImportExport + $(GOBIN)/runsim 50 5 TestGaiaImportExport -sim-gaia-simulation-after-import: +sim-gaia-simulation-after-import: runsim @echo "Running Gaia simulation-after-import. This may take several minutes..." - @bash contrib/sim/multisim.sh 50 5 TestGaiaSimulationAfterImport + $(GOBIN)/runsim 50 5 TestGaiaSimulationAfterImport -sim-gaia-custom-genesis-multi-seed: +sim-gaia-custom-genesis-multi-seed: runsim @echo "Running multi-seed custom genesis simulation..." @echo "By default, ${HOME}/.gaiad/config/genesis.json will be used." - @bash contrib/sim/multisim.sh 400 5 TestFullGaiaSimulation ${HOME}/.gaiad/config/genesis.json + $(GOBIN)/runsim -g ${HOME}/.gaiad/config/genesis.json 400 5 TestFullGaiaSimulation -sim-gaia-multi-seed: +sim-gaia-multi-seed: runsim @echo "Running multi-seed Gaia simulation. This may take awhile!" - @bash contrib/sim/multisim.sh 400 5 TestFullGaiaSimulation + $(GOBIN)/runsim 400 5 TestFullGaiaSimulation sim-benchmark-invariants: @echo "Running simulation invariant benchmarks..."