From 9cf0f257c0627d2f2b4e741fcafa37bd93233730 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Tue, 25 Jul 2017 10:06:07 -0400 Subject: [PATCH 1/2] Add benchmark for abci app --- benchmarks/app_test.go | 199 +++++++++++++++++++++++++++++++++++++++++ modules/coin/helper.go | 9 +- 2 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 benchmarks/app_test.go diff --git a/benchmarks/app_test.go b/benchmarks/app_test.go new file mode 100644 index 000000000000..03e188c3f4c0 --- /dev/null +++ b/benchmarks/app_test.go @@ -0,0 +1,199 @@ +package app + +import ( + "fmt" + "io/ioutil" + "testing" + + wire "github.com/tendermint/go-wire" + eyesApp "github.com/tendermint/merkleeyes/app" + eyes "github.com/tendermint/merkleeyes/client" + cmn "github.com/tendermint/tmlibs/common" + "github.com/tendermint/tmlibs/log" + + "github.com/tendermint/basecoin" + "github.com/tendermint/basecoin/app" + "github.com/tendermint/basecoin/modules/auth" + "github.com/tendermint/basecoin/modules/base" + "github.com/tendermint/basecoin/modules/coin" + "github.com/tendermint/basecoin/modules/fee" + "github.com/tendermint/basecoin/modules/nonce" + "github.com/tendermint/basecoin/modules/roles" + "github.com/tendermint/basecoin/stack" +) + +type BenchApp struct { + App *app.Basecoin + Accounts []*coin.AccountWithKey + ChainID string +} + +// DefaultHandler - placeholder to just handle sendtx +func DefaultHandler(feeDenom string) basecoin.Handler { + // use the default stack + c := coin.NewHandler() + r := roles.NewHandler() + d := stack.NewDispatcher( + stack.WrapHandler(c), + stack.WrapHandler(r), + ) + return stack.New( + base.Logger{}, + stack.Recovery{}, + auth.Signatures{}, + base.Chain{}, + nonce.ReplayCheck{}, + roles.NewMiddleware(), + fee.NewSimpleFeeMiddleware(coin.Coin{feeDenom, 0}, fee.Bank), + ).Use(d) +} + +func NewBenchApp(h basecoin.Handler, chainID string, n int, + persist bool) BenchApp { + + logger := log.NewNopLogger() + // logger := log.NewFilter(log.NewTMLogger(os.Stdout), log.AllowError()) + // logger = log.NewTracingLogger(logger) + + // TODO: disk writing + var eyesCli *eyes.Client + if persist { + tmpDir, _ := ioutil.TempDir("", "bc-app-benchmark") + eyesCli = eyes.NewLocalClient(tmpDir, 500) + } else { + eyesCli = eyes.NewLocalClient("", 0) + } + eyesApp.SetLogger(logger.With("module", "merkle")) + + app := app.NewBasecoin( + h, + eyesCli, + logger.With("module", "app"), + ) + res := app.SetOption("base/chain_id", chainID) + if res != "Success" { + panic("cannot set chain") + } + + // make keys + money := coin.Coins{{"mycoin", 1234567890}} + accts := make([]*coin.AccountWithKey, n) + for i := 0; i < n; i++ { + accts[i] = coin.NewAccountWithKey(money) + res := app.SetOption("coin/account", accts[i].MakeOption()) + if res != "Success" { + panic("can't set account") + } + } + + return BenchApp{ + App: app, + Accounts: accts, + ChainID: chainID, + } +} + +// make a random tx... +func (b BenchApp) makeTx(useFee bool) []byte { + n := len(b.Accounts) + sender := b.Accounts[cmn.RandInt()%n] + recipient := b.Accounts[cmn.RandInt()%n] + amount := coin.Coins{{"mycoin", 123}} + tx := coin.NewSendOneTx(sender.Actor(), recipient.Actor(), amount) + if useFee { + toll := coin.Coin{"mycoin", 2} + tx = fee.NewFee(tx, toll, sender.Actor()) + } + sequence := sender.NextSequence() + tx = nonce.NewTx(sequence, []basecoin.Actor{sender.Actor()}, tx) + tx = base.NewChainTx(b.ChainID, 0, tx) + stx := auth.NewMulti(tx) + auth.Sign(stx, sender.Key) + res := wire.BinaryBytes(stx.Wrap()) + return res +} + +func BenchmarkMakeTx(b *testing.B) { + h := DefaultHandler("mycoin") + app := NewBenchApp(h, "bench-chain", 10, false) + b.ResetTimer() + for i := 1; i <= b.N; i++ { + txBytes := app.makeTx(true) + if len(txBytes) < 2 { + panic("cannot commit") + } + } +} + +func benchmarkTransfers(b *testing.B, app BenchApp, blockSize int, useFee bool) { + // prepare txs + txs := make([][]byte, b.N) + for i := 1; i <= b.N; i++ { + txBytes := app.makeTx(useFee) + if len(txBytes) < 2 { + panic("cannot make bytes") + } + txs[i-1] = txBytes + } + + b.ResetTimer() + + for i := 1; i <= b.N; i++ { + res := app.App.DeliverTx(txs[i-1]) + if res.IsErr() { + panic(res.Error()) + } + if i%blockSize == 0 { + res := app.App.Commit() + if res.IsErr() { + panic("cannot commit") + } + } + } +} + +func BenchmarkSimpleTransfer(b *testing.B) { + benchmarks := []struct { + accounts int + blockSize int + useFee bool + toDisk bool + }{ + {100, 10, false, false}, + {100, 10, true, false}, + {100, 200, false, false}, + {100, 200, true, false}, + {10000, 10, false, false}, + {10000, 10, true, false}, + {10000, 200, false, false}, + {10000, 200, true, false}, + {100, 10, false, true}, + {100, 10, true, true}, + {100, 200, false, true}, + {100, 200, true, true}, + {10000, 10, false, true}, + {10000, 10, true, true}, + {10000, 200, false, true}, + {10000, 200, true, true}, + } + + for _, bb := range benchmarks { + prefix := fmt.Sprintf("%d-%d", bb.accounts, bb.blockSize) + if bb.useFee { + prefix += "-fee" + } else { + prefix += "-nofee" + } + if bb.toDisk { + prefix += "-persist" + } else { + prefix += "-memdb" + } + + h := DefaultHandler("mycoin") + app := NewBenchApp(h, "bench-chain", bb.accounts, bb.toDisk) + b.Run(prefix, func(sub *testing.B) { + benchmarkTransfers(sub, app, bb.blockSize, bb.useFee) + }) + } +} diff --git a/modules/coin/helper.go b/modules/coin/helper.go index 4672e85d33a6..4bf98cddc722 100644 --- a/modules/coin/helper.go +++ b/modules/coin/helper.go @@ -11,7 +11,8 @@ import ( // AccountWithKey is a helper for tests, that includes and account // along with the private key to access it. type AccountWithKey struct { - Key crypto.PrivKey + Key crypto.PrivKey + Sequence uint32 Account } @@ -34,6 +35,12 @@ func (a *AccountWithKey) Actor() basecoin.Actor { return auth.SigPerm(a.Key.PubKey().Address()) } +// NextSequence returns the next sequence to sign with +func (a *AccountWithKey) NextSequence() uint32 { + a.Sequence++ + return a.Sequence +} + // MakeOption returns a string to use with SetOption to initialize this account // // This is intended for use in test cases From 4414e69b7821afa74ad9d85f79a0696f2e23aac6 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Tue, 25 Jul 2017 10:28:54 -0400 Subject: [PATCH 2/2] Remove old test --- tests/tmsp/tmsp_test.go | 159 ---------------------------------------- 1 file changed, 159 deletions(-) delete mode 100644 tests/tmsp/tmsp_test.go diff --git a/tests/tmsp/tmsp_test.go b/tests/tmsp/tmsp_test.go deleted file mode 100644 index 5dca8e07b3bc..000000000000 --- a/tests/tmsp/tmsp_test.go +++ /dev/null @@ -1,159 +0,0 @@ -package tmsp_test - -// TODO: replace with benchmarker - -// import ( -// "encoding/json" -// "testing" - -// "github.com/stretchr/testify/assert" -// "github.com/stretchr/testify/require" -// "github.com/tendermint/basecoin/app" -// "github.com/tendermint/basecoin/types" -// wire "github.com/tendermint/go-wire" -// eyescli "github.com/tendermint/merkleeyes/client" -// cmn "github.com/tendermint/tmlibs/common" -// "github.com/tendermint/tmlibs/log" -// ) - -// func TestSendTx(t *testing.T) { -// eyesCli := eyescli.NewLocalClient("", 0) -// chainID := "test_chain_id" -// bcApp := app.NewBasecoin(eyesCli, log.TestingLogger().With("module", "app")) -// bcApp.SetOption("base/chain_id", chainID) -// // t.Log(bcApp.Info()) - -// test1PrivAcc := types.PrivAccountFromSecret("test1") -// test2PrivAcc := types.PrivAccountFromSecret("test2") - -// // Seed Basecoin with account -// test1Acc := test1PrivAcc.Account -// test1Acc.Balance = coin.Coins{{"", 1000}} -// accOpt, err := json.Marshal(test1Acc) -// require.Nil(t, err) -// bcApp.SetOption("base/account", string(accOpt)) - -// // Construct a SendTx signature -// tx := &types.SendTx{ -// Gas: 0, -// Fee: coin.Coin{"", 0}, -// Inputs: []types.TxInput{ -// types.NewTxInput(test1PrivAcc.Account.PubKey, coin.Coins{{"", 1}}, 1), -// }, -// Outputs: []types.TxOutput{ -// types.TxOutput{ -// Address: test2PrivAcc.Account.PubKey.Address(), -// Coins: coin.Coins{{"", 1}}, -// }, -// }, -// } - -// // Sign request -// signBytes := tx.SignBytes(chainID) -// // t.Log("Sign bytes: %X\n", signBytes) -// sig := test1PrivAcc.Sign(signBytes) -// tx.Inputs[0].Signature = sig -// // t.Log("Signed TX bytes: %X\n", wire.BinaryBytes(types.TxS{tx})) - -// // Write request -// txBytes := wire.BinaryBytes(types.TxS{tx}) -// res := bcApp.DeliverTx(txBytes) -// // t.Log(res) -// assert.True(t, res.IsOK(), "Failed: %v", res.Error()) -// } - -// func TestSequence(t *testing.T) { -// eyesCli := eyescli.NewLocalClient("", 0) -// chainID := "test_chain_id" -// bcApp := app.NewBasecoin(eyesCli, log.TestingLogger().With("module", "app")) -// bcApp.SetOption("base/chain_id", chainID) -// // t.Log(bcApp.Info()) - -// // Get the test account -// test1PrivAcc := types.PrivAccountFromSecret("test1") -// test1Acc := test1PrivAcc.Account -// test1Acc.Balance = coin.Coins{{"", 1 << 53}} -// accOpt, err := json.Marshal(test1Acc) -// require.Nil(t, err) -// bcApp.SetOption("base/account", string(accOpt)) - -// sequence := int(1) -// // Make a bunch of PrivAccounts -// privAccounts := types.RandAccounts(1000, 1000000, 0) -// privAccountSequences := make(map[string]int) -// // Send coins to each account - -// for i := 0; i < len(privAccounts); i++ { -// privAccount := privAccounts[i] - -// tx := &types.SendTx{ -// Gas: 2, -// Fee: coin.Coin{"", 2}, -// Inputs: []types.TxInput{ -// types.NewTxInput(test1Acc.PubKey, coin.Coins{{"", 1000002}}, sequence), -// }, -// Outputs: []types.TxOutput{ -// types.TxOutput{ -// Address: privAccount.Account.PubKey.Address(), -// Coins: coin.Coins{{"", 1000000}}, -// }, -// }, -// } -// sequence += 1 - -// // Sign request -// signBytes := tx.SignBytes(chainID) -// sig := test1PrivAcc.Sign(signBytes) -// tx.Inputs[0].Signature = sig -// // t.Log("ADDR: %X -> %X\n", tx.Inputs[0].Address, tx.Outputs[0].Address) - -// // Write request -// txBytes := wire.BinaryBytes(struct{ types.Tx }{tx}) -// res := bcApp.DeliverTx(txBytes) -// assert.True(t, res.IsOK(), "DeliverTx error: %v", res.Error()) -// } - -// res := bcApp.Commit() -// assert.True(t, res.IsOK(), "Failed Commit: %v", res.Error()) - -// t.Log("-------------------- RANDOM SENDS --------------------") - -// // Now send coins between these accounts -// for i := 0; i < 10000; i++ { -// randA := cmn.RandInt() % len(privAccounts) -// randB := cmn.RandInt() % len(privAccounts) -// if randA == randB { -// continue -// } - -// privAccountA := privAccounts[randA] -// privAccountASequence := privAccountSequences[privAccountA.Account.PubKey.KeyString()] -// privAccountSequences[privAccountA.Account.PubKey.KeyString()] = privAccountASequence + 1 -// privAccountB := privAccounts[randB] - -// tx := &types.SendTx{ -// Gas: 2, -// Fee: coin.Coin{"", 2}, -// Inputs: []types.TxInput{ -// types.NewTxInput(privAccountA.PubKey, coin.Coins{{"", 3}}, privAccountASequence+1), -// }, -// Outputs: []types.TxOutput{ -// types.TxOutput{ -// Address: privAccountB.PubKey.Address(), -// Coins: coin.Coins{{"", 1}}, -// }, -// }, -// } - -// // Sign request -// signBytes := tx.SignBytes(chainID) -// sig := privAccountA.Sign(signBytes) -// tx.Inputs[0].Signature = sig -// // t.Log("ADDR: %X -> %X\n", tx.Inputs[0].Address, tx.Outputs[0].Address) - -// // Write request -// txBytes := wire.BinaryBytes(struct{ types.Tx }{tx}) -// res := bcApp.DeliverTx(txBytes) -// assert.True(t, res.IsOK(), "DeliverTx error: %v", res.Error()) -// } -// }