refactor(systemtests): Extract system test framework (#22578)

Co-authored-by: marbar3778 <[email protected]>

go.work.example

@@ -22,6 +22,7 @@ use (
 	./server/v2/appmanager
 	./store
 	./store/v2
+	./systemtests
 	./runtime/v2
 	./tools/cosmovisor
 	./tools/confix

systemtests/CHANGELOG.md

@@ -0,0 +1,41 @@
+<!--
+Guiding Principles:
+
+Changelogs are for humans, not machines.
+There should be an entry for every single version.
+The same types of changes should be grouped.
+Versions and sections should be linkable.
+The latest version comes first.
+The release date of each version is displayed.
+Mention whether you follow Semantic Versioning.
+
+Usage:
+
+Changelog entries are generated by git cliff ref: https://github.com/orhun/git-cliff
+
+Each commit should be conventional, the following message groups are supported.
+
+* feat: A new feature
+* fix: A bug fix
+* docs: Documentation only changes
+* style: Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc)
+* refactor: A code change that neither fixes a bug nor adds a feature
+* perf: A code change that improves performance
+* test: Adding missing tests or correcting existing tests
+* build: Changes that affect the build system or external dependencies (example scopes: go, npm)
+* ci: Changes to our CI configuration files and scripts (example scopes: GH Actions)
+* chore: Other changes that don't modify src or test files
+* revert: Reverts a previous commit
+
+When a change is made that affects the API or state machine, the commit message prefix should be suffixed with `!`.
+
+Ref: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json
+-->
+
+# Changelog
+
+## [Unreleased]
+
+### Features
+
+* [#22578](https://github.com/cosmos/cosmos-sdk/pull/22578) Extract system test framework

systemtests/README.md

@@ -0,0 +1,63 @@
+# System Tests
+
+This package contains the testing framework for black-box system tests. It includes a test runner that sets up a 
+multi-node blockchain locally for use in tests. The framework provides utilities and helpers for easy access and 
+setup in tests.
+
+## Components
+
+- **CLI**: Command-line interface wrapper for interacting with the chain or keyring
+- **Servers**: Server instances to run the blockchain environment.
+- **Events**: Event listeners
+- **RPC**: Remote Procedure Call setup for communication.
+
+## Dependencies
+
+- **testify**: Testing toolkit.
+- **gjson**: JSON parser.
+- **sjson**: JSON modifier.
+
+Server and client-side operations are executed on the host machine.
+
+## Developer
+
+### Test strategy
+
+System tests cover the full stack via cli and a running (multi node) network. They are more expensive (in terms of time/ cpu) 
+to run compared to unit or integration tests. 
+Therefore, we focus on the **critical path** and do not cover every condition.
+
+## How to use
+
+Read the [getting_started.md](../tests/systemtests/getting_started.md) guide to get started.
+
+### Execute a single test
+
+```sh
+go test -tags system_test -count=1 -v . --run TestStakeUnstake  -verbose
+```
+
+Test cli parameters
+
+* `-verbose` verbose output
+* `-wait-time` duration - time to wait for chain events (default 30s)
+* `-nodes-count` int - number of nodes in the cluster (default 4)
+
+# Port ranges
+
+With *n* nodes:
+
+* `26657` - `26657+n` - RPC
+* `1317` - `1317+n` - API
+* `9090` - `9090+n` - GRPC
+* `16656` - `16656+n` - P2P
+
+For example Node *3* listens on `26660` for RPC calls
+
+## Resources
+
+* [gjson query syntax](https://github.com/tidwall/gjson#path-syntax)
+
+## Disclaimer
+
+This is based on the system test framework in [wasmd](https://github.com/CosmWasm/wasmd) built by Confio.

tests/systemtests/cli.go → systemtests/cli.go

@@ -100,39 +100,31 @@ func (c CLIWrapper) WithRunErrorsIgnored() CLIWrapper {
 
 // WithRunErrorMatcher assert function to ensure run command error value
 func (c CLIWrapper) WithRunErrorMatcher(f RunErrorAssert) CLIWrapper {
-	return *NewCLIWrapperX(
-		c.t,
-		c.execBinary,
-		c.nodeAddress,
-		c.chainID,
-		c.awaitNextBlock,
-		c.nodesCount,
-		c.homeDir,
-		c.fees,
-		c.Debug,
-		f,
-		c.expTXCommitted,
-	)
+	return c.clone(func(r *CLIWrapper) {
+		r.assertErrorFn = f
+	})
 }
 
 func (c CLIWrapper) WithNodeAddress(nodeAddr string) CLIWrapper {
-	return *NewCLIWrapperX(
-		c.t,
-		c.execBinary,
-		nodeAddr,
-		c.chainID,
-		c.awaitNextBlock,
-		c.nodesCount,
-		c.homeDir,
-		c.fees,
-		c.Debug,
-		c.assertErrorFn,
-		c.expTXCommitted,
-	)
+	return c.clone(func(r *CLIWrapper) {
+		r.nodeAddress = nodeAddr
+	})
 }
 
 func (c CLIWrapper) WithAssertTXUncommitted() CLIWrapper {
-	return *NewCLIWrapperX(
+	return c.clone(func(r *CLIWrapper) {
+		r.expTXCommitted = false
+	})
+}
+
+func (c CLIWrapper) WithChainID(newChainID string) CLIWrapper {
+	return c.clone(func(r *CLIWrapper) {
+		r.chainID = newChainID
+	})
+}
+
+func (c CLIWrapper) clone(mutator ...func(r *CLIWrapper)) CLIWrapper {
+	r := NewCLIWrapperX(
 		c.t,
 		c.execBinary,
 		c.nodeAddress,
@@ -143,8 +135,12 @@ func (c CLIWrapper) WithAssertTXUncommitted() CLIWrapper {
 		c.fees,
 		c.Debug,
 		c.assertErrorFn,
-		false,
+		c.expTXCommitted,
 	)
+	for _, m := range mutator {
+		m(r)
+	}
+	return *r
 }
 
 // Run main entry for executing cli commands.
@@ -156,7 +152,7 @@ func (c CLIWrapper) Run(args ...string) string {
 	}) {
 		args = append(args, "--fees="+c.fees) // add default fee
 	}
-	args = c.withTXFlags(args...)
+	args = c.WithTXFlags(args...)
 	execOutput, ok := c.run(args)
 	if !ok {
 		return execOutput
@@ -207,14 +203,14 @@ func (c CLIWrapper) AwaitTxCommitted(submitResp string, timeout ...time.Duration
 
 // Keys wasmd keys CLI command
 func (c CLIWrapper) Keys(args ...string) string {
-	args = c.withKeyringFlags(args...)
+	args = c.WithKeyringFlags(args...)
 	out, _ := c.run(args)
 	return out
 }
 
 // CustomQuery main entrypoint for wasmd CLI queries
 func (c CLIWrapper) CustomQuery(args ...string) string {
-	args = c.withQueryFlags(args...)
+	args = c.WithQueryFlags(args...)
 	out, _ := c.run(args)
 	return out
 }
@@ -254,23 +250,32 @@ func (c CLIWrapper) runWithInput(args []string, input io.Reader) (output string,
 	return strings.TrimSpace(string(gotOut)), ok
 }
 
-func (c CLIWrapper) withQueryFlags(args ...string) []string {
+// WithQueryFlags append the test default query flags to the given args
+func (c CLIWrapper) WithQueryFlags(args ...string) []string {
 	args = append(args, "--output", "json")
-	return c.withChainFlags(args...)
+	return c.WithTargetNodeFlags(args...)
 }
 
-func (c CLIWrapper) withTXFlags(args ...string) []string {
+// WithTXFlags append the test default TX flags to the given args.
+// This includes
+// - broadcast-mode: sync
+// - output: json
+// - chain-id
+// - keyring flags
+// - target-node
+func (c CLIWrapper) WithTXFlags(args ...string) []string {
 	args = append(args,
 		"--broadcast-mode", "sync",
 		"--output", "json",
 		"--yes",
 		"--chain-id", c.chainID,
 	)
-	args = c.withKeyringFlags(args...)
-	return c.withChainFlags(args...)
+	args = c.WithKeyringFlags(args...)
+	return c.WithTargetNodeFlags(args...)
 }
 
-func (c CLIWrapper) withKeyringFlags(args ...string) []string {
+// WithKeyringFlags append the test default keyring flags to the given args
+func (c CLIWrapper) WithKeyringFlags(args ...string) []string {
 	r := append(args,
 		"--home", c.homeDir,
 		"--keyring-backend", "test",
@@ -283,7 +288,8 @@ func (c CLIWrapper) withKeyringFlags(args ...string) []string {
 	return append(r, "--output", "json")
 }
 
-func (c CLIWrapper) withChainFlags(args ...string) []string {
+// WithTargetNodeFlags append the test default target node address flags to the given args
+func (c CLIWrapper) WithTargetNodeFlags(args ...string) []string {
 	return append(args,
 		"--node", c.nodeAddress,
 	)
@@ -297,7 +303,7 @@ func (c CLIWrapper) WasmExecute(contractAddr, msg, from string, args ...string)
 
 // AddKey add key to default keyring. Returns address
 func (c CLIWrapper) AddKey(name string) string {
-	cmd := c.withKeyringFlags("keys", "add", name, "--no-backup")
+	cmd := c.WithKeyringFlags("keys", "add", name, "--no-backup")
 	out, _ := c.run(cmd)
 	addr := gjson.Get(out, "address").String()
 	require.NotEmpty(c.t, addr, "got %q", out)
@@ -306,7 +312,7 @@ func (c CLIWrapper) AddKey(name string) string {
 
 // AddKeyFromSeed recovers the key from given seed and add it to default keyring. Returns address
 func (c CLIWrapper) AddKeyFromSeed(name, mnemoic string) string {
-	cmd := c.withKeyringFlags("keys", "add", name, "--recover")
+	cmd := c.WithKeyringFlags("keys", "add", name, "--recover")
 	out, _ := c.runWithInput(cmd, strings.NewReader(mnemoic))
 	addr := gjson.Get(out, "address").String()
 	require.NotEmpty(c.t, addr, "got %q", out)
@@ -315,7 +321,7 @@ func (c CLIWrapper) AddKeyFromSeed(name, mnemoic string) string {
 
 // GetKeyAddr returns Acc address
 func (c CLIWrapper) GetKeyAddr(name string) string {
-	cmd := c.withKeyringFlags("keys", "show", name, "-a")
+	cmd := c.WithKeyringFlags("keys", "show", name, "-a")
 	out, _ := c.run(cmd)
 	addr := strings.Trim(out, "\n")
 	require.NotEmpty(c.t, addr, "got %q", out)
@@ -324,7 +330,7 @@ func (c CLIWrapper) GetKeyAddr(name string) string {
 
 // GetKeyAddrPrefix returns key address with Beach32 prefix encoding for a key (acc|val|cons)
 func (c CLIWrapper) GetKeyAddrPrefix(name, prefix string) string {
-	cmd := c.withKeyringFlags("keys", "show", name, "-a", "--bech="+prefix)
+	cmd := c.WithKeyringFlags("keys", "show", name, "-a", "--bech="+prefix)
 	out, _ := c.run(cmd)
 	addr := strings.Trim(out, "\n")
 	require.NotEmpty(c.t, addr, "got %q", out)
@@ -413,6 +419,10 @@ func (c CLIWrapper) SubmitAndVoteGovProposal(proposalJson string, args ...string
 	return ourProposalID
 }
 
+func (c CLIWrapper) ChainID() string {
+	return c.chainID
+}
+
 // Version returns the current version of the client binary
 func (c CLIWrapper) Version() string {
 	v, ok := c.run([]string{"version"})