From 20cfa05c105c0fabef1150ce3d0687bbebb99d32 Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Fri, 8 Apr 2022 12:43:53 -0500 Subject: [PATCH] e2e with IBC tx and test (#1216) * second chain with tests * add hermes, comment out balance query for now * eventually * Update Makefile Co-authored-by: Roman <34196718+p0mvn@users.noreply.github.com> * remove gas fees and unused functions * readded check * Nicco changes to own hermes image * Remove unused hermes.Dockerfile * Use a single Dockerfile for both debug and official image * readd build hermes in makefile * remove hermes * Set correct golang image and use correct debug image tag Co-authored-by: Adam Tucker Co-authored-by: Roman <34196718+p0mvn@users.noreply.github.com> Co-authored-by: Niccolo Raspa --- Dockerfile | 7 +- Makefile | 2 +- e2e.Dockerfile | 26 ----- tests/e2e/chain.go | 4 +- tests/e2e/e2e_setup_test.go | 141 ++++++++++++++++++++++---- tests/e2e/e2e_test.go | 59 +++++++++-- tests/e2e/e2e_util_test.go | 108 ++++++++++++++++++++ tests/e2e/scripts/hermes_bootstrap.sh | 72 +++++++++++++ 8 files changed, 362 insertions(+), 57 deletions(-) delete mode 100644 e2e.Dockerfile create mode 100644 tests/e2e/e2e_util_test.go create mode 100644 tests/e2e/scripts/hermes_bootstrap.sh diff --git a/Dockerfile b/Dockerfile index 022e5105bef..2a01c6639ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ # syntax=docker/dockerfile:1 +ARG BASE_IMG_TAG=nonroot + ## Build Image FROM golang:1.18-bullseye as build @@ -13,7 +15,7 @@ RUN sha256sum /lib/libwasmvm_muslc.a | grep d0152067a5609bfdfb3f0d5d6c0f2760f79d RUN BUILD_TAGS=muslc make build ## Deploy image -FROM gcr.io/distroless/base-debian11:nonroot +FROM gcr.io/distroless/base-debian11:${BASE_IMG_TAG} COPY --from=build /osmosis/build/osmosisd /bin/osmosisd @@ -22,6 +24,7 @@ WORKDIR $HOME EXPOSE 26656 EXPOSE 26657 -EXPOSE 1317 +EXPOSE 1317 ENTRYPOINT ["osmosisd"] +CMD [ "start" ] diff --git a/Makefile b/Makefile index b7e2e31668e..a5ee11d11a3 100644 --- a/Makefile +++ b/Makefile @@ -236,7 +236,7 @@ benchmark: @go test -mod=readonly -bench=. $(PACKAGES_UNIT) docker-build-debug: - @docker build -t osmolabs/osmosisd-e2e --build-arg IMG_TAG=debug -f e2e.Dockerfile . + @docker build -t osmosis:debug --build-arg BASE_IMG_TAG=debug -f Dockerfile . ############################################################################### ### Linting ### diff --git a/e2e.Dockerfile b/e2e.Dockerfile deleted file mode 100644 index f0bfd24d8f4..00000000000 --- a/e2e.Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -# syntax=docker/dockerfile:1 -ARG IMG_TAG=latest - -## Build Image -FROM golang:1.17-bullseye as build - -WORKDIR /osmosis -COPY . /osmosis - -# From https://github.com/CosmWasm/wasmd/blob/master/Dockerfile -# For more details see https://github.com/CosmWasm/wasmvm#builds-of-libwasmvm -ADD https://github.com/CosmWasm/wasmvm/releases/download/v1.0.0-beta7/libwasmvm_muslc.a /lib/libwasmvm_muslc.a -RUN sha256sum /lib/libwasmvm_muslc.a | grep d0152067a5609bfdfb3f0d5d6c0f2760f79d5f2cd7fd8513cafa9932d22eb350 -RUN BUILD_TAGS=muslc make build - -## Deploy image -FROM gcr.io/distroless/cc:$IMG_TAG -ARG IMG_TAG -COPY --from=build /osmosis/build/osmosisd /bin/osmosisd - -ENV HOME /osmosis -WORKDIR $HOME - -EXPOSE 26656 26657 1317 9090 - -ENTRYPOINT ["osmosisd", "start"] diff --git a/tests/e2e/chain.go b/tests/e2e/chain.go index 24cea7e9f3b..a5eadfe4230 100644 --- a/tests/e2e/chain.go +++ b/tests/e2e/chain.go @@ -47,14 +47,14 @@ type chain struct { validators []*validator } -func newChain(name string) (*chain, error) { +func newChain(id string) (*chain, error) { tmpDir, err := ioutil.TempDir("", "osmosis-e2e-testnet-") if err != nil { return nil, err } return &chain{ - id: name, + id: id, dataDir: tmpDir, }, nil } diff --git a/tests/e2e/e2e_setup_test.go b/tests/e2e/e2e_setup_test.go index ad6be48df76..53bf8cbd67c 100644 --- a/tests/e2e/e2e_setup_test.go +++ b/tests/e2e/e2e_setup_test.go @@ -4,7 +4,11 @@ import ( "context" "encoding/json" "fmt" + "io" + "io/ioutil" + "net/http" "os" + "path" "path/filepath" "strconv" "strings" @@ -27,16 +31,18 @@ import ( const ( // common - osmoDenom = "uosmo" - stakeDenom = "stake" - minGasPrice = "0.00001" + osmoDenom = "uosmo" + stakeDenom = "stake" + ibcDenom = "ibc/ED07A3391A112B175915CD8FAF43A2DA8E4790EDE12566649D0C2F97716B8518" + minGasPrice = "0.000" + ibcSendAmount = 3300000000 // chainA - chainAName = "osmo-test-a" + chainAID = "osmo-test-a" osmoBalanceA = 200000000000 stakeBalanceA = 110000000000 stakeAmountA = 100000000000 // chainB - chainBName = "osmo-test-b" + chainBID = "osmo-test-b" osmoBalanceB = 500000000000 stakeBalanceB = 440000000000 stakeAmountB = 400000000000 @@ -54,12 +60,13 @@ var ( type IntegrationTestSuite struct { suite.Suite - tmpDirs []string - chainA *chain - chainB *chain - dkrPool *dockertest.Pool - dkrNet *dockertest.Network - valResources map[string][]*dockertest.Resource + tmpDirs []string + chainA *chain + chainB *chain + dkrPool *dockertest.Pool + dkrNet *dockertest.Network + hermesResource *dockertest.Resource + valResources map[string][]*dockertest.Resource } func TestIntegrationTestSuite(t *testing.T) { @@ -70,10 +77,10 @@ func (s *IntegrationTestSuite) SetupSuite() { s.T().Log("setting up e2e integration test suite...") var err error - s.chainA, err = newChain(chainAName) + s.chainA, err = newChain(chainAID) s.Require().NoError(err) - s.chainB, err = newChain(chainBName) + s.chainB, err = newChain(chainBID) s.Require().NoError(err) s.dkrPool, err = dockertest.NewPool("") @@ -101,6 +108,8 @@ func (s *IntegrationTestSuite) SetupSuite() { s.initGenesis(s.chainB) s.initValidatorConfigs(s.chainB) s.runValidators(s.chainB, 10) + + s.runIBCRelayer() } func (s *IntegrationTestSuite) TearDownSuite() { @@ -115,6 +124,8 @@ func (s *IntegrationTestSuite) TearDownSuite() { s.T().Log("tearing down e2e integration test suite...") + s.Require().NoError(s.dkrPool.Purge(s.hermesResource)) + for _, vr := range s.valResources { for _, r := range vr { s.Require().NoError(s.dkrPool.Purge(r)) @@ -137,11 +148,11 @@ func (s *IntegrationTestSuite) initNodes(c *chain) { // initialize a genesis file for the first validator val0ConfigDir := c.validators[0].configDir() for _, val := range c.validators { - if c.id == chainAName { + if c.id == chainAID { s.Require().NoError( addGenesisAccount(val0ConfigDir, "", initBalanceStrA, val.keyInfo.GetAddress()), ) - } else if c.id == chainBName { + } else if c.id == chainBID { s.Require().NoError( addGenesisAccount(val0ConfigDir, "", initBalanceStrB, val.keyInfo.GetAddress()), ) @@ -197,7 +208,7 @@ func (s *IntegrationTestSuite) initGenesis(c *chain) { genTxs := make([]json.RawMessage, len(c.validators)) for i, val := range c.validators { stakeAmountCoin := stakeAmountCoinA - if c.id != chainAName { + if c.id != chainAID { stakeAmountCoin = stakeAmountCoinB } createValmsg, err := val.buildCreateValidatorMsg(stakeAmountCoin) @@ -288,7 +299,8 @@ func (s *IntegrationTestSuite) runValidators(c *chain, portOffset int) { Mounts: []string{ fmt.Sprintf("%s/:/osmosis/.osmosisd", val.configDir()), }, - Repository: "osmolabs/osmosisd-e2e", + Repository: "osmosis", + Tag: "debug", } // expose the first validator for debugging and communication @@ -340,6 +352,101 @@ func (s *IntegrationTestSuite) runValidators(c *chain, portOffset int) { ) } +func (s *IntegrationTestSuite) runIBCRelayer() { + s.T().Log("starting Hermes relayer container...") + + tmpDir, err := ioutil.TempDir("", "gaia-e2e-testnet-hermes-") + s.Require().NoError(err) + s.tmpDirs = append(s.tmpDirs, tmpDir) + + gaiaAVal := s.chainA.validators[0] + gaiaBVal := s.chainB.validators[0] + hermesCfgPath := path.Join(tmpDir, "hermes") + + s.Require().NoError(os.MkdirAll(hermesCfgPath, 0755)) + _, err = copyFile( + filepath.Join("./scripts/", "hermes_bootstrap.sh"), + filepath.Join(hermesCfgPath, "hermes_bootstrap.sh"), + ) + s.Require().NoError(err) + + s.hermesResource, err = s.dkrPool.RunWithOptions( + &dockertest.RunOptions{ + Name: fmt.Sprintf("%s-%s-relayer", s.chainA.id, s.chainB.id), + Repository: "osmolabs/hermes", + Tag: "0.13.0", + NetworkID: s.dkrNet.Network.ID, + Cmd: []string{ + "start", + }, + User: "root:root", + Mounts: []string{ + fmt.Sprintf("%s/:/root/hermes", hermesCfgPath), + }, + ExposedPorts: []string{ + "3031", + }, + PortBindings: map[docker.Port][]docker.PortBinding{ + "3031/tcp": {{HostIP: "", HostPort: "3031"}}, + }, + Env: []string{ + fmt.Sprintf("OSMO_A_E2E_CHAIN_ID=%s", s.chainA.id), + fmt.Sprintf("OSMO_B_E2E_CHAIN_ID=%s", s.chainB.id), + fmt.Sprintf("OSMO_A_E2E_VAL_MNEMONIC=%s", gaiaAVal.mnemonic), + fmt.Sprintf("OSMO_B_E2E_VAL_MNEMONIC=%s", gaiaBVal.mnemonic), + fmt.Sprintf("OSMO_A_E2E_VAL_HOST=%s", s.valResources[s.chainA.id][0].Container.Name[1:]), + fmt.Sprintf("OSMO_B_E2E_VAL_HOST=%s", s.valResources[s.chainB.id][0].Container.Name[1:]), + }, + Entrypoint: []string{ + "sh", + "-c", + "chmod +x /root/hermes/hermes_bootstrap.sh && /root/hermes/hermes_bootstrap.sh", + }, + }, + noRestart, + ) + s.Require().NoError(err) + + endpoint := fmt.Sprintf("http://%s/state", s.hermesResource.GetHostPort("3031/tcp")) + s.Require().Eventually( + func() bool { + resp, err := http.Get(endpoint) + if err != nil { + return false + } + + defer resp.Body.Close() + + bz, err := io.ReadAll(resp.Body) + if err != nil { + return false + } + + var respBody map[string]interface{} + if err := json.Unmarshal(bz, &respBody); err != nil { + return false + } + + status := respBody["status"].(string) + result := respBody["result"].(map[string]interface{}) + + return status == "success" && len(result["chains"].([]interface{})) == 2 + }, + 5*time.Minute, + time.Second, + "hermes relayer not healthy", + ) + + s.T().Logf("started Hermes relayer container: %s", s.hermesResource.Container.ID) + + // XXX: Give time to both networks to start, otherwise we might see gRPC + // transport errors. + time.Sleep(10 * time.Second) + + // create the client, connection and channel between the two Gaia chains + s.connectIBCChains() +} + func noRestart(config *docker.HostConfig) { // in this case we don't want the nodes to restart on failure config.RestartPolicy = docker.RestartPolicy{ diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index 2cc226a7854..8be7b7dd446 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -1,10 +1,10 @@ package e2e import ( - "errors" "fmt" "io" "net/http" + "strings" "time" sdk "github.com/cosmos/cosmos-sdk/types" @@ -13,9 +13,10 @@ import ( func (s *IntegrationTestSuite) TestQueryBalances() { var ( - expectedDenoms = []string{osmoDenom, stakeDenom} - expectedBalancesA = []uint64{osmoBalanceA, stakeBalanceA - stakeAmountA} - expectedBalancesB = []uint64{osmoBalanceB, stakeBalanceB - stakeAmountB} + expectedDenomsA = []string{osmoDenom, stakeDenom} + expectedDenomsB = []string{osmoDenom, stakeDenom, ibcDenom} + expectedBalancesA = []uint64{osmoBalanceA - ibcSendAmount, stakeBalanceA - stakeAmountA} + expectedBalancesB = []uint64{osmoBalanceB, stakeBalanceB - stakeAmountB, ibcSendAmount} ) chainAAPIEndpoint := fmt.Sprintf("http://%s", s.valResources[s.chainA.id][0].GetHostPort("1317/tcp")) @@ -28,7 +29,7 @@ func (s *IntegrationTestSuite) TestQueryBalances() { balancesB, err := queryBalances(chainBAPIEndpoint, s.chainB.validators[0].keyInfo.GetAddress().String()) s.Require().NoError(err) s.Require().NotNil(balancesB) - s.Require().Equal(2, len(balancesB)) + s.Require().Equal(3, len(balancesB)) actualDenomsA := make([]string, 0, 2) actualBalancesA := make([]uint64, 0, 2) @@ -45,10 +46,11 @@ func (s *IntegrationTestSuite) TestQueryBalances() { actualBalancesB = append(actualBalancesB, balanceB.Amount.Uint64()) } - s.Require().ElementsMatch(expectedDenoms, actualDenomsA) + s.Require().ElementsMatch(expectedDenomsA, actualDenomsA) s.Require().ElementsMatch(expectedBalancesA, actualBalancesA) - s.Require().ElementsMatch(expectedDenoms, actualDenomsB) + s.Require().ElementsMatch(expectedDenomsB, actualDenomsB) s.Require().ElementsMatch(expectedBalancesB, actualBalancesB) + } func queryBalances(endpoint, addr string) (sdk.Coins, error) { @@ -56,7 +58,8 @@ func queryBalances(endpoint, addr string) (sdk.Coins, error) { "%s/cosmos/bank/v1beta1/balances/%s", endpoint, addr, ) - resp, err := http.Get(path) + var err error + var resp *http.Response retriesLeft := 5 for { resp, err = http.Get(path) @@ -64,7 +67,7 @@ func queryBalances(endpoint, addr string) (sdk.Coins, error) { if resp.StatusCode == http.StatusServiceUnavailable { retriesLeft-- if retriesLeft == 0 { - return nil, errors.New(fmt.Sprintf("exceeded retry limit of %d with %d", retriesLeft, http.StatusServiceUnavailable)) + return nil, fmt.Errorf("exceeded retry limit of %d with %d", retriesLeft, http.StatusServiceUnavailable) } time.Sleep(10 * time.Second) } else { @@ -90,3 +93,41 @@ func queryBalances(endpoint, addr string) (sdk.Coins, error) { return balancesResp.GetBalances(), nil } + +func (s *IntegrationTestSuite) TestIBCTokenTransfer() { + var ibcStakeDenom string + + s.Run("send_uosmo_to_chainB", func() { + recipient := s.chainB.validators[0].keyInfo.GetAddress().String() + token := sdk.NewInt64Coin(osmoDenom, ibcSendAmount) // 3,300uosmo + s.sendIBC(s.chainA.id, s.chainB.id, recipient, token) + + chainBAPIEndpoint := fmt.Sprintf("http://%s", s.valResources[s.chainB.id][0].GetHostPort("1317/tcp")) + + // require the recipient account receives the IBC tokens (IBC packets ACKd) + var ( + balances sdk.Coins + err error + ) + s.Require().Eventually( + func() bool { + balances, err = queryBalances(chainBAPIEndpoint, recipient) + s.Require().NoError(err) + + return balances.Len() == 3 + }, + time.Minute, + 5*time.Second, + ) + + for _, c := range balances { + if strings.Contains(c.Denom, "ibc/") { + ibcStakeDenom = c.Denom + s.Require().Equal(token.Amount.Int64(), c.Amount.Int64()) + break + } + } + + s.Require().NotEmpty(ibcStakeDenom) + }) +} diff --git a/tests/e2e/e2e_util_test.go b/tests/e2e/e2e_util_test.go new file mode 100644 index 00000000000..efd7cbffed9 --- /dev/null +++ b/tests/e2e/e2e_util_test.go @@ -0,0 +1,108 @@ +package e2e + +import ( + "bytes" + "context" + "fmt" + "time" + + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/ory/dockertest/v3/docker" +) + +func (s *IntegrationTestSuite) connectIBCChains() { + s.T().Logf("connecting %s and %s chains via IBC", s.chainA.id, s.chainB.id) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + exec, err := s.dkrPool.Client.CreateExec(docker.CreateExecOptions{ + Context: ctx, + AttachStdout: true, + AttachStderr: true, + Container: s.hermesResource.Container.ID, + User: "root", + Cmd: []string{ + "hermes", + "create", + "channel", + s.chainA.id, + s.chainB.id, + "--port-a=transfer", + "--port-b=transfer", + }, + }) + s.Require().NoError(err) + + var ( + outBuf bytes.Buffer + errBuf bytes.Buffer + ) + + err = s.dkrPool.Client.StartExec(exec.ID, docker.StartExecOptions{ + Context: ctx, + Detach: false, + OutputStream: &outBuf, + ErrorStream: &errBuf, + }) + s.Require().NoErrorf( + err, + "failed connect chains; stdout: %s, stderr: %s", outBuf.String(), errBuf.String(), + ) + + s.Require().Containsf( + errBuf.String(), + "successfully opened init channel", + "failed to connect chains via IBC: %s", errBuf.String(), + ) + + s.T().Logf("connected %s and %s chains via IBC", s.chainA.id, s.chainB.id) +} + +func (s *IntegrationTestSuite) sendIBC(srcChainID, dstChainID, recipient string, token sdk.Coin) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + s.T().Logf("sending %s from %s to %s (%s)", token, srcChainID, dstChainID, recipient) + + exec, err := s.dkrPool.Client.CreateExec(docker.CreateExecOptions{ + Context: ctx, + AttachStdout: true, + AttachStderr: true, + Container: s.hermesResource.Container.ID, + User: "root", + Cmd: []string{ + "hermes", + "tx", + "raw", + "ft-transfer", + dstChainID, + srcChainID, + "transfer", // source chain port ID + "channel-0", // since only one connection/channel exists, assume 0 + token.Amount.String(), + fmt.Sprintf("--denom=%s", token.Denom), + fmt.Sprintf("--receiver=%s", recipient), + "--timeout-height-offset=1000", + }, + }) + s.Require().NoError(err) + + var ( + outBuf bytes.Buffer + errBuf bytes.Buffer + ) + + err = s.dkrPool.Client.StartExec(exec.ID, docker.StartExecOptions{ + Context: ctx, + Detach: false, + OutputStream: &outBuf, + ErrorStream: &errBuf, + }) + s.Require().NoErrorf( + err, + "failed to send IBC tokens; stdout: %s, stderr: %s", outBuf.String(), errBuf.String(), + ) + + s.T().Log("successfully sent IBC tokens") +} diff --git a/tests/e2e/scripts/hermes_bootstrap.sh b/tests/e2e/scripts/hermes_bootstrap.sh new file mode 100644 index 00000000000..7295a8e14ec --- /dev/null +++ b/tests/e2e/scripts/hermes_bootstrap.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +set -ex + +# initialize Hermes relayer configuration +mkdir -p /root/.hermes/ +touch /root/.hermes/config.toml + +# setup Hermes relayer configuration +tee /root/.hermes/config.toml <