test: more e2e upgrade testing

tests/e2e/chain/config.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/cosmos/cosmos-sdk/server"
 	srvconfig "github.com/cosmos/cosmos-sdk/server/config"
@@ -33,10 +34,10 @@ const (
 	// common
 	OsmoDenom     = "uosmo"
 	StakeDenom    = "stake"
-	IbcDenom      = "ibc/ED07A3391A112B175915CD8FAF43A2DA8E4790EDE12566649D0C2F97716B8518"
+	OsmoIBCDenom  = "ibc/ED07A3391A112B175915CD8FAF43A2DA8E4790EDE12566649D0C2F97716B8518"
+	StakeIBCDenom = "ibc/C053D637CCA2A2BA030E2C5EE1B28A16F71CCB0E45E8BE52766DC1B241B7787"
 	MinGasPrice   = "0.000"
 	IbcSendAmount = 3300000000
-	VotingPeriod  = 30000000000 // 30 seconds
 	// chainA
 	ChainAID      = "osmo-test-a"
 	OsmoBalanceA  = 200000000000
@@ -57,6 +58,8 @@ var (
 
 	InitBalanceStrA = fmt.Sprintf("%d%s,%d%s", OsmoBalanceA, OsmoDenom, StakeBalanceA, StakeDenom)
 	InitBalanceStrB = fmt.Sprintf("%d%s,%d%s", OsmoBalanceB, OsmoDenom, StakeBalanceB, StakeDenom)
+	OsmoToken       = sdk.NewInt64Coin(OsmoDenom, IbcSendAmount)  // 3,300uosmo
+	StakeToken      = sdk.NewInt64Coin(StakeDenom, IbcSendAmount) // 3,300ustake
 )
 
 func addAccount(path, moniker, amountStr string, accAddr sdk.AccAddress) error {
@@ -130,7 +133,7 @@ func addAccount(path, moniker, amountStr string, accAddr sdk.AccAddress) error {
 	return genutil.ExportGenesisFile(genDoc, genFile)
 }
 
-func initGenesis(c *internalChain) error {
+func initGenesis(c *internalChain, votingPeriod time.Duration) error {
 	serverCtx := server.NewDefaultContext()
 	config := serverCtx.Config
 
@@ -174,7 +177,7 @@ func initGenesis(c *internalChain) error {
 	}
 
 	govGenState.VotingParams = govtypes.VotingParams{
-		VotingPeriod: VotingPeriod,
+		VotingPeriod: votingPeriod,
 	}
 
 	gz, err := util.Cdc.MarshalJSON(&govGenState)

tests/e2e/chain/main.go

@@ -1,14 +1,16 @@
 package chain
 
-func Init(id, dataDir string, validatorConfigs []*ValidatorConfig) (*Chain, error) {
+import "time"
+
+func Init(id, dataDir string, validatorConfigs []*ValidatorConfig, votingPeriod time.Duration) (*Chain, error) {
 	chain, err := new(id, dataDir)
 	if err != nil {
 		return nil, err
 	}
 	if err := initNodes(chain, len(validatorConfigs)); err != nil {
 		return nil, err
 	}
-	if err := initGenesis(chain); err != nil {
+	if err := initGenesis(chain, votingPeriod); err != nil {
 		return nil, err
 	}
 	if err := initValidatorConfigs(chain, validatorConfigs); err != nil {

tests/e2e/chain_init/main.go

@@ -5,21 +5,24 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/osmosis-labs/osmosis/v7/tests/e2e/chain"
 )
 
 func main() {
 	var (
-		valConfig []*chain.ValidatorConfig
-		dataDir   string
-		chainId   string
-		config    string
+		valConfig    []*chain.ValidatorConfig
+		dataDir      string
+		chainId      string
+		config       string
+		votingPeriod time.Duration
 	)
 
 	flag.StringVar(&dataDir, "data-dir", "", "chain data directory")
 	flag.StringVar(&chainId, "chain-id", "", "chain ID")
 	flag.StringVar(&config, "config", "", "serialized config")
+	flag.DurationVar(&votingPeriod, "voting-period", 30000000000, "voting period")
 
 	flag.Parse()
 
@@ -36,7 +39,7 @@ func main() {
 		panic(err)
 	}
 
-	createdChain, err := chain.Init(chainId, dataDir, valConfig)
+	createdChain, err := chain.Init(chainId, dataDir, valConfig, votingPeriod)
 	if err != nil {
 		panic(err)
 	}

tests/e2e/e2e_setup_test.go

@@ -25,8 +25,27 @@ import (
 )
 
 var (
-	// common
-	maxRetries             = 10 // max retries for json unmarshalling
+	// voting period for chain A
+	votingPeriodA float32
+	// voting period for chain B
+	votingPeriodB float32
+	// estimated number of blocks it takes to submit for a proposal
+	propSubmitBlocks float32 = 10
+	// estimated number of blocks it takes to deposit for a proposal
+	propDepositBlocks float32 = 10
+	// number of blocks it takes to vote for a single validator to vote for a proposal
+	propVoteBlocks float32 = 1.2
+	// number of blocks used as a calculation buffer
+	propBufferBlocks float32 = 5
+	// variable used to switch between chain A and B prop height in for loop
+	propHeight int
+	// upgrade proposal height for chain A
+	propHeightA int
+	// upgrade proposal height for chain B
+	propHeightB int
+	// max retries for json unmarshalling
+	maxRetries = 60
+	// whatever number of validator configs get posted here are how many validators that will spawn on chain A and B respectively
 	validatorConfigsChainA = []*chain.ValidatorConfig{
 		{
 			Pruning:            "default",
@@ -113,15 +132,18 @@ func (s *IntegrationTestSuite) SetupSuite() {
 	// 3. Run IBC relayer betweeen the two chains.
 	// 4. Execute various e2e tests, including IBC.
 	s.configureDockerResources(chain.ChainAID, chain.ChainBID)
-
 	s.configureChain(chain.ChainAID, validatorConfigsChainA)
 	s.configureChain(chain.ChainBID, validatorConfigsChainB)
 
 	s.runValidators(s.chains[0], 0)
 	s.runValidators(s.chains[1], 10)
 	s.runIBCRelayer()
-	s.initUpgrade()
+	// pre upgrade state creation
+	s.createPreUpgradeState()
+	// initialize and run the upgrade
 	s.upgrade()
+	// post upgrade tests
+	s.runPostUpgradeTests()
 }
 
 func (s *IntegrationTestSuite) TearDownSuite() {
@@ -156,14 +178,17 @@ func (s *IntegrationTestSuite) TearDownSuite() {
 }
 
 func (s *IntegrationTestSuite) runValidators(c *chain.Chain, portOffset int) {
-	s.T().Logf("starting Osmosis %s validator containers...", c.ChainMeta.Id)
+	s.T().Logf("starting %s validator containers...", c.ChainMeta.Id)
 	s.valResources[c.ChainMeta.Id] = make([]*dockertest.Resource, len(c.Validators))
+	pwd, err := os.Getwd()
+	s.Require().NoError(err)
 	for i, val := range c.Validators {
 		runOpts := &dockertest.RunOptions{
 			Name:      val.Name,
 			NetworkID: s.dkrNet.Network.ID,
 			Mounts: []string{
 				fmt.Sprintf("%s/:/osmosis/.osmosisd", val.ConfigDir),
+				fmt.Sprintf("%s/scripts:/osmosis", pwd),
 			},
 			Repository: "osmolabs/osmosis-dev",
 			Tag:        "v7.2.1-debug",
@@ -192,7 +217,7 @@ func (s *IntegrationTestSuite) runValidators(c *chain.Chain, portOffset int) {
 		s.Require().NoError(err)
 
 		s.valResources[c.ChainMeta.Id][i] = resource
-		s.T().Logf("started Osmosis %s validator container: %s", c.ChainMeta.Id, resource.Container.ID)
+		s.T().Logf("started %s validator container: %s", resource.Container.Name[1:], resource.Container.ID)
 	}
 
 	rpcClient, err := rpchttp.New("tcp://localhost:26657", "/websocket")
@@ -327,16 +352,27 @@ func (s *IntegrationTestSuite) configureChain(chainId string, validatorConfigs [
 	b, err := json.Marshal(validatorConfigs)
 	s.Require().NoError(err)
 
+	numVal := float32(len(validatorConfigs))
+	// voting period is number of blocks it takes to deposit, 1.2 seconds per validator to vote on the prop, then a buffer
+	votingPeriodNum := propDepositBlocks + numVal*propVoteBlocks + propBufferBlocks
+	if chainId == chain.ChainAID {
+		votingPeriodA = votingPeriodNum
+	} else if chainId == chain.ChainBID {
+		votingPeriodB = votingPeriodNum
+	}
+	votingPeriod := time.Duration(int(votingPeriodNum) * 1000000000)
+
 	s.initResource, err = s.dkrPool.RunWithOptions(
 		&dockertest.RunOptions{
 			Name:       fmt.Sprintf("%s", chainId),
 			Repository: "osmolabs/osmosis-init",
-			Tag:        "v7.3.0",
+			Tag:        "v7.3.0-1",
 			NetworkID:  s.dkrNet.Network.ID,
 			Cmd: []string{
 				fmt.Sprintf("--data-dir=%s", tmpDir),
 				fmt.Sprintf("--chain-id=%s", chainId),
 				fmt.Sprintf("--config=%s", b),
+				fmt.Sprintf("--voting-period=%v", votingPeriod),
 			},
 			User: "root:root",
 			Mounts: []string{
@@ -391,84 +427,128 @@ func noRestart(config *docker.HostConfig) {
 	}
 }
 
-func (s *IntegrationTestSuite) initUpgrade() {
+func (s *IntegrationTestSuite) upgrade() {
 	// submit, deposit, and vote for upgrade proposal
-	s.submitProposal(s.chains[0])
-	s.submitProposal(s.chains[1])
+	// prop height = current height + voting period + time it takes to submit proposal + small buffer
+	currentHeightA := s.getCurrentChainHeight(s.valResources[s.chains[0].ChainMeta.Id][0].Container.ID)
+	propHeightA = currentHeightA + int(votingPeriodA) + int(propSubmitBlocks) + int(propBufferBlocks)
+	s.submitProposal(s.chains[0], propHeightA)
 	s.depositProposal(s.chains[0])
-	s.depositProposal(s.chains[1])
 	s.voteProposal(s.chains[0])
+	// prop height = current height + voting period + time it takes to submit proposal + small buffer
+	currentHeightB := s.getCurrentChainHeight(s.valResources[s.chains[1].ChainMeta.Id][0].Container.ID)
+	propHeightB = currentHeightB + int(votingPeriodB) + int(propSubmitBlocks) + int(propBufferBlocks)
+	s.submitProposal(s.chains[1], propHeightB)
+	s.depositProposal(s.chains[1])
 	s.voteProposal(s.chains[1])
 
 	// wait till all chains halt at upgrade height
-	for _, chain := range s.chains {
-		for i := range chain.Validators {
-			s.T().Logf("waiting to reach upgrade height on %s validator container: %s", chain.ChainMeta.Id, s.valResources[chain.ChainMeta.Id][i].Container.ID)
+	for _, c := range s.chains {
+		if c.ChainMeta.Id == chain.ChainAID {
+			propHeight = propHeightA
+		} else {
+			propHeight = propHeightB
+		}
+		for i := range c.Validators {
+			// use counter to ensure no new blocks are being created
+			counter := 0
+			s.T().Logf("waiting to reach upgrade height on %s validator container: %s", s.valResources[c.ChainMeta.Id][i].Container.Name[1:], s.valResources[c.ChainMeta.Id][i].Container.ID)
 			s.Require().Eventually(
 				func() bool {
-					out := s.chainStatus(s.valResources[chain.ChainMeta.Id][i].Container.ID)
-					var syncInfo syncInfo
-					json.Unmarshal(out, &syncInfo)
-					if syncInfo.SyncInfo.LatestHeight != "75" {
-						s.T().Logf("current block height is %v, waiting for block 75 container: %s", syncInfo.SyncInfo.LatestHeight, s.valResources[chain.ChainMeta.Id][i].Container.ID)
+					currentHeight := s.getCurrentChainHeight(s.valResources[c.ChainMeta.Id][i].Container.ID)
+					if currentHeight != propHeight {
+						s.T().Logf("current block height on %s is %v, waiting for block %v container: %s", s.valResources[c.ChainMeta.Id][i].Container.Name[1:], currentHeight, propHeight, s.valResources[c.ChainMeta.Id][i].Container.ID)
+					}
+					if currentHeight > propHeight {
+						panic("chain did not halt at upgrade height")
 					}
-					return syncInfo.SyncInfo.LatestHeight == "75"
+					if currentHeight == propHeight {
+						counter++
+					}
+					return counter == 3
 				},
-				2*time.Minute,
-				5*time.Second,
+				5*time.Minute,
+				time.Second,
 			)
-			s.T().Logf("reached upgrade height on %s validator container: %s", chain.ChainMeta.Id, s.valResources[chain.ChainMeta.Id][i].Container.ID)
+			s.T().Logf("reached upgrade height on %s container: %s", s.valResources[c.ChainMeta.Id][i].Container.Name[1:], s.valResources[c.ChainMeta.Id][i].Container.ID)
 		}
 	}
 
 	// remove all containers so we can upgrade them to the new version
 	for _, chain := range s.chains {
 		for i := range chain.Validators {
-			s.Require().NoError(s.dkrPool.RemoveContainerByName(s.valResources[chain.ChainMeta.Id][i].Container.Name))
+			var opts docker.RemoveContainerOptions
+			opts.ID = s.valResources[chain.ChainMeta.Id][i].Container.ID
+			opts.Force = true
+			s.dkrPool.Client.RemoveContainer(opts)
+			s.T().Logf("removed container: %s", s.valResources[chain.ChainMeta.Id][i].Container.Name[1:])
 		}
 	}
+	s.upgradeContainers(s.chains[0])
+	s.upgradeContainers(s.chains[1])
 }
 
-func (s *IntegrationTestSuite) upgrade() {
+func (s *IntegrationTestSuite) upgradeContainers(c *chain.Chain) {
 	// upgrade containers to the locally compiled daemon
-	for _, chain := range s.chains {
-		s.T().Logf("starting upgrade for chain-id: %s...", chain.ChainMeta.Id)
-		for i, val := range chain.Validators {
-			runOpts := &dockertest.RunOptions{
-				Name:       val.Name,
-				Repository: "osmosis",
-				Tag:        "debug",
-				NetworkID:  s.dkrNet.Network.ID,
-				User:       "root:root",
-				Mounts: []string{
-					fmt.Sprintf("%s/:/osmosis/.osmosisd", val.ConfigDir),
-				},
-			}
-			resource, err := s.dkrPool.RunWithOptions(runOpts, noRestart)
-			s.Require().NoError(err)
-
-			s.valResources[chain.ChainMeta.Id][i] = resource
-			s.T().Logf("started Osmosis %s validator container: %s", chain.ChainMeta.Id, resource.Container.ID)
+	s.T().Logf("starting upgrade for chain-id: %s...", c.ChainMeta.Id)
+	pwd, err := os.Getwd()
+	s.Require().NoError(err)
+	for i, val := range c.Validators {
+		runOpts := &dockertest.RunOptions{
+			Name:       val.Name,
+			Repository: "osmosis",
+			Tag:        "debug",
+			NetworkID:  s.dkrNet.Network.ID,
+			User:       "root:root",
+			Mounts: []string{
+				fmt.Sprintf("%s/:/osmosis/.osmosisd", val.ConfigDir),
+				fmt.Sprintf("%s/scripts:/osmosis", pwd),
+			},
 		}
+		resource, err := s.dkrPool.RunWithOptions(runOpts, noRestart)
+		s.Require().NoError(err)
+
+		s.valResources[c.ChainMeta.Id][i] = resource
+		s.T().Logf("started %s validator container: %s", resource.Container.Name[1:], resource.Container.ID)
 	}
 
-	// check that we are hitting blocks again
-	for _, chain := range s.chains {
-		for i := range chain.Validators {
-			s.Require().Eventually(
-				func() bool {
-					out := s.chainStatus(s.valResources[chain.ChainMeta.Id][i].Container.ID)
-					var syncInfo syncInfo
-					json.Unmarshal(out, &syncInfo)
-					if syncInfo.SyncInfo.LatestHeight <= "75" {
-						fmt.Printf("current block height is %v, waiting to hit blocks\n", syncInfo.SyncInfo.LatestHeight)
-					}
-					return syncInfo.SyncInfo.LatestHeight > "75"
-				},
-				2*time.Minute,
-				5*time.Second,
-			)
-			s.T().Logf("upgrade successful on %s validator container: %s", chain.ChainMeta.Id, s.valResources[chain.ChainMeta.Id][i].Container.ID)
+	// check that we are creating blocks again
+	for i := range c.Validators {
+		if c.ChainMeta.Id == chain.ChainAID {
+			propHeight = propHeightA
+		} else {
+			propHeight = propHeightB
 		}
+		s.Require().Eventually(
+			func() bool {
+				currentHeight := s.getCurrentChainHeight(s.valResources[c.ChainMeta.Id][i].Container.ID)
+				if currentHeight <= propHeight {
+					s.T().Logf("current block height on %s is %v, waiting to create blocks container: %s", s.valResources[c.ChainMeta.Id][i].Container.Name[1:], currentHeight, s.valResources[c.ChainMeta.Id][i].Container.ID)
+				}
+				return currentHeight > propHeight
+			},
+			5*time.Minute,
+			time.Second,
+		)
+		s.T().Logf("upgrade successful on %s validator container: %s", s.valResources[c.ChainMeta.Id][i].Container.Name[1:], s.valResources[c.ChainMeta.Id][i].Container.ID)
 	}
+
+}
+
+func (s *IntegrationTestSuite) createPreUpgradeState() {
+	s.sendIBC(s.chains[0], s.chains[1], s.chains[1].Validators[0].PublicAddress, chain.OsmoToken)
+	s.sendIBC(s.chains[1], s.chains[0], s.chains[0].Validators[0].PublicAddress, chain.OsmoToken)
+	s.sendIBC(s.chains[0], s.chains[1], s.chains[1].Validators[0].PublicAddress, chain.StakeToken)
+	s.sendIBC(s.chains[1], s.chains[0], s.chains[0].Validators[0].PublicAddress, chain.StakeToken)
+	s.createPool(s.chains[0], "pool1A.json")
+	s.createPool(s.chains[1], "pool1B.json")
+}
+
+func (s *IntegrationTestSuite) runPostUpgradeTests() {
+	s.sendIBC(s.chains[0], s.chains[1], s.chains[1].Validators[0].PublicAddress, chain.OsmoToken)
+	s.sendIBC(s.chains[1], s.chains[0], s.chains[0].Validators[0].PublicAddress, chain.OsmoToken)
+	s.sendIBC(s.chains[0], s.chains[1], s.chains[1].Validators[0].PublicAddress, chain.StakeToken)
+	s.sendIBC(s.chains[1], s.chains[0], s.chains[0].Validators[0].PublicAddress, chain.StakeToken)
+	s.createPool(s.chains[0], "pool2A.json")
+	s.createPool(s.chains[1], "pool2B.json")
 }