fix: allow retries for messages signed by relayer.

e2e/testsuite/testsuite.go

@@ -60,6 +60,7 @@ type E2ETestSuite struct {
 
 	grpcClients    map[string]GRPCClients
 	paths          map[string]path
+	relayers       relayerMap
 	logger         *zap.Logger
 	DockerClient   *dockerclient.Client
 	network        string
@@ -104,6 +105,25 @@ func newPath(chainA, chainB *cosmos.CosmosChain) path {
 	}
 }
 
+// relayerMap is a mapping from test names to a relayer set for that test.
+type relayerMap map[string]map[ibc.Wallet]bool
+
+// addRelayer adds the given relayer to the relayer set for the given test name.
+func (r relayerMap) addRelayer(testName string, relayer ibc.Wallet) {
+	if _, ok := r[testName]; !ok {
+		r[testName] = make(map[ibc.Wallet]bool)
+	}
+	r[testName][relayer] = true
+}
+
+// containsRelayer returns true if the given relayer is in the relayer set for the given test name.
+func (r relayerMap) containsRelayer(testName string, wallet ibc.Wallet) bool {
+	if relayerSet, ok := r[testName]; ok {
+		return relayerSet[wallet]
+	}
+	return false
+}
+
 // GetRelayerUsers returns two ibc.Wallet instances which can be used for the relayer users
 // on the two chains.
 func (s *E2ETestSuite) GetRelayerUsers(ctx context.Context, chainOpts ...testconfig.ChainOptionConfiguration) (ibc.Wallet, ibc.Wallet) {
@@ -117,6 +137,12 @@ func (s *E2ETestSuite) GetRelayerUsers(ctx context.Context, chainOpts ...testcon
 	chainARelayerUser := cosmos.NewWallet(ChainARelayerName, chainAAccountBytes, "", chainA.Config())
 	chainBRelayerUser := cosmos.NewWallet(ChainBRelayerName, chainBAccountBytes, "", chainB.Config())
 
+	if s.relayers == nil {
+		s.relayers = make(relayerMap)
+	}
+	s.relayers.addRelayer(s.T().Name(), chainARelayerUser)
+	s.relayers.addRelayer(s.T().Name(), chainBRelayerUser)
+
 	return chainARelayerUser, chainBRelayerUser
 }
 
@@ -281,7 +307,18 @@ func (s *E2ETestSuite) BroadcastMessages(ctx context.Context, chain *cosmos.Cosm
 		return factory.WithGas(DefaultGasValue)
 	})
 
-	resp, err := cosmos.BroadcastTx(ctx, broadcaster, user, msgs...)
+	// Retry the operation a few times if the user signing the transaction is a relayer. (See issue #3264)
+	var resp sdk.TxResponse
+	var err error
+	broadcastFunc := func() (sdk.TxResponse, error) {
+		return cosmos.BroadcastTx(ctx, broadcaster, user, msgs...)
+	}
+	if s.relayers.containsRelayer(s.T().Name(), user) {
+		// Retry five times, the value of 5 chosen is arbitrary.
+		resp, err = s.retryNtimes(broadcastFunc, 5)
+	} else {
+		resp, err = broadcastFunc()
+	}
 	if err != nil {
 		return sdk.TxResponse{}, err
 	}
@@ -611,6 +648,36 @@ func (s *E2ETestSuite) QueryGranterGrants(ctx context.Context, chain *cosmos.Cos
 	return grants.Grants, nil
 }
 
+// retryNtimes retries the provided function up to the provided number of attempts.
+func (s *E2ETestSuite) retryNtimes(f func() (sdk.TxResponse, error), attempts int) (sdk.TxResponse, error) {
+	// Ignore account sequence mismatch errors.
+	retryMessages := []string{"account sequence mismatch"}
+	var resp sdk.TxResponse
+	var err error
+	// If the response's raw log doesn't contain any of the allowed prefixes we return, else, we retry.
+	for i := 0; i < attempts; i++ {
+		resp, err = f()
+		if err != nil {
+			return resp, err
+		}
+		if !containsMessage(resp.RawLog, retryMessages) {
+			return sdk.TxResponse{}, err
+		}
+		s.T().Logf("retrying tx due to non deterministic failure: %+v", resp)
+	}
+	return resp, err
+}
+
+// containsMessages returns true if the string s contains any of the messages in the slice.
+func containsMessage(s string, messages []string) bool {
+	for _, message := range messages {
+		if strings.Contains(s, message) {
+			return true
+		}
+	}
+	return false
+}
+
 // GetIBCToken returns the denomination of the full token denom sent to the receiving channel
 func GetIBCToken(fullTokenDenom string, portID, channelID string) transfertypes.DenomTrace {
 	return transfertypes.ParseDenomTrace(fmt.Sprintf("%s/%s/%s", portID, channelID, fullTokenDenom))