fix[e2e]: e2e failure fixes (#5964)

* prevent concurrent map write error e2e

* propper mutex

* add a retry loop to starting a node

* retry node if fails to start

* update changelog

---------

Co-authored-by: devbot-wizard <[email protected]>

CHANGELOG.md

@@ -71,6 +71,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * [#5923] (https://github.com/osmosis-labs/osmosis/pull/5923) CL: Lower gas for initializing ticks
 * [#5927] (https://github.com/osmosis-labs/osmosis/pull/5927) Add gas metering to x/tokenfactory trackBeforeSend hook
 * [#5890](https://github.com/osmosis-labs/osmosis/pull/5890) feat: CreateCLPool & LinkCFMMtoCL pool into one gov-prop
+* [#5964](https://github.com/osmosis-labs/osmosis/pull/5964) fix e2e test concurrency bugs
 
 ### Minor improvements & Bug Fixes
 

tests/e2e/configurer/chain/node.go

@@ -47,35 +47,63 @@ func NewNodeConfig(t *testing.T, initNode *initialization.Node, initConfig *init
 // The node configuration must be already added to the chain config prior to calling this
 // method.
 func (n *NodeConfig) Run() error {
-	n.t.Logf("starting node container: %s", n.Name)
-	resource, err := n.containerManager.RunNodeResource(n.chainId, n.Name, n.ConfigDir)
-	if err != nil {
-		return err
-	}
+	maxRetries := 3
+	currentRetry := 0
+
+	for currentRetry < maxRetries {
+		n.t.Logf("starting node container: %s", n.Name)
+		resource, err := n.containerManager.RunNodeResource(n.chainId, n.Name, n.ConfigDir)
+		if err != nil {
+			return err
+		}
 
-	hostPort := resource.GetHostPort("26657/tcp")
-	rpcClient, err := rpchttp.New("tcp://"+hostPort, "/websocket")
-	if err != nil {
-		return err
-	}
+		hostPort := resource.GetHostPort("26657/tcp")
+		rpcClient, err := rpchttp.New("tcp://"+hostPort, "/websocket")
+		if err != nil {
+			return err
+		}
 
-	n.rpcClient = rpcClient
+		n.rpcClient = rpcClient
+
+		success := false
+		timeout := time.After(time.Second * 20)
+		ticker := time.NewTicker(10 * time.Millisecond)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-timeout:
+				n.t.Logf("Osmosis node failed to produce blocks")
+				break
+			case <-ticker.C:
+				_, err := n.QueryCurrentHeight()
+				if err == nil {
+					n.t.Logf("started node container: %s", n.Name)
+					success = true
+					break
+				}
+			}
+
+			if success {
+				break
+			}
+		}
 
-	require.Eventually(
-		n.t,
-		func() bool {
-			// This fails if unsuccessful.
-			_, err := n.QueryCurrentHeight()
+		if success {
+			break
+		} else {
+			n.t.Logf("failed to start node container, retrying... (%d/%d)", currentRetry+1, maxRetries)
+			err := n.containerManager.RemoveNodeResource(n.Name)
 			if err != nil {
-				return false
+				return err
 			}
-			n.t.Logf("started node container: %s", n.Name)
-			return true
-		},
-		time.Minute,
-		10*time.Millisecond,
-		"Osmosis node failed to produce blocks",
-	)
+			currentRetry++
+		}
+	}
+
+	if currentRetry >= maxRetries {
+		return fmt.Errorf("failed to start node container after %d retries", maxRetries)
+	}
 
 	if err := n.extractOperatorAddressIfValidator(); err != nil {
 		return err

tests/e2e/containers/containers.go

@@ -8,6 +8,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -49,6 +50,7 @@ type Manager struct {
 	pool              *dockertest.Pool
 	network           *dockertest.Network
 	resources         map[string]*dockertest.Resource
+	resourcesMutex    sync.RWMutex
 	isDebugLogEnabled bool
 }
 
@@ -297,7 +299,9 @@ func (m *Manager) RunNodeResource(chainId string, containerName, valCondifDir st
 		return nil, err
 	}
 
+	m.resourcesMutex.Lock()
 	m.resources[containerName] = resource
+	m.resourcesMutex.Unlock()
 
 	return resource, nil
 }