Prevent pollution between scenario tests (#11850)

Co-authored-by: Kasey Kirkham <[email protected]>
Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com>

testing/endtoend/endtoend_test.go

@@ -169,15 +169,15 @@ func (r *testRunner) waitForChainStart() {
 }
 
 // runEvaluators executes assigned evaluators.
-func (r *testRunner) runEvaluators(conns []*grpc.ClientConn, tickingStartTime time.Time) error {
+func (r *testRunner) runEvaluators(ec *e2etypes.EvaluationContext, conns []*grpc.ClientConn, tickingStartTime time.Time) error {
 	t, config := r.t, r.config
 	secondsPerEpoch := uint64(params.BeaconConfig().SlotsPerEpoch.Mul(params.BeaconConfig().SecondsPerSlot))
 	ticker := helpers.NewEpochTicker(tickingStartTime, secondsPerEpoch)
 	for currentEpoch := range ticker.C() {
-		if config.EvalInterceptor(currentEpoch, conns) {
+		if config.EvalInterceptor(ec, currentEpoch, conns) {
 			continue
 		}
-		r.executeProvidedEvaluators(currentEpoch, conns, config.Evaluators)
+		r.executeProvidedEvaluators(ec, currentEpoch, conns, config.Evaluators)
 
 		if t.Failed() || currentEpoch >= config.EpochsToRun-1 {
 			ticker.Done()
@@ -493,8 +493,9 @@ func (r *testRunner) defaultEndToEndRun() error {
 	require.NoError(t, err)
 	tickingStartTime := helpers.EpochTickerStartTime(genesis)
 
+	ec := e2etypes.NewEvaluationContext(r.depositor.History())
 	// Run assigned evaluators.
-	if err := r.runEvaluators(conns, tickingStartTime); err != nil {
+	if err := r.runEvaluators(ec, conns, tickingStartTime); err != nil {
 		return errors.Wrap(err, "one or more evaluators failed")
 	}
 
@@ -574,14 +575,16 @@ func (r *testRunner) scenarioRun() error {
 	require.NoError(t, err)
 	tickingStartTime := helpers.EpochTickerStartTime(genesis)
 
+	ec := e2etypes.NewEvaluationContext(r.depositor.History())
 	// Run assigned evaluators.
-	return r.runEvaluators(conns, tickingStartTime)
+	return r.runEvaluators(ec, conns, tickingStartTime)
 }
+
 func (r *testRunner) addEvent(ev func() error) {
 	r.comHandler.group.Go(ev)
 }
 
-func (r *testRunner) executeProvidedEvaluators(currentEpoch uint64, conns []*grpc.ClientConn, evals []e2etypes.Evaluator) {
+func (r *testRunner) executeProvidedEvaluators(ec *e2etypes.EvaluationContext, currentEpoch uint64, conns []*grpc.ClientConn, evals []e2etypes.Evaluator) {
 	wg := new(sync.WaitGroup)
 	for _, eval := range evals {
 		// Fix reference to evaluator as it will be running
@@ -592,7 +595,6 @@ func (r *testRunner) executeProvidedEvaluators(currentEpoch uint64, conns []*grp
 			continue
 		}
 		wg.Add(1)
-		var ec e2etypes.EvaluationContext = r.depositor.History()
 		go r.t.Run(fmt.Sprintf(evaluator.Name, currentEpoch), func(t *testing.T) {
 			err := evaluator.Evaluation(ec, conns...)
 			assert.NoError(t, err, "Evaluation failed for epoch %d: %v", currentEpoch, err)
@@ -611,7 +613,7 @@ func (r *testRunner) executeProvidedEvaluators(currentEpoch uint64, conns []*grp
 // After the proxy has been sending `SYNCING` responses to the beacon node, we
 // will test this with our optimistic sync evaluator to ensure everything works
 // as expected.
-func (r *testRunner) multiScenarioMulticlient(epoch uint64, conns []*grpc.ClientConn) bool {
+func (r *testRunner) multiScenarioMulticlient(ec *e2etypes.EvaluationContext, epoch uint64, conns []*grpc.ClientConn) bool {
 	type ForkchoiceUpdatedResponse struct {
 		Status    *enginev1.PayloadStatus  `json:"payloadStatus"`
 		PayloadId *enginev1.PayloadIDBytes `json:"payloadId"`
@@ -662,9 +664,8 @@ func (r *testRunner) multiScenarioMulticlient(epoch uint64, conns []*grpc.Client
 		})
 		return true
 	case 15:
-		r.executeProvidedEvaluators(epoch, []*grpc.ClientConn{conns[0]}, []e2etypes.Evaluator{
-			ev.OptimisticSyncEnabled,
-		})
+		evs := []e2etypes.Evaluator{ev.OptimisticSyncEnabled}
+		r.executeProvidedEvaluators(ec, epoch, []*grpc.ClientConn{conns[0]}, evs)
 		// Disable Interceptor
 		component, err := r.comHandler.eth1Proxy.ComponentAtIndex(0)
 		require.NoError(r.t, err)
@@ -690,7 +691,7 @@ func (r *testRunner) multiScenarioMulticlient(epoch uint64, conns []*grpc.Client
 	return false
 }
 
-func (r *testRunner) eeOffline(epoch uint64, _ []*grpc.ClientConn) bool {
+func (r *testRunner) eeOffline(_ *e2etypes.EvaluationContext, epoch uint64, _ []*grpc.ClientConn) bool {
 	switch epoch {
 	case 9:
 		require.NoError(r.t, r.comHandler.eth1Miner.Pause())
@@ -716,7 +717,7 @@ func (r *testRunner) eeOffline(epoch uint64, _ []*grpc.ClientConn) bool {
 // After the proxy has been sending `SYNCING` responses to the beacon node, we
 // will test this with our optimistic sync evaluator to ensure everything works
 // as expected.
-func (r *testRunner) multiScenario(epoch uint64, conns []*grpc.ClientConn) bool {
+func (r *testRunner) multiScenario(ec *e2etypes.EvaluationContext, epoch uint64, conns []*grpc.ClientConn) bool {
 	switch epoch {
 	case 9:
 		require.NoError(r.t, r.comHandler.beaconNodes.PauseAtIndex(0))
@@ -747,9 +748,8 @@ func (r *testRunner) multiScenario(epoch uint64, conns []*grpc.ClientConn) bool
 		})
 		return true
 	case 20:
-		r.executeProvidedEvaluators(epoch, []*grpc.ClientConn{conns[0]}, []e2etypes.Evaluator{
-			ev.OptimisticSyncEnabled,
-		})
+		evs := []e2etypes.Evaluator{ev.OptimisticSyncEnabled}
+		r.executeProvidedEvaluators(ec, epoch, []*grpc.ClientConn{conns[0]}, evs)
 		// Disable Interceptor
 		component, err := r.comHandler.eth1Proxy.ComponentAtIndex(0)
 		require.NoError(r.t, err)
@@ -767,6 +767,6 @@ func (r *testRunner) multiScenario(epoch uint64, conns []*grpc.ClientConn) bool
 }
 
 // All Epochs are valid.
-func defaultInterceptor(_ uint64, _ []*grpc.ClientConn) bool {
+func defaultInterceptor(_ *e2etypes.EvaluationContext, _ uint64, _ []*grpc.ClientConn) bool {
 	return false
 }

testing/endtoend/evaluators/api_gateway_v1alpha1.go

@@ -33,7 +33,7 @@ const (
 
 type apiComparisonFunc func(beaconNodeIdx int, conn *grpc.ClientConn) error
 
-func apiGatewayV1Alpha1Verify(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+func apiGatewayV1Alpha1Verify(_ *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	for beaconNodeIdx, conn := range conns {
 		if err := runAPIComparisonFunctions(
 			beaconNodeIdx,

testing/endtoend/evaluators/api_middleware.go

@@ -51,7 +51,7 @@ const (
 	v1MiddlewarePathTemplate = "http://localhost:%d/eth/v1"
 )
 
-func apiMiddlewareVerify(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+func apiMiddlewareVerify(_ *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	for beaconNodeIdx, conn := range conns {
 		if err := runAPIComparisonFunctions(
 			beaconNodeIdx,

testing/endtoend/evaluators/beaconapi_evaluators/beacon_api_verify.go

@@ -20,7 +20,7 @@ const (
 
 type apiComparisonFunc func(beaconNodeIdx int, conn *grpc.ClientConn) error
 
-func beaconAPIVerify(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+func beaconAPIVerify(_ *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	beacon := []apiComparisonFunc{
 		withCompareBeaconAPIs,
 	}

testing/endtoend/evaluators/data.go

@@ -22,7 +22,7 @@ var ColdStateCheckpoint = e2etypes.Evaluator{
 }
 
 // Checks the first node for an old checkpoint using cold state storage.
-func checkColdStateCheckpoint(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+func checkColdStateCheckpoint(_ *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	ctx := context.Background()
 	client := eth.NewBeaconChainClient(conns[0])
 

testing/endtoend/evaluators/execution_engine.go

@@ -22,7 +22,7 @@ var OptimisticSyncEnabled = types.Evaluator{
 	Evaluation: optimisticSyncEnabled,
 }
 
-func optimisticSyncEnabled(_ types.EvaluationContext, conns ...*grpc.ClientConn) error {
+func optimisticSyncEnabled(_ *types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	for _, conn := range conns {
 		client := service.NewBeaconChainClient(conn)
 		head, err := client.GetBlockV2(context.Background(), &v2.BlockRequestV2{BlockId: []byte("head")})

testing/endtoend/evaluators/fee_recipient.go

@@ -62,7 +62,7 @@ func valKeyMap() (map[string]bool, error) {
 	return km, nil
 }
 
-func feeRecipientIsPresent(_ types.EvaluationContext, conns ...*grpc.ClientConn) error {
+func feeRecipientIsPresent(_ *types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconChainClient(conn)
 	chainHead, err := client.GetChainHead(context.Background(), &emptypb.Empty{})

testing/endtoend/evaluators/finality.go

@@ -23,7 +23,7 @@ var FinalizationOccurs = func(epoch ethtypes.Epoch) types.Evaluator {
 	}
 }
 
-func finalizationOccurs(_ types.EvaluationContext, conns ...*grpc.ClientConn) error {
+func finalizationOccurs(_ *types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := eth.NewBeaconChainClient(conn)
 	chainHead, err := client.GetChainHead(context.Background(), &emptypb.Empty{})

testing/endtoend/evaluators/fork.go

@@ -40,7 +40,7 @@ var BellatrixForkTransition = types.Evaluator{
 	Evaluation: bellatrixForkOccurs,
 }
 
-func altairForkOccurs(_ types.EvaluationContext, conns ...*grpc.ClientConn) error {
+func altairForkOccurs(_ *types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconNodeValidatorClient(conn)
 	ctx, cancel := context.WithTimeout(context.Background(), streamDeadline)
@@ -82,7 +82,7 @@ func altairForkOccurs(_ types.EvaluationContext, conns ...*grpc.ClientConn) erro
 	return nil
 }
 
-func bellatrixForkOccurs(_ types.EvaluationContext, conns ...*grpc.ClientConn) error {
+func bellatrixForkOccurs(_ *types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconNodeValidatorClient(conn)
 	ctx, cancel := context.WithTimeout(context.Background(), streamDeadline)

testing/endtoend/evaluators/metrics.go

@@ -85,7 +85,7 @@ var metricComparisonTests = []comparisonTest{
 	},
 }
 
-func metricsTest(_ types.EvaluationContext, conns ...*grpc.ClientConn) error {
+func metricsTest(_ *types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	genesis, err := eth.NewNodeClient(conns[0]).GetGenesis(context.Background(), &emptypb.Empty{})
 	if err != nil {
 		return err

testing/endtoend/evaluators/node.go

@@ -52,7 +52,7 @@ var AllNodesHaveSameHead = e2etypes.Evaluator{
 	Evaluation: allNodesHaveSameHead,
 }
 
-func healthzCheck(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+func healthzCheck(_ *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	count := len(conns)
 	for i := 0; i < count; i++ {
 		resp, err := http.Get(fmt.Sprintf("http://localhost:%d/healthz", e2e.TestParams.Ports.PrysmBeaconNodeMetricsPort+i))
@@ -94,7 +94,7 @@ func healthzCheck(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error
 	return nil
 }
 
-func peersConnect(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+func peersConnect(_ *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	if len(conns) == 1 {
 		return nil
 	}
@@ -114,7 +114,7 @@ func peersConnect(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error
 	return nil
 }
 
-func finishedSyncing(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+func finishedSyncing(_ *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	syncNodeClient := eth.NewNodeClient(conn)
 	syncStatus, err := syncNodeClient.GetSyncStatus(context.Background(), &emptypb.Empty{})
@@ -127,7 +127,7 @@ func finishedSyncing(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) er
 	return nil
 }
 
-func allNodesHaveSameHead(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+func allNodesHaveSameHead(_ *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	headEpochs := make([]types.Epoch, len(conns))
 	justifiedRoots := make([][]byte, len(conns))
 	prevJustifiedRoots := make([][]byte, len(conns))

testing/endtoend/evaluators/operations.go

@@ -25,8 +25,6 @@ import (
 	"google.golang.org/protobuf/types/known/emptypb"
 )
 
-var exitedVals = make(map[[48]byte]bool)
-
 // churnLimit is normally 4 unless the validator set is extremely large.
 var churnLimit = 4
 var depositValCount = e2e.DepositCount
@@ -97,7 +95,7 @@ func (m mismatch) String() string {
 	return fmt.Sprintf("(%#x:%d:%d)", m.k, m.e, m.o)
 }
 
-func processesDepositsInBlocks(ec e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+func processesDepositsInBlocks(ec *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	expected := ec.Balances(e2etypes.PostGenesisDepositBatch)
 	conn := conns[0]
 	client := ethpb.NewBeaconChainClient(conn)
@@ -138,7 +136,7 @@ func processesDepositsInBlocks(ec e2etypes.EvaluationContext, conns ...*grpc.Cli
 	return nil
 }
 
-func verifyGraffitiInBlocks(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+func verifyGraffitiInBlocks(_ *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconChainClient(conn)
 	chainHead, err := client.GetChainHead(context.Background(), &emptypb.Empty{})
@@ -177,7 +175,7 @@ func verifyGraffitiInBlocks(_ e2etypes.EvaluationContext, conns ...*grpc.ClientC
 	return nil
 }
 
-func activatesDepositedValidators(ec e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+func activatesDepositedValidators(ec *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconChainClient(conn)
 
@@ -258,7 +256,7 @@ func getAllValidators(c ethpb.BeaconChainClient) ([]*ethpb.Validator, error) {
 	return vals, nil
 }
 
-func depositedValidatorsAreActive(ec e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+func depositedValidatorsAreActive(ec *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconChainClient(conn)
 
@@ -282,7 +280,7 @@ func depositedValidatorsAreActive(ec e2etypes.EvaluationContext, conns ...*grpc.
 			continue // we aren't checking for this validator
 		}
 		// ignore voluntary exits when checking balance and active status
-		exited := exitedVals[key]
+		exited := ec.ExitedVals[key]
 		if exited {
 			nexits++
 			delete(expected, key)
@@ -310,7 +308,7 @@ func depositedValidatorsAreActive(ec e2etypes.EvaluationContext, conns ...*grpc.
 	return nil
 }
 
-func proposeVoluntaryExit(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+func proposeVoluntaryExit(ec *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	valClient := ethpb.NewBeaconNodeValidatorClient(conn)
 	beaconClient := ethpb.NewBeaconChainClient(conn)
@@ -354,15 +352,15 @@ func proposeVoluntaryExit(_ e2etypes.EvaluationContext, conns ...*grpc.ClientCon
 		return errors.Wrap(err, "could not propose exit")
 	}
 	pubk := bytesutil.ToBytes48(deposits[exitedIndex].Data.PublicKey)
-	exitedVals[pubk] = true
+	ec.ExitedVals[pubk] = true
 
 	return nil
 }
 
-func validatorsHaveExited(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+func validatorsHaveExited(ec *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconChainClient(conn)
-	for k := range exitedVals {
+	for k := range ec.ExitedVals {
 		validatorRequest := &ethpb.GetValidatorRequest{
 			QueryFilter: &ethpb.GetValidatorRequest_PublicKey{
 				PublicKey: k[:],
@@ -379,7 +377,7 @@ func validatorsHaveExited(_ e2etypes.EvaluationContext, conns ...*grpc.ClientCon
 	return nil
 }
 
-func validatorsVoteWithTheMajority(_ e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
+func validatorsVoteWithTheMajority(ec *e2etypes.EvaluationContext, conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	client := ethpb.NewBeaconChainClient(conn)
 	chainHead, err := client.GetChainHead(context.Background(), &emptypb.Empty{})
@@ -422,7 +420,7 @@ func validatorsVoteWithTheMajority(_ e2etypes.EvaluationContext, conns ...*grpc.
 		default:
 			return errors.New("block neither phase0,altair or bellatrix")
 		}
-		seenVotes[slot] = vote
+		ec.SeenVotes[slot] = vote
 
 		// We treat epoch 1 differently from other epoch for two reasons:
 		// - this evaluator is not executed for epoch 0 so we have to calculate the first slot differently
@@ -438,25 +436,22 @@ func validatorsVoteWithTheMajority(_ e2etypes.EvaluationContext, conns ...*grpc.
 			isFirstSlotInVotingPeriod = slot%slotsPerVotingPeriod == 0
 		}
 		if isFirstSlotInVotingPeriod {
-			expectedEth1DataVote = vote
+			ec.ExpectedEth1DataVote = vote
 			return nil
 		}
 
-		if !bytes.Equal(vote, expectedEth1DataVote) {
+		if !bytes.Equal(vote, ec.ExpectedEth1DataVote) {
 			for i := types.Slot(0); i < slot; i++ {
-				v, ok := seenVotes[i]
+				v, ok := ec.SeenVotes[i]
 				if ok {
 					fmt.Printf("vote at slot=%d = %#x\n", i, v)
 				} else {
 					fmt.Printf("did not see slot=%d\n", i)
 				}
 			}
 			return fmt.Errorf("incorrect eth1data vote for slot %d; expected: %#x vs voted: %#x",
-				slot, expectedEth1DataVote, vote)
+				slot, ec.ExpectedEth1DataVote, vote)
 		}
 	}
 	return nil
 }
-
-var seenVotes = make(map[types.Slot][]byte)
-var expectedEth1DataVote []byte

testing/endtoend/evaluators/peers.go

@@ -19,7 +19,7 @@ var PeersCheck = types.Evaluator{
 	Evaluation: peersTest,
 }
 
-func peersTest(_ types.EvaluationContext, conns ...*grpc.ClientConn) error {
+func peersTest(_ *types.EvaluationContext, conns ...*grpc.ClientConn) error {
 	debugClient := eth.NewDebugClient(conns[0])
 
 	peerResponses, err := debugClient.ListPeers(context.Background(), &emptypb.Empty{})