Add historical summaries to state and processing

WORKSPACE

@@ -188,7 +188,7 @@ filegroup(
     url = "https://github.com/eth-clients/slashing-protection-interchange-tests/archive/b8413ca42dc92308019d0d4db52c87e9e125c4e9.tar.gz",
 )
 
-consensus_spec_version = "v1.3.0-alpha.2"
+consensus_spec_version = "v1.3.0-rc.0"
 
 bls_test_version = "v0.1.1"
 
@@ -204,7 +204,7 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "a92c41058dc17ced811cc85570cd6f8af761aedfcbd2dd7dd4fb64ac961d76f9",
+    sha256 = "44c39a68242a4b731865040ea3009ea1a695ee3377d2891cecf8e6a721f0102b",
     url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/general.tar.gz" % consensus_spec_version,
 )
 
@@ -220,7 +220,7 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "49a7944da92429ac8f41347f19837762247cdbf00e628c285d1b826e58e4096d",
+    sha256 = "c9aaf6f7c1ca1264ec9a45872d5058f3377ebdd9dbcb987796f7302d86f98934",
     url = "https://github.com/ethereum/consensus-spec-tests/releases/download/%s/minimal.tar.gz" % consensus_spec_version,
 )
 
@@ -251,7 +251,7 @@ filegroup(
     visibility = ["//visibility:public"],
 )
     """,
-    sha256 = "a72b7457c403f6b76567d4d7bec19d01bedf7d5ef1d6f2c3a9e09ee86d401a14",
+    sha256 = "531eff00498acf5965d8890d83e3561a64f7b23a75d7f312915291482da2bf52",
     strip_prefix = "consensus-specs-" + consensus_spec_version[1:],
     url = "https://github.com/ethereum/consensus-specs/archive/refs/tags/%s.tar.gz" % consensus_spec_version,
 )

beacon-chain/core/altair/transition.go

@@ -93,7 +93,7 @@ func ProcessEpoch(ctx context.Context, state state.BeaconState) (state.BeaconSta
 	if err != nil {
 		return nil, err
 	}
-	state, err = e.ProcessHistoricalRootsUpdate(state)
+	state, err = e.ProcessHistoricalDataUpdate(state)
 	if err != nil {
 		return nil, err
 	}

beacon-chain/core/epoch/BUILD.bazel

@@ -13,11 +13,14 @@ go_library(
         "//beacon-chain/core/time:go_default_library",
         "//beacon-chain/core/validators:go_default_library",
         "//beacon-chain/state:go_default_library",
+        "//beacon-chain/state/stateutil:go_default_library",
+        "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
         "//consensus-types/primitives:go_default_library",
         "//math:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//proto/prysm/v1alpha1/attestation:go_default_library",
+        "//runtime/version:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
     ],
 )
@@ -33,8 +36,10 @@ go_test(
     deps = [
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/core/time:go_default_library",
+        "//beacon-chain/core/transition:go_default_library",
         "//beacon-chain/state:go_default_library",
         "//beacon-chain/state/state-native:go_default_library",
+        "//beacon-chain/state/stateutil:go_default_library",
         "//config/fieldparams:go_default_library",
         "//config/params:go_default_library",
         "//consensus-types/primitives:go_default_library",

beacon-chain/core/epoch/epoch_processing.go

@@ -14,11 +14,14 @@ import (
 	"github.com/prysmaticlabs/prysm/v3/beacon-chain/core/time"
 	"github.com/prysmaticlabs/prysm/v3/beacon-chain/core/validators"
 	"github.com/prysmaticlabs/prysm/v3/beacon-chain/state"
+	"github.com/prysmaticlabs/prysm/v3/beacon-chain/state/stateutil"
+	fieldparams "github.com/prysmaticlabs/prysm/v3/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v3/config/params"
 	types "github.com/prysmaticlabs/prysm/v3/consensus-types/primitives"
 	"github.com/prysmaticlabs/prysm/v3/math"
 	ethpb "github.com/prysmaticlabs/prysm/v3/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v3/proto/prysm/v1alpha1/attestation"
+	"github.com/prysmaticlabs/prysm/v3/runtime/version"
 )
 
 // sortableIndices implements the Sort interface to sort newly activated validator indices
@@ -349,33 +352,39 @@ func ProcessRandaoMixesReset(state state.BeaconState) (state.BeaconState, error)
 	return state, nil
 }
 
-// ProcessHistoricalRootsUpdate processes the updates to historical root accumulator during epoch processing.
-//
-// Spec pseudocode definition:
-//
-//	def process_historical_roots_update(state: BeaconState) -> None:
-//	  # Set historical root accumulator
-//	  next_epoch = Epoch(get_current_epoch(state) + 1)
-//	  if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0:
-//	      historical_batch = HistoricalBatch(block_roots=state.block_roots, state_roots=state.state_roots)
-//	      state.historical_roots.append(hash_tree_root(historical_batch))
-func ProcessHistoricalRootsUpdate(state state.BeaconState) (state.BeaconState, error) {
+// ProcessHistoricalDataUpdate processes the updates to historical data during epoch processing.
+// For Capella state, per spec, historical summaries is updated instead of historical roots.
+func ProcessHistoricalDataUpdate(state state.BeaconState) (state.BeaconState, error) {
 	currentEpoch := time.CurrentEpoch(state)
 	nextEpoch := currentEpoch + 1
 
 	// Set historical root accumulator.
 	epochsPerHistoricalRoot := params.BeaconConfig().SlotsPerHistoricalRoot.DivSlot(params.BeaconConfig().SlotsPerEpoch)
 	if nextEpoch.Mod(uint64(epochsPerHistoricalRoot)) == 0 {
-		historicalBatch := &ethpb.HistoricalBatch{
-			BlockRoots: state.BlockRoots(),
-			StateRoots: state.StateRoots(),
-		}
-		batchRoot, err := historicalBatch.HashTreeRoot()
-		if err != nil {
-			return nil, errors.Wrap(err, "could not hash historical batch")
-		}
-		if err := state.AppendHistoricalRoots(batchRoot); err != nil {
-			return nil, err
+		if state.Version() >= version.Capella {
+			br, err := stateutil.ArraysRoot(state.BlockRoots(), fieldparams.BlockRootsLength)
+			if err != nil {
+				return nil, err
+			}
+			sr, err := stateutil.ArraysRoot(state.StateRoots(), fieldparams.StateRootsLength)
+			if err != nil {
+				return nil, err
+			}
+			if err := state.AppendHistoricalSummariesUpdate(&ethpb.HistoricalSummary{BlockSummaryRoot: br[:], StateSummaryRoot: sr[:]}); err != nil {
+				return nil, err
+			}
+		} else {
+			historicalBatch := &ethpb.HistoricalBatch{
+				BlockRoots: state.BlockRoots(),
+				StateRoots: state.StateRoots(),
+			}
+			batchRoot, err := historicalBatch.HashTreeRoot()
+			if err != nil {
+				return nil, errors.Wrap(err, "could not hash historical batch")
+			}
+			if err := state.AppendHistoricalRoots(batchRoot); err != nil {
+				return nil, err
+			}
 		}
 	}
 
@@ -426,7 +435,7 @@ func ProcessFinalUpdates(state state.BeaconState) (state.BeaconState, error) {
 	}
 
 	// Set historical root accumulator.
-	state, err = ProcessHistoricalRootsUpdate(state)
+	state, err = ProcessHistoricalDataUpdate(state)
 	if err != nil {
 		return nil, err
 	}

beacon-chain/core/epoch/epoch_processing_test.go

@@ -10,8 +10,10 @@ import (
 	"github.com/prysmaticlabs/prysm/v3/beacon-chain/core/epoch"
 	"github.com/prysmaticlabs/prysm/v3/beacon-chain/core/helpers"
 	"github.com/prysmaticlabs/prysm/v3/beacon-chain/core/time"
+	"github.com/prysmaticlabs/prysm/v3/beacon-chain/core/transition"
 	"github.com/prysmaticlabs/prysm/v3/beacon-chain/state"
 	state_native "github.com/prysmaticlabs/prysm/v3/beacon-chain/state/state-native"
+	"github.com/prysmaticlabs/prysm/v3/beacon-chain/state/stateutil"
 	fieldparams "github.com/prysmaticlabs/prysm/v3/config/fieldparams"
 	"github.com/prysmaticlabs/prysm/v3/config/params"
 	types "github.com/prysmaticlabs/prysm/v3/consensus-types/primitives"
@@ -455,3 +457,71 @@ func TestProcessSlashings_BadValue(t *testing.T) {
 	_, err = epoch.ProcessSlashings(s, params.BeaconConfig().ProportionalSlashingMultiplier)
 	require.ErrorContains(t, "addition overflows", err)
 }
+
+func TestProcessHistoricalRootsUpdate(t *testing.T) {
+	tests := []struct {
+		name     string
+		st       func() state.BeaconState
+		verifier func(state.BeaconState)
+	}{
+		{
+			name: "no change",
+			st: func() state.BeaconState {
+				st, _ := util.DeterministicGenesisState(t, 1)
+				return st
+			},
+			verifier: func(st state.BeaconState) {
+				require.Equal(t, 0, len(st.HistoricalRoots()))
+			},
+		},
+		{
+			name: "before capella can process and get historical root",
+			st: func() state.BeaconState {
+				st, _ := util.DeterministicGenesisState(t, 1)
+				st, err := transition.ProcessSlots(context.Background(), st, params.BeaconConfig().SlotsPerHistoricalRoot-1)
+				require.NoError(t, err)
+				return st
+			},
+			verifier: func(st state.BeaconState) {
+				require.Equal(t, 1, len(st.HistoricalRoots()))
+
+				b := &ethpb.HistoricalBatch{
+					BlockRoots: st.BlockRoots(),
+					StateRoots: st.StateRoots(),
+				}
+				r, err := b.HashTreeRoot()
+				require.NoError(t, err)
+				require.DeepEqual(t, r[:], st.HistoricalRoots()[0])
+			},
+		},
+		{
+			name: "after capella can process and get historical summary",
+			st: func() state.BeaconState {
+				st, _ := util.DeterministicGenesisStateCapella(t, 1)
+				st, err := transition.ProcessSlots(context.Background(), st, params.BeaconConfig().SlotsPerHistoricalRoot-1)
+				require.NoError(t, err)
+				return st
+			},
+			verifier: func(st state.BeaconState) {
+				require.Equal(t, 1, len(st.HistoricalSummaries()))
+
+				br, err := stateutil.ArraysRoot(st.BlockRoots(), fieldparams.BlockRootsLength)
+				require.NoError(t, err)
+				sr, err := stateutil.ArraysRoot(st.StateRoots(), fieldparams.StateRootsLength)
+				require.NoError(t, err)
+				b := &ethpb.HistoricalSummary{
+					BlockSummaryRoot: br[:],
+					StateSummaryRoot: sr[:],
+				}
+				require.DeepEqual(t, b, st.HistoricalSummaries()[0])
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := epoch.ProcessHistoricalDataUpdate(tt.st())
+			require.NoError(t, err)
+			tt.verifier(got)
+		})
+	}
+}

beacon-chain/state/interfaces.go

@@ -58,6 +58,7 @@ type ReadOnlyBeaconState interface {
 	Fork() *ethpb.Fork
 	LatestBlockHeader() *ethpb.BeaconBlockHeader
 	HistoricalRoots() [][]byte
+	HistoricalSummaries() []*ethpb.HistoricalSummary
 	Slashings() []uint64
 	FieldReferencesCount() map[string]uint64
 	MarshalSSZ() ([]byte, error)
@@ -85,6 +86,7 @@ type WriteOnlyBeaconState interface {
 	SetSlashings(val []uint64) error
 	UpdateSlashingsAtIndex(idx, val uint64) error
 	AppendHistoricalRoots(root [32]byte) error
+	AppendHistoricalSummariesUpdate(*ethpb.HistoricalSummary) error
 	SetLatestExecutionPayloadHeader(payload interfaces.ExecutionData) error
 	SetNextWithdrawalIndex(i uint64) error
 	SetNextWithdrawalValidatorIndex(i types.ValidatorIndex) error

beacon-chain/state/state-native/beacon_state_mainnet.go

@@ -27,6 +27,7 @@ type BeaconState struct {
 	blockRoots                          *customtypes.BlockRoots
 	stateRoots                          *customtypes.StateRoots
 	historicalRoots                     customtypes.HistoricalRoots
+	historicalSummaries                 []*ethpb.HistoricalSummary
 	eth1Data                            *ethpb.Eth1Data
 	eth1DataVotes                       []*ethpb.Eth1Data
 	eth1DepositIndex                    uint64

beacon-chain/state/state-native/beacon_state_minimal.go

@@ -27,6 +27,7 @@ type BeaconState struct {
 	blockRoots                          *customtypes.BlockRoots
 	stateRoots                          *customtypes.StateRoots
 	historicalRoots                     customtypes.HistoricalRoots
+	historicalSummaries                 []*ethpb.HistoricalSummary
 	eth1Data                            *ethpb.Eth1Data
 	eth1DataVotes                       []*ethpb.Eth1Data
 	eth1DepositIndex                    uint64

beacon-chain/state/state-native/getters_misc.go

@@ -87,3 +87,21 @@ func (b *BeaconState) balancesLength() int {
 
 	return len(b.balances)
 }
+
+// HistoricalSummaries of the beacon state.
+func (b *BeaconState) HistoricalSummaries() []*ethpb.HistoricalSummary {
+	if b.historicalSummaries == nil {
+		return nil
+	}
+
+	b.lock.RLock()
+	defer b.lock.RUnlock()
+
+	return b.historicalSummariesVal()
+}
+
+// historicalSummariesVal of the beacon state.
+// This assumes that a lock is already held on BeaconState.
+func (b *BeaconState) historicalSummariesVal() []*ethpb.HistoricalSummary {
+	return ethpb.CopyHistoricalSummaries(b.historicalSummaries)
+}

beacon-chain/state/state-native/getters_state.go

@@ -126,6 +126,7 @@ func (b *BeaconState) ToProtoUnsafe() interface{} {
 			LatestExecutionPayloadHeader: b.latestExecutionPayloadHeaderCapella,
 			NextWithdrawalIndex:          b.nextWithdrawalIndex,
 			NextWithdrawalValidatorIndex: b.nextWithdrawalValidatorIndex,
+			HistoricalSummaries:          b.historicalSummaries,
 		}
 	default:
 		return nil
@@ -252,6 +253,7 @@ func (b *BeaconState) ToProto() interface{} {
 			LatestExecutionPayloadHeader: b.latestExecutionPayloadHeaderCapellaVal(),
 			NextWithdrawalIndex:          b.nextWithdrawalIndex,
 			NextWithdrawalValidatorIndex: b.nextWithdrawalValidatorIndex,
+			HistoricalSummaries:          b.historicalSummariesVal(),
 		}
 	default:
 		return nil

beacon-chain/state/state-native/hasher.go

@@ -1,8 +1,10 @@
 package state_native
 
 import (
+	"bytes"
 	"context"
 	"encoding/binary"
+	"fmt"
 
 	"github.com/pkg/errors"
 	nativetypes "github.com/prysmaticlabs/prysm/v3/beacon-chain/state/state-native/types"
@@ -12,6 +14,7 @@ import (
 	"github.com/prysmaticlabs/prysm/v3/crypto/hash"
 	"github.com/prysmaticlabs/prysm/v3/encoding/bytesutil"
 	"github.com/prysmaticlabs/prysm/v3/encoding/ssz"
+	ethpb "github.com/prysmaticlabs/prysm/v3/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/v3/runtime/version"
 	"go.opencensus.io/trace"
 )
@@ -255,7 +258,50 @@ func ComputeFieldRootsWithHasher(ctx context.Context, state *BeaconState) ([][]b
 		nextWithdrawalValidatorIndexRoot := make([]byte, 32)
 		binary.LittleEndian.PutUint64(nextWithdrawalValidatorIndexRoot, uint64(state.nextWithdrawalValidatorIndex))
 		fieldRoots[nativetypes.NextWithdrawalValidatorIndex.RealPosition()] = nextWithdrawalValidatorIndexRoot
+
+		// Historical summary root.
+		historicalSummaryRoot, err := historicalSummaryRoot(state.historicalSummaries)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not compute historical summary merkleization")
+		}
+		fieldRoots[nativetypes.HistoricalSummaries.RealPosition()] = historicalSummaryRoot[:]
 	}
 
 	return fieldRoots, nil
 }
+
+func historicalSummaryRoot(summaries []*ethpb.HistoricalSummary) ([32]byte, error) {
+	max := uint64(fieldparams.HistoricalRootsLength)
+	if uint64(len(summaries)) > max {
+		return [32]byte{}, fmt.Errorf("historical summary exceeds max length %d", max)
+	}
+
+	hasher := hash.CustomSHA256Hasher()
+	roots := make([][32]byte, len(summaries))
+	for i := 0; i < len(summaries); i++ {
+		r, err := summaries[i].HashTreeRoot()
+		if err != nil {
+			return [32]byte{}, errors.Wrap(err, "could not merkleize historical summary")
+		}
+		roots[i] = r
+	}
+
+	summariesRoot, err := ssz.BitwiseMerkleize(
+		hasher,
+		roots,
+		uint64(len(roots)),
+		fieldparams.HistoricalRootsLength,
+	)
+	if err != nil {
+		return [32]byte{}, errors.Wrap(err, "could not compute epoch attestations merkleization")
+	}
+	summariesLenBuf := new(bytes.Buffer)
+	if err := binary.Write(summariesLenBuf, binary.LittleEndian, uint64(len(summaries))); err != nil {
+		return [32]byte{}, errors.Wrap(err, "could not marshal historical summary length")
+	}
+	// We need to mix in the length of the slice.
+	summariesLenRoot := make([]byte, 32)
+	copy(summariesLenRoot, summariesLenBuf.Bytes())
+	res := ssz.MixInLength(summariesRoot, summariesLenRoot)
+	return res, nil
+}