fix: less time intensive slashing migration (#580)

* attempt just skipping deletion to see what time that gets us

* set missed blocks to nil

* re-enable deletes

* add print for debugging

* attempt 1000 per block

* break out of outer loop

* temp change to 1m

* update actual key

* delete pruning key

* change to log

* use var

* test

* lint

* comment update

* add changelog

CHANGELOG.md

@@ -44,6 +44,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * (slashing) [#543](https://github.com/osmosis-labs/cosmos-sdk/pull/543) Make slashing not write sign info every block
 * (authz) [#513](https://github.com/osmosis-labs/cosmos-sdk/pull/513) Limit expired authz grant pruning to 200 per block
 * (gov) [#514](https://github.com/osmosis-labs/cosmos-sdk/pull/514) Let gov hooks return an error
+* (slashing) [#580](https://github.com/osmosis-labs/cosmos-sdk/pull/580) Less time intensive slashing migration
 
 ## [State Compatible]
 

x/slashing/abci.go

@@ -11,6 +11,8 @@ import (
 	"github.com/cosmos/cosmos-sdk/x/slashing/types"
 )
 
+var deprecatedBitArrayPruneLimitPerBlock = 2000
+
 // BeginBlocker check for infraction evidence or downtime of validators
 // on every begin block
 func BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock, k keeper.Keeper) {
@@ -23,4 +25,7 @@ func BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock, k keeper.Keeper)
 	for _, voteInfo := range req.LastCommitInfo.GetVotes() {
 		k.HandleValidatorSignatureWithParams(ctx, params, voteInfo.Validator.Address, voteInfo.Validator.Power, voteInfo.SignedLastBlock)
 	}
+
+	// If there are still entries for the deprecated MissedBlockBitArray, delete them up until we hit the per block limit
+	k.DeleteDeprecatedValidatorMissedBlockBitArray(ctx, deprecatedBitArrayPruneLimitPerBlock)
 }

x/slashing/keeper/keeper.go

@@ -9,6 +9,7 @@ import (
 	"github.com/cosmos/cosmos-sdk/codec"
 	cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types"
 	sdk "github.com/cosmos/cosmos-sdk/types"
+	v4 "github.com/cosmos/cosmos-sdk/x/slashing/migrations/v4"
 	"github.com/cosmos/cosmos-sdk/x/slashing/types"
 	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
 )
@@ -115,3 +116,44 @@ func (k Keeper) deleteAddrPubkeyRelation(ctx sdk.Context, addr cryptotypes.Addre
 	store := ctx.KVStore(k.storeKey)
 	store.Delete(types.AddrPubkeyRelationKey(addr))
 }
+
+func (k Keeper) DeleteDeprecatedValidatorMissedBlockBitArray(ctx sdk.Context, iterationLimit int) {
+	store := ctx.KVStore(k.storeKey)
+	if store.Get(types.IsPruningKey) == nil {
+		return
+	}
+
+	// Iterate over all the validator signing infos and delete the deprecated missed block bit arrays
+	valSignInfoIter := sdk.KVStorePrefixIterator(store, types.ValidatorSigningInfoKeyPrefix)
+	defer valSignInfoIter.Close()
+
+	iterationCounter := 0
+	for ; valSignInfoIter.Valid(); valSignInfoIter.Next() {
+		address := types.ValidatorSigningInfoAddress(valSignInfoIter.Key())
+
+		// Creat anonymous function to scope defer statement
+		func() {
+			iter := storetypes.KVStorePrefixIterator(store, v4.DeprecatedValidatorMissedBlockBitArrayPrefixKey(address))
+			defer iter.Close()
+
+			for ; iter.Valid(); iter.Next() {
+				store.Delete(iter.Key())
+				iterationCounter++
+				if iterationCounter >= iterationLimit {
+					break
+				}
+			}
+		}()
+
+		if iterationCounter >= iterationLimit {
+			break
+		}
+	}
+
+	ctx.Logger().Info("Deleted deprecated missed block bit arrays", "count", iterationCounter)
+
+	// If we have deleted all the deprecated missed block bit arrays, we can delete the pruning key (set to nil)
+	if iterationCounter == 0 {
+		store.Delete(types.IsPruningKey)
+	}
+}

x/slashing/migrations/v4/keys.go

@@ -11,8 +11,13 @@ import (
 const MissedBlockBitmapChunkSize = 1024 // 2^10 bits
 
 var (
-	ValidatorSigningInfoKeyPrefix         = []byte{0x01}
-	validatorMissedBlockBitArrayKeyPrefix = []byte{0x02}
+	ValidatorSigningInfoKeyPrefix                   = []byte{0x01}
+	deprecatedValidatorMissedBlockBitArrayKeyPrefix = []byte{0x02}
+
+	// NOTE: sdk v0.50 uses the same key prefix for both deprecated and new missed block bitmaps.
+	// We needed to use a new key, because we are skipping deletion of all old keys at upgrade time
+	// due to how long this would bring the chain down. We use 0x10 here to prevent overlap with any future keys.
+	validatorMissedBlockBitMapKeyPrefix = []byte{0x10}
 )
 
 func ValidatorSigningInfoKey(v sdk.ConsAddress) []byte {
@@ -27,18 +32,18 @@ func ValidatorSigningInfoAddress(key []byte) (v sdk.ConsAddress) {
 	return sdk.ConsAddress(addr)
 }
 
-func validatorMissedBlockBitArrayPrefixKey(v sdk.ConsAddress) []byte {
-	return append(validatorMissedBlockBitArrayKeyPrefix, address.MustLengthPrefix(v.Bytes())...)
+func DeprecatedValidatorMissedBlockBitArrayPrefixKey(v sdk.ConsAddress) []byte {
+	return append(deprecatedValidatorMissedBlockBitArrayKeyPrefix, address.MustLengthPrefix(v.Bytes())...)
 }
 
-func ValidatorMissedBlockBitArrayKey(v sdk.ConsAddress, i int64) []byte {
+func DeprecatedValidatorMissedBlockBitArrayKey(v sdk.ConsAddress, i int64) []byte {
 	b := make([]byte, 8)
 	binary.LittleEndian.PutUint64(b, uint64(i))
-	return append(validatorMissedBlockBitArrayPrefixKey(v), b...)
+	return append(DeprecatedValidatorMissedBlockBitArrayPrefixKey(v), b...)
 }
 
 func validatorMissedBlockBitmapPrefixKey(v sdk.ConsAddress) []byte {
-	return append(validatorMissedBlockBitArrayKeyPrefix, address.MustLengthPrefix(v.Bytes())...)
+	return append(validatorMissedBlockBitMapKeyPrefix, address.MustLengthPrefix(v.Bytes())...)
 }
 
 func ValidatorMissedBlockBitmapKey(v sdk.ConsAddress, chunkIndex int64) []byte {

x/slashing/migrations/v4/migrate.go

@@ -21,11 +21,13 @@ func Migrate(ctx sdk.Context, cdc codec.BinaryCodec, store storetypes.KVStore, p
 	var missedBlocks []types.ValidatorMissedBlocks
 	iterateValidatorSigningInfos(ctx, cdc, store, func(addr sdk.ConsAddress, info types.ValidatorSigningInfo) (stop bool) {
 		bechAddr := addr.String()
-		localMissedBlocks := GetValidatorMissedBlocks(ctx, cdc, store, addr, params)
+
+		// We opt to reset all validators missed blocks to improve upgrade performance
+		// localMissedBlocks := GetValidatorMissedBlocks(ctx, cdc, store, addr, params)
 
 		missedBlocks = append(missedBlocks, types.ValidatorMissedBlocks{
 			Address:      bechAddr,
-			MissedBlocks: localMissedBlocks,
+			MissedBlocks: []types.MissedBlock{},
 		})
 
 		return false
@@ -39,7 +41,10 @@ func Migrate(ctx sdk.Context, cdc codec.BinaryCodec, store storetypes.KVStore, p
 			return err
 		}
 
-		deleteValidatorMissedBlockBitArray(ctx, store, addr)
+		// We skip the deletion here in favor of spreading out across multiple block for performance reasons
+		// We set the isPruning key to true to indicate that we are in the process of pruning
+		store.Set(types.IsPruningKey, []byte{1})
+		// deleteDeprecatedValidatorMissedBlockBitArray(ctx, store, addr)
 
 		for _, b := range mb.MissedBlocks {
 			// Note: It is not necessary to store entries with missed=false, i.e. where
@@ -86,7 +91,7 @@ func iterateValidatorMissedBlockBitArray(
 ) {
 	for i := int64(0); i < params.SignedBlocksWindow; i++ {
 		var missed gogotypes.BoolValue
-		bz := store.Get(ValidatorMissedBlockBitArrayKey(addr, i))
+		bz := store.Get(DeprecatedValidatorMissedBlockBitArrayKey(addr, i))
 		if bz == nil {
 			continue
 		}
@@ -114,14 +119,15 @@ func GetValidatorMissedBlocks(
 	return missedBlocks
 }
 
-func deleteValidatorMissedBlockBitArray(ctx sdk.Context, store storetypes.KVStore, addr sdk.ConsAddress) {
-	iter := storetypes.KVStorePrefixIterator(store, validatorMissedBlockBitArrayPrefixKey(addr))
-	defer iter.Close()
+// No longer use this
+// func deleteDeprecatedValidatorMissedBlockBitArray(ctx sdk.Context, store storetypes.KVStore, addr sdk.ConsAddress) {
+// 	iter := storetypes.KVStorePrefixIterator(store, DeprecatedValidatorMissedBlockBitArrayPrefixKey(addr))
+// 	defer iter.Close()
 
-	for ; iter.Valid(); iter.Next() {
-		store.Delete(iter.Key())
-	}
-}
+// 	for ; iter.Valid(); iter.Next() {
+// 		store.Delete(iter.Key())
+// 	}
+// }
 
 func setMissedBlockBitmapValue(ctx sdk.Context, store storetypes.KVStore, addr sdk.ConsAddress, index int64, missed bool) error {
 	// get the chunk or "word" in the logical bitmap

x/slashing/migrations/v4/migrate_test.go

@@ -3,7 +3,6 @@ package v4_test
 import (
 	"testing"
 
-	"github.com/bits-and-blooms/bitset"
 	gogotypes "github.com/cosmos/gogoproto/types"
 	"github.com/stretchr/testify/require"
 
@@ -35,7 +34,7 @@ func TestMigrate(t *testing.T) {
 		// all even blocks are missed
 		missed := &gogotypes.BoolValue{Value: i%2 == 0}
 		bz := cdc.MustMarshal(missed)
-		store.Set(v4.ValidatorMissedBlockBitArrayKey(consAddr, i), bz)
+		store.Set(v4.DeprecatedValidatorMissedBlockBitArrayKey(consAddr, i), bz)
 	}
 
 	err := v4.Migrate(ctx, cdc, store, params)
@@ -44,15 +43,18 @@ func TestMigrate(t *testing.T) {
 	for i := int64(0); i < params.SignedBlocksWindow; i++ {
 		chunkIndex := i / v4.MissedBlockBitmapChunkSize
 		chunk := store.Get(v4.ValidatorMissedBlockBitmapKey(consAddr, chunkIndex))
-		require.NotNil(t, chunk)
 
-		bs := bitset.New(uint(v4.MissedBlockBitmapChunkSize))
-		require.NoError(t, bs.UnmarshalBinary(chunk))
+		// We reset all validators missed blocks to improve upgrade performance,
+		// so we expect all chunks to be empty
+		require.Nil(t, chunk)
 
-		// ensure all even blocks are missed
-		bitIndex := uint(i % v4.MissedBlockBitmapChunkSize)
-		require.Equal(t, i%2 == 0, bs.Test(bitIndex))
-		require.Equal(t, i%2 == 1, !bs.Test(bitIndex))
+		// bs := bitset.New(uint(v4.MissedBlockBitmapChunkSize))
+		// require.NoError(t, bs.UnmarshalBinary(chunk))
+
+		// // ensure all even blocks are missed
+		// bitIndex := uint(i % v4.MissedBlockBitmapChunkSize)
+		// require.Equal(t, i%2 == 0, bs.Test(bitIndex))
+		// require.Equal(t, i%2 == 1, !bs.Test(bitIndex))
 	}
 
 	// ensure there's only one chunk for a window of size 100

x/slashing/types/keys.go

@@ -48,10 +48,14 @@ const (
 // - 0x03<accAddrLen (1 Byte)><accAddr_Bytes>: cryptotypes.PubKey
 
 var (
-	ParamsKey                           = []byte{0x00} // Prefix for params key
-	ValidatorSigningInfoKeyPrefix       = []byte{0x01} // Prefix for signing info
-	ValidatorMissedBlockBitmapKeyPrefix = []byte{0x02} // Prefix for missed block bitmap
-	AddrPubkeyRelationKeyPrefix         = []byte{0x03} // Prefix for address-pubkey relation
+	ParamsKey                     = []byte{0x00} // Prefix for params key
+	ValidatorSigningInfoKeyPrefix = []byte{0x01} // Prefix for signing info
+	AddrPubkeyRelationKeyPrefix   = []byte{0x03} // Prefix for address-pubkey relation
+
+	ValidatorMissedBlockBitmapKeyPrefix = []byte{0x10} // Prefix for missed block bitmap
+
+	IsPruningKey  = []byte{0x09}
+	TrueByteValue = []byte{0x01}
 )
 
 // ValidatorSigningInfoKey - stored by *Consensus* address (not operator address)