feat(twap): prune records over multiple blocks

CHANGELOG.md

@@ -63,6 +63,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 * [#7360](https://github.com/osmosis-labs/osmosis/pull/7360) Bump cometbft-db from 0.8.0 to 0.10.0
 * [#7385](https://github.com/osmosis-labs/osmosis/pull/7385) Add missing protobuf interface
+* [#7427](https://github.com/osmosis-labs/osmosis/pull/7427) Prune TWAP records over multiple blocks, instead of all at once at epoch
 
 ## v22.0.5
 

proto/osmosis/twap/v1beta1/twap_record.proto

@@ -74,3 +74,20 @@ message TwapRecord {
     (gogoproto.moretags) = "yaml:\"last_error_time\""
   ];
 }
+
+// PruningState allows us to spread out the pruning of TWAP records over time,
+// instead of pruning all at once at the end of the epoch.
+message PruningState {
+  // is_pruning is true if the pruning process is ongoing.
+  // This tells the module to continue pruning the TWAP records
+  // at the EndBlock.
+  bool is_pruning = 1;
+  // last_kept_time is the time of the last kept TWAP record.
+  // This is used to determine all TWAP records that are older than
+  // last_kept_time and should be pruned.
+  google.protobuf.Timestamp last_kept_time = 2 [
+    (gogoproto.nullable) = false,
+    (gogoproto.stdtime) = true,
+    (gogoproto.moretags) = "yaml:\"last_kept_time\""
+  ];
+}

x/twap/export_test.go

@@ -44,10 +44,6 @@ func (k Keeper) PruneRecordsBeforeTimeButNewest(ctx sdk.Context, lastKeptTime ti
 	return k.pruneRecordsBeforeTimeButNewest(ctx, lastKeptTime)
 }
 
-func (k Keeper) PruneRecords(ctx sdk.Context) error {
-	return k.pruneRecords(ctx)
-}
-
 func (k Keeper) GetInterpolatedRecord(ctx sdk.Context, poolId uint64, asset0Denom string, asset1Denom string, t time.Time) (types.TwapRecord, error) {
 	return k.getInterpolatedRecord(ctx, poolId, t, asset0Denom, asset1Denom)
 }

x/twap/keeper.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/cosmos/gogoproto/proto"
 
 	paramtypes "github.com/cosmos/cosmos-sdk/x/params/types"
 
@@ -100,3 +101,31 @@ func (k Keeper) GetGeometricStrategy() *geometric {
 func (k Keeper) GetArithmeticStrategy() *arithmetic {
 	return &arithmetic{k}
 }
+
+// GetPruningState gets the current pruning state, which is used to determine
+// whether to prune historical records in the EndBlock. This allows us to spread
+// out the computational cost of pruning over time rather than all at once at epoch.
+func (k Keeper) GetPruningState(ctx sdk.Context) types.PruningState {
+	store := ctx.KVStore(k.storeKey)
+	state := types.PruningState{}
+
+	bz := store.Get(types.PruningStateKey)
+	if bz == nil {
+		return state
+	}
+	err := proto.Unmarshal(bz, &state)
+	if err != nil {
+		panic(err)
+	}
+	return state
+}
+
+func (k Keeper) SetPruningState(ctx sdk.Context, state types.PruningState) {
+	store := ctx.KVStore(k.storeKey)
+
+	bz, err := proto.Marshal(&state)
+	if err != nil {
+		panic(err)
+	}
+	store.Set(types.PruningStateKey, bz)
+}

x/twap/listeners.go

@@ -6,6 +6,7 @@ import (
 	"github.com/osmosis-labs/osmosis/osmomath"
 	concentratedliquiditytypes "github.com/osmosis-labs/osmosis/v22/x/concentrated-liquidity/types"
 	gammtypes "github.com/osmosis-labs/osmosis/v22/x/gamm/types"
+	"github.com/osmosis-labs/osmosis/v22/x/twap/types"
 	epochtypes "github.com/osmosis-labs/osmosis/x/epochs/types"
 )
 
@@ -24,9 +25,11 @@ func (k Keeper) EpochHooks() epochtypes.EpochHooks {
 
 func (hook *epochhook) AfterEpochEnd(ctx sdk.Context, epochIdentifier string, epochNumber int64) error {
 	if epochIdentifier == hook.k.PruneEpochIdentifier(ctx) {
-		if err := hook.k.pruneRecords(ctx); err != nil {
-			ctx.Logger().Error("Error pruning old twaps at the epoch end", err)
-		}
+		lastKeptTime := ctx.BlockTime().Add(-hook.k.RecordHistoryKeepPeriod(ctx))
+		hook.k.SetPruningState(ctx, types.PruningState{
+			IsPruning:    true,
+			LastKeptTime: lastKeptTime,
+		})
 	}
 	return nil
 }

x/twap/listeners_test.go

@@ -275,20 +275,22 @@ func (s *TestSuite) TestAfterEpochEnd() {
 		err = s.App.TwapKeeper.EpochHooks().AfterEpochEnd(s.Ctx, allEpochs[i].Identifier, int64(1))
 		s.Require().NoError(err)
 
-		recordsAfterEpoch, err := s.twapkeeper.GetAllHistoricalTimeIndexedTWAPs(s.Ctx)
+		//recordsAfterEpoch, err := s.twapkeeper.GetAllHistoricalTimeIndexedTWAPs(s.Ctx)
 
-		// old record should have been pruned here
-		// however, the newest younger than the prune threshold
-		// is kept.
+		lastKeptTime := s.Ctx.BlockTime().Add(-s.twapkeeper.RecordHistoryKeepPeriod(s.Ctx))
+		pruneState := s.twapkeeper.GetPruningState(s.Ctx)
+
+		// state entry should be set for pruning state
 		if allEpochs[i].Identifier == pruneEpochIdentifier {
-			s.Require().Equal(1, len(recordsAfterEpoch))
-			s.Require().Equal(newestRecord, recordsAfterEpoch[0])
+			s.Require().Equal(true, pruneState.IsPruning)
+			s.Require().Equal(lastKeptTime, pruneState.LastKeptTime)
 
-			// quit test once the record has been pruned
-			return
+			// reset pruning state to make sure other epochs do not modify it
+			s.twapkeeper.SetPruningState(s.Ctx, types.PruningState{})
 		} else { // pruning should not be triggered at first, not pruning epoch
 			s.Require().NoError(err)
-			s.Require().Equal(twapsBeforeEpoch, recordsAfterEpoch)
+			s.Require().Equal(false, pruneState.IsPruning)
+			s.Require().Equal(time.Time{}, pruneState.LastKeptTime)
 		}
 	}
 }

x/twap/logic.go

@@ -115,6 +115,14 @@ func (k Keeper) EndBlock(ctx sdk.Context) {
 					" Skipping record update. Underlying err: %w", id, err).Error())
 		}
 	}
+
+	state := k.GetPruningState(ctx)
+	if state.IsPruning {
+		err := k.pruneRecordsBeforeTimeButNewest(ctx, state.LastKeptTime)
+		if err != nil {
+			ctx.Logger().Error("Error pruning old twaps at the end block", err)
+		}
+	}
 }
 
 // updateRecords updates all records for a given pool id.
@@ -195,18 +203,6 @@ func (k Keeper) updateRecord(ctx sdk.Context, record types.TwapRecord) (types.Tw
 	return newRecord, nil
 }
 
-// pruneRecords prunes twap records that happened earlier than recordHistoryKeepPeriod
-// before current block time while preserving the most recent record before the threshold.
-// Such record is preserved for each pool.
-// See TWAP keeper's `pruneRecordsBeforeTimeButNewest(...)` for more details about the reasons for
-// keeping this record.
-func (k Keeper) pruneRecords(ctx sdk.Context) error {
-	recordHistoryKeepPeriod := k.RecordHistoryKeepPeriod(ctx)
-
-	lastKeptTime := ctx.BlockTime().Add(-recordHistoryKeepPeriod)
-	return k.pruneRecordsBeforeTimeButNewest(ctx, lastKeptTime)
-}
-
 // recordWithUpdatedAccumulators returns a record, with updated accumulator values and time for provided newTime,
 // otherwise referred to as "interpolating the record" to the target time.
 // This does not mutate the passed in record.

x/twap/logic_test.go

@@ -572,62 +572,6 @@ type computeThreeAssetArithmeticTwapTestCase struct {
 	expErr      bool
 }
 
-// TestPruneRecords tests that twap records earlier than
-// current block time - RecordHistoryKeepPeriod are pruned from the store
-// while keeping the newest record before the above time threshold.
-// Such record is kept for each pool.
-func (s *TestSuite) TestPruneRecords() {
-	recordHistoryKeepPeriod := s.twapkeeper.RecordHistoryKeepPeriod(s.Ctx)
-
-	pool1OlderMin2MsRecord, // deleted
-		pool2OlderMin1MsRecordAB, pool2OlderMin1MsRecordAC, pool2OlderMin1MsRecordBC, // deleted
-		pool3OlderBaseRecord,    // kept as newest under keep period
-		pool4OlderPlus1Record := // kept as newest under keep period
-		s.createTestRecordsFromTime(baseTime.Add(2 * -recordHistoryKeepPeriod))
-
-	pool1Min2MsRecord, // kept as newest under keep period
-		pool2Min1MsRecordAB, pool2Min1MsRecordAC, pool2Min1MsRecordBC, // kept as newest under keep period
-		pool3BaseRecord,    // kept as it is at the keep period boundary
-		pool4Plus1Record := // kept as it is above the keep period boundary
-		s.createTestRecordsFromTime(baseTime.Add(-recordHistoryKeepPeriod))
-
-	// non-ascending insertion order.
-	recordsToPreSet := []types.TwapRecord{
-		pool2OlderMin1MsRecordAB, pool2OlderMin1MsRecordAC, pool2OlderMin1MsRecordBC,
-		pool4Plus1Record,
-		pool4OlderPlus1Record,
-		pool3OlderBaseRecord,
-		pool2Min1MsRecordAB, pool2Min1MsRecordAC, pool2Min1MsRecordBC,
-		pool3BaseRecord,
-		pool1Min2MsRecord,
-		pool1OlderMin2MsRecord,
-	}
-
-	// tMin2Record is before the threshold and is pruned away.
-	// tmin1Record is the newest record before current block time - record history keep period.
-	// All other records happen after the threshold and are kept.
-	expectedKeptRecords := []types.TwapRecord{
-		pool3OlderBaseRecord,
-		pool4OlderPlus1Record,
-		pool1Min2MsRecord,
-		pool2Min1MsRecordAB, pool2Min1MsRecordAC, pool2Min1MsRecordBC,
-		pool3BaseRecord,
-		pool4Plus1Record,
-	}
-	s.SetupTest()
-	s.preSetRecords(recordsToPreSet)
-
-	ctx := s.Ctx
-	twapKeeper := s.twapkeeper
-
-	ctx = ctx.WithBlockTime(baseTime)
-
-	err := twapKeeper.PruneRecords(ctx)
-	s.Require().NoError(err)
-
-	s.validateExpectedRecords(expectedKeptRecords)
-}
-
 // TestUpdateRecords tests that the records are updated correctly.
 // It tests the following:
 // - two-asset pools

x/twap/store.go

@@ -11,6 +11,8 @@ import (
 	"github.com/osmosis-labs/osmosis/v22/x/twap/types"
 )
 
+var NumRecordsToPrunePerBlock uint16 = 1000
+
 type timeTooOldError struct {
 	Time time.Time
 }
@@ -93,6 +95,8 @@ func (k Keeper) pruneRecordsBeforeTimeButNewest(ctx sdk.Context, lastKeptTime ti
 	}
 	seenPoolAssetTriplets := map[uniqueTriplet]struct{}{}
 
+	var numPruned uint16
+
 	for ; iter.Valid(); iter.Next() {
 		timeIndexKey := iter.Key()
 		timeS, poolId, asset0, asset1, err := types.ParseFieldsFromHistoricalTimeKey(timeIndexKey)
@@ -117,6 +121,19 @@ func (k Keeper) pruneRecordsBeforeTimeButNewest(ctx sdk.Context, lastKeptTime ti
 		store.Delete(timeIndexKey)
 		poolIndexKey := types.FormatHistoricalPoolIndexTWAPKeyFromStrTime(poolId, asset0, asset1, timeS)
 		store.Delete(poolIndexKey)
+
+		numPruned++
+		if numPruned >= NumRecordsToPrunePerBlock {
+			// We have hit the limit, so we stop pruning.
+			break
+		}
+	}
+
+	if !iter.Valid() {
+		// The iterator is exhausted, so we have pruned all records.
+		state := k.GetPruningState(ctx)
+		state.IsPruning = false
+		k.SetPruningState(ctx, state)
 	}
 	return nil
 }

x/twap/store_test.go

@@ -359,6 +359,8 @@ func (s *TestSuite) TestPruneRecordsBeforeTimeButNewest() {
 		lastKeptTime time.Time
 
 		expectedKeptRecords []types.TwapRecord
+
+		overwriteLimit uint16
 	}{
 		"base time; across pool 3; 4 records; 3 before lastKeptTime; 2 deleted and newest kept": {
 			recordsToPreSet: []types.TwapRecord{
@@ -507,6 +509,20 @@ func (s *TestSuite) TestPruneRecordsBeforeTimeButNewest() {
 
 			expectedKeptRecords: []types.TwapRecord{},
 		},
+		"base time; across pool 3; 4 records; 3 before lastKeptTime; only 1 deleted due to limit set to 1": {
+			recordsToPreSet: []types.TwapRecord{
+				pool3BaseSecMin1Ms, // base time - 1ms; kept since newest before lastKeptTime
+				pool3BaseSecBaseMs, // base time; kept since at lastKeptTime
+				pool3BaseSecMin3Ms, // base time - 3ms; in queue for deletion
+				pool3BaseSecMin2Ms, // base time - 2ms; deleted
+			},
+
+			lastKeptTime: baseTime,
+
+			expectedKeptRecords: []types.TwapRecord{pool3BaseSecMin3Ms, pool3BaseSecMin1Ms, pool3BaseSecBaseMs},
+
+			overwriteLimit: 1,
+		},
 	}
 	for name, tc := range tests {
 		s.Run(name, func() {
@@ -516,6 +532,14 @@ func (s *TestSuite) TestPruneRecordsBeforeTimeButNewest() {
 			ctx := s.Ctx
 			twapKeeper := s.twapkeeper
 
+			if tc.overwriteLimit != 0 {
+				originalLimit := twap.NumRecordsToPrunePerBlock
+				defer func() {
+					twap.NumRecordsToPrunePerBlock = originalLimit
+				}()
+				twap.NumRecordsToPrunePerBlock = tc.overwriteLimit
+			}
+
 			err := twapKeeper.PruneRecordsBeforeTimeButNewest(ctx, tc.lastKeptTime)
 			s.Require().NoError(err)
 

x/twap/types/keys.go

@@ -27,6 +27,7 @@ const (
 )
 
 var (
+	PruningStateKey                    = []byte{0x01}
 	mostRecentTWAPsNoSeparator         = "recent_twap"
 	historicalTWAPTimeIndexNoSeparator = "historical_time_index"
 	historicalTWAPPoolIndexNoSeparator = "historical_pool_index"