feat: track and query protocol rev across all modules

CHANGELOG.md

@@ -42,6 +42,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Features
+
+* [#6804](https://github.com/osmosis-labs/osmosis/pull/6804) feat: track and query protocol rev across all modules
+
 ### State Breaks
 
 * [#6758](https://github.com/osmosis-labs/osmosis/pull/6758) Add codec for MsgUndelegateFromRebalancedValidatorSet

app/apptesting/txfees.go

@@ -0,0 +1,85 @@
+package apptesting
+
+import (
+	cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types"
+	"github.com/cosmos/cosmos-sdk/simapp"
+	"github.com/cosmos/cosmos-sdk/testutil/testdata"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	authsigning "github.com/cosmos/cosmos-sdk/x/auth/signing"
+
+	"github.com/cosmos/cosmos-sdk/client"
+
+	clienttx "github.com/cosmos/cosmos-sdk/client/tx"
+
+	"github.com/osmosis-labs/osmosis/osmomath"
+	"github.com/osmosis-labs/osmosis/v20/x/txfees/keeper"
+	"github.com/osmosis-labs/osmosis/v20/x/txfees/types"
+)
+
+var baseGas = uint64(10000)
+
+func (s *KeeperTestHelper) ExecuteUpgradeFeeTokenProposal(feeToken string, poolId uint64) error {
+	upgradeProp := types.NewUpdateFeeTokenProposal(
+		"Test Proposal",
+		"test",
+		[]types.FeeToken{
+			{
+				Denom:  feeToken,
+				PoolID: poolId,
+			},
+		},
+	)
+	return s.App.TxFeesKeeper.HandleUpdateFeeTokenProposal(s.Ctx, &upgradeProp)
+}
+
+func (s *KeeperTestHelper) SetupTxFeeAnteHandlerAndChargeFee(clientCtx client.Context, minGasPrices sdk.DecCoins, gasRequested uint64, isCheckTx, isSimulate bool, txFee sdk.Coins) error {
+	mempoolFeeOpts := types.NewDefaultMempoolFeeOptions()
+	mempoolFeeOpts.MinGasPriceForHighGasTx = osmomath.MustNewDecFromStr("0.0025")
+
+	uionPoolId := s.PrepareBalancerPoolWithCoins(
+		sdk.NewInt64Coin(sdk.DefaultBondDenom, 500),
+		sdk.NewInt64Coin("uion", 500),
+	)
+	err := s.ExecuteUpgradeFeeTokenProposal("uion", uionPoolId)
+	s.Require().NoError(err)
+
+	if gasRequested == 0 {
+		gasRequested = baseGas
+	}
+	s.Ctx = s.Ctx.WithIsCheckTx(isCheckTx).WithMinGasPrices(minGasPrices)
+
+	// TODO: Cleanup this code.
+	// TxBuilder components reset for every test case
+	txBuilder := clientCtx.TxConfig.NewTxBuilder()
+	priv0, _, addr0 := testdata.KeyTestPubAddr()
+	acc1 := s.App.AccountKeeper.NewAccountWithAddress(s.Ctx, addr0)
+	s.App.AccountKeeper.SetAccount(s.Ctx, acc1)
+	msgs := []sdk.Msg{testdata.NewTestMsg(addr0)}
+	privs, accNums, accSeqs := []cryptotypes.PrivKey{priv0}, []uint64{0}, []uint64{0}
+	signerData := authsigning.SignerData{
+		ChainID:       s.Ctx.ChainID(),
+		AccountNumber: accNums[0],
+		Sequence:      accSeqs[0],
+	}
+
+	gasLimit := gasRequested
+	sigV2, _ := clienttx.SignWithPrivKey(
+		1,
+		signerData,
+		txBuilder,
+		privs[0],
+		clientCtx.TxConfig,
+		accSeqs[0],
+	)
+
+	err = simapp.FundAccount(s.App.BankKeeper, s.Ctx, addr0, txFee)
+	s.Require().NoError(err)
+
+	tx := s.BuildTx(txBuilder, msgs, sigV2, "", txFee, gasLimit)
+
+	mfd := keeper.NewMempoolFeeDecorator(*s.App.TxFeesKeeper, mempoolFeeOpts)
+	dfd := keeper.NewDeductFeeDecorator(*s.App.TxFeesKeeper, *s.App.AccountKeeper, *s.App.BankKeeper, nil)
+	antehandlerMFD := sdk.ChainAnteDecorators(mfd, dfd)
+	_, err = antehandlerMFD(s.Ctx, tx, isSimulate)
+	return err
+}

app/keepers/keepers.go

@@ -369,6 +369,7 @@ func (appKeepers *AppKeepers) InitNormalKeepers(
 		appKeepers.EpochsKeeper,
 		appKeepers.PoolManagerKeeper,
 		appKeepers.ConcentratedLiquidityKeeper,
+		appKeepers.TxFeesKeeper,
 	)
 	appKeepers.ProtoRevKeeper = &protorevKeeper
 	appKeepers.PoolManagerKeeper.SetProtorevKeeper(appKeepers.ProtoRevKeeper)
@@ -383,6 +384,7 @@ func (appKeepers *AppKeepers) InitNormalKeepers(
 		appKeepers.DistrKeeper,
 	)
 	appKeepers.TxFeesKeeper = &txFeesKeeper
+	appKeepers.ProtoRevKeeper.SetTxFeesKeeper(appKeepers.TxFeesKeeper)
 
 	appKeepers.IncentivesKeeper = incentiveskeeper.NewKeeper(
 		appKeepers.keys[incentivestypes.StoreKey],

app/upgrades/v21/upgrades.go

@@ -5,6 +5,7 @@ import (
 	"github.com/cosmos/cosmos-sdk/types/module"
 	upgradetypes "github.com/cosmos/cosmos-sdk/x/upgrade/types"
 
+	"github.com/osmosis-labs/osmosis/osmoutils"
 	"github.com/osmosis-labs/osmosis/v20/app/keepers"
 	"github.com/osmosis-labs/osmosis/v20/app/upgrades"
 )
@@ -23,6 +24,16 @@ func CreateUpgradeHandler(
 			return nil, err
 		}
 
+		// Since we are now tracking all protocol rev, we set the accounting height to the current block height for each module
+		// that generates protocol rev.
+		keepers.PoolManagerKeeper.SetTakerFeeTrackerStartHeight(ctx, uint64(ctx.BlockHeight()))
+		keepers.TxFeesKeeper.SetTxFeesTrackerStartHeight(ctx, uint64(ctx.BlockHeight()))
+		// We start the cyclic arb tracker from the value it currently is at since it has been tracking since inception (without a start height).
+		allCyclicArbProfits := keepers.ProtoRevKeeper.GetAllProfits(ctx)
+		allCyclicArbProfitsCoins := osmoutils.ConvertCoinArrayToCoins(allCyclicArbProfits)
+		keepers.ProtoRevKeeper.SetCyclicArbProfitTrackerValue(ctx, allCyclicArbProfitsCoins)
+		keepers.ProtoRevKeeper.SetCyclicArbProfitTrackerStartHeight(ctx, uint64(ctx.BlockHeight()))
+
 		return migrations, nil
 	}
 }

go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/ory/dockertest/v3 v3.10.0
 	github.com/osmosis-labs/go-mutesting v0.0.0-20221208041716-b43bcd97b3b3
 	github.com/osmosis-labs/osmosis/osmomath v0.0.7-0.20231014001935-1946419d44eb
-	github.com/osmosis-labs/osmosis/osmoutils v0.0.7-0.20231017074304-84e27b5e2aad
+	github.com/osmosis-labs/osmosis/osmoutils v0.0.7-0.20231101190541-d45713e67d41
 	github.com/osmosis-labs/osmosis/x/epochs v0.0.3-0.20231011004221-fd24b80f8366
 	github.com/osmosis-labs/osmosis/x/ibc-hooks v0.0.9-0.20231014001935-1946419d44eb
 	github.com/pkg/errors v0.9.1

go.sum

@@ -966,8 +966,8 @@ github.com/osmosis-labs/go-mutesting v0.0.0-20221208041716-b43bcd97b3b3 h1:Ylmch
 github.com/osmosis-labs/go-mutesting v0.0.0-20221208041716-b43bcd97b3b3/go.mod h1:lV6KnqXYD/ayTe7310MHtM3I2q8Z6bBfMAi+bhwPYtI=
 github.com/osmosis-labs/osmosis/osmomath v0.0.7-0.20231014001935-1946419d44eb h1:pXsC6vqGD+pbMGt+fVBHi9XBk/KDQuRZde2fh4s/1+k=
 github.com/osmosis-labs/osmosis/osmomath v0.0.7-0.20231014001935-1946419d44eb/go.mod h1:jNZ952fypVNMzOsh31LAUS27JbF9naNJGtELxId6ZCg=
-github.com/osmosis-labs/osmosis/osmoutils v0.0.7-0.20231017074304-84e27b5e2aad h1:UcQ/XLz0SqWMrA+BhgDXy9ukD4C+FlN4ULdazZmFOsE=
-github.com/osmosis-labs/osmosis/osmoutils v0.0.7-0.20231017074304-84e27b5e2aad/go.mod h1:16AXMzbTLkYE5If5VLTA07fV9JNcLFwgf/VoW5sHrtU=
+github.com/osmosis-labs/osmosis/osmoutils v0.0.7-0.20231101190541-d45713e67d41 h1:XrXYbkR3i8P8dlWRsmd+16qzfky2yCGdIAeg13JcpyE=
+github.com/osmosis-labs/osmosis/osmoutils v0.0.7-0.20231101190541-d45713e67d41/go.mod h1:16AXMzbTLkYE5If5VLTA07fV9JNcLFwgf/VoW5sHrtU=
 github.com/osmosis-labs/osmosis/x/epochs v0.0.3-0.20231011004221-fd24b80f8366 h1:E6H0V3MKbSNwo1iXE9Kzatd2M02MgZpS5AiJ6CKK5us=
 github.com/osmosis-labs/osmosis/x/epochs v0.0.3-0.20231011004221-fd24b80f8366/go.mod h1:vU0IHK5W38dqMeux3MkSaT3MZU6whAkx7vNuxv1IzeU=
 github.com/osmosis-labs/osmosis/x/ibc-hooks v0.0.9-0.20231014001935-1946419d44eb h1:6lYLEiJERdD+QK925XYyHkvNyvQTghVFufMH5VAQLpg=

osmoutils/coin_helper.go

@@ -114,3 +114,11 @@ func MergeCoinMaps[T comparable](currentEpochExpectedDistributionsOne map[T]sdk.
 	}
 	return newMap
 }
+
+func ConvertCoinArrayToCoins(coinArray []sdk.Coin) sdk.Coins {
+	coins := sdk.Coins{}
+	for _, coin := range coinArray {
+		coins = append(coins, coin)
+	}
+	return coins
+}

osmoutils/coin_helper_test.go

@@ -291,3 +291,35 @@ func TestMergeCoinMaps(t *testing.T) {
 		}
 	})
 }
+
+func TestConvertCoinArrayToCoins(t *testing.T) {
+	tests := []struct {
+		name          string
+		coinArray     []sdk.Coin
+		expectedCoins sdk.Coins
+	}{
+		{
+			name:          "Empty input",
+			coinArray:     []sdk.Coin{},
+			expectedCoins: sdk.NewCoins(),
+		},
+		{
+			name:          "Single coin",
+			coinArray:     []sdk.Coin{sdk.NewCoin("atom", osmomath.NewInt(100000000))},
+			expectedCoins: sdk.NewCoins(sdk.NewCoin("atom", osmomath.NewInt(100000000))),
+		},
+		{
+			name:          "Multiple coins",
+			coinArray:     []sdk.Coin{sdk.NewCoin("atom", osmomath.NewInt(100000000)), sdk.NewCoin("usdc", osmomath.NewInt(500000000))},
+			expectedCoins: sdk.NewCoins(sdk.NewCoin("atom", osmomath.NewInt(100000000)), sdk.NewCoin("usdc", osmomath.NewInt(500000000))),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			result := osmoutils.ConvertCoinArrayToCoins(test.coinArray)
+			require.Equal(t, result, test.expectedCoins)
+
+		})
+	}
+}

proto/osmosis/poolmanager/v1beta1/genesis.proto

@@ -42,6 +42,10 @@ message GenesisState {
   Params params = 2 [ (gogoproto.nullable) = false ];
   // pool_routes is the container of the mappings from pool id to pool type.
   repeated ModuleRoute pool_routes = 3 [ (gogoproto.nullable) = false ];
+
+  // KVStore state
+  TakerFeesToStakersTracker taker_fees_to_stakers_tracker = 4;
+  TakerFeesToCommunityPoolTracker taker_fees_to_community_pool_tracker = 5;
 }
 
 // TakerFeeParams consolidates the taker fee parameters for the poolmanager.
@@ -117,3 +121,21 @@ message TakerFeeDistributionPercentage {
     (gogoproto.nullable) = false
   ];
 }
+
+message TakerFeesToStakersTracker {
+  repeated cosmos.base.v1beta1.Coin taker_fees_to_stakers = 1 [
+    (gogoproto.nullable) = false,
+    (gogoproto.castrepeated) = "github.com/cosmos/cosmos-sdk/types.Coins"
+  ];
+  uint64 height_accounting_starts_from = 2
+      [ (gogoproto.moretags) = "yaml:\"height_accounting_starts_from\"" ];
+}
+
+message TakerFeesToCommunityPoolTracker {
+  repeated cosmos.base.v1beta1.Coin taker_fees_to_community_pool = 1 [
+    (gogoproto.nullable) = false,
+    (gogoproto.castrepeated) = "github.com/cosmos/cosmos-sdk/types.Coins"
+  ];
+  uint64 height_accounting_starts_from = 2
+      [ (gogoproto.moretags) = "yaml:\"height_accounting_starts_from\"" ];
+}

proto/osmosis/protorev/v1beta1/genesis.proto

@@ -72,4 +72,6 @@ message GenesisState {
     (gogoproto.nullable) = false,
     (gogoproto.moretags) = "yaml:\"info_by_pool_type\""
   ];
-}
+  CyclicArbTracker cyclic_arb_tracker = 14
+      [ (gogoproto.moretags) = "yaml:\"cyclic_arb_tracker\"" ];
+}

proto/osmosis/protorev/v1beta1/protorev.proto

@@ -5,6 +5,8 @@ import "cosmos_proto/cosmos.proto";
 import "gogoproto/gogo.proto";
 
 import "cosmos/base/v1beta1/coin.proto";
+import "osmosis/poolmanager/v1beta1/genesis.proto";
+import "osmosis/txfees/v1beta1/genesis.proto";
 
 option go_package = "github.com/osmosis-labs/osmosis/v20/x/protorev/types";
 
@@ -179,4 +181,34 @@ message BaseDenom {
     (gogoproto.nullable) = false,
     (gogoproto.moretags) = "yaml:\"step_size\""
   ];
-}
+}
+
+message AllProtocolRevenue {
+  osmosis.poolmanager.v1beta1.TakerFeesToStakersTracker
+      taker_fees_to_stakers_tracker = 1 [
+    (gogoproto.moretags) = "yaml:\"taker_fees_to_stakers_tracker\"",
+    (gogoproto.nullable) = false
+  ];
+  osmosis.poolmanager.v1beta1.TakerFeesToCommunityPoolTracker
+      taker_fees_to_community_pool_tracker = 2 [
+    (gogoproto.moretags) = "yaml:\"taker_fees_to_community_pool_tracker\"",
+    (gogoproto.nullable) = false
+  ];
+  osmosis.txfees.v1beta1.TxFeesTracker tx_fees_tracker = 3 [
+    (gogoproto.moretags) = "yaml:\"tx_fees_tracker\"",
+    (gogoproto.nullable) = false
+  ];
+  CyclicArbTracker cyclic_arb_tracker = 4 [
+    (gogoproto.moretags) = "yaml:\"cyclic_arb_tracker\"",
+    (gogoproto.nullable) = false
+  ];
+}
+
+message CyclicArbTracker {
+  repeated cosmos.base.v1beta1.Coin cyclic_arb = 1 [
+    (gogoproto.nullable) = false,
+    (gogoproto.castrepeated) = "github.com/cosmos/cosmos-sdk/types.Coins"
+  ];
+  uint64 height_accounting_starts_from = 2
+      [ (gogoproto.moretags) = "yaml:\"height_accounting_starts_from\"" ];
+}

proto/osmosis/protorev/v1beta1/query.proto

@@ -113,6 +113,13 @@ service Query {
       returns (QueryGetProtoRevPoolResponse) {
     option (google.api.http).get = "/osmosis/protorev/pool";
   }
+
+  // GetAllProtocolRevenue queries all of the protocol revenue that has been
+  // accumulated by any module
+  rpc GetAllProtocolRevenue(QueryGetAllProtocolRevenueRequest)
+      returns (QueryGetAllProtocolRevenueResponse) {
+    option (google.api.http).get = "/osmosis/protorev/all_protocol_revenue";
+  }
 }
 
 // QueryParamsRequest is request type for the Query/Params RPC method.
@@ -324,4 +331,13 @@ message QueryGetProtoRevPoolRequest {
 message QueryGetProtoRevPoolResponse {
   // pool_id is the pool_id stored for the denom pair
   uint64 pool_id = 1 [ (gogoproto.moretags) = "yaml:\"pool_id\"" ];
+}
+
+message QueryGetAllProtocolRevenueRequest {}
+
+message QueryGetAllProtocolRevenueResponse {
+  AllProtocolRevenue all_protocol_revenue = 1 [
+    (gogoproto.moretags) = "yaml:\"all_protocol_revenue\"",
+    (gogoproto.nullable) = false
+  ];
 }

proto/osmosis/txfees/v1beta1/genesis.proto

@@ -3,11 +3,24 @@ package osmosis.txfees.v1beta1;
 
 import "gogoproto/gogo.proto";
 import "osmosis/txfees/v1beta1/feetoken.proto";
+import "cosmos/base/v1beta1/coin.proto";
 
 option go_package = "github.com/osmosis-labs/osmosis/v20/x/txfees/types";
 
 // GenesisState defines the txfees module's genesis state.
 message GenesisState {
   string basedenom = 1;
   repeated FeeToken feetokens = 2 [ (gogoproto.nullable) = false ];
+
+  // KVStore state
+  TxFeesTracker txFeesTracker = 3;
 }
+
+message TxFeesTracker {
+  repeated cosmos.base.v1beta1.Coin tx_fees = 1 [
+    (gogoproto.nullable) = false,
+    (gogoproto.castrepeated) = "github.com/cosmos/cosmos-sdk/types.Coins"
+  ];
+  uint64 height_accounting_starts_from = 2
+      [ (gogoproto.moretags) = "yaml:\"height_accounting_starts_from\"" ];
+}

x/epochs/go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/golang/protobuf v1.5.3
 	github.com/gorilla/mux v1.8.0
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0
-	github.com/osmosis-labs/osmosis/osmoutils v0.0.7-0.20231011004221-fd24b80f8366
+	github.com/osmosis-labs/osmosis/osmoutils v0.0.7-0.20231101190541-d45713e67d41
 	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.8.4
 	github.com/tendermint/tendermint v0.37.0-rc1
@@ -89,7 +89,7 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mtibben/percent v0.2.1 // indirect
 	github.com/opencontainers/runc v1.1.5 // indirect
-	github.com/osmosis-labs/osmosis/osmomath v0.0.6 // indirect
+	github.com/osmosis-labs/osmosis/osmomath v0.0.7-0.20231011004221-fd24b80f8366 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
@@ -124,6 +124,7 @@ require (
 	go.etcd.io/bbolt v1.3.6 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
 	golang.org/x/crypto v0.13.0 // indirect
 	golang.org/x/net v0.15.0 // indirect
 	golang.org/x/sys v0.12.0 // indirect