Fabo/query past rewards script (#564)

* initial working state * working, this time is true ;-) * fix validator, comments * fix, cleanup * fix validator * update comment * Use coinReducer * cleanup * cleanup unused vars * cleanup * Flatten era rewards and then aggregate per valid * refactored * typo * uüdated polkadot * load rewards * steps towards storing the rewards * fix era format * fix validator in row needs to be address * fix gql query * format bug * some refactoring * fix missing await * correctly get api * push validator getter inside sources * added address to lunie reward internaly * add address correctly * fix variable naming * filter empty rewards * add height from rewards * added query past rewards script * calling main * missing imports * upsert to prevent conflicts * flatten rewards * fix too many labels issue * typo * write for each validator * await finishing * add validators correctly to the store * remove timing for rewards * correctly write validator and filter 0 * add check and filter out undefined validators * fix amount decimals to 9. allow validator unknown * added address better * refactor * fix tests * only add needed columns to rows * missing refactor * fix getoldpolkadot rewards eras script * possible fix to lastreward of undefined * delete forgotten only * typo Co-authored-by: mariopino <[email protected]> Co-authored-by: Bitcoinera <[email protected]>
luniehq · Apr 6, 2020 · f9c9fc2 · f9c9fc2
1 parent 45512bf
commit f9c9fc2
Show file tree

Hide file tree

Showing 3 changed files with 68 additions and 5 deletions.
diff --git a/lib/reducers/polkadotV0-reducers.js b/lib/reducers/polkadotV0-reducers.js
@@ -222,12 +222,14 @@ function rewardsReducer(network, validators, rewards, reducers) {
 function rewardReducer(network, validators, reward, reducers) {
   let parsedRewards = []
   Object.entries(reward.validators).map(validatorReward => {
-    const reward = {
+    const lunieReward = {
       ...reducers.coinReducer(network, validatorReward[1].toString()),
+      height: reward.era,
+      address: reward.address,
       validator: validators[validatorReward[0]],
       validatorAddress: validatorReward[0] // added for writing the validator to the db even it it is not in the dictionary
     }
-    parsedRewards.push(reward)
+    parsedRewards.push(lunieReward)
   })
   return parsedRewards
 }

diff --git a/lib/source/polkadotV0-source.js b/lib/source/polkadotV0-source.js
@@ -164,20 +164,25 @@ class polkadotAPI {
     // Example execution times:
     // - 10-15s for an account with 1 pending payout
     // - Hangs API for an account with >= 20 eras pending payouts
-    console.time(`getRewards`)
     const api = await this.getAPI()
     const validators = this.store.validators
     const stakingInfo = await api.derive.staking.query(delegatorAddress)
+    if (!stakingInfo.stakingLedger) {
+      // in this case we assume the delegator has no rewards
+      return []
+    }
     const lastEraReward = parseInt(stakingInfo.stakingLedger.lastReward) + 1
     const rewards = await api.derive.staking.stakerRewards(
       delegatorAddress,
       lastEraReward
     )
-    console.timeEnd(`getRewards`)
     return this.reducers.rewardsReducer(
       this.network,
       validators,
-      rewards,
+      rewards.map(reward => ({
+        ...reward,
+        address: delegatorAddress
+      })),
       this.reducers
     )
   }
@@ -196,6 +201,7 @@ class polkadotAPI {
     return delegators
   }
 
+  // ATTENTION: era since to putting a low number will take a long time
   async getEraRewards(era) {
     const api = await this.getAPI()
     const delegators = this.getAllDelegators()

diff --git a/scripts/getOldPolkadotRewardEras.js b/scripts/getOldPolkadotRewardEras.js
@@ -0,0 +1,55 @@
+const database = require('../lib/database')
+const config = require('../config')
+const db = database(config)('polkadot_testnet')
+const { ApiPromise, WsProvider } = require('@polkadot/api')
+const _ = require('lodash')
+
+async function initPolkadotRPC(network, store) {
+  console.time('init polkadot')
+  const api = new ApiPromise({
+    provider: new WsProvider(network.rpc_url)
+  })
+  store.polkadotRPC = api
+  await api.isReady
+  console.timeEnd('init polkadot')
+}
+
+function storeRewards(rewards, chainId) {
+  return db.upsert('rewards', rewards, undefined, chainId) // height is in the rewards rows already
+}
+
+async function main() {
+  const networks = require('../data/networks')
+  const network = networks.find(({ id }) => id === 'polkadot-testnet')
+  const PolkadotApiClass = require('../lib/' + network.source_class_name)
+  const store = {}
+  await initPolkadotRPC(network, store)
+  const polkadotAPI = new PolkadotApiClass(network, store)
+
+  const validators = await polkadotAPI.getAllValidators()
+  store.validators = _.keyBy(validators, 'operatorAddress')
+  const delegators = await polkadotAPI.getAllDelegators()
+
+  await Promise.all(
+    delegators.map(async delegator => {
+      const rewards = await polkadotAPI.getRewards(delegator)
+      const storableRewards = rewards
+        ? rewards.filter(({ amount }) => amount > 0)
+        : []
+      if (storableRewards.length > 0) {
+        await storeRewards(
+          rewards.map(reward => ({
+            amount: reward.amount,
+            height: reward.height,
+            denom: reward.denom,
+            address: reward.address,
+            validator: reward.validatorAddress
+          })),
+          validators[0].chainId
+        )
+      }
+    })
+  )
+}
+
+main()