Fabo/load all txs (even if more then first page in response) (#270)

* load all txs (even if more then first page in response) * improved handling of txs * missing renaming * fixed paginated load * add pagination fix also to cosmosV0-source Co-authored-by: iambeone <[email protected]> Co-authored-by: Ana G. <[email protected]>
luniehq · Jan 17, 2020 · 1208ff8 · 1208ff8
1 parent e0e3d65
commit 1208ff8
Show file tree

Hide file tree

Showing 2 changed files with 51 additions and 23 deletions.
diff --git a/lib/source/cosmosV0-source.js b/lib/source/cosmosV0-source.js
@@ -65,7 +65,7 @@ class CosmosV0API extends RESTDataSource {
   }
 
   async getTransactionsByHeight(height) {
-    const txs = await this.get(`txs?tx.height=${height}`)
+    const txs = await this.loadPaginatedTxs(`txs?tx.height=${height}`)
     return Array.isArray(txs)
       ? txs.map(transaction =>
           this.reducers.transactionReducer(transaction, this.reducers)
@@ -453,30 +453,46 @@ class CosmosV0API extends RESTDataSource {
     )
   }
 
+  async loadPaginatedTxs(url, page = 1, totalAmount = 0) {
+    const pagination = `&limit=1000000000&page=${page}`
+    let allTxs = []
+
+    const { txs, total_count } = await this.get(`${url}${pagination}`)
+    allTxs = allTxs.concat(txs)
+
+    // there is a bug in page_number in gaia-13007 so we can't use is
+    if (allTxs.length + totalAmount < Number(total_count)) {
+      return allTxs.concat(
+        await this.loadPaginatedTxs(url, page + 1, totalAmount + allTxs.length)
+      )
+    }
+
+    return allTxs
+  }
+
   async getTransactions(address) {
     this.checkAddress(address)
-    const pagination = `&limit=${1000000000}`
 
     const txs = await Promise.all([
-      this.get(`/txs?sender=${address}${pagination}`),
-      this.get(`/txs?recipient=${address}${pagination}`),
-      this.get(`/txs?action=submit_proposal&proposer=${address}${pagination}`),
-      this.get(`/txs?action=deposit&depositor=${address}${pagination}`),
-      this.get(`/txs?action=vote&voter=${address}${pagination}`),
+      this.loadPaginatedTxs(`/txs?sender=${address}`),
+      this.loadPaginatedTxsget(`/txs?recipient=${address}`),
+      this.loadPaginatedTxs(`/txs?action=submit_proposal&proposer=${address}`),
+      this.loadPaginatedTxs(`/txs?action=deposit&depositor=${address}`),
+      this.loadPaginatedTxs(`/txs?action=vote&voter=${address}`),
       // this.get(`/txs?action=create_validator&destination-validator=${valAddress}`), // TODO
       // this.get(`/txs?action=edit_validator&destination-validator=${valAddress}`), // TODO
-      this.get(`/txs?action=delegate&delegator=${address}${pagination}`),
-      this.get(
-        `/txs?action=begin_redelegate&delegator=${address}${pagination}`
+      this.loadPaginatedTxs(`/txs?action=delegate&delegator=${address}`),
+      this.loadPaginatedTxs(
+        `/txs?action=begin_redelegate&delegator=${address}`
       ),
-      this.get(`/txs?action=begin_unbonding&delegator=${address}${pagination}`),
+      this.loadPaginatedTxs(`/txs?action=begin_unbonding&delegator=${address}`),
       // this.get(`/txs?action=unjail&source-validator=${address}`), // TODO
       // this.get(`/txs?action=set_withdraw_address&delegator=${address}`), // other
-      this.get(
-        `/txs?action=withdraw_delegator_reward&delegator=${address}${pagination}`
+      this.loadPaginatedTxs(
+        `/txs?action=withdraw_delegator_reward&delegator=${address}`
       ),
-      this.get(
-        `/txs?action=withdraw_validator_rewards_all&source-validator=${address}${pagination}`
+      this.loadPaginatedTxs(
+        `/txs?action=withdraw_validator_rewards_all&source-validator=${address}`
       )
     ]).then(transactionGroups => [].concat(...transactionGroups))
 

diff --git a/lib/source/cosmosV2-source.js b/lib/source/cosmosV2-source.js
@@ -16,17 +16,29 @@ class CosmosV2API extends CosmosV0API {
     return signingInfos
   }
 
+  async loadPaginatedTxs(url, page = 1, totalAmount = 0) {
+    const pagination = `&limit=1000000000&page=${page}`
+    let allTxs = []
+
+    const { txs, total_count } = await this.get(`${url}${pagination}`)
+    allTxs = allTxs.concat(txs)
+
+    // there is a bug in page_number in gaia-13007 so we can't use is
+    if (allTxs.length + totalAmount < Number(total_count)) {
+      return allTxs.concat(
+        await this.loadPaginatedTxs(url, page + 1, totalAmount + allTxs.length)
+      )
+    }
+
+    return allTxs
+  }
+
   async getTransactions(address) {
     this.checkAddress(address)
-    const pagination = `&limit=${1000000000}`
 
     const txs = await Promise.all([
-      this.get(`/txs?message.sender=${address}${pagination}`).then(
-        ({ txs }) => txs
-      ),
-      this.get(`/txs?transfer.recipient=${address}${pagination}`).then(
-        ({ txs }) => txs
-      )
+      this.loadPaginatedTxs(`/txs?message.sender=${address}`),
+      this.loadPaginatedTxs(`/txs?transfer.recipient=${address}`)
     ]).then(([senderTxs, recipientTxs]) => [].concat(senderTxs, recipientTxs))
 
     const dupFreeTxs = uniqWith(txs, (a, b) => a.txhash === b.txhash)
@@ -66,7 +78,7 @@ class CosmosV2API extends CosmosV0API {
   }
 
   async getTransactionsByHeight(height) {
-    const { txs } = await this.get(`txs?tx.height=${height}`)
+    const txs = await this.loadPaginatedTxs(`txs?tx.height=${height}`)
     return Array.isArray(txs)
       ? txs.map(transaction =>
           this.reducers.transactionReducer(transaction, this.reducers)