From adc4474c64238db1eaffe0509635a753c93c6fb3 Mon Sep 17 00:00:00 2001 From: Abdul Rabbani Date: Mon, 27 Jun 2022 15:49:10 -0400 Subject: [PATCH] Commits for node_id array commit dc8b5f0349377e790a6a35feaa52e784c0f1c316 Author: Abdul Rabbani Date: Mon Jun 20 09:57:39 2022 -0400 Duplicate job check commit 867d0104c580cf28c6c4554f6ae327a6b5cdf051 Author: Abdul Rabbani Date: Mon Jun 20 09:54:37 2022 -0400 update db ref commit ad6c51f749e7252e1ab590c0495b9f3bfb502bac Merge: 0a531d011 d629c99d8 Author: Abdul Rabbani Date: Mon Jun 20 09:52:19 2022 -0400 Merge branch 'v1.10.19-statediff-v4' into feature/node_id_array commit 0a531d011a85a6bb943cdc9c7f3213145a2951f0 Author: Abdul Rabbani Date: Mon Jun 20 09:44:08 2022 -0400 update DB ref commit 21df2fbdbe5f0d5b975d9a30f569ea05c34f2135 Author: Abdul Rabbani Date: Wed Jun 15 08:22:24 2022 -0400 minor syntax change commit 4e4d4c27a9cf0bd85a1b72121ca37df99403b6c2 Author: Abdul Rabbani Date: Tue Jun 14 12:32:49 2022 -0400 Update port for testing commit 8e2dec1ffe73d48291598f5498cfa6273502093a Author: Abdul Rabbani Date: Tue Jun 14 12:24:26 2022 -0400 Update compose files commit aee387decd8bbc89adb4d112f9821f686b45fddc Author: Abdul Rabbani Date: Tue Jun 14 12:15:05 2022 -0400 Add uniqueness commit 1c1a460ba48d313fe4423c14351ca6ad22d6e6ec Author: Abdul Rabbani Date: Mon Jun 13 18:43:10 2022 -0400 Update default config commit fce04b542405e62aa84bc65b0e1611c8d9cda2d6 Author: Abdul Rabbani Date: Mon Jun 13 18:37:56 2022 -0400 Use stack orchestrator properly commit 573fdfd920b3d68a812c932d1c15b7639d79d399 Author: Abdul Rabbani Date: Mon Jun 13 18:29:30 2022 -0400 Update "file" and CICD commit d2ee36dd9529bdc713bfb7d4226c755b82feba53 Author: Abdul Rabbani Date: Mon Jun 13 16:06:43 2022 -0400 Update upsert for eth.header_cids commit 8593cfd8ec814f877e8ad2bee2c063b95049b124 Author: Abdul Rabbani Date: Fri May 27 09:40:24 2022 -0400 Update Lt condition commit 10460f4c11335db47390f1e5d893cd19a6db7905 Author: Abdul Rabbani Date: Fri May 27 09:24:46 2022 -0400 Use correct DB and fix syntax commit 6a44b2e1f9ad5adf74db725231b8b4fcc22839ba Author: Abdul Rabbani Date: Fri May 27 09:16:38 2022 -0400 Update the read rows command. commit 0a4079f8bb825dc9af3efb547793cc0891849ec1 Author: Abdul Rabbani Date: Fri May 27 09:03:57 2022 -0400 Fix two tests commit 07ce01d6053efd1479c9895d106b10f0ffb105c1 Author: Abdul Rabbani Date: Fri May 27 08:57:34 2022 -0400 Update test for v4 commit 2bf46e96aeaebdb3d921006f9232a5f3c4a4f9b3 Author: Abdul Rabbani Date: Fri May 27 08:52:17 2022 -0400 Update branch commit 957013e06c3bcc50a18bc3f82f4658181245189c Author: Abdul Rabbani Date: Fri May 27 08:50:52 2022 -0400 Update the test commit 1277fa1312d80048a2e8effd59f16da34f266e44 Author: Abdul Rabbani Date: Fri May 27 08:09:37 2022 -0400 Update CI/CD and match v3 commit 6d5f4ac11181decabfcfa2ffb464875bd81267a5 Merge: af077d7f0 2ca714701 Author: Abdul Rabbani Date: Fri May 27 08:08:52 2022 -0400 Merge branch 'heads/v1.10.18-statediff-3.2.2' into v1.10.18-statediff-v4 commit 2ca71470126cd17ea4caaf3f93ec52ade3e016d1 Merge: 51e8824f7 ab9e73a9c Author: Abdul Rabbani <58230246+abdulrabbani00@users.noreply.github.com> Date: Thu May 26 15:42:34 2022 -0400 Merge pull request #233 from vulcanize/feature/cicd-improvements Feature/cicd improvements commit ab9e73a9c26c3de44331154c308d61c57e51ad2f Author: Abdul Rabbani Date: Thu May 26 15:19:50 2022 -0400 Update readme. commit 1884af321ddd71f39c4f83f93ee21a9842a00973 Author: Abdul Rabbani Date: Thu May 26 15:14:46 2022 -0400 Update CICD commit af077d7f04bdae3f3f002f843ab251aa11f95222 Author: Abdul Rabbani Date: Wed May 25 16:26:58 2022 -0400 Update go.sum commit cef5f08e38c6a802ff6464a573237d377ca286e3 Merge: ef8c9f258 de23cf910 Author: Abdul Rabbani Date: Wed May 25 16:23:37 2022 -0400 Merge tag 'v1.10.18' into v1.10.18-statediff-v4 commit 51e8824f7a23fe313bdfa5d2cdba9d2615df3cdd Merge: 41faf71c0 de23cf910 Author: Abdul Rabbani Date: Wed May 25 14:45:55 2022 -0400 Merge tag 'v1.10.18' into v1.10.18-statediff-v3 commit 41faf71c057a995979b45d7b97806b3181188102 Merge: 17d272d83 d57c260f2 Author: Abdul Rabbani <58230246+abdulrabbani00@users.noreply.github.com> Date: Tue May 17 16:53:08 2022 -0400 Merge pull request #229 from vulcanize/feature/cicd-update-docker Combine build and push for Docker commit d57c260f2938f81a9384eef355d2ac8b45e676ce Author: Abdul Rabbani Date: Tue May 17 16:28:23 2022 -0400 Combine build and push commit ef8c9f2580dd577a7c5eca538fb0ed64d53dc4a4 Merge: 328ab956d 2caedb0c3 Author: Ashwin Phatak Date: Tue May 17 17:21:03 2022 +0530 Merge pull request #228 from deep-stack/pm-single-node-changes Changes for using single-node v4 db schema commit 2caedb0c36198abdf1f639144fdb43075db2a60f Author: prathamesh0 Date: Tue May 17 14:12:10 2022 +0530 Changes for using single-node v4 db schema commit 328ab956df4c40f64cceddb202322de0e6d2eca7 Merge: cef1fc425 41c9ea064 Author: Ashwin Phatak Date: Mon May 16 12:05:27 2022 +0530 Merge pull request #227 from deep-stack/pm-v4-merge Merge branch 'v1.10.17-statediff-3.2.1' into v4 commit 41c9ea064c923b4d2b4252ef62e5d77e66cf1bbc Author: prathamesh0 Date: Mon May 16 11:37:31 2022 +0530 Add new target branch for github actions commit a6891a65a260e3d23f7c96b1e27d5acda62115ca Merge: cef1fc425 17d272d83 Author: prathamesh0 Date: Mon May 16 10:52:45 2022 +0530 Merge branch 'v1.10.17-statediff-3.2.1' into v4 commit 17d272d837e49eecfdfe970a02b2bb64f77cdced Merge: 831685993 4318911a5 Author: Ashwin Phatak Date: Fri May 13 16:09:34 2022 +0530 Merge pull request #226 from deep-stack/ng-public-WatchedAddressesLeafKeys Export builder config param WatchedAddressesLeafKeys commit 4318911a5c57ebe546ec7627fe97b868f6382918 Author: nabarun Date: Thu May 12 19:27:14 2022 +0530 Move over helper function for loading chain config commit aa3fa24bef18e4f1299155a0950e797531797b07 Author: nabarun Date: Thu May 12 15:49:58 2022 +0530 Implemented method to get private watchedAddressesLeafKeys commit cef1fc425fe477dd1eae8be474c02b9a68f02bc8 Merge: de0954a44 8a4583c48 Author: Ashwin Phatak Date: Thu May 12 14:43:06 2022 +0530 Merge pull request #225 from deep-stack/pm-v4-schema-fixes Updates to use v4 multi-node DB setup commit 24058dc9dfaed706df852bc5b925b4abd9e55859 Author: nabarun Date: Wed May 11 18:10:20 2022 +0530 Export builder config param WatchedAddressesLeafKeys commit 8a4583c48ce93993c48cc87ae0bcf905ccc6a183 Author: prathamesh0 Date: Wed May 11 16:15:53 2022 +0530 Run tests using multi-node database setup commit d2c25c6a100b400cc368c2f68caa7f8abf63705c Author: prathamesh0 Date: Thu May 5 19:26:10 2022 +0530 Update more test queries to use COALESCE commit 6ae7719c6412bfbe14b23b9fad4e506de6fb80cd Author: prathamesh0 Date: Thu May 5 17:27:02 2022 +0530 Use COALESCE to avoid possible NULL values from database commit ecba34cdac7585b0a6f209a5b2a5ced056b74b25 Author: prathamesh0 Date: Mon May 2 17:08:01 2022 +0530 Change ON CONFLICT clauses to do nothing commit 9db8c59144e9e0f102d519cccac3e455a225089c Author: prathamesh0 Date: Tue Apr 26 11:03:13 2022 +0530 Update ipld-eth-db build configuration in docker-compose file commit 8551aaee810abaa4d7470aba023b2cc11f734307 Author: prathamesh0 Date: Thu Apr 21 17:43:58 2022 +0530 Fix database insert statements according to v4 schema commit d2144cad4bfc96d6d04a5ae9f1090d5b058f2eb0 Author: prathamesh0 Date: Tue Apr 19 10:40:30 2022 +0530 Fix assertion errors commit de0954a449818bacaac6e5a826a9a8b3aa6675d7 Merge: 4e73e1710 831685993 Author: prathamesh0 Date: Wed May 11 13:42:51 2022 +0530 Merge latest v3 changes into v4 commit 831685993817b2637b682f576ee1803c25689bda Author: Abdul Rabbani Date: Fri Apr 22 08:45:10 2022 -0400 Remove `related-repository` commit 498101102c891c4f8c3cab5649158c642ee1fd6b Merge: e5824ecbe e6ac85fef Author: Abdul Rabbani <58230246+abdulrabbani00@users.noreply.github.com> Date: Wed Apr 6 14:44:49 2022 -0400 Merge pull request #222 from vulcanize/v1.10.17-statediff-v3.2.0 Handle all breaking changes for the patch commit e6ac85fefce187f568fcc69125207fe2492eb081 Author: Abdul Rabbani Date: Wed Apr 6 13:51:03 2022 -0400 Remove redundant deferences for pointers commit 3477d35af46f619295d90e46c6a4998444da2ce6 Author: Abdul Rabbani Date: Wed Apr 6 10:57:57 2022 -0400 Handle all breaking changes for the patch The major change to integrate the patch has been: * Updating the arguments for `rlp.EncodeToBytes`. This function accepts interfaces, but since the last update, it is better to pass in a pointer to the function. * From the Ethereum Release Notes: " Compatibility note about `core/types`: For optimization purposes, `types.Header` and other types in this package now implement the `rlp.Encoder` interface. This change can cause incompatibilities because the new method is implemented with pointer receiver. Attempting to RLP-encode unadressable (i.e. non-pointer) values of type `Header` does not work anymore and will result in an error." * Instead of just updating all the headers. I have updated all parameters for the `rlp.EncodeToBytes` to be pointers instead of values. __Please take a close look at the updates__. The functions won't fail if a non-pointer is passed (in most cases), but we could be unexpected behaviors. commit e5824ecbe9fd313d3c29a216f3012e95beaae942 Merge: a7caed41d 3ef54d3dd Author: Abdul Rabbani <58230246+abdulrabbani00@users.noreply.github.com> Date: Tue Apr 5 15:50:45 2022 -0400 Merge pull request #221 from vulcanize/v1.10.17-statediff-v3.2.0 Update a few go dependencies + GHA workflow commit 3ef54d3ddebb0d14e5d1fa34fa8cf06ead6b3f80 Author: Abdul Rabbani Date: Tue Apr 5 15:49:41 2022 -0400 Update a few go dependencies + GHA workflow commit a7caed41dfad2b61fdbfcfbd204ef1ea46e239a2 Merge: a9ab76cb3 25c9b49fd Author: Abdul Rabbani Date: Tue Apr 5 15:30:47 2022 -0400 Merge tag 'v1.10.17' into v1.10.17-statediff-v3 commit a9ab76cb3361c9f650befe4384c174b5e0e72c0c Merge: 2aaf6bcda b351f8f33 Author: Ashwin Phatak Date: Tue Apr 5 14:21:19 2022 +0530 Merge pull request #210 from deep-stack/pm-removed-storage-nodes Add diffs for storage nodes of destroyed contracts commit b351f8f335323f6e123cb7d45b9ee24c17f27086 Author: prathamesh0 Date: Tue Apr 5 12:38:14 2022 +0530 Increase timeout in a failing test commit c21aa96962ef5af5b5066ff66b026bb89e8c0282 Author: prathamesh0 Date: Tue Mar 22 18:04:19 2022 +0530 Index Removed diffs without leaf keys if an account or storage node is moved commit 9917e6c293f829b6b36cfb599d3cafe66bfa11b7 Author: prathamesh0 Date: Thu Mar 17 18:04:20 2022 +0530 Fix linting errors commit 84365431d183e9473046ce70f874064b7c3c2ad4 Author: prathamesh0 Date: Wed Mar 16 11:44:01 2022 +0530 Update indexer unit tests commit a14745dac57d0e74e53da4f0d3c5840459c72b38 Author: nabarun Date: Tue Mar 15 19:22:09 2022 +0530 Updated unit tests for changes in builder commit 4855d9304e80978e8fcf3fafb9d52662bdcd09e4 Author: prathamesh0 Date: Tue Mar 15 17:10:26 2022 +0530 Index Removed storage diffs on contract destruction commit 2aaf6bcda3bb17f180a52ce4db534c13a1b34d47 Merge: c8b68db48 52df3460d Author: Ashwin Phatak Date: Tue Apr 5 09:33:51 2022 +0530 Merge pull request #212 from deep-stack/pm-watched-addresses-v3 Statediff API (v3) to change addresses watched in direct indexing mode commit 52df3460d6cb74acd95284ef34d083403a461d5f Author: prathamesh0 Date: Mon Mar 21 18:03:48 2022 +0530 Create file for watched addresses only when required commit 710312378a630b16da25c3614049982c9e54512a Author: prathamesh0 Date: Mon Mar 21 14:34:12 2022 +0530 Add indexer unit tests for loading watched addresses commit e7526469db93e03f25e269d0f7230673aeb12d1d Author: prathamesh0 Date: Fri Mar 18 19:49:41 2022 +0530 Add file mode indexer unit tests for watched address methods commit a7f9354f0eee2cca87d3f8da9c752719313436e7 Author: prathamesh0 Date: Fri Mar 18 18:29:15 2022 +0530 Implement watch address indexer methods for file mode commit 12b9f50930a88713005364df07e999ec405c3c18 Author: prathamesh0 Date: Thu Mar 17 10:49:18 2022 +0530 Update context used in watch address indexer functions commit 750e44e9e69bc0658047e17622410ffd7742a9f2 Author: prathamesh0 Date: Wed Mar 16 20:14:44 2022 +0530 Add builder test for removal of watched addresses commit f3761efe76d8df5b7c2a57d2ee6e03a89573ce4e Author: prathamesh0 Date: Wed Mar 16 20:10:06 2022 +0530 Add a fix in builder for removal of a non-watched address commit 7f38afe542a7831ac07ff94db5c5facf7f825088 Author: prathamesh0 Date: Wed Mar 16 19:25:10 2022 +0530 Add tests for the API to change addresses being watched commit af4dbed9d21b3a3ff3d99e6fc168e5751fe9288c Author: prathamesh0 Date: Wed Mar 16 19:01:55 2022 +0530 Add tests for indexer methods used to change addresses being watched commit 8fc2c09634388c5fe33da26ced6e22d03902642f Author: prathamesh0 Date: Wed Mar 16 18:43:39 2022 +0530 Add placeholder indexer methods for file and dump modes commit 9b234ef5b4d943dfb82ddb94761d8ed9acee62d3 Author: prathamesh0 Date: Wed Mar 16 15:35:33 2022 +0530 Statediff API to change addresses being watched in direct indexing commit c8b68db48c6645436d0fcf9883ed372d36bbd198 Merge: 3045068a4 d24a49ef0 Author: Abdul Rabbani <58230246+abdulrabbani00@users.noreply.github.com> Date: Thu Mar 31 14:38:59 2022 -0400 Merge pull request #217 from vulcanize/feature/known-gaps Feature/known gaps commit d24a49ef0356ad8049d59d8b3269a86984850f46 Author: Abdul Rabbani Date: Thu Mar 31 14:14:34 2022 -0400 Add Workflow dispatch for docker builds commit 102f874c63bcb4551b3059246db3c45cc3ba992d Author: Abdul Rabbani Date: Thu Mar 31 14:11:57 2022 -0400 Update known_gaps table name commit 359799dee20dcde40d6b0bf706cdd4837a9349c7 Author: Abdul Rabbani Date: Thu Mar 31 14:02:30 2022 -0400 Update docker compose commit 2e836e869a27d9b03b573fae5cc448e3cb957220 Author: Abdul Rabbani Date: Thu Mar 31 11:37:01 2022 -0400 PR Updates as per Ian's request commit a60d5a016d1b3953eae6f3a31e107547f5e0284c Author: Abdul Rabbani Date: Wed Mar 30 17:30:19 2022 -0400 Code Clean Up commit fc6a9379e148962008b98600d14c75ba1a972820 Author: Abdul Rabbani Date: Wed Mar 30 12:53:44 2022 -0400 Add Prometheus metrics and write SQL file if it has data. commit b960661807db19950ae718580bd3fccb99ee0be0 Author: Abdul Rabbani Date: Tue Mar 29 16:45:26 2022 -0400 Refactor: Decouple knownGaps and Indexer This commit decouples knownGaps and Indexer. All knownGaps logic is in its own file. This makes testing and maintainability easier. We have also removed all efforts to check the `lastProcessedblock` - This is because we won't ever run into this issue (hyptothetically), because geth won't let it happen. commit fc341a90fccc5389656e1a079d7ff810ea7cfb4a Author: Abdul Rabbani Date: Mon Mar 28 15:00:29 2022 -0400 ** Write to file if you can't write to DB This commit contains many new features: 1. Write the insert statement to file if there is an error when trying to write to DB. 2. Test the write to DB and write to file features in unit tests. They work as expected. 3. Insert logic for calling the knownGaps function (untested). This PR makes a major change to the way statediff is started. Unless it is started in `file` mode, it will create a `file` indexer. This indexer will be used for writing errors to disk. commit f566aa780cfff7db33a2993a6c0e0167c33b72ef Author: Abdul Rabbani Date: Fri Mar 25 16:50:43 2022 -0400 % Checkpoint - Added the feature to write to File if writing to DB errors out. NOT TESTED commit f6ff20eb0e21f15181a0f45f8a6b611a2deb6342 Author: Abdul Rabbani Date: Fri Mar 25 09:54:36 2022 -0400 Handle All Gaps within Geth Including an updated doc which keeps track of events in this PR. commit 4e73e171089bc8ddc5dde816e5cdaed8dc154edc Merge: 6b7431094 1e64f486d Author: Ian Norden Date: Thu Mar 24 10:10:35 2022 -0500 Merge pull request #211 from vulcanize/v1.10.16-statediff-4.0.0 update sql indexer to use new v4 schema commit 1e64f486d8e0671da87caa84013f595c120a9118 Author: i-norden Date: Thu Mar 24 10:09:48 2022 -0500 update unit tests commit 7bc4c7520c3d963086698f10a1181f98e467b69d Author: i-norden Date: Thu Mar 24 08:01:26 2022 -0500 update file writing mode commit 1a3a63d00ecadb3db0b88a19ab1faa49c81725e1 Author: Abdul Rabbani Date: Wed Mar 23 14:03:35 2022 -0400 Find Gaps and at them to `known_gaps` This contains all the logic needed to add a gap to the `known_gaps` table. We can now add this code to various parts of the application to calculate if a gap has occured. commit ce66532fcf94a976e6052267e13e754fa0b26002 Author: Abdul Rabbani Date: Tue Mar 22 15:49:30 2022 -0400 Improve Testing for Insert, add a query function commit 73f46a59ef4c95c1887b8b46c40d30c3672b1efd Author: Abdul Rabbani Date: Tue Mar 22 11:52:35 2022 -0400 Remove the need to pass a transaction when upserting to known_gaps Thanks Ian :D commit bca12f9cdf2dff19cf2dc8fda74bc51008d96ee2 Author: Abdul Rabbani Date: Mon Mar 21 17:43:15 2022 -0400 Delete geth-linux-amd64 commit 921538ba640b9dbfb6b76be20c20aad1d0b37273 Author: Abdul Rabbani Date: Mon Mar 21 16:28:13 2022 -0400 Function To Add entry to `known_gaps` This commit contains all the code needed to update the `known_gaps` table. It uses a generic function `PushKnownGaps` to insert gaps to the database. The current test is very "basic." All it does it insert a dummy gap into the database. As long as their is no error it will return succesfully. I have also removed the foundry subfolder. I am using the `related-repository` setup for testing. commit 3045068a45dbf6b92d9e9a21107ccd7dae46826f Merge: 53c1322cb 333dd3ff4 Author: Abdul Rabbani <58230246+abdulrabbani00@users.noreply.github.com> Date: Fri Mar 18 09:44:12 2022 -0400 Merge pull request #205 from vulcanize/feature/155-waitForSync Add a flag to start processing statediff if we are caught up to the head of the chain commit 333dd3ff4fb7788fc2288709d7dd63ab955a055c Author: Abdul Rabbani Date: Fri Mar 18 08:29:36 2022 -0400 Updates as Per Ian's comments 1. Fix some naming inconsistencies. 2. `GetSyncStatus` returns a bool 3. `Start` handles any `errors` from the `WaitingForSync` function. 4. Add a sleep for the `while` loop. 5. Fix test based on a conditional that would never be met. commit 9775355d2b8ca5f009c6b21229d253541f38bc58 Author: i-norden Date: Thu Mar 17 07:17:04 2022 -0500 update sql indexer to use new v4 schema that denormalizes by block_number for the purposes of partitioning & sharding commit c5a36978f89a2ffe4486c164a2f94ae5aa18bce5 Author: Abdul Rabbani Date: Thu Mar 10 13:33:00 2022 -0500 Fix lint errors commit f468636a09127823e543c676a2a382a8a7c8059d Author: Abdul Rabbani Date: Thu Mar 10 13:24:06 2022 -0500 Update param --> parameter to fix conflicting names commit e4090594771f48231fa1a334708883650a6ec998 Author: Abdul Rabbani Date: Thu Mar 10 13:04:25 2022 -0500 Add a flag to start processing statediff if we are caught up to the head of the chain commit 53c1322cb46d271a29a20cdcda618457a43ceb9c Merge: 6b7431094 e4c451467 Author: Abdul Rabbani <58230246+abdulrabbani00@users.noreply.github.com> Date: Wed Mar 16 14:48:33 2022 -0400 Merge pull request #209 from vulcanize/feature/update-foundry Basic updates, add prints and variables commit e4c451467fb4717f85fba3df84dde9c9cc4d8312 Author: Abdul Rabbani Date: Wed Mar 16 14:47:31 2022 -0400 Basic updates, add prints and variables commit 6b74310941fc99e352b05c5785a2ec9aad7c19b2 Merge: f4b7bd4a1 0d70b7ce0 Author: Abdul Rabbani <58230246+abdulrabbani00@users.noreply.github.com> Date: Wed Mar 16 08:04:02 2022 -0400 Merge pull request #208 from vulcanize/feature/gh-208-add-foundry Add Foundry for Manual Integration Testing commit 0d70b7ce042e8acd24d7ace54bf7eb7895aa6afd Author: Abdul Rabbani Date: Tue Mar 15 16:18:50 2022 -0400 Fix Linter commit df55a62430698e145cfed591e516d5ba3a3fb738 Author: Abdul Rabbani Date: Mon Mar 14 15:16:12 2022 -0400 Working Foundry commit 90c89853751a21e0f55ff731145f3704ddcd56fc Author: Abdul Rabbani Date: Mon Mar 14 13:16:03 2022 -0400 forge install: ds-test commit 4704c4416d1c7b9a0592dff626a9b3dcb77d954e Author: Abdul Rabbani Date: Fri Mar 11 17:30:34 2022 -0500 Starting point for Foundry I still need to finish it up, but geth runs properly. commit f4b7bd4a145535eca2c49a8556d3880748bebf2c Author: i-norden Date: Wed Feb 16 14:46:25 2022 -0600 add new target branch for github actions commit 3aead03aeb0d22a0cec6f921fc301f2335e61bb5 Author: Elizabeth Date: Mon Jan 28 15:31:01 2019 -0600 Statediffing geth * Write state diff to CSV (#2) * port statediff from https://github.com/jpmorganchase/quorum/blob/9b7fd9af8082795eeeb6863d9746f12b82dd5078/statediff/statediff.go; minor fixes * integrating state diff extracting, building, and persisting into geth processes * work towards persisting created statediffs in ipfs; based off github.com/vulcanize/eth-block-extractor * Add a state diff service * Remove diff extractor from blockchain * Update imports * Move statediff on/off check to geth cmd config * Update starting state diff service * Add debugging logs for creating diff * Add statediff extractor and builder tests and small refactoring * Start to write statediff to a CSV * Restructure statediff directory * Pull CSV publishing methods into their own file * Reformatting due to go fmt * Add gomega to vendor dir * Remove testing focuses * Update statediff tests to use golang test pkg instead of ginkgo - builder_test - extractor_test - publisher_test * Use hexutil.Encode instead of deprecated common.ToHex * Remove OldValue from DiffBigInt and DiffUint64 fields * Update builder test * Remove old storage value from updated accounts * Remove old values from created/deleted accounts * Update publisher to account for only storing current account values * Update service loop and fetching previous block * Update testing - remove statediff ginkgo test suite file - move mocks to their own dir * Updates per go fmt * Updates to tests * Pass statediff mode and path in through cli * Return filename from publisher * Remove some duplication in builder * Remove code field from state diff output this is the contract byte code, and it can still be obtained by querying the db by the codeHash * Consolidate acct diff structs for updated & updated/deleted accts * Include block number in csv filename * Clean up error logging * Cleanup formatting, spelling, etc * Address PR comments * Add contract address and storage value to csv * Refactor accumulating account row in csv publisher * Add DiffStorage struct * Add storage key to csv * Address PR comments * Fix publisher to include rows for accounts that don't have store updates * Update builder test after merging in release/1.8 * Update test contract to include storage on contract intialization - so that we're able to test that storage diffing works for created and deleted accounts (not just updated accounts). * Factor out a common trie iterator method in builder * Apply goimports to statediff * Apply gosimple changes to statediff * Gracefully exit geth command(#4) * Statediff for full node (#6) * Open a trie from the in-memory database * Use a node's LeafKey as an identifier instead of the address It was proving difficult to find look the address up from a given path with a full node (sometimes the value wouldn't exist in the disk db). So, instead, for now we are using the node's LeafKey with is a Keccak256 hash of the address, so if we know the address we can figure out which LeafKey it matches up to. * Make sure that statediff has been processed before pruning * Use blockchain stateCache.OpenTrie for storage diffs * Clean up log lines and remove unnecessary fields from builder * Apply go fmt changes * Add a sleep to the blockchain test * refactoring/reorganizing packages * refactoring statediff builder and types and adjusted to relay proofs and paths (still need to make this optional) * refactoring state diff service and adding api which allows for streaming state diff payloads over an rpc websocket subscription * make proofs and paths optional + compress service loop into single for loop (may be missing something here) * option to process intermediate nodes * make state diff rlp serializable * cli parameter to limit statediffing to select account addresses + test * review fixes and fixes for issues ran into in integration * review fixes; proper method signature for api; adjust service so that statediff processing is halted/paused until there is at least one subscriber listening for the results * adjust buffering to improve stability; doc.go; fix notifier err handling * relay receipts with the rest of the data + review fixes/changes * rpc method to get statediff at specific block; requires archival node or the block be within the pruning range * fix linter issues * include total difficulty to the payload * fix state diff builder: emit actual leaf nodes instead of value nodes; diff on the leaf not on the value; emit correct path for intermediate nodes * adjust statediff builder tests to changes and extend to test intermediate nodes; golint * add genesis block to test; handle block 0 in StateDiffAt * rlp files for mainnet blocks 0-3, for tests * builder test on mainnet blocks * common.BytesToHash(path) => crypto.Keaccak256(hash) in builder; BytesToHash produces same hash for e.g. []byte{} and []byte{\x00} - prefix \x00 steps are inconsequential to the hash result * complete tests for early mainnet blocks * diff type for representing deleted accounts * fix builder so that we handle account deletions properly and properly diff storage when an account is moved to a new path; update params * remove cli params; moving them to subscriber defined * remove unneeded bc methods * update service and api; statediffing params are now defined by user through api rather than by service provider by cli * update top level tests * add ability to watch specific storage slots (leaf keys) only * comments; explain logic * update mainnet blocks test * update api_test.go * storage leafkey filter test * cleanup chain maker * adjust chain maker for tests to add an empty account in block1 and switch to EIP-158 afterwards (now we just need to generate enough accounts until one causes the empty account to be touched and removed post-EIP-158 so we can simulate and test that process...); also added 2 new blocks where more contract storage is set and old slots are set to zero so they are removed so we can test that * found an account whose creation causes the empty account to be moved to a new path; this should count as 'touching; the empty account and cause it to be removed according to eip-158... but it doesn't * use new contract in unit tests that has self-destruct ability, so we can test eip-158 since simply moving an account to new path doesn't count as 'touchin' it * handle storage deletions * tests for eip-158 account removal and storage value deletions; there is one edge case left to test where we remove 1 account when only two exist such that the remaining account is moved up and replaces the root branch node * finish testing known edge cases * add endpoint to fetch all state and storage nodes at a given blockheight; useful for generating a recent atate cache/snapshot that we can diff forward from rather than needing to collect all diffs from genesis * test for state trie builder * if statediffing is on, lock tries in triedb until the statediffing service signals they are done using them * fix mock blockchain; golint; bump patch * increase maxRequestContentLength; bump patch * log the sizes of the state objects we are sending * CI build (#20) * CI: run build on PR and on push to master * CI: debug building geth * CI: fix coping file * CI: fix coping file v2 * CI: temporary upload file to release asset * CI: get release upload_url by tag, upload asset to current relase * CI: fix tag name * fix ci build on statediff_at_anyblock-1.9.11 branch * fix publishing assets in release * use context deadline for timeout in eth_call * collect and emit codehash=>code mappings for state objects * subscription endpoint for retrieving all the codehash=>code mappings that exist at provided height * Implement WriteStateDiffAt * Writes state diffs directly to postgres * Adds CLI flags to configure PG * Refactors builder output with callbacks * Copies refactored postgres handling code from ipld-eth-indexer * rename PostgresCIDWriter.{index->upsert}* * rm unused * output code & codehash iteratively * had to rf some types for this * prometheus metrics output * duplicate recent eth-indexer changes * migrations and metrics... * [wip] prom.Init() here? another CLI flag? * tidy & DRY * statediff WriteLoop service + CLI flag * [wip] update test mocks * todo - do something meaningful to test write loop * logging * use geth log * port tests to go testing * drop ginkgo/gomega * fix and cleanup tests * fail before defer statement * delete vendor/ dir * fixes after rebase onto 1.9.23 * fix API registration * use golang 1.15.5 version (#34) * bump version meta; add 0.0.11 branch to actions * bump version meta; update github actions workflows * statediff: refactor metrics * Remove redundant statediff/indexer/prom tooling and use existing prometheus integration. * "indexer" namespace for metrics * add reporting loop for db metrics * doc * metrics for statediff stats * metrics namespace/subsystem = statediff/{indexer,service} * statediff: use a worker pool (for direct writes) * fix test * fix chain event subscription * log tweaks * func name * unused import * intermediate chain event channel for metrics * update github actions; linting * add poststate and status to receipt ipld indexes * stateDiffFor endpoints for fetching or writing statediff object by blockhash; bump statediff version * fixes after rebase on to v1.10.1 * update github actions and version meta; go fmt * add leaf key to removed 'nodes' * include Postgres migrations and schema * service documentation * touching up * update github actions after rebase * fix connection leak (misplaced defer) and perform proper rollback on errs * improve error logging; handle PushBlock internal err * build docker image and publish it to Docker Hub on release * add access list tx to unit tests * MarshalBinary and UnmarshalBinary methods for receipt * fix error caused by 2718 by using MarshalBinary instead of EncodeRLP methods * ipld encoding/decoding tests * update TxModel; add AccessListElementModel * index tx type and access lists * add access list metrics * unit tests for tx_type and access list table * unit tests for receipt marshal/unmarshal binary methods * improve documentation of the encoding methods * fix issue identified in linting * update github actions and version meta after rebase * unit test that fails undeterministically on eip2930 txs, giving same error we are seeing in prod * Include genesis block state diff. * documentation on versioning, rebasing, releasing; bump version meta * Add geth and statediff unit test to CI. * Set pgpassword in env. * Added comments. * Add new major branch to github action. * Add support for Dynamic txn(EIP-1559). * Update version meta to 0.0.24 * Verify block base fee in test. * Fix base_fee type and add backward compatible test. * Remove type definition for AccessListElementModel * Change basefee to int64/bigint. * block and uncle reward in PoA network = 0 (#87) * in PoA networks there is no block and uncle rewards * bump meta version * (cherry picked from commit b64ca14689689178b78f915c855fe75010d886c7) * Use Ropsten to test block reward. * Add Makefile target to build static linux binaries. * Strip symbol tables from static binaries. * Fix block_fee to support NULL values. * bump version meta. * Add new major branch to github action. * rename doc.go to README.md * Create a seperate table for storing logs * Bump statediff version to 0.0.26. * add btree index to state/storage_cids.node_type; updated schema * Dedup receipt data. * Fix linter errors. * Address comments. * Bump statediff version to 0.0.27. * new cli flag for initializing db first time service is ran * only write Removed node ipld block (on db init) and reuse constant cid and mhkey * test new handling of Removed nodes; don't require init flag * log metrics * Add new major branch to github action. * Fix build. * Update golang version in CI. * Use ipld-eth-db in testing. * Remove migration from repo. * Add new major branch to github action. *Use `GetTd` instead of `GetTdByHash` https://github.com/ethereum/go-ethereum/commit/62891378274a58c84b9fa35f3c9b99155644b5eb * Add new major branch to github action. * Report DB metrics * batch inserts to public.blocks * v2 => v3 major refactor * fixes and cli integration for new options * update example command in readme * ashwin's fix for failing pgx unit test * update to use new schema; fix pgx driver * indexer that writes sql stmts out to a file * cli integration * fix unit tests * use node_id as PK/FK * misc fixes/adjustments * update README * cleanup; more unit tests * basefee is big.Int, it won't always fit in int64 * adjust for schema updates * finish unit tests * test harnest for arbitrary mainnet blocks and receipts * cache problematic block locally for quicker testing/easier CI testing * fix issue with log/logTrie processing * remove some unecessary hashing operations * handle edge case * add more 'bad blocks' to mainnet_tests * increase file write buffer size * increase buffer further * fix rct trie multicodec type * extend testing * log trie fk fix * bump statediff meta version; use db v0.3.0 in compose * skip file writing tests in CI, for now * prevent parallel execution of tests in different pkgs (suspect this is what causes our deadlock to show up only in CI test env) adjust write buffering * fix rct unit tests * fix README formatting * port retry on deadlock detection feature * new workflow on-'master' targets * update version meta * improve test coverage for logs * fix possible race condition * fix CI * check tx pool state at end of unit tests * better logging of rollbacks and dead lock retries --- .../{publish.yaml => on-pr-publish.yaml} | 28 ++++++++++++++++++- .github/workflows/on-pr.yml | 7 ----- .github/workflows/tests.yml | 2 +- statediff/indexer/database/file/config.go | 2 +- statediff/indexer/database/file/writer.go | 4 ++- .../indexer/database/sql/postgres/database.go | 2 +- statediff/indexer/database/sql/writer.go | 6 ++-- 7 files changed, 37 insertions(+), 14 deletions(-) rename .github/workflows/{publish.yaml => on-pr-publish.yaml} (73%) delete mode 100644 .github/workflows/on-pr.yml diff --git a/.github/workflows/publish.yaml b/.github/workflows/on-pr-publish.yaml similarity index 73% rename from .github/workflows/publish.yaml rename to .github/workflows/on-pr-publish.yaml index 610f8afbff82..51fec9b5abd1 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/on-pr-publish.yaml @@ -1,13 +1,35 @@ -name: Publish geth to release +name: Test, Build, and/or Publish on: release: types: [published] + pull_request: + jobs: + pre_job: + # continue-on-error: true # Uncomment once integration is finished + runs-on: ubuntu-latest + # Map a step output to a job output + outputs: + should_skip: ${{ steps.skip_check.outputs.should_skip }} + steps: + - id: skip_check + uses: fkirc/skip-duplicate-actions@v4 + with: + # All of these options are optional, so you can remove them if you are happy with the defaults + concurrent_skipping: "never" + skip_after_successful_duplicate: "true" + do_not_skip: '["workflow_dispatch", "schedule"]' run-tests: uses: ./.github/workflows/tests.yml + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + needs: pre_job build: name: Run docker build and publish needs: run-tests + if: | + always() && + (needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped') && + github.event_name == 'release' runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -25,6 +47,10 @@ jobs: push_to_registries: name: Publish assets to Release runs-on: ubuntu-latest + if: | + always() && + (needs.build.result == 'success') && + github.event_name == 'release' needs: build steps: - name: Get the version diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml deleted file mode 100644 index 2b05dcda5f8f..000000000000 --- a/.github/workflows/on-pr.yml +++ /dev/null @@ -1,7 +0,0 @@ -name: Build and test - -on: [pull_request] - -jobs: - run-tests: - uses: ./.github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 41f63f235738..31f8a8946771 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,7 +5,7 @@ on: env: stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref || '382aca8e42bc5e33f301f77cdd2e09cc80602fc3'}} - ipld-eth-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref || '65b7bee7a6757c1fc527c8bfdc4f99ab915fcf36' }} + ipld-eth-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref || '67c32e9d4d12b786eb4135d82392cb708f96b1c9' }} GOPATH: /tmp/go jobs: diff --git a/statediff/indexer/database/file/config.go b/statediff/indexer/database/file/config.go index a075e896b3d7..57c20fe8bd1b 100644 --- a/statediff/indexer/database/file/config.go +++ b/statediff/indexer/database/file/config.go @@ -41,7 +41,7 @@ var TestConfig = Config{ GenesisBlock: "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", NetworkID: "1", ChainID: 1, - ID: "mockNodeID", + ID: "1", ClientName: "go-ethereum", }, } diff --git a/statediff/indexer/database/file/writer.go b/statediff/indexer/database/file/writer.go index 3c11b5eea135..2e84250e551c 100644 --- a/statediff/indexer/database/file/writer.go +++ b/statediff/indexer/database/file/writer.go @@ -197,8 +197,10 @@ func (sqw *SQLWriter) upsertIPLDRaw(blockNumber string, codec, mh uint64, raw [] } func (sqw *SQLWriter) upsertHeaderCID(header models.HeaderModel) { + nodeId := fmt.Sprintf("{%s}", header.NodeID) + // {'1'} stmt := fmt.Sprintf(headerInsert, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, - header.TotalDifficulty, header.NodeID, header.Reward, header.StateRoot, header.TxRoot, + header.TotalDifficulty, nodeId, header.Reward, header.StateRoot, header.TxRoot, header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp, header.MhKey, 1, header.Coinbase) sqw.stmts <- []byte(stmt) indexerMetrics.blocks.Inc(1) diff --git a/statediff/indexer/database/sql/postgres/database.go b/statediff/indexer/database/sql/postgres/database.go index 35a9cbc82400..76bdc730f184 100644 --- a/statediff/indexer/database/sql/postgres/database.go +++ b/statediff/indexer/database/sql/postgres/database.go @@ -40,7 +40,7 @@ type DB struct { func (db *DB) InsertHeaderStm() string { return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) - ON CONFLICT (block_hash, block_number) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1, $16)` + ON CONFLICT (block_hash, block_number) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = ($3, $4, $5, ARRAY(SELECT DISTINCT unnest(array_cat(eth.header_cids.node_id, $6))), $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1, $16)` } // InsertUncleStm satisfies the sql.Statements interface diff --git a/statediff/indexer/database/sql/writer.go b/statediff/indexer/database/sql/writer.go index c6c37855653e..bcec19ad5e6e 100644 --- a/statediff/indexer/database/sql/writer.go +++ b/statediff/indexer/database/sql/writer.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/statediff/indexer/models" + "github.com/lib/pq" ) var ( @@ -47,11 +48,12 @@ func (w *Writer) Close() error { /* INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) -ON CONFLICT (block_hash, block_number) DO UPDATE SET (block_number, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = ($1, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1, $16) +ON CONFLICT (block_hash, block_number) DO UPDATE SET (block_number, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) = ($1, $3, $4, $5, ARRAY(SELECT DISTINCT unnest(array_cat(eth.header_cids.node_id, $6))), $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1, $16) */ func (w *Writer) upsertHeaderCID(tx Tx, header models.HeaderModel) error { + nodeIds := []string{w.db.NodeID()} _, err := tx.Exec(w.db.Context(), w.db.InsertHeaderStm(), - header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, w.db.NodeID(), + header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, pq.Array(nodeIds), header.Reward, header.StateRoot, header.TxRoot, header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp, header.MhKey, 1, header.Coinbase) if err != nil {