Merge pull request #180 from vulcanize/v1.10.14-statediff-0.0.29

V1.10.14 statediff 0.0.29

cmd/geth/config.go

@@ -40,6 +40,7 @@ import (
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff"
 )
 
 var (
@@ -165,7 +166,35 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 	if ctx.GlobalIsSet(utils.OverrideTerminalTotalDifficulty.Name) {
 		cfg.Eth.OverrideTerminalTotalDifficulty = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideTerminalTotalDifficulty.Name))
 	}
-	backend, _ := utils.RegisterEthService(stack, &cfg.Eth, ctx.GlobalBool(utils.CatalystFlag.Name))
+	backend, eth := utils.RegisterEthService(stack, &cfg.Eth, ctx.GlobalBool(utils.CatalystFlag.Name))
+
+	if ctx.GlobalBool(utils.StateDiffFlag.Name) {
+		var dbParams *statediff.DBParams
+		if ctx.GlobalIsSet(utils.StateDiffDBFlag.Name) {
+			dbParams = new(statediff.DBParams)
+			dbParams.ConnectionURL = ctx.GlobalString(utils.StateDiffDBFlag.Name)
+			if ctx.GlobalIsSet(utils.StateDiffDBNodeIDFlag.Name) {
+				dbParams.ID = ctx.GlobalString(utils.StateDiffDBNodeIDFlag.Name)
+			} else {
+				utils.Fatalf("Must specify node ID for statediff DB output")
+			}
+			if ctx.GlobalIsSet(utils.StateDiffDBClientNameFlag.Name) {
+				dbParams.ClientName = ctx.GlobalString(utils.StateDiffDBClientNameFlag.Name)
+			} else {
+				utils.Fatalf("Must specify client name for statediff DB output")
+			}
+		} else {
+			if ctx.GlobalBool(utils.StateDiffWritingFlag.Name) {
+				utils.Fatalf("Must pass DB parameters if enabling statediff write loop")
+			}
+		}
+		p := statediff.ServiceParams{
+			DBParams:        dbParams,
+			EnableWriteLoop: ctx.GlobalBool(utils.StateDiffWritingFlag.Name),
+			NumWorkers:      ctx.GlobalUint(utils.StateDiffWorkersFlag.Name),
+		}
+		utils.RegisterStateDiffService(stack, eth, &cfg.Eth, p)
+	}
 
 	// Configure GraphQL if requested
 	if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) {

cmd/utils/flags.go

@@ -796,6 +796,7 @@ var (
 		Name:  "catalyst",
 		Usage: "Catalyst mode (eth2 integration testing)",
 	}
+
 	StateDiffFlag = cli.BoolFlag{
 		Name:  "statediff",
 		Usage: "Enables the processing of state diffs between each block",

params/version.go

@@ -24,7 +24,7 @@ const (
 	VersionMajor = 1                  // Major version component of the current release
 	VersionMinor = 10                 // Minor version component of the current release
 	VersionPatch = 14                 // Patch version component of the current release
-	VersionMeta  = "statediff-0.0.28" // Version metadata to append to the version string
+	VersionMeta  = "statediff-0.0.29" // Version metadata to append to the version string
 )
 
 // Version holds the textual version string.

statediff/README.md

@@ -77,19 +77,18 @@ This state diffing service runs as an auxiliary service concurrent to the regula
 ### CLI configuration
 This service introduces a CLI flag namespace `statediff`
 
-`--statediff` flag is used to turn on the service
-`--statediff.writing` is used to tell the service to write state diff objects it produces from synced ChainEvents directly to a configured Postgres database
-`--statediff.workers` is used to set the number of concurrent workers to process state diff objects and write them into the database
-`--statediff.db` is the connection string for the Postgres database to write to
-`--statediff.db.init` indicates whether we need to initialize a new database; set true if its the first time running the process on a given database
-`--statediff.dbnodeid` is the node id to use in the Postgres database
-`--statediff.dbclientname` is the client name to use in the Postgres database
+`--statediff` flag is used to turn on the service  
+`--statediff.writing` is used to tell the service to write state diff objects it produces from synced ChainEvents directly to a configured Postgres database  
+`--statediff.workers` is used to set the number of concurrent workers to process state diff objects and write them into the database  
+`--statediff.db` is the connection string for the Postgres database to write to  
+`--statediff.dbnodeid` is the node id to use in the Postgres database  
+`--statediff.dbclientname` is the client name to use in the Postgres database  
 
 The service can only operate in full sync mode (`--syncmode=full`), but only the historical RPC endpoints require an archive node (`--gcmode=archive`)
 
 e.g.
 `
-./build/bin/geth --syncmode=full --gcmode=archive --statediff --statediff.writing --statediff.db=postgres://localhost:5432/vulcanize_testing?sslmode=disable --statediff.db.init=true --statediff.dbnodeid={nodeId} --statediff.dbclientname={dbClientName}
+./build/bin/geth --syncmode=full --gcmode=archive --statediff --statediff.writing --statediff.db='postgres://localhost:5432/vulcanize_testing?sslmode=disable' --statediff.dbnodeid={nodeId} --statediff.dbclientname={dbClientName}
 `
 
 ### RPC endpoints

statediff/indexer/indexer_test.go

@@ -125,13 +125,6 @@ func init() {
 	trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx3, multihash.KECCAK_256)
 	trx4CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx4, multihash.KECCAK_256)
 	trx5CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx5, multihash.KECCAK_256)
-	/*
-		rct1Node, _ := ipld.NewReceipt(rcts[0])
-		rct2Node, _ := ipld.NewReceipt(rcts[1])
-		rct3Node, _ := ipld.NewReceipt(rcts[2])
-		rct4Node, _ := ipld.NewReceipt(rcts[3])
-		rct5Node, _ := ipld.NewReceipt(rcts[4])
-	*/
 	state1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, mocks.ContractLeafNode, multihash.KECCAK_256)
 	state2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, mocks.AccountLeafNode, multihash.KECCAK_256)
 	storageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, mocks.StorageLeafNode, multihash.KECCAK_256)
@@ -241,7 +234,7 @@ func TestPublishAndIndexer(t *testing.T) {
 	t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
 		setup(t)
 		defer tearDown(t)
-		// check that txs were properly indexed
+		// check that txs were properly indexed and published
 		trxs := make([]string, 0)
 		pgStr := `SELECT transaction_cids.cid FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
 				WHERE header_cids.block_number = $1`
@@ -255,7 +248,7 @@ func TestPublishAndIndexer(t *testing.T) {
 		expectTrue(t, shared.ListContainsString(trxs, trx3CID.String()))
 		expectTrue(t, shared.ListContainsString(trxs, trx4CID.String()))
 		expectTrue(t, shared.ListContainsString(trxs, trx5CID.String()))
-		// and published
+
 		for _, c := range trxs {
 			dc, err := cid.Decode(c)
 			if err != nil {
@@ -402,7 +395,7 @@ func TestPublishAndIndexer(t *testing.T) {
 		setup(t)
 		defer tearDown(t)
 
-		// check receipts were properly indexed
+		// check receipts were properly indexed and published
 		rcts := make([]string, 0)
 		pgStr := `SELECT receipt_cids.leaf_cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
 				WHERE receipt_cids.tx_id = transaction_cids.id
@@ -413,14 +406,19 @@ func TestPublishAndIndexer(t *testing.T) {
 			t.Fatal(err)
 		}
 		shared.ExpectEqual(t, len(rcts), 5)
+		expectTrue(t, shared.ListContainsString(rcts, rct1CID.String()))
+		expectTrue(t, shared.ListContainsString(rcts, rct2CID.String()))
+		expectTrue(t, shared.ListContainsString(rcts, rct3CID.String()))
+		expectTrue(t, shared.ListContainsString(rcts, rct4CID.String()))
+		expectTrue(t, shared.ListContainsString(rcts, rct5CID.String()))
 
-		for idx, rctLeafCID := range rcts {
+		for idx, c := range rcts {
 			result := make([]ipfs.BlockModel, 0)
 			pgStr = `SELECT data
 					FROM eth.receipt_cids
 					INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = public.blocks.key)
 					WHERE receipt_cids.leaf_cid = $1`
-			err = db.Select(&result, pgStr, rctLeafCID)
+			err = db.Select(&result, pgStr, c)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -434,10 +432,7 @@ func TestPublishAndIndexer(t *testing.T) {
 			require.NoError(t, err)
 
 			shared.ExpectEqual(t, expectedRct, nodeElements[1].([]byte))
-		}
 
-		// and published
-		for _, c := range rcts {
 			dc, err := cid.Decode(c)
 			if err != nil {
 				t.Fatal(err)