diff --git a/.circleci/config.yml b/.circleci/config.yml index c43f550f4..62c1d3a2f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -114,6 +114,7 @@ commands: sudo apt update sudo apt -y install python3 python3-pip python3-setuptools python3-wheel libboost-math-dev libffi-dev pip3 install -r misc/requirements.txt + pip3 install e2e_tests/ - run: name: sync submodules (go-algorand) @@ -143,6 +144,7 @@ commands: - run: make test-generate - run: make fakepackage - run: make e2e + - run: make e2e-conduit run_indexer_vs_algod: steps: diff --git a/.codecov.yml b/.codecov.yml index 19ec22537..921071a9e 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -2,6 +2,11 @@ codecov: require_ci_to_pass: no branch: develop +ignore: + - "idb/mocks" + - "idb/dummy" + - "util/test" + coverage: precision: 2 round: down diff --git a/.gitignore b/.gitignore index b34568c3e..123c01969 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Build artifacts cmd/algorand-indexer/algorand-indexer +cmd/conduit/conduit api/.3tmp.json api/generate tmp/ @@ -28,6 +29,9 @@ _*.json # Python __pycache__ +e2e_tests/dist +e2e_tests/build +e2e_tests/*egg-info* .venv # jetbrains IDE @@ -48,3 +52,6 @@ coverage.txt # asdf .tool-versions + +# conduit example +cmd/conduit/data diff --git a/Makefile b/Makefile index 135cfddd0..3641b9879 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,12 @@ COVERPKG := $(shell go list ./... | grep -v '/cmd/' | egrep -v '(testing|test|m # Used for e2e test export GO_IMAGE = golang:$(shell go version | cut -d ' ' -f 3 | tail -c +3 ) -# This is the default target, build the indexer: +# This is the default target, build everything: +all: conduit cmd/algorand-indexer/algorand-indexer go-algorand idb/postgres/internal/schema/setup_postgres_sql.go idb/mocks/IndexerDb.go + +conduit: go-algorand + go generate ./... && cd cmd/conduit && go build + cmd/algorand-indexer/algorand-indexer: idb/postgres/internal/schema/setup_postgres_sql.go go-algorand cd cmd/algorand-indexer && go build -ldflags="${GOLDFLAGS}" @@ -55,8 +60,8 @@ fakepackage: go-algorand test: idb/mocks/IndexerDb.go cmd/algorand-indexer/algorand-indexer go test -coverpkg=$(COVERPKG) ./... -coverprofile=coverage.txt -covermode=atomic ${TEST_FLAG} -lint: go-algorand - golangci-lint run -c .golangci.yml +lint: conduit + golint -set_exit_status ./... go vet ./... fmt: @@ -68,7 +73,11 @@ integration: cmd/algorand-indexer/algorand-indexer test/postgres_integration_test.sh e2e: cmd/algorand-indexer/algorand-indexer - cd misc && docker-compose build --build-arg GO_IMAGE=${GO_IMAGE} && docker-compose up --exit-code-from e2e + cd e2e_tests/docker/indexer/ && docker-compose build --build-arg GO_IMAGE=${GO_IMAGE} && docker-compose up --exit-code-from e2e + +e2e-conduit: conduit + cd third_party/go-algorand && make install + export PATH=$(PATH):$(shell go env GOPATH)/bin; pip3 install e2e_tests/ && e2econduit --s3-source-net ${CI_E2E_FILENAME} --conduit-bin cmd/conduit/conduit deploy: mule/deploy.sh @@ -96,9 +105,10 @@ indexer-v-algod: nightly-setup indexer-v-algod-swagger nightly-teardown # fetch and update submodule. it's default to latest rel/nightly branch. # to use a different branch, update the branch in .gitmodules for CI build, # and for local testing, you may checkout a specific branch in the submodule. -# after submodule is updated, CI_E2E_FILE in circleci/config.yml should also -# be updated to use a newer artifact. path copied from s3 bucket, s3://algorand-testdata/indexer/e2e4/ +# after submodule is updated, CI_E2E_FILENAME in .circleci/config.yml should +# also be updated to use a newer artifact. path copied from s3 bucket, +# s3://algorand-testdata/indexer/e2e4/ update-submodule: git submodule update --remote -.PHONY: test e2e integration fmt lint deploy sign test-package package fakepackage cmd/algorand-indexer/algorand-indexer idb/mocks/IndexerDb.go go-algorand indexer-v-algod +.PHONY: all test e2e integration fmt lint deploy sign test-package package fakepackage cmd/algorand-indexer/algorand-indexer idb/mocks/IndexerDb.go go-algorand indexer-v-algod conduit diff --git a/README.md b/README.md index 2f7188be5..18e2fc44f 100644 --- a/README.md +++ b/README.md @@ -48,11 +48,11 @@ Indexer is part of the [sandbox](https://github.com/algorand/sandbox) private ne - Search and filter accounts, transactions, assets, and asset balances with many different parameters. - Pagination of results. - Enriched transaction and account data: - - Confirmation round (block containing the transaction) - - Confirmation time - - Signature type - - Close amounts - - Create/delete rounds. + - Confirmation round (block containing the transaction) + - Confirmation time + - Signature type + - Close amounts + - Create/delete rounds. - Human readable field names instead of the space optimized protocol level names. # Contributing diff --git a/api/app_boxes_fixtures_test.go b/api/app_boxes_fixtures_test.go index 6ac030fb4..a55a3906e 100644 --- a/api/app_boxes_fixtures_test.go +++ b/api/app_boxes_fixtures_test.go @@ -6,14 +6,15 @@ import ( "fmt" "testing" + "github.com/stretchr/testify/require" + "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/data/transactions/logic" "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/rpcs" - "github.com/algorand/indexer/processor" + "github.com/algorand/indexer/util/test" - "github.com/stretchr/testify/require" ) func goalEncode(t *testing.T, s string) string { @@ -40,7 +41,7 @@ var goalEncodingExamples map[string]string = map[string]string{ "abi": `(uint64,string,bool[]):[399,"pls pass",[true,false]]`, } -func setupLiveBoxes(t *testing.T, proc processor.Processor, l *ledger.Ledger) { +func setupLiveBoxes(t *testing.T, proc func(cert *rpcs.EncodedBlockCert) error, l *ledger.Ledger) { deleted := "DELETED" firstAppid := basics.AppIndex(1) @@ -69,7 +70,7 @@ func setupLiveBoxes(t *testing.T, proc processor.Processor, l *ledger.Ledger) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createTxn, &payNewAppTxn, &createTxn2, &payNewAppTxn2, &createTxn3, &payNewAppTxn3) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // block header handoff: round 1 --> round 2 @@ -104,7 +105,7 @@ func setupLiveBoxes(t *testing.T, proc processor.Processor, l *ledger.Ledger) { block, err = test.MakeBlockForTxns(blockHdr, boxTxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // block header handoff: round 2 --> round 3 @@ -137,7 +138,7 @@ func setupLiveBoxes(t *testing.T, proc processor.Processor, l *ledger.Ledger) { block, err = test.MakeBlockForTxns(blockHdr, boxTxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // block header handoff: round 3 --> round 4 @@ -165,7 +166,7 @@ func setupLiveBoxes(t *testing.T, proc processor.Processor, l *ledger.Ledger) { block, err = test.MakeBlockForTxns(blockHdr, boxTxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // block header handoff: round 4 --> round 5 @@ -194,7 +195,7 @@ func setupLiveBoxes(t *testing.T, proc processor.Processor, l *ledger.Ledger) { block, err = test.MakeBlockForTxns(blockHdr, boxTxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // block header handoff: round 5 --> round 6 @@ -223,7 +224,7 @@ func setupLiveBoxes(t *testing.T, proc processor.Processor, l *ledger.Ledger) { block, err = test.MakeBlockForTxns(blockHdr, boxTxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // block header handoff: round 6 --> round 7 @@ -250,7 +251,7 @@ func setupLiveBoxes(t *testing.T, proc processor.Processor, l *ledger.Ledger) { block, err = test.MakeBlockForTxns(blockHdr, boxTxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // block header handoff: round 7 --> round 8 @@ -273,7 +274,7 @@ func setupLiveBoxes(t *testing.T, proc processor.Processor, l *ledger.Ledger) { block, err = test.MakeBlockForTxns(blockHdr, boxTxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // block header handoff: round 8 --> round 9 @@ -287,7 +288,7 @@ func setupLiveBoxes(t *testing.T, proc processor.Processor, l *ledger.Ledger) { block, err = test.MakeBlockForTxns(blockHdr, &deleteTxn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // ---- SUMMARY ---- // diff --git a/api/fixtures_test.go b/api/fixtures_test.go index 76fc31899..c6cc8fb7d 100644 --- a/api/fixtures_test.go +++ b/api/fixtures_test.go @@ -15,10 +15,10 @@ import ( "github.com/stretchr/testify/require" "github.com/algorand/go-algorand/ledger" + "github.com/algorand/go-algorand/rpcs" "github.com/algorand/indexer/api/generated/v2" "github.com/algorand/indexer/idb/postgres" - "github.com/algorand/indexer/processor" "github.com/algorand/indexer/util/test" ) @@ -268,7 +268,7 @@ func getProof(path string) (route string, proof prover, err error) { } // WARNING: receiver should not call l.Close() -func setupIdbAndReturnShutdownFunc(t *testing.T) (db *postgres.IndexerDb, proc processor.Processor, l *ledger.Ledger, shutdown func()) { +func setupIdbAndReturnShutdownFunc(t *testing.T) (db *postgres.IndexerDb, proc func(cert *rpcs.EncodedBlockCert) error, l *ledger.Ledger, shutdown func()) { db, dbShutdown, proc, l := setupIdb(t, test.MakeGenesis()) shutdown = func() { diff --git a/api/handlers.go b/api/handlers.go index b0d7f08a0..a030e6816 100644 --- a/api/handlers.go +++ b/api/handlers.go @@ -39,7 +39,7 @@ type ServerImplementation struct { db idb.IndexerDb - fetcher error + dataError func() error timeout time.Duration @@ -142,8 +142,10 @@ func (si *ServerImplementation) MakeHealthCheck(ctx echo.Context) error { errors = append(errors, fmt.Sprintf("database error: %s", health.Error)) } - if si.fetcher != nil && si.fetcher.Error() != "" { - errors = append(errors, fmt.Sprintf("fetcher error: %s", si.fetcher.Error())) + if si.dataError != nil { + if err := si.dataError(); err != nil { + errors = append(errors, fmt.Sprintf("data error: %s", err)) + } } return ctx.JSON(http.StatusOK, common.HealthCheck{ diff --git a/api/handlers_e2e_test.go b/api/handlers_e2e_test.go index e8d276014..a2a388511 100644 --- a/api/handlers_e2e_test.go +++ b/api/handlers_e2e_test.go @@ -29,13 +29,12 @@ import ( "github.com/algorand/go-algorand/data/transactions/logic" "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/rpcs" - "github.com/algorand/indexer/processor" "github.com/algorand/indexer/api/generated/v2" + "github.com/algorand/indexer/conduit/plugins/processors/blockprocessor" "github.com/algorand/indexer/idb" "github.com/algorand/indexer/idb/postgres" pgtest "github.com/algorand/indexer/idb/postgres/testing" - "github.com/algorand/indexer/processor/blockprocessor" "github.com/algorand/indexer/util/test" ) @@ -70,7 +69,7 @@ func testServerImplementation(db idb.IndexerDb) *ServerImplementation { return &ServerImplementation{db: db, timeout: 30 * time.Second, opts: defaultOpts} } -func setupIdb(t *testing.T, genesis bookkeeping.Genesis) (*postgres.IndexerDb, func(), processor.Processor, *ledger.Ledger) { +func setupIdb(t *testing.T, genesis bookkeeping.Genesis) (*postgres.IndexerDb, func(), func(cert *rpcs.EncodedBlockCert) error, *ledger.Ledger) { _, connStr, shutdownFunc := pgtest.SetupPostgres(t) db, _, err := postgres.OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) @@ -87,9 +86,12 @@ func setupIdb(t *testing.T, genesis bookkeeping.Genesis) (*postgres.IndexerDb, f logger, _ := test2.NewNullLogger() l, err := test.MakeTestLedger(logger) require.NoError(t, err) - proc, err := blockprocessor.MakeProcessorWithLedger(logger, l, db.AddBlock) + proc, err := blockprocessor.MakeBlockProcessorWithLedger(logger, l, db.AddBlock) require.NoError(t, err, "failed to open ledger") - return db, newShutdownFunc, proc, l + + f := blockprocessor.MakeBlockProcessorHandlerAdapter(&proc, db.AddBlock) + + return db, newShutdownFunc, f, l } func TestApplicationHandlers(t *testing.T) { @@ -129,7 +131,8 @@ func TestApplicationHandlers(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn, &optInTxnA, &optInTxnB) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) + require.NoError(t, err) ////////// @@ -256,7 +259,7 @@ func TestAccountExcludeParameters(t *testing.T) { &appOptInTxnA, &appOptInTxnB, &assetOptInTxnA, &assetOptInTxnB) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -462,7 +465,7 @@ func TestAccountMaxResultsLimit(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, ptxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -850,7 +853,7 @@ func TestInnerTxn(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) for _, tc := range testcases { @@ -903,7 +906,7 @@ func TestPagingRootTxnDeduplication(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) testcases := []struct { @@ -1051,7 +1054,7 @@ func TestKeyregTransactionWithStateProofKeys(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) e := echo.New() @@ -1156,7 +1159,7 @@ func TestAccountClearsNonUTF8(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAsset) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) verify := func(params generated.AssetParams) { @@ -1278,7 +1281,7 @@ func TestLookupInnerLogs(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) for _, tc := range testcases { @@ -1377,7 +1380,7 @@ func TestLookupMultiInnerLogs(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) for _, tc := range testcases { @@ -1434,7 +1437,7 @@ func TestFetchBlockWithExpiredPartAccts(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCreate) block.ExpiredParticipationAccounts = append(block.ExpiredParticipationAccounts, test.AccountB, test.AccountC) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) //////////// @@ -1501,7 +1504,7 @@ func TestFetchBlockWithOptions(t *testing.T) { txnC := test.MakeCreateAppTxn(test.AccountC) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txnA, &txnB, &txnC) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -1563,7 +1566,7 @@ func TestGetBlocksTransactionsLimit(t *testing.T) { block.BlockHeader.Round = basics.Round(i + 1) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) } @@ -1939,7 +1942,7 @@ func runBoxCreateMutateDelete(t *testing.T, comparator boxTestComparator) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createTxn, &payNewAppTxn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) opts := idb.ApplicationQuery{ApplicationID: uint64(appid)} @@ -1987,7 +1990,7 @@ func runBoxCreateMutateDelete(t *testing.T, comparator boxTestComparator) { block, err = test.MakeBlockForTxns(blockHdr, boxTxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) _, round = db.Applications(context.Background(), opts) require.Equal(t, uint64(currentRound), round) @@ -2025,7 +2028,7 @@ func runBoxCreateMutateDelete(t *testing.T, comparator boxTestComparator) { block, err = test.MakeBlockForTxns(blockHdr, boxTxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) _, round = db.Applications(context.Background(), opts) require.Equal(t, uint64(currentRound), round) @@ -2057,7 +2060,7 @@ func runBoxCreateMutateDelete(t *testing.T, comparator boxTestComparator) { block, err = test.MakeBlockForTxns(blockHdr, boxTxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) _, round = db.Applications(context.Background(), opts) require.Equal(t, uint64(currentRound), round) @@ -2093,7 +2096,7 @@ func runBoxCreateMutateDelete(t *testing.T, comparator boxTestComparator) { block, err = test.MakeBlockForTxns(blockHdr, boxTxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) _, round = db.Applications(context.Background(), opts) require.Equal(t, uint64(currentRound), round) @@ -2124,7 +2127,7 @@ func runBoxCreateMutateDelete(t *testing.T, comparator boxTestComparator) { block, err = test.MakeBlockForTxns(blockHdr, boxTxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) _, round = db.Applications(context.Background(), opts) require.Equal(t, uint64(currentRound), round) diff --git a/api/server.go b/api/server.go index 1b4d794a3..3a30d7a6c 100644 --- a/api/server.go +++ b/api/server.go @@ -87,7 +87,7 @@ func (e ExtraOptions) handlerTimeout() time.Duration { } // Serve starts an http server for the indexer API. This call blocks. -func Serve(ctx context.Context, serveAddr string, db idb.IndexerDb, fetcherError error, log *log.Logger, options ExtraOptions) { +func Serve(ctx context.Context, serveAddr string, db idb.IndexerDb, dataError func() error, log *log.Logger, options ExtraOptions) { e := echo.New() e.HideBanner = true @@ -135,7 +135,7 @@ func Serve(ctx context.Context, serveAddr string, db idb.IndexerDb, fetcherError api := ServerImplementation{ EnableAddressSearchRoundRewind: options.DeveloperMode, db: db, - fetcher: fetcherError, + dataError: dataError, timeout: options.handlerTimeout(), log: log, disabledParams: disabledMap, diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go index b9d728a50..4a816db83 100644 --- a/cmd/algorand-indexer/daemon.go +++ b/cmd/algorand-indexer/daemon.go @@ -9,7 +9,6 @@ import ( "path/filepath" "runtime/pprof" "strings" - "sync" "syscall" "time" @@ -17,19 +16,19 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" - "github.com/algorand/go-algorand/rpcs" "github.com/algorand/go-algorand/util" "github.com/algorand/indexer/api" "github.com/algorand/indexer/api/generated/v2" + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/pipeline" + _ "github.com/algorand/indexer/conduit/plugins/exporters/postgresql" + _ "github.com/algorand/indexer/conduit/plugins/importers/algod" + _ "github.com/algorand/indexer/conduit/plugins/processors/blockprocessor" "github.com/algorand/indexer/config" "github.com/algorand/indexer/fetcher" "github.com/algorand/indexer/idb" - "github.com/algorand/indexer/importer" - "github.com/algorand/indexer/processor" - "github.com/algorand/indexer/processor/blockprocessor" iutil "github.com/algorand/indexer/util" - "github.com/algorand/indexer/util/metrics" ) // GetConfigFromDataDir Given the data directory, configuration filename and a list of types, see if @@ -242,30 +241,6 @@ func loadIndexerParamConfig(cfg *daemonConfig) error { return err } -func createIndexerPidFile(pidFilePath string) error { - var err error - logger.Infof("Creating PID file at: %s\n", pidFilePath) - fout, err := os.Create(pidFilePath) - if err != nil { - err = fmt.Errorf("%s: could not create pid file, %v", pidFilePath, err) - logger.Error(err) - return err - } - _, err = fmt.Fprintf(fout, "%d", os.Getpid()) - if err != nil { - err = fmt.Errorf("%s: could not write pid file, %v", pidFilePath, err) - logger.Error(err) - return err - } - err = fout.Close() - if err != nil { - err = fmt.Errorf("%s: could not close pid file, %v", pidFilePath, err) - logger.Error(err) - return err - } - return err -} - func runDaemon(daemonConfig *daemonConfig) error { var err error config.BindFlagSet(daemonConfig.flags) @@ -297,7 +272,7 @@ func runDaemon(daemonConfig *daemonConfig) error { logger.Infof("Using configuration file: %s", viper.ConfigFileUsed()) if daemonConfig.pidFilePath != "" { - err = createIndexerPidFile(daemonConfig.pidFilePath) + err = iutil.CreateIndexerPidFile(logger, daemonConfig.pidFilePath) if err != nil { return err } @@ -343,19 +318,18 @@ func runDaemon(daemonConfig *daemonConfig) error { }() } - var bot fetcher.Fetcher - if daemonConfig.noAlgod { - logger.Info("algod block following disabled") - } else if daemonConfig.algodAddr != "" && daemonConfig.algodToken != "" { - bot, err = fetcher.ForNetAndToken(daemonConfig.algodAddr, daemonConfig.algodToken, logger) - maybeFail(err, "fetcher setup, %v", err) - } else if daemonConfig.algodDataDir != "" { - bot, err = fetcher.ForDataDir(daemonConfig.algodDataDir, logger) - maybeFail(err, "fetcher setup, %v", err) - } else { + if daemonConfig.algodDataDir != "" { + daemonConfig.algodAddr, daemonConfig.algodToken, _, err = fetcher.AlgodArgsForDataDir(daemonConfig.algodDataDir) + maybeFail(err, "algod data dir err, %v", err) + } else if daemonConfig.algodAddr == "" || daemonConfig.algodToken == "" { // no algod was found + logger.Info("no algod was found, provide either --algod OR --algod-net and --algod-token to enable") daemonConfig.noAlgod = true } + if daemonConfig.noAlgod { + logger.Info("algod block following disabled") + } + opts := idb.IndexerDbOptions{} if daemonConfig.noAlgod && !daemonConfig.allowMigration { opts.ReadOnly = true @@ -369,10 +343,13 @@ func runDaemon(daemonConfig *daemonConfig) error { db, availableCh := indexerDbFromFlags(opts) defer db.Close() - var wg sync.WaitGroup - if bot != nil { - wg.Add(1) - go runBlockImporter(ctx, daemonConfig, &wg, db, availableCh, bot, opts) + var dataError func() error + if daemonConfig.noAlgod != true { + pipeline := runConduitPipeline(ctx, availableCh, daemonConfig) + if pipeline != nil { + dataError = pipeline.Error + defer pipeline.Stop() + } } else { logger.Info("No block importer configured.") } @@ -382,53 +359,68 @@ func runDaemon(daemonConfig *daemonConfig) error { options := makeOptions(daemonConfig) - api.Serve(ctx, daemonConfig.daemonServerAddr, db, bot, logger, options) - wg.Wait() + api.Serve(ctx, daemonConfig.daemonServerAddr, db, dataError, logger, options) return err } -func runBlockImporter(ctx context.Context, cfg *daemonConfig, wg *sync.WaitGroup, db idb.IndexerDb, dbAvailable chan struct{}, bot fetcher.Fetcher, opts idb.IndexerDbOptions) { +func makeConduitConfig(dCfg *daemonConfig) pipeline.Config { + return pipeline.Config{ + ConduitConfig: &conduit.Config{ + ConduitDataDir: dCfg.indexerDataDir, + }, + PipelineLogLevel: logger.GetLevel().String(), + Importer: pipeline.NameConfigPair{ + Name: "algod", + Config: map[string]interface{}{ + "netaddr": dCfg.algodAddr, + "token": dCfg.algodToken, + }, + }, + Processors: []pipeline.NameConfigPair{ + { + Name: "block_evaluator", + Config: map[string]interface{}{ + "catchpoint": dCfg.catchpoint, + "data-dir": dCfg.indexerDataDir, + "algod-data-dir": dCfg.algodDataDir, + "algod-token": dCfg.algodToken, + "algod-addr": dCfg.algodAddr, + }, + }, + }, + Exporter: pipeline.NameConfigPair{ + Name: "postgresql", + Config: map[string]interface{}{ + "connection-string": postgresAddr, + "max-conn": dCfg.maxConn, + "test": dummyIndexerDb, + }, + }, + } + +} + +func runConduitPipeline(ctx context.Context, dbAvailable chan struct{}, dCfg *daemonConfig) pipeline.Pipeline { // Need to redefine exitHandler() for every go-routine defer exitHandler() - defer wg.Done() // Wait until the database is available. <-dbAvailable - // Initial import if needed. - genesisReader := importer.GetGenesisFile(cfg.genesisJSONPath, bot.Algod(), logger) - genesis, err := iutil.ReadGenesis(genesisReader) - maybeFail(err, "Error reading genesis file") - - _, err = importer.EnsureInitialImport(db, genesis) - maybeFail(err, "importer.EnsureInitialImport() error") - - // sync local ledger - nextDBRound, err := db.GetNextRoundToAccount() - maybeFail(err, "Error getting DB round") - - logger.Info("Initializing block import handler.") - imp := importer.NewImporter(db) - - logger.Info("Initializing local ledger.") - proc, err := blockprocessor.MakeProcessorWithLedgerInit(ctx, logger, cfg.catchpoint, &genesis, nextDBRound, opts, imp.ImportBlock) - if err != nil { - maybeFail(err, "blockprocessor.MakeProcessor() err %v", err) + var conduit pipeline.Pipeline + var err error + pcfg := makeConduitConfig(dCfg) + if conduit, err = pipeline.MakePipeline(ctx, &pcfg, logger); err != nil { + logger.Errorf("%v", err) + panic(exit{1}) } - - bot.SetNextRound(proc.NextRoundToProcess()) - handler := blockHandler(proc, 1*time.Second) - bot.SetBlockHandler(handler) - - logger.Info("Starting block importer.") - err = bot.Run(ctx) + err = conduit.Init() if err != nil { - // If context is not expired. - if ctx.Err() == nil { - logger.WithError(err).Errorf("fetcher exited with error") - panic(exit{1}) - } + logger.Errorf("%v", err) + panic(exit{1}) } + conduit.Start() + return conduit } // makeOptions converts CLI options to server options @@ -497,54 +489,3 @@ func makeOptions(daemonConfig *daemonConfig) (options api.ExtraOptions) { return } - -// blockHandler creates a handler complying to the fetcher block handler interface. In case of a failure it keeps -// attempting to add the block until the fetcher shuts down. -func blockHandler(proc processor.Processor, retryDelay time.Duration) func(context.Context, *rpcs.EncodedBlockCert) error { - return func(ctx context.Context, block *rpcs.EncodedBlockCert) error { - for { - err := handleBlock(block, proc) - if err == nil { - // return on success. - return nil - } - - // Delay or terminate before next attempt. - select { - case <-ctx.Done(): - return err - case <-time.After(retryDelay): - // NOOP - } - } - } -} - -func handleBlock(block *rpcs.EncodedBlockCert, proc processor.Processor) error { - start := time.Now() - err := proc.Process(block) - if err != nil { - logger.WithError(err).Errorf( - "block %d import failed", block.Block.Round()) - return fmt.Errorf("handleBlock() err: %w", err) - } - dt := time.Since(start) - - // Ignore round 0 (which is empty). - if block.Block.Round() > 0 { - metrics.BlockImportTimeSeconds.Observe(dt.Seconds()) - metrics.ImportedTxnsPerBlock.Observe(float64(len(block.Block.Payset))) - metrics.ImportedRoundGauge.Set(float64(block.Block.Round())) - txnCountByType := make(map[string]int) - for _, txn := range block.Block.Payset { - txnCountByType[string(txn.Txn.Type)]++ - } - for k, v := range txnCountByType { - metrics.ImportedTxns.WithLabelValues(k).Set(float64(v)) - } - } - - logger.Infof("round r=%d (%d txn) imported in %s", block.Block.Round(), len(block.Block.Payset), dt.String()) - - return nil -} diff --git a/cmd/algorand-indexer/daemon_test.go b/cmd/algorand-indexer/daemon_test.go index 2d25491c9..26a7b1466 100644 --- a/cmd/algorand-indexer/daemon_test.go +++ b/cmd/algorand-indexer/daemon_test.go @@ -2,87 +2,21 @@ package main import ( "bytes" - "context" - "errors" "fmt" "io/fs" "os" "path" "path/filepath" - "sync" "testing" - "time" - "github.com/sirupsen/logrus/hooks/test" + log "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/stretchr/testify/assert" - "github.com/algorand/go-algorand/data/bookkeeping" - "github.com/algorand/go-algorand/ledger/ledgercore" - "github.com/algorand/go-algorand/rpcs" - "github.com/algorand/indexer/config" - "github.com/algorand/indexer/processor/blockprocessor" - itest "github.com/algorand/indexer/util/test" + "github.com/algorand/indexer/util" ) -type mockImporter struct { -} - -var errMockImportBlock = errors.New("Process() invalid round blockCert.Block.Round(): 1234 nextRoundToProcess: 1") - -func (imp *mockImporter) ImportBlock(vb *ledgercore.ValidatedBlock) error { - return nil -} - -func TestImportRetryAndCancel(t *testing.T) { - // connect debug logger - nullLogger, hook := test.NewNullLogger() - logger = nullLogger - - // cancellable context - ctx := context.Background() - cctx, cancel := context.WithCancel(ctx) - - // create handler with mock importer and start, it should generate errors until cancelled. - imp := &mockImporter{} - ledgerLogger, _ := test.NewNullLogger() - l, err := itest.MakeTestLedger(ledgerLogger) - assert.NoError(t, err) - defer l.Close() - proc, err := blockprocessor.MakeProcessorWithLedger(logger, l, nil) - assert.Nil(t, err) - proc.SetHandler(imp.ImportBlock) - handler := blockHandler(proc, 50*time.Millisecond) - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - block := rpcs.EncodedBlockCert{ - Block: bookkeeping.Block{ - BlockHeader: bookkeeping.BlockHeader{ - Round: 1234, - }, - }, - } - handler(cctx, &block) - }() - - // accumulate some errors - for len(hook.Entries) < 5 { - time.Sleep(25 * time.Millisecond) - } - - for _, entry := range hook.Entries { - assert.Equal(t, entry.Message, "block 1234 import failed") - assert.Equal(t, entry.Data["error"], errMockImportBlock) - } - - // Wait for handler to exit. - cancel() - wg.Wait() -} - func createTempDir(t *testing.T) string { dir, err := os.MkdirTemp("", "indexer") if err != nil { @@ -259,7 +193,7 @@ func TestIndexerPidFileExpectSuccess(t *testing.T) { defer os.RemoveAll(indexerDataDir) pidFilePath := path.Join(indexerDataDir, "pidFile") - assert.NoError(t, createIndexerPidFile(pidFilePath)) + assert.NoError(t, util.CreateIndexerPidFile(log.New(), pidFilePath)) } func TestIndexerPidFileCreateFailExpectError(t *testing.T) { @@ -277,6 +211,6 @@ func TestIndexerPidFileCreateFailExpectError(t *testing.T) { cfg.indexerDataDir = indexerDataDir assert.ErrorContains(t, runDaemon(cfg), "pid file") - assert.Error(t, createIndexerPidFile(cfg.pidFilePath)) + assert.Error(t, util.CreateIndexerPidFile(log.New(), cfg.pidFilePath)) } } diff --git a/cmd/block-generator/generator/generate.go b/cmd/block-generator/generator/generate.go index ebb45f28d..55807856a 100644 --- a/cmd/block-generator/generator/generate.go +++ b/cmd/block-generator/generator/generate.go @@ -159,6 +159,7 @@ type Generator interface { WriteGenesis(output io.Writer) error WriteBlock(output io.Writer, round uint64) error WriteAccount(output io.Writer, accountString string) error + WriteStatus(output io.Writer) error Accounts() <-chan basics.Address } @@ -247,6 +248,14 @@ func (g *generator) WriteReport(output io.Writer) error { return err } +func (g *generator) WriteStatus(output io.Writer) error { + response := generated.NodeStatusResponse{ + LastRound: g.round, + } + _, err := output.Write(protocol.EncodeJSON(response)) + return err +} + func (g *generator) WriteGenesis(output io.Writer) error { defer g.recordData(track(genesis)) var allocations []bookkeeping.GenesisAllocation diff --git a/cmd/block-generator/generator/server.go b/cmd/block-generator/generator/server.go index 12750bc86..d2444b418 100644 --- a/cmd/block-generator/generator/server.go +++ b/cmd/block-generator/generator/server.go @@ -57,6 +57,7 @@ func MakeServerWithMiddleware(configFile string, addr string, blocksMiddleware B mux.HandleFunc("/v2/accounts/", getAccountHandler(gen)) mux.HandleFunc("/genesis", getGenesisHandler(gen)) mux.HandleFunc("/report", getReportHandler(gen)) + mux.HandleFunc("/v2/status/wait-for-block-after/", getStatusWaitHandler(gen)) return &http.Server{ Addr: addr, @@ -81,6 +82,12 @@ func getReportHandler(gen Generator) func(w http.ResponseWriter, r *http.Request } } +func getStatusWaitHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + maybeWriteError(w, gen.WriteStatus(w)) + } +} + func getGenesisHandler(gen Generator) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { maybeWriteError(w, gen.WriteGenesis(w)) diff --git a/cmd/block-generator/runner/run.go b/cmd/block-generator/runner/run.go index 67844d21f..216025f26 100644 --- a/cmd/block-generator/runner/run.go +++ b/cmd/block-generator/runner/run.go @@ -161,10 +161,6 @@ func recordDataToFile(start time.Time, entry Entry, prefix string, out *os.File) record("_cumulative", metrics.BlockImportTimeName, floatTotal) record("_average", metrics.ImportedTxnsPerBlockName, rate) record("_cumulative", metrics.ImportedTxnsPerBlockName, intTotal) - record("_average", metrics.BlockUploadTimeName, rate) - record("_cumulative", metrics.BlockUploadTimeName, floatTotal) - record("_average", metrics.PostgresEvalName, rate) - record("_cumulative", metrics.PostgresEvalName, floatTotal) record("", metrics.ImportedRoundGaugeName, intTotal) if len(writeErrors) > 0 { @@ -441,7 +437,7 @@ func startIndexer(dataDir string, logfile string, loglevel string, indexerBinary // Ensure that the health endpoint can be queried. // The service should start very quickly because the DB is empty. - time.Sleep(5 * time.Second) + time.Sleep(10 * time.Second) resp, err := http.Get(fmt.Sprintf("http://%s/health", indexerNet)) if err != nil { fmt.Fprintf(os.Stderr, "stdout:\n%s\n", stdout.String()) diff --git a/cmd/conduit/conduit.yml.example b/cmd/conduit/conduit.yml.example new file mode 100644 index 000000000..e5301c016 --- /dev/null +++ b/cmd/conduit/conduit.yml.example @@ -0,0 +1,20 @@ +# Generated conduit configuration file. + +# The importer is typically an algod archival instance. +importer: + name: algod + config: + - netaddr: "your algod address here" + token: "your algod token here" + +# One or more processors may be defined to manipulate what data +# reaches the exporter. +processors: + +# An exporter is defined to do something with the data. +# Here the filewriter is defined which writes the raw block +# data to files. +exporter: + name: filewriter + config: + - block-dir: "path where data should be written" \ No newline at end of file diff --git a/cmd/conduit/internal/list/list.go b/cmd/conduit/internal/list/list.go new file mode 100644 index 000000000..a4ea3eaf7 --- /dev/null +++ b/cmd/conduit/internal/list/list.go @@ -0,0 +1,94 @@ +package list + +import ( + "fmt" + "io" + "os" + "sort" + "strings" + + "github.com/spf13/cobra" + + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/pipeline" +) + +// Command is the list command to embed in a root cobra command. +var Command = &cobra.Command{ + Use: "list", + Short: "lists all plugins available to conduit", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + printAll() + return nil + }, + // Silence errors because our logger will catch and print any errors + SilenceErrors: true, +} + +func makeDetailsCommand(use string, data func() []conduit.Metadata) *cobra.Command { + return &cobra.Command{ + Use: use + "s", + Aliases: []string{use}, + Short: fmt.Sprintf("usage detail for %s plugins", use), + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + if len(args) == 0 { + printMetadata(os.Stdout, data(), 0) + } else { + printDetails(args[0], data()) + } + }, + } +} + +func init() { + Command.AddCommand(makeDetailsCommand("importer", pipeline.ImporterMetadata)) + Command.AddCommand(makeDetailsCommand("processor", pipeline.ProcessorMetadata)) + Command.AddCommand(makeDetailsCommand("exporter", pipeline.ExporterMetadata)) +} + +func printDetails(name string, plugins []conduit.Metadata) { + for _, data := range plugins { + if data.Name == name { + fmt.Println(data.SampleConfig) + return + } + } + + fmt.Printf("Plugin not found: %s\n", name) +} + +func printMetadata(w io.Writer, plugins []conduit.Metadata, leftIndent int) { + sort.Slice(plugins, func(i, j int) bool { + return plugins[i].Name < plugins[j].Name + }) + + largestPluginNameSize := 0 + for _, data := range plugins { + if len(data.Name) > largestPluginNameSize { + largestPluginNameSize = len(data.Name) + } + } + + for _, data := range plugins { + depthStr := strings.Repeat(" ", leftIndent) + // pad with right with spaces + fmtString := fmt.Sprintf("%s%%-%ds - %%s\n", depthStr, largestPluginNameSize) + + if data.Deprecated { + fmtString = "[DEPRECATED] " + fmtString + } + + fmt.Fprintf(w, fmtString, data.Name, data.Description) + } +} + +func printAll() { + fmt.Fprint(os.Stdout, "importers:\n") + printMetadata(os.Stdout, pipeline.ImporterMetadata(), 2) + fmt.Fprint(os.Stdout, "\nprocessors:\n") + printMetadata(os.Stdout, pipeline.ProcessorMetadata(), 2) + fmt.Fprint(os.Stdout, "\nexporters:\n") + printMetadata(os.Stdout, pipeline.ExporterMetadata(), 2) +} diff --git a/cmd/conduit/main.go b/cmd/conduit/main.go new file mode 100644 index 000000000..cf3b3ec4c --- /dev/null +++ b/cmd/conduit/main.go @@ -0,0 +1,199 @@ +package main + +import ( + "context" + _ "embed" + "fmt" + "os" + "path/filepath" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/algorand/indexer/cmd/conduit/internal/list" + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/pipeline" + "github.com/algorand/indexer/loggers" + "github.com/algorand/indexer/util/metrics" + + // We need to import these so that the package wide init() function gets called + _ "github.com/algorand/indexer/conduit/plugins/exporters/all" + _ "github.com/algorand/indexer/conduit/plugins/importers/all" + _ "github.com/algorand/indexer/conduit/plugins/processors/all" +) + +var ( + loggerManager *loggers.LoggerManager + logger *log.Logger + conduitCmd = makeConduitCmd() + initCmd = makeInitCmd() + defaultDataDirectory = "data" +) + +// init() function for main package +func init() { + + loggerManager = loggers.MakeLoggerManager(os.Stdout) + // Setup logger + logger = loggerManager.MakeLogger() + + formatter := pipeline.PluginLogFormatter{ + Formatter: &log.JSONFormatter{ + DisableHTMLEscape: true, + }, + Type: "Conduit", + Name: "main", + } + + logger.SetFormatter(&formatter) + + conduitCmd.AddCommand(initCmd) + metrics.RegisterPrometheusMetrics() +} + +// runConduitCmdWithConfig run the main logic with a supplied conduit config +func runConduitCmdWithConfig(cfg *conduit.Config) error { + defer pipeline.HandlePanic(logger) + + // From docs: + // BindEnv takes one or more parameters. The first parameter is the key name, the rest are the name of the + // environment variables to bind to this key. If more than one are provided, they will take precedence in + // the specified order. The name of the environment variable is case sensitive. If the ENV variable name is not + // provided, then Viper will automatically assume that the ENV variable matches the following format: + // prefix + "_" + the key name in ALL CAPS. When you explicitly provide the ENV variable name (the second + // parameter), it does not automatically add the prefix. For example if the second parameter is "id", + // Viper will look for the ENV variable "ID". + // + // One important thing to recognize when working with ENV variables is that the value will be read each time + // it is accessed. Viper does not fix the value when the BindEnv is called. + err := viper.BindEnv("data-dir", "CONDUIT_DATA_DIR") + if err != nil { + return err + } + + // Changed returns true if the key was set during parse and IsSet determines if it is in the internal + // viper hashmap + if !cfg.Flags.Changed("data-dir") && viper.IsSet("data-dir") { + cfg.ConduitDataDir = viper.Get("data-dir").(string) + } + + logger.Info(cfg) + + pCfg, err := pipeline.MakePipelineConfig(logger, cfg) + + if err != nil { + return err + } + + logger.Info("Conduit configuration is valid") + + if pCfg.LogFile != "" { + logger.Infof("Conduit log file: %s", pCfg.LogFile) + } + + ctx := context.Background() + + pipeline, err := pipeline.MakePipeline(ctx, pCfg, logger) + if err != nil { + return fmt.Errorf("pipeline creation error: %w", err) + } + + err = pipeline.Init() + if err != nil { + return fmt.Errorf("pipeline init error: %w", err) + } + pipeline.Start() + defer pipeline.Stop() + pipeline.Wait() + return pipeline.Error() +} + +// makeConduitCmd creates the main cobra command, initializes flags, and viper aliases +func makeConduitCmd() *cobra.Command { + cfg := &conduit.Config{} + cmd := &cobra.Command{ + Use: "conduit", + Short: "run the conduit framework", + Long: "run the conduit framework", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return runConduitCmdWithConfig(cfg) + }, + SilenceUsage: true, + // Silence errors because our logger will catch and print any errors + SilenceErrors: true, + } + cfg.Flags = cmd.Flags() + cfg.Flags.StringVarP(&cfg.ConduitDataDir, "data-dir", "d", "", "set the data directory for the conduit binary") + cfg.Flags.Uint64VarP(&cfg.NextRoundOverride, "next-round-override", "r", 0, "set the starting round. Overrides next-round in metadata.json") + + cmd.AddCommand(list.Command) + + return cmd +} + +//go:embed conduit.yml.example +var sampleConfig string + +func runConduitInit(path string) error { + var location string + if path == "" { + path = defaultDataDirectory + location = "in the current working directory" + } else { + location = fmt.Sprintf("at '%s'", path) + } + + if err := os.MkdirAll(path, 0755); err != nil { + return err + } + + configFilePath := filepath.Join(path, conduit.DefaultConfigName) + f, err := os.Create(configFilePath) + if err != nil { + return fmt.Errorf("runConduitInit(): failed to create %s", configFilePath) + } + defer f.Close() + + f.WriteString(sampleConfig) + if err != nil { + return fmt.Errorf("runConduitInit(): failed to write sample config: %w", err) + } + + fmt.Printf("A data directory has been created %s.\n", location) + fmt.Printf("\nBefore it can be used, the config file needs to be updated\n") + fmt.Printf("by setting the algod address/token and the block-dir path where\n") + fmt.Printf("Conduit should write the block files.\n") + fmt.Printf("\nOnce the config file is updated, start Conduit with:\n") + fmt.Printf(" ./conduit -d %s\n", path) + return nil +} + +// makeInitCmd creates a sample data directory. +func makeInitCmd() *cobra.Command { + var data string + cmd := &cobra.Command{ + Use: "init", + Short: "initializes a sample data directory", + Long: "initializes a Conduit data directory and conduit.yml file configured with the file_writer plugin. The config file needs to be modified slightly to include an algod address and token. Once ready, launch conduit with './conduit -d /path/to/data'.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return runConduitInit(data) + }, + SilenceUsage: true, + } + + cmd.Flags().StringVarP(&data, "data", "d", "", "Full path to new data directory. If not set, a directory named 'data' will be created in the current directory.") + + return cmd +} + +func main() { + if err := conduitCmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + + os.Exit(0) +} diff --git a/cmd/conduit/main_test.go b/cmd/conduit/main_test.go new file mode 100644 index 000000000..686589bd8 --- /dev/null +++ b/cmd/conduit/main_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestInitDataDirectory tests the initialization of the data directory +func TestInitDataDirectory(t *testing.T) { + verifyFile := func(file string) { + require.FileExists(t, file) + data, err := os.ReadFile(file) + require.NoError(t, err) + require.Equal(t, sampleConfig, string(data)) + } + + // avoid clobbering an existing data directory + defaultDataDirectory = "override" + require.NoDirExists(t, defaultDataDirectory) + + runConduitInit("") + verifyFile(fmt.Sprintf("%s/conduit.yml", defaultDataDirectory)) + + runConduitInit(fmt.Sprintf("%s/provided_directory", defaultDataDirectory)) + verifyFile(fmt.Sprintf("%s/provided_directory/conduit.yml", defaultDataDirectory)) + + os.RemoveAll(defaultDataDirectory) +} diff --git a/conduit/config.go b/conduit/config.go new file mode 100644 index 000000000..aa455b6dc --- /dev/null +++ b/conduit/config.go @@ -0,0 +1,50 @@ +package conduit + +import ( + "fmt" + "strings" + + "github.com/spf13/pflag" + + "github.com/algorand/go-algorand/util" +) + +// DefaultConfigName is the default conduit configuration filename. +const DefaultConfigName = "conduit.yml" + +// Config configuration for conduit running. +// This is needed to support a CONDUIT_DATA_DIR environment variable. +type Config struct { + Flags *pflag.FlagSet + ConduitDataDir string `yaml:"data-dir"` + NextRoundOverride uint64 `yaml:"next-round-override"` +} + +func (cfg *Config) String() string { + var sb strings.Builder + + var dataDirToPrint string + if cfg.ConduitDataDir == "" { + dataDirToPrint = "[EMPTY]" + } else { + dataDirToPrint = cfg.ConduitDataDir + } + + fmt.Fprintf(&sb, "Data Directory: %s ", dataDirToPrint) + + return sb.String() +} + +// Valid validates a supplied configuration +func (cfg *Config) Valid() error { + + if cfg.ConduitDataDir == "" { + return fmt.Errorf("supplied data directory was empty") + } + + if !util.IsDir(cfg.ConduitDataDir) { + return fmt.Errorf("supplied data directory (%s) was not valid", cfg.ConduitDataDir) + } + + return nil +} diff --git a/conduit/config_test.go b/conduit/config_test.go new file mode 100644 index 000000000..3a5150a97 --- /dev/null +++ b/conduit/config_test.go @@ -0,0 +1,37 @@ +package conduit + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestConfigString test basic configuration stringification +func TestConfigString(t *testing.T) { + dataDir := "/tmp/dd" + cfg := Config{ConduitDataDir: dataDir} + + outputStr := fmt.Sprintf("Data Directory: %s ", dataDir) + require.Equal(t, cfg.String(), outputStr) +} + +// TestConfigValid tests validity scenarios for config +func TestConfigValid(t *testing.T) { + tests := []struct { + name string + dataDir string + err error + }{ + {"valid", t.TempDir(), nil}, + {"nil data dir", "", fmt.Errorf("supplied data directory was empty")}, + {"invalid data dir", "/tmp/this_directory_should_not_exist", fmt.Errorf("supplied data directory (/tmp/this_directory_should_not_exist) was not valid")}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cfg := Config{ConduitDataDir: test.dataDir} + assert.Equal(t, test.err, cfg.Valid()) + }) + } +} diff --git a/conduit/errors.go b/conduit/errors.go new file mode 100644 index 000000000..4ceb11aa4 --- /dev/null +++ b/conduit/errors.go @@ -0,0 +1,11 @@ +package conduit + +import "fmt" + +// CriticalError an error that causes the entire conduit pipeline to +// stop +type CriticalError struct{} + +func (e *CriticalError) Error() string { + return fmt.Sprintf("critical error occured") +} diff --git a/conduit/gen/lookup.go b/conduit/gen/lookup.go new file mode 100644 index 000000000..37b40dad5 --- /dev/null +++ b/conduit/gen/lookup.go @@ -0,0 +1,312 @@ +// Code generated via go generate. DO NOT EDIT. +package fields + +import "github.com/algorand/go-algorand/data/transactions" + +// SignedTxnMap generates a map with the key as the codec tag and the value as a function +// that returns the associated variable +var SignedTxnMap = map[string]func(*transactions.SignedTxnInBlock) interface{}{ + "aca": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.ApplyData.AssetClosingAmount) + }, + "apid": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.ApplyData.ApplicationID) + }, + "ca": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.ApplyData.ClosingAmount) + }, + "caid": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.ApplyData.ConfigAsset) + }, + "dt": func(i *transactions.SignedTxnInBlock) interface{} { return &((*i).SignedTxnWithAD.ApplyData.EvalDelta) }, + "dt.gd": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.ApplyData.EvalDelta.GlobalDelta) + }, + "dt.itx": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.ApplyData.EvalDelta.InnerTxns) + }, + "dt.ld": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.ApplyData.EvalDelta.LocalDeltas) + }, + "dt.lg": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.ApplyData.EvalDelta.Logs) + }, + "hgh": func(i *transactions.SignedTxnInBlock) interface{} { return &((*i).HasGenesisHash) }, + "hgi": func(i *transactions.SignedTxnInBlock) interface{} { return &((*i).HasGenesisID) }, + "lsig": func(i *transactions.SignedTxnInBlock) interface{} { return &((*i).SignedTxnWithAD.SignedTxn.Lsig) }, + "lsig.arg": func(i *transactions.SignedTxnInBlock) interface{} { return &((*i).SignedTxnWithAD.SignedTxn.Lsig.Args) }, + "lsig.l": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Lsig.Logic) + }, + "lsig.msig": func(i *transactions.SignedTxnInBlock) interface{} { return &((*i).SignedTxnWithAD.SignedTxn.Lsig.Msig) }, + "lsig.msig.subsig": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Lsig.Msig.Subsigs) + }, + "lsig.msig.thr": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Lsig.Msig.Threshold) + }, + "lsig.msig.v": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Lsig.Msig.Version) + }, + "lsig.sig": func(i *transactions.SignedTxnInBlock) interface{} { return &((*i).SignedTxnWithAD.SignedTxn.Lsig.Sig) }, + "msig": func(i *transactions.SignedTxnInBlock) interface{} { return &((*i).SignedTxnWithAD.SignedTxn.Msig) }, + "msig.subsig": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Msig.Subsigs) + }, + "msig.thr": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Msig.Threshold) + }, + "msig.v": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Msig.Version) + }, + "rc": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.ApplyData.CloseRewards) + }, + "rr": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.ApplyData.ReceiverRewards) + }, + "rs": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.ApplyData.SenderRewards) + }, + "sgnr": func(i *transactions.SignedTxnInBlock) interface{} { return &((*i).SignedTxnWithAD.SignedTxn.AuthAddr) }, + "sig": func(i *transactions.SignedTxnInBlock) interface{} { return &((*i).SignedTxnWithAD.SignedTxn.Sig) }, + "txn": func(i *transactions.SignedTxnInBlock) interface{} { return &((*i).SignedTxnWithAD.SignedTxn.Txn) }, + "txn.aamt": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetTransferTxnFields.AssetAmount) + }, + "txn.aclose": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetTransferTxnFields.AssetCloseTo) + }, + "txn.afrz": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetFreezeTxnFields.AssetFrozen) + }, + "txn.amt": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Amount) + }, + "txn.apaa": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.ApplicationArgs) + }, + "txn.apan": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.OnCompletion) + }, + "txn.apap": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.ApprovalProgram) + }, + "txn.apar": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams) + }, + "txn.apar.am": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.MetadataHash) + }, + "txn.apar.an": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.AssetName) + }, + "txn.apar.au": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.URL) + }, + "txn.apar.c": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.Clawback) + }, + "txn.apar.dc": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.Decimals) + }, + "txn.apar.df": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.DefaultFrozen) + }, + "txn.apar.f": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.Freeze) + }, + "txn.apar.m": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.Manager) + }, + "txn.apar.r": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.Reserve) + }, + "txn.apar.t": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.Total) + }, + "txn.apar.un": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.UnitName) + }, + "txn.apas": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.ForeignAssets) + }, + "txn.apat": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.Accounts) + }, + "txn.apep": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.ExtraProgramPages) + }, + "txn.apfa": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.ForeignApps) + }, + "txn.apgs": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.GlobalStateSchema) + }, + "txn.apgs.nbs": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.GlobalStateSchema.NumByteSlice) + }, + "txn.apgs.nui": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.GlobalStateSchema.NumUint) + }, + "txn.apid": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.ApplicationID) + }, + "txn.apls": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.LocalStateSchema) + }, + "txn.apls.nbs": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.LocalStateSchema.NumByteSlice) + }, + "txn.apls.nui": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.LocalStateSchema.NumUint) + }, + "txn.apsu": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.ClearStateProgram) + }, + "txn.arcv": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetTransferTxnFields.AssetReceiver) + }, + "txn.asnd": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetTransferTxnFields.AssetSender) + }, + "txn.caid": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.ConfigAsset) + }, + "txn.close": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.CloseRemainderTo) + }, + "txn.fadd": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetFreezeTxnFields.FreezeAccount) + }, + "txn.faid": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetFreezeTxnFields.FreezeAsset) + }, + "txn.fee": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.Header.Fee) + }, + "txn.fv": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.Header.FirstValid) + }, + "txn.gen": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.Header.GenesisID) + }, + "txn.gh": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.Header.GenesisHash) + }, + "txn.grp": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.Header.Group) + }, + "txn.lv": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.Header.LastValid) + }, + "txn.lx": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.Header.Lease) + }, + "txn.nonpart": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.KeyregTxnFields.Nonparticipation) + }, + "txn.note": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.Header.Note) + }, + "txn.rcv": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Receiver) + }, + "txn.rekey": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.Header.RekeyTo) + }, + "txn.selkey": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.KeyregTxnFields.SelectionPK) + }, + "txn.snd": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.Header.Sender) + }, + "txn.sp": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof) + }, + "txn.sp.P": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.PartProofs) + }, + "txn.sp.P.hsh": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.PartProofs.HashFactory) + }, + "txn.sp.P.hsh.t": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.PartProofs.HashFactory.HashType) + }, + "txn.sp.P.pth": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.PartProofs.Path) + }, + "txn.sp.P.td": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.PartProofs.TreeDepth) + }, + "txn.sp.S": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.SigProofs) + }, + "txn.sp.S.hsh": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.SigProofs.HashFactory) + }, + "txn.sp.S.hsh.t": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.SigProofs.HashFactory.HashType) + }, + "txn.sp.S.pth": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.SigProofs.Path) + }, + "txn.sp.S.td": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.SigProofs.TreeDepth) + }, + "txn.sp.c": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.SigCommit) + }, + "txn.sp.pr": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.PositionsToReveal) + }, + "txn.sp.r": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.Reveals) + }, + "txn.sp.v": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.MerkleSignatureSaltVersion) + }, + "txn.sp.w": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.SignedWeight) + }, + "txn.spmsg": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.Message) + }, + "txn.spmsg.P": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.Message.LnProvenWeight) + }, + "txn.spmsg.b": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.Message.BlockHeadersCommitment) + }, + "txn.spmsg.f": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.Message.FirstAttestedRound) + }, + "txn.spmsg.l": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.Message.LastAttestedRound) + }, + "txn.spmsg.v": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.Message.VotersCommitment) + }, + "txn.sprfkey": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.KeyregTxnFields.StateProofPK) + }, + "txn.sptype": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProofType) + }, + "txn.type": func(i *transactions.SignedTxnInBlock) interface{} { return &((*i).SignedTxnWithAD.SignedTxn.Txn.Type) }, + "txn.votefst": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.KeyregTxnFields.VoteFirst) + }, + "txn.votekd": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.KeyregTxnFields.VoteKeyDilution) + }, + "txn.votekey": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.KeyregTxnFields.VotePK) + }, + "txn.votelst": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.KeyregTxnFields.VoteLast) + }, + "txn.xaid": func(i *transactions.SignedTxnInBlock) interface{} { + return &((*i).SignedTxnWithAD.SignedTxn.Txn.AssetTransferTxnFields.XferAsset) + }, +} diff --git a/conduit/hooks.go b/conduit/hooks.go new file mode 100644 index 000000000..40a608942 --- /dev/null +++ b/conduit/hooks.go @@ -0,0 +1,26 @@ +package conduit + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/algorand/indexer/data" +) + +// OnCompleteFunc is the signature for the Completed functional interface. +type OnCompleteFunc func(input data.BlockData) error + +// Completed is called by the conduit pipeline after every exporter has +// finished. It can be used for things like finalizing state. +type Completed interface { + // OnComplete will be called by the Conduit framework when the pipeline + // finishes processing a round. + OnComplete(input data.BlockData) error +} + +// ProvideMetricsFunc is the signature for the PluginMetrics interface. +type ProvideMetricsFunc func() []prometheus.Collector + +// PluginMetrics is for defining plugin specific metrics +type PluginMetrics interface { + ProvideMetrics() []prometheus.Collector +} diff --git a/conduit/init_provider.go b/conduit/init_provider.go new file mode 100644 index 000000000..2303513b5 --- /dev/null +++ b/conduit/init_provider.go @@ -0,0 +1,30 @@ +package conduit + +import ( + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" +) + +// PipelineInitProvider algod based init provider +type PipelineInitProvider struct { + currentRound *basics.Round + genesis *bookkeeping.Genesis +} + +// MakePipelineInitProvider constructs an init provider. +func MakePipelineInitProvider(currentRound *basics.Round, genesis *bookkeeping.Genesis) *PipelineInitProvider { + return &PipelineInitProvider{ + currentRound: currentRound, + genesis: genesis, + } +} + +// GetGenesis produces genesis pointer +func (a *PipelineInitProvider) GetGenesis() *bookkeeping.Genesis { + return a.genesis +} + +// NextDBRound provides next database round +func (a *PipelineInitProvider) NextDBRound() basics.Round { + return *a.currentRound +} diff --git a/conduit/metadata.go b/conduit/metadata.go new file mode 100644 index 000000000..de51d8ec9 --- /dev/null +++ b/conduit/metadata.go @@ -0,0 +1,15 @@ +package conduit + +// Metadata returns fields relevant to identification and description of plugins. +type Metadata struct { + Name string + Description string + Deprecated bool + SampleConfig string +} + +// PluginMetadata is the common interface for providing plugin metadata. +type PluginMetadata interface { + // Metadata associated with the plugin. + Metadata() Metadata +} diff --git a/conduit/pipeline/common.go b/conduit/pipeline/common.go new file mode 100644 index 000000000..49090b620 --- /dev/null +++ b/conduit/pipeline/common.go @@ -0,0 +1,10 @@ +package pipeline + +import log "github.com/sirupsen/logrus" + +// HandlePanic function to log panics in a common way +func HandlePanic(logger *log.Logger) { + if r := recover(); r != nil { + logger.Panicf("conduit pipeline experienced a panic: %v", r) + } +} diff --git a/conduit/pipeline/logging.go b/conduit/pipeline/logging.go new file mode 100644 index 000000000..40a628c98 --- /dev/null +++ b/conduit/pipeline/logging.go @@ -0,0 +1,30 @@ +package pipeline + +import ( + log "github.com/sirupsen/logrus" +) + +// PluginLogFormatter formats the log message with special conduit tags +type PluginLogFormatter struct { + Formatter *log.JSONFormatter + Type string + Name string +} + +// Format allows this to be used as a logrus formatter +func (f PluginLogFormatter) Format(entry *log.Entry) ([]byte, error) { + // Underscores force these to be in the front in order type -> name + entry.Data["__type"] = f.Type + entry.Data["_name"] = f.Name + return f.Formatter.Format(entry) +} + +func makePluginLogFormatter(pluginType string, pluginName string) PluginLogFormatter { + return PluginLogFormatter{ + Formatter: &log.JSONFormatter{ + DisableHTMLEscape: true, + }, + Type: pluginType, + Name: pluginName, + } +} diff --git a/conduit/pipeline/logging_test.go b/conduit/pipeline/logging_test.go new file mode 100644 index 000000000..dcff8aa0b --- /dev/null +++ b/conduit/pipeline/logging_test.go @@ -0,0 +1,34 @@ +package pipeline + +import ( + "testing" + "time" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +// TestPluginLogFormatter_Format tests the output of the formatter while pondering philosophy +func TestPluginLogFormatter_Format(t *testing.T) { + + pluginType := "A Question" + pluginName := "What's in a name?" + + pluginFormatter := makePluginLogFormatter(pluginType, pluginName) + + l := log.New() + + entry := &log.Entry{ + Time: time.Time{}, + Level: log.InfoLevel, + Message: "That which we call a rose by any other name would smell just as sweet.", + Data: log.Fields{}, + Logger: l, + } + + bytes, err := pluginFormatter.Format(entry) + assert.Nil(t, err) + str := string(bytes) + assert.Equal(t, str, "{\"__type\":\"A Question\",\"_name\":\"What's in a name?\",\"level\":\"info\",\"msg\":\"That which we call a rose by any other name would smell just as sweet.\",\"time\":\"0001-01-01T00:00:00Z\"}\n") + +} diff --git a/conduit/pipeline/metadata.go b/conduit/pipeline/metadata.go new file mode 100644 index 000000000..8cfc0f104 --- /dev/null +++ b/conduit/pipeline/metadata.go @@ -0,0 +1,45 @@ +package pipeline + +// TODO: move this to plugins package after reorganizing plugin packages. + +import ( + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/plugins/exporters" + "github.com/algorand/indexer/conduit/plugins/importers" + "github.com/algorand/indexer/conduit/plugins/processors" +) + +// AllMetadata gets a slice with metadata from all registered plugins. +func AllMetadata() (results []conduit.Metadata) { + results = append(results, ImporterMetadata()...) + results = append(results, ProcessorMetadata()...) + results = append(results, ExporterMetadata()...) + return +} + +// ImporterMetadata gets a slice with metadata for all importers.Importer plugins. +func ImporterMetadata() (results []conduit.Metadata) { + for _, constructor := range importers.Importers { + plugin := constructor.New() + results = append(results, plugin.Metadata()) + } + return +} + +// ProcessorMetadata gets a slice with metadata for all importers.Processor plugins. +func ProcessorMetadata() (results []conduit.Metadata) { + for _, constructor := range processors.Processors { + plugin := constructor.New() + results = append(results, plugin.Metadata()) + } + return +} + +// ExporterMetadata gets a slice with metadata for all importers.Exporter plugins. +func ExporterMetadata() (results []conduit.Metadata) { + for _, constructor := range exporters.Exporters { + plugin := constructor.New() + results = append(results, plugin.Metadata()) + } + return +} diff --git a/conduit/pipeline/metadata_test.go b/conduit/pipeline/metadata_test.go new file mode 100644 index 000000000..efa55ad6e --- /dev/null +++ b/conduit/pipeline/metadata_test.go @@ -0,0 +1,30 @@ +package pipeline + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "gopkg.in/yaml.v3" +) + +import ( + _ "github.com/algorand/indexer/conduit/plugins/exporters/all" + _ "github.com/algorand/indexer/conduit/plugins/exporters/example" + _ "github.com/algorand/indexer/conduit/plugins/importers/all" + _ "github.com/algorand/indexer/conduit/plugins/processors/all" +) + +// TestSamples ensures that all plugins contain a sample file with valid yaml. +func TestSamples(t *testing.T) { + metadata := AllMetadata() + for _, data := range metadata { + data := data + t.Run(data.Name, func(t *testing.T) { + t.Parallel() + var config NameConfigPair + assert.NoError(t, yaml.Unmarshal([]byte(data.SampleConfig), &config)) + assert.Equal(t, data.Name, config.Name) + }) + } +} diff --git a/conduit/pipeline/pipeline.go b/conduit/pipeline/pipeline.go new file mode 100644 index 000000000..98a7fcc30 --- /dev/null +++ b/conduit/pipeline/pipeline.go @@ -0,0 +1,608 @@ +package pipeline + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "path" + "path/filepath" + "runtime/pprof" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" + + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "gopkg.in/yaml.v3" + + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/exporters" + "github.com/algorand/indexer/conduit/plugins/importers" + "github.com/algorand/indexer/conduit/plugins/processors" + "github.com/algorand/indexer/data" + "github.com/algorand/indexer/util" + "github.com/algorand/indexer/util/metrics" +) + +func init() { + viper.SetConfigType("yaml") +} + +// NameConfigPair is a generic structure used across plugin configuration ser/de +type NameConfigPair struct { + Name string `yaml:"name"` + Config map[string]interface{} `yaml:"config"` +} + +// Metrics configs for turning on Prometheus endpoint /metrics +type Metrics struct { + Mode string `yaml:"mode"` + Addr string `yaml:"addr"` +} + +// Config stores configuration specific to the conduit pipeline +type Config struct { + ConduitConfig *conduit.Config + + CPUProfile string `yaml:"cpu-profile"` + PIDFilePath string `yaml:"pid-filepath"` + + LogFile string `yaml:"log-file"` + PipelineLogLevel string `yaml:"log-level"` + // Store a local copy to access parent variables + Importer NameConfigPair `yaml:"importer"` + Processors []NameConfigPair `yaml:"processors"` + Exporter NameConfigPair `yaml:"exporter"` + Metrics Metrics `yaml:"metrics"` +} + +// Valid validates pipeline config +func (cfg *Config) Valid() error { + if cfg.ConduitConfig == nil { + return fmt.Errorf("Config.Valid(): conduit configuration was nil") + } + + if _, err := log.ParseLevel(cfg.PipelineLogLevel); err != nil { + return fmt.Errorf("Config.Valid(): pipeline log level (%s) was invalid: %w", cfg.PipelineLogLevel, err) + } + + if len(cfg.Importer.Config) == 0 { + return fmt.Errorf("Config.Valid(): importer configuration was empty") + } + + if len(cfg.Exporter.Config) == 0 { + return fmt.Errorf("Config.Valid(): exporter configuration was empty") + } + + return nil +} + +// MakePipelineConfig creates a pipeline configuration +func MakePipelineConfig(logger *log.Logger, cfg *conduit.Config) (*Config, error) { + if cfg == nil { + return nil, fmt.Errorf("MakePipelineConfig(): empty conduit config") + } + + // double check that it is valid + if err := cfg.Valid(); err != nil { + return nil, fmt.Errorf("MakePipelineConfig(): %w", err) + } + pCfg := Config{PipelineLogLevel: logger.Level.String(), ConduitConfig: cfg} + + // Search for pipeline configuration in data directory + autoloadParamConfigPath := filepath.Join(cfg.ConduitDataDir, conduit.DefaultConfigName) + + _, err := os.Stat(autoloadParamConfigPath) + paramConfigFound := err == nil + + if !paramConfigFound { + return nil, fmt.Errorf("MakePipelineConfig(): could not find %s in data directory (%s)", conduit.DefaultConfigName, cfg.ConduitDataDir) + } + + logger.Infof("Auto-loading Conduit Configuration: %s", autoloadParamConfigPath) + + file, err := os.ReadFile(autoloadParamConfigPath) + if err != nil { + return nil, fmt.Errorf("MakePipelineConfig(): reading config error: %w", err) + } + err = yaml.Unmarshal(file, &pCfg) + if err != nil { + return nil, fmt.Errorf("MakePipelineConfig(): config file (%s) was mal-formed yaml: %w", autoloadParamConfigPath, err) + } + + if err := pCfg.Valid(); err != nil { + return nil, fmt.Errorf("MakePipelineConfig(): config file (%s) had mal-formed schema: %w", autoloadParamConfigPath, err) + } + + return &pCfg, nil + +} + +// Pipeline is a struct that orchestrates the entire +// sequence of events, taking in importers, processors and +// exporters and generating the result +type Pipeline interface { + Init() error + Start() + Stop() + Error() error + Wait() +} + +type pipelineImpl struct { + ctx context.Context + cf context.CancelFunc + wg sync.WaitGroup + cfg *Config + logger *log.Logger + profFile *os.File + err error + mu sync.RWMutex + + initProvider *data.InitProvider + + importer *importers.Importer + processors []*processors.Processor + exporter *exporters.Exporter + completeCallback []conduit.OnCompleteFunc + + pipelineMetadata state + + metricsCallback []conduit.ProvideMetricsFunc +} + +// state contains the pipeline state. +type state struct { + GenesisHash string `json:"genesis-hash"` + Network string `json:"network"` + NextRound uint64 `json:"next-round"` +} + +func (p *pipelineImpl) Error() error { + p.mu.RLock() + defer p.mu.RUnlock() + return p.err +} + +func (p *pipelineImpl) setError(err error) { + p.mu.Lock() + defer p.mu.Unlock() + p.err = err +} + +func (p *pipelineImpl) registerLifecycleCallbacks() { + if v, ok := (*p.importer).(conduit.Completed); ok { + p.completeCallback = append(p.completeCallback, v.OnComplete) + } + for _, processor := range p.processors { + if v, ok := (*processor).(conduit.Completed); ok { + p.completeCallback = append(p.completeCallback, v.OnComplete) + } + } + if v, ok := (*p.exporter).(conduit.Completed); ok { + p.completeCallback = append(p.completeCallback, v.OnComplete) + } +} + +func (p *pipelineImpl) registerPluginMetricsCallbacks() { + if v, ok := (*p.importer).(conduit.PluginMetrics); ok { + p.metricsCallback = append(p.metricsCallback, v.ProvideMetrics) + } + for _, processor := range p.processors { + if v, ok := (*processor).(conduit.PluginMetrics); ok { + p.metricsCallback = append(p.metricsCallback, v.ProvideMetrics) + } + } + if v, ok := (*p.exporter).(conduit.PluginMetrics); ok { + p.metricsCallback = append(p.metricsCallback, v.ProvideMetrics) + } +} + +// Init prepares the pipeline for processing block data +func (p *pipelineImpl) Init() error { + p.logger.Infof("Starting Pipeline Initialization") + + if p.cfg.CPUProfile != "" { + p.logger.Infof("Creating CPU Profile file at %s", p.cfg.CPUProfile) + var err error + profFile, err := os.Create(p.cfg.CPUProfile) + if err != nil { + p.logger.WithError(err).Errorf("%s: create, %v", p.cfg.CPUProfile, err) + return err + } + p.profFile = profFile + err = pprof.StartCPUProfile(profFile) + if err != nil { + p.logger.WithError(err).Errorf("%s: start pprof, %v", p.cfg.CPUProfile, err) + return err + } + } + + if p.cfg.PIDFilePath != "" { + err := util.CreateIndexerPidFile(p.logger, p.cfg.PIDFilePath) + if err != nil { + return err + } + } + + // TODO Need to change interfaces to accept config of map[string]interface{} + + // Initialize Importer + importerLogger := log.New() + // Make sure we are thread-safe + importerLogger.SetOutput(p.logger.Out) + importerName := (*p.importer).Metadata().Name + importerLogger.SetFormatter(makePluginLogFormatter(plugins.Importer, importerName)) + + configs, err := yaml.Marshal(p.cfg.Importer.Config) + if err != nil { + return fmt.Errorf("Pipeline.Start(): could not serialize Importer.Config: %w", err) + } + genesis, err := (*p.importer).Init(p.ctx, plugins.PluginConfig(configs), importerLogger) + if err != nil { + return fmt.Errorf("Pipeline.Start(): could not initialize importer (%s): %w", importerName, err) + } + + // initialize or load pipeline metadata + gh := crypto.HashObj(genesis).String() + p.pipelineMetadata.GenesisHash = gh + p.pipelineMetadata.Network = string(genesis.Network) + p.pipelineMetadata, err = p.initializeOrLoadBlockMetadata() + if err != nil { + return fmt.Errorf("Pipeline.Start(): could not read metadata: %w", err) + } + if p.pipelineMetadata.GenesisHash != gh { + return fmt.Errorf("Pipeline.Start(): genesis hash in metadata does not match expected value: actual %s, expected %s", gh, p.pipelineMetadata.GenesisHash) + } + // overriding NextRound if NextRoundOverride is set + if p.cfg.ConduitConfig.NextRoundOverride > 0 { + p.logger.Infof("Overriding default next round from %d to %d.", p.pipelineMetadata.NextRound, p.cfg.ConduitConfig.NextRoundOverride) + p.pipelineMetadata.NextRound = p.cfg.ConduitConfig.NextRoundOverride + } + + p.logger.Infof("Initialized Importer: %s", importerName) + + // InitProvider + round := basics.Round(p.pipelineMetadata.NextRound) + var initProvider data.InitProvider = conduit.MakePipelineInitProvider(&round, genesis) + p.initProvider = &initProvider + + // Initialize Processors + for idx, processor := range p.processors { + processorLogger := log.New() + // Make sure we are thread-safe + processorLogger.SetOutput(p.logger.Out) + processorLogger.SetFormatter(makePluginLogFormatter(plugins.Processor, (*processor).Metadata().Name)) + configs, err = yaml.Marshal(p.cfg.Processors[idx].Config) + if err != nil { + return fmt.Errorf("Pipeline.Start(): could not serialize Processors[%d].Config : %w", idx, err) + } + err := (*processor).Init(p.ctx, *p.initProvider, plugins.PluginConfig(configs), processorLogger) + processorName := (*processor).Metadata().Name + if err != nil { + return fmt.Errorf("Pipeline.Init(): could not initialize processor (%s): %w", processorName, err) + } + p.logger.Infof("Initialized Processor: %s", processorName) + } + + // Initialize Exporter + exporterLogger := log.New() + // Make sure we are thread-safe + exporterLogger.SetOutput(p.logger.Out) + exporterLogger.SetFormatter(makePluginLogFormatter(plugins.Exporter, (*p.exporter).Metadata().Name)) + + configs, err = yaml.Marshal(p.cfg.Exporter.Config) + if err != nil { + return fmt.Errorf("Pipeline.Start(): could not serialize Exporter.Config : %w", err) + } + err = (*p.exporter).Init(p.ctx, *p.initProvider, plugins.PluginConfig(configs), exporterLogger) + exporterName := (*p.exporter).Metadata().Name + if err != nil { + return fmt.Errorf("Pipeline.Start(): could not initialize Exporter (%s): %w", exporterName, err) + } + p.logger.Infof("Initialized Exporter: %s", exporterName) + + // Register callbacks. + p.registerLifecycleCallbacks() + + // start metrics server + if p.cfg.Metrics.Mode == "ON" { + p.registerPluginMetricsCallbacks() + for _, cb := range p.metricsCallback { + collectors := cb() + for _, c := range collectors { + prometheus.Register(c) + } + } + go p.startMetricsServer() + } + + return err +} + +func (p *pipelineImpl) Stop() { + p.cf() + p.wg.Wait() + + if p.profFile != nil { + if err := p.profFile.Close(); err != nil { + p.logger.WithError(err).Errorf("%s: could not close CPUProf file", p.profFile.Name()) + } + pprof.StopCPUProfile() + } + + if p.cfg.PIDFilePath != "" { + if err := os.Remove(p.cfg.PIDFilePath); err != nil { + p.logger.WithError(err).Errorf("%s: could not remove pid file", p.cfg.PIDFilePath) + } + } + + if err := (*p.importer).Close(); err != nil { + // Log and continue on closing the rest of the pipeline + p.logger.Errorf("Pipeline.Stop(): Importer (%s) error on close: %v", (*p.importer).Metadata().Name, err) + } + + for _, processor := range p.processors { + if err := (*processor).Close(); err != nil { + // Log and continue on closing the rest of the pipeline + p.logger.Errorf("Pipeline.Stop(): Processor (%s) error on close: %v", (*processor).Metadata().Name, err) + } + } + + if err := (*p.exporter).Close(); err != nil { + p.logger.Errorf("Pipeline.Stop(): Exporter (%s) error on close: %v", (*p.exporter).Metadata().Name, err) + } +} + +func (p *pipelineImpl) addMetrics(block data.BlockData, importTime time.Duration) { + metrics.BlockImportTimeSeconds.Observe(importTime.Seconds()) + metrics.ImportedTxnsPerBlock.Observe(float64(len(block.Payset))) + metrics.ImportedRoundGauge.Set(float64(block.Round())) + txnCountByType := make(map[string]int) + for _, txn := range block.Payset { + txnCountByType[string(txn.Txn.Type)]++ + } + for k, v := range txnCountByType { + metrics.ImportedTxns.WithLabelValues(k).Set(float64(v)) + } +} + +// Start pushes block data through the pipeline +func (p *pipelineImpl) Start() { + p.wg.Add(1) + retry := 0 + go func() { + defer p.wg.Done() + // We need to add a separate recover function here since it launches its own go-routine + defer HandlePanic(p.logger) + for { + pipelineRun: + metrics.PipelineRetryCount.Observe(float64(retry)) + select { + case <-p.ctx.Done(): + return + default: + { + p.logger.Infof("Pipeline round: %v", p.pipelineMetadata.NextRound) + // fetch block + importStart := time.Now() + blkData, err := (*p.importer).GetBlock(p.pipelineMetadata.NextRound) + if err != nil { + p.logger.Errorf("%v", err) + p.setError(err) + retry++ + goto pipelineRun + } + metrics.ImporterTimeSeconds.Observe(time.Since(importStart).Seconds()) + // Start time currently measures operations after block fetching is complete. + // This is for backwards compatibility w/ Indexer's metrics + // run through processors + start := time.Now() + for _, proc := range p.processors { + processorStart := time.Now() + blkData, err = (*proc).Process(blkData) + if err != nil { + p.logger.Errorf("%v", err) + p.setError(err) + retry++ + goto pipelineRun + } + metrics.ProcessorTimeSeconds.WithLabelValues((*proc).Metadata().Name).Observe(time.Since(processorStart).Seconds()) + } + // run through exporter + exporterStart := time.Now() + err = (*p.exporter).Receive(blkData) + if err != nil { + p.logger.Errorf("%v", err) + p.setError(err) + retry++ + goto pipelineRun + } + + // Increment Round, update metadata + p.pipelineMetadata.NextRound++ + p.encodeMetadataToFile() + + // Callback Processors + for _, cb := range p.completeCallback { + err = cb(blkData) + if err != nil { + p.logger.Errorf("%v", err) + p.setError(err) + retry++ + goto pipelineRun + } + } + metrics.ExporterTimeSeconds.Observe(time.Since(exporterStart).Seconds()) + // Ignore round 0 (which is empty). + if p.pipelineMetadata.NextRound > 1 { + p.addMetrics(blkData, time.Since(start)) + } + p.setError(nil) + retry = 0 + } + } + + } + }() +} + +func (p *pipelineImpl) Wait() { + p.wg.Wait() +} + +func metadataPath(dataDir string) string { + return path.Join(dataDir, "metadata.json") +} + +func (p *pipelineImpl) encodeMetadataToFile() error { + pipelineMetadataFilePath := metadataPath(p.cfg.ConduitConfig.ConduitDataDir) + tempFilename := fmt.Sprintf("%s.temp", pipelineMetadataFilePath) + file, err := os.Create(tempFilename) + if err != nil { + return fmt.Errorf("encodeMetadataToFile(): failed to create temp metadata file: %w", err) + } + defer file.Close() + err = json.NewEncoder(file).Encode(p.pipelineMetadata) + if err != nil { + return fmt.Errorf("encodeMetadataToFile(): failed to write temp metadata: %w", err) + } + + err = os.Rename(tempFilename, pipelineMetadataFilePath) + if err != nil { + return fmt.Errorf("encodeMetadataToFile(): failed to replace metadata file: %w", err) + } + return nil +} + +func (p *pipelineImpl) initializeOrLoadBlockMetadata() (state, error) { + pipelineMetadataFilePath := metadataPath(p.cfg.ConduitConfig.ConduitDataDir) + if stat, err := os.Stat(pipelineMetadataFilePath); errors.Is(err, os.ErrNotExist) || (stat != nil && stat.Size() == 0) { + fmt.Println(err) + fmt.Println(stat) + if stat != nil && stat.Size() == 0 { + err = os.Remove(pipelineMetadataFilePath) + if err != nil { + return p.pipelineMetadata, fmt.Errorf("Init(): error creating file: %w", err) + } + } + err = p.encodeMetadataToFile() + if err != nil { + return p.pipelineMetadata, fmt.Errorf("Init(): error creating file: %w", err) + } + } else { + if err != nil { + return p.pipelineMetadata, fmt.Errorf("error opening file: %w", err) + } + var data []byte + data, err = os.ReadFile(pipelineMetadataFilePath) + if err != nil { + return p.pipelineMetadata, fmt.Errorf("error reading metadata: %w", err) + } + err = json.Unmarshal(data, &p.pipelineMetadata) + if err != nil { + return p.pipelineMetadata, fmt.Errorf("error reading metadata: %w", err) + } + } + return p.pipelineMetadata, nil +} + +// start a http server serving /metrics +func (p *pipelineImpl) startMetricsServer() { + http.Handle("/metrics", promhttp.Handler()) + http.ListenAndServe(p.cfg.Metrics.Addr, nil) + p.logger.Infof("conduit metrics serving on %s", p.cfg.Metrics.Addr) +} + +// MakePipeline creates a Pipeline +func MakePipeline(ctx context.Context, cfg *Config, logger *log.Logger) (Pipeline, error) { + + if cfg == nil { + return nil, fmt.Errorf("MakePipeline(): pipeline config was empty") + } + + if err := cfg.Valid(); err != nil { + return nil, fmt.Errorf("MakePipeline(): %w", err) + } + + if logger == nil { + return nil, fmt.Errorf("MakePipeline(): logger was empty") + } + + if cfg.LogFile != "" { + f, err := os.OpenFile(cfg.LogFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return nil, fmt.Errorf("MakePipeline(): %w", err) + } + logger.SetOutput(f) + } + + logLevel, err := log.ParseLevel(cfg.PipelineLogLevel) + if err != nil { + // Belt and suspenders. Valid() should have caught this + return nil, fmt.Errorf("MakePipeline(): config had mal-formed log level: %w", err) + } + logger.SetLevel(logLevel) + + cancelContext, cancelFunc := context.WithCancel(ctx) + + pipeline := &pipelineImpl{ + ctx: cancelContext, + cf: cancelFunc, + cfg: cfg, + logger: logger, + initProvider: nil, + importer: nil, + processors: []*processors.Processor{}, + exporter: nil, + } + + importerName := cfg.Importer.Name + + importerBuilder, err := importers.ImporterBuilderByName(importerName) + if err != nil { + return nil, fmt.Errorf("MakePipeline(): could not find importer builder with name: %s", importerName) + } + + importer := importerBuilder.New() + pipeline.importer = &importer + logger.Infof("Found Importer: %s", importerName) + + // --- + + for _, processorConfig := range cfg.Processors { + processorName := processorConfig.Name + + processorBuilder, err := processors.ProcessorBuilderByName(processorName) + if err != nil { + return nil, fmt.Errorf("MakePipeline(): could not find processor builder with name: %s", processorName) + } + + processor := processorBuilder.New() + pipeline.processors = append(pipeline.processors, &processor) + logger.Infof("Found Processor: %s", processorName) + } + + // --- + + exporterName := cfg.Exporter.Name + + exporterBuilder, err := exporters.ExporterBuilderByName(exporterName) + if err != nil { + return nil, fmt.Errorf("MakePipeline(): could not find exporter builder with name: %s", exporterName) + } + + exporter := exporterBuilder.New() + pipeline.exporter = &exporter + logger.Infof("Found Exporter: %s", exporterName) + + return pipeline, nil +} diff --git a/conduit/pipeline/pipeline_test.go b/conduit/pipeline/pipeline_test.go new file mode 100644 index 000000000..ba1f41b5a --- /dev/null +++ b/conduit/pipeline/pipeline_test.go @@ -0,0 +1,849 @@ +package pipeline + +import ( + "context" + "fmt" + "net/http" + "os" + "path" + "path/filepath" + "testing" + "time" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/exporters" + "github.com/algorand/indexer/conduit/plugins/importers" + "github.com/algorand/indexer/conduit/plugins/processors" + "github.com/algorand/indexer/data" +) + +// TestPipelineConfigValidity tests the Valid() function for the Config +func TestPipelineConfigValidity(t *testing.T) { + tests := []struct { + name string + toTest Config + errContains string + }{ + {"valid", Config{ + ConduitConfig: &conduit.Config{ConduitDataDir: ""}, + PipelineLogLevel: "info", + Importer: NameConfigPair{"test", map[string]interface{}{"a": "a"}}, + Processors: nil, + Exporter: NameConfigPair{"test", map[string]interface{}{"a": "a"}}, + }, ""}, + + {"valid 2", Config{ + ConduitConfig: &conduit.Config{ConduitDataDir: ""}, + PipelineLogLevel: "info", + Importer: NameConfigPair{"test", map[string]interface{}{"a": "a"}}, + Processors: []NameConfigPair{{"test", map[string]interface{}{"a": "a"}}}, + Exporter: NameConfigPair{"test", map[string]interface{}{"a": "a"}}, + }, ""}, + + {"empty config", Config{ConduitConfig: nil}, "Config.Valid(): conduit configuration was nil"}, + {"invalid log level", Config{ConduitConfig: &conduit.Config{ConduitDataDir: ""}, PipelineLogLevel: "asdf"}, "Config.Valid(): pipeline log level (asdf) was invalid:"}, + {"importer config was 0", + Config{ + ConduitConfig: &conduit.Config{ConduitDataDir: ""}, + PipelineLogLevel: "info", + Importer: NameConfigPair{"test", map[string]interface{}{}}, + }, "Config.Valid(): importer configuration was empty"}, + + {"exporter config was 0", + Config{ + ConduitConfig: &conduit.Config{ConduitDataDir: ""}, + PipelineLogLevel: "info", + Importer: NameConfigPair{"test", map[string]interface{}{"a": "a"}}, + Processors: []NameConfigPair{{"test", map[string]interface{}{"a": "a"}}}, + Exporter: NameConfigPair{"test", map[string]interface{}{}}, + }, "Config.Valid(): exporter configuration was empty"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.toTest.Valid() + + if test.errContains == "" { + assert.Nil(t, err) + return + } + + assert.Contains(t, err.Error(), test.errContains) + }) + } +} + +// TestMakePipelineConfig tests making the pipeline configuration +func TestMakePipelineConfig(t *testing.T) { + + l := log.New() + + _, err := MakePipelineConfig(l, nil) + assert.Equal(t, fmt.Errorf("MakePipelineConfig(): empty conduit config"), err) + + dataDir := t.TempDir() + + validConfigFile := `--- +log-level: info +importer: + name: "algod" + config: + netaddr: "http://127.0.0.1:8080" + token: "e36c01fc77e490f23e61899c0c22c6390d0fff1443af2c95d056dc5ce4e61302" +processors: + - name: "noop" + config: + catchpoint: "7560000#3OUX3TLXZNOK6YJXGETKRRV2MHMILF5CCIVZUOJCT6SLY5H2WWTQ" +exporter: + name: "noop" + config: + connectionstring: ""` + + err = os.WriteFile(filepath.Join(dataDir, conduit.DefaultConfigName), []byte(validConfigFile), 0777) + assert.Nil(t, err) + + cfg := &conduit.Config{ConduitDataDir: dataDir} + + pCfg, err := MakePipelineConfig(l, cfg) + assert.Nil(t, err) + assert.Equal(t, pCfg.PipelineLogLevel, "info") + assert.Equal(t, pCfg.Valid(), nil) + assert.Equal(t, pCfg.Importer.Name, "algod") + assert.Equal(t, pCfg.Importer.Config["token"], "e36c01fc77e490f23e61899c0c22c6390d0fff1443af2c95d056dc5ce4e61302") + assert.Equal(t, pCfg.Processors[0].Name, "noop") + assert.Equal(t, pCfg.Processors[0].Config["catchpoint"], "7560000#3OUX3TLXZNOK6YJXGETKRRV2MHMILF5CCIVZUOJCT6SLY5H2WWTQ") + assert.Equal(t, pCfg.Exporter.Name, "noop") + assert.Equal(t, pCfg.Exporter.Config["connectionstring"], "") + + // invalidDataDir has no auto load file + invalidDataDir := t.TempDir() + assert.Nil(t, err) + + cfgBad := &conduit.Config{ConduitDataDir: invalidDataDir} + _, err = MakePipelineConfig(l, cfgBad) + assert.Equal(t, err, + fmt.Errorf("MakePipelineConfig(): could not find %s in data directory (%s)", conduit.DefaultConfigName, cfgBad.ConduitDataDir)) + +} + +// a unique block data to validate with tests +var uniqueBlockData = data.BlockData{ + BlockHeader: bookkeeping.BlockHeader{ + Round: 1337, + }, +} + +type mockImporter struct { + mock.Mock + importers.Importer + genesis bookkeeping.Genesis + finalRound basics.Round + returnError bool + onCompleteError bool +} + +func (m *mockImporter) Init(_ context.Context, _ plugins.PluginConfig, _ *log.Logger) (*bookkeeping.Genesis, error) { + return &m.genesis, nil +} + +func (m *mockImporter) Close() error { + return nil +} + +func (m *mockImporter) Metadata() conduit.Metadata { + return conduit.Metadata{Name: "mockImporter"} +} + +func (m *mockImporter) GetBlock(rnd uint64) (data.BlockData, error) { + var err error + if m.returnError { + err = fmt.Errorf("importer") + } + m.Called(rnd) + // Return an error to make sure we + return uniqueBlockData, err +} + +func (m *mockImporter) OnComplete(input data.BlockData) error { + var err error + if m.onCompleteError { + err = fmt.Errorf("on complete") + } + m.finalRound = input.BlockHeader.Round + m.Called(input) + return err +} + +type mockProcessor struct { + mock.Mock + processors.Processor + finalRound basics.Round + returnError bool + onCompleteError bool +} + +func (m *mockProcessor) Init(_ context.Context, _ data.InitProvider, _ plugins.PluginConfig, _ *log.Logger) error { + return nil +} + +func (m *mockProcessor) Close() error { + return nil +} + +func (m *mockProcessor) Metadata() conduit.Metadata { + return conduit.Metadata{ + Name: "mockProcessor", + } +} + +func (m *mockProcessor) Process(input data.BlockData) (data.BlockData, error) { + var err error + if m.returnError { + err = fmt.Errorf("process") + } + m.Called(input) + input.BlockHeader.Round++ + return input, err +} + +func (m *mockProcessor) OnComplete(input data.BlockData) error { + var err error + if m.onCompleteError { + err = fmt.Errorf("on complete") + } + m.finalRound = input.BlockHeader.Round + m.Called(input) + return err +} + +type mockExporter struct { + mock.Mock + exporters.Exporter + finalRound basics.Round + returnError bool + onCompleteError bool +} + +func (m *mockExporter) Metadata() conduit.Metadata { + return conduit.Metadata{ + Name: "mockExporter", + } +} + +func (m *mockExporter) Init(_ context.Context, _ data.InitProvider, _ plugins.PluginConfig, _ *log.Logger) error { + return nil +} + +func (m *mockExporter) Close() error { + return nil +} + +func (m *mockExporter) Receive(exportData data.BlockData) error { + var err error + if m.returnError { + err = fmt.Errorf("receive") + } + m.Called(exportData) + return err +} + +func (m *mockExporter) OnComplete(input data.BlockData) error { + var err error + if m.onCompleteError { + err = fmt.Errorf("on complete") + } + m.finalRound = input.BlockHeader.Round + m.Called(input) + return err +} + +type mockedImporterNew struct{} + +func (c mockedImporterNew) New() importers.Importer { return &mockImporter{} } + +type mockedExporterNew struct{} + +func (c mockedExporterNew) New() exporters.Exporter { return &mockExporter{} } + +// TestPipelineRun tests that running the pipeline calls the correct functions with mocking +func TestPipelineRun(t *testing.T) { + + mImporter := mockImporter{} + mImporter.On("GetBlock", mock.Anything).Return(uniqueBlockData, nil) + mProcessor := mockProcessor{} + processorData := uniqueBlockData + processorData.BlockHeader.Round++ + mProcessor.On("Process", mock.Anything).Return(processorData) + mProcessor.On("OnComplete", mock.Anything).Return(nil) + mExporter := mockExporter{} + mExporter.On("Receive", mock.Anything).Return(nil) + + var pImporter importers.Importer = &mImporter + var pProcessor processors.Processor = &mProcessor + var pExporter exporters.Exporter = &mExporter + var cbComplete conduit.Completed = &mProcessor + + ctx, cf := context.WithCancel(context.Background()) + + pImpl := pipelineImpl{ + ctx: ctx, + cf: cf, + logger: log.New(), + initProvider: nil, + importer: &pImporter, + processors: []*processors.Processor{&pProcessor}, + exporter: &pExporter, + completeCallback: []conduit.OnCompleteFunc{cbComplete.OnComplete}, + pipelineMetadata: state{ + NextRound: 0, + GenesisHash: "", + }, + cfg: &Config{ + ConduitConfig: &conduit.Config{ + ConduitDataDir: t.TempDir(), + }, + }, + } + + go func() { + time.Sleep(1 * time.Second) + cf() + }() + + pImpl.Start() + pImpl.Wait() + assert.NoError(t, pImpl.Error()) + + assert.Equal(t, mProcessor.finalRound, uniqueBlockData.BlockHeader.Round+1) + + mock.AssertExpectationsForObjects(t, &mImporter, &mProcessor, &mExporter) + +} + +// TestPipelineCpuPidFiles tests that cpu and pid files are created when specified +func TestPipelineCpuPidFiles(t *testing.T) { + + var pImporter importers.Importer = &mockImporter{} + var pProcessor processors.Processor = &mockProcessor{} + var pExporter exporters.Exporter = &mockExporter{} + + tempDir := t.TempDir() + pidFilePath := filepath.Join(tempDir, "pidfile") + cpuFilepath := filepath.Join(tempDir, "cpufile") + + pImpl := pipelineImpl{ + cfg: &Config{ + ConduitConfig: &conduit.Config{ + Flags: nil, + ConduitDataDir: tempDir, + }, + Importer: NameConfigPair{ + Name: "", + Config: map[string]interface{}{}, + }, + Processors: []NameConfigPair{ + { + Name: "", + Config: map[string]interface{}{}, + }, + }, + Exporter: NameConfigPair{ + Name: "", + Config: map[string]interface{}{}, + }, + }, + logger: log.New(), + initProvider: nil, + importer: &pImporter, + processors: []*processors.Processor{&pProcessor}, + exporter: &pExporter, + pipelineMetadata: state{ + GenesisHash: "", + Network: "", + NextRound: 0, + }, + } + + err := pImpl.Init() + assert.NoError(t, err) + + // Test that file is not created + _, err = os.Stat(pidFilePath) + assert.ErrorIs(t, err, os.ErrNotExist) + + _, err = os.Stat(cpuFilepath) + assert.ErrorIs(t, err, os.ErrNotExist) + + // Test that they were created + + pImpl.cfg.PIDFilePath = pidFilePath + pImpl.cfg.CPUProfile = cpuFilepath + + err = pImpl.Init() + assert.NoError(t, err) + + // Test that file is created + _, err = os.Stat(cpuFilepath) + assert.Nil(t, err) + + _, err = os.Stat(pidFilePath) + assert.Nil(t, err) +} + +// TestPipelineErrors tests the pipeline erroring out at different stages +func TestPipelineErrors(t *testing.T) { + tempDir := t.TempDir() + + mImporter := mockImporter{} + mImporter.On("GetBlock", mock.Anything).Return(uniqueBlockData, nil) + mProcessor := mockProcessor{} + processorData := uniqueBlockData + processorData.BlockHeader.Round++ + mProcessor.On("Process", mock.Anything).Return(processorData) + mProcessor.On("OnComplete", mock.Anything).Return(nil) + mExporter := mockExporter{} + mExporter.On("Receive", mock.Anything).Return(nil) + + var pImporter importers.Importer = &mImporter + var pProcessor processors.Processor = &mProcessor + var pExporter exporters.Exporter = &mExporter + var cbComplete conduit.Completed = &mProcessor + + ctx, cf := context.WithCancel(context.Background()) + pImpl := pipelineImpl{ + ctx: ctx, + cf: cf, + cfg: &Config{ + ConduitConfig: &conduit.Config{ + ConduitDataDir: tempDir, + }, + }, + logger: log.New(), + initProvider: nil, + importer: &pImporter, + processors: []*processors.Processor{&pProcessor}, + exporter: &pExporter, + completeCallback: []conduit.OnCompleteFunc{cbComplete.OnComplete}, + pipelineMetadata: state{}, + } + + mImporter.returnError = true + + go pImpl.Start() + time.Sleep(time.Millisecond) + pImpl.cf() + pImpl.Wait() + assert.Error(t, pImpl.Error(), fmt.Errorf("importer")) + + mImporter.returnError = false + mProcessor.returnError = true + pImpl.ctx, pImpl.cf = context.WithCancel(context.Background()) + pImpl.setError(nil) + go pImpl.Start() + time.Sleep(time.Millisecond) + pImpl.cf() + pImpl.Wait() + assert.Error(t, pImpl.Error(), fmt.Errorf("process")) + + mProcessor.returnError = false + mProcessor.onCompleteError = true + pImpl.ctx, pImpl.cf = context.WithCancel(context.Background()) + pImpl.setError(nil) + go pImpl.Start() + time.Sleep(time.Millisecond) + pImpl.cf() + pImpl.Wait() + assert.Error(t, pImpl.Error(), fmt.Errorf("on complete")) + + mProcessor.onCompleteError = false + mExporter.returnError = true + pImpl.ctx, pImpl.cf = context.WithCancel(context.Background()) + pImpl.setError(nil) + go pImpl.Start() + time.Sleep(time.Millisecond) + pImpl.cf() + pImpl.Wait() + assert.Error(t, pImpl.Error(), fmt.Errorf("exporter")) +} + +func Test_pipelineImpl_registerLifecycleCallbacks(t *testing.T) { + mImporter := mockImporter{} + mImporter.On("GetBlock", mock.Anything).Return(uniqueBlockData, nil) + mProcessor := mockProcessor{} + processorData := uniqueBlockData + processorData.BlockHeader.Round++ + mProcessor.On("Process", mock.Anything).Return(processorData) + mProcessor.On("OnComplete", mock.Anything).Return(nil) + mExporter := mockExporter{} + mExporter.On("Receive", mock.Anything).Return(nil) + + var pImporter importers.Importer = &mImporter + var pProcessor processors.Processor = &mProcessor + var pExporter exporters.Exporter = &mExporter + + ctx, cf := context.WithCancel(context.Background()) + pImpl := pipelineImpl{ + ctx: ctx, + cf: cf, + cfg: &Config{}, + logger: log.New(), + initProvider: nil, + importer: &pImporter, + processors: []*processors.Processor{&pProcessor, &pProcessor}, + exporter: &pExporter, + } + + // Each plugin implements the Completed interface, so there should be 4 + // plugins registered (one of them is registered twice) + pImpl.registerLifecycleCallbacks() + assert.Len(t, pImpl.completeCallback, 4) +} + +// TestBlockMetaDataFile tests that metadata.json file is created as expected +func TestBlockMetaDataFile(t *testing.T) { + + var pImporter importers.Importer = &mockImporter{} + var pProcessor processors.Processor = &mockProcessor{} + var pExporter exporters.Exporter = &mockExporter{} + + datadir := t.TempDir() + pImpl := pipelineImpl{ + cfg: &Config{ + ConduitConfig: &conduit.Config{ + Flags: nil, + ConduitDataDir: datadir, + }, + Importer: NameConfigPair{ + Name: "", + Config: map[string]interface{}{}, + }, + Processors: []NameConfigPair{ + { + Name: "", + Config: map[string]interface{}{}, + }, + }, + Exporter: NameConfigPair{ + Name: "", + Config: map[string]interface{}{}, + }, + }, + logger: log.New(), + initProvider: nil, + importer: &pImporter, + processors: []*processors.Processor{&pProcessor}, + exporter: &pExporter, + pipelineMetadata: state{ + NextRound: 3, + }, + } + + err := pImpl.Init() + assert.NoError(t, err) + + // Test that file is created + blockMetaDataFile := filepath.Join(datadir, "metadata.json") + _, err = os.Stat(blockMetaDataFile) + assert.NoError(t, err) + + // Test that file loads correctly + metaData, err := pImpl.initializeOrLoadBlockMetadata() + assert.NoError(t, err) + assert.Equal(t, pImpl.pipelineMetadata.GenesisHash, metaData.GenesisHash) + assert.Equal(t, pImpl.pipelineMetadata.NextRound, metaData.NextRound) + assert.Equal(t, pImpl.pipelineMetadata.Network, metaData.Network) + + // Test that file encodes correctly + pImpl.pipelineMetadata.GenesisHash = "HASH" + pImpl.pipelineMetadata.NextRound = 7 + err = pImpl.encodeMetadataToFile() + assert.NoError(t, err) + metaData, err = pImpl.initializeOrLoadBlockMetadata() + assert.NoError(t, err) + assert.Equal(t, "HASH", metaData.GenesisHash) + assert.Equal(t, uint64(7), metaData.NextRound) + assert.Equal(t, pImpl.pipelineMetadata.Network, metaData.Network) + + // invalid file directory + pImpl.cfg.ConduitConfig.ConduitDataDir = "datadir" + metaData, err = pImpl.initializeOrLoadBlockMetadata() + assert.Contains(t, err.Error(), "Init(): error creating file") + err = pImpl.encodeMetadataToFile() + assert.Contains(t, err.Error(), "encodeMetadataToFile(): failed to create temp metadata file") +} + +func TestGenesisHash(t *testing.T) { + var pImporter importers.Importer = &mockImporter{genesis: bookkeeping.Genesis{Network: "test"}} + var pProcessor processors.Processor = &mockProcessor{} + var pExporter exporters.Exporter = &mockExporter{} + datadir := t.TempDir() + pImpl := pipelineImpl{ + cfg: &Config{ + ConduitConfig: &conduit.Config{ + Flags: nil, + ConduitDataDir: datadir, + }, + Importer: NameConfigPair{ + Name: "", + Config: map[string]interface{}{}, + }, + Processors: []NameConfigPair{ + { + Name: "", + Config: map[string]interface{}{}, + }, + }, + Exporter: NameConfigPair{ + Name: "", + Config: map[string]interface{}{}, + }, + }, + logger: log.New(), + initProvider: nil, + importer: &pImporter, + processors: []*processors.Processor{&pProcessor}, + exporter: &pExporter, + pipelineMetadata: state{ + GenesisHash: "", + Network: "", + NextRound: 3, + }, + } + + // write genesis hash to metadata.json + err := pImpl.Init() + assert.NoError(t, err) + + // read genesis hash from metadata.json + blockmetaData, err := pImpl.initializeOrLoadBlockMetadata() + assert.NoError(t, err) + assert.Equal(t, blockmetaData.GenesisHash, crypto.HashObj(&bookkeeping.Genesis{Network: "test"}).String()) + assert.Equal(t, blockmetaData.Network, "test") + + // mock a different genesis hash + pImporter = &mockImporter{genesis: bookkeeping.Genesis{Network: "dev"}} + pImpl.importer = &pImporter + err = pImpl.Init() + assert.Contains(t, err.Error(), "genesis hash in metadata does not match") +} + +func TestInitError(t *testing.T) { + var pImporter importers.Importer = &mockImporter{genesis: bookkeeping.Genesis{Network: "test"}} + var pProcessor processors.Processor = &mockProcessor{} + var pExporter exporters.Exporter = &mockExporter{} + datadir := "data" + pImpl := pipelineImpl{ + cfg: &Config{ + ConduitConfig: &conduit.Config{ + Flags: nil, + ConduitDataDir: datadir, + }, + Importer: NameConfigPair{ + Name: "", + Config: map[string]interface{}{}, + }, + Processors: []NameConfigPair{ + { + Name: "", + Config: map[string]interface{}{}, + }, + }, + Exporter: NameConfigPair{ + Name: "unknown", + Config: map[string]interface{}{}, + }, + }, + logger: log.New(), + initProvider: nil, + importer: &pImporter, + processors: []*processors.Processor{&pProcessor}, + exporter: &pExporter, + pipelineMetadata: state{ + GenesisHash: "", + Network: "", + NextRound: 3, + }, + } + + // could not read metadata + err := pImpl.Init() + assert.Contains(t, err.Error(), "could not read metadata") +} + +func TestPipelineMetricsConfigs(t *testing.T) { + var pImporter importers.Importer = &mockImporter{} + var pProcessor processors.Processor = &mockProcessor{} + var pExporter exporters.Exporter = &mockExporter{} + ctx, cf := context.WithCancel(context.Background()) + pImpl := pipelineImpl{ + cfg: &Config{ + ConduitConfig: &conduit.Config{ + Flags: nil, + ConduitDataDir: t.TempDir(), + }, + Importer: NameConfigPair{ + Name: "", + Config: map[string]interface{}{}, + }, + Processors: []NameConfigPair{ + { + Name: "", + Config: map[string]interface{}{}, + }, + }, + Exporter: NameConfigPair{ + Name: "", + Config: map[string]interface{}{}, + }, + Metrics: Metrics{}, + }, + logger: log.New(), + initProvider: nil, + importer: &pImporter, + processors: []*processors.Processor{&pProcessor}, + exporter: &pExporter, + cf: cf, + ctx: ctx, + pipelineMetadata: state{ + GenesisHash: "", + Network: "", + NextRound: 0, + }, + } + defer pImpl.cf() + + getMetrics := func() (*http.Response, error) { + resp0, err0 := http.Get(fmt.Sprintf("http://localhost%s/metrics", pImpl.cfg.Metrics.Addr)) + return resp0, err0 + } + // metrics should be OFF by default + err := pImpl.Init() + assert.NoError(t, err) + time.Sleep(1 * time.Second) + _, err = getMetrics() + assert.Error(t, err) + + // metrics mode OFF + pImpl.cfg.Metrics = Metrics{ + Mode: "OFF", + Addr: ":8081", + } + pImpl.Init() + time.Sleep(1 * time.Second) + _, err = getMetrics() + assert.Error(t, err) + + // metrics mode ON + pImpl.cfg.Metrics = Metrics{ + Mode: "ON", + Addr: ":8081", + } + pImpl.Init() + time.Sleep(1 * time.Second) + resp, err := getMetrics() + assert.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) +} + +// TestPipelineLogFile tests that log file is created when specified +func TestPipelineLogFile(t *testing.T) { + + var mockedImpNew mockedImporterNew + var mockedExpNew mockedExporterNew + importers.Register("mockedImporter", mockedImpNew) + exporters.Register("mockedExporter", mockedExpNew) + + tempDir := t.TempDir() + logfilePath := path.Join(tempDir, "conduit.log") + configs := &Config{ + ConduitConfig: &conduit.Config{ + Flags: nil, + ConduitDataDir: tempDir, + }, + Importer: NameConfigPair{ + Name: "mockedImporter", + Config: map[string]interface{}{"key": "value"}, + }, + Exporter: NameConfigPair{ + Name: "mockedExporter", + Config: map[string]interface{}{"key": "value"}, + }, + PipelineLogLevel: "INFO", + } + + _, err := MakePipeline(context.Background(), configs, log.New()) + require.NoError(t, err) + + // Test that file is not created + _, err = os.Stat(logfilePath) + assert.ErrorIs(t, err, os.ErrNotExist) + + // Test that it is created + configs.LogFile = logfilePath + _, err = MakePipeline(context.Background(), configs, log.New()) + require.NoError(t, err) + + _, err = os.Stat(logfilePath) + assert.Nil(t, err) +} + +// TestPipelineLogFile tests that log file is created when specified +func TestRoundOverwrite(t *testing.T) { + var pImporter importers.Importer = &mockImporter{genesis: bookkeeping.Genesis{Network: "test"}} + var pProcessor processors.Processor = &mockProcessor{} + var pExporter exporters.Exporter = &mockExporter{} + pImpl := pipelineImpl{ + cfg: &Config{ + ConduitConfig: &conduit.Config{ + Flags: nil, + ConduitDataDir: t.TempDir(), + NextRoundOverride: 0, + }, + Importer: NameConfigPair{ + Name: "", + Config: map[string]interface{}{}, + }, + Processors: []NameConfigPair{ + { + Name: "", + Config: map[string]interface{}{}, + }, + }, + Exporter: NameConfigPair{ + Name: "unknown", + Config: map[string]interface{}{}, + }, + }, + logger: log.New(), + initProvider: nil, + importer: &pImporter, + processors: []*processors.Processor{&pProcessor}, + exporter: &pExporter, + pipelineMetadata: state{ + GenesisHash: "", + Network: "", + NextRound: 3, + }, + } + + // pipeline should initialize if NextRoundOverride is not set + err := pImpl.Init() + assert.Nil(t, err) + + // override NextRound + for i := 1; i < 10; i++ { + pImpl.cfg.ConduitConfig.NextRoundOverride = uint64(i) + err = pImpl.Init() + assert.Nil(t, err) + assert.Equal(t, uint64(i), pImpl.pipelineMetadata.NextRound) + } +} diff --git a/conduit/plugins/config.go b/conduit/plugins/config.go new file mode 100644 index 000000000..4f428b1cd --- /dev/null +++ b/conduit/plugins/config.go @@ -0,0 +1,4 @@ +package plugins + +// PluginConfig is a generic string which can be deserialized by each individual Plugin +type PluginConfig string diff --git a/conduit/plugins/exporters/README.md b/conduit/plugins/exporters/README.md new file mode 100644 index 000000000..8667d83f2 --- /dev/null +++ b/conduit/plugins/exporters/README.md @@ -0,0 +1,49 @@ +# Exporter Plugins +## Developing a new Exporter ## +Each Exporter lives in its own module inside the `exporters` directory. We have provided an `example` exporter in `exporters/example` which should be used as a starting point. +Run `cp -r exporters/example exporters/my_exporter` to begin defining a new exporter definition. Interface methods in the example exporter have not been defined, but the Exporter interface attempts to give some background their purpose. +``` +type Exporter interface { + // Metadata associated with each Exporter. + Metadata() ExporterMetadata + + // Init will be called during initialization, before block data starts going through the pipeline. + // Typically used for things like initializing network connections. + // The ExporterConfig passed to Connect will contain the Unmarhsalled config file specific to this plugin. + // Should return an error if it fails--this will result in the Indexer process terminating. + Init(cfg plugins.PluginConfig, logger *logrus.Logger) error + + // Config returns the configuration options used to create an Exporter. + // Initialized during Connect, it should return nil until the Exporter has been Connected. + Config() plugins.PluginConfig + + // Close will be called during termination of the Indexer process. + // There is no guarantee that plugin lifecycle hooks will be invoked in any specific order in relation to one another. + // Returns an error if it fails which will be surfaced in the logs, but the process is already terminating. + Close() error + + // Receive is called for each block to be processed by the exporter. + // Should return an error on failure--retries are configurable. + Receive(exportData data.BlockData) error + +} +``` + +## Exporter Configuration +In order to avoid requiring the Conduit framework from knowing how to configure every single plugin, it is necessary to be able to construct every plugin without configuration data. +The configuration for a plugin will be passed into the constructed exporter object via the `Init` function as a `plugin.PluginConfig` (string) object where it can be deserialized inside the plugin. + +Existing plugins and their configurations use yaml for their serialization format, so all new exporters should also define their configs via a yaml object. For an example of this, take a look at the [postgresql exporter config](postgresql/postgresql_exporter_config.go). + +## Testing +The provided example has a few basic test definitions which can be used as a starting point for unit testing. All exporter modules should include unit tests out of the box. +Support for integration testing is TBD. + +## Common Code +Any code necessary for the execution of an Exporter should be defined within the exporter's directory. For example, a MongoDB exporter located in `exporters/mongodb/` might have connection utils located in `exporters/mongodb/connections/`. +If code can be useful across multiple exporters it may be placed in its own module external to the exporters. + +## Work In Progress +* There is currently no defined process for constructing a Conduit pipeline, so you cannot actually use your new Exporter. We are working on the Conduit framework which will allow users to easily construct block data pipelines. +* Config files are read in based on a predefined path which is different for each plugin. That means users' Exporter configs will need to be defined in their own file. +* We are working to determine the best way to support integration test definitions for Exporter plugins. \ No newline at end of file diff --git a/conduit/plugins/exporters/all/all.go b/conduit/plugins/exporters/all/all.go new file mode 100644 index 000000000..52cf359e4 --- /dev/null +++ b/conduit/plugins/exporters/all/all.go @@ -0,0 +1,8 @@ +package all + +import ( + // Call package wide init function + _ "github.com/algorand/indexer/conduit/plugins/exporters/filewriter" + _ "github.com/algorand/indexer/conduit/plugins/exporters/noop" + _ "github.com/algorand/indexer/conduit/plugins/exporters/postgresql" +) diff --git a/conduit/plugins/exporters/example/example_exporter.go b/conduit/plugins/exporters/example/example_exporter.go new file mode 100644 index 000000000..e0e39df09 --- /dev/null +++ b/conduit/plugins/exporters/example/example_exporter.go @@ -0,0 +1,66 @@ +package example + +import ( + "context" + _ "embed" // used to embed config + + "github.com/sirupsen/logrus" + + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/exporters" + "github.com/algorand/indexer/data" +) + +// This is our exporter object. It should store all the in memory data required to run the Exporter. +type exampleExporter struct{} + +//go:embed sample.yaml +var sampleConfig string + +// Each Exporter should implement its own Metadata object. These fields shouldn't change at runtime so there is +// no reason to construct more than a single metadata object. +var metadata = conduit.Metadata{ + Name: "example", + Description: "example exporter", + Deprecated: false, + SampleConfig: sampleConfig, +} + +// Metadata returns the Exporter's Metadata object +func (exp *exampleExporter) Metadata() conduit.Metadata { + return metadata +} + +// Init provides the opportunity for your Exporter to initialize connections, store config variables, etc. +func (exp *exampleExporter) Init(_ context.Context, _ data.InitProvider, _ plugins.PluginConfig, _ *logrus.Logger) error { + panic("not implemented") +} + +// Config returns the unmarshaled config object +func (exp *exampleExporter) Config() plugins.PluginConfig { + panic("not implemented") +} + +// Close provides the opportunity to close connections, flush buffers, etc. when the process is terminating +func (exp *exampleExporter) Close() error { + panic("not implemented") +} + +// Receive is the main handler function for blocks +func (exp *exampleExporter) Receive(exportData data.BlockData) error { + panic("not implemented") +} + +// Round should return the round number of the next expected round that should be provided to the Exporter +func (exp *exampleExporter) Round() uint64 { + panic("not implemented") +} + +func init() { + // In order to provide a Constructor to the exporter_factory, we register our Exporter in the init block. + // To load this Exporter into the factory, simply import the package. + exporters.Register(metadata.Name, exporters.ExporterConstructorFunc(func() exporters.Exporter { + return &exampleExporter{} + })) +} diff --git a/conduit/plugins/exporters/example/example_exporter_test.go b/conduit/plugins/exporters/example/example_exporter_test.go new file mode 100644 index 000000000..5f4efb1c5 --- /dev/null +++ b/conduit/plugins/exporters/example/example_exporter_test.go @@ -0,0 +1,40 @@ +package example + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/algorand/indexer/conduit/plugins/exporters" + "github.com/algorand/indexer/data" +) + +var exCons = exporters.ExporterConstructorFunc(func() exporters.Exporter { + return &exampleExporter{} +}) + +var exExp = exCons.New() + +func TestExporterMetadata(t *testing.T) { + meta := exExp.Metadata() + assert.Equal(t, metadata.Name, meta.Name) + assert.Equal(t, metadata.Description, meta.Description) + assert.Equal(t, metadata.Deprecated, meta.Deprecated) +} + +func TestExporterInit(t *testing.T) { + assert.Panics(t, func() { exExp.Init(context.Background(), nil, "", nil) }) +} + +func TestExporterConfig(t *testing.T) { + assert.Panics(t, func() { exExp.Config() }) +} + +func TestExporterClose(t *testing.T) { + assert.Panics(t, func() { exExp.Close() }) +} + +func TestExporterReceive(t *testing.T) { + assert.Panics(t, func() { exExp.Receive(data.BlockData{}) }) +} diff --git a/conduit/plugins/exporters/example/sample.yaml b/conduit/plugins/exporters/example/sample.yaml new file mode 100644 index 000000000..e181e4d29 --- /dev/null +++ b/conduit/plugins/exporters/example/sample.yaml @@ -0,0 +1,3 @@ +name: example +# example has no config +config: \ No newline at end of file diff --git a/conduit/plugins/exporters/exporter.go b/conduit/plugins/exporters/exporter.go new file mode 100644 index 000000000..976edbd5c --- /dev/null +++ b/conduit/plugins/exporters/exporter.go @@ -0,0 +1,36 @@ +package exporters + +import ( + "context" + + "github.com/sirupsen/logrus" + + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/data" +) + +// Exporter defines the interface for plugins +type Exporter interface { + // PluginMetadata implement this interface. + conduit.PluginMetadata + + // Init will be called during initialization, before block data starts going through the pipeline. + // Typically used for things like initializing network connections. + // The ExporterConfig passed to Connect will contain the Unmarhsalled config file specific to this plugin. + // Should return an error if it fails--this will result in the Indexer process terminating. + Init(ctx context.Context, initProvider data.InitProvider, cfg plugins.PluginConfig, logger *logrus.Logger) error + + // Config returns the configuration options used to create an Exporter. + // Initialized during Connect, it should return nil until the Exporter has been Connected. + Config() plugins.PluginConfig + + // Close will be called during termination of the Indexer process. + // There is no guarantee that plugin lifecycle hooks will be invoked in any specific order in relation to one another. + // Returns an error if it fails which will be surfaced in the logs, but the process is already terminating. + Close() error + + // Receive is called for each block to be processed by the exporter. + // Should return an error on failure--retries are configurable. + Receive(exportData data.BlockData) error +} diff --git a/conduit/plugins/exporters/exporter_factory.go b/conduit/plugins/exporters/exporter_factory.go new file mode 100644 index 000000000..d3682af1d --- /dev/null +++ b/conduit/plugins/exporters/exporter_factory.go @@ -0,0 +1,41 @@ +package exporters + +import ( + "fmt" +) + +// ExporterConstructor must be implemented by each Exporter. +// It provides a basic no-arg constructor for instances of an ExporterImpl. +type ExporterConstructor interface { + // New should return an instantiation of an Exporter. + // Configuration values should be passed and can be processed during `Init()`. + New() Exporter +} + +// ExporterConstructorFunc is Constructor implementation for exporters +type ExporterConstructorFunc func() Exporter + +// New initializes an exporter constructor +func (f ExporterConstructorFunc) New() Exporter { + return f() +} + +// Exporters are the constructors to build exporter plugins. +var Exporters = make(map[string]ExporterConstructor) + +// Register is used to register ExporterConstructor implementations. This mechanism allows +// for loose coupling between the configuration and the implementation. It is extremely similar to the way sql.DB +// drivers are configured and used. +func Register(name string, constructor ExporterConstructor) { + Exporters[name] = constructor +} + +// ExporterBuilderByName returns a Processor constructor for the name provided +func ExporterBuilderByName(name string) (ExporterConstructor, error) { + constructor, ok := Exporters[name] + if !ok { + return nil, fmt.Errorf("no Exporter Constructor for %s", name) + } + + return constructor, nil +} diff --git a/conduit/plugins/exporters/exporter_factory_test.go b/conduit/plugins/exporters/exporter_factory_test.go new file mode 100644 index 000000000..fbb0806ff --- /dev/null +++ b/conduit/plugins/exporters/exporter_factory_test.go @@ -0,0 +1,49 @@ +package exporters + +import ( + "testing" + + "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/assert" + + "github.com/algorand/indexer/conduit" +) + +var logger *logrus.Logger + +func init() { + logger, _ = test.NewNullLogger() +} + +type mockExporter struct { + Exporter +} + +func (m *mockExporter) Metadata() conduit.Metadata { + return conduit.Metadata{} +} + +type mockExporterConstructor struct { + me *mockExporter +} + +func (c *mockExporterConstructor) New() Exporter { + return c.me +} + +func TestExporterByNameSuccess(t *testing.T) { + me := mockExporter{} + Register("foobar", &mockExporterConstructor{&me}) + + expC, err := ExporterBuilderByName("foobar") + assert.NoError(t, err) + exp := expC.New() + assert.Implements(t, (*Exporter)(nil), exp) +} + +func TestExporterByNameNotFound(t *testing.T) { + _, err := ExporterBuilderByName("barfoo") + expectedErr := "no Exporter Constructor for barfoo" + assert.EqualError(t, err, expectedErr) +} diff --git a/conduit/plugins/exporters/filewriter/file_exporter.go b/conduit/plugins/exporters/filewriter/file_exporter.go new file mode 100644 index 000000000..4c19cdece --- /dev/null +++ b/conduit/plugins/exporters/filewriter/file_exporter.go @@ -0,0 +1,115 @@ +package filewriter + +import ( + "context" + _ "embed" // used to embed config + "errors" + "fmt" + "os" + "path" + + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/exporters" + "github.com/algorand/indexer/data" + "github.com/algorand/indexer/util" +) + +const ( + exporterName = "file_writer" + // FilePattern is used to name the output files. + FilePattern = "%[1]d_block.json" +) + +type fileExporter struct { + round uint64 + cfg Config + logger *logrus.Logger +} + +//go:embed sample.yaml +var sampleFile string + +var metadata = conduit.Metadata{ + Name: exporterName, + Description: "Exporter for writing data to a file.", + Deprecated: false, + SampleConfig: sampleFile, +} + +func (exp *fileExporter) Metadata() conduit.Metadata { + return metadata +} + +func (exp *fileExporter) Init(_ context.Context, initProvider data.InitProvider, cfg plugins.PluginConfig, logger *logrus.Logger) error { + exp.logger = logger + var err error + exp.cfg, err = unmarshalConfig(string(cfg)) + if err != nil { + return fmt.Errorf("connect failure in unmarshalConfig: %w", err) + } + if exp.cfg.FilenamePattern == "" { + exp.cfg.FilenamePattern = FilePattern + } + // create block directory + err = os.Mkdir(exp.cfg.BlocksDir, 0755) + if err != nil && errors.Is(err, os.ErrExist) { + // Ignore mkdir if the dir exists + err = nil + } else if err != nil { + return fmt.Errorf("Init() error: %w", err) + } + exp.round = uint64(initProvider.NextDBRound()) + return err +} + +func (exp *fileExporter) Config() plugins.PluginConfig { + ret, _ := yaml.Marshal(exp.cfg) + return plugins.PluginConfig(ret) +} + +func (exp *fileExporter) Close() error { + exp.logger.Infof("latest round on file: %d", exp.round) + return nil +} + +func (exp *fileExporter) Receive(exportData data.BlockData) error { + if exp.logger == nil { + return fmt.Errorf("exporter not initialized") + } + if exportData.Round() != exp.round { + return fmt.Errorf("Receive(): wrong block: received round %d, expected round %d", exportData.Round(), exp.round) + } + + // write block to file + { + if exp.cfg.DropCertificate { + exportData.Certificate = nil + } + + blockFile := path.Join(exp.cfg.BlocksDir, fmt.Sprintf(exp.cfg.FilenamePattern, exportData.Round())) + err := util.EncodeToFile(blockFile, exportData, true) + if err != nil { + return fmt.Errorf("Receive(): failed to write file %s: %w", blockFile, err) + } + exp.logger.Infof("Wrote block %d to %s", exportData.Round(), blockFile) + } + + exp.round++ + return nil +} + +func unmarshalConfig(cfg string) (Config, error) { + var config Config + err := yaml.Unmarshal([]byte(cfg), &config) + return config, err +} + +func init() { + exporters.Register(exporterName, exporters.ExporterConstructorFunc(func() exporters.Exporter { + return &fileExporter{} + })) +} diff --git a/conduit/plugins/exporters/filewriter/file_exporter_config.go b/conduit/plugins/exporters/filewriter/file_exporter_config.go new file mode 100644 index 000000000..a375ac404 --- /dev/null +++ b/conduit/plugins/exporters/filewriter/file_exporter_config.go @@ -0,0 +1,21 @@ +package filewriter + +// Config specific to the file exporter +type Config struct { + // BlocksDir is the path to a directory where block data should be stored. + // The directory is created if it doesn't exist. + BlocksDir string `yaml:"block-dir"` + // FilenamePattern is the format used to write block files. It uses go + // string formatting and should accept one number for the round. + // If the file has a '.gz' extension, blocks will be gzipped. + // Default: "%[1]d_block.json" + FilenamePattern string `yaml:"filename-pattern"` + // DropCertificate is used to remove the vote certificate from the block data before writing files. + DropCertificate bool `yaml:"drop-certificate"` + + // TODO: compression level - Default, Fastest, Best compression, etc + + // TODO: How to avoid having millions of files in a directory? + // Write batches of blocks to a single file? + // Tree of directories +} diff --git a/conduit/plugins/exporters/filewriter/file_exporter_test.go b/conduit/plugins/exporters/filewriter/file_exporter_test.go new file mode 100644 index 000000000..d681729f5 --- /dev/null +++ b/conduit/plugins/exporters/filewriter/file_exporter_test.go @@ -0,0 +1,190 @@ +package filewriter + +import ( + "context" + "fmt" + "testing" + + "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/algorand/go-algorand/agreement" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/ledger/ledgercore" + + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/exporters" + "github.com/algorand/indexer/data" + "github.com/algorand/indexer/util" + testutil "github.com/algorand/indexer/util/test" +) + +var logger *logrus.Logger +var fileCons = exporters.ExporterConstructorFunc(func() exporters.Exporter { + return &fileExporter{} +}) +var configTemplate = "block-dir: %s/blocks\n" +var round = basics.Round(2) + +func init() { + logger, _ = test.NewNullLogger() +} + +func getConfig(t *testing.T) (config, tempdir string) { + tempdir = t.TempDir() + config = fmt.Sprintf(configTemplate, tempdir) + return +} + +func TestExporterMetadata(t *testing.T) { + fileExp := fileCons.New() + meta := fileExp.Metadata() + assert.Equal(t, metadata.Name, meta.Name) + assert.Equal(t, metadata.Description, meta.Description) + assert.Equal(t, metadata.Deprecated, meta.Deprecated) +} + +func TestExporterInit(t *testing.T) { + config, _ := getConfig(t) + fileExp := fileCons.New() + defer fileExp.Close() + + // creates a new output file + err := fileExp.Init(context.Background(), testutil.MockedInitProvider(&round), plugins.PluginConfig(config), logger) + pluginConfig := fileExp.Config() + configWithDefault := config + "filename-pattern: '%[1]d_block.json'\n" + "drop-certificate: false\n" + assert.Equal(t, configWithDefault, string(pluginConfig)) + fileExp.Close() + + // can open existing file + err = fileExp.Init(context.Background(), testutil.MockedInitProvider(&round), plugins.PluginConfig(config), logger) + assert.NoError(t, err) + fileExp.Close() +} + +func sendData(t *testing.T, fileExp exporters.Exporter, config string, numRounds int) { + // Test invalid block receive + block := data.BlockData{ + BlockHeader: bookkeeping.BlockHeader{ + Round: 3, + }, + Payset: nil, + Delta: nil, + Certificate: nil, + } + + err := fileExp.Receive(block) + require.Contains(t, err.Error(), "exporter not initialized") + + // initialize + rnd := basics.Round(0) + err = fileExp.Init(context.Background(), testutil.MockedInitProvider(&rnd), plugins.PluginConfig(config), logger) + require.NoError(t, err) + + // incorrect round + err = fileExp.Receive(block) + require.Contains(t, err.Error(), "received round 3, expected round 0") + + // write block to file + for i := 0; i < numRounds; i++ { + block = data.BlockData{ + BlockHeader: bookkeeping.BlockHeader{ + Round: basics.Round(i), + }, + Payset: nil, + Delta: &ledgercore.StateDelta{ + PrevTimestamp: 1234, + }, + Certificate: &agreement.Certificate{ + Round: basics.Round(i), + Period: 2, + Step: 2, + }, + } + err = fileExp.Receive(block) + require.NoError(t, err) + } + + require.NoError(t, fileExp.Close()) +} + +func TestExporterReceive(t *testing.T) { + config, tempdir := getConfig(t) + fileExp := fileCons.New() + numRounds := 5 + sendData(t, fileExp, config, numRounds) + + // block data is valid + for i := 0; i < 5; i++ { + filename := fmt.Sprintf(FilePattern, i) + path := fmt.Sprintf("%s/blocks/%s", tempdir, filename) + assert.FileExists(t, path) + + var blockData data.BlockData + err := util.DecodeFromFile(path, &blockData) + require.Equal(t, basics.Round(i), blockData.BlockHeader.Round) + require.NoError(t, err) + require.NotNil(t, blockData.Certificate) + } +} + +func TestExporterClose(t *testing.T) { + config, _ := getConfig(t) + fileExp := fileCons.New() + rnd := basics.Round(0) + fileExp.Init(context.Background(), testutil.MockedInitProvider(&rnd), plugins.PluginConfig(config), logger) + require.NoError(t, fileExp.Close()) +} + +func TestPatternOverride(t *testing.T) { + config, tempdir := getConfig(t) + fileExp := fileCons.New() + + patternOverride := "PREFIX_%[1]d_block.json" + config = fmt.Sprintf("%sfilename-pattern: '%s'\n", config, patternOverride) + + numRounds := 5 + sendData(t, fileExp, config, numRounds) + + // block data is valid + for i := 0; i < 5; i++ { + filename := fmt.Sprintf(patternOverride, i) + path := fmt.Sprintf("%s/blocks/%s", tempdir, filename) + assert.FileExists(t, path) + + var blockData data.BlockData + err := util.DecodeFromFile(path, &blockData) + require.Equal(t, basics.Round(i), blockData.BlockHeader.Round) + require.NoError(t, err) + require.NotNil(t, blockData.Certificate) + } +} + +func TestDropCertificate(t *testing.T) { + tempdir := t.TempDir() + cfg := Config{ + BlocksDir: tempdir, + DropCertificate: true, + } + config, err := yaml.Marshal(cfg) + require.NoError(t, err) + + numRounds := 10 + exporter := fileCons.New() + sendData(t, exporter, string(config), numRounds) + + // block data is valid + for i := 0; i < numRounds; i++ { + filename := fmt.Sprintf(FilePattern, i) + path := fmt.Sprintf("%s/%s", tempdir, filename) + assert.FileExists(t, path) + var blockData data.BlockData + err := util.DecodeFromFile(path, &blockData) + assert.NoError(t, err) + assert.Nil(t, blockData.Certificate) + } +} diff --git a/conduit/plugins/exporters/filewriter/sample.yaml b/conduit/plugins/exporters/filewriter/sample.yaml new file mode 100644 index 000000000..64d885cb6 --- /dev/null +++ b/conduit/plugins/exporters/filewriter/sample.yaml @@ -0,0 +1,13 @@ +name: "file_writer" +config: + # BlocksDir is the path to a directory where block data should be stored. + # The directory is created if it doesn't exist. + block-dir: "/path/to/block/files" + # FilenamePattern is the format used to write block files. It uses go + # string formatting and should accept one number for the round. + # If the file has a '.gz' extension, blocks will be gzipped. + # Default: "%[1]d_block.json" + filename-pattern: "%[1]d_block.json" + # DropCertificate is used to remove the vote certificate from the block data before writing files. + drop-certificate: true + diff --git a/conduit/plugins/exporters/noop/noop_exporter.go b/conduit/plugins/exporters/noop/noop_exporter.go new file mode 100644 index 000000000..3c0c817bf --- /dev/null +++ b/conduit/plugins/exporters/noop/noop_exporter.go @@ -0,0 +1,72 @@ +package noop + +import ( + "context" + _ "embed" // used to embed config + "fmt" + + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/exporters" + "github.com/algorand/indexer/data" +) + +var implementationName = "noop" + +// `noopExporter`s will function without ever erroring. This means they will also process out of order blocks +// which may or may not be desirable for different use cases--it can hide errors in actual exporters expecting in order +// block processing. +// The `noopExporter` will maintain `Round` state according to the round of the last block it processed. +type noopExporter struct { + round uint64 + cfg ExporterConfig +} + +//go:embed sample.yaml +var sampleConfig string + +var metadata = conduit.Metadata{ + Name: implementationName, + Description: "noop exporter", + Deprecated: false, + SampleConfig: sampleConfig, +} + +func (exp *noopExporter) Metadata() conduit.Metadata { + return metadata +} + +func (exp *noopExporter) Init(_ context.Context, initProvider data.InitProvider, cfg plugins.PluginConfig, _ *logrus.Logger) error { + if err := yaml.Unmarshal([]byte(cfg), &exp.cfg); err != nil { + return fmt.Errorf("init failure in unmarshalConfig: %v", err) + } + exp.round = exp.cfg.Round + return nil +} + +func (exp *noopExporter) Config() plugins.PluginConfig { + ret, _ := yaml.Marshal(exp.cfg) + return plugins.PluginConfig(ret) +} + +func (exp *noopExporter) Close() error { + return nil +} + +func (exp *noopExporter) Receive(exportData data.BlockData) error { + exp.round = exportData.Round() + 1 + return nil +} + +func (exp *noopExporter) Round() uint64 { + return exp.round +} + +func init() { + exporters.Register(implementationName, exporters.ExporterConstructorFunc(func() exporters.Exporter { + return &noopExporter{} + })) +} diff --git a/conduit/plugins/exporters/noop/noop_exporter_config.go b/conduit/plugins/exporters/noop/noop_exporter_config.go new file mode 100644 index 000000000..5a39b2533 --- /dev/null +++ b/conduit/plugins/exporters/noop/noop_exporter_config.go @@ -0,0 +1,7 @@ +package noop + +// ExporterConfig specific to the noop exporter +type ExporterConfig struct { + // Optionally specify the round to start on + Round uint64 `yaml:"round"` +} diff --git a/conduit/plugins/exporters/noop/noop_exporter_test.go b/conduit/plugins/exporters/noop/noop_exporter_test.go new file mode 100644 index 000000000..8a0a8dbfa --- /dev/null +++ b/conduit/plugins/exporters/noop/noop_exporter_test.go @@ -0,0 +1,63 @@ +package noop + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" + + "github.com/algorand/go-algorand/data/bookkeeping" + + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/exporters" + "github.com/algorand/indexer/data" + testutil "github.com/algorand/indexer/util/test" +) + +var nc = exporters.ExporterConstructorFunc(func() exporters.Exporter { + return &noopExporter{} +}) +var ne = nc.New() + +func TestExporterBuilderByName(t *testing.T) { + exporters.Register(metadata.Name, nc) + neBuilder, err := exporters.ExporterBuilderByName(metadata.Name) + assert.NoError(t, err) + ne := neBuilder.New() + assert.Implements(t, (*exporters.Exporter)(nil), ne) +} + +func TestExporterMetadata(t *testing.T) { + meta := ne.Metadata() + assert.Equal(t, metadata.Name, meta.Name) + assert.Equal(t, metadata.Description, meta.Description) + assert.Equal(t, metadata.Deprecated, meta.Deprecated) +} + +func TestExporterInit(t *testing.T) { + assert.NoError(t, ne.Init(context.Background(), testutil.MockedInitProvider(nil), "", nil)) +} + +func TestExporterConfig(t *testing.T) { + defaultConfig := &ExporterConfig{} + expected, err := yaml.Marshal(defaultConfig) + if err != nil { + t.Fatalf("unable to Marshal default noop.ExporterConfig: %v", err) + } + assert.NoError(t, ne.Init(context.Background(), testutil.MockedInitProvider(nil), "", nil)) + assert.Equal(t, plugins.PluginConfig(expected), ne.Config()) +} + +func TestExporterClose(t *testing.T) { + assert.NoError(t, ne.Close()) +} + +func TestExporterRoundReceive(t *testing.T) { + eData := data.BlockData{ + BlockHeader: bookkeeping.BlockHeader{ + Round: 5, + }, + } + assert.NoError(t, ne.Receive(eData)) +} diff --git a/conduit/plugins/exporters/noop/sample.yaml b/conduit/plugins/exporters/noop/sample.yaml new file mode 100644 index 000000000..a4e995631 --- /dev/null +++ b/conduit/plugins/exporters/noop/sample.yaml @@ -0,0 +1,3 @@ +name: noop +# noop has no config +config: diff --git a/conduit/plugins/exporters/postgresql/postgresql_exporter.go b/conduit/plugins/exporters/postgresql/postgresql_exporter.go new file mode 100644 index 000000000..00d5d753d --- /dev/null +++ b/conduit/plugins/exporters/postgresql/postgresql_exporter.go @@ -0,0 +1,158 @@ +package postgresql + +import ( + "context" + _ "embed" // used to embed config + "fmt" + "sync" + "sync/atomic" + + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/ledger/ledgercore" + + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/exporters" + "github.com/algorand/indexer/conduit/plugins/exporters/postgresql/util" + "github.com/algorand/indexer/data" + "github.com/algorand/indexer/idb" + "github.com/algorand/indexer/importer" + // Necessary to ensure the postgres implementation has been registered in the idb factory + _ "github.com/algorand/indexer/idb/postgres" +) + +const exporterName = "postgresql" + +type postgresqlExporter struct { + round uint64 + cfg ExporterConfig + db idb.IndexerDb + logger *logrus.Logger + wg sync.WaitGroup + ctx context.Context + cf context.CancelFunc + dm util.DataManager +} + +//go:embed sample.yaml +var sampleConfig string + +var metadata = conduit.Metadata{ + Name: exporterName, + Description: "Exporter for writing data to a postgresql instance.", + Deprecated: false, + SampleConfig: sampleConfig, +} + +func (exp *postgresqlExporter) Metadata() conduit.Metadata { + return metadata +} + +func (exp *postgresqlExporter) Init(ctx context.Context, initProvider data.InitProvider, cfg plugins.PluginConfig, logger *logrus.Logger) error { + exp.ctx, exp.cf = context.WithCancel(ctx) + dbName := "postgres" + exp.logger = logger + if err := exp.unmarhshalConfig(string(cfg)); err != nil { + return fmt.Errorf("connect failure in unmarshalConfig: %v", err) + } + // Inject a dummy db for unit testing + if exp.cfg.Test { + dbName = "dummy" + } + var opts idb.IndexerDbOptions + opts.MaxConn = exp.cfg.MaxConn + opts.ReadOnly = false + + // for some reason when ConnectionString is empty, it's automatically + // connecting to a local instance that's running. + // this behavior can be reproduced in TestConnectDbFailure. + if !exp.cfg.Test && exp.cfg.ConnectionString == "" { + return fmt.Errorf("connection string is empty for %s", dbName) + } + db, ready, err := idb.IndexerDbByName(dbName, exp.cfg.ConnectionString, opts, exp.logger) + if err != nil { + return fmt.Errorf("connect failure constructing db, %s: %v", dbName, err) + } + exp.db = db + <-ready + _, err = importer.EnsureInitialImport(exp.db, *initProvider.GetGenesis()) + if err != nil { + return fmt.Errorf("error importing genesis: %v", err) + } + dbRound, err := db.GetNextRoundToAccount() + if err != nil { + return fmt.Errorf("error getting next db round : %v", err) + } + if uint64(initProvider.NextDBRound()) != dbRound { + return fmt.Errorf("initializing block round %d but next round to account is %d", initProvider.NextDBRound(), dbRound) + } + exp.round = uint64(initProvider.NextDBRound()) + + // if data pruning is enabled + if !exp.cfg.Test && exp.cfg.Delete.Rounds > 0 { + exp.dm = util.MakeDataManager(exp.ctx, &exp.cfg.Delete, exp.db, logger) + exp.wg.Add(1) + go exp.dm.DeleteLoop(&exp.wg, &exp.round) + } + return nil +} + +func (exp *postgresqlExporter) Config() plugins.PluginConfig { + ret, _ := yaml.Marshal(exp.cfg) + return plugins.PluginConfig(ret) +} + +func (exp *postgresqlExporter) Close() error { + if exp.db != nil { + exp.db.Close() + } + + exp.cf() + exp.wg.Wait() + return nil +} + +func (exp *postgresqlExporter) Receive(exportData data.BlockData) error { + if exportData.Delta == nil { + if exportData.Round() == 0 { + exportData.Delta = &ledgercore.StateDelta{} + } else { + return fmt.Errorf("receive got an invalid block: %#v", exportData) + } + } + // Do we need to test for consensus protocol here? + /* + _, ok := config.Consensus[block.CurrentProtocol] + if !ok { + return fmt.Errorf("protocol %s not found", block.CurrentProtocol) + } + */ + var delta ledgercore.StateDelta + if exportData.Delta != nil { + delta = *exportData.Delta + } + vb := ledgercore.MakeValidatedBlock( + bookkeeping.Block{ + BlockHeader: exportData.BlockHeader, + Payset: exportData.Payset, + }, + delta) + if err := exp.db.AddBlock(&vb); err != nil { + return err + } + atomic.StoreUint64(&exp.round, exportData.Round()+1) + return nil +} + +func (exp *postgresqlExporter) unmarhshalConfig(cfg string) error { + return yaml.Unmarshal([]byte(cfg), &exp.cfg) +} + +func init() { + exporters.Register(exporterName, exporters.ExporterConstructorFunc(func() exporters.Exporter { + return &postgresqlExporter{} + })) +} diff --git a/conduit/plugins/exporters/postgresql/postgresql_exporter_config.go b/conduit/plugins/exporters/postgresql/postgresql_exporter_config.go new file mode 100644 index 000000000..ecb33dfba --- /dev/null +++ b/conduit/plugins/exporters/postgresql/postgresql_exporter_config.go @@ -0,0 +1,23 @@ +package postgresql + +import ( + "github.com/algorand/indexer/conduit/plugins/exporters/postgresql/util" +) + +// serde for converting an ExporterConfig to/from a PostgresqlExporterConfig + +// ExporterConfig specific to the postgresql exporter +type ExporterConfig struct { + // Pgsql connection string + // See https://github.com/jackc/pgconn for more details + ConnectionString string `yaml:"connection-string"` + // Maximum connection number for connection pool + // This means the total number of active queries that can be running + // concurrently can never be more than this + MaxConn uint32 `yaml:"max-conn"` + // The test flag will replace an actual DB connection being created via the connection string, + // with a mock DB for unit testing. + Test bool `yaml:"test"` + // Delete has the configuration for data pruning. + Delete util.PruneConfigurations `yaml:"delete-task"` +} diff --git a/conduit/plugins/exporters/postgresql/postgresql_exporter_test.go b/conduit/plugins/exporters/postgresql/postgresql_exporter_test.go new file mode 100644 index 000000000..e91729994 --- /dev/null +++ b/conduit/plugins/exporters/postgresql/postgresql_exporter_test.go @@ -0,0 +1,175 @@ +package postgresql + +import ( + "context" + "fmt" + "testing" + + "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" + + "github.com/algorand/go-algorand/agreement" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/ledger/ledgercore" + + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/exporters" + "github.com/algorand/indexer/conduit/plugins/exporters/postgresql/util" + "github.com/algorand/indexer/data" + _ "github.com/algorand/indexer/idb/dummy" + testutil "github.com/algorand/indexer/util/test" +) + +var pgsqlConstructor = exporters.ExporterConstructorFunc(func() exporters.Exporter { + return &postgresqlExporter{} +}) +var logger *logrus.Logger +var round = basics.Round(0) + +func init() { + logger, _ = test.NewNullLogger() +} + +func TestExporterMetadata(t *testing.T) { + pgsqlExp := pgsqlConstructor.New() + meta := pgsqlExp.Metadata() + assert.Equal(t, metadata.Name, meta.Name) + assert.Equal(t, metadata.Description, meta.Description) + assert.Equal(t, metadata.Deprecated, meta.Deprecated) +} + +func TestConnectDisconnectSuccess(t *testing.T) { + pgsqlExp := pgsqlConstructor.New() + cfg := plugins.PluginConfig("test: true\nconnection-string: ''") + assert.NoError(t, pgsqlExp.Init(context.Background(), testutil.MockedInitProvider(&round), cfg, logger)) + assert.NoError(t, pgsqlExp.Close()) +} + +func TestConnectUnmarshalFailure(t *testing.T) { + pgsqlExp := pgsqlConstructor.New() + cfg := plugins.PluginConfig("'") + assert.ErrorContains(t, pgsqlExp.Init(context.Background(), testutil.MockedInitProvider(&round), cfg, logger), "connect failure in unmarshalConfig") +} + +func TestConnectDbFailure(t *testing.T) { + pgsqlExp := pgsqlConstructor.New() + cfg := plugins.PluginConfig("") + assert.ErrorContains(t, pgsqlExp.Init(context.Background(), testutil.MockedInitProvider(&round), cfg, logger), "connection string is empty for postgres") +} + +func TestConfigDefault(t *testing.T) { + pgsqlExp := pgsqlConstructor.New() + defaultConfig := &ExporterConfig{} + expected, err := yaml.Marshal(defaultConfig) + if err != nil { + t.Fatalf("unable to Marshal default postgresql.ExporterConfig: %v", err) + } + assert.Equal(t, plugins.PluginConfig(expected), pgsqlExp.Config()) +} + +func TestReceiveInvalidBlock(t *testing.T) { + pgsqlExp := pgsqlConstructor.New() + cfg := plugins.PluginConfig("test: true") + assert.NoError(t, pgsqlExp.Init(context.Background(), testutil.MockedInitProvider(&round), cfg, logger)) + + invalidBlock := data.BlockData{ + BlockHeader: bookkeeping.BlockHeader{ + Round: 1, + }, + Payset: transactions.Payset{}, + Certificate: &agreement.Certificate{}, + Delta: nil, + } + expectedErr := fmt.Sprintf("receive got an invalid block: %#v", invalidBlock) + assert.EqualError(t, pgsqlExp.Receive(invalidBlock), expectedErr) +} + +func TestReceiveAddBlockSuccess(t *testing.T) { + pgsqlExp := pgsqlConstructor.New() + cfg := plugins.PluginConfig("test: true") + assert.NoError(t, pgsqlExp.Init(context.Background(), testutil.MockedInitProvider(&round), cfg, logger)) + + block := data.BlockData{ + BlockHeader: bookkeeping.BlockHeader{}, + Payset: transactions.Payset{}, + Certificate: &agreement.Certificate{}, + Delta: &ledgercore.StateDelta{}, + } + assert.NoError(t, pgsqlExp.Receive(block)) +} + +func TestPostgresqlExporterInit(t *testing.T) { + pgsqlExp := pgsqlConstructor.New() + cfg := plugins.PluginConfig("test: true") + + // genesis hash mismatch + initProvider := testutil.MockedInitProvider(&round) + initProvider.Genesis = &bookkeeping.Genesis{ + Network: "test", + } + err := pgsqlExp.Init(context.Background(), initProvider, cfg, logger) + assert.Contains(t, err.Error(), "error importing genesis: genesis hash not matching") + + // incorrect round + round = 1 + err = pgsqlExp.Init(context.Background(), testutil.MockedInitProvider(&round), cfg, logger) + assert.Contains(t, err.Error(), "initializing block round 1 but next round to account is 0") +} + +func TestUnmarshalConfigsContainingDeleteTask(t *testing.T) { + // configured delete task + pgsqlExp := postgresqlExporter{} + cfg := ExporterConfig{ + ConnectionString: "", + MaxConn: 0, + Test: true, + Delete: util.PruneConfigurations{ + Rounds: 3000, + Interval: 3, + }, + } + data, err := yaml.Marshal(cfg) + assert.NoError(t, err) + assert.NoError(t, pgsqlExp.unmarhshalConfig(string(data))) + assert.Equal(t, 3, int(pgsqlExp.cfg.Delete.Interval)) + assert.Equal(t, uint64(3000), pgsqlExp.cfg.Delete.Rounds) + + // delete task with fields default to 0 + pgsqlExp = postgresqlExporter{} + cfg = ExporterConfig{ + ConnectionString: "", + MaxConn: 0, + Test: true, + Delete: util.PruneConfigurations{}, + } + data, err = yaml.Marshal(cfg) + assert.NoError(t, err) + assert.NoError(t, pgsqlExp.unmarhshalConfig(string(data))) + assert.Equal(t, 0, int(pgsqlExp.cfg.Delete.Interval)) + assert.Equal(t, uint64(0), pgsqlExp.cfg.Delete.Rounds) + + // delete task with negative interval + pgsqlExp = postgresqlExporter{} + cfg = ExporterConfig{ + ConnectionString: "", + MaxConn: 0, + Test: true, + Delete: util.PruneConfigurations{ + Rounds: 1, + Interval: -1, + }, + } + data, err = yaml.Marshal(cfg) + assert.NoError(t, pgsqlExp.unmarhshalConfig(string(data))) + assert.Equal(t, -1, int(pgsqlExp.cfg.Delete.Interval)) + + // delete task with negative round + pgsqlExp = postgresqlExporter{} + cfgstr := "test: true\ndelete-task:\n rounds: -1\n interval: 2" + assert.ErrorContains(t, pgsqlExp.unmarhshalConfig(cfgstr), "unmarshal errors") + +} diff --git a/conduit/plugins/exporters/postgresql/sample.yaml b/conduit/plugins/exporters/postgresql/sample.yaml new file mode 100644 index 000000000..325df8124 --- /dev/null +++ b/conduit/plugins/exporters/postgresql/sample.yaml @@ -0,0 +1,19 @@ +name: postgresql +config: + # Pgsql connection string + # See https://github.com/jackc/pgconn for more details + connection-string: "host= port=5432 user= password= dbname=" + # Maximum connection number for connection pool + # This means the total number of active queries that can be running + # concurrently can never be more than this + max-conn: 20 + # The test flag will replace an actual DB connection being created via the connection string, + # with a mock DB for unit testing. + test: false + # Delete has the configuration for data pruning. + delete-task: + # Rounds to keep + rounds: 100000 + # Interval used to prune the data. The values can be -1 to run at startup, + # 0 to disable or N to run every N rounds. + interval: 0 diff --git a/conduit/plugins/exporters/postgresql/util/prune.go b/conduit/plugins/exporters/postgresql/util/prune.go new file mode 100644 index 000000000..93c641aa6 --- /dev/null +++ b/conduit/plugins/exporters/postgresql/util/prune.go @@ -0,0 +1,96 @@ +package util + +import ( + "context" + "sync" + "time" + + "github.com/algorand/indexer/idb" + "github.com/sirupsen/logrus" +) + +// Interval determines how often to delete data +type Interval int + +const ( + once Interval = -1 + disabled Interval = 0 + d = 2 * time.Second +) + +// PruneConfigurations contains the configurations for data pruning +type PruneConfigurations struct { + // Rounds to keep + Rounds uint64 `yaml:"rounds"` + // Interval used to prune the data. The values can be -1 to run at startup, + // 0 to disable or N to run every N rounds. + Interval Interval `yaml:"interval"` +} + +// DataManager is a data pruning interface +type DataManager interface { + DeleteLoop(*sync.WaitGroup, *uint64) +} + +type postgresql struct { + config *PruneConfigurations + db idb.IndexerDb + logger *logrus.Logger + ctx context.Context + duration time.Duration +} + +// MakeDataManager initializes resources need for removing data from data source +func MakeDataManager(ctx context.Context, cfg *PruneConfigurations, db idb.IndexerDb, logger *logrus.Logger) DataManager { + + dm := &postgresql{ + config: cfg, + db: db, + logger: logger, + ctx: ctx, + duration: d, + } + + return dm +} + +// DeleteLoop removes data from the txn table in Postgres DB +func (p *postgresql) DeleteLoop(wg *sync.WaitGroup, nextRound *uint64) { + + defer wg.Done() + // round value used for interval calculation + round := *nextRound + for { + select { + case <-p.ctx.Done(): + return + case <-time.After(p.duration): + currentRound := *nextRound + // keep, remove data older than keep + keep := currentRound - p.config.Rounds + if p.config.Interval == once { + if currentRound > p.config.Rounds { + err := p.db.DeleteTransactions(p.ctx, keep) + if err != nil { + p.logger.Warnf("MakeDataManager(): data pruning err: %v", err) + } + } + return + } else if p.config.Interval > disabled { + // *nextRound should increment as exporter receives new block + if currentRound > p.config.Rounds && currentRound-round >= uint64(p.config.Interval) { + err := p.db.DeleteTransactions(p.ctx, keep) + if err != nil { + p.logger.Warnf("DeleteLoop(): data pruning err: %v", err) + return + } + // update round value for next interval calculation + round = currentRound + } + } else { + p.logger.Fatalf("DeleteLoop(): unsupported interval value %v", p.config.Interval) + return + } + } + } +} diff --git a/conduit/plugins/exporters/postgresql/util/prune_test.go b/conduit/plugins/exporters/postgresql/util/prune_test.go new file mode 100644 index 000000000..9f909415f --- /dev/null +++ b/conduit/plugins/exporters/postgresql/util/prune_test.go @@ -0,0 +1,267 @@ +package util + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/jackc/pgx/v4" + "github.com/jackc/pgx/v4/pgxpool" + "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/assert" + + "github.com/algorand/indexer/idb" + "github.com/algorand/indexer/idb/postgres" + pgtest "github.com/algorand/indexer/idb/postgres/testing" +) + +var logger *logrus.Logger +var hook *test.Hook + +func init() { + logger, hook = test.NewNullLogger() +} + +var config = PruneConfigurations{ + Interval: -1, + Rounds: 10, +} + +func delete(idb idb.IndexerDb, nextround uint64) DataManager { + var wg sync.WaitGroup + ctx, cancel := context.WithCancel(context.Background()) + dm := MakeDataManager(ctx, &config, idb, logger) + wg.Add(1) + go dm.DeleteLoop(&wg, &nextround) + go func() { + time.Sleep(3 * time.Second) + cancel() + }() + wg.Wait() + return dm +} +func TestDeleteEmptyTxnTable(t *testing.T) { + db, connStr, shutdownFunc := pgtest.SetupPostgres(t) + defer shutdownFunc() + + // init the tables + idb, _, err := postgres.OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) + assert.NoError(t, err) + defer idb.Close() + + // empty txn table + _, err = populateTxnTable(db, 1, 0) + assert.NoError(t, err) + assert.Equal(t, 0, rowsInTxnTable(db)) + + delete(idb, 0) + assert.Equal(t, 0, rowsInTxnTable(db)) +} + +func TestDeleteTxns(t *testing.T) { + db, connStr, shutdownFunc := pgtest.SetupPostgres(t) + defer shutdownFunc() + + // init the tables + idb, _, err := postgres.OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) + assert.NoError(t, err) + defer idb.Close() + + // add 20 records to txn table + ntxns := 20 + nextRound, err := populateTxnTable(db, 1, ntxns) + assert.NoError(t, err) + assert.Equal(t, ntxns, rowsInTxnTable(db)) + delete(idb, uint64(nextRound)) + // 10 rounds removed + assert.Equal(t, 10, rowsInTxnTable(db)) + // check remaining rounds are correct + assert.True(t, validateTxnTable(db, 11, 20)) +} + +func TestDeleteConfigs(t *testing.T) { + db, connStr, shutdownFunc := pgtest.SetupPostgres(t) + defer shutdownFunc() + + // init the tables + idb, _, err := postgres.OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) + assert.NoError(t, err) + defer idb.Close() + + // add 3 records to txn table + ntxns := 3 + nextRound, err := populateTxnTable(db, 1, ntxns) + assert.NoError(t, err) + assert.Equal(t, ntxns, rowsInTxnTable(db)) + + // config.Rounds > rounds in DB + config = PruneConfigurations{ + Interval: -1, + Rounds: 5, + } + + // config.Rounds > rounds in DB + delete(idb, uint64(nextRound)) + // delete didn't happen + assert.Equal(t, 3, rowsInTxnTable(db)) + + // config.Rounds == rounds in DB + config.Rounds = 3 + delete(idb, uint64(nextRound)) + // delete didn't happen + assert.Equal(t, 3, rowsInTxnTable(db)) + + // run delete once + config = PruneConfigurations{ + Interval: -1, + Rounds: 2, + } + var wg sync.WaitGroup + ctx := context.Background() + dm := postgresql{ + config: &config, + db: idb, + logger: logger, + ctx: ctx, + duration: 500 * time.Millisecond, + } + + wg.Add(1) + round := uint64(nextRound) + go dm.DeleteLoop(&wg, &round) + wg.Wait() + assert.Equal(t, 2, rowsInTxnTable(db)) + +} + +func TestDeleteInterval(t *testing.T) { + db, connStr, shutdownFunc := pgtest.SetupPostgres(t) + defer shutdownFunc() + + // init the tables + idb, _, err := postgres.OpenPostgres(connStr, idb.IndexerDbOptions{}, nil) + assert.NoError(t, err) + defer idb.Close() + + // add 5 records to txn table + ntxns := 5 + nextRound, err := populateTxnTable(db, 1, ntxns) + assert.NoError(t, err) + assert.Equal(t, ntxns, rowsInTxnTable(db)) + + config = PruneConfigurations{ + Interval: 3, + Rounds: 3, + } + var wg sync.WaitGroup + ctx, cf := context.WithCancel(context.Background()) + dm := postgresql{ + config: &config, + db: idb, + logger: logger, + ctx: ctx, + duration: 500 * time.Millisecond, + } + + wg.Add(1) + round := uint64(nextRound) + go dm.DeleteLoop(&wg, &round) + time.Sleep(1 * time.Second) + assert.Equal(t, 5, rowsInTxnTable(db)) + + // remove data every 3 round + // add round 6. no data removed + nextRound, err = populateTxnTable(db, nextRound, 1) + assert.NoError(t, err) + atomic.AddUint64(&round, 1) + time.Sleep(1 * time.Second) + assert.Equal(t, 6, rowsInTxnTable(db)) + + // add round 7. no data removed + nextRound, err = populateTxnTable(db, nextRound, 1) + assert.NoError(t, err) + atomic.AddUint64(&round, 1) + time.Sleep(1 * time.Second) + assert.Equal(t, 7, rowsInTxnTable(db)) + + // add round 8. data removed + nextRound, err = populateTxnTable(db, nextRound, 1) + assert.NoError(t, err) + atomic.AddUint64(&round, 1) + time.Sleep(1 * time.Second) + assert.Equal(t, 3, rowsInTxnTable(db)) + + go func() { + time.Sleep(1 * time.Second) + cf() + }() + wg.Wait() + + // reconnect + config = PruneConfigurations{ + Interval: -1, + Rounds: 1, + } + delete(idb, uint64(nextRound)) + assert.Equal(t, 1, rowsInTxnTable(db)) +} + +// populate n records starting with round starting at r. +// return next round +func populateTxnTable(db *pgxpool.Pool, r int, n int) (int, error) { + batch := &pgx.Batch{} + // txn round starts at 1 because genesis block is empty + for i := 1; i <= n; i++ { + query := "INSERT INTO txn(round, intra, typeenum,asset,txn,extra) VALUES ($1,$2,$3,$4,$5,$6)" + batch.Queue(query, r, i, 1, 0, "{}", "{}") + r++ + } + results := db.SendBatch(context.Background(), batch) + defer results.Close() + for i := 0; i < batch.Len(); i++ { + _, err := results.Exec() + if err != nil { + return 0, fmt.Errorf("populateTxnTable() exec err: %w", err) + } + } + return r, nil +} + +func rowsInTxnTable(db *pgxpool.Pool) int { + var rows int + r, err := db.Query(context.Background(), "SELECT count(*) FROM txn") + if err != nil { + return 0 + } + defer r.Close() + for r.Next() { + err = r.Scan(&rows) + if err != nil { + return 0 + } + } + return rows +} + +func validateTxnTable(db *pgxpool.Pool, first, last uint64) bool { + res, err := db.Query(context.Background(), "SELECT round FROM txn") + if err != nil { + return false + } + defer res.Close() + var round uint64 + // expected round + expected := first + for res.Next() { + err = res.Scan(&round) + if err != nil || round != expected { + return false + } + expected++ + } + return expected-1 == last +} diff --git a/conduit/plugins/importers/algod/algod_importer.go b/conduit/plugins/importers/algod/algod_importer.go new file mode 100644 index 000000000..408cea22f --- /dev/null +++ b/conduit/plugins/importers/algod/algod_importer.go @@ -0,0 +1,159 @@ +package algodimporter + +import ( + "context" + _ "embed" // used to embed config + "fmt" + "net/url" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + + "github.com/algorand/go-algorand-sdk/client/v2/algod" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/rpcs" + + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/importers" + "github.com/algorand/indexer/data" +) + +const importerName = "algod" + +type algodImporter struct { + aclient *algod.Client + logger *logrus.Logger + cfg Config + ctx context.Context + cancel context.CancelFunc +} + +//go:embed sample.yaml +var sampleConfig string + +var algodImporterMetadata = conduit.Metadata{ + Name: importerName, + Description: "Importer for fetching blocks from an algod REST API.", + Deprecated: false, + SampleConfig: sampleConfig, +} + +// New initializes an algod importer +func New() importers.Importer { + return &algodImporter{} +} + +func (algodImp *algodImporter) Metadata() conduit.Metadata { + return algodImporterMetadata +} + +// package-wide init function +func init() { + importers.Register(importerName, importers.ImporterConstructorFunc(func() importers.Importer { + return &algodImporter{} + })) +} + +func (algodImp *algodImporter) Init(ctx context.Context, cfg plugins.PluginConfig, logger *logrus.Logger) (*bookkeeping.Genesis, error) { + algodImp.ctx, algodImp.cancel = context.WithCancel(ctx) + algodImp.logger = logger + var err error + algodImp.cfg, err = unmarshalConfig(string(cfg)) + if err != nil { + return nil, fmt.Errorf("connect failure in unmarshalConfig: %v", err) + } + var client *algod.Client + u, err := url.Parse(algodImp.cfg.NetAddr) + if err != nil { + return nil, err + } + + if u.Scheme != "http" && u.Scheme != "https" { + algodImp.cfg.NetAddr = "http://" + algodImp.cfg.NetAddr + algodImp.logger.Infof("Algod Importer added http prefix to NetAddr: %s", algodImp.cfg.NetAddr) + } + client, err = algod.MakeClient(algodImp.cfg.NetAddr, algodImp.cfg.Token) + if err != nil { + return nil, err + } + algodImp.aclient = client + + genesisResponse, err := client.GetGenesis().Do(ctx) + if err != nil { + return nil, err + } + + genesis := bookkeeping.Genesis{} + + err = protocol.DecodeJSON([]byte(genesisResponse), &genesis) + if err != nil { + return nil, err + } + + return &genesis, err +} + +func (algodImp *algodImporter) Config() plugins.PluginConfig { + s, _ := yaml.Marshal(algodImp.cfg) + return plugins.PluginConfig(s) +} + +func (algodImp *algodImporter) Close() error { + if algodImp.cancel != nil { + algodImp.cancel() + } + return nil +} + +func (algodImp *algodImporter) GetBlock(rnd uint64) (data.BlockData, error) { + var blockbytes []byte + var err error + var blk data.BlockData + + for retries := 0; retries < 3; retries++ { + // nextRound - 1 because the endpoint waits until the subsequent block is committed to return + _, err = algodImp.aclient.StatusAfterBlock(rnd - 1).Do(algodImp.ctx) + if err != nil { + // If context has expired. + if algodImp.ctx.Err() != nil { + return blk, fmt.Errorf("GetBlock ctx error: %w", err) + } + algodImp.logger.Errorf( + "r=%d error getting status %d", retries, rnd) + continue + } + start := time.Now() + blockbytes, err = algodImp.aclient.BlockRaw(rnd).Do(algodImp.ctx) + dt := time.Since(start) + GetAlgodRawBlockTimeSeconds.Observe(dt.Seconds()) + if err != nil { + return blk, err + } + tmpBlk := new(rpcs.EncodedBlockCert) + err = protocol.Decode(blockbytes, tmpBlk) + + blk = data.BlockData{ + BlockHeader: tmpBlk.Block.BlockHeader, + Payset: tmpBlk.Block.Payset, + Certificate: &tmpBlk.Certificate, + } + return blk, err + } + algodImp.logger.Error("GetBlock finished retries without fetching a block.") + return blk, fmt.Errorf("finished retries without fetching a block") +} + +func (algodImp *algodImporter) ProvideMetrics() []prometheus.Collector { + return []prometheus.Collector{ + GetAlgodRawBlockTimeSeconds, + } +} + +func unmarshalConfig(cfg string) (config Config, err error) { + err = yaml.Unmarshal([]byte(cfg), &config) + return +} diff --git a/conduit/plugins/importers/algod/algod_importer_config.go b/conduit/plugins/importers/algod/algod_importer_config.go new file mode 100644 index 000000000..60379fd39 --- /dev/null +++ b/conduit/plugins/importers/algod/algod_importer_config.go @@ -0,0 +1,9 @@ +package algodimporter + +// Config specific to the algod importer +type Config struct { + // Algod netaddr string + NetAddr string `yaml:"netaddr"` + // Algod rest endpoint token + Token string `yaml:"token"` +} diff --git a/conduit/plugins/importers/algod/algod_importer_test.go b/conduit/plugins/importers/algod/algod_importer_test.go new file mode 100644 index 000000000..6a270b6bc --- /dev/null +++ b/conduit/plugins/importers/algod/algod_importer_test.go @@ -0,0 +1,133 @@ +package algodimporter + +import ( + "context" + "os" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" + + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/importers" + "github.com/algorand/indexer/util/test" +) + +var ( + logger *logrus.Logger + ctx context.Context + cancel context.CancelFunc + testImporter importers.Importer +) + +func init() { + logger = logrus.New() + logger.SetOutput(os.Stdout) + logger.SetLevel(logrus.InfoLevel) + ctx, cancel = context.WithCancel(context.Background()) +} + +func TestImporterMetadata(t *testing.T) { + testImporter = New() + metadata := testImporter.Metadata() + assert.Equal(t, metadata.Name, algodImporterMetadata.Name) + assert.Equal(t, metadata.Description, algodImporterMetadata.Description) + assert.Equal(t, metadata.Deprecated, algodImporterMetadata.Deprecated) +} + +func TestCloseSuccess(t *testing.T) { + ts := test.NewAlgodServer(test.GenesisResponder) + testImporter = New() + _, err := testImporter.Init(ctx, plugins.PluginConfig("netaddr: "+ts.URL), logger) + assert.NoError(t, err) + err = testImporter.Close() + assert.NoError(t, err) +} + +func TestInitSuccess(t *testing.T) { + ts := test.NewAlgodServer(test.GenesisResponder) + testImporter = New() + _, err := testImporter.Init(ctx, plugins.PluginConfig("netaddr: "+ts.URL), logger) + assert.NoError(t, err) + assert.NotEqual(t, testImporter, nil) + testImporter.Close() +} + +func TestInitUnmarshalFailure(t *testing.T) { + testImporter = New() + _, err := testImporter.Init(ctx, "`", logger) + assert.Error(t, err) + assert.ErrorContains(t, err, "connect failure in unmarshalConfig") + testImporter.Close() +} + +func TestConfigDefault(t *testing.T) { + testImporter = New() + expected, err := yaml.Marshal(&Config{}) + if err != nil { + t.Fatalf("unable to Marshal default algodimporter.Config: %v", err) + } + assert.Equal(t, plugins.PluginConfig(expected), testImporter.Config()) +} + +func TestWaitForBlockBlockFailure(t *testing.T) { + ts := test.NewAlgodServer(test.GenesisResponder) + testImporter = New() + _, err := testImporter.Init(ctx, plugins.PluginConfig("netaddr: "+ts.URL), logger) + assert.NoError(t, err) + assert.NotEqual(t, testImporter, nil) + + blk, err := testImporter.GetBlock(uint64(10)) + assert.Error(t, err) + assert.True(t, blk.Empty()) +} + +func TestGetBlockSuccess(t *testing.T) { + ctx, cancel = context.WithCancel(context.Background()) + ts := test.NewAlgodServer( + test.GenesisResponder, + test.BlockResponder, + test.BlockAfterResponder) + testImporter = New() + _, err := testImporter.Init(ctx, plugins.PluginConfig("netaddr: "+ts.URL), logger) + assert.NoError(t, err) + assert.NotEqual(t, testImporter, nil) + + downloadedBlk, err := testImporter.GetBlock(uint64(10)) + assert.NoError(t, err) + assert.Equal(t, downloadedBlk.Round(), uint64(10)) + assert.True(t, downloadedBlk.Empty()) + cancel() +} + +func TestGetBlockContextCancelled(t *testing.T) { + ctx, cancel = context.WithCancel(context.Background()) + ts := test.NewAlgodServer( + test.GenesisResponder, + test.BlockResponder, + test.BlockAfterResponder) + testImporter = New() + _, err := testImporter.Init(ctx, plugins.PluginConfig("netaddr: "+ts.URL), logger) + assert.NoError(t, err) + assert.NotEqual(t, testImporter, nil) + + cancel() + _, err = testImporter.GetBlock(uint64(10)) + assert.Error(t, err) +} + +func TestGetBlockFailure(t *testing.T) { + ctx, cancel = context.WithCancel(context.Background()) + ts := test.NewAlgodServer( + test.GenesisResponder, + test.BlockAfterResponder) + testImporter = New() + _, err := testImporter.Init(ctx, plugins.PluginConfig("netaddr: "+ts.URL), logger) + assert.NoError(t, err) + assert.NotEqual(t, testImporter, nil) + + _, err = testImporter.GetBlock(uint64(10)) + assert.Error(t, err) + cancel() +} diff --git a/conduit/plugins/importers/algod/metrics.go b/conduit/plugins/importers/algod/metrics.go new file mode 100644 index 000000000..488e387d7 --- /dev/null +++ b/conduit/plugins/importers/algod/metrics.go @@ -0,0 +1,18 @@ +package algodimporter + +import "github.com/prometheus/client_golang/prometheus" + +// Prometheus metric names +const ( + GetAlgodRawBlockTimeName = "get_algod_raw_block_time_sec" +) + +// Initialize the prometheus objects +var ( + GetAlgodRawBlockTimeSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Subsystem: "indexer_daemon", + Name: GetAlgodRawBlockTimeName, + Help: "Total response time from Algod's raw block endpoint in seconds.", + }) +) diff --git a/conduit/plugins/importers/algod/sample.yaml b/conduit/plugins/importers/algod/sample.yaml new file mode 100644 index 000000000..8894764d9 --- /dev/null +++ b/conduit/plugins/importers/algod/sample.yaml @@ -0,0 +1,6 @@ +name: algod +config: + # Algod netaddr string + netaddr: "http://url" + # Algod rest endpoint token + token: "" diff --git a/conduit/plugins/importers/all/all.go b/conduit/plugins/importers/all/all.go new file mode 100644 index 000000000..2704b613b --- /dev/null +++ b/conduit/plugins/importers/all/all.go @@ -0,0 +1,7 @@ +package all + +import ( + // Call package wide init function + _ "github.com/algorand/indexer/conduit/plugins/importers/algod" + _ "github.com/algorand/indexer/conduit/plugins/importers/filereader" +) diff --git a/conduit/plugins/importers/filereader/filereader.go b/conduit/plugins/importers/filereader/filereader.go new file mode 100644 index 000000000..eba530a14 --- /dev/null +++ b/conduit/plugins/importers/filereader/filereader.go @@ -0,0 +1,129 @@ +package fileimporter + +import ( + "context" + _ "embed" // used to embed config + "errors" + "fmt" + "io/fs" + "path" + "time" + + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + + "github.com/algorand/go-algorand/data/bookkeeping" + + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/exporters/filewriter" + "github.com/algorand/indexer/conduit/plugins/importers" + "github.com/algorand/indexer/data" + "github.com/algorand/indexer/util" +) + +const importerName = "file_reader" + +type fileReader struct { + logger *logrus.Logger + cfg Config + ctx context.Context + cancel context.CancelFunc +} + +// New initializes an algod importer +func New() importers.Importer { + return &fileReader{} +} + +//go:embed sample.yaml +var sampleConfig string + +var metadata = conduit.Metadata{ + Name: importerName, + Description: "Importer for fetching blocks from files in a directory created by the 'file_writer' plugin.", + Deprecated: false, + SampleConfig: sampleConfig, +} + +func (r *fileReader) Metadata() conduit.Metadata { + return metadata +} + +// package-wide init function +func init() { + importers.Register(importerName, importers.ImporterConstructorFunc(func() importers.Importer { + return &fileReader{} + })) +} + +func (r *fileReader) Init(ctx context.Context, cfg plugins.PluginConfig, logger *logrus.Logger) (*bookkeeping.Genesis, error) { + r.ctx, r.cancel = context.WithCancel(ctx) + r.logger = logger + var err error + r.cfg, err = unmarshalConfig(string(cfg)) + if err != nil { + return nil, fmt.Errorf("invalid configuration: %v", err) + } + + if r.cfg.FilenamePattern == "" { + r.cfg.FilenamePattern = filewriter.FilePattern + } + + genesisFile := path.Join(r.cfg.BlocksDir, "genesis.json") + var genesis bookkeeping.Genesis + err = util.DecodeFromFile(genesisFile, &genesis) + if err != nil { + return nil, fmt.Errorf("Init(): failed to process genesis file: %w", err) + } + + return &genesis, err +} + +func (r *fileReader) Config() plugins.PluginConfig { + s, _ := yaml.Marshal(r.cfg) + return plugins.PluginConfig(s) +} + +func (r *fileReader) Close() error { + if r.cancel != nil { + r.cancel() + } + return nil +} + +func (r *fileReader) GetBlock(rnd uint64) (data.BlockData, error) { + attempts := r.cfg.RetryCount + for { + filename := path.Join(r.cfg.BlocksDir, fmt.Sprintf(r.cfg.FilenamePattern, rnd)) + var blockData data.BlockData + start := time.Now() + err := util.DecodeFromFile(filename, &blockData) + if err != nil && errors.Is(err, fs.ErrNotExist) { + // If the file read failed because the file didn't exist, wait before trying again + if attempts == 0 { + return data.BlockData{}, fmt.Errorf("GetBlock(): block not found after (%d) attempts", r.cfg.RetryCount) + } + attempts-- + + select { + case <-time.After(r.cfg.RetryDuration): + case <-r.ctx.Done(): + return data.BlockData{}, fmt.Errorf("GetBlock() context finished: %w", r.ctx.Err()) + } + } else if err != nil { + // Other error, return error to pipeline + return data.BlockData{}, fmt.Errorf("GetBlock(): unable to read block file '%s': %w", filename, err) + } else { + r.logger.Infof("Block %d read time: %s", rnd, time.Since(start)) + // The read was fine, return the data. + return blockData, nil + } + } +} + +func unmarshalConfig(cfg string) (Config, error) { + var config Config + err := yaml.Unmarshal([]byte(cfg), &config) + return config, err +} diff --git a/conduit/plugins/importers/filereader/filereader_config.go b/conduit/plugins/importers/filereader/filereader_config.go new file mode 100644 index 000000000..c336f0a17 --- /dev/null +++ b/conduit/plugins/importers/filereader/filereader_config.go @@ -0,0 +1,21 @@ +package fileimporter + +import "time" + +// Config specific to the file importer +type Config struct { + // BlocksDir is the path to a directory where block data should be stored. + // The directory is created if it doesn't exist. + BlocksDir string `yaml:"block-dir"` + // RetryDuration controls the delay between checks when the importer has + // caught up and is waiting for new blocks to appear. + RetryDuration time.Duration `yaml:"retry-duration"` + // RetryCount controls the number of times to check for a missing block + // before generating an error. The retry count and retry duration should + // be configured according the expected round time. + RetryCount uint64 `yaml:"retry-count"` + // FilenamePattern is the format used to find block files. It uses go string formatting and should accept one number for the round. + FilenamePattern string `yaml:"filename-pattern"` + + // TODO: Option to delete files after processing them +} diff --git a/conduit/plugins/importers/filereader/filereader_test.go b/conduit/plugins/importers/filereader/filereader_test.go new file mode 100644 index 000000000..c11174470 --- /dev/null +++ b/conduit/plugins/importers/filereader/filereader_test.go @@ -0,0 +1,186 @@ +package fileimporter + +import ( + "context" + "fmt" + "os" + "path" + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/ledger/ledgercore" + + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/exporters/filewriter" + "github.com/algorand/indexer/conduit/plugins/importers" + "github.com/algorand/indexer/data" + "github.com/algorand/indexer/util" +) + +var ( + logger *logrus.Logger + ctx context.Context + cancel context.CancelFunc + testImporter importers.Importer +) + +func init() { + logger = logrus.New() + logger.SetOutput(os.Stdout) + logger.SetLevel(logrus.InfoLevel) +} + +func TestImporterorterMetadata(t *testing.T) { + testImporter = New() + m := testImporter.Metadata() + assert.Equal(t, metadata.Name, m.Name) + assert.Equal(t, metadata.Description, m.Description) + assert.Equal(t, metadata.Deprecated, m.Deprecated) +} + +// initializeTestData fills a data directory with some dummy data for the importer to read. +func initializeTestData(t *testing.T, dir string, numRounds int) bookkeeping.Genesis { + genesisA := bookkeeping.Genesis{ + SchemaID: "test", + Network: "test", + Proto: "test", + Allocation: nil, + RewardsPool: "AAAAAAA", + FeeSink: "AAAAAAA", + Timestamp: 1234, + Comment: "", + DevMode: true, + } + + err := util.EncodeToFile(path.Join(dir, "genesis.json"), genesisA, true) + require.NoError(t, err) + + for i := 0; i < numRounds; i++ { + block := data.BlockData{ + BlockHeader: bookkeeping.BlockHeader{ + Round: basics.Round(i), + }, + Payset: make([]transactions.SignedTxnInBlock, 0), + Delta: &ledgercore.StateDelta{}, + Certificate: nil, + } + blockFile := path.Join(dir, fmt.Sprintf(filewriter.FilePattern, i)) + err = util.EncodeToFile(blockFile, block, true) + require.NoError(t, err) + } + + return genesisA +} + +func initializeImporter(t *testing.T, numRounds int) (importer importers.Importer, tempdir string, genesis *bookkeeping.Genesis, err error) { + tempdir = t.TempDir() + genesisExpected := initializeTestData(t, tempdir, numRounds) + importer = New() + defer importer.Config() + cfg := Config{ + BlocksDir: tempdir, + RetryDuration: 0, + } + data, err := yaml.Marshal(cfg) + require.NoError(t, err) + genesis, err = importer.Init(context.Background(), plugins.PluginConfig(data), logger) + assert.NoError(t, err) + require.NotNil(t, genesis) + require.Equal(t, genesisExpected, *genesis) + return +} + +func TestInitSuccess(t *testing.T) { + _, _, _, err := initializeImporter(t, 1) + require.NoError(t, err) +} + +func TestInitUnmarshalFailure(t *testing.T) { + testImporter = New() + _, err := testImporter.Init(context.Background(), "`", logger) + assert.Error(t, err) + assert.ErrorContains(t, err, "invalid configuration") + testImporter.Close() +} + +func TestConfigDefault(t *testing.T) { + testImporter = New() + expected, err := yaml.Marshal(&Config{}) + require.NoError(t, err) + assert.Equal(t, plugins.PluginConfig(expected), testImporter.Config()) +} + +func TestGetBlockSuccess(t *testing.T) { + numRounds := 10 + importer, tempdir, genesis, err := initializeImporter(t, 10) + require.NoError(t, err) + require.NotEqual(t, "", tempdir) + require.NotNil(t, genesis) + + for i := 0; i < numRounds; i++ { + block, err := importer.GetBlock(uint64(i)) + require.NoError(t, err) + require.Equal(t, basics.Round(i), block.BlockHeader.Round) + } +} + +func TestRetryAndDuration(t *testing.T) { + tempdir := t.TempDir() + initializeTestData(t, tempdir, 0) + importer := New() + defer importer.Config() + cfg := Config{ + BlocksDir: tempdir, + RetryDuration: 10 * time.Millisecond, + RetryCount: 3, + } + data, err := yaml.Marshal(cfg) + require.NoError(t, err) + _, err = importer.Init(context.Background(), plugins.PluginConfig(data), logger) + assert.NoError(t, err) + + start := time.Now() + _, err = importer.GetBlock(0) + assert.ErrorContains(t, err, "GetBlock(): block not found after (3) attempts") + + expectedDuration := cfg.RetryDuration*time.Duration(cfg.RetryCount) + 10*time.Millisecond + assert.WithinDuration(t, start, time.Now(), expectedDuration, "Error should generate after retry count * retry duration") +} + +func TestRetryWithCancel(t *testing.T) { + tempdir := t.TempDir() + initializeTestData(t, tempdir, 0) + importer := New() + defer importer.Config() + cfg := Config{ + BlocksDir: tempdir, + RetryDuration: 1 * time.Hour, + RetryCount: 3, + } + data, err := yaml.Marshal(cfg) + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, err) + _, err = importer.Init(ctx, plugins.PluginConfig(data), logger) + assert.NoError(t, err) + + // Cancel after delay + delay := time.Millisecond + go func() { + time.Sleep(delay) + cancel() + }() + start := time.Now() + _, err = importer.GetBlock(0) + assert.ErrorContains(t, err, "GetBlock() context finished: context canceled") + + // within 1ms of the expected time (but much less than the 3hr configuration. + assert.WithinDuration(t, start, time.Now(), delay+time.Millisecond) +} diff --git a/conduit/plugins/importers/filereader/sample.yaml b/conduit/plugins/importers/filereader/sample.yaml new file mode 100644 index 000000000..a88153640 --- /dev/null +++ b/conduit/plugins/importers/filereader/sample.yaml @@ -0,0 +1,14 @@ +name: file_reader +config: + # BlocksDir is the path to a directory where block data should be stored. + # The directory is created if it doesn't exist. + block-dir: "/path/to/directory" + # RetryDuration controls the delay between checks when the importer has + # caught up and is waiting for new blocks to appear. + retry-duration: "5s" + # RetryCount controls the number of times to check for a missing block + # before generating an error. The retry count and retry duration should + # be configured according the expected round time. + retry-count: 5 + # FilenamePattern is the format used to find block files. It uses go string formatting and should accept one number for the round. + filename-pattern: "%[1]d_block.json" diff --git a/conduit/plugins/importers/importer.go b/conduit/plugins/importers/importer.go new file mode 100644 index 000000000..ba8030c94 --- /dev/null +++ b/conduit/plugins/importers/importer.go @@ -0,0 +1,34 @@ +package importers + +import ( + "context" + + "github.com/sirupsen/logrus" + + "github.com/algorand/go-algorand/data/bookkeeping" + + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/data" +) + +// Importer defines the interface for importer plugins +type Importer interface { + // PluginMetadata implement this interface. + conduit.PluginMetadata + + // Init will initialize each importer with a given config. This config will contain the Unmarhsalled config file specific to this plugin. + // It is called during initialization of an importer plugin such as setting up network connections, file buffers etc. + // Importers will also be responsible for returning a valid Genesis object pointer + Init(ctx context.Context, cfg plugins.PluginConfig, logger *logrus.Logger) (*bookkeeping.Genesis, error) + + // Config returns the configuration options used to create an Importer. Initialized during Init. + Config() plugins.PluginConfig + + // Close function is used for closing network connections, files, flushing buffers etc. + Close() error + + // GetBlock given any round number-rnd fetches the block at that round + // It returns an object of type BlockData defined in data + GetBlock(rnd uint64) (data.BlockData, error) +} diff --git a/conduit/plugins/importers/importer_factory.go b/conduit/plugins/importers/importer_factory.go new file mode 100644 index 000000000..ed9ba00e5 --- /dev/null +++ b/conduit/plugins/importers/importer_factory.go @@ -0,0 +1,41 @@ +package importers + +import ( + "fmt" +) + +// Constructor must be implemented by each Importer. +// It provides a basic no-arg constructor for instances of an ImporterImpl. +type Constructor interface { + // New should return an instantiation of a Importer. + // Configuration values should be passed and can be processed during `Init()`. + New() Importer +} + +// ImporterConstructorFunc is Constructor implementation for importers +type ImporterConstructorFunc func() Importer + +// New initializes an importer constructor +func (f ImporterConstructorFunc) New() Importer { + return f() +} + +// Importers are the constructors to build importer plugins. +var Importers = make(map[string]Constructor) + +// Register is used to register Constructor implementations. This mechanism allows +// for loose coupling between the configuration and the implementation. It is extremely similar to the way sql.DB +// drivers are configured and used. +func Register(name string, constructor Constructor) { + Importers[name] = constructor +} + +// ImporterBuilderByName returns a Importer constructor for the name provided +func ImporterBuilderByName(name string) (Constructor, error) { + constructor, ok := Importers[name] + if !ok { + return nil, fmt.Errorf("no Importer Constructor for %s", name) + } + + return constructor, nil +} diff --git a/conduit/plugins/metadata.go b/conduit/plugins/metadata.go new file mode 100644 index 000000000..83102329f --- /dev/null +++ b/conduit/plugins/metadata.go @@ -0,0 +1,23 @@ +package plugins + +// PluginType is defined for each plugin category +type PluginType string + +const ( + // Exporter PluginType + Exporter = "exporter" + + // Processor PluginType + Processor = "processors" + + // Importer PluginType + Importer = "importer" +) + +// PluginMetadata provides static per-plugin data +type PluginMetadata interface { + // Type used to differentiate behaviour across plugin categories + Type() PluginType + // Name used to differentiate behavior across plugin implementations + Name() string +} diff --git a/conduit/plugins/processors/all/all.go b/conduit/plugins/processors/all/all.go new file mode 100644 index 000000000..6b8158ff4 --- /dev/null +++ b/conduit/plugins/processors/all/all.go @@ -0,0 +1,8 @@ +package all + +import ( + // Call package wide init function + _ "github.com/algorand/indexer/conduit/plugins/processors/blockprocessor" + _ "github.com/algorand/indexer/conduit/plugins/processors/filterprocessor" + _ "github.com/algorand/indexer/conduit/plugins/processors/noop" +) diff --git a/conduit/plugins/processors/blockprocessor/block_processor.go b/conduit/plugins/processors/blockprocessor/block_processor.go new file mode 100644 index 000000000..66072a463 --- /dev/null +++ b/conduit/plugins/processors/blockprocessor/block_processor.go @@ -0,0 +1,372 @@ +package blockprocessor + +import ( + "context" + _ "embed" // used to embed config + "fmt" + "time" + + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + + "github.com/algorand/go-algorand/agreement" + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/ledger" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/rpcs" + + "github.com/algorand/indexer/accounting" + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/processors" + indexerledger "github.com/algorand/indexer/conduit/plugins/processors/eval" + "github.com/algorand/indexer/data" + "github.com/algorand/indexer/util" +) + +const implementationName = "block_evaluator" + +// BlockProcessor is the block processors interface +type BlockProcessor interface { + NextRoundToProcess() uint64 + + processors.Processor + conduit.Completed +} + +// package-wide init function +func init() { + processors.Register(implementationName, processors.ProcessorConstructorFunc(func() processors.Processor { + return &blockProcessor{} + })) +} + +type blockProcessor struct { + handler func(block *ledgercore.ValidatedBlock) error + ledger *ledger.Ledger + logger *log.Logger + + cfg plugins.PluginConfig + ctx context.Context + + // lastValidatedBlock is the last validated block that was made via the Process() function + // to be used with the OnComplete() function + lastValidatedBlock ledgercore.ValidatedBlock + // lastValidatedBlockRound is the round at which to add the last validated block + lastValidatedBlockRound basics.Round + lastValidatedBlockCertificate agreement.Certificate +} + +//go:embed sample.yaml +var sampleConfig string + +func (proc *blockProcessor) Metadata() conduit.Metadata { + return conduit.Metadata{ + Name: implementationName, + Description: "Local Ledger Block Processor", + Deprecated: false, + SampleConfig: sampleConfig, + } +} + +func (proc *blockProcessor) Config() plugins.PluginConfig { + return proc.cfg +} + +func (proc *blockProcessor) Init(ctx context.Context, initProvider data.InitProvider, cfg plugins.PluginConfig, logger *log.Logger) error { + proc.ctx = ctx + proc.logger = logger + + // First get the configuration from the string + var pCfg Config + err := yaml.Unmarshal([]byte(cfg), &pCfg) + if err != nil { + return fmt.Errorf("blockprocessor init error: %w", err) + } + proc.cfg = cfg + + genesis := initProvider.GetGenesis() + round := uint64(initProvider.NextDBRound()) + + err = InitializeLedger(ctx, proc.logger, round, *genesis, &pCfg) + if err != nil { + return fmt.Errorf("could not initialize ledger: %w", err) + } + + l, err := util.MakeLedger(proc.logger, false, genesis, pCfg.IndexerDatadir) + if err != nil { + return fmt.Errorf("could not make ledger: %w", err) + } + if l == nil { + return fmt.Errorf("ledger was created with nil pointer") + } + proc.ledger = l + + if uint64(l.Latest()) > round { + return fmt.Errorf("the ledger cache is ahead of the required round (%d > %d) and must be re-initialized", l.Latest(), round) + } + + return nil +} + +func (proc *blockProcessor) extractValidatedBlockAndPayset(blockCert *rpcs.EncodedBlockCert) (ledgercore.ValidatedBlock, transactions.Payset, error) { + var vb ledgercore.ValidatedBlock + var payset transactions.Payset + if blockCert == nil { + return vb, payset, fmt.Errorf("cannot process a nil block") + } + if blockCert.Block.Round() == 0 && proc.ledger.Latest() == 0 { + vb = ledgercore.MakeValidatedBlock(blockCert.Block, ledgercore.StateDelta{}) + return vb, blockCert.Block.Payset, nil + } + if blockCert.Block.Round() != (proc.ledger.Latest() + 1) { + return vb, payset, fmt.Errorf("invalid round blockCert.Block.Round(): %d nextRoundToProcess: %d", blockCert.Block.Round(), uint64(proc.ledger.Latest())+1) + } + + // Make sure "AssetCloseAmount" is enabled. If it isn't, override the + // protocol and update the blocks to include transactions with modified + // apply data. + proto, ok := config.Consensus[blockCert.Block.BlockHeader.CurrentProtocol] + if !ok { + return vb, payset, fmt.Errorf( + "cannot find proto version %s", blockCert.Block.BlockHeader.CurrentProtocol) + } + protoChanged := !proto.EnableAssetCloseAmount + proto.EnableAssetCloseAmount = true + + ledgerForEval := indexerledger.MakeLedgerForEvaluator(proc.ledger) + + resources, err := prepareEvalResources(&ledgerForEval, &blockCert.Block) + if err != nil { + proc.logger.Panicf("ProcessBlockCert() resources err: %v", err) + } + + start := time.Now() + delta, payset, err := ledger.EvalForIndexer(ledgerForEval, &blockCert.Block, proto, resources) + if err != nil { + return vb, transactions.Payset{}, fmt.Errorf("eval err: %w", err) + } + EvalTimeSeconds.Observe(time.Since(start).Seconds()) + + // validated block + if protoChanged { + block := bookkeeping.Block{ + BlockHeader: blockCert.Block.BlockHeader, + Payset: payset, + } + vb = ledgercore.MakeValidatedBlock(block, delta) + return vb, payset, nil + } + vb = ledgercore.MakeValidatedBlock(blockCert.Block, delta) + return vb, blockCert.Block.Payset, nil +} + +func (proc *blockProcessor) saveLastValidatedInformation(lastValidatedBlock ledgercore.ValidatedBlock, lastValidatedBlockRound basics.Round, lastValidatedBlockCertificate agreement.Certificate) { + + // Set last validated block for later + proc.lastValidatedBlock = lastValidatedBlock + proc.lastValidatedBlockRound = lastValidatedBlockRound + proc.lastValidatedBlockCertificate = lastValidatedBlockCertificate + +} + +func (proc *blockProcessor) Close() error { + if proc.ledger != nil { + proc.ledger.Close() + } + return nil +} + +// MakeBlockProcessorWithLedger creates a block processorswith a given ledger +func MakeBlockProcessorWithLedger(logger *log.Logger, l *ledger.Ledger, handler func(block *ledgercore.ValidatedBlock) error) (BlockProcessor, error) { + if l == nil { + return nil, fmt.Errorf("MakeBlockProcessorWithLedger() err: local ledger not initialized") + } + err := addGenesisBlock(l, handler) + if err != nil { + return nil, fmt.Errorf("MakeBlockProcessorWithLedger() err: %w", err) + } + return &blockProcessor{logger: logger, ledger: l, handler: handler}, nil +} + +// MakeBlockProcessorWithLedgerInit creates a block processor and initializes the ledger. +func MakeBlockProcessorWithLedgerInit(ctx context.Context, logger *log.Logger, nextDbRound uint64, genesis *bookkeeping.Genesis, config Config, handler func(block *ledgercore.ValidatedBlock) error) (BlockProcessor, error) { + err := InitializeLedger(ctx, logger, nextDbRound, *genesis, &config) + if err != nil { + return nil, fmt.Errorf("MakeBlockProcessorWithLedgerInit() err: %w", err) + } + return MakeBlockProcessor(logger, genesis, nextDbRound, config.IndexerDatadir, handler) +} + +// MakeBlockProcessor creates a block processor +func MakeBlockProcessor(logger *log.Logger, genesis *bookkeeping.Genesis, dbRound uint64, datadir string, handler func(block *ledgercore.ValidatedBlock) error) (BlockProcessor, error) { + l, err := util.MakeLedger(logger, false, genesis, datadir) + if err != nil { + return nil, fmt.Errorf("MakeBlockProcessor() err: %w", err) + } + if uint64(l.Latest()) > dbRound { + return nil, fmt.Errorf("MakeBlockProcessor() err: the ledger cache is ahead of the required round (%d > %d) and must be re-initialized", l.Latest(), dbRound) + } + return MakeBlockProcessorWithLedger(logger, l, handler) +} + +func addGenesisBlock(l *ledger.Ledger, handler func(block *ledgercore.ValidatedBlock) error) error { + if handler != nil && l != nil && l.Latest() == 0 { + blk, err := l.Block(0) + if err != nil { + return fmt.Errorf("addGenesisBlock() err: %w", err) + } + vb := ledgercore.MakeValidatedBlock(blk, ledgercore.StateDelta{}) + err = handler(&vb) + if err != nil { + return fmt.Errorf("addGenesisBlock() handler err: %w", err) + } + } + return nil +} + +func (proc *blockProcessor) Process(input data.BlockData) (data.BlockData, error) { + start := time.Now() + blockCert := input.EncodedBlockCertificate() + + vb, modifiedTxns, err := proc.extractValidatedBlockAndPayset(&blockCert) + + if err != nil { + return data.BlockData{}, fmt.Errorf("processing error: %w", err) + } + + // Set last validated block for later + proc.saveLastValidatedInformation(vb, blockCert.Block.Round(), blockCert.Certificate) + + delta := vb.Delta() + input.Payset = modifiedTxns + input.Delta = &delta + + proc.logger.Debugf("Block processor: processed block %d (%s)", input.Round(), time.Since(start)) + + return input, nil +} + +func (proc *blockProcessor) OnComplete(_ data.BlockData) error { + + if proc.lastValidatedBlockRound == basics.Round(0) { + return nil + } + // write to ledger + err := proc.ledger.AddValidatedBlock(proc.lastValidatedBlock, proc.lastValidatedBlockCertificate) + if err != nil { + return fmt.Errorf("add validated block err: %w", err) + } + + // wait for commit to disk + proc.ledger.WaitForCommit(proc.lastValidatedBlockRound) + return nil + +} + +func (proc *blockProcessor) NextRoundToProcess() uint64 { + return uint64(proc.ledger.Latest()) + 1 +} + +func (proc *blockProcessor) ProvideMetrics() []prometheus.Collector { + return []prometheus.Collector{ + EvalTimeSeconds, + } +} + +// Preload all resources (account data, account resources, asset/app creators) for the +// evaluator. +func prepareEvalResources(l *indexerledger.LedgerForEvaluator, block *bookkeeping.Block) (ledger.EvalForIndexerResources, error) { + assetCreators, appCreators, err := prepareCreators(l, block.Payset) + if err != nil { + return ledger.EvalForIndexerResources{}, + fmt.Errorf("prepareEvalResources() err: %w", err) + } + + res := ledger.EvalForIndexerResources{ + Accounts: nil, + Resources: nil, + Creators: make(map[ledger.Creatable]ledger.FoundAddress), + } + + for index, foundAddress := range assetCreators { + creatable := ledger.Creatable{ + Index: basics.CreatableIndex(index), + Type: basics.AssetCreatable, + } + res.Creators[creatable] = foundAddress + } + for index, foundAddress := range appCreators { + creatable := ledger.Creatable{ + Index: basics.CreatableIndex(index), + Type: basics.AppCreatable, + } + res.Creators[creatable] = foundAddress + } + + res.Accounts, res.Resources, err = prepareAccountsResources(l, block.Payset, assetCreators, appCreators) + if err != nil { + return ledger.EvalForIndexerResources{}, + fmt.Errorf("prepareEvalResources() err: %w", err) + } + + return res, nil +} + +// Preload asset and app creators. +func prepareCreators(l *indexerledger.LedgerForEvaluator, payset transactions.Payset) (map[basics.AssetIndex]ledger.FoundAddress, map[basics.AppIndex]ledger.FoundAddress, error) { + assetsReq, appsReq := accounting.MakePreloadCreatorsRequest(payset) + + assets, err := l.GetAssetCreator(assetsReq) + if err != nil { + return nil, nil, fmt.Errorf("prepareCreators() err: %w", err) + } + apps, err := l.GetAppCreator(appsReq) + if err != nil { + return nil, nil, fmt.Errorf("prepareCreators() err: %w", err) + } + + return assets, apps, nil +} + +// Preload account data and account resources. +func prepareAccountsResources(l *indexerledger.LedgerForEvaluator, payset transactions.Payset, assetCreators map[basics.AssetIndex]ledger.FoundAddress, appCreators map[basics.AppIndex]ledger.FoundAddress) (map[basics.Address]*ledgercore.AccountData, map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource, error) { + addressesReq, resourcesReq := + accounting.MakePreloadAccountsResourcesRequest(payset, assetCreators, appCreators) + + accounts, err := l.LookupWithoutRewards(addressesReq) + if err != nil { + return nil, nil, fmt.Errorf("prepareAccountsResources() err: %w", err) + } + resources, err := l.LookupResources(resourcesReq) + if err != nil { + return nil, nil, fmt.Errorf("prepareAccountsResources() err: %w", err) + } + + return accounts, resources, nil +} + +// MakeBlockProcessorHandlerAdapter makes an adapter function that emulates original behavior of block processor +func MakeBlockProcessorHandlerAdapter(proc *BlockProcessor, handler func(block *ledgercore.ValidatedBlock) error) func(cert *rpcs.EncodedBlockCert) error { + return func(cert *rpcs.EncodedBlockCert) error { + blockData, err := (*proc).Process(data.MakeBlockDataFromEncodedBlockCertificate(cert)) + if err != nil { + return err + } + + vb := blockData.ValidatedBlock() + + if handler != nil { + err = handler(&vb) + if err != nil { + return err + } + } + + return (*proc).OnComplete(blockData) + } +} diff --git a/processor/blockprocessor/block_processor_test.go b/conduit/plugins/processors/blockprocessor/block_processor_test.go similarity index 75% rename from processor/blockprocessor/block_processor_test.go rename to conduit/plugins/processors/blockprocessor/block_processor_test.go index 316234cd7..f1500b0e4 100644 --- a/processor/blockprocessor/block_processor_test.go +++ b/conduit/plugins/processors/blockprocessor/block_processor_test.go @@ -15,8 +15,7 @@ import ( "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/rpcs" - "github.com/algorand/indexer/idb" - "github.com/algorand/indexer/processor/blockprocessor" + "github.com/algorand/indexer/conduit/plugins/processors/blockprocessor" "github.com/algorand/indexer/util/test" ) @@ -32,7 +31,8 @@ func TestProcess(t *testing.T) { genesisBlock, err := l.Block(basics.Round(0)) assert.Nil(t, err) // create processor - pr, _ := blockprocessor.MakeProcessorWithLedger(logger, l, noopHandler) + pr, _ := blockprocessor.MakeBlockProcessorWithLedger(logger, l, noopHandler) + proc := blockprocessor.MakeBlockProcessorHandlerAdapter(&pr, noopHandler) prevHeader := genesisBlock.BlockHeader assert.Equal(t, basics.Round(0), l.Latest()) // create a few rounds @@ -41,7 +41,7 @@ func TestProcess(t *testing.T) { block, err := test.MakeBlockForTxns(prevHeader, &txn) assert.Nil(t, err) rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) assert.Nil(t, err) // check round assert.Equal(t, basics.Round(i), l.Latest()) @@ -61,12 +61,13 @@ func TestFailedProcess(t *testing.T) { require.NoError(t, err) defer l.Close() // invalid processor - pr, err := blockprocessor.MakeProcessorWithLedger(logger, nil, nil) - assert.Contains(t, err.Error(), "MakeProcessorWithLedger() err: local ledger not initialized") - pr, err = blockprocessor.MakeProcessorWithLedger(logger, l, nil) + pr, err := blockprocessor.MakeBlockProcessorWithLedger(logger, nil, nil) + assert.Contains(t, err.Error(), "local ledger not initialized") + pr, err = blockprocessor.MakeBlockProcessorWithLedger(logger, l, nil) + proc := blockprocessor.MakeBlockProcessorHandlerAdapter(&pr, nil) + assert.Nil(t, err) + err = proc(nil) assert.Nil(t, err) - err = pr.Process(nil) - assert.Contains(t, err.Error(), "Process(): cannot process a nil block") genesisBlock, err := l.Block(basics.Round(0)) assert.Nil(t, err) @@ -76,15 +77,15 @@ func TestFailedProcess(t *testing.T) { block.BlockHeader.Round = 10 assert.Nil(t, err) rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) - assert.Contains(t, err.Error(), "Process() invalid round blockCert.Block.Round()") + err = proc(&rawBlock) + assert.Contains(t, err.Error(), "invalid round blockCert.Block.Round()") // non-zero balance after close remainder to sender address txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, test.AccountA, test.AccountA) block, err = test.MakeBlockForTxns(genesisBlock.BlockHeader, &txn) assert.Nil(t, err) rawBlock = rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) assert.Contains(t, err.Error(), "ProcessBlockForIndexer() err") // stxn GenesisID not empty @@ -93,7 +94,7 @@ func TestFailedProcess(t *testing.T) { assert.Nil(t, err) block.Payset[0].Txn.GenesisID = "genesisID" rawBlock = rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) assert.Contains(t, err.Error(), "ProcessBlockForIndexer() err") // eval error: concensus protocol not supported @@ -102,23 +103,33 @@ func TestFailedProcess(t *testing.T) { block.BlockHeader.CurrentProtocol = "testing" assert.Nil(t, err) rawBlock = rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) - assert.Contains(t, err.Error(), "Process() cannot find proto version testing") + err = proc(&rawBlock) + assert.Contains(t, err.Error(), "cannot find proto version testing") - // handler error + // handler error, errors out if this is set to true + throwError := true handler := func(vb *ledgercore.ValidatedBlock) error { - return fmt.Errorf("handler error") + if throwError { + return fmt.Errorf("handler error") + } + return nil } - _, err = blockprocessor.MakeProcessorWithLedger(logger, l, handler) + _, err = blockprocessor.MakeBlockProcessorWithLedger(logger, l, handler) assert.Contains(t, err.Error(), "handler error") - pr, _ = blockprocessor.MakeProcessorWithLedger(logger, l, nil) + // We don't want it to throw an error when we create the ledger but after + throwError = false + pr, err = blockprocessor.MakeBlockProcessorWithLedger(logger, l, handler) + proc = blockprocessor.MakeBlockProcessorHandlerAdapter(&pr, handler) + assert.NotNil(t, pr) + assert.NoError(t, err) + // enable this so it will throw an error when we process the block + throwError = true txn = test.MakePaymentTxn(0, 10, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err = test.MakeBlockForTxns(genesisBlock.BlockHeader, &txn) assert.Nil(t, err) - pr.SetHandler(handler) rawBlock = rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) - assert.Contains(t, err.Error(), "Process() handler err") + err = proc(&rawBlock) + assert.Contains(t, err.Error(), "handler error") } // TestMakeProcessorWithLedgerInit_CatchpointErrors verifies that the catchpoint error handling works properly. @@ -154,13 +165,13 @@ func TestMakeProcessorWithLedgerInit_CatchpointErrors(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - _, err := blockprocessor.MakeProcessorWithLedgerInit( + config := blockprocessor.Config{Catchpoint: tc.catchpoint} + _, err := blockprocessor.MakeBlockProcessorWithLedgerInit( context.Background(), logger, - tc.catchpoint, - &genesis, tc.round, - idb.IndexerDbOptions{}, + &genesis, + config, noopHandler) require.ErrorContains(t, err, tc.errMsg) }) diff --git a/conduit/plugins/processors/blockprocessor/config.go b/conduit/plugins/processors/blockprocessor/config.go new file mode 100644 index 000000000..96d1a56e5 --- /dev/null +++ b/conduit/plugins/processors/blockprocessor/config.go @@ -0,0 +1,12 @@ +package blockprocessor + +// Config configuration for a block processor +type Config struct { + // Catchpoint to initialize the local ledger to + Catchpoint string `yaml:"catchpoint"` + + IndexerDatadir string `yaml:"data-dir"` + AlgodDataDir string `yaml:"algod-data-dir"` + AlgodToken string `yaml:"algod-token"` + AlgodAddr string `yaml:"algod-addr"` +} diff --git a/conduit/plugins/processors/blockprocessor/config_test.go b/conduit/plugins/processors/blockprocessor/config_test.go new file mode 100644 index 000000000..7d82cd07c --- /dev/null +++ b/conduit/plugins/processors/blockprocessor/config_test.go @@ -0,0 +1,29 @@ +package blockprocessor + +import ( + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestConfigDeserialize(t *testing.T) { + + configStr := `--- + catchpoint: "acatch" + data-dir: "idx_data_dir" + algod-data-dir: "algod_data_dir" + algod-token: "algod_token" + algod-addr: "algod_addr" + ` + + var processorConfig Config + err := yaml.Unmarshal([]byte(configStr), &processorConfig) + require.Nil(t, err) + require.Equal(t, processorConfig.Catchpoint, "acatch") + require.Equal(t, processorConfig.IndexerDatadir, "idx_data_dir") + require.Equal(t, processorConfig.AlgodDataDir, "algod_data_dir") + require.Equal(t, processorConfig.AlgodToken, "algod_token") + require.Equal(t, processorConfig.AlgodAddr, "algod_addr") + +} diff --git a/processor/blockprocessor/initialize.go b/conduit/plugins/processors/blockprocessor/initialize.go similarity index 66% rename from processor/blockprocessor/initialize.go rename to conduit/plugins/processors/blockprocessor/initialize.go index 59af78bd6..c780e3282 100644 --- a/processor/blockprocessor/initialize.go +++ b/conduit/plugins/processors/blockprocessor/initialize.go @@ -8,36 +8,34 @@ import ( log "github.com/sirupsen/logrus" - "github.com/algorand/indexer/fetcher" - "github.com/algorand/indexer/idb" - "github.com/algorand/indexer/processor" - "github.com/algorand/indexer/processor/blockprocessor/internal" - "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/rpcs" + + "github.com/algorand/indexer/conduit/plugins/processors/blockprocessor/internal" + "github.com/algorand/indexer/fetcher" ) // InitializeLedger will initialize a ledger to the directory given by the // IndexerDbOpts. // nextRound - next round to process after initializing. // catchpoint - if provided, attempt to use fast catchup. -func InitializeLedger(ctx context.Context, logger *log.Logger, catchpoint string, nextRound uint64, genesis bookkeeping.Genesis, opts *idb.IndexerDbOptions) error { - if nextRound > 0 { - if catchpoint != "" { - round, _, err := ledgercore.ParseCatchpointLabel(catchpoint) +func InitializeLedger(ctx context.Context, logger *log.Logger, nextDbRound uint64, genesis bookkeeping.Genesis, config *Config) error { + if nextDbRound > 0 { + if config.Catchpoint != "" { + round, _, err := ledgercore.ParseCatchpointLabel(config.Catchpoint) if err != nil { return fmt.Errorf("InitializeLedger() label err: %w", err) } - if uint64(round) >= nextRound { - return fmt.Errorf("invalid catchpoint: catchpoint round %d should not be ahead of target round %d", uint64(round), nextRound-1) + if uint64(round) >= nextDbRound { + return fmt.Errorf("invalid catchpoint: catchpoint round %d should not be ahead of target round %d", uint64(round), nextDbRound-1) } - err = InitializeLedgerFastCatchup(ctx, logger, catchpoint, opts.IndexerDatadir, genesis) + err = InitializeLedgerFastCatchup(ctx, logger, config.Catchpoint, config.IndexerDatadir, genesis) if err != nil { return fmt.Errorf("InitializeLedger() fast catchup err: %w", err) } } - err := InitializeLedgerSimple(ctx, logger, nextRound-1, &genesis, opts) + err := InitializeLedgerSimple(ctx, logger, nextDbRound-1, &genesis, config) if err != nil { return fmt.Errorf("InitializeLedger() simple catchup err: %w", err) } @@ -60,22 +58,22 @@ func InitializeLedgerFastCatchup(ctx context.Context, logger *log.Logger, catchp // InitializeLedgerSimple initializes a ledger with the block processor by // sending it one block at a time and letting it update the ledger as usual. -func InitializeLedgerSimple(ctx context.Context, logger *log.Logger, round uint64, genesis *bookkeeping.Genesis, opts *idb.IndexerDbOptions) error { +func InitializeLedgerSimple(ctx context.Context, logger *log.Logger, round uint64, genesis *bookkeeping.Genesis, config *Config) error { ctx, cf := context.WithCancel(ctx) defer cf() var bot fetcher.Fetcher var err error - if opts.IndexerDatadir == "" { + if config.IndexerDatadir == "" { return fmt.Errorf("InitializeLedgerSimple() err: indexer data directory missing") } // create algod client - bot, err = getFetcher(logger, opts) + bot, err = getFetcher(logger, config) if err != nil { return fmt.Errorf("InitializeLedgerSimple() err: %w", err) } logger.Info("initializing ledger") - proc, err := MakeProcessor(logger, genesis, round, opts.IndexerDatadir, nil) + proc, err := MakeBlockProcessor(logger, genesis, round, config.IndexerDatadir, nil) if err != nil { return fmt.Errorf("RunMigration() err: %w", err) } @@ -84,7 +82,10 @@ func InitializeLedgerSimple(ctx context.Context, logger *log.Logger, round uint6 return nil } bot.SetNextRound(proc.NextRoundToProcess()) - handler := blockHandler(logger, round, proc, cf, 1*time.Second) + + procHandler := MakeBlockProcessorHandlerAdapter(&proc, nil) + + handler := blockHandler(logger, round, procHandler, cf, 1*time.Second) bot.SetBlockHandler(handler) logger.Info("Starting ledger migration.") @@ -101,10 +102,10 @@ func InitializeLedgerSimple(ctx context.Context, logger *log.Logger, round uint6 // blockHandler creates a handler complying to the fetcher block handler interface. In case of a failure it keeps // attempting to add the block until the fetcher shuts down. -func blockHandler(logger *log.Logger, dbRound uint64, proc processor.Processor, cancel context.CancelFunc, retryDelay time.Duration) func(context.Context, *rpcs.EncodedBlockCert) error { +func blockHandler(logger *log.Logger, dbRound uint64, procHandler func(block *rpcs.EncodedBlockCert) error, cancel context.CancelFunc, retryDelay time.Duration) func(context.Context, *rpcs.EncodedBlockCert) error { return func(ctx context.Context, block *rpcs.EncodedBlockCert) error { for { - err := handleBlock(logger, block, proc) + err := handleBlock(logger, block, procHandler) if err == nil { if uint64(block.Block.Round()) == dbRound { // migration completes @@ -125,27 +126,28 @@ func blockHandler(logger *log.Logger, dbRound uint64, proc processor.Processor, } } -func handleBlock(logger *log.Logger, block *rpcs.EncodedBlockCert, proc processor.Processor) error { - err := proc.Process(block) +func handleBlock(logger *log.Logger, block *rpcs.EncodedBlockCert, procHandler func(block *rpcs.EncodedBlockCert) error) error { + start := time.Now() + err := procHandler(block) if err != nil { logger.WithError(err).Errorf( "block %d import failed", block.Block.Round()) return fmt.Errorf("handleBlock() err: %w", err) } - logger.Infof("Initialize Ledger: added block %d to ledger", block.Block.Round()) + logger.Infof("Initialize Ledger: added block %d to ledger (%s)", block.Block.Round(), time.Since(start)) return nil } -func getFetcher(logger *log.Logger, opts *idb.IndexerDbOptions) (fetcher.Fetcher, error) { +func getFetcher(logger *log.Logger, config *Config) (fetcher.Fetcher, error) { var err error var bot fetcher.Fetcher - if opts.AlgodDataDir != "" { - bot, err = fetcher.ForDataDir(opts.AlgodDataDir, logger) + if config.AlgodDataDir != "" { + bot, err = fetcher.ForDataDir(config.AlgodDataDir, logger) if err != nil { return nil, fmt.Errorf("InitializeLedgerFastCatchup() err: %w", err) } - } else if opts.AlgodAddr != "" && opts.AlgodToken != "" { - bot, err = fetcher.ForNetAndToken(opts.AlgodAddr, opts.AlgodToken, logger) + } else if config.AlgodAddr != "" && config.AlgodToken != "" { + bot, err = fetcher.ForNetAndToken(config.AlgodAddr, config.AlgodToken, logger) if err != nil { return nil, fmt.Errorf("InitializeLedgerFastCatchup() err: %w", err) } diff --git a/processor/blockprocessor/initialize_test.go b/conduit/plugins/processors/blockprocessor/initialize_test.go similarity index 92% rename from processor/blockprocessor/initialize_test.go rename to conduit/plugins/processors/blockprocessor/initialize_test.go index ba34d8ab7..244a18c6c 100644 --- a/processor/blockprocessor/initialize_test.go +++ b/conduit/plugins/processors/blockprocessor/initialize_test.go @@ -19,8 +19,7 @@ import ( "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/rpcs" - "github.com/algorand/indexer/idb" - "github.com/algorand/indexer/processor/blockprocessor/internal" + "github.com/algorand/indexer/conduit/plugins/processors/blockprocessor/internal" "github.com/algorand/indexer/util" "github.com/algorand/indexer/util/test" ) @@ -61,7 +60,7 @@ func TestRunMigration(t *testing.T) { dname, err := os.MkdirTemp("", "indexer") defer os.RemoveAll(dname) - opts := idb.IndexerDbOptions{ + config := Config{ IndexerDatadir: dname, AlgodAddr: "localhost", AlgodToken: "AAAAA", @@ -69,19 +68,19 @@ func TestRunMigration(t *testing.T) { // migrate 3 rounds log, _ := test2.NewNullLogger() - err = InitializeLedgerSimple(context.Background(), log, 3, &genesis, &opts) + err = InitializeLedgerSimple(context.Background(), log, 3, &genesis, &config) assert.NoError(t, err) - l, err := util.MakeLedger(log, false, &genesis, opts.IndexerDatadir) + l, err := util.MakeLedger(log, false, &genesis, config.IndexerDatadir) assert.NoError(t, err) // check 3 rounds written to ledger assert.Equal(t, uint64(3), uint64(l.Latest())) l.Close() // migration continues from last round - err = InitializeLedgerSimple(context.Background(), log, 6, &genesis, &opts) + err = InitializeLedgerSimple(context.Background(), log, 6, &genesis, &config) assert.NoError(t, err) - l, err = util.MakeLedger(log, false, &genesis, opts.IndexerDatadir) + l, err = util.MakeLedger(log, false, &genesis, config.IndexerDatadir) assert.NoError(t, err) assert.Equal(t, uint64(6), uint64(l.Latest())) l.Close() diff --git a/processor/blockprocessor/internal/catchupservice.go b/conduit/plugins/processors/blockprocessor/internal/catchupservice.go similarity index 100% rename from processor/blockprocessor/internal/catchupservice.go rename to conduit/plugins/processors/blockprocessor/internal/catchupservice.go diff --git a/processor/blockprocessor/internal/catchupservice_test.go b/conduit/plugins/processors/blockprocessor/internal/catchupservice_test.go similarity index 100% rename from processor/blockprocessor/internal/catchupservice_test.go rename to conduit/plugins/processors/blockprocessor/internal/catchupservice_test.go diff --git a/conduit/plugins/processors/blockprocessor/metrics.go b/conduit/plugins/processors/blockprocessor/metrics.go new file mode 100644 index 000000000..84051123c --- /dev/null +++ b/conduit/plugins/processors/blockprocessor/metrics.go @@ -0,0 +1,18 @@ +package blockprocessor + +import "github.com/prometheus/client_golang/prometheus" + +// Prometheus metric names +const ( + EvalTimeName = "eval_time_sec" +) + +// Initialize the prometheus objects +var ( + EvalTimeSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Subsystem: "indexer_daemon", + Name: EvalTimeName, + Help: "Time spent calling Eval function in seconds.", + }) +) diff --git a/conduit/plugins/processors/blockprocessor/sample.yaml b/conduit/plugins/processors/blockprocessor/sample.yaml new file mode 100644 index 000000000..070f4bab9 --- /dev/null +++ b/conduit/plugins/processors/blockprocessor/sample.yaml @@ -0,0 +1,15 @@ +name: block_evaluator +config: + # Catchpoint is the catchpoint you wish the block evaluator to use + catchpoint: "acatch" + # DataDir is the data directory for the Indexer process. It should be a full directory + # path and is used for reading in config and writing some output. See + # Algorand docs for further documentation + data-dir: "/tmp" + # AlgodDataDir is the algod data directory. It should be a full directory path. + # See Algorand docs for further documentation + algod-data-dir: "algod_data_dir" + # AlgodToken is the token to connect to algod. + algod-token: "algod_token" + # AlgodAddr is the address to connect to the algod instance. + algod-addr: "algod_addr" diff --git a/processor/eval/ledger_for_evaluator.go b/conduit/plugins/processors/eval/ledger_for_evaluator.go similarity index 100% rename from processor/eval/ledger_for_evaluator.go rename to conduit/plugins/processors/eval/ledger_for_evaluator.go diff --git a/processor/eval/ledger_for_evaluator_test.go b/conduit/plugins/processors/eval/ledger_for_evaluator_test.go similarity index 91% rename from processor/eval/ledger_for_evaluator_test.go rename to conduit/plugins/processors/eval/ledger_for_evaluator_test.go index 106d373f9..dc179d18e 100644 --- a/processor/eval/ledger_for_evaluator_test.go +++ b/conduit/plugins/processors/eval/ledger_for_evaluator_test.go @@ -17,8 +17,8 @@ import ( "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/rpcs" - block_processor "github.com/algorand/indexer/processor/blockprocessor" - indxLedger "github.com/algorand/indexer/processor/eval" + block_processor "github.com/algorand/indexer/conduit/plugins/processors/blockprocessor" + indxLedger "github.com/algorand/indexer/conduit/plugins/processors/eval" "github.com/algorand/indexer/util/test" ) @@ -33,13 +33,14 @@ func TestLedgerForEvaluatorLatestBlockHdr(t *testing.T) { l := makeTestLedger(t) defer l.Close() logger, _ := test2.NewNullLogger() - pr, _ := block_processor.MakeProcessorWithLedger(logger, l, nil) + pr, _ := block_processor.MakeBlockProcessorWithLedger(logger, l, nil) + proc := block_processor.MakeBlockProcessorHandlerAdapter(&pr, nil) txn := test.MakePaymentTxn(0, 100, 0, 1, 1, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) assert.Nil(t, err) rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) assert.Nil(t, err) ld := indxLedger.MakeLedgerForEvaluator(l) @@ -55,7 +56,7 @@ func TestLedgerForEvaluatorAccountDataBasic(t *testing.T) { l := makeTestLedger(t) defer l.Close() logger, _ := test2.NewNullLogger() - block_processor.MakeProcessorWithLedger(logger, l, nil) + block_processor.MakeBlockProcessorWithLedger(logger, l, nil) accountData, _, err := l.LookupWithoutRewards(0, test.AccountB) require.NoError(t, err) @@ -91,7 +92,8 @@ func TestLedgerForEvaluatorAsset(t *testing.T) { l := makeTestLedger(t) defer l.Close() logger, _ := test2.NewNullLogger() - pr, _ := block_processor.MakeProcessorWithLedger(logger, l, nil) + pr, _ := block_processor.MakeBlockProcessorWithLedger(logger, l, nil) + proc := block_processor.MakeBlockProcessorHandlerAdapter(&pr, nil) txn0 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountA) txn1 := test.MakeAssetConfigTxn(0, 4, 0, false, "", "", "", test.AccountA) @@ -102,7 +104,7 @@ func TestLedgerForEvaluatorAsset(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2, &txn3, &txn4) assert.Nil(t, err) rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) assert.Nil(t, err) ld := indxLedger.MakeLedgerForEvaluator(l) @@ -156,7 +158,8 @@ func TestLedgerForEvaluatorApp(t *testing.T) { l := makeTestLedger(t) defer l.Close() logger, _ := test2.NewNullLogger() - pr, _ := block_processor.MakeProcessorWithLedger(logger, l, nil) + pr, _ := block_processor.MakeBlockProcessorWithLedger(logger, l, nil) + proc := block_processor.MakeBlockProcessorHandlerAdapter(&pr, nil) txn0 := test.MakeSimpleAppCallTxn(0, test.AccountA) txn1 := test.MakeAppCallTxnWithLogs(0, test.AccountA, []string{"testing"}) @@ -168,7 +171,7 @@ func TestLedgerForEvaluatorApp(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2, &txn3, &txn4, &txn5) assert.Nil(t, err) rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) assert.Nil(t, err) ld := indxLedger.MakeLedgerForEvaluator(l) @@ -238,7 +241,8 @@ func TestLedgerForEvaluatorFetchAllResourceTypes(t *testing.T) { l := makeTestLedger(t) defer l.Close() logger, _ := test2.NewNullLogger() - pr, _ := block_processor.MakeProcessorWithLedger(logger, l, nil) + pr, _ := block_processor.MakeBlockProcessorWithLedger(logger, l, nil) + proc := block_processor.MakeBlockProcessorHandlerAdapter(&pr, nil) txn0 := test.MakeSimpleAppCallTxn(0, test.AccountA) txn1 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountA) @@ -246,7 +250,7 @@ func TestLedgerForEvaluatorFetchAllResourceTypes(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1) assert.Nil(t, err) rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) assert.Nil(t, err) ld := indxLedger.MakeLedgerForEvaluator(l) @@ -288,7 +292,7 @@ func TestLedgerForEvaluatorLookupMultipleAccounts(t *testing.T) { l := makeTestLedger(t) defer l.Close() logger, _ := test2.NewNullLogger() - block_processor.MakeProcessorWithLedger(logger, l, nil) + block_processor.MakeBlockProcessorWithLedger(logger, l, nil) addresses := []basics.Address{ test.AccountA, test.AccountB, test.AccountC, test.AccountD} @@ -317,14 +321,15 @@ func TestLedgerForEvaluatorAssetCreatorBasic(t *testing.T) { l := makeTestLedger(t) defer l.Close() logger, _ := test2.NewNullLogger() - pr, _ := block_processor.MakeProcessorWithLedger(logger, l, nil) + pr, _ := block_processor.MakeBlockProcessorWithLedger(logger, l, nil) + proc := block_processor.MakeBlockProcessorHandlerAdapter(&pr, nil) txn0 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountA) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0) assert.Nil(t, err) rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) assert.Nil(t, err) ld := indxLedger.MakeLedgerForEvaluator(l) @@ -348,7 +353,8 @@ func TestLedgerForEvaluatorAssetCreatorDeleted(t *testing.T) { l := makeTestLedger(t) defer l.Close() logger, _ := test2.NewNullLogger() - pr, _ := block_processor.MakeProcessorWithLedger(logger, l, nil) + pr, _ := block_processor.MakeBlockProcessorWithLedger(logger, l, nil) + proc := block_processor.MakeBlockProcessorHandlerAdapter(&pr, nil) txn0 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountA) txn1 := test.MakeAssetDestroyTxn(1, test.AccountA) @@ -356,7 +362,7 @@ func TestLedgerForEvaluatorAssetCreatorDeleted(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1) assert.Nil(t, err) rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) assert.Nil(t, err) ld := indxLedger.MakeLedgerForEvaluator(l) @@ -377,7 +383,8 @@ func TestLedgerForEvaluatorAssetCreatorMultiple(t *testing.T) { l := makeTestLedger(t) defer l.Close() logger, _ := test2.NewNullLogger() - pr, _ := block_processor.MakeProcessorWithLedger(logger, l, nil) + pr, _ := block_processor.MakeBlockProcessorWithLedger(logger, l, nil) + proc := block_processor.MakeBlockProcessorHandlerAdapter(&pr, nil) txn0 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountA) txn1 := test.MakeAssetConfigTxn(0, 2, 0, false, "", "", "", test.AccountB) @@ -387,7 +394,7 @@ func TestLedgerForEvaluatorAssetCreatorMultiple(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2, &txn3) assert.Nil(t, err) rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) assert.Nil(t, err) ld := indxLedger.MakeLedgerForEvaluator(l) @@ -431,14 +438,15 @@ func TestLedgerForEvaluatorAppCreatorBasic(t *testing.T) { l := makeTestLedger(t) defer l.Close() logger, _ := test2.NewNullLogger() - pr, _ := block_processor.MakeProcessorWithLedger(logger, l, nil) + pr, _ := block_processor.MakeBlockProcessorWithLedger(logger, l, nil) + proc := block_processor.MakeBlockProcessorHandlerAdapter(&pr, nil) txn0 := test.MakeSimpleAppCallTxn(0, test.AccountA) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0) assert.Nil(t, err) rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) assert.Nil(t, err) ld := indxLedger.MakeLedgerForEvaluator(l) @@ -464,7 +472,8 @@ func TestLedgerForEvaluatorAppCreatorDeleted(t *testing.T) { l := makeTestLedger(t) defer l.Close() logger, _ := test2.NewNullLogger() - pr, _ := block_processor.MakeProcessorWithLedger(logger, l, nil) + pr, _ := block_processor.MakeBlockProcessorWithLedger(logger, l, nil) + proc := block_processor.MakeBlockProcessorHandlerAdapter(&pr, nil) txn0 := test.MakeSimpleAppCallTxn(0, test.AccountA) txn1 := test.MakeAppDestroyTxn(1, test.AccountA) @@ -472,7 +481,7 @@ func TestLedgerForEvaluatorAppCreatorDeleted(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1) assert.Nil(t, err) rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) assert.Nil(t, err) ld := indxLedger.MakeLedgerForEvaluator(l) @@ -493,7 +502,8 @@ func TestLedgerForEvaluatorAppCreatorMultiple(t *testing.T) { l := makeTestLedger(t) defer l.Close() logger, _ := test2.NewNullLogger() - pr, _ := block_processor.MakeProcessorWithLedger(logger, l, nil) + pr, _ := block_processor.MakeBlockProcessorWithLedger(logger, l, nil) + proc := block_processor.MakeBlockProcessorHandlerAdapter(&pr, nil) txn0 := test.MakeSimpleAppCallTxn(0, test.AccountA) txn1 := test.MakeSimpleAppCallTxn(0, test.AccountB) @@ -503,7 +513,7 @@ func TestLedgerForEvaluatorAppCreatorMultiple(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2, &txn3) assert.Nil(t, err) rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) assert.Nil(t, err) ld := indxLedger.MakeLedgerForEvaluator(l) @@ -586,7 +596,8 @@ func compareAppBoxesAgainstLedger(t *testing.T, ld indxLedger.LedgerForEvaluator func TestLedgerForEvaluatorLookupKv(t *testing.T) { logger, _ := test2.NewNullLogger() l := makeTestLedger(t) - pr, _ := block_processor.MakeProcessorWithLedger(logger, l, nil) + pr, _ := block_processor.MakeBlockProcessorWithLedger(logger, l, nil) + proc := block_processor.MakeBlockProcessorHandlerAdapter(&pr, nil) ld := indxLedger.MakeLedgerForEvaluator(l) defer l.Close() defer ld.Close() @@ -606,7 +617,7 @@ func TestLedgerForEvaluatorLookupKv(t *testing.T) { require.NoError(t, err) rawBlock := rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) require.NoError(t, err) ret, err := ld.LookupKv(currentRound, "sanity check") @@ -648,7 +659,7 @@ func TestLedgerForEvaluatorLookupKv(t *testing.T) { require.NoError(t, err) rawBlock = rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) require.NoError(t, err) compareAppBoxesAgainstLedger(t, ld, currentRound, expectedAppBoxes) @@ -685,7 +696,7 @@ func TestLedgerForEvaluatorLookupKv(t *testing.T) { require.NoError(t, err) rawBlock = rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) require.NoError(t, err) compareAppBoxesAgainstLedger(t, ld, currentRound, expectedAppBoxes) @@ -716,7 +727,7 @@ func TestLedgerForEvaluatorLookupKv(t *testing.T) { require.NoError(t, err) rawBlock = rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) require.NoError(t, err) deletedBoxes := make(map[basics.AppIndex]map[string]bool) @@ -752,7 +763,7 @@ func TestLedgerForEvaluatorLookupKv(t *testing.T) { require.NoError(t, err) rawBlock = rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) require.NoError(t, err) compareAppBoxesAgainstLedger(t, ld, currentRound, expectedAppBoxes) @@ -782,7 +793,7 @@ func TestLedgerForEvaluatorLookupKv(t *testing.T) { require.NoError(t, err) rawBlock = rpcs.EncodedBlockCert{Block: block, Certificate: agreement.Certificate{}} - err = pr.Process(&rawBlock) + err = proc(&rawBlock) require.NoError(t, err) compareAppBoxesAgainstLedger(t, ld, currentRound, expectedAppBoxes, deletedBoxes) diff --git a/conduit/plugins/processors/filterprocessor/config.go b/conduit/plugins/processors/filterprocessor/config.go new file mode 100644 index 000000000..1d6362c20 --- /dev/null +++ b/conduit/plugins/processors/filterprocessor/config.go @@ -0,0 +1,21 @@ +package filterprocessor + +import ( + "github.com/algorand/indexer/conduit/plugins/processors/filterprocessor/expression" +) + +// SubConfig is the configuration needed for each additional filter +type SubConfig struct { + // FilterTag the tag of the struct to analyze + FilterTag string `yaml:"tag"` + // ExpressionType the type of expression to search for (i.e. "exact" or "regex") + ExpressionType expression.FilterType `yaml:"expression-type"` + // Expression the expression to search + Expression string `yaml:"expression"` +} + +// Config configuration for the filter processor +type Config struct { + // Filters are a list of SubConfig objects with an operation acting as the string key in the map + Filters []map[string][]SubConfig `yaml:"filters"` +} diff --git a/conduit/plugins/processors/filterprocessor/expression/expression.go b/conduit/plugins/processors/filterprocessor/expression/expression.go new file mode 100644 index 000000000..f06e2bfe5 --- /dev/null +++ b/conduit/plugins/processors/filterprocessor/expression/expression.go @@ -0,0 +1,152 @@ +package expression + +import ( + "fmt" + "reflect" + "regexp" + "strconv" + + "github.com/algorand/go-algorand/data/basics" +) + +// FilterType is the type of the filter (i.e. const, regex, etc) +type FilterType string + +const ( + // ExactFilter a filter that looks at a constant string in its entirety + ExactFilter FilterType = "exact" + // RegexFilter a filter that applies regex rules to the matching + RegexFilter FilterType = "regex" + + // LessThanFilter a filter that applies numerical less than operation + LessThanFilter FilterType = "less-than" + // LessThanEqualFilter a filter that applies numerical less than or equal operation + LessThanEqualFilter FilterType = "less-than-equal" + // GreaterThanFilter a filter that applies numerical greater than operation + GreaterThanFilter FilterType = "greater-than" + // GreaterThanEqualFilter a filter that applies numerical greater than or equal operation + GreaterThanEqualFilter FilterType = "greater-than-equal" + // EqualToFilter a filter that applies numerical equal to operation + EqualToFilter FilterType = "equal" + // NotEqualToFilter a filter that applies numerical NOT equal to operation + NotEqualToFilter FilterType = "not-equal" +) + +// TypeToFunctionMap maps the expression-type with the needed function for the expression. +// For instance the exact or regex expression-type might need the String() function +// A blank string means a function is not required +// Can't make this const because there are no constant maps in go... +var TypeToFunctionMap = map[FilterType]string{ + ExactFilter: "String", + RegexFilter: "String", + LessThanFilter: "", + LessThanEqualFilter: "", + GreaterThanFilter: "", + GreaterThanEqualFilter: "", + EqualToFilter: "", + NotEqualToFilter: "", +} + +// Expression the expression interface +type Expression interface { + Search(input interface{}) (bool, error) +} + +type regexExpression struct { + Regex *regexp.Regexp +} + +func (e regexExpression) Search(input interface{}) (bool, error) { + return e.Regex.MatchString(input.(string)), nil +} + +// MakeExpression creates an expression based on an expression type +func MakeExpression(expressionType FilterType, expressionSearchStr string, targetKind reflect.Kind) (*Expression, error) { + switch expressionType { + case ExactFilter: + { + r, err := regexp.Compile("^" + expressionSearchStr + "$") + if err != nil { + return nil, err + } + + var exp Expression = regexExpression{Regex: r} + return &exp, nil + } + case RegexFilter: + { + r, err := regexp.Compile(expressionSearchStr) + if err != nil { + return nil, err + } + + var exp Expression = regexExpression{Regex: r} + return &exp, nil + } + + case LessThanFilter: + fallthrough + case LessThanEqualFilter: + fallthrough + case GreaterThanFilter: + fallthrough + case GreaterThanEqualFilter: + fallthrough + case EqualToFilter: + fallthrough + case NotEqualToFilter: + { + var exp Expression + + switch targetKind { + case reflect.TypeOf(basics.MicroAlgos{}).Kind(): + { + v, err := strconv.ParseUint(expressionSearchStr, 10, 64) + if err != nil { + return nil, err + } + + exp = microAlgoExpression{ + FilterValue: basics.MicroAlgos{Raw: v}, + Op: expressionType, + } + } + + case reflect.Int64: + { + v, err := strconv.ParseInt(expressionSearchStr, 10, 64) + if err != nil { + return nil, err + } + + exp = int64NumericalExpression{ + FilterValue: v, + Op: expressionType, + } + } + + case reflect.Uint64: + { + v, err := strconv.ParseUint(expressionSearchStr, 10, 64) + if err != nil { + return nil, err + } + + exp = uint64NumericalExpression{ + FilterValue: v, + Op: expressionType, + } + } + + default: + return nil, fmt.Errorf("unknown target kind (%s) for filter type: %s", targetKind.String(), expressionType) + } + + return &exp, nil + + } + + default: + return nil, fmt.Errorf("unknown expression type: %s", expressionType) + } +} diff --git a/conduit/plugins/processors/filterprocessor/expression/numerical_expressions.go b/conduit/plugins/processors/filterprocessor/expression/numerical_expressions.go new file mode 100644 index 000000000..42f969925 --- /dev/null +++ b/conduit/plugins/processors/filterprocessor/expression/numerical_expressions.go @@ -0,0 +1,108 @@ +package expression + +import ( + "fmt" + "reflect" + + "github.com/algorand/go-algorand/data/basics" +) + +type microAlgoExpression struct { + FilterValue basics.MicroAlgos + Op FilterType +} + +func (m microAlgoExpression) Search(input interface{}) (bool, error) { + + inputValue, ok := input.(basics.MicroAlgos) + if !ok { + return false, fmt.Errorf("supplied type (%s) was not microalgos", reflect.TypeOf(input).String()) + } + + switch m.Op { + case LessThanFilter: + return inputValue.Raw < m.FilterValue.Raw, nil + case LessThanEqualFilter: + return inputValue.Raw <= m.FilterValue.Raw, nil + case EqualToFilter: + return inputValue.Raw == m.FilterValue.Raw, nil + case NotEqualToFilter: + return inputValue.Raw != m.FilterValue.Raw, nil + case GreaterThanFilter: + return inputValue.Raw > m.FilterValue.Raw, nil + case GreaterThanEqualFilter: + return inputValue.Raw >= m.FilterValue.Raw, nil + default: + return false, fmt.Errorf("unknown op: %s", m.Op) + } +} + +type int64NumericalExpression struct { + FilterValue int64 + Op FilterType +} + +func (s int64NumericalExpression) Search(input interface{}) (bool, error) { + inputValue, ok := input.(int64) + if !ok { + // If the input interface{} isn't int64 but an alias for int64, we want to check that + if reflect.TypeOf(input).ConvertibleTo(reflect.TypeOf(int64(0))) { + inputValue = reflect.ValueOf(input).Convert(reflect.TypeOf(int64(0))).Int() + } else { + return false, fmt.Errorf("supplied type (%s) was not int64", reflect.TypeOf(input).String()) + } + } + + switch s.Op { + case LessThanFilter: + return inputValue < s.FilterValue, nil + case LessThanEqualFilter: + return inputValue <= s.FilterValue, nil + case EqualToFilter: + return inputValue == s.FilterValue, nil + case NotEqualToFilter: + return inputValue != s.FilterValue, nil + case GreaterThanFilter: + return inputValue > s.FilterValue, nil + case GreaterThanEqualFilter: + return inputValue >= s.FilterValue, nil + default: + return false, fmt.Errorf("unknown op: %s", s.Op) + } + +} + +type uint64NumericalExpression struct { + FilterValue uint64 + Op FilterType +} + +func (u uint64NumericalExpression) Search(input interface{}) (bool, error) { + inputValue, ok := input.(uint64) + if !ok { + // If the input interface{} isn't uint64 but an alias for uint64, we want to check that + if reflect.TypeOf(input).ConvertibleTo(reflect.TypeOf(uint64(0))) { + inputValue = reflect.ValueOf(input).Convert(reflect.TypeOf(uint64(0))).Uint() + } else { + return false, fmt.Errorf("supplied type (%s) was not uint64", reflect.TypeOf(input).String()) + } + } + + switch u.Op { + case LessThanFilter: + return inputValue < u.FilterValue, nil + case LessThanEqualFilter: + return inputValue <= u.FilterValue, nil + case EqualToFilter: + return inputValue == u.FilterValue, nil + case NotEqualToFilter: + return inputValue != u.FilterValue, nil + case GreaterThanFilter: + return inputValue > u.FilterValue, nil + case GreaterThanEqualFilter: + return inputValue >= u.FilterValue, nil + + default: + return false, fmt.Errorf("unknown op: %s", u.Op) + } +} diff --git a/conduit/plugins/processors/filterprocessor/fields/filter.go b/conduit/plugins/processors/filterprocessor/fields/filter.go new file mode 100644 index 000000000..fd67785c4 --- /dev/null +++ b/conduit/plugins/processors/filterprocessor/fields/filter.go @@ -0,0 +1,100 @@ +package fields + +import ( + "fmt" + + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/indexer/data" +) + +// Operation an operation like "any" or "all" for boolean logic +type Operation string + +const anyFieldOperation Operation = "any" +const allFieldOperation Operation = "all" +const noneFieldOperation Operation = "none" + +// ValidFieldOperation returns true if the input is a valid operation +func ValidFieldOperation(input string) bool { + if input != string(anyFieldOperation) && input != string(allFieldOperation) && input != string(noneFieldOperation) { + return false + } + + return true +} + +// Filter an object that combines field searches with a boolean operator +type Filter struct { + Op Operation + Searchers []*Searcher +} + +// SearchAndFilter searches through the block data and applies the operation to the results +func (f Filter) SearchAndFilter(input data.BlockData) (data.BlockData, error) { + var newPayset []transactions.SignedTxnInBlock + switch f.Op { + case noneFieldOperation: + for _, txn := range input.Payset { + + allFalse := true + for _, fs := range f.Searchers { + b, err := fs.search(txn) + if err != nil { + return data.BlockData{}, err + } + if b { + allFalse = false + break + } + } + + if allFalse { + newPayset = append(newPayset, txn) + } + + } + break + + case anyFieldOperation: + for _, txn := range input.Payset { + for _, fs := range f.Searchers { + b, err := fs.search(txn) + if err != nil { + return data.BlockData{}, err + } + if b { + newPayset = append(newPayset, txn) + break + } + } + } + + break + case allFieldOperation: + for _, txn := range input.Payset { + + allTrue := true + for _, fs := range f.Searchers { + b, err := fs.search(txn) + if err != nil { + return data.BlockData{}, err + } + if !b { + allTrue = false + break + } + } + + if allTrue { + newPayset = append(newPayset, txn) + } + + } + break + default: + return data.BlockData{}, fmt.Errorf("unknown operation: %s", f.Op) + } + + input.Payset = newPayset + return input, nil +} diff --git a/conduit/plugins/processors/filterprocessor/fields/generated_signed_txn_map.go b/conduit/plugins/processors/filterprocessor/fields/generated_signed_txn_map.go new file mode 100644 index 000000000..26c02ed54 --- /dev/null +++ b/conduit/plugins/processors/filterprocessor/fields/generated_signed_txn_map.go @@ -0,0 +1,239 @@ +// Code generated via go generate. DO NOT EDIT. + +package fields + +import ( + "fmt" + + "github.com/algorand/go-algorand/data/transactions" +) + +// SignedTxnFunc takes a tag and associated SignedTxnInBlock and returns the value +// referenced by the tag. An error is returned if the tag does not exist +func SignedTxnFunc(tag string, input *transactions.SignedTxnInBlock) (interface{}, error) { + + switch tag { + case "aca": + return &input.SignedTxnWithAD.ApplyData.AssetClosingAmount, nil + case "apid": + return &input.SignedTxnWithAD.ApplyData.ApplicationID, nil + case "ca": + return &input.SignedTxnWithAD.ApplyData.ClosingAmount, nil + case "caid": + return &input.SignedTxnWithAD.ApplyData.ConfigAsset, nil + case "dt": + return &input.SignedTxnWithAD.ApplyData.EvalDelta, nil + case "dt.gd": + return &input.SignedTxnWithAD.ApplyData.EvalDelta.GlobalDelta, nil + case "dt.itx": + return &input.SignedTxnWithAD.ApplyData.EvalDelta.InnerTxns, nil + case "dt.ld": + return &input.SignedTxnWithAD.ApplyData.EvalDelta.LocalDeltas, nil + case "dt.lg": + return &input.SignedTxnWithAD.ApplyData.EvalDelta.Logs, nil + case "hgh": + return &input.HasGenesisHash, nil + case "hgi": + return &input.HasGenesisID, nil + case "lsig": + return &input.SignedTxnWithAD.SignedTxn.Lsig, nil + case "lsig.arg": + return &input.SignedTxnWithAD.SignedTxn.Lsig.Args, nil + case "lsig.l": + return &input.SignedTxnWithAD.SignedTxn.Lsig.Logic, nil + case "lsig.msig": + return &input.SignedTxnWithAD.SignedTxn.Lsig.Msig, nil + case "lsig.msig.subsig": + return &input.SignedTxnWithAD.SignedTxn.Lsig.Msig.Subsigs, nil + case "lsig.msig.thr": + return &input.SignedTxnWithAD.SignedTxn.Lsig.Msig.Threshold, nil + case "lsig.msig.v": + return &input.SignedTxnWithAD.SignedTxn.Lsig.Msig.Version, nil + case "lsig.sig": + return &input.SignedTxnWithAD.SignedTxn.Lsig.Sig, nil + case "msig": + return &input.SignedTxnWithAD.SignedTxn.Msig, nil + case "msig.subsig": + return &input.SignedTxnWithAD.SignedTxn.Msig.Subsigs, nil + case "msig.thr": + return &input.SignedTxnWithAD.SignedTxn.Msig.Threshold, nil + case "msig.v": + return &input.SignedTxnWithAD.SignedTxn.Msig.Version, nil + case "rc": + return &input.SignedTxnWithAD.ApplyData.CloseRewards, nil + case "rr": + return &input.SignedTxnWithAD.ApplyData.ReceiverRewards, nil + case "rs": + return &input.SignedTxnWithAD.ApplyData.SenderRewards, nil + case "sgnr": + return &input.SignedTxnWithAD.SignedTxn.AuthAddr, nil + case "sig": + return &input.SignedTxnWithAD.SignedTxn.Sig, nil + case "txn": + return &input.SignedTxnWithAD.SignedTxn.Txn, nil + case "txn.aamt": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetTransferTxnFields.AssetAmount, nil + case "txn.aclose": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetTransferTxnFields.AssetCloseTo, nil + case "txn.afrz": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetFreezeTxnFields.AssetFrozen, nil + case "txn.amt": + return &input.SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Amount, nil + case "txn.apaa": + return &input.SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.ApplicationArgs, nil + case "txn.apan": + return &input.SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.OnCompletion, nil + case "txn.apap": + return &input.SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.ApprovalProgram, nil + case "txn.apar": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams, nil + case "txn.apar.am": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.MetadataHash, nil + case "txn.apar.an": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.AssetName, nil + case "txn.apar.au": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.URL, nil + case "txn.apar.c": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.Clawback, nil + case "txn.apar.dc": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.Decimals, nil + case "txn.apar.df": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.DefaultFrozen, nil + case "txn.apar.f": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.Freeze, nil + case "txn.apar.m": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.Manager, nil + case "txn.apar.r": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.Reserve, nil + case "txn.apar.t": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.Total, nil + case "txn.apar.un": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.AssetParams.UnitName, nil + case "txn.apas": + return &input.SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.ForeignAssets, nil + case "txn.apat": + return &input.SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.Accounts, nil + case "txn.apbx": + return &input.SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.Boxes, nil + case "txn.apep": + return &input.SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.ExtraProgramPages, nil + case "txn.apfa": + return &input.SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.ForeignApps, nil + case "txn.apgs": + return &input.SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.GlobalStateSchema, nil + case "txn.apgs.nbs": + return &input.SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.GlobalStateSchema.NumByteSlice, nil + case "txn.apgs.nui": + return &input.SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.GlobalStateSchema.NumUint, nil + case "txn.apid": + return &input.SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.ApplicationID, nil + case "txn.apls": + return &input.SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.LocalStateSchema, nil + case "txn.apls.nbs": + return &input.SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.LocalStateSchema.NumByteSlice, nil + case "txn.apls.nui": + return &input.SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.LocalStateSchema.NumUint, nil + case "txn.apsu": + return &input.SignedTxnWithAD.SignedTxn.Txn.ApplicationCallTxnFields.ClearStateProgram, nil + case "txn.arcv": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetTransferTxnFields.AssetReceiver, nil + case "txn.asnd": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetTransferTxnFields.AssetSender, nil + case "txn.caid": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetConfigTxnFields.ConfigAsset, nil + case "txn.close": + return &input.SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.CloseRemainderTo, nil + case "txn.fadd": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetFreezeTxnFields.FreezeAccount, nil + case "txn.faid": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetFreezeTxnFields.FreezeAsset, nil + case "txn.fee": + return &input.SignedTxnWithAD.SignedTxn.Txn.Header.Fee, nil + case "txn.fv": + return &input.SignedTxnWithAD.SignedTxn.Txn.Header.FirstValid, nil + case "txn.gen": + return &input.SignedTxnWithAD.SignedTxn.Txn.Header.GenesisID, nil + case "txn.gh": + return &input.SignedTxnWithAD.SignedTxn.Txn.Header.GenesisHash, nil + case "txn.grp": + return &input.SignedTxnWithAD.SignedTxn.Txn.Header.Group, nil + case "txn.lv": + return &input.SignedTxnWithAD.SignedTxn.Txn.Header.LastValid, nil + case "txn.lx": + return &input.SignedTxnWithAD.SignedTxn.Txn.Header.Lease, nil + case "txn.nonpart": + return &input.SignedTxnWithAD.SignedTxn.Txn.KeyregTxnFields.Nonparticipation, nil + case "txn.note": + return &input.SignedTxnWithAD.SignedTxn.Txn.Header.Note, nil + case "txn.rcv": + return &input.SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Receiver, nil + case "txn.rekey": + return &input.SignedTxnWithAD.SignedTxn.Txn.Header.RekeyTo, nil + case "txn.selkey": + return &input.SignedTxnWithAD.SignedTxn.Txn.KeyregTxnFields.SelectionPK, nil + case "txn.snd": + return &input.SignedTxnWithAD.SignedTxn.Txn.Header.Sender, nil + case "txn.sp": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof, nil + case "txn.sp.P": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.PartProofs, nil + case "txn.sp.P.hsh": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.PartProofs.HashFactory, nil + case "txn.sp.P.hsh.t": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.PartProofs.HashFactory.HashType, nil + case "txn.sp.P.pth": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.PartProofs.Path, nil + case "txn.sp.P.td": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.PartProofs.TreeDepth, nil + case "txn.sp.S": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.SigProofs, nil + case "txn.sp.S.hsh": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.SigProofs.HashFactory, nil + case "txn.sp.S.hsh.t": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.SigProofs.HashFactory.HashType, nil + case "txn.sp.S.pth": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.SigProofs.Path, nil + case "txn.sp.S.td": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.SigProofs.TreeDepth, nil + case "txn.sp.c": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.SigCommit, nil + case "txn.sp.pr": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.PositionsToReveal, nil + case "txn.sp.r": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.Reveals, nil + case "txn.sp.v": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.MerkleSignatureSaltVersion, nil + case "txn.sp.w": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProof.SignedWeight, nil + case "txn.spmsg": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.Message, nil + case "txn.spmsg.P": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.Message.LnProvenWeight, nil + case "txn.spmsg.b": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.Message.BlockHeadersCommitment, nil + case "txn.spmsg.f": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.Message.FirstAttestedRound, nil + case "txn.spmsg.l": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.Message.LastAttestedRound, nil + case "txn.spmsg.v": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.Message.VotersCommitment, nil + case "txn.sprfkey": + return &input.SignedTxnWithAD.SignedTxn.Txn.KeyregTxnFields.StateProofPK, nil + case "txn.sptype": + return &input.SignedTxnWithAD.SignedTxn.Txn.StateProofTxnFields.StateProofType, nil + case "txn.type": + return &input.SignedTxnWithAD.SignedTxn.Txn.Type, nil + case "txn.votefst": + return &input.SignedTxnWithAD.SignedTxn.Txn.KeyregTxnFields.VoteFirst, nil + case "txn.votekd": + return &input.SignedTxnWithAD.SignedTxn.Txn.KeyregTxnFields.VoteKeyDilution, nil + case "txn.votekey": + return &input.SignedTxnWithAD.SignedTxn.Txn.KeyregTxnFields.VotePK, nil + case "txn.votelst": + return &input.SignedTxnWithAD.SignedTxn.Txn.KeyregTxnFields.VoteLast, nil + case "txn.xaid": + return &input.SignedTxnWithAD.SignedTxn.Txn.AssetTransferTxnFields.XferAsset, nil + default: + return nil, fmt.Errorf("unknown tag: %s", tag) + } +} diff --git a/conduit/plugins/processors/filterprocessor/fields/searcher.go b/conduit/plugins/processors/filterprocessor/fields/searcher.go new file mode 100644 index 000000000..558eb8433 --- /dev/null +++ b/conduit/plugins/processors/filterprocessor/fields/searcher.go @@ -0,0 +1,91 @@ +package fields + +//go:generate go run ../gen/generate.go fields ./generated_signed_txn_map.go + +import ( + "fmt" + "reflect" + + "github.com/algorand/go-algorand/data/transactions" + + "github.com/algorand/indexer/conduit/plugins/processors/filterprocessor/expression" +) + +// Searcher searches the struct with an expression and method to call +type Searcher struct { + Exp *expression.Expression + Tag string + MethodToCall string +} + +// This function is ONLY to be used by the filter.field function. +// The reason being is that without validation of the tag (which is provided by +// MakeFieldSearcher) then this can panic +func (f Searcher) search(input transactions.SignedTxnInBlock) (bool, error) { + + val, err := SignedTxnFunc(f.Tag, &input) + if err != nil { + return false, err + } + + e := reflect.ValueOf(val).Elem() + + var toSearch interface{} + if f.MethodToCall != "" { + // If there is a function, search what is returned + toSearch = e.MethodByName(f.MethodToCall).Call([]reflect.Value{})[0].Interface() + } else { + // Otherwise, get the original value + toSearch = e.Interface() + } + + b, err := (*f.Exp).Search(toSearch) + if err != nil { + return false, err + } + + return b, nil +} + +// checks that the supplied tag exists in the struct and recovers from any panics +func checkTagExistsAndHasCorrectFunction(expressionType expression.FilterType, tag string) (outError error) { + defer func() { + // This defer'd function is a belt and suspenders type thing. We check every reflected + // evaluation's IsValid() function to make sure not to operate on a zero value. Therfore we can't + // actually reach inside the if conditional unless we intentionally panic. + // However, having this function gives additional safety to a critical function + if r := recover(); r != nil { + outError = fmt.Errorf("error occured regarding tag %s - %v", tag, r) + } + }() + + val, err := SignedTxnFunc(tag, &transactions.SignedTxnInBlock{}) + + if err != nil { + return fmt.Errorf("%s does not exist in transactions.SignedTxnInBlock struct", tag) + } + + e := reflect.ValueOf(val).Elem() + + method, ok := expression.TypeToFunctionMap[expressionType] + + if !ok { + return fmt.Errorf("expression type (%s) is not supported. tag value: %s", expressionType, tag) + } + + if method != "" && !e.MethodByName(method).IsValid() { + return fmt.Errorf("variable referenced by tag %s does not contain the needed method: %s", tag, method) + } + + return nil +} + +// MakeFieldSearcher will check that the field exists and that it contains the necessary "conversion" function +func MakeFieldSearcher(e *expression.Expression, expressionType expression.FilterType, tag string) (*Searcher, error) { + + if err := checkTagExistsAndHasCorrectFunction(expressionType, tag); err != nil { + return nil, err + } + + return &Searcher{Exp: e, Tag: tag, MethodToCall: expression.TypeToFunctionMap[expressionType]}, nil +} diff --git a/conduit/plugins/processors/filterprocessor/fields/searcher_test.go b/conduit/plugins/processors/filterprocessor/fields/searcher_test.go new file mode 100644 index 000000000..8f2035b66 --- /dev/null +++ b/conduit/plugins/processors/filterprocessor/fields/searcher_test.go @@ -0,0 +1,100 @@ +package fields + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/indexer/conduit/plugins/processors/filterprocessor/expression" +) + +// TestInternalSearch tests the internal search functionality +func TestInternalSearch(t *testing.T) { + + defer func() { + // Since this function should only be called after validation is performed, + // this recovery function lets us recover is the schema changes in anyway in the future + if r := recover(); r != nil { + assert.True(t, false) + } + }() + + address1 := basics.Address{1} + address2 := basics.Address{2} + + var expressionType expression.FilterType = "exact" + tag := "sgnr" + exp, err := expression.MakeExpression(expressionType, address1.String(), reflect.String) + assert.NoError(t, err) + searcher, err := MakeFieldSearcher(exp, expressionType, tag) + assert.NoError(t, err) + + result, err := searcher.search( + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + AuthAddr: address1, + }, + }, + }, + ) + + assert.NoError(t, err) + assert.True(t, result) + + result, err = searcher.search( + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + AuthAddr: address2, + }, + }, + }, + ) + + assert.NoError(t, err) + assert.False(t, result) +} + +// TestMakeFieldSearcher tests making a field searcher is valid +func TestMakeFieldSearcher(t *testing.T) { + var expressionType expression.FilterType = "exact" + tag := "sgnr" + sampleExpressionStr := "sample" + exp, err := expression.MakeExpression(expressionType, sampleExpressionStr, reflect.String) + assert.NoError(t, err) + searcher, err := MakeFieldSearcher(exp, expressionType, tag) + assert.NoError(t, err) + assert.NotNil(t, searcher) + assert.Equal(t, searcher.Tag, tag) + assert.Equal(t, searcher.MethodToCall, expression.TypeToFunctionMap[expressionType]) + + searcher, err = MakeFieldSearcher(exp, "made-up-expression-type", sampleExpressionStr) + assert.Error(t, err) + assert.Nil(t, searcher) + +} + +// TestCheckTagExistsAndHasCorrectFunction tests that the check tag exists and function relation works +func TestCheckTagExistsAndHasCorrectFunction(t *testing.T) { + // check that something that doesn't exist throws an error + err := checkTagExistsAndHasCorrectFunction("exact", "SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.LoreumIpsum.SDF") + assert.ErrorContains(t, err, "does not exist in transactions") + + err = checkTagExistsAndHasCorrectFunction("exact", "LoreumIpsum") + assert.ErrorContains(t, err, "does not exist in transactions") + + // Fee does not have a "String" Function so we cant use exact with it. + err = checkTagExistsAndHasCorrectFunction("exact", "txn.fee") + assert.ErrorContains(t, err, "does not contain the needed method") + + // a made up expression type should throw an error + err = checkTagExistsAndHasCorrectFunction("made-up-expression-type", "sgnr") + assert.ErrorContains(t, err, "is not supported") + + err = checkTagExistsAndHasCorrectFunction("exact", "sgnr") + assert.NoError(t, err) +} diff --git a/conduit/plugins/processors/filterprocessor/filter_processor.go b/conduit/plugins/processors/filterprocessor/filter_processor.go new file mode 100644 index 000000000..4b81fe08e --- /dev/null +++ b/conduit/plugins/processors/filterprocessor/filter_processor.go @@ -0,0 +1,144 @@ +package filterprocessor + +import ( + "context" + _ "embed" // used to embed config + "fmt" + "reflect" + + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + + "github.com/algorand/go-algorand/data/transactions" + + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/processors" + "github.com/algorand/indexer/conduit/plugins/processors/filterprocessor/expression" + "github.com/algorand/indexer/conduit/plugins/processors/filterprocessor/fields" + "github.com/algorand/indexer/data" +) + +const implementationName = "filter_processor" + +// package-wide init function +func init() { + processors.Register(implementationName, processors.ProcessorConstructorFunc(func() processors.Processor { + return &FilterProcessor{} + })) +} + +// FilterProcessor filters transactions by a variety of means +type FilterProcessor struct { + FieldFilters []fields.Filter + + logger *log.Logger + cfg plugins.PluginConfig + ctx context.Context +} + +//go:embed sample.yaml +var sampleConfig string + +// Metadata returns metadata +func (a *FilterProcessor) Metadata() conduit.Metadata { + return conduit.Metadata{ + Name: implementationName, + Description: "FilterProcessor Filter Processor", + Deprecated: false, + SampleConfig: sampleConfig, + } +} + +// Config returns the config +func (a *FilterProcessor) Config() plugins.PluginConfig { + return a.cfg +} + +// Init initializes the filter processor +func (a *FilterProcessor) Init(ctx context.Context, _ data.InitProvider, cfg plugins.PluginConfig, logger *log.Logger) error { + a.logger = logger + a.cfg = cfg + a.ctx = ctx + + // First get the configuration from the string + pCfg := Config{} + + err := yaml.Unmarshal([]byte(cfg), &pCfg) + if err != nil { + return fmt.Errorf("filter processor init error: %w", err) + } + + // configMaps is the "- any: ...." portion of the filter config + for _, configMaps := range pCfg.Filters { + + // We only want one key in the map (i.e. either "any" or "all"). The reason we use a list is that want + // to maintain ordering of the filters and a straight-up map doesn't do that. + if len(configMaps) != 1 { + return fmt.Errorf("filter processor Init(): illegal filter tag formation. tag length was: %d", len(configMaps)) + } + + for key, subConfigs := range configMaps { + + if !fields.ValidFieldOperation(key) { + return fmt.Errorf("filter processor Init(): filter key was not a valid value: %s", key) + } + + var searcherList []*fields.Searcher + + for _, subConfig := range subConfigs { + + t, err := fields.SignedTxnFunc(subConfig.FilterTag, &transactions.SignedTxnInBlock{}) + if err != nil { + return err + } + + // We need the Elem() here because SignedTxnFunc returns a pointer underneath the interface{} + targetKind := reflect.TypeOf(t).Elem().Kind() + + exp, err := expression.MakeExpression(subConfig.ExpressionType, subConfig.Expression, targetKind) + if err != nil { + return fmt.Errorf("filter processor Init(): could not make expression with string %s for filter tag %s - %w", subConfig.Expression, subConfig.FilterTag, err) + } + + searcher, err := fields.MakeFieldSearcher(exp, subConfig.ExpressionType, subConfig.FilterTag) + if err != nil { + return fmt.Errorf("filter processor Init(): error making field searcher - %w", err) + } + + searcherList = append(searcherList, searcher) + } + + ff := fields.Filter{ + Op: fields.Operation(key), + Searchers: searcherList, + } + + a.FieldFilters = append(a.FieldFilters, ff) + + } + } + + return nil + +} + +// Close a no-op for this processor +func (a *FilterProcessor) Close() error { + return nil +} + +// Process processes the input data +func (a *FilterProcessor) Process(input data.BlockData) (data.BlockData, error) { + + var err error + + for _, searcher := range a.FieldFilters { + input, err = searcher.SearchAndFilter(input) + if err != nil { + return data.BlockData{}, err + } + } + + return input, err +} diff --git a/conduit/plugins/processors/filterprocessor/filter_processor_test.go b/conduit/plugins/processors/filterprocessor/filter_processor_test.go new file mode 100644 index 000000000..1cbacd25e --- /dev/null +++ b/conduit/plugins/processors/filterprocessor/filter_processor_test.go @@ -0,0 +1,1032 @@ +package filterprocessor + +import ( + "context" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/transactions" + + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/processors" + "github.com/algorand/indexer/data" +) + +// TestFilterProcessor_Init_None +func TestFilterProcessor_Init_None(t *testing.T) { + + sampleAddr1 := basics.Address{1} + sampleAddr2 := basics.Address{2} + sampleAddr3 := basics.Address{3} + + sampleCfgStr := `--- +filters: + - none: + - tag: sgnr + expression-type: exact + expression: "` + sampleAddr1.String() + `" + - tag: txn.asnd + expression-type: regex + expression: "` + sampleAddr3.String() + `" + - all: + - tag: txn.rcv + expression-type: regex + expression: "` + sampleAddr2.String() + `" + - tag: txn.snd + expression-type: exact + expression: "` + sampleAddr2.String() + `" + - any: + - tag: txn.aclose + expression-type: exact + expression: "` + sampleAddr2.String() + `" + - tag: txn.arcv + expression-type: regex + expression: "` + sampleAddr2.String() + `" +` + + fpBuilder, err := processors.ProcessorBuilderByName(implementationName) + assert.NoError(t, err) + + fp := fpBuilder.New() + err = fp.Init(context.Background(), &conduit.PipelineInitProvider{}, plugins.PluginConfig(sampleCfgStr), logrus.New()) + assert.NoError(t, err) + + bd := data.BlockData{} + bd.Payset = append(bd.Payset, + + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + AuthAddr: sampleAddr1, + }, + }, + }, + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + AuthAddr: sampleAddr1, + Txn: transactions.Transaction{ + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: sampleAddr2, + }, + Header: transactions.Header{ + Sender: sampleAddr2, + }, + AssetTransferTxnFields: transactions.AssetTransferTxnFields{ + AssetCloseTo: sampleAddr2, + }, + }, + }, + }, + }, + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + AuthAddr: sampleAddr1, + Txn: transactions.Transaction{ + AssetTransferTxnFields: transactions.AssetTransferTxnFields{ + AssetSender: sampleAddr3, + }, + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: sampleAddr3, + }, + }, + }, + }, + }, + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + AuthAddr: sampleAddr1, + Txn: transactions.Transaction{ + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: sampleAddr2, + }, + Header: transactions.Header{ + Sender: sampleAddr2, + }, + AssetTransferTxnFields: transactions.AssetTransferTxnFields{ + AssetSender: sampleAddr3, + AssetCloseTo: sampleAddr2, + AssetReceiver: sampleAddr2, + }, + }, + }, + }, + }, + // The one transaction that will be allowed through + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + AuthAddr: sampleAddr2, + Txn: transactions.Transaction{ + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: sampleAddr2, + }, + Header: transactions.Header{ + Sender: sampleAddr2, + }, + AssetTransferTxnFields: transactions.AssetTransferTxnFields{ + AssetSender: sampleAddr1, + AssetCloseTo: sampleAddr2, + AssetReceiver: sampleAddr2, + }, + }, + }, + }, + }, + ) + + output, err := fp.Process(bd) + assert.NoError(t, err) + assert.Equal(t, len(output.Payset), 1) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Receiver, sampleAddr2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.Header.Sender, sampleAddr2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.AssetTransferTxnFields.AssetSender, sampleAddr1) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.AssetTransferTxnFields.AssetCloseTo, sampleAddr2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.AssetTransferTxnFields.AssetReceiver, sampleAddr2) +} + +// TestFilterProcessor_Illegal tests that numerical operations won't occur on non-supported types +func TestFilterProcessor_Illegal(t *testing.T) { + tests := []struct { + name string + cfg string + errorContains string + }{ + { + "illegal 1", `--- +filters: + - any: + - tag: txn.type + expression-type: less-than + expression: 4 +`, "unknown target kind"}, + + { + "illegal 2", `--- +filters: + - any: + - tag: txn.type + expression-type: less-than-equal + expression: 4 +`, "unknown target kind"}, + + { + "illegal 3", `--- +filters: + - any: + - tag: txn.type + expression-type: greater-than + expression: 4 +`, "unknown target kind"}, + + { + "illegal 4", `--- +filters: + - any: + - tag: txn.type + expression-type: greater-than-equal + expression: 4 +`, "unknown target kind"}, + + { + "illegal 4", `--- +filters: + - any: + - tag: txn.type + expression-type: equal + expression: 4 +`, "unknown target kind"}, + + { + "illegal 5", `--- +filters: + - any: + - tag: txn.type + expression-type: not-equal + expression: 4 +`, "unknown target kind"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + fpBuilder, err := processors.ProcessorBuilderByName(implementationName) + assert.NoError(t, err) + + fp := fpBuilder.New() + err = fp.Init(context.Background(), &conduit.PipelineInitProvider{}, plugins.PluginConfig(test.cfg), logrus.New()) + assert.ErrorContains(t, err, test.errorContains) + }) + } +} + +// TestFilterProcessor_Alias tests the various numerical operations on integers that are aliased +func TestFilterProcessor_Alias(t *testing.T) { + tests := []struct { + name string + cfg string + fxn func(t *testing.T, output *data.BlockData) + }{ + + {"alias 1", `--- +filters: + - any: + - tag: apid + expression-type: less-than + expression: 4 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 1) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.ApplicationID, basics.AppIndex(2)) + }, + }, + {"alias 2", `--- +filters: + - any: + - tag: apid + expression-type: less-than + expression: 5 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.ApplicationID, basics.AppIndex(4)) + assert.Equal(t, output.Payset[1].SignedTxnWithAD.ApplicationID, basics.AppIndex(2)) + }, + }, + + {"alias 3", `--- +filters: + - any: + - tag: apid + expression-type: less-than-equal + expression: 4 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.ApplicationID, basics.AppIndex(4)) + assert.Equal(t, output.Payset[1].SignedTxnWithAD.ApplicationID, basics.AppIndex(2)) + }, + }, + {"alias 4", `--- +filters: + - any: + - tag: apid + expression-type: equal + expression: 11 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 1) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.ApplicationID, basics.AppIndex(11)) + }, + }, + + {"alias 5", `--- +filters: + - any: + - tag: apid + expression-type: not-equal + expression: 11 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.ApplicationID, basics.AppIndex(4)) + assert.Equal(t, output.Payset[1].SignedTxnWithAD.ApplicationID, basics.AppIndex(2)) + }, + }, + + {"alias 6", `--- +filters: + - any: + - tag: apid + expression-type: greater-than + expression: 4 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 1) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.ApplicationID, basics.AppIndex(11)) + }, + }, + {"alias 7", `--- +filters: + - any: + - tag: apid + expression-type: greater-than + expression: 3 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.ApplicationID, basics.AppIndex(4)) + assert.Equal(t, output.Payset[1].SignedTxnWithAD.ApplicationID, basics.AppIndex(11)) + }, + }, + + {"alias 8", `--- +filters: + - any: + - tag: apid + expression-type: greater-than-equal + expression: 4 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.ApplicationID, basics.AppIndex(4)) + assert.Equal(t, output.Payset[1].SignedTxnWithAD.ApplicationID, basics.AppIndex(11)) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + fpBuilder, err := processors.ProcessorBuilderByName(implementationName) + assert.NoError(t, err) + + fp := fpBuilder.New() + err = fp.Init(context.Background(), &conduit.PipelineInitProvider{}, plugins.PluginConfig(test.cfg), logrus.New()) + assert.NoError(t, err) + + bd := data.BlockData{} + bd.Payset = append(bd.Payset, + + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + ApplyData: transactions.ApplyData{ + ApplicationID: 4, + }, + }, + }, + transactions.SignedTxnInBlock{ + + SignedTxnWithAD: transactions.SignedTxnWithAD{ + ApplyData: transactions.ApplyData{ + ApplicationID: 2, + }, + }, + }, + transactions.SignedTxnInBlock{ + + SignedTxnWithAD: transactions.SignedTxnWithAD{ + ApplyData: transactions.ApplyData{ + ApplicationID: 11, + }, + }, + }, + ) + + output, err := fp.Process(bd) + assert.NoError(t, err) + test.fxn(t, &output) + }) + } +} + +// TestFilterProcessor_Numerical tests the various numerical operations on integers +func TestFilterProcessor_Numerical(t *testing.T) { + tests := []struct { + name string + cfg string + fxn func(t *testing.T, output *data.BlockData) + }{ + + {"numerical 1", `--- +filters: + - any: + - tag: aca + expression-type: less-than + expression: 4 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 1) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.AssetClosingAmount, uint64(2)) + }, + }, + {"numerical 2", `--- +filters: + - any: + - tag: aca + expression-type: less-than + expression: 5 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.AssetClosingAmount, uint64(4)) + assert.Equal(t, output.Payset[1].SignedTxnWithAD.AssetClosingAmount, uint64(2)) + }, + }, + + {"numerical 3", `--- +filters: + - any: + - tag: aca + expression-type: less-than-equal + expression: 4 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.AssetClosingAmount, uint64(4)) + assert.Equal(t, output.Payset[1].SignedTxnWithAD.AssetClosingAmount, uint64(2)) + }, + }, + {"numerical 4", `--- +filters: + - any: + - tag: aca + expression-type: equal + expression: 11 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 1) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.AssetClosingAmount, uint64(11)) + }, + }, + + {"numerical 5", `--- +filters: + - any: + - tag: aca + expression-type: not-equal + expression: 11 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.AssetClosingAmount, uint64(4)) + assert.Equal(t, output.Payset[1].SignedTxnWithAD.AssetClosingAmount, uint64(2)) + }, + }, + + {"numerical 6", `--- +filters: + - any: + - tag: aca + expression-type: greater-than + expression: 4 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 1) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.AssetClosingAmount, uint64(11)) + }, + }, + {"numerical 7", `--- +filters: + - any: + - tag: aca + expression-type: greater-than + expression: 3 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.AssetClosingAmount, uint64(4)) + assert.Equal(t, output.Payset[1].SignedTxnWithAD.AssetClosingAmount, uint64(11)) + }, + }, + + {"numerical 8", `--- +filters: + - any: + - tag: aca + expression-type: greater-than-equal + expression: 4 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.AssetClosingAmount, uint64(4)) + assert.Equal(t, output.Payset[1].SignedTxnWithAD.AssetClosingAmount, uint64(11)) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + fpBuilder, err := processors.ProcessorBuilderByName(implementationName) + assert.NoError(t, err) + + fp := fpBuilder.New() + err = fp.Init(context.Background(), &conduit.PipelineInitProvider{}, plugins.PluginConfig(test.cfg), logrus.New()) + assert.NoError(t, err) + + bd := data.BlockData{} + bd.Payset = append(bd.Payset, + + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + ApplyData: transactions.ApplyData{ + AssetClosingAmount: 4, + }, + }, + }, + transactions.SignedTxnInBlock{ + + SignedTxnWithAD: transactions.SignedTxnWithAD{ + ApplyData: transactions.ApplyData{ + AssetClosingAmount: 2, + }, + }, + }, + transactions.SignedTxnInBlock{ + + SignedTxnWithAD: transactions.SignedTxnWithAD{ + ApplyData: transactions.ApplyData{ + AssetClosingAmount: 11, + }, + }, + }, + ) + + output, err := fp.Process(bd) + assert.NoError(t, err) + test.fxn(t, &output) + }) + } +} + +// TestFilterProcessor_MicroAlgos tests the various numerical operations on microalgos +func TestFilterProcessor_MicroAlgos(t *testing.T) { + tests := []struct { + name string + cfg string + fxn func(t *testing.T, output *data.BlockData) + }{ + {"micro algo 1", `--- +filters: + - any: + - tag: txn.amt + expression-type: less-than + expression: 4 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 1) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Amount.Raw, uint64(2)) + }, + }, + {"micro algo 2", `--- +filters: + - any: + - tag: txn.amt + expression-type: less-than + expression: 5 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Amount.Raw, uint64(4)) + assert.Equal(t, output.Payset[1].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Amount.Raw, uint64(2)) + }, + }, + + {"micro algo 3", `--- +filters: + - any: + - tag: txn.amt + expression-type: less-than-equal + expression: 4 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Amount.Raw, uint64(4)) + assert.Equal(t, output.Payset[1].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Amount.Raw, uint64(2)) + }, + }, + {"micro algo 4", `--- +filters: + - any: + - tag: txn.amt + expression-type: equal + expression: 11 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 1) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Amount.Raw, uint64(11)) + }, + }, + + {"micro algo 5", `--- +filters: + - any: + - tag: txn.amt + expression-type: not-equal + expression: 11 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Amount.Raw, uint64(4)) + assert.Equal(t, output.Payset[1].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Amount.Raw, uint64(2)) + }, + }, + + {"micro algo 6", `--- +filters: + - any: + - tag: txn.amt + expression-type: greater-than + expression: 4 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 1) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Amount.Raw, uint64(11)) + }, + }, + {"micro algo 7", `--- +filters: + - any: + - tag: txn.amt + expression-type: greater-than + expression: 3 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Amount.Raw, uint64(4)) + assert.Equal(t, output.Payset[1].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Amount.Raw, uint64(11)) + }, + }, + + {"micro algo 8", `--- +filters: + - any: + - tag: txn.amt + expression-type: greater-than-equal + expression: 4 +`, func(t *testing.T, output *data.BlockData) { + + assert.Equal(t, len(output.Payset), 2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Amount.Raw, uint64(4)) + assert.Equal(t, output.Payset[1].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Amount.Raw, uint64(11)) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + fpBuilder, err := processors.ProcessorBuilderByName(implementationName) + assert.NoError(t, err) + + fp := fpBuilder.New() + err = fp.Init(context.Background(), &conduit.PipelineInitProvider{}, plugins.PluginConfig(test.cfg), logrus.New()) + assert.NoError(t, err) + + bd := data.BlockData{} + bd.Payset = append(bd.Payset, + + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + Txn: transactions.Transaction{ + PaymentTxnFields: transactions.PaymentTxnFields{ + Amount: basics.MicroAlgos{Raw: 4}, + }, + }, + }, + }, + }, + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + Txn: transactions.Transaction{ + PaymentTxnFields: transactions.PaymentTxnFields{ + Amount: basics.MicroAlgos{Raw: 2}, + }, + }, + }, + }, + }, + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + Txn: transactions.Transaction{ + PaymentTxnFields: transactions.PaymentTxnFields{ + Amount: basics.MicroAlgos{Raw: 11}, + }, + }, + }, + }, + }, + ) + + output, err := fp.Process(bd) + assert.NoError(t, err) + test.fxn(t, &output) + }) + } +} + +// TestFilterProcessor_VariousErrorPathsOnInit tests the various error paths in the filter processor init function +func TestFilterProcessor_VariousErrorPathsOnInit(t *testing.T) { + tests := []struct { + name string + sampleCfgStr string + errorContainsStr string + }{ + + {"MakeExpressionError", `--- +filters: + - any: + - tag: DoesNot.ExistIn.Struct + expression-type: exact + expression: "sample" +`, "unknown tag"}, + + {"MakeExpressionError", `--- +filters: + - any: + - tag: sgnr + expression-type: wrong-expression-type + expression: "sample" +`, "could not make expression with string"}, + + {"CorrectFilterType", `--- +filters: + - wrong-filter-type: + - tag: sgnr + expression-type: exact + expression: "sample" + +`, "filter key was not a valid value"}, + + {"FilterTagFormation", `--- +filters: + - any: + - tag: sgnr + expression-type: exact + expression: "sample" + all: + - tag: sgnr + expression-type: exact + expression: "sample" + + +`, "illegal filter tag formation"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + fpBuilder, err := processors.ProcessorBuilderByName(implementationName) + assert.NoError(t, err) + + fp := fpBuilder.New() + err = fp.Init(context.Background(), &conduit.PipelineInitProvider{}, plugins.PluginConfig(test.sampleCfgStr), logrus.New()) + assert.ErrorContains(t, err, test.errorContainsStr) + }) + } +} + +// TestFilterProcessor_Init_Multi tests initialization of the filter processor with the "all" and "any" filter types +func TestFilterProcessor_Init_Multi(t *testing.T) { + + sampleAddr1 := basics.Address{1} + sampleAddr2 := basics.Address{2} + sampleAddr3 := basics.Address{3} + + sampleCfgStr := `--- +filters: + - any: + - tag: sgnr + expression-type: exact + expression: "` + sampleAddr1.String() + `" + - tag: txn.asnd + expression-type: regex + expression: "` + sampleAddr3.String() + `" + - all: + - tag: txn.rcv + expression-type: regex + expression: "` + sampleAddr2.String() + `" + - tag: txn.snd + expression-type: exact + expression: "` + sampleAddr2.String() + `" + - any: + - tag: txn.aclose + expression-type: exact + expression: "` + sampleAddr2.String() + `" + - tag: txn.arcv + expression-type: regex + expression: "` + sampleAddr2.String() + `" +` + + fpBuilder, err := processors.ProcessorBuilderByName(implementationName) + assert.NoError(t, err) + + fp := fpBuilder.New() + err = fp.Init(context.Background(), &conduit.PipelineInitProvider{}, plugins.PluginConfig(sampleCfgStr), logrus.New()) + assert.NoError(t, err) + + bd := data.BlockData{} + bd.Payset = append(bd.Payset, + + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + AuthAddr: sampleAddr1, + }, + }, + }, + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + Txn: transactions.Transaction{ + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: sampleAddr2, + }, + Header: transactions.Header{ + Sender: sampleAddr2, + }, + AssetTransferTxnFields: transactions.AssetTransferTxnFields{ + AssetCloseTo: sampleAddr2, + }, + }, + }, + }, + }, + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + Txn: transactions.Transaction{ + AssetTransferTxnFields: transactions.AssetTransferTxnFields{ + AssetSender: sampleAddr3, + }, + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: sampleAddr3, + }, + }, + }, + }, + }, + // The one transaction that will be allowed through + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + Txn: transactions.Transaction{ + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: sampleAddr2, + }, + Header: transactions.Header{ + Sender: sampleAddr2, + }, + AssetTransferTxnFields: transactions.AssetTransferTxnFields{ + AssetSender: sampleAddr3, + AssetCloseTo: sampleAddr2, + AssetReceiver: sampleAddr2, + }, + }, + }, + }, + }, + ) + + output, err := fp.Process(bd) + assert.NoError(t, err) + assert.Equal(t, len(output.Payset), 1) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Receiver, sampleAddr2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.Header.Sender, sampleAddr2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.AssetTransferTxnFields.AssetSender, sampleAddr3) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.AssetTransferTxnFields.AssetCloseTo, sampleAddr2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.AssetTransferTxnFields.AssetReceiver, sampleAddr2) + +} + +// TestFilterProcessor_Init_All tests initialization of the filter processor with the "all" filter type +func TestFilterProcessor_Init_All(t *testing.T) { + + sampleAddr1 := basics.Address{1} + sampleAddr2 := basics.Address{2} + sampleAddr3 := basics.Address{3} + + sampleCfgStr := `--- +filters: + - all: + - tag: txn.rcv + expression-type: regex + expression: "` + sampleAddr2.String() + `" + - tag: txn.snd + expression-type: exact + expression: "` + sampleAddr2.String() + `" +` + + fpBuilder, err := processors.ProcessorBuilderByName(implementationName) + assert.NoError(t, err) + + fp := fpBuilder.New() + err = fp.Init(context.Background(), &conduit.PipelineInitProvider{}, plugins.PluginConfig(sampleCfgStr), logrus.New()) + assert.NoError(t, err) + + bd := data.BlockData{} + bd.Payset = append(bd.Payset, + + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + Txn: transactions.Transaction{ + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: sampleAddr1, + }, + }, + }, + }, + }, + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + Txn: transactions.Transaction{ + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: sampleAddr2, + }, + Header: transactions.Header{ + Sender: sampleAddr2, + }, + }, + }, + }, + }, + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + Txn: transactions.Transaction{ + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: sampleAddr3, + }, + }, + }, + }, + }, + ) + + output, err := fp.Process(bd) + assert.NoError(t, err) + assert.Equal(t, len(output.Payset), 1) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Receiver, sampleAddr2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.Header.Sender, sampleAddr2) +} + +// TestFilterProcessor_Init_Some tests initialization of the filter processor with the "any" filter type +func TestFilterProcessor_Init(t *testing.T) { + + sampleAddr1 := basics.Address{1} + sampleAddr2 := basics.Address{2} + sampleAddr3 := basics.Address{3} + + sampleCfgStr := `--- +filters: + - any: + - tag: txn.rcv + expression-type: regex + expression: "` + sampleAddr1.String() + `" + - tag: txn.rcv + expression-type: exact + expression: "` + sampleAddr2.String() + `" +` + + fpBuilder, err := processors.ProcessorBuilderByName(implementationName) + assert.NoError(t, err) + + fp := fpBuilder.New() + err = fp.Init(context.Background(), &conduit.PipelineInitProvider{}, plugins.PluginConfig(sampleCfgStr), logrus.New()) + assert.NoError(t, err) + + bd := data.BlockData{} + bd.Payset = append(bd.Payset, + + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + Txn: transactions.Transaction{ + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: sampleAddr1, + }, + }, + }, + }, + }, + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + Txn: transactions.Transaction{ + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: sampleAddr2, + }, + }, + }, + }, + }, + transactions.SignedTxnInBlock{ + SignedTxnWithAD: transactions.SignedTxnWithAD{ + SignedTxn: transactions.SignedTxn{ + Txn: transactions.Transaction{ + PaymentTxnFields: transactions.PaymentTxnFields{ + Receiver: sampleAddr3, + }, + }, + }, + }, + }, + ) + + output, err := fp.Process(bd) + assert.NoError(t, err) + assert.Equal(t, len(output.Payset), 2) + assert.Equal(t, output.Payset[0].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Receiver, sampleAddr1) + assert.Equal(t, output.Payset[1].SignedTxnWithAD.SignedTxn.Txn.PaymentTxnFields.Receiver, sampleAddr2) +} diff --git a/conduit/plugins/processors/filterprocessor/gen/generate.go b/conduit/plugins/processors/filterprocessor/gen/generate.go new file mode 100644 index 000000000..48cfc8384 --- /dev/null +++ b/conduit/plugins/processors/filterprocessor/gen/generate.go @@ -0,0 +1,145 @@ +package main + +import ( + "bytes" + "fmt" + "go/format" + "os" + "reflect" + "sort" + "strings" + + "github.com/algorand/go-algorand/data/transactions" +) + +// recursiveTagFields recursively gets all field names in a struct +// Output will contain a key of the full tag along with the fully qualified struct +func recursiveTagFields(theStruct interface{}, output map[string]string, tagLevel []string, fieldLevel []string) { + rStruct := reflect.TypeOf(theStruct) + numFields := rStruct.NumField() + for i := 0; i < numFields; i++ { + field := rStruct.Field(i) + name := field.Name + + var tagValue string + var foundTag bool + // If there is a codec tag... + if tagValue, foundTag = field.Tag.Lookup("codec"); foundTag { + + vals := strings.Split(tagValue, ",") + // Get the first value (the one we care about) + tagValue = vals[0] + // If it is empty ignore it + if tagValue == "" { + continue + } + + fullTag := strings.Join(append(tagLevel, tagValue), ".") + output[fullTag] = strings.Join(append(fieldLevel, name), ".") + } + + if field.Type.Kind() == reflect.Struct { + var passedTagLevel []string + if foundTag { + passedTagLevel = append(tagLevel, tagValue) + } else { + passedTagLevel = tagLevel + } + recursiveTagFields(reflect.New(field.Type).Elem().Interface(), output, passedTagLevel, append(fieldLevel, name)) + } + } +} + +// usage: +// go run generate.go packagename outputfile +func main() { + + var packageName string + var outputFilepath string + + if len(os.Args) == 3 { + packageName = os.Args[1] + outputFilepath = os.Args[2] + } + + if packageName == "" { + packageName = "NULL" + } + + output := make(map[string]string) + tagLevel := []string{} + fieldLevel := []string{} + + recursiveTagFields(transactions.SignedTxnInBlock{}, output, tagLevel, fieldLevel) + + var err error + var bb bytes.Buffer + + initialStr := `// Code generated via go generate. DO NOT EDIT. + +package %s + +import ( +"fmt" + +"github.com/algorand/go-algorand/data/transactions" +) + +// SignedTxnFunc takes a tag and associated SignedTxnInBlock and returns the value +// referenced by the tag. An error is returned if the tag does not exist +func SignedTxnFunc(tag string, input *transactions.SignedTxnInBlock) (interface{}, error) { + +` + _, err = fmt.Fprintf(&bb, initialStr, packageName) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to write to %s: %v\n", outputFilepath, err) + os.Exit(1) + } + + keys := []string{} + + for k := range output { + keys = append(keys, k) + } + + sort.Strings(keys) + + _, err = fmt.Fprintf(&bb, "switch tag {\n") + if err != nil { + fmt.Fprintf(os.Stderr, "failed to write to %s: %v\n", outputFilepath, err) + os.Exit(1) + } + + for _, k := range keys { + fmt.Fprintf(&bb, "case \"%s\":\nreturn &input.%s, nil\n", k, output[k]) + } + + _, err = fmt.Fprint(&bb, "default:\n"+ + "return nil, fmt.Errorf(\"unknown tag: %s\", tag)\n"+ + "}\n}") + if err != nil { + fmt.Fprintf(os.Stderr, "failed to write to %s: %v\n", outputFilepath, err) + os.Exit(1) + } + + bbuf, err := format.Source(bb.Bytes()) + if err != nil { + fmt.Fprintf(os.Stderr, "formatting error: %v", err) + os.Exit(1) + } + + outputStr := string(bbuf) + + if outputFilepath == "" { + fmt.Printf("%s", outputStr) + } else { + fout, err := os.Create(outputFilepath) + if err != nil { + fmt.Fprintf(os.Stderr, "cannot open %s for writing: %v\n", outputFilepath, err) + os.Exit(1) + } + defer fout.Close() + fmt.Fprintf(fout, "%s", outputStr) + } + +} diff --git a/conduit/plugins/processors/filterprocessor/sample.yaml b/conduit/plugins/processors/filterprocessor/sample.yaml new file mode 100644 index 000000000..e8733ca7a --- /dev/null +++ b/conduit/plugins/processors/filterprocessor/sample.yaml @@ -0,0 +1,9 @@ +name: filter_processor +config: + # Filters is a list of boolean expressions that can search the payset transactions. + # See README for more info. + filters: + - any: + - tag: txn.rcv + expression-type: exact + expression: "ADDRESS" diff --git a/conduit/plugins/processors/filterprocessor/sample_test.go b/conduit/plugins/processors/filterprocessor/sample_test.go new file mode 100644 index 000000000..f58f274e8 --- /dev/null +++ b/conduit/plugins/processors/filterprocessor/sample_test.go @@ -0,0 +1,48 @@ +package filterprocessor + +import ( + "context" + "gopkg.in/yaml.v3" + "os" + "reflect" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/processors" +) + +// TestFilterProcessorSampleConfigInit validates that all fields in the sample config are valid for a filter processor +func TestFilterProcessorSampleConfigInit(t *testing.T) { + + fpBuilder, err := processors.ProcessorBuilderByName("filter_processor") + assert.NoError(t, err) + + sampleConfigStr, err := os.ReadFile("sample.yaml") + assert.NoError(t, err) + + fp := fpBuilder.New() + err = fp.Init(context.Background(), &conduit.PipelineInitProvider{}, plugins.PluginConfig(sampleConfigStr), logrus.New()) + assert.NoError(t, err) +} + +// TestFilterProcessorSampleConfigNotEmpty tests that all fields in the sample config are filled in +func TestFilterProcessorSampleConfigNotEmpty(t *testing.T) { + + sampleConfigStr, err := os.ReadFile("sample.yaml") + assert.NoError(t, err) + pCfg := Config{} + + err = yaml.Unmarshal(sampleConfigStr, &pCfg) + assert.NoError(t, err) + + v := reflect.ValueOf(pCfg) + + for i := 0; i < v.NumField(); i++ { + assert.True(t, v.Field(i).Interface() != nil) + } + +} diff --git a/conduit/plugins/processors/noop/noop_processor.go b/conduit/plugins/processors/noop/noop_processor.go new file mode 100644 index 000000000..375402d17 --- /dev/null +++ b/conduit/plugins/processors/noop/noop_processor.go @@ -0,0 +1,58 @@ +package noop + +import ( + "context" + _ "embed" // used to embed config + + "github.com/sirupsen/logrus" + + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/conduit/plugins/processors" + "github.com/algorand/indexer/data" +) + +const implementationName = "noop" + +// package-wide init function +func init() { + processors.Register(implementationName, processors.ProcessorConstructorFunc(func() processors.Processor { + return &Processor{} + })) +} + +// Processor noop +type Processor struct{} + +//go:embed sample.yaml +var sampleConfig string + +// Metadata noop +func (p *Processor) Metadata() conduit.Metadata { + return conduit.Metadata{ + Name: implementationName, + Description: "noop processor", + Deprecated: false, + SampleConfig: sampleConfig, + } +} + +// Config noop +func (p *Processor) Config() plugins.PluginConfig { + return "" +} + +// Init noop +func (p *Processor) Init(_ context.Context, _ data.InitProvider, _ plugins.PluginConfig, _ *logrus.Logger) error { + return nil +} + +// Close noop +func (p *Processor) Close() error { + return nil +} + +// Process noop +func (p *Processor) Process(input data.BlockData) (data.BlockData, error) { + return input, nil +} diff --git a/conduit/plugins/processors/noop/sample.yaml b/conduit/plugins/processors/noop/sample.yaml new file mode 100644 index 000000000..a4e995631 --- /dev/null +++ b/conduit/plugins/processors/noop/sample.yaml @@ -0,0 +1,3 @@ +name: noop +# noop has no config +config: diff --git a/conduit/plugins/processors/processor.go b/conduit/plugins/processors/processor.go new file mode 100644 index 000000000..75c1a8295 --- /dev/null +++ b/conduit/plugins/processors/processor.go @@ -0,0 +1,32 @@ +package processors + +import ( + "context" + + "github.com/sirupsen/logrus" + + "github.com/algorand/indexer/conduit" + "github.com/algorand/indexer/conduit/plugins" + "github.com/algorand/indexer/data" +) + +// Processor an interface that defines an object that can filter and process transactions +type Processor interface { + // PluginMetadata implement this interface. + conduit.PluginMetadata + + // Config returns the configuration options used to create an Processor. + Config() plugins.PluginConfig + + // Init will be called during initialization, before block data starts going through the pipeline. + // Typically, used for things like initializing network connections. + // The Context passed to Init() will be used for deadlines, cancel signals and other early terminations + // The Config passed to Init() will contain the unmarshalled config file specific to this plugin. + Init(ctx context.Context, initProvider data.InitProvider, cfg plugins.PluginConfig, logger *logrus.Logger) error + + // Close will be called during termination of the Indexer process. + Close() error + + // Process will be called with provided optional inputs. It is up to the plugin to check that required inputs are provided. + Process(input data.BlockData) (data.BlockData, error) +} diff --git a/conduit/plugins/processors/processor_factory.go b/conduit/plugins/processors/processor_factory.go new file mode 100644 index 000000000..f99e51143 --- /dev/null +++ b/conduit/plugins/processors/processor_factory.go @@ -0,0 +1,41 @@ +package processors + +import ( + "fmt" +) + +// ProcessorConstructor must be implemented by each Processor. +// It provides a basic no-arg constructor for instances of an ProcessorImpl. +type ProcessorConstructor interface { + // New should return an instantiation of a Processor. + // Configuration values should be passed and can be processed during `Init()`. + New() Processor +} + +// ProcessorConstructorFunc is Constructor implementation for processors +type ProcessorConstructorFunc func() Processor + +// New initializes a processor constructor +func (f ProcessorConstructorFunc) New() Processor { + return f() +} + +// Processors are the constructors to build processor plugins. +var Processors = make(map[string]ProcessorConstructor) + +// Register is used to register ProcessorConstructor implementations. This mechanism allows +// for loose coupling between the configuration and the implementation. It is extremely similar to the way sql.DB +// drivers are configured and used. +func Register(name string, constructor ProcessorConstructor) { + Processors[name] = constructor +} + +// ProcessorBuilderByName returns a Processor constructor for the name provided +func ProcessorBuilderByName(name string) (ProcessorConstructor, error) { + constructor, ok := Processors[name] + if !ok { + return nil, fmt.Errorf("no Processor Constructor for %s", name) + } + + return constructor, nil +} diff --git a/conduit/plugins/processors/processor_factory_test.go b/conduit/plugins/processors/processor_factory_test.go new file mode 100644 index 000000000..8f5a34e7a --- /dev/null +++ b/conduit/plugins/processors/processor_factory_test.go @@ -0,0 +1,54 @@ +package processors + +import ( + "testing" + + "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/assert" + + "github.com/algorand/indexer/conduit" +) + +var logger *logrus.Logger + +func init() { + logger, _ = test.NewNullLogger() +} + +type mockProcessor struct { + Processor +} + +func (m *mockProcessor) Metadata() conduit.Metadata { + return conduit.Metadata{ + Name: "foobar", + Description: "", + Deprecated: false, + SampleConfig: "", + } +} + +type mockProcessorConstructor struct { + me *mockProcessor +} + +func (c *mockProcessorConstructor) New() Processor { + return c.me +} + +func TestProcessorBuilderByNameSuccess(t *testing.T) { + me := mockProcessor{} + Register("foobar", &mockProcessorConstructor{&me}) + + expBuilder, err := ProcessorBuilderByName("foobar") + assert.NoError(t, err) + exp := expBuilder.New() + assert.Implements(t, (*Processor)(nil), exp) +} + +func TestProcessorBuilderByNameNotFound(t *testing.T) { + _, err := ProcessorBuilderByName("barfoo") + expectedErr := "no Processor Constructor for barfoo" + assert.EqualError(t, err, expectedErr) +} diff --git a/data/block_export_data.go b/data/block_export_data.go new file mode 100644 index 000000000..741e04573 --- /dev/null +++ b/data/block_export_data.go @@ -0,0 +1,116 @@ +package data + +import ( + "github.com/algorand/go-algorand/agreement" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/transactions" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/rpcs" +) + +// RoundProvider is the interface which all data types sent to Exporters should implement +type RoundProvider interface { + Round() uint64 + Empty() bool +} + +// InitProvider is the interface that can be used when initializing to get common algod related +// variables +type InitProvider interface { + GetGenesis() *bookkeeping.Genesis + NextDBRound() basics.Round +} + +// BlockData is provided to the Exporter on each round. +type BlockData struct { + + // BlockHeader is the immutable header from the block + BlockHeader bookkeeping.BlockHeader `json:"block,omitempty"` + + // Payset is the set of data the block is carrying--can be modified as it is processed + Payset []transactions.SignedTxnInBlock `json:"payset,omitempty"` + + // Delta contains a list of account changes resulting from the block. Processor plugins may have modify this data. + Delta *ledgercore.StateDelta `json:"delta,omitempty"` + + // Certificate contains voting data that certifies the block. The certificate is non deterministic, a node stops collecting votes once the voting threshold is reached. + Certificate *agreement.Certificate `json:"cert,omitempty"` +} + +// MakeBlockDataFromValidatedBlock makes BlockData from agreement.ValidatedBlock +func MakeBlockDataFromValidatedBlock(input ledgercore.ValidatedBlock) BlockData { + blockData := BlockData{} + blockData.UpdateFromValidatedBlock(input) + return blockData +} + +// UpdateFromValidatedBlock updates BlockData from ValidatedBlock input +func (blkData *BlockData) UpdateFromValidatedBlock(input ledgercore.ValidatedBlock) { + blkData.BlockHeader = input.Block().BlockHeader + blkData.Payset = input.Block().Payset + delta := input.Delta() + blkData.Delta = &delta +} + +// UpdateFromEncodedBlockCertificate updates BlockData from EncodedBlockCert info +func (blkData *BlockData) UpdateFromEncodedBlockCertificate(input *rpcs.EncodedBlockCert) { + if input == nil { + return + } + + blkData.BlockHeader = input.Block.BlockHeader + blkData.Payset = input.Block.Payset + + cert := input.Certificate + blkData.Certificate = &cert +} + +// MakeBlockDataFromEncodedBlockCertificate makes BlockData from rpcs.EncodedBlockCert +func MakeBlockDataFromEncodedBlockCertificate(input *rpcs.EncodedBlockCert) BlockData { + blkData := BlockData{} + blkData.UpdateFromEncodedBlockCertificate(input) + return blkData +} + +// ValidatedBlock returns a validated block from the BlockData object +func (blkData BlockData) ValidatedBlock() ledgercore.ValidatedBlock { + tmpBlock := bookkeeping.Block{ + BlockHeader: blkData.BlockHeader, + Payset: blkData.Payset, + } + + tmpDelta := ledgercore.StateDelta{} + if blkData.Delta != nil { + tmpDelta = *blkData.Delta + } + return ledgercore.MakeValidatedBlock(tmpBlock, tmpDelta) +} + +// EncodedBlockCertificate returns an encoded block certificate from the BlockData object +func (blkData BlockData) EncodedBlockCertificate() rpcs.EncodedBlockCert { + + tmpBlock := bookkeeping.Block{ + BlockHeader: blkData.BlockHeader, + Payset: blkData.Payset, + } + + tmpCert := agreement.Certificate{} + if blkData.Certificate != nil { + tmpCert = *blkData.Certificate + } + return rpcs.EncodedBlockCert{ + Block: tmpBlock, + Certificate: tmpCert, + } +} + +// Round returns the round to which the BlockData corresponds +func (blkData BlockData) Round() uint64 { + return uint64(blkData.BlockHeader.Round) +} + +// Empty returns whether the Block contains Txns. Assumes the Block is never nil +func (blkData BlockData) Empty() bool { + return len(blkData.Payset) == 0 +} diff --git a/docs/Conduit.md b/docs/Conduit.md new file mode 100644 index 000000000..1d252c0e6 --- /dev/null +++ b/docs/Conduit.md @@ -0,0 +1,68 @@ +
+ + Algorand + + + + +[![CircleCI](https://img.shields.io/circleci/build/github/algorand/indexer/develop?label=develop)](https://circleci.com/gh/algorand/indexer/tree/develop) +[![CircleCI](https://img.shields.io/circleci/build/github/algorand/indexer/master?label=master)](https://circleci.com/gh/algorand/indexer/tree/master) +![Github](https://img.shields.io/github/license/algorand/indexer) +[![Contribute](https://img.shields.io/badge/contributor-guide-blue?logo=github)](https://github.com/algorand/go-algorand/blob/master/CONTRIBUTING.md) +
+ +# Algorand Conduit + +Conduit is a framework which provides reusable components necessary for ingesting blocks from the Algorand blockchain into external applications. It is designed around a modular plugin system that allows users to configure their own data pipelines for filtering, aggregation, and storage of transactions and accounts on any Algorand network. + +## Building from source + +Development is done using the [Go Programming Language](https://golang.org/), the version is specified in the project's [go.mod](go.mod) file. + +Run `make` to build Conduit, the binary is located at `cmd/algorand-indexer/conduit`. + +# Quickstart + +See the [Getting Started](conduit/GettingStarted.md) page. + +# Configuration + +See the [Configuration](conduit/Configuration.md) page. + +# Develoment + +See the [Development](conduit/Development.md) page for building a plugin. + +# Features + +# Plugin System +A Conduit pipeline is composed of 3 components, [Importers](../conduit/plugins/importers/README.md), [Processors](../conduit/plugins/processors/README.md), and [Exporters](../conduit/plugins/exporters/README.md). +Every pipeline must define exactly 1 Importer, 1 Exporter, and can optionally define a series of 0 or more Processors. + +The original Algorand Indexer has been defined as a Conduit pipeline via the [algorand-indexer](../cmd/algorand-indexer/daemon.go) executable, see [Migrating from Indexer](#migrating-from-indexer) + +# Contributing + +Contributions welcome! Please refer to our [CONTRIBUTING](https://github.com/algorand/go-algorand/blob/master/CONTRIBUTING.md) document for general contribution guidelines, and individual plugin documentation for contributing to new and existing Conduit plugins. + +## RFCs +If you have an idea for how to improve Conduit that would require significant changes, open a [Feature Request Issue](https://github.com/algorand/indexer/issues/new/choose) to begin discussion. If the proposal is accepted, the next step is to define the technical direction and answer implementation questions via a PR containing an [RFC](./rfc/template.md). + +You do _not_ need to open an RFC for adding a new plugin--follow the development guide for the corresponding plugin type. + + + +# Common Setups + +The most common usage of Conduit is expecting to be getting validated blocks from a local `algod` Algorand node, adding them to a database (such as [PostgreSQL](https://www.postgresql.org/)), and serving an API to make available a variety of prepared queries. Some users may wish to directly write SQL queries of the database. + +Conduit works by fetching blocks one at a time via the configured Importer, sending the block data through the configured Processors, and terminating block handling via an Exporter (traditionally a database). + + + + +# Migrating from Indexer + +Indexer was built in a way that strongly coupled it to Postgresql, and the defined REST API. We've built Conduit in a way which is backwards compatible with the preexisting Indexer application. Running the `algorand-indexer` binary will use Conduit to construct a pipeline that replicates the Indexer functionality. + +Going forward we will continue to maintain the Indexer application, however our main focus will be enabling and optimizing a multitude of use cases through the Conduit pipeline design rather the singular Indexer pipeline. diff --git a/docs/conduit/Configuration.md b/docs/conduit/Configuration.md new file mode 100644 index 000000000..a02b384c4 --- /dev/null +++ b/docs/conduit/Configuration.md @@ -0,0 +1,50 @@ +# Configuration + +Configuration is stored in a file in the data directory named `conduit.yml`. +Use `./conduit -h` for command options. + +## conduit.yml + +There are several top level configurations for configuring behavior of the conduit process. Most detailed configuration is made on a per-plugin basis. These are split between `Importer`, `Processor` and `Exporter` plugins. + +Here is an example configuration which shows the general format: +```yaml +# optional: if present perform runtime profiling and put results in this file. +cpu-profile: "path to cpu profile file." + +# optional: maintain a pidfile for the life of the conduit process. +pid-filepath: "path to pid file." + +# optional: level to use for logging. +log-level: "INFO, WARN, ERROR" + +# optional: path to log file +log-file: "" + +# optional: setting to turn on Prometheus metrics server +metrics: + mode: "ON, OFF" + addr: ":" + +# Define one importer. +importer: + name: + config: + +# Define one or more processors. +processors: + - name: + config: + - name: + config: + +# Define one exporter. +exporter: + name: + config: +``` + +## Plugin configuration + +See [plugin list](plugins/home.md) for details. +Each plugin is identified by a `name`, and provided the `config` during initialization. diff --git a/docs/conduit/Development.md b/docs/conduit/Development.md new file mode 100644 index 000000000..9e8e05d02 --- /dev/null +++ b/docs/conduit/Development.md @@ -0,0 +1,80 @@ +# Creating A Plugin + +There are three different interfaces to implement, depending on what sort of functionality you are adding: +* Importer: for sourcing data into the system. +* Processor: for manipulating data as it goes through the system. +* Exporter: for sending processed data somewhere. + +All plugins should be implemented in the respective `importers`, `processors`, or `exporters` package. + +# Registering a plugin + +## Register the Constructor + +The constructor is registered to the system by name in the init this is how the configuration is able to dynamically create pipelines: +``` +func init() { + exporters.RegisterExporter(noopExporterMetadata.ExpName, exporters.ExporterConstructorFunc(func() exporters.Exporter { + return &noopExporter{} + })) +} +``` + +There are similar interfaces for each plugin type. + +## Load the Plugin + +Each plugin package contains an `all.go` file. Add your plugin to the import statement, this causes the init function to be called and ensures the plugin is registered. + +# Implement the interface + +Generally speaking, you can follow the code in one of the existing plugins. + +# Lifecycle + +## Init + +Each plugin will have it's `Init` function called once as the pipline is constructed. + +The context provided to this function should be saved, and used to terminate any long-running operations if necessary. + +## Per-round function + +Each plugin type has a function which is called once per round: +* Importer: `GetBlock` called when a particular round is required. Generally this will be increasing over time. +* Processor: `Process` called to process a round. +* Exporter: `Receive` for consuming a round. + +## Close + +Called during a graceful shutdown. We make every effort to call this function, but it is not guaranteed. + +## Hooks + +There are special lifecycle hooks that can be registered on any plugin by implementing additional interfaces. + +### Completed + +When all processing has completed for a round, the `OnComplete` function is called on any plugin that implements it. + +```go +// Completed is called by the conduit pipeline after every exporter has +// finished. It can be used for things like finalizing state. +type Completed interface { + // OnComplete will be called by the Conduit framework when the pipeline + // finishes processing a round. + OnComplete(input data.BlockData) error +} +``` + +### PluginMetrics + +After the pipeline has been initialized, and before it has been started, plugins may provide prometheus metric handlers. +The ProvideMetrics function will only be called once. + +```go +// PluginMetrics is for defining plugin specific metrics +type PluginMetrics interface { + ProvideMetrics() []prometheus.Collector +} +``` diff --git a/docs/conduit/GettingStarted.md b/docs/conduit/GettingStarted.md new file mode 100644 index 000000000..173e345eb --- /dev/null +++ b/docs/conduit/GettingStarted.md @@ -0,0 +1,17 @@ +# Getting Started + +## Install from Source + +1. Checkout the repo, or download the source. +2. Run `make conduit`. +3. The binary is created at `cmd/conduit/conduit`. + +## Quick Start + +1. Run `./conduit init` to setup a sample data directory. +2. Follow the on screen instructions to update the sample configuration in `data/conduit.yml`. +3. Run `./conduit -d data` to launch conduit with the `data` directory. + +# Configuration + +See [Configuration.md](Configuration.md) for more details. diff --git a/docs/conduit/plugins/algod.md b/docs/conduit/plugins/algod.md new file mode 100644 index 000000000..bcc238c3c --- /dev/null +++ b/docs/conduit/plugins/algod.md @@ -0,0 +1,16 @@ +# Algod Importer + +Fetch blocks one by one from the [algod REST API](https://developer.algorand.org/docs/rest-apis/algod/v2/). The node must be configured as an archival node in order to +provide old blocks. + +Block data from the Algod REST API contains the block header, transactions, and a vote certificate. + +# Config +```yaml +importer: + name: algod + config: + - netaddr: "algod URL" + token: "algod REST API token" +``` + diff --git a/docs/conduit/plugins/block_evaluator.md b/docs/conduit/plugins/block_evaluator.md new file mode 100644 index 000000000..2f77bec0e --- /dev/null +++ b/docs/conduit/plugins/block_evaluator.md @@ -0,0 +1,38 @@ +# Block Evaluator + +Enrich raw block data with a "State Delta" object which contains new account values. For example: new account balances, new application states, which assets have been created or deleted, etc. + +The block evaluator computes the delta by maintaining a local ledger and evaluating each block locally. + +State Delta's are required by some exporters. + +## algod requirement + +It is sometimes useful, or necessary to re-initialize the local ledger. To expedite that process a direct connection to algod is used. + +This is configured by providing one of: +* `algod-data-dir`: this will automatically lookup the algod address and token. This is convenient when algod is installed locally to conduit. +* `algod-addr` and `algod-token`: explicitly provide the algod connection information. + +## catchpoint + +Fast catchup can be used for Mainnet, Testnet and Betanet. To use, lookup the most recent catchpoint for your network **without going over the desired round**. Use the following links: +* [Mainnet](https://algorand-catchpoints.s3.us-east-2.amazonaws.com/consolidated/mainnet_catchpoints.txt) +* [Testnet](https://algorand-catchpoints.s3.us-east-2.amazonaws.com/consolidated/testnet_catchpoints.txt) +* [Betanet](https://algorand-catchpoints.s3.us-east-2.amazonaws.com/consolidated/betanet_catchpoints.txt) + +For example, if you want to get **Mainnet** round `22212765`, you would refer to the Mainnet link above and find: +* `22210000#MZZIOYXYPPGNYRQHROXCPILIWIMQQRN7ZNLQJVM2QVSKT3QX6O4A` + +# Config +```yaml +processors: + - name: block_evaluator + config: + - data-dir: "location where the local ledger will be stored." + algod-data-dir: "local algod data directory" + algod-addr: "algod URL" + algod-token: "algod token" + catchpoint: "a catchpoint ID" +``` + diff --git a/docs/conduit/plugins/file_writer.md b/docs/conduit/plugins/file_writer.md new file mode 100644 index 000000000..84f0f0538 --- /dev/null +++ b/docs/conduit/plugins/file_writer.md @@ -0,0 +1,18 @@ +# Filewriter Exporter + +Write the block data to a file. + +Data is written to one file per block in JSON format. + +# Config +```yaml +exporter: + - name: file_writer + config: + - block-dir: "path to write block data" + # override the filename pattern. + filename-pattern: "%[1]d_block.json" + # exclude the vote certificate from the file. + drop-certificate: false +``` + diff --git a/docs/conduit/plugins/filter_processor.md b/docs/conduit/plugins/filter_processor.md new file mode 100644 index 000000000..dde711e55 --- /dev/null +++ b/docs/conduit/plugins/filter_processor.md @@ -0,0 +1,66 @@ +# Filter Processor + +This is used to filter transactions to include only the ones that you want. This may be useful for some deployments +which only require specific applications or accounts. + +## any / all +One or more top-level operations should be provided. +* any: transactions are included if they match `any` of the nested sub expressions. +* all: transactions are included if they match `all` of the nested sub expressions. + +If `any` and `all` are both provided, the transaction must pass both checks. + +## Sub expressions + +Parts of an expression: +* `tag`: the transaction field being considering. +* `expression-type`: The type of expression. +* `expression`: Input to the expression + +### tag +The full path to a given field. Uses the messagepack encoded names of a canonical transaction. For example: +* `txn.snd` is the sender. +* `tsn.amt` is the amount. + +For information about the structure of transactions, refer to the [Transaction Structure](https://developer.algorand.org/docs/get-details/transactions/) documentation. For detail about individual fields, refer to the [Transaction Reference](https://developer.algorand.org/docs/get-details/transactions/transactions/) documentation. + +**Note**: The "Apply Data" information is also available for filtering. These fields are not well documented. Advanced users can inspect raw transactions returned by algod to see what fields are available. + +### expression-type + +What type of expression to use for filtering the tag. +* `exact`: exact match for string values. +* `regex`: applies regex rules to the matching. +* `less-than` applies numerical less than expression. +* `less-than-equal` applies numerical less than or equal expression. +* `greater-than` applies numerical greater than expression. +* `greater-than-equal` applies numerical greater than or equal expression. +* `equal` applies numerical equal expression. +* `not-equal` applies numerical not equal expression. + +### expression + +The input to the expression. A number or string depending on the expression type. + +# Config +```yaml +processors: + - name: filter_processor + config: + - filters: + - any + - tag: + expression-type: + expression: + - tag: + expression-type: + expression: + - all + - tag: + expression-type: + expression: + - tag: + expression-type: + expression: +``` + diff --git a/docs/conduit/plugins/home.md b/docs/conduit/plugins/home.md new file mode 100644 index 000000000..a04cb1bf6 --- /dev/null +++ b/docs/conduit/plugins/home.md @@ -0,0 +1,19 @@ +# Plugin Configuration + +Each plugin is identified by a `name`, and provided the `config` during initialization. + +## Importers + +* [algod](algod.md) +* [file_reader](file_reader.md) + +## Processors +* [block_evaluator](block_evaluator.md) +* [filter_processor](filter_processor.md) +* [noop_processor](noop_processor.md) + +## Exporters +* [file_writer](file_writer.md) +* [postgresql](postgresql.md) +* [noop_exporter](noop_exporter.md) + diff --git a/docs/conduit/plugins/noop_exporter.md b/docs/conduit/plugins/noop_exporter.md new file mode 100644 index 000000000..f9b074c22 --- /dev/null +++ b/docs/conduit/plugins/noop_exporter.md @@ -0,0 +1,11 @@ +# Noop Exporter + +For testing purposes, the noop processor discards any data it receives. + +# Config +```yaml +processors: + - name: noop + config: +``` + diff --git a/docs/conduit/plugins/noop_processor.md b/docs/conduit/plugins/noop_processor.md new file mode 100644 index 000000000..ed696d0a0 --- /dev/null +++ b/docs/conduit/plugins/noop_processor.md @@ -0,0 +1,11 @@ +# Noop Processor + +For testing purposes, the noop processor simply passes the input to the output. + +# Config +```yaml +processors: + - name: noop + config: +``` + diff --git a/docs/conduit/plugins/postgresql.md b/docs/conduit/plugins/postgresql.md new file mode 100644 index 000000000..101515e1f --- /dev/null +++ b/docs/conduit/plugins/postgresql.md @@ -0,0 +1,23 @@ +# PostgreSQL Exporter + +Write block data to a postgres database with the Indexer REST API schema. + +## Connection string + +We are using the [pgx](https://github.com/jackc/pgconn) database driver, which dictates the connection string format. + +For most deployments, you can use the following format: +`host={url} port={port} user={user} password={password} dbname={db_name} sslmode={enable|disable}` + +For additional details, refer to the [parsing documentation here](https://pkg.go.dev/github.com/jackc/pgx/v4/pgxpool@v4.11.0#ParseConfig). + +# Config +```yaml +exporter: + - name: postgresql + config: + - connection-string: "postgres connection string" + max-conn: "connection pool setting, maximum active queries" + test: "a boolean, when true a mock database is used" +``` + diff --git a/docs/rfc/0001-exporter-interface.md b/docs/rfc/0001-exporter-interface.md index 56a20fedc..ae4dea7bf 100644 --- a/docs/rfc/0001-exporter-interface.md +++ b/docs/rfc/0001-exporter-interface.md @@ -76,9 +76,6 @@ type Exporter interface { // Receive is called for each block to be processed by the exporter. Receive(exportData ExportData) error - // HandleGenesis is an Exporter's opportunity to do initial validation and handling of the Genesis block. - HandleGenesis(genesis bookkeeping.Genesis) error - // Round returns the next round not yet processed by the Exporter. Atomically updated when Receive successfully completes. Round() uint64 } diff --git a/docs/rfc/0002-processor-interface.md b/docs/rfc/0002-processor-interface.md new file mode 100644 index 000000000..7f0b93bfc --- /dev/null +++ b/docs/rfc/0002-processor-interface.md @@ -0,0 +1,151 @@ +# Processor Interface RFC + +- Contribution Name: Processor Interface/Plugin Framework +- Implementation Owner: Stephen Akiki + +## Problem Statement + +Users of the Indexer each have their own needs when it comes to the data gathered by the Indexer. Whether it is +extracting certain fields of information or transforming it entirely, the data gathered is something that every user +consumes in a different way. + +Currently, users must "drink from a firehouse" of information in that all information is presented at all times. No +custom "pre-processing" of the data can be done so no derivative information can be created. This means that the user +must save all the data and only after that is accomplished can they use the information. This can result in an +incredible amount of "wasted" storage if the user only needs to save a derived value and not all underlying data. + +Additionally, there might be various transformations that are useful enough that one would wish to "pipe" them from one +to another, similar to how commands are piped in Linux. This would allow for modules to be saved and re-used across +Indexer deployments. + +## Proposal + +### Plugin Interface + +A sample interface and pipeline effort is shown below: + +```go +package processor + +// Config will act as a placeholder for now. It will end up providing an interface for +// serialization/deserialization of config files. +// Derived types will provide plugin-specific data fields. +type Config string + +type MetaData string + +type Processor interface { + // Metadata associated with each Processor. + Metadata() MetaData + + // Config returns the configuration options used to create an Processor. + Config() Config + + // Init will be called during initialization, before block data starts going through the pipeline. + // Typically, used for things like initializing network connections. + // The Context passed to Init() will be used for deadlines, cancel signals and other early terminations + // The Config passed to Init() will contain the unmarshalled config file specific to this plugin. + Init(ctx context.Context, initProvider data.InitProvider, cfg plugins.PluginConfig) error + + // Close will be called during termination of the Indexer process. + Close() error + + // Process will be called with provided optional inputs. It is up to the plugin to check that required inputs are provided. + Process(input data.BlockData) (data.BlockData, error) +} +``` + +### Interface Description + +#### Metadata and Configuration + +The `Metadata()` and `Config()` functions are used to get the supplied information of the interface. The information +will be returned as a string, which can be used along-side conduit tooling to de-serialize into appropriate objects. + +Plugin Metadata shall adhere to the Conduit `PluginMetadata` interface and return the appropriate type depending on the +defined plugin. For more info, see Conduit documentation. + +Likewise, plugin Config data shall be a string that is convertable from the YAML specification into a Go-Lang object. +This is also a common interface defined in the Conduit framework. + +#### Init and Close + +The `Init()` and `Close()` functions serve as "hooks" into the Conduit start up interface. They allow the plugin writer +to write startup and shutdown functions in a sequence defined by the Conduit framework. An example would be to have a +processor setup stateful information (like a database or special file) during the `Init()` call and flush to disk and/or +close that stateful information on `Close()`. + +The `Init()` function is given a configuration parameter which it can use to determine how to initialize. The +configuration is a string type that can be de-serialized by the Conduit framework into an object that can be +programmatically accessed. It is also given a `context` parameter which will be used by the Conduit framework to +terminate the runtime early if needed. Finally, a `InitProvider` parameter is provided so that a processor can access +information needed from the blockchain. + +Note: the `Close()` function will only be called on an orderly shutdown of the Conduit framework. The Conduit framework +can not guarantee this function is called if any abnormal shutdown (i.e. signals, un-caught panics) occurs. + + + +Conduit documentation will delve into further detail about order of operations and how these functions will be called +relative to the lifecycle of the plugin and the framework itself. + +#### Process + +The `Process()` function is the heart of any processor. It is given an input parameter of type `BlockData` and is +expected to perform processing based on these inputs. However, to propagate the processing done on the input data one +must return a corresponding `BlockData` object in the return value. + +The `BlockData` object is a common object shared among all plugins within the Conduit framework. It is passed into the +processor plugin via the input in an already initialized and correctly formed state. + +Valid processing of the block is signified by having the returned `error` value be nil. If this is the case then the +output `BlockData` will be assumed to be correctly processed and be free of any errors. **It is imperative that if +the `error` value is nil then the `BlockData` is assumed to contain all valid data.** + +It is valid to return the same `BlockData` values provided in the input as the output if one wishes to perform a "no +operation" on the block. However, note that further plugins (whether more processors and/or exporters) will take action +on this data. + +It is also valid to return an "empty" block. An empty block is one in which `BlockData.Empty()` returns true without any +errors. Returning an empty block could indicate that all data has been filtered out which would cause all downstream +plugins to perform no operation. + +Below is a collection of common scenarios and the appropriate way to fill out the data in the output `BlockData`: + +**Scenario 1: An error occurred when processing the block** + +The values within the output `BlockData` can be anything, however the returned error must be non-nil. This represents +that the data contained within the `BlockData` is incorrect and/or could not be processed. Depending on the error type +returned, this could result in a retry of the pipeline but isn't guaranteed to. + +**Scenario 2: Some processing occurred successfully** + +The values within the output `BlockData` match what the processing says should have happened and the returned error must +be nil. + +**Scenario 3: Processing filtered out all data** + +If a processor has filtered out all data, then it will have the `Payset` field set to nil. This will indicate that no +transactions are available to process. + +##### Default Output Values + +If a plugin does not modify a variable of the output `BlockData` then the value of the variable shall be what the +corresponding variable was in the input `BlockData`. + +#### On Complete + +Often it is useful to know when a block has passed through the entire Conduit framework and has successfully been written by an Exporter. For instance, +a stateful Processor plugin might want to commit block information to a private database once it is certain that the block information has been fully processed. + +The `OnComplete()` function will be called by the Conduit framework when the exporter plugin has finished executing **and** did not return any errors. The data supplied to this function +will be the final block data written to disk. Note that this block data may have been processed by other Processor plugins since it was last "seen" by any specific Processor plugin. + +It is implemented as a dynamic interface and can be added to any plugin. +```go +type Completed interface { + // OnComplete will be called by the Conduit framework when the exporter has successfully written the block + // The input will be the finalized information written to disk + OnComplete(input data.BlockData) error +} +``` diff --git a/docs/rfc/0003-importer-interface.md b/docs/rfc/0003-importer-interface.md new file mode 100644 index 000000000..bf0911259 --- /dev/null +++ b/docs/rfc/0003-importer-interface.md @@ -0,0 +1,58 @@ +# Importer Interface RFC + +- Contribution Name: Importer Interface/Plugin Framework +- Implementation Owner: Ganesh Vanahalli + +## Problem Statement + +Users would like to choose how to fetch blocks, either by downloading from an algod rest endpoint or directly from the gossip network or read blocks from a file which was previously written by the exporter. + +The current implementation of indexer only allows downloading blocks sequentially from an algod rest endpoint starting at the round number of recently written block to postgres db and users cannot choose a specific block to download. + +## Proposal + +Importer interface allows users to fetch any particular block either from algod rest endpoint (or directly from the network or from a file written to by the exporter plugin). + +### Plugin Interface +Importer plugins that are native to the Indexer (maintained within the Indexer repository) will each implementation the importer interface: + +```GO +package importers + +// Importer defines the interface for importer plugins +type Importer interface { + // Metadata associated with each Importer. + Metadata() ImporterMetadata + + // Init will initialize each importer with a given config. This config will contain the Unmarhsalled config file specific to this plugin. + // It is called during initialization of an importer plugin such as setting up network connections, file buffers etc. + Init(ctx context.Context, cfg plugins.PluginConfig, logger *logrus.Logger) error + + // Config returns the configuration options used to create an Importer. Initialized during Init. + Config() plugins.PluginConfig + + // Close function is used for closing network connections, files, flushing buffers etc. + Close() error + + // GetBlock, given any round number-rnd fetches the block at that round + // It returns an object of type BlockData defined in data + GetBlock(rnd uint64) (data.BlockData, error) +} +``` + +### Interface Description + +#### Metadata +This function returns the metadata associated with an importer plugin that implements the above importer interface + +#### Init +This function is used to initialize the importer plugin such as establishing network connections, file buffers, context etc. + +#### Config +This function returns the configuration options used to create an Importer. Initialized during Init. + +#### Close +This function is used to close/end resources used by the importer plugin, such as closing of open network connections, file buffers, context etc. + +#### GetBlock +Every importer plugin implements GetBlock function that fetches a block at given round number and returns this data in a standardized format (data.BlockData). diff --git a/e2e_tests/README.md b/e2e_tests/README.md new file mode 100644 index 000000000..f51b271a5 --- /dev/null +++ b/e2e_tests/README.md @@ -0,0 +1,65 @@ +# e2elive + +An integration testing tool. It leverages an artifact generated by the go-algorand integration tests to ensure all features are exercised. + +# Usage + +A python virtual environment is highly encouraged. + +## Setup environment with venv +```sh +cd e2e_tests + +# create venv dir, I believe you could specify the version of python here +python3 -m venv venv_dir + +# This command depends on your shell +source venv_dir/bin/activate.fish + +# Install 'e2elive' and 'e2econduit' +python3 -m pip install . +``` + +## Running locally + + +### e2elive + +This tests runs the `algorand-indexer` binary. Depending on what you're doing, you may want to use a different value for `--s3-source-net`. After the test finishes, a few simple queries are executed. + +This command requires a postgres database, you can run one in docker: + +```sh +docker run --rm -it --name some-postgres -p 5555:5432 -e POSTGRES_PASSWORD=pgpass -e POSTGRES_USER=pguser -e POSTGRES_DB=mydb postgres +``` + +Running e2elive + +**Note**: you will need to restart this container between tests, otherwise the database will already be initialized. Being out of sync with the data directory may cause unexpected behavior and failing tests. +```sh +e2elive --connection-string "host=127.0.0.1 port=5555 user=pguser password=pgpass dbname=mydb" --s3-source-net "fafa8862/rel-nightly" --indexer-bin ../cmd/algorand-indexer/algorand-indexer --indexer-port 9890 +``` + +### e2econduit + +This runs a series of pipeline scenarios with conduit. Each module manages its own resources and may have dependencies. For example the PostgreSQL exporter starts a docker container to initialize a database server. +```sh +e2econduit --s3-source-net rel-nightly --conduit-bin ../cmd/conduit/conduit +``` + +## Run with docker + +The e2elive script has a docker configuration available. + +```sh +make e2e +``` + +You may need to set (and export) the `CI_E2E_FILENAME` environment variable. + +```sh +export CI_E2E_FILENAME=rel-nightly +make e2e +``` + +Alternatively, modify `e2e_tests/docker/indexer/Dockerfile` by swapping out `$CI_E2E_FILENAME` with the test artifact you want to use. See `.circleci/config.yml` for the current values used by CI. diff --git a/misc/Dockerfile b/e2e_tests/docker/indexer/Dockerfile similarity index 63% rename from misc/Dockerfile rename to e2e_tests/docker/indexer/Dockerfile index 5a02ad1db..e81efb612 100644 --- a/misc/Dockerfile +++ b/e2e_tests/docker/indexer/Dockerfile @@ -13,14 +13,16 @@ RUN mkdir -p /opt/go/indexer COPY . /opt/go/indexer WORKDIR /opt/go/indexer RUN rm -f $HOME/go/bin/algod -RUN rm /opt/go/indexer/cmd/algorand-indexer/algorand-indexer +RUN rm -f /opt/go/indexer/cmd/algorand-indexer/algorand-indexer WORKDIR /opt/go/indexer/third_party/go-algorand RUN make install WORKDIR /opt/go/indexer RUN make ENV PATH="${HOME}/go/bin/:${PATH}" -RUN pip3 install -r misc/requirements.txt +WORKDIR ./e2e_tests +RUN pip3 install ./ ENV INDEXER_DATA="${HOME}/indexer/" +WORKDIR /opt/go/indexer # Run test script -ENTRYPOINT ["/bin/bash", "-c", "sleep 5 && python3 misc/e2elive.py --connection-string \"$CONNECTION_STRING\" --s3-source-net \"$CI_E2E_FILENAME\" --indexer-bin /opt/go/indexer/cmd/algorand-indexer/algorand-indexer --indexer-port 9890"] +ENTRYPOINT ["/bin/bash", "-c", "sleep 5 && e2elive --connection-string \"$CONNECTION_STRING\" --s3-source-net \"$CI_E2E_FILENAME\" --indexer-bin /opt/go/indexer/cmd/algorand-indexer/algorand-indexer --indexer-port 9890"] diff --git a/misc/docker-compose.yml b/e2e_tests/docker/indexer/docker-compose.yml similarity index 83% rename from misc/docker-compose.yml rename to e2e_tests/docker/indexer/docker-compose.yml index 56cd6442e..a6c05022c 100644 --- a/misc/docker-compose.yml +++ b/e2e_tests/docker/indexer/docker-compose.yml @@ -3,8 +3,8 @@ version: '3' services: e2e: build: - context: ../. - dockerfile: ./misc/Dockerfile + context: ../../../. + dockerfile: ./e2e_tests/docker/indexer/Dockerfile environment: CONNECTION_STRING: "host=e2e-db port=5432 user=algorand password=algorand dbname=indexer_db sslmode=disable" CI_E2E_FILENAME: "${CI_E2E_FILENAME}" @@ -18,4 +18,3 @@ services: POSTGRES_PASSWORD: algorand POSTGRES_DB: indexer_db - diff --git a/e2e_tests/pyproject.toml b/e2e_tests/pyproject.toml new file mode 100644 index 000000000..37f9ea1f5 --- /dev/null +++ b/e2e_tests/pyproject.toml @@ -0,0 +1,23 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "indexer_e2e_tests" +description = "End to end tests for Algorand Indexer/Conduit" +version = "0.0.1" +requires-python = ">=3.8" +dependencies = [ + "boto3==1.24.71", + "msgpack==1.0.4", + "py-algorand-sdk==1.17.0", + "pytest==6.2.5", + "PyYAML==6.0", + "setuptools ==65.3.0", +] + + +[project.scripts] +e2elive = "e2e_indexer.e2elive:main" +e2econduit = "e2e_conduit.e2econduit:main" +validate-accounting = "e2e_indexer.validate_accounting:main" diff --git a/e2e_tests/src/e2e_common/__init__.py b/e2e_tests/src/e2e_common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/misc/util.py b/e2e_tests/src/e2e_common/util.py similarity index 87% rename from misc/util.py rename to e2e_tests/src/e2e_common/util.py index cf137200a..218b6cc44 100644 --- a/misc/util.py +++ b/e2e_tests/src/e2e_common/util.py @@ -4,6 +4,7 @@ import logging import os import random +import sqlite3 import subprocess import sys import time @@ -125,16 +126,16 @@ def atexitrun(cmd, *args, **kwargs): atexit.register(xrun, *cargs, **kwargs) -def find_indexer(indexer_bin, exc=True): - if indexer_bin: - return indexer_bin - # manually search local build and PATH for algorand-indexer - path = ["cmd/algorand-indexer"] + os.getenv("PATH").split(":") +def find_binary(binary, exc=True, binary_name="algorand-indexer"): + if binary: + return binary + # manually search local build and PATH for binary_name + path = [f"cmd/{binary_name}"] + os.getenv("PATH").split(":") for pd in path: - ib = os.path.join(pd, "algorand-indexer") + ib = os.path.join(pd, binary_name) if os.path.exists(ib): return ib - msg = "could not find algorand-indexer. use --indexer-bin or PATH environment variable." + msg = f"could not find {binary_name} at the provided location or PATH environment variable." if exc: raise Exception(msg) logger.error(msg) @@ -175,13 +176,14 @@ def report(): atexit.register(report) - response = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, MaxKeys=50) + response = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, MaxKeys=100) if (not response.get("KeyCount")) or ("Contents" not in response): raise Exception("nothing found in s3://{}/{}".format(bucket, prefix)) for x in response["Contents"]: path = x["Key"] haystack.append(path) - if desired_filename in path: + _, fname = path.rsplit("/", 1) + if fname == desired_filename: if outpath is None: if outdir is None: outdir = "." @@ -193,3 +195,20 @@ def report(): logger.warning("file not found in s3://{}/{}".format(bucket, prefix)) return found_needle + + +def countblocks(path): + db = sqlite3.connect(path) + cursor = db.cursor() + cursor.execute("SELECT max(rnd) FROM blocks") + row = cursor.fetchone() + cursor.close() + db.close() + return row[0] + + +def hassuffix(x, *suffixes): + for s in suffixes: + if x.endswith(s): + return True + return False diff --git a/e2e_tests/src/e2e_conduit/__init__.py b/e2e_tests/src/e2e_conduit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/e2e_tests/src/e2e_conduit/e2econduit.py b/e2e_tests/src/e2e_conduit/e2econduit.py new file mode 100644 index 000000000..59de0020b --- /dev/null +++ b/e2e_tests/src/e2e_conduit/e2econduit.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# + +import argparse +import logging +import os +import sys + +from e2e_common.util import find_binary +import e2e_conduit.fixtures.importers as importers +import e2e_conduit.fixtures.processors as processors +import e2e_conduit.fixtures.exporters as exporters +from e2e_conduit.runner import ConduitE2ETestRunner +from e2e_conduit.scenarios import scenarios +from e2e_conduit.scenarios.indexer_scenario import indexer_scenario + +logger = logging.getLogger(__name__) + + +def main(): + ap = argparse.ArgumentParser() + # TODO FIXME convert keep_temps to debug mode which will leave all resources running/around + # So files will not be deleted and docker containers will be left running + ap.add_argument("--keep-temps", default=False, action="store_true") + ap.add_argument( + "--conduit-bin", + default=None, + help="path to conduit binary, otherwise search PATH", + ) + ap.add_argument( + "--source-net", + help="Path to test network directory containing Primary and other nodes. May be a tar file.", + ) + ap.add_argument( + "--s3-source-net", + help="AWS S3 key suffix to test network tarball containing Primary and other nodes. Must be a tar bz2 file.", + ) + ap.add_argument("--verbose", default=False, action="store_true") + args = ap.parse_args() + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + sourcenet = args.source_net + source_is_tar = False + if not sourcenet: + e2edata = os.getenv("E2EDATA") + sourcenet = e2edata and os.path.join(e2edata, "net") + importer_source = sourcenet if sourcenet else args.s3_source_net + if importer_source: + scenarios.append(indexer_scenario(importer_source)) + + runner = ConduitE2ETestRunner(args.conduit_bin, keep_temps=args.keep_temps) + + success = True + for scenario in scenarios: + runner.setup_scenario(scenario) + if runner.run_scenario(scenario) != 0: + success = False + return 0 if success else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/e2e_tests/src/e2e_conduit/fixtures/__init__.py b/e2e_tests/src/e2e_conduit/fixtures/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/e2e_tests/src/e2e_conduit/fixtures/exporters/__init__.py b/e2e_tests/src/e2e_conduit/fixtures/exporters/__init__.py new file mode 100644 index 000000000..6f37e731d --- /dev/null +++ b/e2e_tests/src/e2e_conduit/fixtures/exporters/__init__.py @@ -0,0 +1 @@ +from e2e_conduit.fixtures.exporters.postgresql import PostgresqlExporter diff --git a/e2e_tests/src/e2e_conduit/fixtures/exporters/postgresql.py b/e2e_tests/src/e2e_conduit/fixtures/exporters/postgresql.py new file mode 100644 index 000000000..cfb53a622 --- /dev/null +++ b/e2e_tests/src/e2e_conduit/fixtures/exporters/postgresql.py @@ -0,0 +1,62 @@ +import logging +import random +import string +import sys +import time + +from e2e_conduit.fixtures.plugin_fixture import PluginFixture +from e2e_common.util import atexitrun, xrun + +logger = logging.getLogger(__name__) + + +class PostgresqlExporter(PluginFixture): + def __init__(self, max_conn=0): + self.user = "algorand" + self.password = "algorand" + self.db_name = "e2e_db" + # Should we have a random port here so that we can run multiple of these in parallel? + self.port = "45432" + self.container_name = "" + self.max_conn = max_conn + super().__init__() + + @property + def name(self): + return "postgresql" + + def setup(self, _): + self.container_name = "".join( + random.choice(string.ascii_lowercase) for i in range(10) + ) + try: + xrun( + [ + "docker", + "run", + "-d", + "--name", + self.container_name, + "-p", + f"{self.port}:5432", + "-e", + f"POSTGRES_PASSWORD={self.password}", + "-e", + f"POSTGRES_USER={self.user}", + "-e", + f"POSTGRES_DB={self.db_name}", + "postgres:13-alpine", + ] + ) + # Sleep 15 seconds to let postgresql start--otherwise conduit might fail on startup + time.sleep(15) + atexitrun(["docker", "kill", self.container_name]) + except Exception as exc: + logger.error(f"docker postgres container startup failed: {exc}") + sys.exit(1) + + def resolve_config_input(self): + self.config_input = { + "connection-string": f"host=localhost port={self.port} user={self.user} password={self.password} dbname={self.db_name} sslmode=disable", + "max-conn": self.max_conn, + } diff --git a/e2e_tests/src/e2e_conduit/fixtures/importers/__init__.py b/e2e_tests/src/e2e_conduit/fixtures/importers/__init__.py new file mode 100644 index 000000000..045f727a0 --- /dev/null +++ b/e2e_tests/src/e2e_conduit/fixtures/importers/__init__.py @@ -0,0 +1 @@ +from e2e_conduit.fixtures.importers.algod import AlgodImporter diff --git a/e2e_tests/src/e2e_conduit/fixtures/importers/algod.py b/e2e_tests/src/e2e_conduit/fixtures/importers/algod.py new file mode 100644 index 000000000..de8d9c3d8 --- /dev/null +++ b/e2e_tests/src/e2e_conduit/fixtures/importers/algod.py @@ -0,0 +1,104 @@ +import atexit +import glob +import logging +import os +import shutil +import tempfile + +import boto3 +from botocore.config import Config +from botocore import UNSIGNED + +from e2e_common.util import hassuffix, xrun, firstFromS3Prefix, countblocks, atexitrun +from e2e_conduit.fixtures.importers.importer_plugin import ImporterPlugin + +logger = logging.getLogger(__name__) + + +class AlgodImporter(ImporterPlugin): + def __init__(self, sourcenet): + self.algoddir = None + self.sourcenet = sourcenet + self.last = None + super().__init__() + + @property + def name(self): + return "algod" + + @property + def lastblock(self): + if self.last is None: + raise RuntimeError("algod importer has no blockfiles configured") + return self.last + + def resolve_config_input(self): + with open(os.path.join(self.algoddir, "algod.net"), "r") as algod_net: + self.config_input["netaddr"] = "http://" + algod_net.read().strip() + with open(os.path.join(self.algoddir, "algod.token"), "r") as algod_token: + self.config_input["token"] = algod_token.read().strip() + + def resolve_config_output(self): + self.config_output["algod_token"] = self.config_input["token"] + self.config_output["algod_net"] = self.config_input["netaddr"] + self.config_output["algod_data_dir"] = self.algoddir + + def setup(self, accumulated_config): + try: + conduit_dir = accumulated_config["conduit_dir"] + except KeyError as exc: + logger.error( + f"AlgodImporter needs to be provided with the proper config: {exc}" + ) + raise + source_is_tar = hassuffix( + self.sourcenet, ".tar", ".tar.gz", ".tar.bz2", ".tar.xz" + ) + if not (source_is_tar or (self.sourcenet and os.path.isdir(self.sourcenet))): + # Assuming self.sourcenet is an s3 source net + tarname = f"{self.sourcenet}.tar.bz2" + # fetch test data from S3 + bucket = "algorand-testdata" + + s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED)) + prefix = "indexer/e2e4" + if "/" in tarname: + cmhash_tarnme = tarname.split("/") + cmhash = cmhash_tarnme[0] + tarname =cmhash_tarnme[1] + prefix+="/"+cmhash + tarpath = os.path.join(conduit_dir, tarname) + else: + tarpath = os.path.join(conduit_dir, tarname) + success = firstFromS3Prefix(s3, bucket, prefix, tarname, outpath=tarpath) + if not success: + raise Exception( + f"failed to locate tarname={tarname} from AWS S3 path {bucket}/{prefix}" + ) + source_is_tar = True + self.sourcenet = tarpath + tempnet = os.path.join(conduit_dir, "net") + if source_is_tar: + xrun(["tar", "-C", conduit_dir, "-x", "-f", self.sourcenet]) + else: + xrun(["rsync", "-a", self.sourcenet + "/", tempnet + "/"]) + blockfiles = glob.glob( + os.path.join(conduit_dir, "net", "Primary", "*", "*.block.sqlite") + ) + self.last = countblocks(blockfiles[0]) + try: + xrun(["goal", "network", "start", "-r", tempnet]) + except Exception: + logger.error("failed to start private network, looking for node.log") + for root, dirs, files in os.walk(tempnet): + for f in files: + if f == "node.log": + p = os.path.join(root, f) + logger.error("found node.log: {}".format(p)) + with open(p) as nf: + for line in nf: + logger.error(" {}".format(line)) + raise + + atexitrun(["goal", "network", "stop", "-r", tempnet]) + self.algoddir = os.path.join(tempnet, "Primary") diff --git a/e2e_tests/src/e2e_conduit/fixtures/importers/importer_plugin.py b/e2e_tests/src/e2e_conduit/fixtures/importers/importer_plugin.py new file mode 100644 index 000000000..bbef9ff3d --- /dev/null +++ b/e2e_tests/src/e2e_conduit/fixtures/importers/importer_plugin.py @@ -0,0 +1,10 @@ +from e2e_conduit.fixtures.plugin_fixture import PluginFixture + + +class ImporterPlugin(PluginFixture): + def __init__(self): + super().__init__() + + @property + def lastblock(self): + raise NotImplementedError diff --git a/e2e_tests/src/e2e_conduit/fixtures/plugin_fixture.py b/e2e_tests/src/e2e_conduit/fixtures/plugin_fixture.py new file mode 100644 index 000000000..f633cf8c8 --- /dev/null +++ b/e2e_tests/src/e2e_conduit/fixtures/plugin_fixture.py @@ -0,0 +1,21 @@ +class PluginFixture: + def __init__(self): + self.config_input = {} + self.config_output = {} + + @property + def name(self): + raise NotImplementedError + + def resolve_config(self): + self.resolve_config_input() + self.resolve_config_output() + + def resolve_config_input(self): + pass + + def resolve_config_output(self): + pass + + def setup(self, accumulated_config): + raise NotImplementedError diff --git a/e2e_tests/src/e2e_conduit/fixtures/processors/__init__.py b/e2e_tests/src/e2e_conduit/fixtures/processors/__init__.py new file mode 100644 index 000000000..64e2beb09 --- /dev/null +++ b/e2e_tests/src/e2e_conduit/fixtures/processors/__init__.py @@ -0,0 +1 @@ +from e2e_conduit.fixtures.processors.block_evaluator import BlockEvaluator diff --git a/e2e_tests/src/e2e_conduit/fixtures/processors/block_evaluator.py b/e2e_tests/src/e2e_conduit/fixtures/processors/block_evaluator.py new file mode 100644 index 000000000..c7b5b8e54 --- /dev/null +++ b/e2e_tests/src/e2e_conduit/fixtures/processors/block_evaluator.py @@ -0,0 +1,40 @@ +import logging + +from e2e_conduit.fixtures.plugin_fixture import PluginFixture + +logger = logging.getLogger(__name__) + + +class BlockEvaluator(PluginFixture): + def __init__(self, catchpoint=""): + self.catchpoint = catchpoint + self.algod_data_dir = None + self.algod_token = None + self.algod_net = None + self.indexer_data_dir = None + super().__init__() + + @property + def name(self): + return "block_evaluator" + + def setup(self, accumulated_config): + try: + self.algod_data_dir = accumulated_config["algod_data_dir"] + self.algod_token = accumulated_config["algod_token"] + self.algod_net = accumulated_config["algod_net"] + self.indexer_data_dir = accumulated_config["conduit_dir"] + except KeyError as exc: + logger.error( + f"BlockEvaluator must be provided with the proper config values {exc}" + ) + raise + + def resolve_config_input(self): + self.config_input = { + "catchpoint": self.catchpoint, + "data-dir": self.indexer_data_dir, + "algod-data-dir": self.algod_data_dir, + "algod-token": self.algod_token, + "algod-addr": self.algod_net, + } diff --git a/e2e_tests/src/e2e_conduit/runner.py b/e2e_tests/src/e2e_conduit/runner.py new file mode 100644 index 000000000..ce296174e --- /dev/null +++ b/e2e_tests/src/e2e_conduit/runner.py @@ -0,0 +1,102 @@ +import atexit +import os +import logging +import shutil +import subprocess +import sys +import tempfile +import time + +import yaml + +from e2e_common.util import find_binary +from e2e_conduit.subslurp import subslurp + +logger = logging.getLogger(__name__) + + +class ConduitE2ETestRunner: + def __init__(self, conduit_bin, keep_temps=False): + self.conduit_bin = find_binary(conduit_bin, binary_name="conduit") + self.keep_temps = keep_temps + + def setup_scenario(self, scenario): + + # Setup conduit_dir for conduit data dir + scenario.conduit_dir = tempfile.mkdtemp() + if not self.keep_temps: + atexit.register(shutil.rmtree, scenario.conduit_dir, onerror=logger.error) + else: + logger.info(f"leaving temp dir {scenario.conduit_dir}") + + scenario.accumulated_config = { + "conduit_dir": scenario.conduit_dir, + } + + for plugin in [scenario.importer, *scenario.processors, scenario.exporter]: + plugin.setup(scenario.accumulated_config) + plugin.resolve_config() + scenario.accumulated_config = ( + scenario.accumulated_config | plugin.config_output + ) + + # Write conduit config to data directory + with open( + os.path.join(scenario.conduit_dir, "conduit.yml"), "w" + ) as conduit_cfg: + yaml.dump( + { + "log-level": "info", + "importer": { + "name": scenario.importer.name, + "config": scenario.importer.config_input, + }, + "processors": [ + { + "name": processor.name, + "config": processor.config_input, + } + for processor in scenario.processors + ], + "exporter": { + "name": scenario.exporter.name, + "config": scenario.exporter.config_input, + }, + }, + conduit_cfg, + ) + + def run_scenario(self, scenario): + # Run conduit + start = time.time() + cmd = [self.conduit_bin, "-d", scenario.conduit_dir] + logger.info(f"running scenario {scenario.name}") + logger.debug("%s", " ".join(map(repr, cmd))) + indexerdp = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + atexit.register(indexerdp.kill) + indexerout = subslurp(indexerdp.stdout) + + logger.info(f"Waiting for conduit to reach round {scenario.importer.lastblock}") + + try: + indexerout.run(scenario.importer.lastblock) + except RuntimeError as exc: + logger.error(f"{exc}") + logger.error( + f"conduit hit an error during execution: {indexerout.error_log}" + ) + sys.stderr.write(indexerout.dump()) + return 1 + + if indexerout.round >= scenario.importer.lastblock: + logger.info("reached expected round={}".format(scenario.importer.lastblock)) + dt = time.time() - start + sys.stdout.write("conduit e2etest OK ({:.1f}s)\n".format(dt)) + return 0 + logger.error( + "conduit did not reach round={}".format(scenario.importer.lastblock) + ) + sys.stderr.write(indexerout.dump()) + return 1 diff --git a/e2e_tests/src/e2e_conduit/scenarios/__init__.py b/e2e_tests/src/e2e_conduit/scenarios/__init__.py new file mode 100644 index 000000000..d9b1a97d1 --- /dev/null +++ b/e2e_tests/src/e2e_conduit/scenarios/__init__.py @@ -0,0 +1,18 @@ +from dataclasses import dataclass, field + +from e2e_conduit.fixtures.importers.importer_plugin import ImporterPlugin +from e2e_conduit.fixtures.plugin_fixture import PluginFixture + + +@dataclass +class Scenario: + """Data class for conduit E2E test pipelines""" + name: str + importer: ImporterPlugin + processors: list[PluginFixture] + exporter: PluginFixture + accumulated_config: dict = field(default_factory=dict) + conduit_dir: str = "" + + +scenarios = [] diff --git a/e2e_tests/src/e2e_conduit/scenarios/indexer_scenario.py b/e2e_tests/src/e2e_conduit/scenarios/indexer_scenario.py new file mode 100644 index 000000000..2f52ca97e --- /dev/null +++ b/e2e_tests/src/e2e_conduit/scenarios/indexer_scenario.py @@ -0,0 +1,15 @@ +import e2e_conduit.fixtures.importers as importers +import e2e_conduit.fixtures.processors as processors +import e2e_conduit.fixtures.exporters as exporters +from e2e_conduit.scenarios import Scenario + + +def indexer_scenario(sourcenet): + return Scenario( + "indexer_scenario", + importer=importers.AlgodImporter(sourcenet), + processors=[ + processors.BlockEvaluator(), + ], + exporter=exporters.PostgresqlExporter(), + ) diff --git a/e2e_tests/src/e2e_conduit/subslurp.py b/e2e_tests/src/e2e_conduit/subslurp.py new file mode 100644 index 000000000..aa5d373a9 --- /dev/null +++ b/e2e_tests/src/e2e_conduit/subslurp.py @@ -0,0 +1,60 @@ +from datetime import datetime, timedelta +import gzip +import io +import logging +import re + +logger = logging.getLogger(__name__) + + +class subslurp: + """accumulate stdout or stderr from a subprocess and hold it for debugging if something goes wrong""" + def __init__(self, f): + self.f = f + self.buf = io.BytesIO() + self.gz = gzip.open(self.buf, "wb") + self.timeout = timedelta(seconds=120) + # Matches conduit log output: "Pipeline round: 110" + self.round_re = re.compile(b'.*"Pipeline round: ([0-9]+)"') + self.round = 0 + self.error_log = None + + def logIsError(self, log_line): + if b"error" in log_line: + self.error_log = log_line + return True + return False + + def tryParseRound(self, log_line): + m = self.round_re.match(log_line) + if m is not None and m.group(1) is not None: + self.round = int(m.group(1)) + + def run(self, lastround): + if len(self.f.peek().strip()) == 0: + logger.info("No Conduit output found") + return + + start = datetime.now() + lastlog = datetime.now() + while ( + datetime.now() - start < self.timeout + and datetime.now() - lastlog < timedelta(seconds=15) + ): + for line in self.f: + lastlog = datetime.now() + if self.gz is not None: + self.gz.write(line) + self.tryParseRound(line) + if self.round >= lastround: + logger.info(f"Conduit reached desired lastround: {lastround}") + return + if self.logIsError(line): + raise RuntimeError(f"E2E tests logged an error: {self.error_log}") + + def dump(self): + self.gz.close() + self.gz = None + self.buf.seek(0) + r = gzip.open(self.buf, "rt") + return r.read() diff --git a/e2e_tests/src/e2e_indexer/__init__.py b/e2e_tests/src/e2e_indexer/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/misc/e2elive.py b/e2e_tests/src/e2e_indexer/e2elive.py similarity index 90% rename from misc/e2elive.py rename to e2e_tests/src/e2e_indexer/e2elive.py index b03742533..89668b789 100644 --- a/misc/e2elive.py +++ b/e2e_tests/src/e2e_indexer/e2elive.py @@ -10,7 +10,6 @@ import os import random import shutil -import sqlite3 import subprocess import sys import tempfile @@ -18,7 +17,15 @@ import time import urllib.request -from util import xrun, atexitrun, find_indexer, ensure_test_db, firstFromS3Prefix +from e2e_common.util import ( + xrun, + atexitrun, + find_binary, + ensure_test_db, + firstFromS3Prefix, + hassuffix, + countblocks, +) logger = logging.getLogger(__name__) @@ -58,7 +65,7 @@ def main(): logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO) - indexer_bin = find_indexer(args.indexer_bin) + indexer_bin = find_binary(args.indexer_bin) sourcenet = args.source_net source_is_tar = False if not sourcenet: @@ -74,7 +81,9 @@ def main(): if not (source_is_tar or (sourcenet and os.path.isdir(sourcenet))): tarname = args.s3_source_net if not tarname: - raise Exception("Must provide either local or s3 network to run test against") + raise Exception( + "Must provide either local or s3 network to run test against" + ) tarname = f"{tarname}.tar.bz2" # fetch test data from S3 @@ -82,10 +91,10 @@ def main(): import boto3 from botocore.config import Config from botocore import UNSIGNED + s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED)) prefix = "indexer/e2e4" if "/" in tarname: - # parse tarname like fafa8862/rel-nightly cmhash_tarnme = tarname.split("/") cmhash = cmhash_tarnme[0] tarname =cmhash_tarnme[1] @@ -129,11 +138,12 @@ def main(): psqlstring = ensure_test_db(args.connection_string, args.keep_temps) algoddir = os.path.join(tempnet, "Primary") aiport = args.indexer_port or random.randint(4000, 30000) + indexerdir = os.path.join(tempdir, "indexer_data_dir") cmd = [ indexer_bin, "daemon", "--data-dir", - tempdir, + indexerdir, "-P", psqlstring, "--dev-mode", @@ -169,8 +179,7 @@ def main(): logger.info("reached expected round={}".format(lastblock)) xrun( [ - "python3", - "misc/validate_accounting.py", + "validate-accounting", "--verbose", "--algod", algoddir, @@ -191,23 +200,6 @@ def main(): return 0 -def hassuffix(x, *suffixes): - for s in suffixes: - if x.endswith(s): - return True - return False - - -def countblocks(path): - db = sqlite3.connect(path) - cursor = db.cursor() - cursor.execute("SELECT max(rnd) FROM blocks") - row = cursor.fetchone() - cursor.close() - db.close() - return row[0] - - def tryhealthurl(healthurl, verbose=False, waitforround=200): try: response = urllib.request.urlopen(healthurl) diff --git a/misc/validate_accounting.py b/e2e_tests/src/e2e_indexer/validate_accounting.py similarity index 99% rename from misc/validate_accounting.py rename to e2e_tests/src/e2e_indexer/validate_accounting.py index fdf6dbd9b..e5f521453 100644 --- a/misc/validate_accounting.py +++ b/e2e_tests/src/e2e_indexer/validate_accounting.py @@ -23,7 +23,7 @@ import algosdk from algosdk.v2client.algod import AlgodClient -from util import maybedecode, mloads, unmsgpack +from e2e_common.util import maybedecode, mloads, unmsgpack logger = logging.getLogger(__name__) diff --git a/exporters/exporter.go b/exporters/exporter.go deleted file mode 100644 index ad2a837e3..000000000 --- a/exporters/exporter.go +++ /dev/null @@ -1,72 +0,0 @@ -package exporters - -import ( - "github.com/algorand/go-algorand/agreement" - "github.com/algorand/go-algorand/data/bookkeeping" - "github.com/algorand/go-algorand/ledger/ledgercore" -) - -// ExporterConfig is a generic string which can be deserialized by each individual Exporter. -type ExporterConfig string - -// ExportData is the interface which all data types sent to Exporters should implement -type ExportData interface { - Round() uint64 -} - -// BlockExportData is provided to the Exporter on each round. -type BlockExportData struct { - // Block is the block data written to the blockchain. - Block bookkeeping.Block - - // Delta contains a list of account changes resulting from the block. Processor plugins may have modify this data. - Delta ledgercore.StateDelta - - // Certificate contains voting data that certifies the block. The certificate is non deterministic, a node stops collecting votes once the voting threshold is reached. - Certificate agreement.Certificate -} - -// Round returns the round to which the BlockExportData corresponds -func (blkData *BlockExportData) Round() uint64 { - return uint64(blkData.Block.Round()) -} - -// ExporterMetadata contains fields relevant to identification and description of plugins. -type ExporterMetadata struct { - Name string - Description string - Deprecated bool -} - -// Exporter defines the interface for plugins -type Exporter interface { - // Metadata associated with each Exporter. - Metadata() ExporterMetadata - - // Connect will be called during initialization, before block data starts going through the pipeline. - // Typically used for things like initializing network connections. - // The ExporterConfig passed to Connect will contain the Unmarhsalled config file specific to this plugin. - // Should return an error if it fails--this will result in the Indexer process terminating. - Connect(cfg ExporterConfig) error - - // Config returns the configuration options used to create an Exporter. - // Initialized during Connect, it should return nil until the Exporter has been Connected. - Config() ExporterConfig - - // Disconnect will be called during termination of the Indexer process. - // There is no guarantee that plugin lifecycle hooks will be invoked in any specific order in relation to one another. - // Returns an error if it fails which will be surfaced in the logs, but the process is already terminating. - Disconnect() error - - // Receive is called for each block to be processed by the exporter. - // Should return an error on failure--retries are configurable. - Receive(exportData ExportData) error - - // HandleGenesis is an Exporter's opportunity to do initial validation and handling of the Genesis block. - // If validation (such as a check to ensure `genesis` matches a previously stored genesis block) or handling fails, - // it returns an error. - HandleGenesis(genesis bookkeeping.Genesis) error - - // Round returns the next round not yet processed by the Exporter. Atomically updated when Receive successfully completes. - Round() uint64 -} diff --git a/exporters/exporter_factory.go b/exporters/exporter_factory.go deleted file mode 100644 index 5c29b511f..000000000 --- a/exporters/exporter_factory.go +++ /dev/null @@ -1,40 +0,0 @@ -package exporters - -import ( - "fmt" -) - -// ExporterConstructor must be implemented by each Exporter. -// It provides a basic no-arg constructor for instances of an ExporterImpl. -type ExporterConstructor interface { - // New should return an instantiation of an Exporter. - // Configuration values should be passed and can be processed during `Connect()`. - New() Exporter -} - -// exporterImpls is a k/v store from exporter names to their constructor implementations. -// This layer of indirection allows for different exporter integrations to be compiled in or compiled out by `go build --tags ...` -var exporterImpls = make(map[string]ExporterConstructor) - -// RegisterExporter is used to register ExporterConstructor implementations. This mechanism allows -// for loose coupling between the configuration and the implementation. It is extremely similar to the way sql.DB -// driver's are configured and used. -func RegisterExporter(name string, constructor ExporterConstructor) { - exporterImpls[name] = constructor -} - -// ExporterByName is used to construct an Exporter object by name. -// Returns an Exporter object, an availability channel that closes when the database -// becomes available, and an error object. -func ExporterByName(name string, cfg ExporterConfig) (Exporter, error) { - var constructor ExporterConstructor - var ok bool - if constructor, ok = exporterImpls[name]; !ok { - return nil, fmt.Errorf("no Exporter Constructor for %s", name) - } - val := constructor.New() - if err := val.Connect(cfg); err != nil { - return nil, err - } - return val, nil -} diff --git a/exporters/exporter_factory_test.go b/exporters/exporter_factory_test.go deleted file mode 100644 index 0bf71b8ba..000000000 --- a/exporters/exporter_factory_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package exporters - -import ( - "fmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "testing" -) - -type mockExporter struct { - mock.Mock - Exporter -} - -func (m *mockExporter) Connect(config ExporterConfig) error { - args := m.Called(config) - return args.Error(0) -} - -type mockExporterConstructor struct { - me *mockExporter -} - -func (c *mockExporterConstructor) New() Exporter { - return c.me -} - -func TestExporterByNameSuccess(t *testing.T) { - me := mockExporter{} - me.On("Connect", mock.Anything).Return(nil) - RegisterExporter("foobar", &mockExporterConstructor{&me}) - - exp, err := ExporterByName("foobar", "") - assert.NoError(t, err) - assert.Implements(t, (*Exporter)(nil), exp) -} - -func TestExporterByNameNotFound(t *testing.T) { - _, err := ExporterByName("barfoo", "") - expectedErr := "no Exporter Constructor for barfoo" - assert.EqualError(t, err, expectedErr) -} - -func TestExporterByNameConnectFailure(t *testing.T) { - me := mockExporter{} - expectedErr := fmt.Errorf("connect failure") - me.On("Connect", mock.Anything).Return(expectedErr) - RegisterExporter("baz", &mockExporterConstructor{&me}) - _, err := ExporterByName("baz", "") - assert.EqualError(t, err, expectedErr.Error()) -} diff --git a/exporters/noop/noop_exporter.go b/exporters/noop/noop_exporter.go deleted file mode 100644 index 156996d1c..000000000 --- a/exporters/noop/noop_exporter.go +++ /dev/null @@ -1,66 +0,0 @@ -package noop - -import ( - "github.com/algorand/indexer/exporters" - - "github.com/algorand/go-algorand/data/bookkeeping" -) - -// `noopExporter`s will function without ever erroring. This means they will also process out of order blocks -// which may or may not be desirable for different use cases--it can hide errors in actual exporters expecting in order -// block processing. -// The `noopExporter` will maintain `Round` state according to the round of the last block it processed. -type noopExporter struct { - round uint64 - cfg exporters.ExporterConfig -} - -var noopExporterMetadata exporters.ExporterMetadata = exporters.ExporterMetadata{ - Name: "noop", - Description: "noop exporter", - Deprecated: false, -} - -// Constructor is the ExporterConstructor implementation for the "noop" exporter -type Constructor struct{} - -// New initializes a noopExporter -func (c *Constructor) New() exporters.Exporter { - return &noopExporter{ - round: 0, - cfg: "", - } -} - -func (exp *noopExporter) Metadata() exporters.ExporterMetadata { - return noopExporterMetadata -} - -func (exp *noopExporter) Connect(_ exporters.ExporterConfig) error { - return nil -} - -func (exp *noopExporter) Config() exporters.ExporterConfig { - return exp.cfg -} - -func (exp *noopExporter) Disconnect() error { - return nil -} - -func (exp *noopExporter) Receive(exportData exporters.ExportData) error { - exp.round = exportData.Round() + 1 - return nil -} - -func (exp *noopExporter) HandleGenesis(_ bookkeeping.Genesis) error { - return nil -} - -func (exp *noopExporter) Round() uint64 { - return exp.round -} - -func init() { - exporters.RegisterExporter(noopExporterMetadata.Name, &Constructor{}) -} diff --git a/exporters/noop/noop_exporter_test.go b/exporters/noop/noop_exporter_test.go deleted file mode 100644 index 438189a84..000000000 --- a/exporters/noop/noop_exporter_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package noop - -import ( - "github.com/algorand/go-algorand/data/bookkeeping" - "github.com/algorand/indexer/exporters" - "github.com/stretchr/testify/assert" - "testing" -) - -var nc = &Constructor{} - -var ne = nc.New() - -func TestExporterByName(t *testing.T) { - exporters.RegisterExporter(noopExporterMetadata.Name, nc) - ne, err := exporters.ExporterByName(noopExporterMetadata.Name, "") - assert.NoError(t, err) - assert.Implements(t, (*exporters.Exporter)(nil), ne) -} - -func TestExporterMetadata(t *testing.T) { - meta := ne.Metadata() - assert.Equal(t, noopExporterMetadata.Name, meta.Name) - assert.Equal(t, noopExporterMetadata.Description, meta.Description) - assert.Equal(t, noopExporterMetadata.Deprecated, meta.Deprecated) -} - -func TestExporterConnect(t *testing.T) { - assert.NoError(t, ne.Connect("")) -} - -func TestExporterConfig(t *testing.T) { - assert.Equal(t, exporters.ExporterConfig(""), ne.Config()) -} - -func TestExporterDisconnect(t *testing.T) { - assert.NoError(t, ne.Disconnect()) -} - -func TestExporterHandleGenesis(t *testing.T) { - assert.NoError(t, ne.HandleGenesis(bookkeeping.Genesis{})) -} - -func TestExporterRoundReceive(t *testing.T) { - eData := &exporters.BlockExportData{ - Block: bookkeeping.Block{ - BlockHeader: bookkeeping.BlockHeader{ - Round: 5, - }, - }, - } - assert.Equal(t, uint64(0), ne.Round()) - assert.NoError(t, ne.Receive(eData)) - assert.Equal(t, uint64(6), ne.Round()) -} diff --git a/fetcher/fetcher.go b/fetcher/fetcher.go index 1ba75df1a..f93543f27 100644 --- a/fetcher/fetcher.go +++ b/fetcher/fetcher.go @@ -318,7 +318,17 @@ func algodStat(netpath, tokenpath string) (lastmod time.Time, err error) { } func algodClientForDataDir(datadir string) (client *algod.Client, lastmod time.Time, err error) { - // TODO: move this to go-algorand-sdk + netAddr, token, lastmod, err := AlgodArgsForDataDir(datadir) + if err != nil { + return + } + client, err = algod.MakeClient(netAddr, token) + + return +} + +// AlgodArgsForDataDir opens the token and network files in the data directory, returning data for constructing client +func AlgodArgsForDataDir(datadir string) (netAddr string, token string, lastmod time.Time, err error) { netpath, tokenpath := algodPaths(datadir) var netaddrbytes []byte netaddrbytes, err = ioutil.ReadFile(netpath) @@ -326,18 +336,21 @@ func algodClientForDataDir(datadir string) (client *algod.Client, lastmod time.T err = fmt.Errorf("%s: %v", netpath, err) return } - netaddr := strings.TrimSpace(string(netaddrbytes)) - if !strings.HasPrefix(netaddr, "http") { - netaddr = "http://" + netaddr + netAddr = strings.TrimSpace(string(netaddrbytes)) + if !strings.HasPrefix(netAddr, "http") { + netAddr = "http://" + netAddr } - token, err := ioutil.ReadFile(tokenpath) + + tokenBytes, err := ioutil.ReadFile(tokenpath) if err != nil { err = fmt.Errorf("%s: %v", tokenpath, err) return } - client, err = algod.MakeClient(netaddr, strings.TrimSpace(string(token))) + token = strings.TrimSpace(string(tokenBytes)) + if err == nil { lastmod, err = algodStat(netpath, tokenpath) } + return } diff --git a/fetcher/fetcher_test.go b/fetcher/fetcher_test.go index eba54c14d..67b14ed97 100644 --- a/fetcher/fetcher_test.go +++ b/fetcher/fetcher_test.go @@ -3,8 +3,12 @@ package fetcher import ( "context" "fmt" + "github.com/algorand/indexer/util/test" + "github.com/stretchr/testify/assert" + "io/fs" "net/http" - "net/http/httptest" + "os" + "path/filepath" "strings" "testing" @@ -12,21 +16,11 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "github.com/algorand/go-algorand-sdk/client/v2/algod" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/rpcs" ) -type AlgodHandler struct { - mock.Mock -} - -func (handler *AlgodHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - handler.Called(w, req) - return -} - type BlockHandler struct { mock.Mock } @@ -36,29 +30,23 @@ func (handler *BlockHandler) handlerFunc(ctx context.Context, cert *rpcs.Encoded return args.Error(0) } -func mockAClient(t *testing.T, algodHandler *AlgodHandler) *algod.Client { - mockServer := httptest.NewServer(algodHandler) - aclient, err := algod.MakeClient(mockServer.URL, "") - if err != nil { - t.FailNow() - } - return aclient -} - func TestFetcherImplErrorInitialization(t *testing.T) { - aclient := mockAClient(t, &AlgodHandler{}) + aclient, err := test.MockAClient(test.NewAlgodHandler()) + assert.NoError(t, err) fetcher := &fetcherImpl{aclient: aclient, log: logrus.New()} require.Equal(t, "", fetcher.Error(), "Initialization of fetcher caused an unexpected error.") } func TestFetcherImplAlgodReturnsClient(t *testing.T) { - aclient := mockAClient(t, &AlgodHandler{}) + aclient, err := test.MockAClient(test.NewAlgodHandler()) + assert.NoError(t, err) fetcher := &fetcherImpl{aclient: aclient, log: logrus.New()} require.Equal(t, aclient, fetcher.Algod(), "Algod client returned from fetcherImpl does not match expected instance.") } func TestFetcherImplSetError(t *testing.T) { - aclient := mockAClient(t, &AlgodHandler{}) + aclient, err := test.MockAClient(test.NewAlgodHandler()) + assert.NoError(t, err) fetcher := &fetcherImpl{aclient: aclient, log: logrus.New()} expectedErr := fmt.Errorf("foobar") fetcher.setError(expectedErr) @@ -66,56 +54,88 @@ func TestFetcherImplSetError(t *testing.T) { } func TestFetcherImplProcessQueueHandlerError(t *testing.T) { - mockAlgodHandler := &AlgodHandler{} - aclient := mockAClient(t, mockAlgodHandler) + aclient, err := test.MockAClient(test.NewAlgodHandler(test.BlockResponder)) + assert.NoError(t, err) fetcher := &fetcherImpl{aclient: aclient, log: logrus.New()} bHandler := &BlockHandler{} expectedError := fmt.Errorf("handlerError") // The block handler function will immediately return an error on any block passed to it bHandler.On("handlerFunc", mock.Anything, mock.Anything).Return(expectedError) fetcher.SetBlockHandler(bHandler.handlerFunc) - // Mock algod server to continually return empty blocks - mockAlgodHandler.On("ServeHTTP", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - respWriter := args.Get(0).(http.ResponseWriter) - req := args.Get(1).(*http.Request) - path := req.URL.Path - if strings.Contains(path, "v2/blocks/") { - var block bookkeeping.Block - respWriter.Write(protocol.Encode(&block)) - } - }) require.ErrorIsf(t, fetcher.Run(context.Background()), expectedError, "FetcherImpl did not return expected error in processQueue handler.") } func TestFetcherImplCatchupLoopBlockError(t *testing.T) { - mockAlgodHandler := &AlgodHandler{} - aclient := mockAClient(t, mockAlgodHandler) passingCalls := 5 + aclient, err := test.MockAClient(test.NewAlgodHandler( + // Our mock algod client will process /v2/blocks/{round} calls + // returning an empty block `passingCalls` times before throwing 500s + func(path string, w http.ResponseWriter) bool { + if strings.Contains(path, "v2/blocks/") { + if passingCalls == 0 { + w.WriteHeader(http.StatusInternalServerError) + } else { + var block bookkeeping.Block + w.WriteHeader(http.StatusOK) + w.Write(protocol.Encode(&block)) + passingCalls-- + } + return true + } + return false + }), + ) + assert.NoError(t, err) // Initializing blockQueue here needs buffer since we have no other goroutines receiving from it fetcher := &fetcherImpl{aclient: aclient, log: logrus.New(), blockQueue: make(chan *rpcs.EncodedBlockCert, 256)} bHandler := &BlockHandler{} // the handler will do nothing here bHandler.On("handlerFunc", mock.Anything, mock.Anything).Return(nil) fetcher.SetBlockHandler(bHandler.handlerFunc) - - // Our mock algod client will process /v2/blocks/{round} calls - // returning an empty block `passingCalls` times before throwing 500s - mockAlgodHandler.On("ServeHTTP", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - respWriter := args.Get(0).(http.ResponseWriter) - req := args.Get(1).(*http.Request) - path := req.URL.Path - if strings.Contains(path, "v2/blocks/") { - if passingCalls <= 0 { - respWriter.WriteHeader(http.StatusInternalServerError) - } else { - var block bookkeeping.Block - respWriter.WriteHeader(http.StatusOK) - respWriter.Write(protocol.Encode(&block)) - passingCalls-- - } - } - }) - err := fetcher.catchupLoop(context.Background()) + err = fetcher.catchupLoop(context.Background()) require.NoError(t, err, "FetcherImpl returned an unexpected error from catchupLoop") require.Equal(t, "", fetcher.Error(), "FetcherImpl set an unexpected error from algod client during catchupLoop") } + +func TestAlgodArgsForDataDirNetDoesNotExist(t *testing.T) { + _, _, _, err := AlgodArgsForDataDir("foobar") + assert.Error(t, err) + assert.Contains(t, err.Error(), "foobar/algod.net: ") +} + +func TestAlgodArgsForDataDirTokenDoesNotExist(t *testing.T) { + dir, err := os.MkdirTemp("", "datadir") + if err != nil { + t.Fatalf(err.Error()) + } + err = os.WriteFile(filepath.Join(dir, "algod.net"), []byte("127.0.0.1:8080"), fs.ModePerm) + if err != nil { + t.Fatalf(err.Error()) + } + defer os.RemoveAll(dir) + _, _, _, err = AlgodArgsForDataDir(dir) + assert.Error(t, err) + assert.Contains(t, err.Error(), fmt.Sprintf("%v/algod.token: ", dir)) +} + +func TestAlgodArgsForDataDirSuccess(t *testing.T) { + dir, err := os.MkdirTemp("", "datadir") + if err != nil { + t.Fatalf(err.Error()) + } + err = os.WriteFile(filepath.Join(dir, "algod.net"), []byte("127.0.0.1:8080"), fs.ModePerm) + if err != nil { + t.Fatalf(err.Error()) + } + err = os.WriteFile(filepath.Join(dir, "algod.token"), []byte("abc123"), fs.ModePerm) + if err != nil { + t.Fatalf(err.Error()) + } + defer os.RemoveAll(dir) + netAddr, token, lastmod, err := AlgodArgsForDataDir(dir) + assert.NoError(t, err) + assert.Equal(t, netAddr, "http://127.0.0.1:8080") + assert.Equal(t, token, "abc123") + assert.NotNil(t, lastmod) + +} diff --git a/go.sum b/go.sum index 955980681..7a1422b62 100644 --- a/go.sum +++ b/go.sum @@ -176,6 +176,10 @@ github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cucumber/godog v0.8.1/go.mod h1:vSh3r/lM+psC1BPXvdkSEuNjmXfpVqrMGYAElF6hxnA= +<<<<<<< HEAD +======= +github.com/cyberdelia/templates v0.0.0-20191230040416-20a325f050d4/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= +>>>>>>> d672efe (conduit: Initial conduit pipeline tool. (#1326)) github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -227,8 +231,14 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +<<<<<<< HEAD github.com/getkin/kin-openapi v0.107.0 h1:bxhL6QArW7BXQj8NjXfIJQy680NsMKd25nwhvpCXchg= github.com/getkin/kin-openapi v0.107.0/go.mod h1:9Dhr+FasATJZjS4iOLvB0hkaxgYdulrNYm2e9epLWOo= +======= +github.com/getkin/kin-openapi v0.3.1/go.mod h1:W8dhxZgpE84ciM+VIItFqkmZ4eHtuomrdIHtASQIqi0= +github.com/getkin/kin-openapi v0.22.0 h1:J5IFyKd/5yuB6AZAgwK0CMBKnabWcmkowtsl6bRkz4s= +github.com/getkin/kin-openapi v0.22.0/go.mod h1:WGRs2ZMM1Q8LR1QBEwUxC6RJEfaBcD0s+pcEVXFuAjw= +>>>>>>> d672efe (conduit: Initial conduit pipeline tool. (#1326)) github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -446,8 +456,11 @@ github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +<<<<<<< HEAD github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc= github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= +======= +>>>>>>> d672efe (conduit: Initial conduit pipeline tool. (#1326)) github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= @@ -542,6 +555,11 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/echo-contrib v0.11.0 h1:/B7meUKBP7AAoSEOrawpSivhFvu7GQG+kDhlzi5v0Wo= github.com/labstack/echo-contrib v0.11.0/go.mod h1:Hk8Iyxe2GrYR/ch0cbI3BK7ZhR2Y60YEqtkoZilqDOc= +<<<<<<< HEAD +======= +github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI= +github.com/labstack/echo/v4 v4.3.0 h1:DCP6cbtT+Zu++K6evHOJzSgA2115cPMuCx0xg55q1EQ= +>>>>>>> d672efe (conduit: Initial conduit pipeline tool. (#1326)) github.com/labstack/echo/v4 v4.3.0/go.mod h1:PvmtTvhVqKDzDQy4d3bWzPjZLzom4iQbAZy2sgZ/qI8= github.com/labstack/echo/v4 v4.9.1 h1:GliPYSpzGKlyOhqIbG8nmHBo3i1saKWFOgh41AN3b+Y= github.com/labstack/echo/v4 v4.9.1/go.mod h1:Pop5HLc+xoc4qhTZ1ip6C0RtP7Z+4VzRLWZZFKqbbjo= @@ -1077,8 +1095,13 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +<<<<<<< HEAD golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +======= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +>>>>>>> d672efe (conduit: Initial conduit pipeline tool. (#1326)) golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/idb/dummy/dummy.go b/idb/dummy/dummy.go index 0fa142b32..ff3ad894e 100644 --- a/idb/dummy/dummy.go +++ b/idb/dummy/dummy.go @@ -3,6 +3,8 @@ package dummy import ( "context" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/ledger/ledgercore" @@ -95,10 +97,15 @@ func (db *dummyIndexerDb) Health(ctx context.Context) (state idb.Health, err err // GetNetworkState is part of idb.IndexerDB func (db *dummyIndexerDb) GetNetworkState() (state idb.NetworkState, err error) { - return idb.NetworkState{}, nil + return idb.NetworkState{GenesisHash: crypto.HashObj(bookkeeping.Genesis{})}, nil } // SetNetworkState is part of idb.IndexerDB func (db *dummyIndexerDb) SetNetworkState(genesis bookkeeping.Genesis) error { return nil } + +// SetNetworkState is part of idb.IndexerDB +func (db *dummyIndexerDb) DeleteTransactions(ctx context.Context, keep uint64) error { + return nil +} diff --git a/idb/dummy/dummy_factory.go b/idb/dummy/dummy_factory.go index c093a3244..45f0e5f4a 100644 --- a/idb/dummy/dummy_factory.go +++ b/idb/dummy/dummy_factory.go @@ -16,7 +16,9 @@ func (df dummyFactory) Name() string { // Build is part of the IndexerFactory interface. func (df dummyFactory) Build(arg string, opts idb.IndexerDbOptions, log *log.Logger) (idb.IndexerDb, chan struct{}, error) { - return &dummyIndexerDb{log: log}, nil, nil + ch := make(chan struct{}) + close(ch) + return &dummyIndexerDb{log: log}, ch, nil } func init() { diff --git a/idb/idb.go b/idb/idb.go index 413a66a16..2f040dfd8 100644 --- a/idb/idb.go +++ b/idb/idb.go @@ -185,6 +185,8 @@ type IndexerDb interface { ApplicationBoxes(ctx context.Context, filter ApplicationBoxQuery) (<-chan ApplicationBoxRow, uint64) Health(ctx context.Context) (status Health, err error) + + DeleteTransactions(ctx context.Context, keep uint64) error } // GetBlockOptions contains the options when requesting to load a block from the database. diff --git a/idb/mocks/IndexerDb.go b/idb/mocks/IndexerDb.go index dbd7ca94d..da13a12b9 100644 --- a/idb/mocks/IndexerDb.go +++ b/idb/mocks/IndexerDb.go @@ -157,6 +157,20 @@ func (_m *IndexerDb) Close() { _m.Called() } +// DeleteTransactions provides a mock function with given fields: ctx, keep +func (_m *IndexerDb) DeleteTransactions(ctx context.Context, keep uint64) error { + ret := _m.Called(ctx, keep) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, uint64) error); ok { + r0 = rf(ctx, keep) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // GetAccounts provides a mock function with given fields: ctx, opts func (_m *IndexerDb) GetAccounts(ctx context.Context, opts idb.AccountQueryOptions) (<-chan idb.AccountRow, uint64) { ret := _m.Called(ctx, opts) diff --git a/idb/postgres/internal/encoding/encoding.go b/idb/postgres/internal/encoding/encoding.go index 659ae4c82..e3be2acb0 100644 --- a/idb/postgres/internal/encoding/encoding.go +++ b/idb/postgres/internal/encoding/encoding.go @@ -742,3 +742,19 @@ func DecodeTrimmedLcAccountData(data []byte) (ledgercore.AccountData, error) { return unconvertTrimmedLcAccountData(ba), nil } + +// EncodeDeleteStatus encodes network metastate into json. +func EncodeDeleteStatus(p *types.DeleteStatus) []byte { + return encodeJSON(p) +} + +// DecodeDeleteStatus decodes network metastate from json. +func DecodeDeleteStatus(data []byte) (types.DeleteStatus, error) { + var status types.DeleteStatus + err := DecodeJSON(data, &status) + if err != nil { + return types.DeleteStatus{}, fmt.Errorf("DecodeDeleteStatus() err: %w", err) + } + + return status, nil +} diff --git a/idb/postgres/internal/schema/metastate.go b/idb/postgres/internal/schema/metastate.go index 8256358ee..1d94f7c77 100644 --- a/idb/postgres/internal/schema/metastate.go +++ b/idb/postgres/internal/schema/metastate.go @@ -7,4 +7,5 @@ const ( SpecialAccountsMetastateKey = "accounts" AccountTotals = "totals" NetworkMetaStateKey = "network" + DeleteStatusKey = "pruned" ) diff --git a/idb/postgres/internal/types/types.go b/idb/postgres/internal/types/types.go index 14e16481b..8bd884b7f 100644 --- a/idb/postgres/internal/types/types.go +++ b/idb/postgres/internal/types/types.go @@ -1,6 +1,8 @@ package types -import "github.com/algorand/go-algorand/crypto" +import ( + "github.com/algorand/go-algorand/crypto" +) // ImportState encodes an import round counter. type ImportState struct { @@ -26,3 +28,9 @@ type MigrationState struct { type NetworkState struct { GenesisHash crypto.Digest `codec:"genesis-hash"` } + +// DeleteStatus encodes pruned metastate. +type DeleteStatus struct { + LastPruned string `codec:"last_pruned"` + OldestRound uint64 `codec:"oldest_txn_round"` +} diff --git a/idb/postgres/postgres.go b/idb/postgres/postgres.go index cfc78395e..9aff1d4c1 100644 --- a/idb/postgres/postgres.go +++ b/idb/postgres/postgres.go @@ -21,7 +21,6 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/ledger/ledgercore" - "github.com/jackc/pgconn" "github.com/jackc/pgerrcode" "github.com/jackc/pgx/v4" @@ -2647,3 +2646,36 @@ func (db *IndexerDb) SetNetworkState(genesis bookkeeping.Genesis) error { } return db.setNetworkState(nil, &networkState) } + +// DeleteTransactions removes old transactions +// keep is the number of rounds to keep in db +func (db *IndexerDb) DeleteTransactions(ctx context.Context, keep uint64) error { + // delete old transactions and update metastate + deleteTxns := func(tx pgx.Tx) error { + db.log.Infof("deleteTxns(): removing transactions before round %d", keep) + // delete query + query := "DELETE FROM txn WHERE round < $1" + cmd, err2 := tx.Exec(ctx, query, keep) + if err2 != nil { + return fmt.Errorf("deleteTxns(): transaction delete err %w", err2) + } + t := time.Now().UTC() + // update metastate + status := types.DeleteStatus{ + // format time, "2006-01-02T15:04:05Z07:00" + LastPruned: t.Format(time.RFC3339), + OldestRound: keep, + } + err2 = db.setMetastate(tx, schema.DeleteStatusKey, string(encoding.EncodeDeleteStatus(&status))) + if err2 != nil { + return fmt.Errorf("deleteTxns(): metastate update err %w", err2) + } + db.log.Infof("%d transactions deleted, last pruned at %s", cmd.RowsAffected(), status.LastPruned) + return nil + } + err := db.txWithRetry(serializable, deleteTxns) + if err != nil { + return fmt.Errorf("DeleteTransactions err: %w", err) + } + return nil +} diff --git a/idb/postgres/postgres_boxes_test.go b/idb/postgres/postgres_boxes_test.go index 666bc384b..6d1fd70eb 100644 --- a/idb/postgres/postgres_boxes_test.go +++ b/idb/postgres/postgres_boxes_test.go @@ -125,7 +125,7 @@ func runBoxCreateMutateDelete(t *testing.T, comparator boxTestComparator) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createTxn, &payNewAppTxn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) opts := idb.ApplicationQuery{ApplicationID: uint64(appid)} @@ -173,7 +173,7 @@ func runBoxCreateMutateDelete(t *testing.T, comparator boxTestComparator) { block, err = test.MakeBlockForTxns(blockHdr, boxTxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) _, round = db.Applications(context.Background(), opts) require.Equal(t, uint64(currentRound), round) @@ -211,7 +211,7 @@ func runBoxCreateMutateDelete(t *testing.T, comparator boxTestComparator) { block, err = test.MakeBlockForTxns(blockHdr, boxTxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) _, round = db.Applications(context.Background(), opts) require.Equal(t, uint64(currentRound), round) @@ -243,7 +243,7 @@ func runBoxCreateMutateDelete(t *testing.T, comparator boxTestComparator) { block, err = test.MakeBlockForTxns(blockHdr, boxTxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) _, round = db.Applications(context.Background(), opts) require.Equal(t, uint64(currentRound), round) @@ -279,7 +279,7 @@ func runBoxCreateMutateDelete(t *testing.T, comparator boxTestComparator) { block, err = test.MakeBlockForTxns(blockHdr, boxTxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) _, round = db.Applications(context.Background(), opts) require.Equal(t, uint64(currentRound), round) @@ -310,7 +310,7 @@ func runBoxCreateMutateDelete(t *testing.T, comparator boxTestComparator) { block, err = test.MakeBlockForTxns(blockHdr, boxTxns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) _, round = db.Applications(context.Background(), opts) require.Equal(t, uint64(currentRound), round) diff --git a/idb/postgres/postgres_integration_common_test.go b/idb/postgres/postgres_integration_common_test.go index 76838cf42..4e9b09d16 100644 --- a/idb/postgres/postgres_integration_common_test.go +++ b/idb/postgres/postgres_integration_common_test.go @@ -4,16 +4,16 @@ import ( "context" "testing" + "github.com/jackc/pgx/v4/pgxpool" test2 "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/require" "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/ledger" - "github.com/algorand/indexer/processor" - "github.com/algorand/indexer/processor/blockprocessor" + "github.com/algorand/go-algorand/rpcs" "github.com/algorand/indexer/util/test" - "github.com/jackc/pgx/v4/pgxpool" - "github.com/stretchr/testify/require" + "github.com/algorand/indexer/conduit/plugins/processors/blockprocessor" "github.com/algorand/indexer/idb" pgtest "github.com/algorand/indexer/idb/postgres/internal/testing" ) @@ -28,7 +28,8 @@ func setupIdbWithConnectionString(t *testing.T, connStr string, genesis bookkeep return idb } -func setupIdb(t *testing.T, genesis bookkeeping.Genesis) (*IndexerDb, func(), processor.Processor, *ledger.Ledger) { +func setupIdb(t *testing.T, genesis bookkeeping.Genesis) (*IndexerDb, func(), func(cert *rpcs.EncodedBlockCert) error, *ledger.Ledger) { + _, connStr, shutdownFunc := pgtest.SetupPostgres(t) db := setupIdbWithConnectionString(t, connStr, genesis) @@ -40,10 +41,12 @@ func setupIdb(t *testing.T, genesis bookkeeping.Genesis) (*IndexerDb, func(), pr logger, _ := test2.NewNullLogger() l, err := test.MakeTestLedger(logger) require.NoError(t, err) - proc, err := blockprocessor.MakeProcessorWithLedger(logger, l, db.AddBlock) + proc, err := blockprocessor.MakeBlockProcessorWithLedger(logger, l, db.AddBlock) require.NoError(t, err, "failed to open ledger") - return db, newShutdownFunc, proc, l + f := blockprocessor.MakeBlockProcessorHandlerAdapter(&proc, db.AddBlock) + + return db, newShutdownFunc, f, l } // Helper to execute a query returning an integer, for example COUNT(*). Returns -1 on an error. diff --git a/idb/postgres/postgres_integration_test.go b/idb/postgres/postgres_integration_test.go index aba7791da..6100c6220 100644 --- a/idb/postgres/postgres_integration_test.go +++ b/idb/postgres/postgres_integration_test.go @@ -24,13 +24,13 @@ import ( "github.com/algorand/go-codec/codec" "github.com/algorand/indexer/api/generated/v2" + "github.com/algorand/indexer/conduit/plugins/processors/blockprocessor" "github.com/algorand/indexer/idb" "github.com/algorand/indexer/idb/postgres/internal/encoding" "github.com/algorand/indexer/idb/postgres/internal/schema" pgtest "github.com/algorand/indexer/idb/postgres/internal/testing" pgutil "github.com/algorand/indexer/idb/postgres/internal/util" "github.com/algorand/indexer/importer" - "github.com/algorand/indexer/processor/blockprocessor" "github.com/algorand/indexer/util" "github.com/algorand/indexer/util/test" ) @@ -181,7 +181,7 @@ func TestAssetCloseReopenTransfer(t *testing.T) { ////////// // When // We commit the block to the database ////////// - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -232,7 +232,7 @@ func TestReCreateAssetHolding(t *testing.T) { ////////// // When // We commit the round accounting to the database. ////////// - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -269,7 +269,7 @@ func TestNoopOptins(t *testing.T) { ////////// // When // We commit the round accounting to the database. ////////// - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -308,7 +308,7 @@ func TestMultipleWriters(t *testing.T) { go func() { defer wg.Done() <-start - errors <- proc.Process(&rpcs.EncodedBlockCert{Block: block}) + errors <- proc(&rpcs.EncodedBlockCert{Block: block}) }() } close(start) @@ -369,7 +369,7 @@ func TestBlockWithTransactions(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, txns...) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -409,7 +409,7 @@ func TestRekeyBasic(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -438,7 +438,7 @@ func TestRekeyToItself(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) txn = test.MakePaymentTxn( @@ -447,7 +447,7 @@ func TestRekeyToItself(t *testing.T) { block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -484,7 +484,7 @@ func TestRekeyThreeTimesInSameRound(t *testing.T) { test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -517,7 +517,7 @@ func TestRekeyToItselfHasNotBeenRekeyed(t *testing.T) { ////////// // Then // No error when committing to the DB. ////////// - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) } @@ -549,7 +549,7 @@ func TestIgnoreDefaultFrozenConfigUpdate(t *testing.T) { ////////// // When // We commit the round accounting to the database. ////////// - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -581,7 +581,7 @@ func TestZeroTotalAssetCreate(t *testing.T) { ////////// // When // We commit the round accounting to the database. ////////// - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -636,7 +636,7 @@ func TestDestroyAssetBasic(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Destroy an asset. @@ -644,7 +644,7 @@ func TestDestroyAssetBasic(t *testing.T) { block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Check that the asset is deleted. @@ -673,7 +673,7 @@ func TestDestroyAssetZeroSupply(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn0, &txn1) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Check that the asset is deleted. @@ -729,7 +729,7 @@ func TestDestroyAssetDeleteCreatorsHolding(t *testing.T) { block, err := test.MakeBlockForTxns( test.MakeGenesisBlock().BlockHeader, &txn0, &txn1, &txn2) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Check that the creator's asset holding is deleted. @@ -777,7 +777,7 @@ func TestAssetFreezeTxnParticipation(t *testing.T) { ////////// // When // We import the block. ////////// - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -821,7 +821,7 @@ func TestInnerTxnParticipation(t *testing.T) { ////////// // When // We import the block. ////////// - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) ////////// @@ -874,7 +874,7 @@ func TestAppExtraPages(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) row := db.db.QueryRow(context.Background(), "SELECT index, params FROM app WHERE creator = $1", test.AccountA[:]) @@ -950,7 +950,7 @@ func TestKeytypeBasic(t *testing.T) { block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) keytype := generated.AccountSigTypeSig @@ -964,7 +964,7 @@ func TestKeytypeBasic(t *testing.T) { block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) keytype = "msig" @@ -985,7 +985,7 @@ func TestLargeAssetAmount(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) { @@ -1134,7 +1134,7 @@ func TestNonDisplayableUTF8(t *testing.T) { require.NoError(t, err) // Test 1: import/accounting should work. - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Test 2: asset results properly serialized @@ -1215,7 +1215,7 @@ func TestReconfigAsset(t *testing.T) { 0, math.MaxUint64, 0, false, unit, name, url, test.AccountA) block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) txn = transactions.SignedTxnWithAD{ @@ -1240,7 +1240,7 @@ func TestReconfigAsset(t *testing.T) { } block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Test 2: asset results properly serialized @@ -1275,7 +1275,7 @@ func TestKeytypeResetsOnRekey(t *testing.T) { block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) keytype := generated.AccountSigTypeSig @@ -1287,7 +1287,7 @@ func TestKeytypeResetsOnRekey(t *testing.T) { block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) assertKeytype(t, db, test.AccountA, nil) @@ -1301,7 +1301,7 @@ func TestKeytypeResetsOnRekey(t *testing.T) { block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) keytype = generated.AccountSigTypeMsig @@ -1323,7 +1323,7 @@ func TestKeytypeDeletedAccount(t *testing.T) { block, err := test.MakeBlockForTxns(block.BlockHeader, &closeTxn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) keytype := generated.AccountSigTypeSig @@ -1380,7 +1380,7 @@ func TestAddBlockAssetCloseAmountInTxnExtra(t *testing.T) { &optinC, &closeB) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Check asset close amount in the `closeB` transaction. @@ -1421,8 +1421,9 @@ func TestAddBlockIncrementsMaxRoundAccounted(t *testing.T) { l, err := test.MakeTestLedger(logger) require.NoError(t, err) defer l.Close() - proc, err := blockprocessor.MakeProcessorWithLedger(logger, l, db.AddBlock) + pr, err := blockprocessor.MakeBlockProcessorWithLedger(logger, l, db.AddBlock) require.NoError(t, err, "failed to open ledger") + proc := blockprocessor.MakeBlockProcessorHandlerAdapter(&pr, db.AddBlock) round, err = db.GetNextRoundToAccount() require.NoError(t, err) @@ -1431,7 +1432,7 @@ func TestAddBlockIncrementsMaxRoundAccounted(t *testing.T) { block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader) require.NoError(t, err) blockCert := rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&blockCert) + err = proc(&blockCert) require.NoError(t, err) round, err = db.GetNextRoundToAccount() @@ -1441,7 +1442,7 @@ func TestAddBlockIncrementsMaxRoundAccounted(t *testing.T) { block, err = test.MakeBlockForTxns(block.BlockHeader) require.NoError(t, err) blockCert = rpcs.EncodedBlockCert{Block: block} - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) round, err = db.GetNextRoundToAccount() @@ -1465,7 +1466,7 @@ func TestAddBlockCreateDeleteAccountSameRound(t *testing.T) { test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) opts := idb.AccountQueryOptions{ @@ -1500,7 +1501,7 @@ func TestAddBlockCreateDeleteAssetSameRound(t *testing.T) { test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Asset global state. @@ -1557,7 +1558,7 @@ func TestAddBlockCreateDeleteAppSameRound(t *testing.T) { test.MakeGenesisBlock().BlockHeader, &createTxn, &deleteTxn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) opts := idb.ApplicationQuery{ @@ -1593,7 +1594,7 @@ func TestAddBlockAppOptInOutSameRound(t *testing.T) { test.MakeGenesisBlock().BlockHeader, &createTxn, &optInTxn, &optOutTxn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) opts := idb.AccountQueryOptions{ @@ -1719,10 +1720,11 @@ func TestSearchForInnerTransactionReturnsRootTransaction(t *testing.T) { l, err := test.MakeTestLedger(logger) require.NoError(t, err) defer l.Close() - proc, err := blockprocessor.MakeProcessorWithLedger(logger, l, db.AddBlock) + pr, err := blockprocessor.MakeBlockProcessorWithLedger(logger, l, db.AddBlock) require.NoError(t, err, "failed to open ledger") blockCert := rpcs.EncodedBlockCert{Block: block} - return proc.Process(&blockCert) + proc := blockprocessor.MakeBlockProcessorHandlerAdapter(&pr, db.AddBlock) + return proc(&blockCert) }, nil) require.NoError(t, err) @@ -1830,7 +1832,7 @@ func TestNonUTF8Logs(t *testing.T) { require.NoError(t, err) // Test 1: import/accounting should work. - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Test 2: transaction results properly serialized @@ -1892,7 +1894,7 @@ func TestTxnAssetID(t *testing.T) { &createAppTxn, &destroyAppTxn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) txnRowsCh, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) @@ -1919,7 +1921,7 @@ func TestBadTxnJsonEncoding(t *testing.T) { // Need to import a block header because the transactions query joins on it. block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) rootTxid := "abc" @@ -1990,7 +1992,7 @@ func TestKeytypeDoNotResetReceiver(t *testing.T) { 0, 0, 0, 0, 0, 0, test.AccountB, test.AccountB, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // Sigtype of account A becomes "sig" and B remains the same. @@ -1998,7 +2000,7 @@ func TestKeytypeDoNotResetReceiver(t *testing.T) { 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountB, basics.Address{}, basics.Address{}) block, err = test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) keytype := generated.AccountSigTypeSig @@ -2032,7 +2034,7 @@ func TestAddBlockTxnTxnParticipationAhead(t *testing.T) { 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) } @@ -2048,7 +2050,7 @@ func TestAddBlockTxnParticipationAdded(t *testing.T) { 0, 0, 0, 0, 0, 0, test.AccountA, test.AccountA, basics.Address{}, basics.Address{}) block, err := test.MakeBlockForTxns(block.BlockHeader, &txn) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) tf := idb.TransactionFilter{ @@ -2090,7 +2092,7 @@ func TestTransactionsTxnAhead(t *testing.T) { { block, err := test.MakeBlockForTxns(block.BlockHeader) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) } { @@ -2270,7 +2272,7 @@ func TestTransactionFilterAssetAmount(t *testing.T) { block, err := test.MakeBlockForTxns(block.BlockHeader, &txnA, &txnB, &txnC) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // query @@ -2290,7 +2292,7 @@ func TestTransactionFilterAssetAmount(t *testing.T) { block, err = test.MakeBlockForTxns(block.BlockHeader, &txnD, &txnE, &txnF) require.NoError(t, err) - err = proc.Process(&rpcs.EncodedBlockCert{Block: block}) + err = proc(&rpcs.EncodedBlockCert{Block: block}) require.NoError(t, err) // query @@ -2304,3 +2306,76 @@ func TestTransactionFilterAssetAmount(t *testing.T) { assert.Equal(t, txnF, *row.Txn) } + +func TestDeleteTransactions(t *testing.T) { + block := test.MakeGenesisBlock() + db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis()) + defer shutdownFunc() + defer l.Close() + + txnA := test.MakeCreateAppTxn(test.AccountA) + txnB := test.MakeCreateAppTxn(test.AccountB) + txnC := test.MakeCreateAppTxn(test.AccountC) + txnD := test.MakeCreateAppTxn(test.AccountD) + + txns := []transactions.SignedTxnWithAD{txnA, txnB, txnC, txnD} + + // add 4 rounds of txns + for i := 1; i <= 4; i++ { + block, err := test.MakeBlockForTxns(block.BlockHeader, &txns[i-1]) + block.BlockHeader.Round = basics.Round(i) + require.NoError(t, err) + err = proc(&rpcs.EncodedBlockCert{Block: block}) + require.NoError(t, err) + } + + // keep rounds >= 2 + err := db.DeleteTransactions(context.Background(), 2) + assert.NoError(t, err) + + // query txns + rowsCh, _ := db.Transactions(context.Background(), idb.TransactionFilter{}) + + // check remaining transactions are correct + i := 1 + for row := range rowsCh { + require.NoError(t, row.Error) + require.NotNil(t, row.Txn) + assert.Equal(t, txns[i], *row.Txn) + i++ + } + + // verify metastate + deleteStatus, err := db.getMetastate(context.Background(), nil, schema.DeleteStatusKey) + assert.NoError(t, err) + status, err := encoding.DecodeDeleteStatus([]byte(deleteStatus)) + assert.NoError(t, err) + assert.Equal(t, uint64(2), status.OldestRound) + + // add 2 txns for round 5 + block, err = test.MakeBlockForTxns(block.BlockHeader, &txnA, &txnB) + block.BlockHeader.Round = basics.Round(5) + require.NoError(t, err) + err = proc(&rpcs.EncodedBlockCert{Block: block}) + require.NoError(t, err) + + // keep round 5 + err = db.DeleteTransactions(context.Background(), 5) + assert.NoError(t, err) + + // 2 txns in round 5 + rowsCh, _ = db.Transactions(context.Background(), idb.TransactionFilter{}) + i = 0 + for row := range rowsCh { + require.NoError(t, row.Error) + require.NotNil(t, row.Txn) + assert.Equal(t, txns[i], *row.Txn) + i++ + } + + deleteStatus, err = db.getMetastate(context.Background(), nil, schema.DeleteStatusKey) + assert.NoError(t, err) + status, err = encoding.DecodeDeleteStatus([]byte(deleteStatus)) + assert.NoError(t, err) + assert.Equal(t, uint64(5), status.OldestRound) +} diff --git a/importer/helper.go b/importer/helper.go index 9eb185e70..7939be373 100644 --- a/importer/helper.go +++ b/importer/helper.go @@ -26,6 +26,11 @@ import ( "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/rpcs" + "github.com/algorand/indexer/conduit/plugins/processors/blockprocessor" + "github.com/algorand/indexer/util" + log "github.com/sirupsen/logrus" + + "github.com/algorand/indexer/idb" ) // NewImportHelper builds an ImportHelper @@ -143,14 +148,19 @@ func importTar(imp Importer, tarfile io.Reader, logger *log.Logger, genesisReade maybeFail(err, logger, "error decoding genesis, %v", err) } - ld, err := util.MakeLedger(logger, false, &genesis, "") + dir, err := os.MkdirTemp(os.TempDir(), "indexer_import_tempdir") + maybeFail(err, logger, "Failed to make temp dir") + + ld, err := util.MakeLedger(logger, false, &genesis, dir) maybeFail(err, logger, "Cannot open ledger") - proc, err := blockprocessor.MakeProcessorWithLedger(logger, ld, imp.ImportBlock) + proc, err := blockprocessor.MakeBlockProcessorWithLedger(logger, ld, imp.ImportBlock) maybeFail(err, logger, "Error creating processor") + f := blockprocessor.MakeBlockProcessorHandlerAdapter(&proc, imp.ImportBlock) + for _, blockContainer := range blocks[1:] { - err = proc.Process(&blockContainer) + err = f(&blockContainer) if err != nil { return } diff --git a/loggers/loggers.go b/loggers/loggers.go new file mode 100644 index 000000000..410e84159 --- /dev/null +++ b/loggers/loggers.go @@ -0,0 +1,48 @@ +package loggers + +import ( + "io" + "sync" + + log "github.com/sirupsen/logrus" +) + +// LoggerManager a manager that can produce loggers that are synchronized internally +type LoggerManager struct { + internalWriter ThreadSafeWriter +} + +// AdaptLogger will take a logger instance and wrap it for synchronization +func AdaptLogger(log *log.Logger) *LoggerManager { + return MakeLoggerManager(log.Out) +} + +// MakeLogger returns a logger that is synchronized with the internal mutex +func (l *LoggerManager) MakeLogger() *log.Logger { + logger := log.New() + logger.SetOutput(l.internalWriter) + return logger +} + +// MakeLoggerManager returns a logger manager +func MakeLoggerManager(writer io.Writer) *LoggerManager { + return &LoggerManager{ + internalWriter: ThreadSafeWriter{ + Writer: writer, + Mutex: &sync.Mutex{}, + }, + } +} + +// ThreadSafeWriter a struct that implements io.Writer in a threadsafe way +type ThreadSafeWriter struct { + Writer io.Writer + Mutex *sync.Mutex +} + +// Write writes p bytes with the mutex +func (w ThreadSafeWriter) Write(p []byte) (n int, err error) { + w.Mutex.Lock() + defer w.Mutex.Unlock() + return w.Writer.Write(p) +} diff --git a/loggers/loggers_test.go b/loggers/loggers_test.go new file mode 100644 index 000000000..33246405a --- /dev/null +++ b/loggers/loggers_test.go @@ -0,0 +1,89 @@ +package loggers + +import ( + "encoding/json" + "math/rand" + "sync" + "sync/atomic" + "testing" + "time" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +type FakeIoWriter struct { + // A list of strings that represent ints + ListOfInts []string +} + +func (f *FakeIoWriter) Write(p []byte) (n int, err error) { + f.ListOfInts = append(f.ListOfInts, string(p)) + return len(p), nil +} + +// TestThreadSafetyOfLogger ensures that multiple threads writing to a single source +// don't get corrupted +func TestThreadSafetyOfLogger(t *testing.T) { + var atomicInt int64 = 0 + + fakeWriter := FakeIoWriter{} + lMgr := MakeLoggerManager(&fakeWriter) + + const numberOfWritesPerLogger = 20 + const numberOfLoggers = 15 + + var wg sync.WaitGroup + wg.Add(numberOfLoggers) + + // Launch go routines + for i := 0; i < numberOfLoggers; i++ { + go func() { + // Sleep a random number of milliseconds before and after to test + // that creating a logger doesn't affect thread-safety + time.Sleep(time.Duration(rand.Intn(50)) * time.Millisecond) + l := lMgr.MakeLogger() + l.SetFormatter(&log.JSONFormatter{ + // We want to disable timestamps to stop logrus from sorting our output + DisableTimestamp: true, + }) + time.Sleep(time.Duration(rand.Intn(50)) * time.Millisecond) + + for j := 0; j < numberOfWritesPerLogger; j++ { + + // Atomically adds 1 and returns new value + localInt := atomic.AddInt64(&atomicInt, 1) + l.Infof("%d", localInt) + + } + wg.Done() + }() + + } + wg.Wait() + + assert.Equal(t, len(fakeWriter.ListOfInts), numberOfLoggers*numberOfWritesPerLogger) + + // We can't assume that the writes are in order since the call to atomically update + // and log are not atomic *together*...just independently. So we test that all + // numbers are present with a map and have no duplicates + numMap := make(map[string]bool) + + for i := 0; i < numberOfLoggers*numberOfWritesPerLogger; i++ { + var jsonText map[string]interface{} + err := json.Unmarshal([]byte(fakeWriter.ListOfInts[i]), &jsonText) + assert.NoError(t, err) + + sourceString := jsonText["msg"].(string) + + _, ok := numMap[sourceString] + // We shouldn't have seen this before + assert.False(t, ok) + + numMap[sourceString] = true + + } + + assert.Equal(t, len(numMap), numberOfLoggers*numberOfWritesPerLogger) + +} diff --git a/misc/blockarchiver.py b/misc/blockarchiver.py index d8f4627df..ba09645a9 100755 --- a/misc/blockarchiver.py +++ b/misc/blockarchiver.py @@ -17,7 +17,7 @@ import algosdk -from util import maybedecode, mloads, unmsgpack +from e2e_common.util import maybedecode, mloads, unmsgpack logger = logging.getLogger(__name__) diff --git a/misc/liveindextest.py b/misc/liveindextest.py index 43bb09441..6b4b71830 100644 --- a/misc/liveindextest.py +++ b/misc/liveindextest.py @@ -34,7 +34,7 @@ import algosdk import algosdk.v2client -from util import xrun, atexitrun, find_indexer, ensure_test_db +from e2e_common.util import xrun, atexitrun, find_indexer, ensure_test_db logger = logging.getLogger(__name__) diff --git a/misc/release.py b/misc/release.py index 0d0ea1b58..fc132b547 100755 --- a/misc/release.py +++ b/misc/release.py @@ -43,7 +43,7 @@ def groupname(gid): channel = "indexer" -filespec = [ +indexer_filespec = [ # [files], source path, deb path, tar path [ ["algorand-indexer.service", "algorand-indexer@.service"], @@ -65,6 +65,21 @@ def groupname(gid): ], ] +conduit_filespec = [ + [ + ["conduit"], + "cmd/conduit", + "usr/bin", + "" + ], + [ + ["LICENSE"], + "", + None, + "", + ], +] + debian_copyright_top = """Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: Algorand Indexer Upstream-Contact: Algorand developers @@ -161,7 +176,7 @@ def compile_version_opts(release_version=None, allow_mismatch=False): return ldflags -def compile(goos=None, goarch=None, ldflags=None): +def compile(path="cmd/algorand-indexer", goos=None, goarch=None, ldflags=None): env = dict(os.environ) if goos is not None: env["GOOS"] = goos @@ -170,10 +185,11 @@ def compile(goos=None, goarch=None, ldflags=None): cmd = ["go", "build"] if ldflags is not None: cmd.append(ldflags) - subprocess.run(cmd, cwd="cmd/algorand-indexer", env=env).check_returncode() + subprocess.run(["go", "generate", "./..."], env=env).check_returncode() + subprocess.run(cmd, cwd=path, env=env).check_returncode() -def build_deb(debarch, version, outdir): +def build_deb(debarch, version, filespec, outdir): os.makedirs(".deb_tmp/DEBIAN", exist_ok=True) debian_copyright(".deb_tmp/DEBIAN/copyright") arch_ver(".deb_tmp/DEBIAN/control", "misc/debian/control", debarch, version) @@ -228,8 +244,8 @@ def usage_html(): return _usage_html -def build_tar(goos, goarch, version, outdir): - rootdir = "algorand-indexer_{}_{}_{}".format(goos, goarch, version) +def build_tar(name, goos, goarch, version, filespec, outdir): + rootdir = "{}_{}_{}_{}".format(name, goos, goarch, version) tarname = os.path.join(outdir, rootdir) + ".tar.bz2" tf = tarfile.open(tarname, "w:bz2") for files, source_path, _, tar_path in filespec: @@ -319,14 +335,17 @@ def main(): logger.debug("skip %s %s", goos, goarch) continue logger.info("GOOS=%s GOARCH=%s DEB_HOST_ARCH=%s", goos, goarch, debarch) - compile(goos, goarch, ldflags) + compile("cmd/algorand-indexer", goos, goarch, ldflags) + compile("cmd/conduit", goos, goarch) if args.build_only: logger.debug("skip packaging") continue - tarname = build_tar(goos, goarch, version, outdir) - logger.info("\t%s", tarname) + indexer_tarname = build_tar("algorand-indexer", goos, goarch, version, indexer_filespec, outdir) + logger.info("\t%s", indexer_tarname) + conduit_tarname = build_tar("conduit", goos, goarch, version, conduit_filespec, outdir) + logger.info("\t%s", conduit_tarname) if (not args.no_deb) and (debarch is not None): - debname = build_deb(debarch, version, outdir) + debname = build_deb(debarch, version, indexer_filespec, outdir) logger.info("\t%s", debname) dt = time.time() - start logger.info("done %0.1fs", dt) diff --git a/misc/requirements.txt b/misc/requirements.txt index 767aaa776..f6a3ca5b1 100644 --- a/misc/requirements.txt +++ b/misc/requirements.txt @@ -1,8 +1,7 @@ # packages needed for test and release -boto3 >=1.14.9,<2 markdown2 >=2.3.9,<3 msgpack >=1,<2 py-algorand-sdk >=1.3.0,<2 pytest==6.2.5 GitPython==3.1.26 -PyYAML==6.0 \ No newline at end of file +PyYAML==6.0 diff --git a/processor/blockprocessor/block_processor.go b/processor/blockprocessor/block_processor.go deleted file mode 100644 index a670ac5b7..000000000 --- a/processor/blockprocessor/block_processor.go +++ /dev/null @@ -1,218 +0,0 @@ -package blockprocessor - -import ( - "context" - "fmt" - - log "github.com/sirupsen/logrus" - - "github.com/algorand/indexer/accounting" - "github.com/algorand/indexer/idb" - "github.com/algorand/indexer/processor" - indexerledger "github.com/algorand/indexer/processor/eval" - "github.com/algorand/indexer/util" - - "github.com/algorand/go-algorand/config" - "github.com/algorand/go-algorand/data/basics" - "github.com/algorand/go-algorand/data/bookkeeping" - "github.com/algorand/go-algorand/data/transactions" - "github.com/algorand/go-algorand/ledger" - "github.com/algorand/go-algorand/ledger/ledgercore" - "github.com/algorand/go-algorand/rpcs" -) - -type blockProcessor struct { - handler func(block *ledgercore.ValidatedBlock) error - ledger *ledger.Ledger - logger *log.Logger -} - -// MakeProcessorWithLedger creates a block processor with a given ledger -func MakeProcessorWithLedger(logger *log.Logger, l *ledger.Ledger, handler func(block *ledgercore.ValidatedBlock) error) (processor.Processor, error) { - if l == nil { - return nil, fmt.Errorf("MakeProcessorWithLedger() err: local ledger not initialized") - } - err := addGenesisBlock(l, handler) - if err != nil { - return nil, fmt.Errorf("MakeProcessorWithLedger() err: %w", err) - } - return &blockProcessor{logger: logger, ledger: l, handler: handler}, nil -} - -// MakeProcessorWithLedgerInit creates a block processor and initializes the ledger. -func MakeProcessorWithLedgerInit(ctx context.Context, logger *log.Logger, catchpoint string, genesis *bookkeeping.Genesis, nextDBRound uint64, opts idb.IndexerDbOptions, handler func(block *ledgercore.ValidatedBlock) error) (processor.Processor, error) { - err := InitializeLedger(ctx, logger, catchpoint, nextDBRound, *genesis, &opts) - if err != nil { - return nil, fmt.Errorf("MakeProcessorWithLedgerInit() err: %w", err) - } - return MakeProcessor(logger, genesis, nextDBRound, opts.IndexerDatadir, handler) -} - -// MakeProcessor creates a block processor -func MakeProcessor(logger *log.Logger, genesis *bookkeeping.Genesis, dbRound uint64, datadir string, handler func(block *ledgercore.ValidatedBlock) error) (processor.Processor, error) { - l, err := util.MakeLedger(logger, false, genesis, datadir) - if err != nil { - return nil, fmt.Errorf("MakeProcessor() err: %w", err) - } - if uint64(l.Latest()) > dbRound { - return nil, fmt.Errorf("MakeProcessor() err: the ledger cache is ahead of the required round and must be re-initialized") - } - return MakeProcessorWithLedger(logger, l, handler) -} - -func addGenesisBlock(l *ledger.Ledger, handler func(block *ledgercore.ValidatedBlock) error) error { - if handler != nil && l != nil && l.Latest() == 0 { - blk, err := l.Block(0) - if err != nil { - return fmt.Errorf("addGenesisBlock() err: %w", err) - } - vb := ledgercore.MakeValidatedBlock(blk, ledgercore.StateDelta{}) - err = handler(&vb) - if err != nil { - return fmt.Errorf("addGenesisBlock() handler err: %w", err) - } - } - return nil -} - -// Process a raw algod block -func (proc *blockProcessor) Process(blockCert *rpcs.EncodedBlockCert) error { - - if blockCert == nil { - return fmt.Errorf("Process(): cannot process a nil block") - } - if blockCert.Block.Round() != (proc.ledger.Latest() + 1) { - return fmt.Errorf("Process() invalid round blockCert.Block.Round(): %d nextRoundToProcess: %d", blockCert.Block.Round(), uint64(proc.ledger.Latest())+1) - } - - // Make sure "AssetCloseAmount" is enabled. If it isn't, override the - // protocol and update the blocks to include transactions with modified - // apply data. - proto, ok := config.Consensus[blockCert.Block.BlockHeader.CurrentProtocol] - if !ok { - return fmt.Errorf( - "Process() cannot find proto version %s", blockCert.Block.BlockHeader.CurrentProtocol) - } - protoChanged := !proto.EnableAssetCloseAmount - proto.EnableAssetCloseAmount = true - - ledgerForEval := indexerledger.MakeLedgerForEvaluator(proc.ledger) - - resources, err := prepareEvalResources(&ledgerForEval, &blockCert.Block) - if err != nil { - proc.logger.Panicf("Process() resources err: %v", err) - } - - delta, modifiedTxns, err := - ledger.EvalForIndexer(ledgerForEval, &blockCert.Block, proto, resources) - if err != nil { - return fmt.Errorf("Process() eval err: %w", err) - } - // validated block - var vb ledgercore.ValidatedBlock - if protoChanged { - block := bookkeeping.Block{ - BlockHeader: blockCert.Block.BlockHeader, - Payset: modifiedTxns, - } - vb = ledgercore.MakeValidatedBlock(block, delta) - } else { - vb = ledgercore.MakeValidatedBlock(blockCert.Block, delta) - } - - // execute handler before writing to local ledger - if proc.handler != nil { - err = proc.handler(&vb) - if err != nil { - return fmt.Errorf("Process() handler err: %w", err) - } - } - // write to ledger - err = proc.ledger.AddValidatedBlock(vb, blockCert.Certificate) - if err != nil { - return fmt.Errorf("Process() add validated block err: %w", err) - } - // wait for commit to disk - proc.ledger.WaitForCommit(blockCert.Block.Round()) - return nil -} - -func (proc *blockProcessor) SetHandler(handler func(block *ledgercore.ValidatedBlock) error) { - proc.handler = handler -} - -func (proc *blockProcessor) NextRoundToProcess() uint64 { - return uint64(proc.ledger.Latest()) + 1 -} - -// Preload all resources (account data, account resources, asset/app creators) for the -// evaluator. -func prepareEvalResources(l *indexerledger.LedgerForEvaluator, block *bookkeeping.Block) (ledger.EvalForIndexerResources, error) { - assetCreators, appCreators, err := prepareCreators(l, block.Payset) - if err != nil { - return ledger.EvalForIndexerResources{}, - fmt.Errorf("prepareEvalResources() err: %w", err) - } - - res := ledger.EvalForIndexerResources{ - Accounts: nil, - Resources: nil, - Creators: make(map[ledger.Creatable]ledger.FoundAddress), - } - - for index, foundAddress := range assetCreators { - creatable := ledger.Creatable{ - Index: basics.CreatableIndex(index), - Type: basics.AssetCreatable, - } - res.Creators[creatable] = foundAddress - } - for index, foundAddress := range appCreators { - creatable := ledger.Creatable{ - Index: basics.CreatableIndex(index), - Type: basics.AppCreatable, - } - res.Creators[creatable] = foundAddress - } - - res.Accounts, res.Resources, err = prepareAccountsResources(l, block.Payset, assetCreators, appCreators) - if err != nil { - return ledger.EvalForIndexerResources{}, - fmt.Errorf("prepareEvalResources() err: %w", err) - } - - return res, nil -} - -// Preload asset and app creators. -func prepareCreators(l *indexerledger.LedgerForEvaluator, payset transactions.Payset) (map[basics.AssetIndex]ledger.FoundAddress, map[basics.AppIndex]ledger.FoundAddress, error) { - assetsReq, appsReq := accounting.MakePreloadCreatorsRequest(payset) - - assets, err := l.GetAssetCreator(assetsReq) - if err != nil { - return nil, nil, fmt.Errorf("prepareCreators() err: %w", err) - } - apps, err := l.GetAppCreator(appsReq) - if err != nil { - return nil, nil, fmt.Errorf("prepareCreators() err: %w", err) - } - - return assets, apps, nil -} - -// Preload account data and account resources. -func prepareAccountsResources(l *indexerledger.LedgerForEvaluator, payset transactions.Payset, assetCreators map[basics.AssetIndex]ledger.FoundAddress, appCreators map[basics.AppIndex]ledger.FoundAddress) (map[basics.Address]*ledgercore.AccountData, map[basics.Address]map[ledger.Creatable]ledgercore.AccountResource, error) { - addressesReq, resourcesReq := - accounting.MakePreloadAccountsResourcesRequest(payset, assetCreators, appCreators) - - accounts, err := l.LookupWithoutRewards(addressesReq) - if err != nil { - return nil, nil, fmt.Errorf("prepareAccountsResources() err: %w", err) - } - resources, err := l.LookupResources(resourcesReq) - if err != nil { - return nil, nil, fmt.Errorf("prepareAccountsResources() err: %w", err) - } - - return accounts, resources, nil -} diff --git a/processor/processor.go b/processor/processor.go deleted file mode 100644 index 43de31441..000000000 --- a/processor/processor.go +++ /dev/null @@ -1,13 +0,0 @@ -package processor - -import ( - "github.com/algorand/go-algorand/ledger/ledgercore" - "github.com/algorand/go-algorand/rpcs" -) - -// Processor is the block processor interface -type Processor interface { - Process(cert *rpcs.EncodedBlockCert) error - SetHandler(handler func(block *ledgercore.ValidatedBlock) error) - NextRoundToProcess() uint64 -} diff --git a/test/common.sh b/test/common.sh index b644538f0..83a2845ea 100755 --- a/test/common.sh +++ b/test/common.sh @@ -3,6 +3,7 @@ # The cleanup hook ensures these containers are removed when the script exits. POSTGRES_CONTAINER=test-container export INDEXER_DATA=/tmp/e2e_test/ +TEST_DATA_DIR=$(mktemp) NET=localhost:8981 CURL_TEMPFILE=curl_out.txt @@ -202,7 +203,7 @@ function start_indexer_with_connection_string() { ALGORAND_DATA= ../cmd/algorand-indexer/algorand-indexer daemon \ -S $NET "$RO" \ -P "$1" \ - -i /tmp \ + -i "$TEST_DATA_DIR" \ --enable-all-parameters \ "$RO" \ --pidfile $PIDFILE 2>&1 > /dev/null & @@ -212,7 +213,7 @@ function start_indexer_with_connection_string() { # $2 - if set, halts execution function start_indexer() { if [ ! -z $2 ]; then - echo "daemon -i /tmp -S $NET -P \"${CONNECTION_STRING/DB_NAME_HERE/$1}\"" + echo "daemon -i "$TEST_DATA_DIR" -S $NET -P \"${CONNECTION_STRING/DB_NAME_HERE/$1}\"" sleep_forever fi @@ -292,9 +293,9 @@ function kill_indexer() { kill -9 $(cat "$PIDFILE") > /dev/null 2>&1 || true rm $PIDFILE rm -rf $INDEXER_DATA + rm -rf $TEST_DATA_DIR/* pwd ls -l - rm ledger*sqlite* || true fi } diff --git a/util/ledger_util.go b/util/ledger_util.go index e4f184e7c..e9bbc650b 100644 --- a/util/ledger_util.go +++ b/util/ledger_util.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "io/ioutil" + "os" "path/filepath" log "github.com/sirupsen/logrus" @@ -64,6 +65,10 @@ func createInitState(genesis *bookkeeping.Genesis) (ledgercore.InitState, error) // MakeLedger opens a ledger, initializing if necessary. func MakeLedger(logger *log.Logger, inMemory bool, genesis *bookkeeping.Genesis, dataDir string) (*ledger.Ledger, error) { const prefix = "ledger" + err := os.MkdirAll(dataDir, 0755) + if err != nil { + return nil, fmt.Errorf("MakeProcessor() failed to create '%s': %w", dataDir, err) + } dbPrefix := filepath.Join(dataDir, prefix) initState, err := createInitState(genesis) if err != nil { diff --git a/util/metrics/metrics.go b/util/metrics/metrics.go index e9b9f1a09..4f7924fe2 100644 --- a/util/metrics/metrics.go +++ b/util/metrics/metrics.go @@ -5,34 +5,34 @@ import "github.com/prometheus/client_golang/prometheus" // RegisterPrometheusMetrics register all prometheus metrics with the global // metrics handler. func RegisterPrometheusMetrics() { - prometheus.Register(BlockImportTimeSeconds) - prometheus.Register(ImportedTxnsPerBlock) - prometheus.Register(ImportedRoundGauge) - prometheus.Register(BlockUploadTimeSeconds) - prometheus.Register(PostgresEvalTimeSeconds) - prometheus.Register(GetAlgodRawBlockTimeSeconds) - prometheus.Register(ImportedTxns) + for _, c := range collectors { + prometheus.Register(c) + } } // Prometheus metric names broken out for reuse. const ( BlockImportTimeName = "import_time_sec" - BlockUploadTimeName = "block_upload_time_sec" ImportedTxnsPerBlockName = "imported_tx_per_block" ImportedRoundGaugeName = "imported_round" - PostgresEvalName = "postgres_eval_time_sec" GetAlgodRawBlockTimeName = "get_algod_raw_block_time_sec" ImportedTxnsName = "imported_txns" + ImporterTimeName = "importer_time_sec" + ProcessorTimeName = "processor_time_sec" + ExporterTimeName = "exporter_time_sec" + PipelineRetryCountName = "pipeline_retry_count" ) // AllMetricNames is a reference for all the custom metric names. var AllMetricNames = []string{ BlockImportTimeName, - BlockUploadTimeName, ImportedTxnsPerBlockName, ImportedRoundGaugeName, - PostgresEvalName, GetAlgodRawBlockTimeName, + ImporterTimeName, + ProcessorTimeName, + ExporterTimeName, + PipelineRetryCountName, } // Initialize the prometheus objects. @@ -44,13 +44,6 @@ var ( Help: "Total block upload and processing time in seconds.", }) - BlockUploadTimeSeconds = prometheus.NewSummary( - prometheus.SummaryOpts{ - Subsystem: "indexer_daemon", - Name: BlockUploadTimeName, - Help: "Block upload time in seconds.", - }) - ImportedTxnsPerBlock = prometheus.NewSummary( prometheus.SummaryOpts{ Subsystem: "indexer_daemon", @@ -75,17 +68,53 @@ var ( Help: "The most recent round indexer has imported.", }) - PostgresEvalTimeSeconds = prometheus.NewSummary( + GetAlgodRawBlockTimeSeconds = prometheus.NewSummary( prometheus.SummaryOpts{ Subsystem: "indexer_daemon", - Name: PostgresEvalName, - Help: "Time spent calling Eval function in seconds.", + Name: GetAlgodRawBlockTimeName, + Help: "Total response time from Algod's raw block endpoint in seconds.", }) - GetAlgodRawBlockTimeSeconds = prometheus.NewSummary( + ImporterTimeSeconds = prometheus.NewSummary( prometheus.SummaryOpts{ Subsystem: "indexer_daemon", - Name: GetAlgodRawBlockTimeName, - Help: "Total response time from Algod's raw block endpoint in seconds.", + Name: ImporterTimeName, + Help: "Time spent at importer step", + }) + + ProcessorTimeSeconds = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Subsystem: "indexer_daemon", + Name: ProcessorTimeName, + Help: "Time spent running a processor", + }, + []string{"processor_name"}, + ) + + ExporterTimeSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Subsystem: "indexer_daemon", + Name: ExporterTimeName, + Help: "Time spent at exporter step", + }) + + PipelineRetryCount = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Subsystem: "indexer_daemon", + Name: PipelineRetryCountName, + Help: "Total pipeline retries since last successful run", }) ) + +var collectors = []prometheus.Collector{ + BlockImportTimeSeconds, + BlockImportTimeSeconds, + ImportedTxnsPerBlock, + ImportedRoundGauge, + GetAlgodRawBlockTimeSeconds, + ImportedTxns, + ImporterTimeSeconds, + ProcessorTimeSeconds, + ExporterTimeSeconds, + PipelineRetryCount, +} diff --git a/util/pidfile.go b/util/pidfile.go new file mode 100644 index 000000000..552293ccd --- /dev/null +++ b/util/pidfile.go @@ -0,0 +1,34 @@ +package util + +import ( + "fmt" + "os" + + log "github.com/sirupsen/logrus" +) + +// CreateIndexerPidFile creates the pid file at the specified location +func CreateIndexerPidFile(logger *log.Logger, pidFilePath string) error { + var err error + logger.Infof("Creating PID file at: %s", pidFilePath) + fout, err := os.Create(pidFilePath) + if err != nil { + err = fmt.Errorf("%s: could not create pid file, %v", pidFilePath, err) + logger.Error(err) + return err + } + + if _, err = fmt.Fprintf(fout, "%d", os.Getpid()); err != nil { + err = fmt.Errorf("%s: could not write pid file, %v", pidFilePath, err) + logger.Error(err) + return err + } + + err = fout.Close() + if err != nil { + err = fmt.Errorf("%s: could not close pid file, %v", pidFilePath, err) + logger.Error(err) + return err + } + return err +} diff --git a/util/test/mock_algod.go b/util/test/mock_algod.go new file mode 100644 index 000000000..b0e2afdda --- /dev/null +++ b/util/test/mock_algod.go @@ -0,0 +1,82 @@ +package test + +import ( + "github.com/algorand/go-algorand-sdk/client/v2/algod" + "github.com/algorand/go-algorand-sdk/client/v2/common/models" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/rpcs" + "net/http" + "net/http/httptest" + "path" + "strconv" + "strings" +) + +// AlgodHandler is used to handle http requests to a mock algod server +type AlgodHandler struct { + responders []func(path string, w http.ResponseWriter) bool +} + +// NewAlgodServer creates an httptest server with an algodHandler using the provided responders +func NewAlgodServer(responders ...func(path string, w http.ResponseWriter) bool) *httptest.Server { + return httptest.NewServer(&AlgodHandler{responders}) +} + +// NewAlgodHandler creates an AlgodHandler using the provided responders +func NewAlgodHandler(responders ...func(path string, w http.ResponseWriter) bool) *AlgodHandler { + return &AlgodHandler{responders} +} + +// ServeHTTP implements the http.Handler interface for AlgodHandler +func (handler *AlgodHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { + for _, responder := range handler.responders { + if responder(req.URL.Path, w) { + return + } + } + w.WriteHeader(http.StatusBadRequest) +} + +// MockAClient creates an algod client using an AlgodHandler based server +func MockAClient(handler *AlgodHandler) (*algod.Client, error) { + mockServer := httptest.NewServer(handler) + return algod.MakeClient(mockServer.URL, "") +} + +// BlockResponder handles /v2/blocks requests and returns an empty Block object +func BlockResponder(reqPath string, w http.ResponseWriter) bool { + if strings.Contains(reqPath, "v2/blocks/") { + rnd, _ := strconv.Atoi(path.Base(reqPath)) + blk := rpcs.EncodedBlockCert{Block: bookkeeping.Block{BlockHeader: bookkeeping.BlockHeader{Round: basics.Round(rnd)}}} + blockbytes := protocol.Encode(&blk) + w.WriteHeader(http.StatusOK) + w.Write(blockbytes) + return true + } + return false +} + +// GenesisResponder handles /v2/genesis requests and returns an empty Genesis object +func GenesisResponder(reqPath string, w http.ResponseWriter) bool { + if strings.Contains(reqPath, "/genesis") { + w.WriteHeader(http.StatusOK) + genesis := &bookkeeping.Genesis{} + blockbytes := protocol.EncodeJSON(*genesis) + w.Write(blockbytes) + return true + } + return false +} + +// BlockAfterResponder handles /v2/wait-for-block-after requests and returns an empty NodeStatus object +func BlockAfterResponder(reqPath string, w http.ResponseWriter) bool { + if strings.Contains(reqPath, "/wait-for-block-after") { + w.WriteHeader(http.StatusOK) + nStatus := models.NodeStatus{} + w.Write(protocol.EncodeJSON(nStatus)) + return true + } + return false +} diff --git a/util/test/testutil.go b/util/test/testutil.go index 17631f1cc..5563c875f 100644 --- a/util/test/testutil.go +++ b/util/test/testutil.go @@ -7,6 +7,7 @@ import ( "os" "runtime" + "github.com/algorand/go-algorand/data/bookkeeping" log "github.com/sirupsen/logrus" "github.com/algorand/indexer/idb" @@ -123,3 +124,27 @@ func MakeTestLedger(logger *log.Logger) (*ledger.Ledger, error) { genesis := MakeGenesis() return util.MakeLedger(logger, true, &genesis, "ledger") } + +// MockInitProvider mock an init provider +type MockInitProvider struct { + CurrentRound *basics.Round + Genesis *bookkeeping.Genesis +} + +// GetGenesis produces genesis pointer +func (m *MockInitProvider) GetGenesis() *bookkeeping.Genesis { + return m.Genesis +} + +// NextDBRound provides next database round +func (m *MockInitProvider) NextDBRound() basics.Round { + return *m.CurrentRound +} + +// MockedInitProvider returns an InitProvider for testing +func MockedInitProvider(round *basics.Round) *MockInitProvider { + return &MockInitProvider{ + CurrentRound: round, + Genesis: &bookkeeping.Genesis{}, + } +} diff --git a/util/util.go b/util/util.go index 52a366ea7..43d3fd284 100644 --- a/util/util.go +++ b/util/util.go @@ -1,15 +1,72 @@ package util import ( + "bytes" + "compress/gzip" "fmt" + "io" + "io/ioutil" "os" "strings" "unicode" "unicode/utf8" + "github.com/algorand/go-algorand-sdk/encoding/json" "github.com/algorand/go-codec/codec" ) +// EncodeToFile is used to encode an object to a file. If the file ends in .gz it will be gzipped. +func EncodeToFile(filename string, v interface{}, pretty bool) error { + var writer io.Writer + + file, err := os.Create(filename) + if err != nil { + return fmt.Errorf("EncodeToFile(): failed to create %s: %w", filename, err) + } + defer file.Close() + + if strings.HasSuffix(filename, ".gz") { + gz := gzip.NewWriter(file) + gz.Name = filename + defer gz.Close() + writer = gz + } else { + writer = file + } + + handle := json.CodecHandle + if pretty { + handle.Indent = 2 + } else { + handle.Indent = 0 + } + enc := codec.NewEncoder(writer, handle) + return enc.Encode(v) +} + +// DecodeFromFile is used to decode a file to an object. +func DecodeFromFile(filename string, v interface{}) error { + // Streaming into the decoder was slow. + fileBytes, err := ioutil.ReadFile(filename) + if err != nil { + return fmt.Errorf("DecodeFromFile(): failed to read %s: %w", filename, err) + } + + var reader io.Reader = bytes.NewReader(fileBytes) + + if strings.HasSuffix(filename, ".gz") { + gz, err := gzip.NewReader(reader) + defer gz.Close() + if err != nil { + return fmt.Errorf("DecodeFromFile(): failed to make gzip reader: %w", err) + } + reader = gz + } + + enc := codec.NewDecoder(reader, json.CodecHandle) + return enc.Decode(v) +} + // PrintableUTF8OrEmpty checks to see if the entire string is a UTF8 printable string. // If this is the case, the string is returned as is. Otherwise, the empty string is returned. func PrintableUTF8OrEmpty(in string) string { diff --git a/util/util_test.go b/util/util_test.go index bcfe3917f..0d01dc2be 100644 --- a/util/util_test.go +++ b/util/util_test.go @@ -2,9 +2,62 @@ package util import ( "encoding/base64" + "io/ioutil" + "path" "testing" + + "github.com/stretchr/testify/require" ) +func TestEncodeToAndFromFile(t *testing.T) { + tempdir := t.TempDir() + + type test struct { + One string `json:"one"` + Two int `json:"two"` + } + data := test{ + One: "one", + Two: 2, + } + + { + pretty := path.Join(tempdir, "pretty.json") + err := EncodeToFile(pretty, data, true) + require.NoError(t, err) + require.FileExists(t, pretty) + var testDecode test + err = DecodeFromFile(pretty, &testDecode) + require.Equal(t, data, testDecode) + + // Check the pretty printing + bytes, err := ioutil.ReadFile(pretty) + require.NoError(t, err) + require.Contains(t, string(bytes), " \"one\": \"one\",\n") + } + + { + small := path.Join(tempdir, "small.json") + err := EncodeToFile(small, data, false) + require.NoError(t, err) + require.FileExists(t, small) + var testDecode test + err = DecodeFromFile(small, &testDecode) + require.Equal(t, data, testDecode) + } + + // gzip test + { + small := path.Join(tempdir, "small.json.gz") + err := EncodeToFile(small, data, false) + require.NoError(t, err) + require.FileExists(t, small) + var testDecode test + err = DecodeFromFile(small, &testDecode) + require.Equal(t, data, testDecode) + } +} + func TestPrintableUTF8OrEmpty(t *testing.T) { encodeArg := func(str string) string { return base64.StdEncoding.EncodeToString([]byte("input"))