From 5bca3f47d39c75cb8d9a254d669e4a01a04edc86 Mon Sep 17 00:00:00 2001 From: Eric Warehime Date: Wed, 15 Jun 2022 14:12:24 -0700 Subject: [PATCH 1/8] Lock to mockery minor version --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 27e281335..033df9807 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ idb/postgres/internal/schema/setup_postgres_sql.go: idb/postgres/internal/schema cd idb/postgres/internal/schema && go generate idb/mocks/IndexerDb.go: idb/idb.go - go install github.com/vektra/mockery/v2@latest + go install github.com/vektra/mockery/v2@v2.12.3 cd idb && mockery --name=IndexerDb # check that all packages (except tests) compile From aede594f47020cfe627208b8fd616b1d15ab9967 Mon Sep 17 00:00:00 2001 From: Eric Warehime Date: Tue, 28 Jun 2022 09:12:33 -0700 Subject: [PATCH 2/8] Add Exporter interface --- exporters/exporter.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 exporters/exporter.go diff --git a/exporters/exporter.go b/exporters/exporter.go new file mode 100644 index 000000000..c15f74047 --- /dev/null +++ b/exporters/exporter.go @@ -0,0 +1,27 @@ +package exporters + +import "github.com/algorand/go-algorand/rpcs" + +// ExporterConfig is a generic type providing serialization/deserialization for exporter config files +type ExporterConfig interface{} + +// Exporter defines the interface for plugins +type Exporter interface { + // Connect will be called during initialization, before block data starts going through the pipeline. + // Typically used for things like initializing network connections. + // The ExporterConfig passed to Connect will contain the Unmarhsalled config file specific to this plugin. + // Should return an error if it fails--this will result in the Indexer process terminating. + Connect(cfg ExporterConfig) error + + // Shutdown will be called during termination of the Indexer process. + // There is no guarantee that plugin lifecycle hooks will be invoked in any specific order in relation to one another. + // Returns an error if it fails which will be surfaced in the logs, but the process is already terminating. + Shutdown() error + + // Receive is called for each block to be processed by the exporter. + // Should return an error on failure--retries are configurable. + Receive(blockData *rpcs.EncodedBlockCert) error + + // Round returns the next round not yet processed by the Exporter. Atomically updated when Receive successfully completes. + Round() uint64 +} From b758ca7ddc54cd5b59a27f586f483f1e10e4bd1f Mon Sep 17 00:00:00 2001 From: Eric Warehime Date: Tue, 28 Jun 2022 09:15:13 -0700 Subject: [PATCH 3/8] Rename Shutdown method --- exporters/exporter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exporters/exporter.go b/exporters/exporter.go index c15f74047..5cac58bb5 100644 --- a/exporters/exporter.go +++ b/exporters/exporter.go @@ -13,10 +13,10 @@ type Exporter interface { // Should return an error if it fails--this will result in the Indexer process terminating. Connect(cfg ExporterConfig) error - // Shutdown will be called during termination of the Indexer process. + // Disconnect will be called during termination of the Indexer process. // There is no guarantee that plugin lifecycle hooks will be invoked in any specific order in relation to one another. // Returns an error if it fails which will be surfaced in the logs, but the process is already terminating. - Shutdown() error + Disconnect() error // Receive is called for each block to be processed by the exporter. // Should return an error on failure--retries are configurable. From 5210f78fca331fbb645a348fadfa724c0fec5bb8 Mon Sep 17 00:00:00 2001 From: Eric Warehime Date: Tue, 5 Jul 2022 15:19:21 -0700 Subject: [PATCH 4/8] update exporter interface --- exporters/exporter.go | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/exporters/exporter.go b/exporters/exporter.go index 5cac58bb5..7ee9f9b42 100644 --- a/exporters/exporter.go +++ b/exporters/exporter.go @@ -1,18 +1,48 @@ package exporters -import "github.com/algorand/go-algorand/rpcs" +import ( + "github.com/algorand/go-algorand/agreement" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/ledger/ledgercore" +) -// ExporterConfig is a generic type providing serialization/deserialization for exporter config files -type ExporterConfig interface{} +// ExporterConfig is a generic string which can be deserialized by each individual Exporter. +type ExporterConfig string +type ExportData interface { + Round() uint64 +} + +// BlockExportData is provided to the Exporter on each round. +type BlockExportData struct { + // Block is the block data written to the blockchain. + Block bookkeeping.Block + + // Delta contains a list of account changes resulting from the block. Processor plugins may have modify this data. + Delta ledgercore.StateDelta + + // Certificate contains voting data that certifies the block. The certificate is non deterministic, a node stops collecting votes once the voting threshold is reached. + Certificate agreement.Certificate +} + +func (blkData *BlockExportData) Round() uint64 { + return uint64(blkData.Block.Round()) +} // Exporter defines the interface for plugins type Exporter interface { + // Name is a UID for each Exporter. + Name() string + // Connect will be called during initialization, before block data starts going through the pipeline. // Typically used for things like initializing network connections. // The ExporterConfig passed to Connect will contain the Unmarhsalled config file specific to this plugin. // Should return an error if it fails--this will result in the Indexer process terminating. Connect(cfg ExporterConfig) error + // Config returns the configuration options used to create an Exporter. + // Initialized during Connect, it should return nil until the Exporter has been Connected. + Config() ExporterConfig + // Disconnect will be called during termination of the Indexer process. // There is no guarantee that plugin lifecycle hooks will be invoked in any specific order in relation to one another. // Returns an error if it fails which will be surfaced in the logs, but the process is already terminating. @@ -20,7 +50,12 @@ type Exporter interface { // Receive is called for each block to be processed by the exporter. // Should return an error on failure--retries are configurable. - Receive(blockData *rpcs.EncodedBlockCert) error + Receive(exportData ExportData) error + + // HandleGenesis is an Exporter's opportunity to do initial validation and handling of the Genesis block. + // If validation (such as a check to ensure `genesis` matches a previously stored genesis block) or handling fails, + // it returns an error. + HandleGenesis(genesis bookkeeping.Genesis) error // Round returns the next round not yet processed by the Exporter. Atomically updated when Receive successfully completes. Round() uint64 From a65a7b320f7f2208f4c8b42a19691e8a98fa383e Mon Sep 17 00:00:00 2001 From: Eric Warehime Date: Tue, 5 Jul 2022 15:24:59 -0700 Subject: [PATCH 5/8] add comments --- exporters/exporter.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/exporters/exporter.go b/exporters/exporter.go index 7ee9f9b42..7c40b6340 100644 --- a/exporters/exporter.go +++ b/exporters/exporter.go @@ -8,6 +8,8 @@ import ( // ExporterConfig is a generic string which can be deserialized by each individual Exporter. type ExporterConfig string + +// ExportData is the interface which all data types sent to Exporters should implement type ExportData interface { Round() uint64 } @@ -24,6 +26,7 @@ type BlockExportData struct { Certificate agreement.Certificate } +// Round returns the round to which the BlockExportData corresponds func (blkData *BlockExportData) Round() uint64 { return uint64(blkData.Block.Round()) } From 437e6916b66b2e08a8e3fd6cc3a809b2ec1f5c01 Mon Sep 17 00:00:00 2001 From: Eric Warehime Date: Wed, 6 Jul 2022 11:27:29 -0700 Subject: [PATCH 6/8] add noop exporter --- exporters/noop/noop_exporter.go | 64 ++++++++++++++++++++++++++++ exporters/noop/noop_exporter_test.go | 48 +++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 exporters/noop/noop_exporter.go create mode 100644 exporters/noop/noop_exporter_test.go diff --git a/exporters/noop/noop_exporter.go b/exporters/noop/noop_exporter.go new file mode 100644 index 000000000..05180e4c6 --- /dev/null +++ b/exporters/noop/noop_exporter.go @@ -0,0 +1,64 @@ +package noop + +import ( + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/indexer/exporters" +) + +// `noopExporter`s will function without ever erroring. This means they will also process out of order blocks +// which may or may not be desirable for different use cases--it can hide errors in actual exporters expecting in order +// block processing. +// The `noopExporter` will maintain `Round` state according to the round of the last block it processed. +type noopExporter struct { + round uint64 + cfg exporters.ExporterConfig +} + +var NoopExporterMetadata exporters.ExporterMetadata = exporters.ExporterMetadata{ + Name: "noop", + Description: "noop exporter", + Deprecated: false, +} + +// NoopConstructor is the ExporterConstructor implementation for the "noop" exporter +type NoopConstructor struct{} + +func (c *NoopConstructor) New() exporters.Exporter { + return &noopExporter{ + round: 0, + cfg: "", + } +} + +func (exp *noopExporter) Metadata() exporters.ExporterMetadata { + return NoopExporterMetadata +} + +func (exp *noopExporter) Connect(_ exporters.ExporterConfig) error { + return nil +} + +func (exp *noopExporter) Config() exporters.ExporterConfig { + return exp.cfg +} + +func (exp *noopExporter) Disconnect() error { + return nil +} + +func (exp *noopExporter) Receive(exportData exporters.ExportData) error { + exp.round = exportData.Round() + 1 + return nil +} + +func (exp *noopExporter) HandleGenesis(_ bookkeeping.Genesis) error { + return nil +} + +func (exp *noopExporter) Round() uint64 { + return exp.round +} + +func init() { + exporters.RegisterExporter(NoopExporterMetadata.Name, &NoopConstructor{}) +} diff --git a/exporters/noop/noop_exporter_test.go b/exporters/noop/noop_exporter_test.go new file mode 100644 index 000000000..c788472a4 --- /dev/null +++ b/exporters/noop/noop_exporter_test.go @@ -0,0 +1,48 @@ +package noop + +import ( + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/indexer/exporters" + "github.com/stretchr/testify/assert" + "testing" +) + +var nc = &NoopConstructor{} + +var ne = nc.New() + +func TestExporterMetadata(t *testing.T) { + meta := ne.Metadata() + assert.Equal(t, NoopExporterMetadata.Name, meta.Name) + assert.Equal(t, NoopExporterMetadata.Description, meta.Description) + assert.Equal(t, NoopExporterMetadata.Deprecated, meta.Deprecated) +} + +func TestExporterConnect(t *testing.T) { + assert.NoError(t, ne.Connect("")) +} + +func TestExporterConfig(t *testing.T) { + assert.Equal(t, exporters.ExporterConfig(""), ne.Config()) +} + +func TestExporterDisconnect(t *testing.T) { + assert.NoError(t, ne.Disconnect()) +} + +func TestExporterHandleGenesis(t *testing.T) { + assert.NoError(t, ne.HandleGenesis(bookkeeping.Genesis{})) +} + +func TestExporterRoundReceive(t *testing.T) { + eData := &exporters.BlockExportData{ + Block: bookkeeping.Block{ + BlockHeader: bookkeeping.BlockHeader{ + Round: 5, + }, + }, + } + assert.Equal(t, uint64(0), ne.Round()) + assert.NoError(t, ne.Receive(eData)) + assert.Equal(t, uint64(6), ne.Round()) +} From 95b01935f41fe9831dadc2fece0347a4cafec3a1 Mon Sep 17 00:00:00 2001 From: Eric Warehime Date: Wed, 6 Jul 2022 11:44:28 -0700 Subject: [PATCH 7/8] update exporters module --- exporters/exporter.go | 11 ++++++-- exporters/exporter_factory.go | 40 ++++++++++++++++++++++++++++ exporters/noop/noop_exporter.go | 13 ++++----- exporters/noop/noop_exporter_test.go | 8 +++--- 4 files changed, 60 insertions(+), 12 deletions(-) create mode 100644 exporters/exporter_factory.go diff --git a/exporters/exporter.go b/exporters/exporter.go index 7c40b6340..ad2a837e3 100644 --- a/exporters/exporter.go +++ b/exporters/exporter.go @@ -31,10 +31,17 @@ func (blkData *BlockExportData) Round() uint64 { return uint64(blkData.Block.Round()) } +// ExporterMetadata contains fields relevant to identification and description of plugins. +type ExporterMetadata struct { + Name string + Description string + Deprecated bool +} + // Exporter defines the interface for plugins type Exporter interface { - // Name is a UID for each Exporter. - Name() string + // Metadata associated with each Exporter. + Metadata() ExporterMetadata // Connect will be called during initialization, before block data starts going through the pipeline. // Typically used for things like initializing network connections. diff --git a/exporters/exporter_factory.go b/exporters/exporter_factory.go new file mode 100644 index 000000000..5c29b511f --- /dev/null +++ b/exporters/exporter_factory.go @@ -0,0 +1,40 @@ +package exporters + +import ( + "fmt" +) + +// ExporterConstructor must be implemented by each Exporter. +// It provides a basic no-arg constructor for instances of an ExporterImpl. +type ExporterConstructor interface { + // New should return an instantiation of an Exporter. + // Configuration values should be passed and can be processed during `Connect()`. + New() Exporter +} + +// exporterImpls is a k/v store from exporter names to their constructor implementations. +// This layer of indirection allows for different exporter integrations to be compiled in or compiled out by `go build --tags ...` +var exporterImpls = make(map[string]ExporterConstructor) + +// RegisterExporter is used to register ExporterConstructor implementations. This mechanism allows +// for loose coupling between the configuration and the implementation. It is extremely similar to the way sql.DB +// driver's are configured and used. +func RegisterExporter(name string, constructor ExporterConstructor) { + exporterImpls[name] = constructor +} + +// ExporterByName is used to construct an Exporter object by name. +// Returns an Exporter object, an availability channel that closes when the database +// becomes available, and an error object. +func ExporterByName(name string, cfg ExporterConfig) (Exporter, error) { + var constructor ExporterConstructor + var ok bool + if constructor, ok = exporterImpls[name]; !ok { + return nil, fmt.Errorf("no Exporter Constructor for %s", name) + } + val := constructor.New() + if err := val.Connect(cfg); err != nil { + return nil, err + } + return val, nil +} diff --git a/exporters/noop/noop_exporter.go b/exporters/noop/noop_exporter.go index 05180e4c6..651992ce1 100644 --- a/exporters/noop/noop_exporter.go +++ b/exporters/noop/noop_exporter.go @@ -14,16 +14,17 @@ type noopExporter struct { cfg exporters.ExporterConfig } -var NoopExporterMetadata exporters.ExporterMetadata = exporters.ExporterMetadata{ +var noopExporterMetadata exporters.ExporterMetadata = exporters.ExporterMetadata{ Name: "noop", Description: "noop exporter", Deprecated: false, } -// NoopConstructor is the ExporterConstructor implementation for the "noop" exporter -type NoopConstructor struct{} +// Constructor is the ExporterConstructor implementation for the "noop" exporter +type Constructor struct{} -func (c *NoopConstructor) New() exporters.Exporter { +// New initializes a noopExporter +func (c *Constructor) New() exporters.Exporter { return &noopExporter{ round: 0, cfg: "", @@ -31,7 +32,7 @@ func (c *NoopConstructor) New() exporters.Exporter { } func (exp *noopExporter) Metadata() exporters.ExporterMetadata { - return NoopExporterMetadata + return noopExporterMetadata } func (exp *noopExporter) Connect(_ exporters.ExporterConfig) error { @@ -60,5 +61,5 @@ func (exp *noopExporter) Round() uint64 { } func init() { - exporters.RegisterExporter(NoopExporterMetadata.Name, &NoopConstructor{}) + exporters.RegisterExporter(noopExporterMetadata.Name, &Constructor{}) } diff --git a/exporters/noop/noop_exporter_test.go b/exporters/noop/noop_exporter_test.go index c788472a4..7dceb1126 100644 --- a/exporters/noop/noop_exporter_test.go +++ b/exporters/noop/noop_exporter_test.go @@ -7,15 +7,15 @@ import ( "testing" ) -var nc = &NoopConstructor{} +var nc = &Constructor{} var ne = nc.New() func TestExporterMetadata(t *testing.T) { meta := ne.Metadata() - assert.Equal(t, NoopExporterMetadata.Name, meta.Name) - assert.Equal(t, NoopExporterMetadata.Description, meta.Description) - assert.Equal(t, NoopExporterMetadata.Deprecated, meta.Deprecated) + assert.Equal(t, noopExporterMetadata.Name, meta.Name) + assert.Equal(t, noopExporterMetadata.Description, meta.Description) + assert.Equal(t, noopExporterMetadata.Deprecated, meta.Deprecated) } func TestExporterConnect(t *testing.T) { From b601d9ab33f88e93ca4b9e14b5d3ee703ba31728 Mon Sep 17 00:00:00 2001 From: Eric Warehime Date: Wed, 6 Jul 2022 13:31:35 -0700 Subject: [PATCH 8/8] Add exporter_factory tests --- exporters/exporter_factory_test.go | 51 ++++++++++++++++++++++++++++ exporters/noop/noop_exporter_test.go | 7 ++++ 2 files changed, 58 insertions(+) create mode 100644 exporters/exporter_factory_test.go diff --git a/exporters/exporter_factory_test.go b/exporters/exporter_factory_test.go new file mode 100644 index 000000000..0bf71b8ba --- /dev/null +++ b/exporters/exporter_factory_test.go @@ -0,0 +1,51 @@ +package exporters + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "testing" +) + +type mockExporter struct { + mock.Mock + Exporter +} + +func (m *mockExporter) Connect(config ExporterConfig) error { + args := m.Called(config) + return args.Error(0) +} + +type mockExporterConstructor struct { + me *mockExporter +} + +func (c *mockExporterConstructor) New() Exporter { + return c.me +} + +func TestExporterByNameSuccess(t *testing.T) { + me := mockExporter{} + me.On("Connect", mock.Anything).Return(nil) + RegisterExporter("foobar", &mockExporterConstructor{&me}) + + exp, err := ExporterByName("foobar", "") + assert.NoError(t, err) + assert.Implements(t, (*Exporter)(nil), exp) +} + +func TestExporterByNameNotFound(t *testing.T) { + _, err := ExporterByName("barfoo", "") + expectedErr := "no Exporter Constructor for barfoo" + assert.EqualError(t, err, expectedErr) +} + +func TestExporterByNameConnectFailure(t *testing.T) { + me := mockExporter{} + expectedErr := fmt.Errorf("connect failure") + me.On("Connect", mock.Anything).Return(expectedErr) + RegisterExporter("baz", &mockExporterConstructor{&me}) + _, err := ExporterByName("baz", "") + assert.EqualError(t, err, expectedErr.Error()) +} diff --git a/exporters/noop/noop_exporter_test.go b/exporters/noop/noop_exporter_test.go index 7dceb1126..438189a84 100644 --- a/exporters/noop/noop_exporter_test.go +++ b/exporters/noop/noop_exporter_test.go @@ -11,6 +11,13 @@ var nc = &Constructor{} var ne = nc.New() +func TestExporterByName(t *testing.T) { + exporters.RegisterExporter(noopExporterMetadata.Name, nc) + ne, err := exporters.ExporterByName(noopExporterMetadata.Name, "") + assert.NoError(t, err) + assert.Implements(t, (*exporters.Exporter)(nil), ne) +} + func TestExporterMetadata(t *testing.T) { meta := ne.Metadata() assert.Equal(t, noopExporterMetadata.Name, meta.Name)