diff --git a/cmd/conduit/internal/initialize/init.go b/cmd/conduit/internal/initialize/init.go index 79677a8b..b7575dc9 100644 --- a/cmd/conduit/internal/initialize/init.go +++ b/cmd/conduit/internal/initialize/init.go @@ -11,6 +11,8 @@ import ( "github.com/algorand/conduit/conduit" "github.com/algorand/conduit/conduit/pipeline" + "github.com/algorand/conduit/conduit/plugins/exporters/filewriter" + algodimporter "github.com/algorand/indexer/conduit/plugins/importers/algod" ) // InitCommand is the init subcommand. @@ -60,7 +62,7 @@ func runConduitInit(path string, importerFlag string, processorsFlag []string, e var importer string if importerFlag == "" { - importerFlag = "algod" + importerFlag = algodimporter.PluginName } for _, metadata := range pipeline.ImporterMetadata() { if metadata.Name == importerFlag { @@ -74,7 +76,7 @@ func runConduitInit(path string, importerFlag string, processorsFlag []string, e var exporter string if exporterFlag == "" { - exporterFlag = "file_writer" + exporterFlag = filewriter.PluginName } for _, metadata := range pipeline.ExporterMetadata() { if metadata.Name == exporterFlag { diff --git a/cmd/conduit/internal/initialize/init_test.go b/cmd/conduit/internal/initialize/init_test.go index 2f7bab1f..2f5f0da2 100644 --- a/cmd/conduit/internal/initialize/init_test.go +++ b/cmd/conduit/internal/initialize/init_test.go @@ -40,16 +40,19 @@ func TestInitDataDirectory(t *testing.T) { // Defaults dataDirectory := t.TempDir() - runConduitInit(dataDirectory, "", []string{}, "") + err := runConduitInit(dataDirectory, "", []string{}, "") + require.NoError(t, err) verifyFile(fmt.Sprintf("%s/conduit.yml", dataDirectory), algodimporter.PluginName, filewriter.PluginName, nil) // Explicit defaults dataDirectory = t.TempDir() - runConduitInit(dataDirectory, algodimporter.PluginName, []string{noopProcessor.PluginName}, filewriter.PluginName) + err = runConduitInit(dataDirectory, algodimporter.PluginName, []string{noopProcessor.PluginName}, filewriter.PluginName) + require.NoError(t, err) verifyFile(fmt.Sprintf("%s/conduit.yml", dataDirectory), algodimporter.PluginName, filewriter.PluginName, []string{noopProcessor.PluginName}) // Different dataDirectory = t.TempDir() - runConduitInit(dataDirectory, fileimporter.PluginName, []string{noopProcessor.PluginName, filterprocessor.PluginName}, noopExporter.PluginName) + err = runConduitInit(dataDirectory, fileimporter.PluginName, []string{noopProcessor.PluginName, filterprocessor.PluginName}, noopExporter.PluginName) + require.NoError(t, err) verifyFile(fmt.Sprintf("%s/conduit.yml", dataDirectory), fileimporter.PluginName, noopExporter.PluginName, []string{noopProcessor.PluginName, filterprocessor.PluginName}) } diff --git a/cmd/conduit/internal/list/list.go b/cmd/conduit/internal/list/list.go index a4ea3eaf..0a5e1237 100644 --- a/cmd/conduit/internal/list/list.go +++ b/cmd/conduit/internal/list/list.go @@ -9,8 +9,8 @@ import ( "github.com/spf13/cobra" - "github.com/algorand/indexer/conduit" - "github.com/algorand/indexer/conduit/pipeline" + "github.com/algorand/conduit/conduit" + "github.com/algorand/conduit/conduit/pipeline" ) // Command is the list command to embed in a root cobra command. diff --git a/conduit/metrics/metrics.go b/conduit/metrics/metrics.go new file mode 100644 index 00000000..4b462122 --- /dev/null +++ b/conduit/metrics/metrics.go @@ -0,0 +1,143 @@ +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +// This is helpful for tests to ensure there are never uninitialized values. +func init() { + RegisterPrometheusMetrics("uninitialized") +} + +// RegisterPrometheusMetrics register all prometheus metrics with the global +// metrics handler. +func RegisterPrometheusMetrics(subsystem string) { + // deregister metric objects in case the register function was called more than once. + // This helps with testing. + deregister() + instantiateCollectors(subsystem) + + _ = prometheus.Register(BlockImportTimeSeconds) + _ = prometheus.Register(BlockImportTimeSeconds) + _ = prometheus.Register(ImportedTxnsPerBlock) + _ = prometheus.Register(ImportedRoundGauge) + _ = prometheus.Register(ImportedTxns) + _ = prometheus.Register(ImporterTimeSeconds) + _ = prometheus.Register(ProcessorTimeSeconds) + _ = prometheus.Register(ExporterTimeSeconds) + _ = prometheus.Register(PipelineRetryCount) +} +func deregister() { + // Use ImportedTxns as a sentinel value. None or all should be initialized. + if ImportedTxns != nil { + prometheus.Unregister(BlockImportTimeSeconds) + prometheus.Unregister(BlockImportTimeSeconds) + prometheus.Unregister(ImportedTxnsPerBlock) + prometheus.Unregister(ImportedRoundGauge) + prometheus.Unregister(ImportedTxns) + prometheus.Unregister(ImporterTimeSeconds) + prometheus.Unregister(ProcessorTimeSeconds) + prometheus.Unregister(ExporterTimeSeconds) + prometheus.Unregister(PipelineRetryCount) + } +} + +func instantiateCollectors(subsystem string) { + BlockImportTimeSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Subsystem: subsystem, + Name: BlockImportTimeName, + Help: "Total block upload and processing time in seconds.", + }) + + ImportedTxnsPerBlock = prometheus.NewSummary( + prometheus.SummaryOpts{ + Subsystem: subsystem, + Name: ImportedTxnsPerBlockName, + Help: "Transactions per block.", + }, + ) + + ImportedTxns = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: subsystem, + Name: ImportedTxnsName, + Help: "Imported transactions grouped by type", + }, + []string{"txn_type"}, + ) + + ImportedRoundGauge = prometheus.NewGauge( + prometheus.GaugeOpts{ + Subsystem: subsystem, + Name: ImportedRoundGaugeName, + Help: "The most recent round indexer has imported.", + }) + + ImporterTimeSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Subsystem: subsystem, + Name: ImporterTimeName, + Help: "Time spent at importer step", + }) + + ProcessorTimeSeconds = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Subsystem: subsystem, + Name: ProcessorTimeName, + Help: "Time spent running a processor", + }, + []string{"processor_name"}, + ) + + ExporterTimeSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Subsystem: subsystem, + Name: ExporterTimeName, + Help: "Time spent at exporter step", + }) + + PipelineRetryCount = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Subsystem: subsystem, + Name: PipelineRetryCountName, + Help: "Total pipeline retries since last successful run", + }) +} + +// Prometheus metric names broken out for reuse. +const ( + BlockImportTimeName = "import_time_sec" + ImportedTxnsPerBlockName = "imported_tx_per_block" + ImportedRoundGaugeName = "imported_round" + GetAlgodRawBlockTimeName = "get_algod_raw_block_time_sec" + ImportedTxnsName = "imported_txns" + ImporterTimeName = "importer_time_sec" + ProcessorTimeName = "processor_time_sec" + ExporterTimeName = "exporter_time_sec" + PipelineRetryCountName = "pipeline_retry_count" +) + +// AllMetricNames is a reference for all the custom metric names. +var AllMetricNames = []string{ + BlockImportTimeName, + ImportedTxnsPerBlockName, + ImportedRoundGaugeName, + GetAlgodRawBlockTimeName, + ImporterTimeName, + ProcessorTimeName, + ExporterTimeName, + PipelineRetryCountName, +} + +// Initialize the prometheus objects. +var ( + // used by pipeline + + BlockImportTimeSeconds prometheus.Summary + ImportedTxnsPerBlock prometheus.Summary + ImportedTxns *prometheus.GaugeVec + ImportedRoundGauge prometheus.Gauge + ImporterTimeSeconds prometheus.Summary + ProcessorTimeSeconds *prometheus.SummaryVec + ExporterTimeSeconds prometheus.Summary + PipelineRetryCount prometheus.Histogram +) diff --git a/conduit/pipeline/pipeline.go b/conduit/pipeline/pipeline.go index 2145de7f..e272bd80 100644 --- a/conduit/pipeline/pipeline.go +++ b/conduit/pipeline/pipeline.go @@ -20,10 +20,10 @@ import ( sdk "github.com/algorand/go-algorand-sdk/v2/types" "github.com/algorand/indexer/util" - "github.com/algorand/indexer/util/metrics" "github.com/algorand/conduit/conduit" "github.com/algorand/conduit/conduit/data" + "github.com/algorand/conduit/conduit/metrics" "github.com/algorand/conduit/conduit/plugins" "github.com/algorand/conduit/conduit/plugins/exporters" "github.com/algorand/conduit/conduit/plugins/importers" diff --git a/conduit/pipeline/pipeline_test.go b/conduit/pipeline/pipeline_test.go index 2c9ebdbe..2839389a 100644 --- a/conduit/pipeline/pipeline_test.go +++ b/conduit/pipeline/pipeline_test.go @@ -22,11 +22,11 @@ import ( "github.com/algorand/conduit/conduit" "github.com/algorand/conduit/conduit/data" + _ "github.com/algorand/conduit/conduit/metrics" "github.com/algorand/conduit/conduit/plugins" "github.com/algorand/conduit/conduit/plugins/exporters" "github.com/algorand/conduit/conduit/plugins/importers" "github.com/algorand/conduit/conduit/plugins/processors" - _ "github.com/algorand/indexer/util/metrics" ) // TestPipelineConfigValidity tests the Valid() function for the Config diff --git a/conduit/plugins/exporters/all/all.go b/conduit/plugins/exporters/all/all.go index 52cf359e..4a49b30e 100644 --- a/conduit/plugins/exporters/all/all.go +++ b/conduit/plugins/exporters/all/all.go @@ -2,7 +2,7 @@ package all import ( // Call package wide init function - _ "github.com/algorand/indexer/conduit/plugins/exporters/filewriter" - _ "github.com/algorand/indexer/conduit/plugins/exporters/noop" - _ "github.com/algorand/indexer/conduit/plugins/exporters/postgresql" + _ "github.com/algorand/conduit/conduit/plugins/exporters/filewriter" + _ "github.com/algorand/conduit/conduit/plugins/exporters/noop" + _ "github.com/algorand/conduit/conduit/plugins/exporters/postgresql" ) diff --git a/conduit/plugins/exporters/example/example_exporter.go b/conduit/plugins/exporters/example/example_exporter.go index 25e93187..b1b0b056 100644 --- a/conduit/plugins/exporters/example/example_exporter.go +++ b/conduit/plugins/exporters/example/example_exporter.go @@ -6,10 +6,10 @@ import ( "github.com/sirupsen/logrus" - "github.com/algorand/indexer/conduit" - "github.com/algorand/indexer/conduit/data" - "github.com/algorand/indexer/conduit/plugins" - "github.com/algorand/indexer/conduit/plugins/exporters" + "github.com/algorand/conduit/conduit" + "github.com/algorand/conduit/conduit/data" + "github.com/algorand/conduit/conduit/plugins" + "github.com/algorand/conduit/conduit/plugins/exporters" ) // This is our exporter object. It should store all the in memory data required to run the Exporter. diff --git a/conduit/plugins/exporters/example/example_exporter_test.go b/conduit/plugins/exporters/example/example_exporter_test.go index 37e94fee..6e51a9bf 100644 --- a/conduit/plugins/exporters/example/example_exporter_test.go +++ b/conduit/plugins/exporters/example/example_exporter_test.go @@ -6,9 +6,9 @@ import ( "github.com/stretchr/testify/assert" - "github.com/algorand/indexer/conduit/data" - "github.com/algorand/indexer/conduit/plugins" - "github.com/algorand/indexer/conduit/plugins/exporters" + "github.com/algorand/conduit/conduit/data" + "github.com/algorand/conduit/conduit/plugins" + "github.com/algorand/conduit/conduit/plugins/exporters" ) var exCons = exporters.ExporterConstructorFunc(func() exporters.Exporter { diff --git a/conduit/plugins/exporters/exporter_factory_test.go b/conduit/plugins/exporters/exporter_factory_test.go index fbb0806f..f16de4ab 100644 --- a/conduit/plugins/exporters/exporter_factory_test.go +++ b/conduit/plugins/exporters/exporter_factory_test.go @@ -7,7 +7,7 @@ import ( "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" - "github.com/algorand/indexer/conduit" + "github.com/algorand/conduit/conduit" ) var logger *logrus.Logger diff --git a/conduit/plugins/exporters/filewriter/file_exporter.go b/conduit/plugins/exporters/filewriter/file_exporter.go index cfb2c869..f08393d0 100644 --- a/conduit/plugins/exporters/filewriter/file_exporter.go +++ b/conduit/plugins/exporters/filewriter/file_exporter.go @@ -11,10 +11,10 @@ import ( "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" - "github.com/algorand/indexer/conduit" - "github.com/algorand/indexer/conduit/data" - "github.com/algorand/indexer/conduit/plugins" - "github.com/algorand/indexer/conduit/plugins/exporters" + "github.com/algorand/conduit/conduit" + "github.com/algorand/conduit/conduit/data" + "github.com/algorand/conduit/conduit/plugins" + "github.com/algorand/conduit/conduit/plugins/exporters" ) const ( diff --git a/conduit/plugins/exporters/filewriter/file_exporter_test.go b/conduit/plugins/exporters/filewriter/file_exporter_test.go index 81084484..75ffbc6a 100644 --- a/conduit/plugins/exporters/filewriter/file_exporter_test.go +++ b/conduit/plugins/exporters/filewriter/file_exporter_test.go @@ -13,10 +13,10 @@ import ( "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" - "github.com/algorand/indexer/conduit/data" - "github.com/algorand/indexer/conduit/plugins" - "github.com/algorand/indexer/conduit/plugins/exporters" - "github.com/algorand/indexer/conduit/plugins/tools/testutil" + "github.com/algorand/conduit/conduit/data" + "github.com/algorand/conduit/conduit/plugins" + "github.com/algorand/conduit/conduit/plugins/exporters" + "github.com/algorand/conduit/conduit/plugins/tools/testutil" sdk "github.com/algorand/go-algorand-sdk/v2/types" ) diff --git a/conduit/plugins/exporters/noop/noop_exporter.go b/conduit/plugins/exporters/noop/noop_exporter.go index 82fe11be..12be5350 100644 --- a/conduit/plugins/exporters/noop/noop_exporter.go +++ b/conduit/plugins/exporters/noop/noop_exporter.go @@ -8,10 +8,10 @@ import ( "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" - "github.com/algorand/indexer/conduit" - "github.com/algorand/indexer/conduit/data" - "github.com/algorand/indexer/conduit/plugins" - "github.com/algorand/indexer/conduit/plugins/exporters" + "github.com/algorand/conduit/conduit" + "github.com/algorand/conduit/conduit/data" + "github.com/algorand/conduit/conduit/plugins" + "github.com/algorand/conduit/conduit/plugins/exporters" ) // PluginName to use when configuring. diff --git a/conduit/plugins/exporters/noop/noop_exporter_test.go b/conduit/plugins/exporters/noop/noop_exporter_test.go index 04301748..42363709 100644 --- a/conduit/plugins/exporters/noop/noop_exporter_test.go +++ b/conduit/plugins/exporters/noop/noop_exporter_test.go @@ -7,10 +7,10 @@ import ( "github.com/stretchr/testify/assert" "gopkg.in/yaml.v3" - "github.com/algorand/indexer/conduit/data" - "github.com/algorand/indexer/conduit/plugins" - "github.com/algorand/indexer/conduit/plugins/exporters" - "github.com/algorand/indexer/conduit/plugins/tools/testutil" + "github.com/algorand/conduit/conduit/data" + "github.com/algorand/conduit/conduit/plugins" + "github.com/algorand/conduit/conduit/plugins/exporters" + "github.com/algorand/conduit/conduit/plugins/tools/testutil" sdk "github.com/algorand/go-algorand-sdk/v2/types" ) diff --git a/conduit/plugins/exporters/postgresql/postgresql_exporter.go b/conduit/plugins/exporters/postgresql/postgresql_exporter.go index 44761ed9..72724369 100644 --- a/conduit/plugins/exporters/postgresql/postgresql_exporter.go +++ b/conduit/plugins/exporters/postgresql/postgresql_exporter.go @@ -10,13 +10,14 @@ import ( "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" - "github.com/algorand/indexer/conduit" - "github.com/algorand/indexer/conduit/data" - "github.com/algorand/indexer/conduit/plugins" - "github.com/algorand/indexer/conduit/plugins/exporters" - "github.com/algorand/indexer/conduit/plugins/exporters/postgresql/util" - "github.com/algorand/indexer/idb" + "github.com/algorand/conduit/conduit" + "github.com/algorand/conduit/conduit/data" + "github.com/algorand/conduit/conduit/plugins" + "github.com/algorand/conduit/conduit/plugins/exporters" + "github.com/algorand/conduit/conduit/plugins/exporters/postgresql/util" + // Necessary to ensure the postgres implementation has been registered in the idb factory + "github.com/algorand/indexer/idb" _ "github.com/algorand/indexer/idb/postgres" "github.com/algorand/indexer/types" iutil "github.com/algorand/indexer/util" diff --git a/conduit/plugins/exporters/postgresql/postgresql_exporter_config.go b/conduit/plugins/exporters/postgresql/postgresql_exporter_config.go index 0d178157..2f632da4 100644 --- a/conduit/plugins/exporters/postgresql/postgresql_exporter_config.go +++ b/conduit/plugins/exporters/postgresql/postgresql_exporter_config.go @@ -3,7 +3,7 @@ package postgresql //go:generate go run ../../../../cmd/conduit-docs/main.go ../../../../conduit-docs/ import ( - "github.com/algorand/indexer/conduit/plugins/exporters/postgresql/util" + "github.com/algorand/conduit/conduit/plugins/exporters/postgresql/util" ) //Name: conduit_exporters_postgresql diff --git a/conduit/plugins/exporters/postgresql/postgresql_exporter_test.go b/conduit/plugins/exporters/postgresql/postgresql_exporter_test.go index cfcdf6b3..f429293e 100644 --- a/conduit/plugins/exporters/postgresql/postgresql_exporter_test.go +++ b/conduit/plugins/exporters/postgresql/postgresql_exporter_test.go @@ -11,11 +11,11 @@ import ( "github.com/stretchr/testify/assert" "gopkg.in/yaml.v3" - "github.com/algorand/indexer/conduit/data" - "github.com/algorand/indexer/conduit/plugins" - "github.com/algorand/indexer/conduit/plugins/exporters" - "github.com/algorand/indexer/conduit/plugins/exporters/postgresql/util" - "github.com/algorand/indexer/conduit/plugins/tools/testutil" + "github.com/algorand/conduit/conduit/data" + "github.com/algorand/conduit/conduit/plugins" + "github.com/algorand/conduit/conduit/plugins/exporters" + "github.com/algorand/conduit/conduit/plugins/exporters/postgresql/util" + "github.com/algorand/conduit/conduit/plugins/tools/testutil" _ "github.com/algorand/indexer/idb/dummy" )