Skip to content

Commit

Permalink
tests: Get them pasing. (#1)
Browse files Browse the repository at this point in the history
Fix imports which weren't updated.
Bring in metrics package.
Minor code cleanup noticed while fixing specific tests.
  • Loading branch information
winder authored Mar 9, 2023
1 parent c22990e commit 23eabcc
Show file tree
Hide file tree
Showing 17 changed files with 197 additions and 48 deletions.
6 changes: 4 additions & 2 deletions cmd/conduit/internal/initialize/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import (

"github.com/algorand/conduit/conduit"
"github.com/algorand/conduit/conduit/pipeline"
"github.com/algorand/conduit/conduit/plugins/exporters/filewriter"
algodimporter "github.com/algorand/indexer/conduit/plugins/importers/algod"
)

// InitCommand is the init subcommand.
Expand Down Expand Up @@ -60,7 +62,7 @@ func runConduitInit(path string, importerFlag string, processorsFlag []string, e

var importer string
if importerFlag == "" {
importerFlag = "algod"
importerFlag = algodimporter.PluginName
}
for _, metadata := range pipeline.ImporterMetadata() {
if metadata.Name == importerFlag {
Expand All @@ -74,7 +76,7 @@ func runConduitInit(path string, importerFlag string, processorsFlag []string, e

var exporter string
if exporterFlag == "" {
exporterFlag = "file_writer"
exporterFlag = filewriter.PluginName
}
for _, metadata := range pipeline.ExporterMetadata() {
if metadata.Name == exporterFlag {
Expand Down
9 changes: 6 additions & 3 deletions cmd/conduit/internal/initialize/init_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,16 +40,19 @@ func TestInitDataDirectory(t *testing.T) {

// Defaults
dataDirectory := t.TempDir()
runConduitInit(dataDirectory, "", []string{}, "")
err := runConduitInit(dataDirectory, "", []string{}, "")
require.NoError(t, err)
verifyFile(fmt.Sprintf("%s/conduit.yml", dataDirectory), algodimporter.PluginName, filewriter.PluginName, nil)

// Explicit defaults
dataDirectory = t.TempDir()
runConduitInit(dataDirectory, algodimporter.PluginName, []string{noopProcessor.PluginName}, filewriter.PluginName)
err = runConduitInit(dataDirectory, algodimporter.PluginName, []string{noopProcessor.PluginName}, filewriter.PluginName)
require.NoError(t, err)
verifyFile(fmt.Sprintf("%s/conduit.yml", dataDirectory), algodimporter.PluginName, filewriter.PluginName, []string{noopProcessor.PluginName})

// Different
dataDirectory = t.TempDir()
runConduitInit(dataDirectory, fileimporter.PluginName, []string{noopProcessor.PluginName, filterprocessor.PluginName}, noopExporter.PluginName)
err = runConduitInit(dataDirectory, fileimporter.PluginName, []string{noopProcessor.PluginName, filterprocessor.PluginName}, noopExporter.PluginName)
require.NoError(t, err)
verifyFile(fmt.Sprintf("%s/conduit.yml", dataDirectory), fileimporter.PluginName, noopExporter.PluginName, []string{noopProcessor.PluginName, filterprocessor.PluginName})
}
4 changes: 2 additions & 2 deletions cmd/conduit/internal/list/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ import (

"github.com/spf13/cobra"

"github.com/algorand/indexer/conduit"
"github.com/algorand/indexer/conduit/pipeline"
"github.com/algorand/conduit/conduit"
"github.com/algorand/conduit/conduit/pipeline"
)

// Command is the list command to embed in a root cobra command.
Expand Down
143 changes: 143 additions & 0 deletions conduit/metrics/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
package metrics

import "github.com/prometheus/client_golang/prometheus"

// This is helpful for tests to ensure there are never uninitialized values.
func init() {
RegisterPrometheusMetrics("uninitialized")
}

// RegisterPrometheusMetrics register all prometheus metrics with the global
// metrics handler.
func RegisterPrometheusMetrics(subsystem string) {
// deregister metric objects in case the register function was called more than once.
// This helps with testing.
deregister()
instantiateCollectors(subsystem)

_ = prometheus.Register(BlockImportTimeSeconds)
_ = prometheus.Register(BlockImportTimeSeconds)
_ = prometheus.Register(ImportedTxnsPerBlock)
_ = prometheus.Register(ImportedRoundGauge)
_ = prometheus.Register(ImportedTxns)
_ = prometheus.Register(ImporterTimeSeconds)
_ = prometheus.Register(ProcessorTimeSeconds)
_ = prometheus.Register(ExporterTimeSeconds)
_ = prometheus.Register(PipelineRetryCount)
}
func deregister() {
// Use ImportedTxns as a sentinel value. None or all should be initialized.
if ImportedTxns != nil {
prometheus.Unregister(BlockImportTimeSeconds)
prometheus.Unregister(BlockImportTimeSeconds)
prometheus.Unregister(ImportedTxnsPerBlock)
prometheus.Unregister(ImportedRoundGauge)
prometheus.Unregister(ImportedTxns)
prometheus.Unregister(ImporterTimeSeconds)
prometheus.Unregister(ProcessorTimeSeconds)
prometheus.Unregister(ExporterTimeSeconds)
prometheus.Unregister(PipelineRetryCount)
}
}

func instantiateCollectors(subsystem string) {
BlockImportTimeSeconds = prometheus.NewSummary(
prometheus.SummaryOpts{
Subsystem: subsystem,
Name: BlockImportTimeName,
Help: "Total block upload and processing time in seconds.",
})

ImportedTxnsPerBlock = prometheus.NewSummary(
prometheus.SummaryOpts{
Subsystem: subsystem,
Name: ImportedTxnsPerBlockName,
Help: "Transactions per block.",
},
)

ImportedTxns = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Subsystem: subsystem,
Name: ImportedTxnsName,
Help: "Imported transactions grouped by type",
},
[]string{"txn_type"},
)

ImportedRoundGauge = prometheus.NewGauge(
prometheus.GaugeOpts{
Subsystem: subsystem,
Name: ImportedRoundGaugeName,
Help: "The most recent round indexer has imported.",
})

ImporterTimeSeconds = prometheus.NewSummary(
prometheus.SummaryOpts{
Subsystem: subsystem,
Name: ImporterTimeName,
Help: "Time spent at importer step",
})

ProcessorTimeSeconds = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Subsystem: subsystem,
Name: ProcessorTimeName,
Help: "Time spent running a processor",
},
[]string{"processor_name"},
)

ExporterTimeSeconds = prometheus.NewSummary(
prometheus.SummaryOpts{
Subsystem: subsystem,
Name: ExporterTimeName,
Help: "Time spent at exporter step",
})

PipelineRetryCount = prometheus.NewHistogram(
prometheus.HistogramOpts{
Subsystem: subsystem,
Name: PipelineRetryCountName,
Help: "Total pipeline retries since last successful run",
})
}

// Prometheus metric names broken out for reuse.
const (
BlockImportTimeName = "import_time_sec"
ImportedTxnsPerBlockName = "imported_tx_per_block"
ImportedRoundGaugeName = "imported_round"
GetAlgodRawBlockTimeName = "get_algod_raw_block_time_sec"
ImportedTxnsName = "imported_txns"
ImporterTimeName = "importer_time_sec"
ProcessorTimeName = "processor_time_sec"
ExporterTimeName = "exporter_time_sec"
PipelineRetryCountName = "pipeline_retry_count"
)

// AllMetricNames is a reference for all the custom metric names.
var AllMetricNames = []string{
BlockImportTimeName,
ImportedTxnsPerBlockName,
ImportedRoundGaugeName,
GetAlgodRawBlockTimeName,
ImporterTimeName,
ProcessorTimeName,
ExporterTimeName,
PipelineRetryCountName,
}

// Initialize the prometheus objects.
var (
// used by pipeline

BlockImportTimeSeconds prometheus.Summary
ImportedTxnsPerBlock prometheus.Summary
ImportedTxns *prometheus.GaugeVec
ImportedRoundGauge prometheus.Gauge
ImporterTimeSeconds prometheus.Summary
ProcessorTimeSeconds *prometheus.SummaryVec
ExporterTimeSeconds prometheus.Summary
PipelineRetryCount prometheus.Histogram
)
2 changes: 1 addition & 1 deletion conduit/pipeline/pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@ import (

sdk "github.com/algorand/go-algorand-sdk/v2/types"
"github.com/algorand/indexer/util"
"github.com/algorand/indexer/util/metrics"

"github.com/algorand/conduit/conduit"
"github.com/algorand/conduit/conduit/data"
"github.com/algorand/conduit/conduit/metrics"
"github.com/algorand/conduit/conduit/plugins"
"github.com/algorand/conduit/conduit/plugins/exporters"
"github.com/algorand/conduit/conduit/plugins/importers"
Expand Down
2 changes: 1 addition & 1 deletion conduit/pipeline/pipeline_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,11 @@ import (

"github.com/algorand/conduit/conduit"
"github.com/algorand/conduit/conduit/data"
_ "github.com/algorand/conduit/conduit/metrics"
"github.com/algorand/conduit/conduit/plugins"
"github.com/algorand/conduit/conduit/plugins/exporters"
"github.com/algorand/conduit/conduit/plugins/importers"
"github.com/algorand/conduit/conduit/plugins/processors"
_ "github.com/algorand/indexer/util/metrics"
)

// TestPipelineConfigValidity tests the Valid() function for the Config
Expand Down
6 changes: 3 additions & 3 deletions conduit/plugins/exporters/all/all.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package all

import (
// Call package wide init function
_ "github.com/algorand/indexer/conduit/plugins/exporters/filewriter"
_ "github.com/algorand/indexer/conduit/plugins/exporters/noop"
_ "github.com/algorand/indexer/conduit/plugins/exporters/postgresql"
_ "github.com/algorand/conduit/conduit/plugins/exporters/filewriter"
_ "github.com/algorand/conduit/conduit/plugins/exporters/noop"
_ "github.com/algorand/conduit/conduit/plugins/exporters/postgresql"
)
8 changes: 4 additions & 4 deletions conduit/plugins/exporters/example/example_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ import (

"github.com/sirupsen/logrus"

"github.com/algorand/indexer/conduit"
"github.com/algorand/indexer/conduit/data"
"github.com/algorand/indexer/conduit/plugins"
"github.com/algorand/indexer/conduit/plugins/exporters"
"github.com/algorand/conduit/conduit"
"github.com/algorand/conduit/conduit/data"
"github.com/algorand/conduit/conduit/plugins"
"github.com/algorand/conduit/conduit/plugins/exporters"
)

// This is our exporter object. It should store all the in memory data required to run the Exporter.
Expand Down
6 changes: 3 additions & 3 deletions conduit/plugins/exporters/example/example_exporter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ import (

"github.com/stretchr/testify/assert"

"github.com/algorand/indexer/conduit/data"
"github.com/algorand/indexer/conduit/plugins"
"github.com/algorand/indexer/conduit/plugins/exporters"
"github.com/algorand/conduit/conduit/data"
"github.com/algorand/conduit/conduit/plugins"
"github.com/algorand/conduit/conduit/plugins/exporters"
)

var exCons = exporters.ExporterConstructorFunc(func() exporters.Exporter {
Expand Down
2 changes: 1 addition & 1 deletion conduit/plugins/exporters/exporter_factory_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"

"github.com/algorand/indexer/conduit"
"github.com/algorand/conduit/conduit"
)

var logger *logrus.Logger
Expand Down
8 changes: 4 additions & 4 deletions conduit/plugins/exporters/filewriter/file_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@ import (
"github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"

"github.com/algorand/indexer/conduit"
"github.com/algorand/indexer/conduit/data"
"github.com/algorand/indexer/conduit/plugins"
"github.com/algorand/indexer/conduit/plugins/exporters"
"github.com/algorand/conduit/conduit"
"github.com/algorand/conduit/conduit/data"
"github.com/algorand/conduit/conduit/plugins"
"github.com/algorand/conduit/conduit/plugins/exporters"
)

const (
Expand Down
8 changes: 4 additions & 4 deletions conduit/plugins/exporters/filewriter/file_exporter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ import (
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"

"github.com/algorand/indexer/conduit/data"
"github.com/algorand/indexer/conduit/plugins"
"github.com/algorand/indexer/conduit/plugins/exporters"
"github.com/algorand/indexer/conduit/plugins/tools/testutil"
"github.com/algorand/conduit/conduit/data"
"github.com/algorand/conduit/conduit/plugins"
"github.com/algorand/conduit/conduit/plugins/exporters"
"github.com/algorand/conduit/conduit/plugins/tools/testutil"

sdk "github.com/algorand/go-algorand-sdk/v2/types"
)
Expand Down
8 changes: 4 additions & 4 deletions conduit/plugins/exporters/noop/noop_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@ import (
"github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"

"github.com/algorand/indexer/conduit"
"github.com/algorand/indexer/conduit/data"
"github.com/algorand/indexer/conduit/plugins"
"github.com/algorand/indexer/conduit/plugins/exporters"
"github.com/algorand/conduit/conduit"
"github.com/algorand/conduit/conduit/data"
"github.com/algorand/conduit/conduit/plugins"
"github.com/algorand/conduit/conduit/plugins/exporters"
)

// PluginName to use when configuring.
Expand Down
8 changes: 4 additions & 4 deletions conduit/plugins/exporters/noop/noop_exporter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ import (
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v3"

"github.com/algorand/indexer/conduit/data"
"github.com/algorand/indexer/conduit/plugins"
"github.com/algorand/indexer/conduit/plugins/exporters"
"github.com/algorand/indexer/conduit/plugins/tools/testutil"
"github.com/algorand/conduit/conduit/data"
"github.com/algorand/conduit/conduit/plugins"
"github.com/algorand/conduit/conduit/plugins/exporters"
"github.com/algorand/conduit/conduit/plugins/tools/testutil"

sdk "github.com/algorand/go-algorand-sdk/v2/types"
)
Expand Down
13 changes: 7 additions & 6 deletions conduit/plugins/exporters/postgresql/postgresql_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,14 @@ import (
"github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"

"github.com/algorand/indexer/conduit"
"github.com/algorand/indexer/conduit/data"
"github.com/algorand/indexer/conduit/plugins"
"github.com/algorand/indexer/conduit/plugins/exporters"
"github.com/algorand/indexer/conduit/plugins/exporters/postgresql/util"
"github.com/algorand/indexer/idb"
"github.com/algorand/conduit/conduit"
"github.com/algorand/conduit/conduit/data"
"github.com/algorand/conduit/conduit/plugins"
"github.com/algorand/conduit/conduit/plugins/exporters"
"github.com/algorand/conduit/conduit/plugins/exporters/postgresql/util"

// Necessary to ensure the postgres implementation has been registered in the idb factory
"github.com/algorand/indexer/idb"
_ "github.com/algorand/indexer/idb/postgres"
"github.com/algorand/indexer/types"
iutil "github.com/algorand/indexer/util"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package postgresql
//go:generate go run ../../../../cmd/conduit-docs/main.go ../../../../conduit-docs/

import (
"github.com/algorand/indexer/conduit/plugins/exporters/postgresql/util"
"github.com/algorand/conduit/conduit/plugins/exporters/postgresql/util"
)

//Name: conduit_exporters_postgresql
Expand Down
10 changes: 5 additions & 5 deletions conduit/plugins/exporters/postgresql/postgresql_exporter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@ import (
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v3"

"github.com/algorand/indexer/conduit/data"
"github.com/algorand/indexer/conduit/plugins"
"github.com/algorand/indexer/conduit/plugins/exporters"
"github.com/algorand/indexer/conduit/plugins/exporters/postgresql/util"
"github.com/algorand/indexer/conduit/plugins/tools/testutil"
"github.com/algorand/conduit/conduit/data"
"github.com/algorand/conduit/conduit/plugins"
"github.com/algorand/conduit/conduit/plugins/exporters"
"github.com/algorand/conduit/conduit/plugins/exporters/postgresql/util"
"github.com/algorand/conduit/conduit/plugins/tools/testutil"
_ "github.com/algorand/indexer/idb/dummy"
)

Expand Down

0 comments on commit 23eabcc

Please sign in to comment.