From ee05f58618c69d8bc31c7d3b15690af97040afe4 Mon Sep 17 00:00:00 2001 From: Chengxuan Xing Date: Thu, 5 Dec 2024 16:36:28 +0000 Subject: [PATCH 1/5] restructure config Signed-off-by: Chengxuan Xing --- config.md | 14 +++++++-- internal/ethereum/config.go | 24 ++++++++++------ internal/ethereum/event_enricher.go | 6 ++-- internal/msgs/en_config_descriptions.go | 38 ++++++++++++++----------- 4 files changed, 52 insertions(+), 30 deletions(-) diff --git a/config.md b/config.md index 61ea695..be65e4b 100644 --- a/config.md +++ b/config.md @@ -91,6 +91,14 @@ |password|Password|`string`|`` |username|Username|`string`|`` +## connector.blockListenerRetry + +|Key|Description|Type|Default Value| +|---|-----------|----|-------------| +|factor|Factor to increase the delay by, between each block indexing http requests retry to the RPC endpoint|`int`|`2` +|initialDelay|Initial delay for retrying block indexing http requests to the RPC endpoint|[`time.Duration`](https://pkg.go.dev/time#Duration)|`100ms` +|maxDelay|Maximum delay for between each block indexing http requests retry to the RPC endpoint|[`time.Duration`](https://pkg.go.dev/time#Duration)|`30s` + ## connector.events |Key|Description|Type|Default Value| @@ -115,10 +123,10 @@ |count|The maximum number of times to retry|`int`|`5` |enabled|Enables retries|`boolean`|`false` |errorStatusCodeRegex|The regex that the error response status code must match to trigger retry|`string`|`` -|factor|The retry backoff factor|`float32`|`2` +|factor| use 'connector.blockListenerRetry.factor' instead|`int`|`` |initWaitTime|The initial retry delay|[`time.Duration`](https://pkg.go.dev/time#Duration)|`250ms` -|initialDelay|The initial retry delay|[`time.Duration`](https://pkg.go.dev/time#Duration)|`100ms` -|maxDelay|The maximum retry delay|[`time.Duration`](https://pkg.go.dev/time#Duration)|`30s` +|initialDelay| use 'connector.blockListenerRetry.initialDelay' instead|[`time.Duration`](https://pkg.go.dev/time#Duration)|`` +|maxDelay| use 'connector.blockListenerRetry.maxDelay' instead|[`time.Duration`](https://pkg.go.dev/time#Duration)|`` |maxWaitTime|The maximum retry delay|[`time.Duration`](https://pkg.go.dev/time#Duration)|`30s` ## connector.throttle diff --git a/internal/ethereum/config.go b/internal/ethereum/config.go index 0cc1c09..6b57bb0 100644 --- a/internal/ethereum/config.go +++ b/internal/ethereum/config.go @@ -32,14 +32,19 @@ const ( EventsCheckpointBlockGap = "events.checkpointBlockGap" EventsBlockTimestamps = "events.blockTimestamps" EventsFilterPollingInterval = "events.filterPollingInterval" - RetryInitDelay = "retry.initialDelay" - RetryMaxDelay = "retry.maxDelay" - RetryFactor = "retry.factor" - MaxConcurrentRequests = "maxConcurrentRequests" - TxCacheSize = "txCacheSize" - HederaCompatibilityMode = "hederaCompatibilityMode" - TraceTXForRevertReason = "traceTXForRevertReason" - WebSocketsEnabled = "ws.enabled" + RetryInitDelay = "blockListenerRetry.initialDelay" + RetryMaxDelay = "blockListenerRetry.maxDelay" + RetryFactor = "blockListenerRetry.factor" + + DeprecatedRetryInitDelay = "retry.initialDelay" + DeprecatedRetryMaxDelay = "retry.maxDelay" + DeprecatedRetryFactor = "retry.factor" + + MaxConcurrentRequests = "maxConcurrentRequests" + TxCacheSize = "txCacheSize" + HederaCompatibilityMode = "hederaCompatibilityMode" + TraceTXForRevertReason = "traceTXForRevertReason" + WebSocketsEnabled = "ws.enabled" ) const ( @@ -72,6 +77,9 @@ func InitConfig(conf config.Section) { conf.AddKnownKey(RetryFactor, DefaultRetryDelayFactor) conf.AddKnownKey(RetryInitDelay, DefaultRetryInitDelay) conf.AddKnownKey(RetryMaxDelay, DefaultRetryMaxDelay) + conf.AddKnownKey(DeprecatedRetryFactor) + conf.AddKnownKey(DeprecatedRetryInitDelay) + conf.AddKnownKey(DeprecatedRetryMaxDelay) conf.AddKnownKey(MaxConcurrentRequests, 50) conf.AddKnownKey(TxCacheSize, 250) conf.AddKnownKey(HederaCompatibilityMode, false) diff --git a/internal/ethereum/event_enricher.go b/internal/ethereum/event_enricher.go index b131155..5c02558 100644 --- a/internal/ethereum/event_enricher.go +++ b/internal/ethereum/event_enricher.go @@ -96,9 +96,9 @@ func (ee *eventEnricher) filterEnrichEthLog(ctx context.Context, f *eventFilter, Signature: signature, BlockHash: ethLog.BlockHash.String(), TransactionHash: ethLog.TransactionHash.String(), - BlockNumber: fftypes.FFuint64(blockNumber), - TransactionIndex: fftypes.FFuint64(transactionIndex), - LogIndex: fftypes.FFuint64(logIndex), + BlockNumber: fftypes.FFuint64(ethLog.BlockNumber.BigInt().Uint64()), + TransactionIndex: fftypes.FFuint64(ethLog.TransactionIndex.BigInt().Uint64()), + LogIndex: fftypes.FFuint64(ethLog.LogIndex.BigInt().Uint64()), Timestamp: timestamp, }, Info: &info, diff --git a/internal/msgs/en_config_descriptions.go b/internal/msgs/en_config_descriptions.go index aba6020..f8a317b 100644 --- a/internal/msgs/en_config_descriptions.go +++ b/internal/msgs/en_config_descriptions.go @@ -27,20 +27,26 @@ var ffc = func(key, translation string, fieldType string) i18n.ConfigMessageKey //revive:disable var ( - ConfigEthereumURL = ffc("config.connector.url", "URL of JSON/RPC endpoint for the Ethereum node/gateway", "string") - ConfigEthereumWSEnabled = ffc("config.connector.ws.enabled", "When true a WebSocket is established for block listening, in addition to the HTTP RPC connections used for other functions", i18n.BooleanType) - ConfigEthereumDataFormat = ffc("config.connector.dataFormat", "Configure the JSON data format for query output and events", "map,flat_array,self_describing") - ConfigEthereumGasEstimationFactor = ffc("config.connector.gasEstimationFactor", "The factor to apply to the gas estimation to determine the gas limit", "float") - ConfigBlockCacheSize = ffc("config.connector.blockCacheSize", "Maximum of blocks to hold in the block info cache", i18n.IntType) - ConfigBlockPollingInterval = ffc("config.connector.blockPollingInterval", "Interval for polling to check for new blocks", i18n.TimeDurationType) - ConfigEventsBlockTimestamps = ffc("config.connector.events.blockTimestamps", "Whether to include the block timestamps in the event information", i18n.BooleanType) - ConfigEventsCatchupPageSize = ffc("config.connector.events.catchupPageSize", "Number of blocks to query per poll when catching up to the head of the blockchain", i18n.IntType) - ConfigEventsCatchupThreshold = ffc("config.connector.events.catchupThreshold", "How many blocks behind the chain head an event stream or listener must be on startup, to enter catchup mode", i18n.IntType) - ConfigEventsCatchupDownscaleRegex = ffc("config.connector.events.catchupDownscaleRegex", "An error pattern to check for from JSON/RPC providers if they limit response sizes to eth_getLogs(). If an error is returned from eth_getLogs() and that error matches the configured pattern, the number of logs requested (catchupPageSize) will be reduced automatically.", "string") - ConfigEventsCheckpointBlockGap = ffc("config.connector.events.checkpointBlockGap", "The number of blocks at the head of the chain that should be considered unstable (could be dropped from the canonical chain after a re-org). Unless events with a full set of confirmations are detected, the restart checkpoint will this many blocks behind the chain head.", i18n.IntType) - ConfigEventsFilterPollingInterval = ffc("config.connector.events.filterPollingInterval", "The interval between polling calls to a filter, when checking for newly arrived events", i18n.TimeDurationType) - ConfigTxCacheSize = ffc("config.connector.txCacheSize", "Maximum of transactions to hold in the transaction info cache", i18n.IntType) - ConfigMaxConcurrentRequests = ffc("config.connector.maxConcurrentRequests", "Maximum of concurrent requests to be submitted to the blockchain", i18n.IntType) - ConfigHederaCompatibilityMode = ffc("config.connector.hederaCompatibilityMode", "Compatibility mode for Hedera, allowing non-standard block header hashes to be processed", i18n.BooleanType) - ConfigTraceTXForRevertReason = ffc("config.connector.traceTXForRevertReason", "Enable the use of transaction trace functions (e.g. debug_traceTransaction) to obtain transaction revert reasons. This can place a high load on the EVM client.", i18n.BooleanType) + ConfigEthereumURL = ffc("config.connector.url", "URL of JSON/RPC endpoint for the Ethereum node/gateway", "string") + ConfigEthereumWSEnabled = ffc("config.connector.ws.enabled", "When true a WebSocket is established for block listening, in addition to the HTTP RPC connections used for other functions", i18n.BooleanType) + ConfigEthereumDataFormat = ffc("config.connector.dataFormat", "Configure the JSON data format for query output and events", "map,flat_array,self_describing") + ConfigEthereumGasEstimationFactor = ffc("config.connector.gasEstimationFactor", "The factor to apply to the gas estimation to determine the gas limit", "float") + ConfigBlockCacheSize = ffc("config.connector.blockCacheSize", "Maximum of blocks to hold in the block info cache", i18n.IntType) + ConfigBlockPollingInterval = ffc("config.connector.blockPollingInterval", "Interval for polling to check for new blocks", i18n.TimeDurationType) + ConfigBlockIndexRetryInitialDelay = ffc("config.connector.blockListenerRetry.initialDelay", "Initial delay for retrying block indexing http requests to the RPC endpoint", i18n.TimeDurationType) + ConfigBlockIndexRetryFactor = ffc("config.connector.blockListenerRetry.factor", "Factor to increase the delay by, between each block indexing http requests retry to the RPC endpoint", i18n.IntType) + ConfigBlockIndexRetryMaxDelay = ffc("config.connector.blockListenerRetry.maxDelay", "Maximum delay for between each block indexing http requests retry to the RPC endpoint", i18n.TimeDurationType) + DeprecatedConfigBlockIndexRetryInitialDelay = ffc("config.connector.retry.initialDelay", " use 'connector.blockListenerRetry.initialDelay' instead", i18n.TimeDurationType) + DeprecatedConfigBlockIndexRetryFactor = ffc("config.connector.retry.factor", " use 'connector.blockListenerRetry.factor' instead", i18n.IntType) + DeprecatedConfigBlockIndexRetryMaxDelay = ffc("config.connector.retry.maxDelay", " use 'connector.blockListenerRetry.maxDelay' instead", i18n.TimeDurationType) + ConfigEventsBlockTimestamps = ffc("config.connector.events.blockTimestamps", "Whether to include the block timestamps in the event information", i18n.BooleanType) + ConfigEventsCatchupPageSize = ffc("config.connector.events.catchupPageSize", "Number of blocks to query per poll when catching up to the head of the blockchain", i18n.IntType) + ConfigEventsCatchupThreshold = ffc("config.connector.events.catchupThreshold", "How many blocks behind the chain head an event stream or listener must be on startup, to enter catchup mode", i18n.IntType) + ConfigEventsCatchupDownscaleRegex = ffc("config.connector.events.catchupDownscaleRegex", "An error pattern to check for from JSON/RPC providers if they limit response sizes to eth_getLogs(). If an error is returned from eth_getLogs() and that error matches the configured pattern, the number of logs requested (catchupPageSize) will be reduced automatically.", "string") + ConfigEventsCheckpointBlockGap = ffc("config.connector.events.checkpointBlockGap", "The number of blocks at the head of the chain that should be considered unstable (could be dropped from the canonical chain after a re-org). Unless events with a full set of confirmations are detected, the restart checkpoint will this many blocks behind the chain head.", i18n.IntType) + ConfigEventsFilterPollingInterval = ffc("config.connector.events.filterPollingInterval", "The interval between polling calls to a filter, when checking for newly arrived events", i18n.TimeDurationType) + ConfigTxCacheSize = ffc("config.connector.txCacheSize", "Maximum of transactions to hold in the transaction info cache", i18n.IntType) + ConfigMaxConcurrentRequests = ffc("config.connector.maxConcurrentRequests", "Maximum of concurrent requests to be submitted to the blockchain", i18n.IntType) + ConfigHederaCompatibilityMode = ffc("config.connector.hederaCompatibilityMode", "Compatibility mode for Hedera, allowing non-standard block header hashes to be processed", i18n.BooleanType) + ConfigTraceTXForRevertReason = ffc("config.connector.traceTXForRevertReason", "Enable the use of transaction trace functions (e.g. debug_traceTransaction) to obtain transaction revert reasons. This can place a high load on the EVM client.", i18n.BooleanType) ) From c220338f0364b4dbd10239cfd5e90a65bc127617 Mon Sep 17 00:00:00 2001 From: Chengxuan Xing Date: Mon, 9 Dec 2024 14:07:22 +0000 Subject: [PATCH 2/5] adding migration Signed-off-by: Chengxuan Xing --- config.md | 6 ++-- internal/ethereum/ethereum.go | 25 ++++++++++++---- internal/ethereum/ethereum_test.go | 38 +++++++++++++++++++++++++ internal/msgs/en_config_descriptions.go | 6 ++-- 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/config.md b/config.md index be65e4b..c8807d3 100644 --- a/config.md +++ b/config.md @@ -70,7 +70,7 @@ |connectionTimeout|The maximum amount of time that a connection is allowed to remain with no data transmitted|[`time.Duration`](https://pkg.go.dev/time#Duration)|`30s` |dataFormat|Configure the JSON data format for query output and events|map,flat_array,self_describing|`map` |expectContinueTimeout|See [ExpectContinueTimeout in the Go docs](https://pkg.go.dev/net/http#Transport)|[`time.Duration`](https://pkg.go.dev/time#Duration)|`1s` -|gasEstimationFactor|The factor to apply to the gas estimation to determine the gas limit|float|`1.5` +|gasEstimationFactor|The factor to apply to the gas estimation to determine the gas limit|`float32`|`1.5` |headers|Adds custom headers to HTTP requests|`map[string]string`|`` |hederaCompatibilityMode|Compatibility mode for Hedera, allowing non-standard block header hashes to be processed|`boolean`|`false` |idleTimeout|The max duration to hold a HTTP keepalive connection between calls|[`time.Duration`](https://pkg.go.dev/time#Duration)|`475ms` @@ -95,7 +95,7 @@ |Key|Description|Type|Default Value| |---|-----------|----|-------------| -|factor|Factor to increase the delay by, between each block indexing http requests retry to the RPC endpoint|`int`|`2` +|factor|Factor to increase the delay by, between each block indexing http requests retry to the RPC endpoint|`float32`|`2` |initialDelay|Initial delay for retrying block indexing http requests to the RPC endpoint|[`time.Duration`](https://pkg.go.dev/time#Duration)|`100ms` |maxDelay|Maximum delay for between each block indexing http requests retry to the RPC endpoint|[`time.Duration`](https://pkg.go.dev/time#Duration)|`30s` @@ -123,7 +123,7 @@ |count|The maximum number of times to retry|`int`|`5` |enabled|Enables retries|`boolean`|`false` |errorStatusCodeRegex|The regex that the error response status code must match to trigger retry|`string`|`` -|factor| use 'connector.blockListenerRetry.factor' instead|`int`|`` +|factor| use 'connector.blockListenerRetry.factor' instead|`float32`|`` |initWaitTime|The initial retry delay|[`time.Duration`](https://pkg.go.dev/time#Duration)|`250ms` |initialDelay| use 'connector.blockListenerRetry.initialDelay' instead|[`time.Duration`](https://pkg.go.dev/time#Duration)|`` |maxDelay| use 'connector.blockListenerRetry.maxDelay' instead|[`time.Duration`](https://pkg.go.dev/time#Duration)|`` diff --git a/internal/ethereum/ethereum.go b/internal/ethereum/ethereum.go index c2b5ee9..8736c28 100644 --- a/internal/ethereum/ethereum.go +++ b/internal/ethereum/ethereum.go @@ -66,12 +66,27 @@ func NewEthereumConnector(ctx context.Context, conf config.Section) (cc ffcapi.A eventBlockTimestamps: conf.GetBool(EventsBlockTimestamps), eventFilterPollingInterval: conf.GetDuration(EventsFilterPollingInterval), traceTXForRevertReason: conf.GetBool(TraceTXForRevertReason), - retry: &retry.Retry{ - InitialDelay: conf.GetDuration(RetryInitDelay), - MaximumDelay: conf.GetDuration(RetryMaxDelay), - Factor: conf.GetFloat64(RetryFactor), - }, + retry: &retry.Retry{}, } + + if !conf.IsSet(DeprecatedRetryInitDelay) || (conf.IsSet(DeprecatedRetryInitDelay) && conf.IsSet(RetryInitDelay)) { + c.retry.InitialDelay = conf.GetDuration(RetryInitDelay) + } else { + c.retry.InitialDelay = conf.GetDuration(DeprecatedRetryInitDelay) + } + + if !conf.IsSet(DeprecatedRetryFactor) || (conf.IsSet(DeprecatedRetryFactor) && conf.IsSet(RetryFactor)) { + c.retry.Factor = conf.GetFloat64(RetryFactor) + } else { + c.retry.Factor = conf.GetFloat64(DeprecatedRetryFactor) + } + + if !conf.IsSet(DeprecatedRetryMaxDelay) || (conf.IsSet(DeprecatedRetryMaxDelay) && conf.IsSet(RetryMaxDelay)) { + c.retry.MaximumDelay = conf.GetDuration(RetryMaxDelay) + } else { + c.retry.MaximumDelay = conf.GetDuration(DeprecatedRetryMaxDelay) + } + if c.catchupThreshold < c.catchupPageSize { log.L(ctx).Warnf("Catchup threshold %d must be at least as large as the catchup page size %d (overridden to %d)", c.catchupThreshold, c.catchupPageSize, c.catchupPageSize) c.catchupThreshold = c.catchupPageSize diff --git a/internal/ethereum/ethereum_test.go b/internal/ethereum/ethereum_test.go index c37575b..b69376f 100644 --- a/internal/ethereum/ethereum_test.go +++ b/internal/ethereum/ethereum_test.go @@ -19,6 +19,7 @@ package ethereum import ( "context" "testing" + "time" "github.com/hyperledger/firefly-common/pkg/config" "github.com/hyperledger/firefly-common/pkg/ffresty" @@ -153,3 +154,40 @@ func TestConnectorInit(t *testing.T) { cc, err = NewEthereumConnector(context.Background(), conf) assert.Regexp(t, "FF23051", err) } +func TestNewEthereumConnector(t *testing.T) { + // Test deprecated fields + config.RootConfigReset() + conf := config.RootSection("unittest") + InitConfig(conf) + conf.Set(ffresty.HTTPConfigURL, "http://localhost:8545") + + // check default + cc, err := NewEthereumConnector(context.Background(), conf) + assert.NoError(t, err) + assert.NotNil(t, cc) + assert.Equal(t, 100*time.Millisecond, cc.(*ethConnector).retry.InitialDelay) + assert.Equal(t, 2.0, cc.(*ethConnector).retry.Factor) + assert.Equal(t, 30*time.Second, cc.(*ethConnector).retry.MaximumDelay) + + // check default + conf.Set(DeprecatedRetryInitDelay, "100ms") + conf.Set(DeprecatedRetryFactor, 2.0) + conf.Set(DeprecatedRetryMaxDelay, "30s") + cc, err = NewEthereumConnector(context.Background(), conf) + assert.NoError(t, err) + assert.NotNil(t, cc) + assert.Equal(t, 100*time.Millisecond, cc.(*ethConnector).retry.InitialDelay) + assert.Equal(t, 2.0, cc.(*ethConnector).retry.Factor) + assert.Equal(t, 30*time.Second, cc.(*ethConnector).retry.MaximumDelay) + + // check new values set + conf.Set(RetryInitDelay, "10s") + conf.Set(RetryFactor, 4.0) + conf.Set(RetryMaxDelay, "30s") + cc, err = NewEthereumConnector(context.Background(), conf) + assert.NoError(t, err) + assert.NotNil(t, cc) + assert.Equal(t, 10*time.Second, cc.(*ethConnector).retry.InitialDelay) + assert.Equal(t, 4.0, cc.(*ethConnector).retry.Factor) + assert.Equal(t, 30*time.Second, cc.(*ethConnector).retry.MaximumDelay) +} diff --git a/internal/msgs/en_config_descriptions.go b/internal/msgs/en_config_descriptions.go index f8a317b..b4fa35c 100644 --- a/internal/msgs/en_config_descriptions.go +++ b/internal/msgs/en_config_descriptions.go @@ -30,14 +30,14 @@ var ( ConfigEthereumURL = ffc("config.connector.url", "URL of JSON/RPC endpoint for the Ethereum node/gateway", "string") ConfigEthereumWSEnabled = ffc("config.connector.ws.enabled", "When true a WebSocket is established for block listening, in addition to the HTTP RPC connections used for other functions", i18n.BooleanType) ConfigEthereumDataFormat = ffc("config.connector.dataFormat", "Configure the JSON data format for query output and events", "map,flat_array,self_describing") - ConfigEthereumGasEstimationFactor = ffc("config.connector.gasEstimationFactor", "The factor to apply to the gas estimation to determine the gas limit", "float") + ConfigEthereumGasEstimationFactor = ffc("config.connector.gasEstimationFactor", "The factor to apply to the gas estimation to determine the gas limit", i18n.FloatType) ConfigBlockCacheSize = ffc("config.connector.blockCacheSize", "Maximum of blocks to hold in the block info cache", i18n.IntType) ConfigBlockPollingInterval = ffc("config.connector.blockPollingInterval", "Interval for polling to check for new blocks", i18n.TimeDurationType) ConfigBlockIndexRetryInitialDelay = ffc("config.connector.blockListenerRetry.initialDelay", "Initial delay for retrying block indexing http requests to the RPC endpoint", i18n.TimeDurationType) - ConfigBlockIndexRetryFactor = ffc("config.connector.blockListenerRetry.factor", "Factor to increase the delay by, between each block indexing http requests retry to the RPC endpoint", i18n.IntType) + ConfigBlockIndexRetryFactor = ffc("config.connector.blockListenerRetry.factor", "Factor to increase the delay by, between each block indexing http requests retry to the RPC endpoint", i18n.FloatType) ConfigBlockIndexRetryMaxDelay = ffc("config.connector.blockListenerRetry.maxDelay", "Maximum delay for between each block indexing http requests retry to the RPC endpoint", i18n.TimeDurationType) DeprecatedConfigBlockIndexRetryInitialDelay = ffc("config.connector.retry.initialDelay", " use 'connector.blockListenerRetry.initialDelay' instead", i18n.TimeDurationType) - DeprecatedConfigBlockIndexRetryFactor = ffc("config.connector.retry.factor", " use 'connector.blockListenerRetry.factor' instead", i18n.IntType) + DeprecatedConfigBlockIndexRetryFactor = ffc("config.connector.retry.factor", " use 'connector.blockListenerRetry.factor' instead", i18n.FloatType) DeprecatedConfigBlockIndexRetryMaxDelay = ffc("config.connector.retry.maxDelay", " use 'connector.blockListenerRetry.maxDelay' instead", i18n.TimeDurationType) ConfigEventsBlockTimestamps = ffc("config.connector.events.blockTimestamps", "Whether to include the block timestamps in the event information", i18n.BooleanType) ConfigEventsCatchupPageSize = ffc("config.connector.events.catchupPageSize", "Number of blocks to query per poll when catching up to the head of the blockchain", i18n.IntType) From 2a87b942adff3f72a7640fa9e481efee6df7ec59 Mon Sep 17 00:00:00 2001 From: Chengxuan Xing Date: Mon, 9 Dec 2024 14:09:39 +0000 Subject: [PATCH 3/5] split tests Signed-off-by: Chengxuan Xing --- internal/ethereum/ethereum_test.go | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/internal/ethereum/ethereum_test.go b/internal/ethereum/ethereum_test.go index b69376f..b6f5651 100644 --- a/internal/ethereum/ethereum_test.go +++ b/internal/ethereum/ethereum_test.go @@ -154,37 +154,39 @@ func TestConnectorInit(t *testing.T) { cc, err = NewEthereumConnector(context.Background(), conf) assert.Regexp(t, "FF23051", err) } -func TestNewEthereumConnector(t *testing.T) { + +// TODO: remove once deprecated fields are removed +func TestNewEthereumConnectorConfigDeprecates(t *testing.T) { // Test deprecated fields config.RootConfigReset() conf := config.RootSection("unittest") InitConfig(conf) conf.Set(ffresty.HTTPConfigURL, "http://localhost:8545") - // check default - cc, err := NewEthereumConnector(context.Background(), conf) - assert.NoError(t, err) - assert.NotNil(t, cc) - assert.Equal(t, 100*time.Millisecond, cc.(*ethConnector).retry.InitialDelay) - assert.Equal(t, 2.0, cc.(*ethConnector).retry.Factor) - assert.Equal(t, 30*time.Second, cc.(*ethConnector).retry.MaximumDelay) - - // check default + // check deprecates conf.Set(DeprecatedRetryInitDelay, "100ms") conf.Set(DeprecatedRetryFactor, 2.0) conf.Set(DeprecatedRetryMaxDelay, "30s") - cc, err = NewEthereumConnector(context.Background(), conf) + cc, err := NewEthereumConnector(context.Background(), conf) assert.NoError(t, err) assert.NotNil(t, cc) assert.Equal(t, 100*time.Millisecond, cc.(*ethConnector).retry.InitialDelay) assert.Equal(t, 2.0, cc.(*ethConnector).retry.Factor) assert.Equal(t, 30*time.Second, cc.(*ethConnector).retry.MaximumDelay) +} + +func TestNewEthereumConnectorConfig(t *testing.T) { + // Test deprecated fields + config.RootConfigReset() + conf := config.RootSection("unittest") + InitConfig(conf) + conf.Set(ffresty.HTTPConfigURL, "http://localhost:8545") // check new values set conf.Set(RetryInitDelay, "10s") conf.Set(RetryFactor, 4.0) conf.Set(RetryMaxDelay, "30s") - cc, err = NewEthereumConnector(context.Background(), conf) + cc, err := NewEthereumConnector(context.Background(), conf) assert.NoError(t, err) assert.NotNil(t, cc) assert.Equal(t, 10*time.Second, cc.(*ethConnector).retry.InitialDelay) From bcc3bcb862fa710fc11b9039c304855c40d2a5a4 Mon Sep 17 00:00:00 2001 From: Chengxuan Xing Date: Wed, 11 Dec 2024 06:37:22 +0000 Subject: [PATCH 4/5] rename configuration and update description Signed-off-by: Chengxuan Xing --- config.md | 22 +++++++++++----------- internal/ethereum/config.go | 6 +++--- internal/msgs/en_config_descriptions.go | 12 ++++++------ 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/config.md b/config.md index c8807d3..232bb76 100644 --- a/config.md +++ b/config.md @@ -91,14 +91,6 @@ |password|Password|`string`|`` |username|Username|`string`|`` -## connector.blockListenerRetry - -|Key|Description|Type|Default Value| -|---|-----------|----|-------------| -|factor|Factor to increase the delay by, between each block indexing http requests retry to the RPC endpoint|`float32`|`2` -|initialDelay|Initial delay for retrying block indexing http requests to the RPC endpoint|[`time.Duration`](https://pkg.go.dev/time#Duration)|`100ms` -|maxDelay|Maximum delay for between each block indexing http requests retry to the RPC endpoint|[`time.Duration`](https://pkg.go.dev/time#Duration)|`30s` - ## connector.events |Key|Description|Type|Default Value| @@ -116,6 +108,14 @@ |---|-----------|----|-------------| |url|Optional HTTP proxy server to connect through|`string`|`` +## connector.queryLoopRetry + +|Key|Description|Type|Default Value| +|---|-----------|----|-------------| +|factor|Factor to increase the delay by, between each query request retry to the RPC endpoint, applicable to all the query loops|`float32`|`2` +|initialDelay|Initial delay for retrying query requests to the RPC endpoint, applicable to all the query loops|[`time.Duration`](https://pkg.go.dev/time#Duration)|`100ms` +|maxDelay|Maximum delay for between each query request retry to the RPC endpoint, applicable to all the query loops|[`time.Duration`](https://pkg.go.dev/time#Duration)|`30s` + ## connector.retry |Key|Description|Type|Default Value| @@ -123,10 +123,10 @@ |count|The maximum number of times to retry|`int`|`5` |enabled|Enables retries|`boolean`|`false` |errorStatusCodeRegex|The regex that the error response status code must match to trigger retry|`string`|`` -|factor| use 'connector.blockListenerRetry.factor' instead|`float32`|`` +|factor| Please refer to 'connector.queryLoopRetry.factor' to understand its original purpose and use that instead|`float32`|`` |initWaitTime|The initial retry delay|[`time.Duration`](https://pkg.go.dev/time#Duration)|`250ms` -|initialDelay| use 'connector.blockListenerRetry.initialDelay' instead|[`time.Duration`](https://pkg.go.dev/time#Duration)|`` -|maxDelay| use 'connector.blockListenerRetry.maxDelay' instead|[`time.Duration`](https://pkg.go.dev/time#Duration)|`` +|initialDelay| Please refer to 'connector.queryLoopRetry.initialDelay' to understand its original purpose and use that instead|[`time.Duration`](https://pkg.go.dev/time#Duration)|`` +|maxDelay| Please refer to 'connector.queryLoopRetry.maxDelay' to understand its original purpose and use that instead|[`time.Duration`](https://pkg.go.dev/time#Duration)|`` |maxWaitTime|The maximum retry delay|[`time.Duration`](https://pkg.go.dev/time#Duration)|`30s` ## connector.throttle diff --git a/internal/ethereum/config.go b/internal/ethereum/config.go index 6b57bb0..fe40f49 100644 --- a/internal/ethereum/config.go +++ b/internal/ethereum/config.go @@ -32,9 +32,9 @@ const ( EventsCheckpointBlockGap = "events.checkpointBlockGap" EventsBlockTimestamps = "events.blockTimestamps" EventsFilterPollingInterval = "events.filterPollingInterval" - RetryInitDelay = "blockListenerRetry.initialDelay" - RetryMaxDelay = "blockListenerRetry.maxDelay" - RetryFactor = "blockListenerRetry.factor" + RetryInitDelay = "queryLoopRetry.initialDelay" + RetryMaxDelay = "queryLoopRetry.maxDelay" + RetryFactor = "queryLoopRetry.factor" DeprecatedRetryInitDelay = "retry.initialDelay" DeprecatedRetryMaxDelay = "retry.maxDelay" diff --git a/internal/msgs/en_config_descriptions.go b/internal/msgs/en_config_descriptions.go index b4fa35c..a1fab28 100644 --- a/internal/msgs/en_config_descriptions.go +++ b/internal/msgs/en_config_descriptions.go @@ -33,12 +33,12 @@ var ( ConfigEthereumGasEstimationFactor = ffc("config.connector.gasEstimationFactor", "The factor to apply to the gas estimation to determine the gas limit", i18n.FloatType) ConfigBlockCacheSize = ffc("config.connector.blockCacheSize", "Maximum of blocks to hold in the block info cache", i18n.IntType) ConfigBlockPollingInterval = ffc("config.connector.blockPollingInterval", "Interval for polling to check for new blocks", i18n.TimeDurationType) - ConfigBlockIndexRetryInitialDelay = ffc("config.connector.blockListenerRetry.initialDelay", "Initial delay for retrying block indexing http requests to the RPC endpoint", i18n.TimeDurationType) - ConfigBlockIndexRetryFactor = ffc("config.connector.blockListenerRetry.factor", "Factor to increase the delay by, between each block indexing http requests retry to the RPC endpoint", i18n.FloatType) - ConfigBlockIndexRetryMaxDelay = ffc("config.connector.blockListenerRetry.maxDelay", "Maximum delay for between each block indexing http requests retry to the RPC endpoint", i18n.TimeDurationType) - DeprecatedConfigBlockIndexRetryInitialDelay = ffc("config.connector.retry.initialDelay", " use 'connector.blockListenerRetry.initialDelay' instead", i18n.TimeDurationType) - DeprecatedConfigBlockIndexRetryFactor = ffc("config.connector.retry.factor", " use 'connector.blockListenerRetry.factor' instead", i18n.FloatType) - DeprecatedConfigBlockIndexRetryMaxDelay = ffc("config.connector.retry.maxDelay", " use 'connector.blockListenerRetry.maxDelay' instead", i18n.TimeDurationType) + ConfigBlockIndexRetryInitialDelay = ffc("config.connector.queryLoopRetry.initialDelay", "Initial delay for retrying query requests to the RPC endpoint, applicable to all the query loops", i18n.TimeDurationType) + ConfigBlockIndexRetryFactor = ffc("config.connector.queryLoopRetry.factor", "Factor to increase the delay by, between each query request retry to the RPC endpoint, applicable to all the query loops", i18n.FloatType) + ConfigBlockIndexRetryMaxDelay = ffc("config.connector.queryLoopRetry.maxDelay", "Maximum delay for between each query request retry to the RPC endpoint, applicable to all the query loops", i18n.TimeDurationType) + DeprecatedConfigBlockIndexRetryInitialDelay = ffc("config.connector.retry.initialDelay", " Please refer to 'connector.queryLoopRetry.initialDelay' to understand its original purpose and use that instead", i18n.TimeDurationType) + DeprecatedConfigBlockIndexRetryFactor = ffc("config.connector.retry.factor", " Please refer to 'connector.queryLoopRetry.factor' to understand its original purpose and use that instead", i18n.FloatType) + DeprecatedConfigBlockIndexRetryMaxDelay = ffc("config.connector.retry.maxDelay", " Please refer to 'connector.queryLoopRetry.maxDelay' to understand its original purpose and use that instead", i18n.TimeDurationType) ConfigEventsBlockTimestamps = ffc("config.connector.events.blockTimestamps", "Whether to include the block timestamps in the event information", i18n.BooleanType) ConfigEventsCatchupPageSize = ffc("config.connector.events.catchupPageSize", "Number of blocks to query per poll when catching up to the head of the blockchain", i18n.IntType) ConfigEventsCatchupThreshold = ffc("config.connector.events.catchupThreshold", "How many blocks behind the chain head an event stream or listener must be on startup, to enter catchup mode", i18n.IntType) From 14438ef85c181e1cda03c66e8f4a5126ce605760 Mon Sep 17 00:00:00 2001 From: Chengxuan Xing Date: Wed, 11 Dec 2024 06:39:16 +0000 Subject: [PATCH 5/5] remove useless names Signed-off-by: Chengxuan Xing --- internal/msgs/en_config_descriptions.go | 44 ++++++++++++------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/internal/msgs/en_config_descriptions.go b/internal/msgs/en_config_descriptions.go index a1fab28..ea7bf8d 100644 --- a/internal/msgs/en_config_descriptions.go +++ b/internal/msgs/en_config_descriptions.go @@ -27,26 +27,26 @@ var ffc = func(key, translation string, fieldType string) i18n.ConfigMessageKey //revive:disable var ( - ConfigEthereumURL = ffc("config.connector.url", "URL of JSON/RPC endpoint for the Ethereum node/gateway", "string") - ConfigEthereumWSEnabled = ffc("config.connector.ws.enabled", "When true a WebSocket is established for block listening, in addition to the HTTP RPC connections used for other functions", i18n.BooleanType) - ConfigEthereumDataFormat = ffc("config.connector.dataFormat", "Configure the JSON data format for query output and events", "map,flat_array,self_describing") - ConfigEthereumGasEstimationFactor = ffc("config.connector.gasEstimationFactor", "The factor to apply to the gas estimation to determine the gas limit", i18n.FloatType) - ConfigBlockCacheSize = ffc("config.connector.blockCacheSize", "Maximum of blocks to hold in the block info cache", i18n.IntType) - ConfigBlockPollingInterval = ffc("config.connector.blockPollingInterval", "Interval for polling to check for new blocks", i18n.TimeDurationType) - ConfigBlockIndexRetryInitialDelay = ffc("config.connector.queryLoopRetry.initialDelay", "Initial delay for retrying query requests to the RPC endpoint, applicable to all the query loops", i18n.TimeDurationType) - ConfigBlockIndexRetryFactor = ffc("config.connector.queryLoopRetry.factor", "Factor to increase the delay by, between each query request retry to the RPC endpoint, applicable to all the query loops", i18n.FloatType) - ConfigBlockIndexRetryMaxDelay = ffc("config.connector.queryLoopRetry.maxDelay", "Maximum delay for between each query request retry to the RPC endpoint, applicable to all the query loops", i18n.TimeDurationType) - DeprecatedConfigBlockIndexRetryInitialDelay = ffc("config.connector.retry.initialDelay", " Please refer to 'connector.queryLoopRetry.initialDelay' to understand its original purpose and use that instead", i18n.TimeDurationType) - DeprecatedConfigBlockIndexRetryFactor = ffc("config.connector.retry.factor", " Please refer to 'connector.queryLoopRetry.factor' to understand its original purpose and use that instead", i18n.FloatType) - DeprecatedConfigBlockIndexRetryMaxDelay = ffc("config.connector.retry.maxDelay", " Please refer to 'connector.queryLoopRetry.maxDelay' to understand its original purpose and use that instead", i18n.TimeDurationType) - ConfigEventsBlockTimestamps = ffc("config.connector.events.blockTimestamps", "Whether to include the block timestamps in the event information", i18n.BooleanType) - ConfigEventsCatchupPageSize = ffc("config.connector.events.catchupPageSize", "Number of blocks to query per poll when catching up to the head of the blockchain", i18n.IntType) - ConfigEventsCatchupThreshold = ffc("config.connector.events.catchupThreshold", "How many blocks behind the chain head an event stream or listener must be on startup, to enter catchup mode", i18n.IntType) - ConfigEventsCatchupDownscaleRegex = ffc("config.connector.events.catchupDownscaleRegex", "An error pattern to check for from JSON/RPC providers if they limit response sizes to eth_getLogs(). If an error is returned from eth_getLogs() and that error matches the configured pattern, the number of logs requested (catchupPageSize) will be reduced automatically.", "string") - ConfigEventsCheckpointBlockGap = ffc("config.connector.events.checkpointBlockGap", "The number of blocks at the head of the chain that should be considered unstable (could be dropped from the canonical chain after a re-org). Unless events with a full set of confirmations are detected, the restart checkpoint will this many blocks behind the chain head.", i18n.IntType) - ConfigEventsFilterPollingInterval = ffc("config.connector.events.filterPollingInterval", "The interval between polling calls to a filter, when checking for newly arrived events", i18n.TimeDurationType) - ConfigTxCacheSize = ffc("config.connector.txCacheSize", "Maximum of transactions to hold in the transaction info cache", i18n.IntType) - ConfigMaxConcurrentRequests = ffc("config.connector.maxConcurrentRequests", "Maximum of concurrent requests to be submitted to the blockchain", i18n.IntType) - ConfigHederaCompatibilityMode = ffc("config.connector.hederaCompatibilityMode", "Compatibility mode for Hedera, allowing non-standard block header hashes to be processed", i18n.BooleanType) - ConfigTraceTXForRevertReason = ffc("config.connector.traceTXForRevertReason", "Enable the use of transaction trace functions (e.g. debug_traceTransaction) to obtain transaction revert reasons. This can place a high load on the EVM client.", i18n.BooleanType) + _ = ffc("config.connector.url", "URL of JSON/RPC endpoint for the Ethereum node/gateway", "string") + _ = ffc("config.connector.ws.enabled", "When true a WebSocket is established for block listening, in addition to the HTTP RPC connections used for other functions", i18n.BooleanType) + _ = ffc("config.connector.dataFormat", "Configure the JSON data format for query output and events", "map,flat_array,self_describing") + _ = ffc("config.connector.gasEstimationFactor", "The factor to apply to the gas estimation to determine the gas limit", i18n.FloatType) + _ = ffc("config.connector.blockCacheSize", "Maximum of blocks to hold in the block info cache", i18n.IntType) + _ = ffc("config.connector.blockPollingInterval", "Interval for polling to check for new blocks", i18n.TimeDurationType) + _ = ffc("config.connector.queryLoopRetry.initialDelay", "Initial delay for retrying query requests to the RPC endpoint, applicable to all the query loops", i18n.TimeDurationType) + _ = ffc("config.connector.queryLoopRetry.factor", "Factor to increase the delay by, between each query request retry to the RPC endpoint, applicable to all the query loops", i18n.FloatType) + _ = ffc("config.connector.queryLoopRetry.maxDelay", "Maximum delay for between each query request retry to the RPC endpoint, applicable to all the query loops", i18n.TimeDurationType) + _ = ffc("config.connector.retry.initialDelay", " Please refer to 'connector.queryLoopRetry.initialDelay' to understand its original purpose and use that instead", i18n.TimeDurationType) + _ = ffc("config.connector.retry.factor", " Please refer to 'connector.queryLoopRetry.factor' to understand its original purpose and use that instead", i18n.FloatType) + _ = ffc("config.connector.retry.maxDelay", " Please refer to 'connector.queryLoopRetry.maxDelay' to understand its original purpose and use that instead", i18n.TimeDurationType) + _ = ffc("config.connector.events.blockTimestamps", "Whether to include the block timestamps in the event information", i18n.BooleanType) + _ = ffc("config.connector.events.catchupPageSize", "Number of blocks to query per poll when catching up to the head of the blockchain", i18n.IntType) + _ = ffc("config.connector.events.catchupThreshold", "How many blocks behind the chain head an event stream or listener must be on startup, to enter catchup mode", i18n.IntType) + _ = ffc("config.connector.events.catchupDownscaleRegex", "An error pattern to check for from JSON/RPC providers if they limit response sizes to eth_getLogs(). If an error is returned from eth_getLogs() and that error matches the configured pattern, the number of logs requested (catchupPageSize) will be reduced automatically.", "string") + _ = ffc("config.connector.events.checkpointBlockGap", "The number of blocks at the head of the chain that should be considered unstable (could be dropped from the canonical chain after a re-org). Unless events with a full set of confirmations are detected, the restart checkpoint will this many blocks behind the chain head.", i18n.IntType) + _ = ffc("config.connector.events.filterPollingInterval", "The interval between polling calls to a filter, when checking for newly arrived events", i18n.TimeDurationType) + _ = ffc("config.connector.txCacheSize", "Maximum of transactions to hold in the transaction info cache", i18n.IntType) + _ = ffc("config.connector.maxConcurrentRequests", "Maximum of concurrent requests to be submitted to the blockchain", i18n.IntType) + _ = ffc("config.connector.hederaCompatibilityMode", "Compatibility mode for Hedera, allowing non-standard block header hashes to be processed", i18n.BooleanType) + _ = ffc("config.connector.traceTXForRevertReason", "Enable the use of transaction trace functions (e.g. debug_traceTransaction) to obtain transaction revert reasons. This can place a high load on the EVM client.", i18n.BooleanType) )