Merge branch 'develop' into fix_release_2.14.0

# Conflicts:
#	api/generated/v2/routes.go
#	third_party/go-algorand

.circleci/config.yml

@@ -41,10 +41,12 @@ workflows:
 jobs:
   test:
     machine:
-      image: << pipeline.parameters.ubuntu_image >> 
+      image: << pipeline.parameters.ubuntu_image >>
     parameters:
       go_version:
         type: string
+    environment:
+      CI_E2E_FILENAME: "fafa8862/rel-nightly"
     steps:
       - go/install:
           version: << parameters.go_version >>
@@ -53,10 +55,12 @@ jobs:
       - codecov/upload
   test_nightly:
     machine:
-      image: << pipeline.parameters.ubuntu_image >> 
+      image: << pipeline.parameters.ubuntu_image >>
     parameters:
       go_version:
         type: string
+    environment:
+      CI_E2E_FILENAME: "rel-nightly"
     steps:
       - go/install:
           version: << parameters.go_version >>
@@ -119,7 +123,7 @@ commands:
 
       - run: echo 'export PATH=$PATH:/usr/local/go/bin' >> $BASH_ENV
 
-      - run: 
+      - run:
           name: Install golint
           command: go install golang.org/x/lint/golint@latest
 

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "go-algorand"]
 	path = third_party/go-algorand
 	url = https://github.com/algorand/go-algorand
+	branch = rel/nightly

Makefile

@@ -34,7 +34,7 @@ idb/postgres/internal/schema/setup_postgres_sql.go:	idb/postgres/internal/schema
 	cd idb/postgres/internal/schema && go generate
 
 idb/mocks/IndexerDb.go:	idb/idb.go
-	go install github.com/vektra/mockery/[email protected].3
+	go install github.com/vektra/mockery/[email protected].1
 	cd idb && mockery --name=IndexerDb
 
 # check that all packages (except tests) compile
@@ -93,4 +93,12 @@ indexer-v-algod-swagger:
 
 indexer-v-algod: nightly-setup indexer-v-algod-swagger nightly-teardown
 
+# fetch and update submodule. it's default to latest rel/nightly branch.
+# to use a different branch, update the branch in .gitmodules for CI build,
+# and for local testing, you may checkout a specific branch in the submodule.
+# after submodule is updated, CI_E2E_FILE in circleci/config.yml should also
+# be updated to use a newer artifact. path copied from s3 bucket, s3://algorand-testdata/indexer/e2e4/
+update-submodule:
+	git submodule update --remote
+
 .PHONY: test e2e integration fmt lint deploy sign test-package package fakepackage cmd/algorand-indexer/algorand-indexer idb/mocks/IndexerDb.go go-algorand indexer-v-algod

api/handlers_test.go

@@ -628,6 +628,15 @@ func TestFetchTransactions(t *testing.T) {
 				loadTransactionFromFile("test_resources/state_proof.response"),
 			},
 		},
+		{
+			name: "State Proof Txn - High Reveal Index",
+			txnBytes: [][]byte{
+				loadResourceFileOrPanic("test_resources/state_proof_with_index.txn"),
+			},
+			response: []generated.Transaction{
+				loadTransactionFromFile("test_resources/state_proof_with_index.response"),
+			},
+		},
 	}
 
 	// use for the brach below and createTxn helper func to add a new test case
@@ -1270,6 +1279,11 @@ func TestFetchBlock(t *testing.T) {
 			blockBytes: loadResourceFileOrPanic("test_resources/stpf_block.block"),
 			expected:   loadBlockFromFile("test_resources/stpf_block_response.json"),
 		},
+		{
+			name:       "State Proof Block - High Reveal Index",
+			blockBytes: loadResourceFileOrPanic("test_resources/stpf_block_high_index.block"),
+			expected:   loadBlockFromFile("test_resources/stpf_block_high_index_response.json"),
+		},
 	}
 
 	for _, tc := range testcases {

api/indexer.oas2.json

@@ -80,7 +80,8 @@
           },
           {
             "type": "integer",
-            "description": "Include results for the specified round. For performance reasons, this parameter may be disabled on some configurations.",
+            "description": "Include results for the specified round. For performance reasons, this parameter may be disabled on some configurations. Using application-id or asset-id filters will return both creator and opt-in accounts. Filtering by include-all will return creator and opt-in accounts for deleted assets and accounts. Non-opt-in managers are not included in the results when asset-id is used.",
+
             "name": "round",
             "in": "query"
           },

api/indexer.oas3.yml

@@ -2314,7 +2314,7 @@
             "x-algorand-format": "Address"
           },
           {
-            "description": "Include results for the specified round. For performance reasons, this parameter may be disabled on some configurations.",
+            "description": "Include results for the specified round. For performance reasons, this parameter may be disabled on some configurations. Using application-id or asset-id filters will return both creator and opt-in accounts. Filtering by include-all will return creator and opt-in accounts for deleted assets and accounts. Non-opt-in managers are not included in the results when asset-id is used.",
             "in": "query",
             "name": "round",
             "schema": {

cmd/algorand-indexer/daemon.go

@@ -32,6 +32,32 @@ import (
 	"github.com/algorand/indexer/util/metrics"
 )
 
+// GetConfigFromDataDir Given the data directory, configuration filename and a list of types, see if
+// a configuration file that matches was located there.  If no configuration file was there then an
+// empty string is returned.  If more than one filetype was matched, an error is returned.
+func GetConfigFromDataDir(dataDirectory string, configFilename string, configFileTypes []string) (string, error) {
+	count := 0
+	fullPath := ""
+	var err error
+
+	for _, configFileType := range configFileTypes {
+		autoloadParamConfigPath := filepath.Join(dataDirectory, configFilename+"."+configFileType)
+		if util.FileExists(autoloadParamConfigPath) {
+			count++
+			fullPath = autoloadParamConfigPath
+		}
+	}
+
+	if count > 1 {
+		return "", fmt.Errorf("config filename (%s) in data directory (%s) matched more than one filetype: %v",
+			configFilename, dataDirectory, configFileTypes)
+	}
+
+	// if count == 0 then the fullpath will be set to "" and error will be nil
+	// if count == 1 then it fullpath will be correct
+	return fullPath, err
+}
+
 type daemonConfig struct {
 	flags                     *pflag.FlagSet
 	algodDataDir              string
@@ -146,17 +172,22 @@ func configureIndexerDataDir(indexerDataDir string) error {
 func loadIndexerConfig(indexerDataDir string, configFile string) error {
 	var err error
 	var resolvedConfigPath string
-	indexerConfigAutoLoadPath := filepath.Join(indexerDataDir, autoLoadIndexerConfigName)
-	indexerConfigFound := util.FileExists(indexerConfigAutoLoadPath)
+	potentialIndexerConfigPath, err := GetConfigFromDataDir(indexerDataDir, autoLoadIndexerConfigFileName, config.FileTypes[:])
+	if err != nil {
+		logger.Error(err)
+		return err
+	}
+	indexerConfigFound := potentialIndexerConfigPath != ""
+
 	if indexerConfigFound {
 		//autoload
 		if configFile != "" {
 			err = fmt.Errorf("indexer configuration was found in data directory (%s) as well as supplied via command line.  Only provide one",
-				indexerConfigAutoLoadPath)
+				potentialIndexerConfigPath)
 			logger.Error(err)
 			return err
 		}
-		resolvedConfigPath = indexerConfigAutoLoadPath
+		resolvedConfigPath = potentialIndexerConfigPath
 	} else if configFile != "" {
 		// user specified
 		resolvedConfigPath = configFile
@@ -188,17 +219,21 @@ func loadIndexerParamConfig(cfg *daemonConfig) error {
 		logger.WithError(err).Errorf("API Parameter Error: %v", err)
 		return err
 	}
-	autoloadParamConfigPath := filepath.Join(cfg.indexerDataDir, autoLoadParameterConfigName)
-	paramConfigFound := util.FileExists(autoloadParamConfigPath)
+	potentialParamConfigPath, err := GetConfigFromDataDir(cfg.indexerDataDir, autoLoadParameterConfigFileName, config.FileTypes[:])
+	if err != nil {
+		logger.Error(err)
+		return err
+	}
+	paramConfigFound := potentialParamConfigPath != ""
 	// If we auto-loaded configs but a user supplied them as well, we have an error
 	if paramConfigFound {
 		if cfg.suppliedAPIConfigFile != "" {
 			err = fmt.Errorf("api parameter configuration was found in data directory (%s) as well as supplied via command line.  Only provide one",
-				filepath.Join(cfg.indexerDataDir, autoLoadParameterConfigName))
+				potentialParamConfigPath)
 			logger.WithError(err).Errorf("indexer parameter config error: %v", err)
 			return err
 		}
-		cfg.suppliedAPIConfigFile = autoloadParamConfigPath
+		cfg.suppliedAPIConfigFile = potentialParamConfigPath
 		logger.Infof("Auto-loading parameter configuration file: %s", suppliedAPIConfigFile)
 	}
 	return err

cmd/algorand-indexer/daemon_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/algorand/go-algorand/ledger/ledgercore"
 	"github.com/algorand/go-algorand/rpcs"
 
+	"github.com/algorand/indexer/config"
 	"github.com/algorand/indexer/processor/blockprocessor"
 	itest "github.com/algorand/indexer/util/test"
 )
@@ -90,21 +91,61 @@ func createTempDir(t *testing.T) string {
 	return dir
 }
 
-// Make sure we output and return an error when both an API Config and
-// enable all parameters are provided together.
-func TestConfigWithEnableAllParamsExpectError(t *testing.T) {
+// TestParameterConfigErrorWhenBothFileTypesArePresent test that if both file types are there then it is an error
+func TestParameterConfigErrorWhenBothFileTypesArePresent(t *testing.T) {
+
 	indexerDataDir := createTempDir(t)
 	defer os.RemoveAll(indexerDataDir)
-	autoloadPath := filepath.Join(indexerDataDir, autoLoadIndexerConfigName)
-	os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
+	for _, configFiletype := range config.FileTypes {
+		autoloadPath := filepath.Join(indexerDataDir, autoLoadParameterConfigFileName+"."+configFiletype)
+		os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
+	}
+
 	daemonConfig := &daemonConfig{}
 	daemonConfig.flags = pflag.NewFlagSet("indexer", 0)
 	daemonConfig.indexerDataDir = indexerDataDir
-	daemonConfig.enableAllParameters = true
-	daemonConfig.suppliedAPIConfigFile = "foobar"
 	err := runDaemon(daemonConfig)
-	errorStr := "not allowed to supply an api config file and enable all parameters"
-	assert.EqualError(t, err, errorStr)
+	errorStr := fmt.Errorf("config filename (%s) in data directory (%s) matched more than one filetype: %v",
+		autoLoadParameterConfigFileName, indexerDataDir, config.FileTypes)
+	assert.EqualError(t, err, errorStr.Error())
+}
+
+// TestIndexerConfigErrorWhenBothFileTypesArePresent test that if both file types are there then it is an error
+func TestIndexerConfigErrorWhenBothFileTypesArePresent(t *testing.T) {
+
+	indexerDataDir := createTempDir(t)
+	defer os.RemoveAll(indexerDataDir)
+	for _, configFiletype := range config.FileTypes {
+		autoloadPath := filepath.Join(indexerDataDir, autoLoadIndexerConfigFileName+"."+configFiletype)
+		os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
+	}
+
+	daemonConfig := &daemonConfig{}
+	daemonConfig.flags = pflag.NewFlagSet("indexer", 0)
+	daemonConfig.indexerDataDir = indexerDataDir
+	err := runDaemon(daemonConfig)
+	errorStr := fmt.Errorf("config filename (%s) in data directory (%s) matched more than one filetype: %v",
+		autoLoadIndexerConfigFileName, indexerDataDir, config.FileTypes)
+	assert.EqualError(t, err, errorStr.Error())
+}
+
+// Make sure we output and return an error when both an API Config and
+// enable all parameters are provided together.
+func TestConfigWithEnableAllParamsExpectError(t *testing.T) {
+	for _, configFiletype := range config.FileTypes {
+		indexerDataDir := createTempDir(t)
+		defer os.RemoveAll(indexerDataDir)
+		autoloadPath := filepath.Join(indexerDataDir, autoLoadIndexerConfigFileName+"."+configFiletype)
+		os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
+		daemonConfig := &daemonConfig{}
+		daemonConfig.flags = pflag.NewFlagSet("indexer", 0)
+		daemonConfig.indexerDataDir = indexerDataDir
+		daemonConfig.enableAllParameters = true
+		daemonConfig.suppliedAPIConfigFile = "foobar"
+		err := runDaemon(daemonConfig)
+		errorStr := "not allowed to supply an api config file and enable all parameters"
+		assert.EqualError(t, err, errorStr)
+	}
 }
 
 func TestConfigDoesNotExistExpectError(t *testing.T) {
@@ -153,20 +194,23 @@ func TestConfigSpecifiedTwiceExpectError(t *testing.T) {
 }
 
 func TestLoadAPIConfigGivenAutoLoadAndUserSuppliedExpectError(t *testing.T) {
-	indexerDataDir := createTempDir(t)
-	defer os.RemoveAll(indexerDataDir)
-
-	autoloadPath := filepath.Join(indexerDataDir, autoLoadParameterConfigName)
-	userSuppliedPath := filepath.Join(indexerDataDir, "foobar.yml")
-	os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
-	cfg := &daemonConfig{}
-	cfg.indexerDataDir = indexerDataDir
-	cfg.suppliedAPIConfigFile = userSuppliedPath
 
-	err := loadIndexerParamConfig(cfg)
-	errorStr := fmt.Sprintf("api parameter configuration was found in data directory (%s) as well as supplied via command line.  Only provide one",
-		autoloadPath)
-	assert.EqualError(t, err, errorStr)
+	for _, configFiletype := range config.FileTypes {
+		indexerDataDir := createTempDir(t)
+		defer os.RemoveAll(indexerDataDir)
+
+		autoloadPath := filepath.Join(indexerDataDir, autoLoadParameterConfigFileName+"."+configFiletype)
+		userSuppliedPath := filepath.Join(indexerDataDir, "foobar.yml")
+		os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
+		cfg := &daemonConfig{}
+		cfg.indexerDataDir = indexerDataDir
+		cfg.suppliedAPIConfigFile = userSuppliedPath
+
+		err := loadIndexerParamConfig(cfg)
+		errorStr := fmt.Sprintf("api parameter configuration was found in data directory (%s) as well as supplied via command line.  Only provide one",
+			autoloadPath)
+		assert.EqualError(t, err, errorStr)
+	}
 }
 
 func TestLoadAPIConfigGivenUserSuppliedExpectSuccess(t *testing.T) {
@@ -183,17 +227,19 @@ func TestLoadAPIConfigGivenUserSuppliedExpectSuccess(t *testing.T) {
 }
 
 func TestLoadAPIConfigGivenAutoLoadExpectSuccess(t *testing.T) {
-	indexerDataDir := createTempDir(t)
-	defer os.RemoveAll(indexerDataDir)
-
-	autoloadPath := filepath.Join(indexerDataDir, autoLoadParameterConfigName)
-	os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
-	cfg := &daemonConfig{}
-	cfg.indexerDataDir = indexerDataDir
-
-	err := loadIndexerParamConfig(cfg)
-	assert.NoError(t, err)
-	assert.Equal(t, autoloadPath, cfg.suppliedAPIConfigFile)
+	for _, configFiletype := range config.FileTypes {
+		indexerDataDir := createTempDir(t)
+		defer os.RemoveAll(indexerDataDir)
+
+		autoloadPath := filepath.Join(indexerDataDir, autoLoadParameterConfigFileName+"."+configFiletype)
+		os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
+		cfg := &daemonConfig{}
+		cfg.indexerDataDir = indexerDataDir
+
+		err := loadIndexerParamConfig(cfg)
+		assert.NoError(t, err)
+		assert.Equal(t, autoloadPath, cfg.suppliedAPIConfigFile)
+	}
 }
 
 func TestIndexerDataDirNotProvidedExpectError(t *testing.T) {
@@ -217,18 +263,20 @@ func TestIndexerPidFileExpectSuccess(t *testing.T) {
 }
 
 func TestIndexerPidFileCreateFailExpectError(t *testing.T) {
-	indexerDataDir := createTempDir(t)
-	defer os.RemoveAll(indexerDataDir)
-	autoloadPath := filepath.Join(indexerDataDir, autoLoadIndexerConfigName)
-	os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
+	for _, configFiletype := range config.FileTypes {
+		indexerDataDir := createTempDir(t)
+		defer os.RemoveAll(indexerDataDir)
+		autoloadPath := filepath.Join(indexerDataDir, autoLoadIndexerConfigFileName+"."+configFiletype)
+		os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
 
-	invalidDir := filepath.Join(indexerDataDir, "foo", "bar")
-	cfg := &daemonConfig{}
-	cfg.pidFilePath = invalidDir
+		invalidDir := filepath.Join(indexerDataDir, "foo", "bar")
+		cfg := &daemonConfig{}
+		cfg.pidFilePath = invalidDir
 
-	cfg.flags = pflag.NewFlagSet("indexer", 0)
-	cfg.indexerDataDir = indexerDataDir
+		cfg.flags = pflag.NewFlagSet("indexer", 0)
+		cfg.indexerDataDir = indexerDataDir
 
-	assert.ErrorContains(t, runDaemon(cfg), "pid file")
-	assert.Error(t, createIndexerPidFile(cfg.pidFilePath))
+		assert.ErrorContains(t, runDaemon(cfg), "pid file")
+		assert.Error(t, createIndexerPidFile(cfg.pidFilePath))
+	}
 }