Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Local Ledger #1011

Merged
merged 38 commits into from
Jun 8, 2022
Merged

Local Ledger #1011

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
a96fa70
integrate block processor
shiqizng May 12, 2022
ed32cb1
update ledger path
shiqizng May 12, 2022
8568fa2
add ledger close
shiqizng May 12, 2022
fda48cf
refactor AddBlock
shiqizng May 13, 2022
075f265
api handler test fails
shiqizng May 16, 2022
ac04503
Merge branch 'develop' into shiqi/AddBlock
shiqizng May 16, 2022
d7b8012
fixed handler e2e
shiqizng May 17, 2022
4dbda7d
updating postgres integration tests
shiqizng May 18, 2022
c0726b9
more test updates
shiqizng May 19, 2022
aee8e16
refactor processor to use EvalForIndexer
shiqizng May 19, 2022
a12262c
more test refactoring
shiqizng May 20, 2022
4c15d63
make integration failing
shiqizng May 20, 2022
7a8c05d
adding ledger for eval tests
shiqizng May 23, 2022
8de6a18
more tests
shiqizng May 24, 2022
088434b
make integration working
shiqizng May 25, 2022
dfd2e83
fix linting errors
shiqizng May 25, 2022
a40789f
remove debugging lines
shiqizng May 25, 2022
1abe9f9
error fix
shiqizng May 25, 2022
d0fe6a4
fix test failure
shiqizng May 25, 2022
39dda1b
refactoring
shiqizng May 25, 2022
1e7c0a6
more refactoring
shiqizng May 25, 2022
2ca22f3
fix failed test
shiqizng May 25, 2022
c4ab181
fix
shiqizng May 25, 2022
4d38249
fix failing test
shiqizng May 25, 2022
6ea76de
ready for review
shiqizng May 26, 2022
baaeeb2
test changes
shiqizng Jun 3, 2022
ce32607
fix vet issues
shiqizng Jun 4, 2022
c6411ce
fix failing test
shiqizng Jun 4, 2022
27b0a07
fix failing test
shiqizng Jun 4, 2022
d06852d
update e2e test
shiqizng Jun 4, 2022
f36732e
fix env var
shiqizng Jun 6, 2022
531588f
add an alias
shiqizng Jun 6, 2022
93b3988
use dynamic values in test
shiqizng Jun 6, 2022
2a5cb8b
remove env var alias
shiqizng Jun 6, 2022
66c3c70
pr reviews
shiqizng Jun 7, 2022
aef1b3b
update ledger init
shiqizng Jun 7, 2022
b1bc03d
format readme
shiqizng Jun 7, 2022
664bdd0
format readme
shiqizng Jun 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ Settings can be provided from the command line, a configuration file, or an envi
| default-balances-limit | | default-balances-limit | INDEXER_DEFAULT_BALANCES_LIMIT |
| max-applications-limit | | max-applications-limit | INDEXER_MAX_APPLICATIONS_LIMIT |
| default-applications-limit | | default-applications-limit | INDEXER_DEFAULT_APPLICATIONS_LIMIT |
| data-dir | i | data | INDEXER_DATA |

## Command line

Expand Down
89 changes: 52 additions & 37 deletions api/handlers_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,19 @@ import (
"time"

"github.com/algorand/go-algorand/crypto"
"github.com/algorand/go-algorand/crypto/merklesignature"
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/data/bookkeeping"
"github.com/algorand/go-algorand/ledger"
"github.com/algorand/go-algorand/rpcs"
"github.com/algorand/indexer/processor"
"github.com/algorand/indexer/processor/blockprocessor"
"github.com/labstack/echo/v4"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/algorand/go-algorand-sdk/encoding/json"
"github.com/algorand/go-algorand/crypto/merklesignature"
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/data/bookkeeping"
"github.com/algorand/go-algorand/data/transactions"

"github.com/algorand/indexer/api/generated/v2"
Expand Down Expand Up @@ -56,7 +60,7 @@ func testServerImplementation(db idb.IndexerDb) *ServerImplementation {
return &ServerImplementation{db: db, timeout: 30 * time.Second, opts: defaultOpts}
}

func setupIdb(t *testing.T, genesis bookkeeping.Genesis, genesisBlock bookkeeping.Block) (*postgres.IndexerDb /*db*/, func() /*shutdownFunc*/) {
func setupIdb(t *testing.T, genesis bookkeeping.Genesis) (*postgres.IndexerDb, func(), processor.Processor, *ledger.Ledger) {
_, connStr, shutdownFunc := pgtest.SetupPostgres(t)

db, _, err := postgres.OpenPostgres(connStr, idb.IndexerDbOptions{}, nil)
Expand All @@ -70,15 +74,16 @@ func setupIdb(t *testing.T, genesis bookkeeping.Genesis, genesisBlock bookkeepin
err = db.LoadGenesis(genesis)
require.NoError(t, err)

err = db.AddBlock(&genesisBlock)
require.NoError(t, err)

return db, newShutdownFunc
l := test.MakeTestLedger("ledger")
proc, err := blockprocessor.MakeProcessorWithLedger(l, db.AddBlock)
require.NoError(t, err, "failed to open ledger")
return db, newShutdownFunc, proc, l
}

func TestApplicationHandlers(t *testing.T) {
db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // A block containing an app call txn with ExtraProgramPages, that the creator and another account have opted into
Expand Down Expand Up @@ -112,8 +117,8 @@ func TestApplicationHandlers(t *testing.T) {
block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn, &optInTxnA, &optInTxnB)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

//////////
// When // We query the app
Expand Down Expand Up @@ -218,8 +223,9 @@ func TestApplicationHandlers(t *testing.T) {
}

func TestAccountExcludeParameters(t *testing.T) {
db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // A block containing a creator of an app, an asset, who also holds and has opted-into those apps.
Expand All @@ -238,8 +244,8 @@ func TestAccountExcludeParameters(t *testing.T) {
&appOptInTxnA, &appOptInTxnB, &assetOptInTxnA, &assetOptInTxnB)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

//////////
// When // We look up the address using various exclude parameters.
Expand Down Expand Up @@ -389,8 +395,9 @@ type accountsErrorResponse struct {
}

func TestAccountMaxResultsLimit(t *testing.T) {
db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // A block containing an address that has created 10 apps, deleted 5 apps, and created 10 assets,
Expand Down Expand Up @@ -443,8 +450,8 @@ func TestAccountMaxResultsLimit(t *testing.T) {
block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, ptxns...)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

//////////
// When // We look up the address using a ServerImplementation with a maxAccountsAPIResults limit set,
Expand Down Expand Up @@ -768,8 +775,9 @@ func TestAccountMaxResultsLimit(t *testing.T) {
}

func TestBlockNotFound(t *testing.T) {
db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, _, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // An empty database.
Expand Down Expand Up @@ -833,8 +841,9 @@ func TestInnerTxn(t *testing.T) {
},
}

db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // a DB with some inner txns in it.
Expand All @@ -845,8 +854,8 @@ func TestInnerTxn(t *testing.T) {
block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
Expand Down Expand Up @@ -881,8 +890,9 @@ func TestInnerTxn(t *testing.T) {
// transaction group does not allow the root transaction to be returned on both
// pages.
func TestPagingRootTxnDeduplication(t *testing.T) {
db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // a DB with some inner txns in it.
Expand All @@ -897,8 +907,8 @@ func TestPagingRootTxnDeduplication(t *testing.T) {
block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

testcases := []struct {
name string
Expand Down Expand Up @@ -1004,8 +1014,9 @@ func TestPagingRootTxnDeduplication(t *testing.T) {
}

func TestKeyregTransactionWithStateProofKeys(t *testing.T) {
db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // A block containing a key reg txn with state proof key
Expand Down Expand Up @@ -1044,8 +1055,8 @@ func TestKeyregTransactionWithStateProofKeys(t *testing.T) {
block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &txn)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

e := echo.New()
{
Expand Down Expand Up @@ -1098,8 +1109,9 @@ func TestVersion(t *testing.T) {
///////////
// Given // An API and context
///////////
db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, _, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()
api := testServerImplementation(db)

e := echo.New()
Expand Down Expand Up @@ -1127,8 +1139,9 @@ func TestVersion(t *testing.T) {
}

func TestAccountClearsNonUTF8(t *testing.T) {
db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // a DB with some inner txns in it.
Expand All @@ -1147,8 +1160,8 @@ func TestAccountClearsNonUTF8(t *testing.T) {
block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &createAsset)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

verify := func(params generated.AssetParams) {
compareB64 := func(expected string, actual *[]byte) {
Expand Down Expand Up @@ -1257,8 +1270,9 @@ func TestLookupInnerLogs(t *testing.T) {
},
}

db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // a DB with some inner txns in it.
Expand All @@ -1268,8 +1282,8 @@ func TestLookupInnerLogs(t *testing.T) {
block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
Expand Down Expand Up @@ -1355,8 +1369,9 @@ func TestLookupMultiInnerLogs(t *testing.T) {
},
}

db, shutdownFunc := setupIdb(t, test.MakeGenesis(), test.MakeGenesisBlock())
db, shutdownFunc, proc, l := setupIdb(t, test.MakeGenesis())
defer shutdownFunc()
defer l.Close()

///////////
// Given // a DB with some inner txns in it.
Expand All @@ -1366,8 +1381,8 @@ func TestLookupMultiInnerLogs(t *testing.T) {
block, err := test.MakeBlockForTxns(test.MakeGenesisBlock().BlockHeader, &appCall)
require.NoError(t, err)

err = db.AddBlock(&block)
require.NoError(t, err, "failed to commit")
err = proc.Process(&rpcs.EncodedBlockCert{Block: block})
require.NoError(t, err)

for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
Expand Down
Loading