AVM: Implement lsig size pooling (#6057)

cmd/goal/clerk.go

@@ -980,16 +980,6 @@ func assembleFileImpl(fname string, printWarnings bool) *logic.OpStream {
 		ops.ReportMultipleErrors(fname, os.Stderr)
 		reportErrorf("%s: %s", fname, err)
 	}
-	_, params := getProto(protoVersion)
-	if ops.HasStatefulOps {
-		if len(ops.Program) > config.MaxAvailableAppProgramLen {
-			reportErrorf(tealAppSize, fname, len(ops.Program), config.MaxAvailableAppProgramLen)
-		}
-	} else {
-		if uint64(len(ops.Program)) > params.LogicSigMaxSize {
-			reportErrorf(tealLogicSigSize, fname, len(ops.Program), params.LogicSigMaxSize)
-		}
-	}
 
 	if printWarnings && len(ops.Warnings) != 0 {
 		for _, warning := range ops.Warnings {
@@ -1179,14 +1169,19 @@ var dryrunCmd = &cobra.Command{
 		if timeStamp <= 0 {
 			timeStamp = time.Now().Unix()
 		}
+
+		lSigPooledSize := 0
 		for i, txn := range stxns {
 			if txn.Lsig.Blank() {
 				continue
 			}
-			if uint64(txn.Lsig.Len()) > params.LogicSigMaxSize {
+			lsigLen := txn.Lsig.Len()
+			lSigPooledSize += lsigLen
+			if !params.EnableLogicSigSizePooling && uint64(lsigLen) > params.LogicSigMaxSize {
 				reportErrorf("program size too large: %d > %d", len(txn.Lsig.Logic), params.LogicSigMaxSize)
 			}
 			ep := logic.NewSigEvalParams(stxns, &params, logic.NoHeaderLedger{})
+
 			err := logic.CheckSignature(i, ep)
 			if err != nil {
 				reportErrorf("program failed Check: %s", err)
@@ -1204,6 +1199,10 @@ var dryrunCmd = &cobra.Command{
 				fmt.Fprintf(os.Stdout, "ERROR: %s\n", err.Error())
 			}
 		}
+		lSigMaxPooledSize := len(stxns) * int(params.LogicSigMaxSize)
+		if params.EnableLogicSigSizePooling && lSigPooledSize > lSigMaxPooledSize {
+			reportErrorf("total lsigs size too large: %d > %d", lSigPooledSize, lSigMaxPooledSize)
+		}
 
 	},
 }

config/consensus.go

@@ -113,9 +113,13 @@ type ConsensusParams struct {
 	EnableAppCostPooling bool
 
 	// EnableLogicSigCostPooling specifies LogicSig budgets are pooled across a
-	// group. The total available is len(group) * LogicSigMaxCost)
+	// group. The total available is len(group) * LogicSigMaxCost
 	EnableLogicSigCostPooling bool
 
+	// EnableLogicSigSizePooling specifies LogicSig sizes are pooled across a
+	// group. The total available is len(group) * LogicSigMaxSize
+	EnableLogicSigSizePooling bool
+
 	// RewardUnit specifies the number of MicroAlgos corresponding to one reward
 	// unit.
 	//
@@ -228,7 +232,7 @@ type ConsensusParams struct {
 	// 0 for no support, otherwise highest version supported
 	LogicSigVersion uint64
 
-	// len(LogicSig.Logic) + len(LogicSig.Args[*]) must be less than this
+	// len(LogicSig.Logic) + len(LogicSig.Args[*]) must be less than this (unless pooling is enabled)
 	LogicSigMaxSize uint64
 
 	// sum of estimated op cost must be less than this
@@ -765,7 +769,7 @@ func checkSetAllocBounds(p ConsensusParams) {
 	checkSetMax(p.MaxAppProgramLen, &MaxStateDeltaKeys)
 	checkSetMax(p.MaxAppProgramLen, &MaxEvalDeltaAccounts)
 	checkSetMax(p.MaxAppProgramLen, &MaxAppProgramLen)
-	checkSetMax(int(p.LogicSigMaxSize), &MaxLogicSigMaxSize)
+	checkSetMax((int(p.LogicSigMaxSize) * p.MaxTxGroupSize), &MaxLogicSigMaxSize)
 	checkSetMax(p.MaxTxnNoteBytes, &MaxTxnNoteBytes)
 	checkSetMax(p.MaxTxGroupSize, &MaxTxGroupSize)
 	// MaxBytesKeyValueLen is max of MaxAppKeyLen and MaxAppBytesValueLen
@@ -1512,6 +1516,8 @@ func initConsensusProtocols() {
 
 	vFuture.LogicSigVersion = 11 // When moving this to a release, put a new higher LogicSigVersion here
 
+	vFuture.EnableLogicSigSizePooling = true
+
 	vFuture.Payouts.Enabled = true
 	vFuture.Payouts.Percent = 75
 	vFuture.Payouts.GoOnlineFee = 2_000_000         // 2 algos

data/transactions/logic/eval.go

@@ -1000,6 +1000,7 @@ func (pe panicError) Error() string {
 
 var errLogicSigNotSupported = errors.New("LogicSig not supported")
 var errTooManyArgs = errors.New("LogicSig has too many arguments")
+var errLogicSigArgTooLarge = errors.New("LogicSig argument too large")
 
 // EvalError indicates AVM evaluation failure
 type EvalError struct {
@@ -1305,8 +1306,15 @@ func eval(program []byte, cx *EvalContext) (pass bool, err error) {
 	if (cx.EvalParams.Proto == nil) || (cx.EvalParams.Proto.LogicSigVersion == 0) {
 		return false, errLogicSigNotSupported
 	}
-	if cx.txn.Lsig.Args != nil && len(cx.txn.Lsig.Args) > transactions.EvalMaxArgs {
-		return false, errTooManyArgs
+	if cx.txn.Lsig.Args != nil {
+		if len(cx.txn.Lsig.Args) > transactions.EvalMaxArgs {
+			return false, errTooManyArgs
+		}
+		for _, arg := range cx.txn.Lsig.Args {
+			if len(arg) > transactions.MaxLogicSigArgSize {
+				return false, errLogicSigArgTooLarge
+			}
+		}
 	}
 	if verr != nil {
 		return false, verr

data/transactions/logic/eval_test.go

@@ -258,6 +258,24 @@ func TestTooManyArgs(t *testing.T) {
 	}
 }
 
+func TestArgTooLarge(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	t.Parallel()
+	for v := uint64(1); v <= AssemblerMaxVersion; v++ {
+		t.Run(fmt.Sprintf("v=%d", v), func(t *testing.T) {
+			ops := testProg(t, "int 1", v)
+			var txn transactions.SignedTxn
+			txn.Lsig.Logic = ops.Program
+			txn.Lsig.Args = [][]byte{make([]byte, transactions.MaxLogicSigArgSize+1)}
+			pass, err := EvalSignature(0, defaultSigParams(txn))
+			require.Error(t, err)
+			require.False(t, pass)
+		})
+	}
+
+}
+
 func TestEmptyProgram(t *testing.T) {
 	partitiontest.PartitionTest(t)
 

data/transactions/logicsig.go

@@ -25,6 +25,11 @@ import (
 // EvalMaxArgs is the maximum number of arguments to an LSig
 const EvalMaxArgs = 255
 
+// MaxLogicSigArgSize is the maximum size of an argument to an LSig
+// We use 4096 to match the maximum size of a TEAL value
+// (as defined in `const maxStringSize` in package logic)
+const MaxLogicSigArgSize = 4096
+
 // LogicSig contains logic for validating a transaction.
 // LogicSig is signed by an account, allowing delegation of operations.
 // OR
@@ -39,7 +44,7 @@ type LogicSig struct {
 	Msig crypto.MultisigSig `codec:"msig"`
 
 	// Args are not signed, but checked by Logic
-	Args [][]byte `codec:"arg,allocbound=EvalMaxArgs,allocbound=config.MaxLogicSigMaxSize"`
+	Args [][]byte `codec:"arg,allocbound=EvalMaxArgs,allocbound=MaxLogicSigArgSize,maxtotalbytes=config.MaxLogicSigMaxSize"`
 }
 
 // Blank returns true if there is no content in this LogicSig

data/transactions/verify/txn.go

@@ -92,7 +92,7 @@ type TxGroupErrorReason int
 const (
 	// TxGroupErrorReasonGeneric is a generic (not tracked) reason code
 	TxGroupErrorReasonGeneric TxGroupErrorReason = iota
-	// TxGroupErrorReasonNotWellFormed is txn.WellFormed failure
+	// TxGroupErrorReasonNotWellFormed is txn.WellFormed failure or malformed logic signature
 	TxGroupErrorReasonNotWellFormed
 	// TxGroupErrorReasonInvalidFee is invalid fee pooling in transaction group
 	TxGroupErrorReasonInvalidFee
@@ -213,6 +213,7 @@ func txnGroupBatchPrep(stxs []transactions.SignedTxn, contextHdr *bookkeeping.Bl
 
 	minFeeCount := uint64(0)
 	feesPaid := uint64(0)
+	lSigPooledSize := 0
 	for i, stxn := range stxs {
 		prepErr := txnBatchPrep(i, groupCtx, verifier)
 		if prepErr != nil {
@@ -224,6 +225,17 @@ func txnGroupBatchPrep(stxs []transactions.SignedTxn, contextHdr *bookkeeping.Bl
 			minFeeCount++
 		}
 		feesPaid = basics.AddSaturate(feesPaid, stxn.Txn.Fee.Raw)
+		lSigPooledSize += stxn.Lsig.Len()
+	}
+	if groupCtx.consensusParams.EnableLogicSigSizePooling {
+		lSigMaxPooledSize := len(stxs) * int(groupCtx.consensusParams.LogicSigMaxSize)
+		if lSigPooledSize > lSigMaxPooledSize {
+			errorMsg := fmt.Errorf(
+				"txgroup had %d bytes of LogicSigs, more than the available pool of %d bytes",
+				lSigPooledSize, lSigMaxPooledSize,
+			)
+			return nil, &TxGroupError{err: errorMsg, GroupIndex: -1, Reason: TxGroupErrorReasonNotWellFormed}
+		}
 	}
 	feeNeeded, overflow := basics.OMul(groupCtx.consensusParams.MinTxnFee, minFeeCount)
 	if overflow {
@@ -360,8 +372,8 @@ func logicSigSanityCheckBatchPrep(gi int, groupCtx *GroupContext, batchVerifier
 	if version > groupCtx.consensusParams.LogicSigVersion {
 		return errors.New("LogicSig.Logic version too new")
 	}
-	if uint64(lsig.Len()) > groupCtx.consensusParams.LogicSigMaxSize {
-		return errors.New("LogicSig.Logic too long")
+	if !groupCtx.consensusParams.EnableLogicSigSizePooling && uint64(lsig.Len()) > groupCtx.consensusParams.LogicSigMaxSize {
+		return errors.New("LogicSig too long")
 	}
 
 	err := logic.CheckSignature(gi, groupCtx.evalParams)

data/transactions/verify/txn_test.go

@@ -649,6 +649,70 @@ func BenchmarkPaysetGroups(b *testing.B) {
 	b.StopTimer()
 }
 
+func TestLsigSize(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	secrets, addresses, _ := generateAccounts(2)
+
+	execPool := execpool.MakePool(t)
+	verificationPool := execpool.MakeBacklog(execPool, 64, execpool.LowPriority, t)
+	defer verificationPool.Shutdown()
+
+	// From consensus version 18, we have lsigs with a maximum size of 1000 bytes.
+	// We need to use pragma 1 for teal in v18
+	pragma := uint(1)
+	consensusVersionPreSizePooling := protocol.ConsensusV18
+	consensusVersionPostSizePooling := protocol.ConsensusFuture
+
+	// We will do tests based on a transaction group of 2 payment transactions,
+	// the first signed by a lsig and the second a vanilla payment transaction.
+	testCases := []struct {
+		consensusVersion protocol.ConsensusVersion
+		lsigSize         uint
+		success          bool
+	}{
+		{consensusVersionPreSizePooling, 1000, true},
+		{consensusVersionPreSizePooling, 1001, false},
+		{consensusVersionPostSizePooling, 2000, true},
+		{consensusVersionPostSizePooling, 2001, false},
+	}
+
+	blkHdr := createDummyBlockHeader()
+	for _, test := range testCases {
+		blkHdr.UpgradeState.CurrentProtocol = test.consensusVersion
+
+		lsig, err := txntest.GenerateProgramOfSize(test.lsigSize, pragma)
+		require.NoError(t, err)
+
+		lsigPay := txntest.Txn{
+			Type:     protocol.PaymentTx,
+			Sender:   basics.Address(logic.HashProgram(lsig)),
+			Receiver: addresses[0],
+			Fee:      config.Consensus[test.consensusVersion].MinTxnFee,
+		}
+
+		vanillaPay := txntest.Txn{
+			Type:     protocol.PaymentTx,
+			Sender:   addresses[0],
+			Receiver: addresses[1],
+			Fee:      config.Consensus[test.consensusVersion].MinTxnFee,
+		}
+
+		group := txntest.Group(&lsigPay, &vanillaPay)
+		group[0].Lsig = transactions.LogicSig{
+			Logic: lsig,
+		}
+		group[1].Sig = secrets[0].Sign(group[1].Txn)
+
+		err = PaysetGroups(context.Background(), [][]transactions.SignedTxn{group}, blkHdr, verificationPool, MakeVerifiedTransactionCache(50000), &DummyLedgerForSignature{})
+		if test.success {
+			require.NoError(t, err)
+		} else {
+			require.Error(t, err)
+		}
+	}
+}
+
 func TestTxnGroupMixedSignatures(t *testing.T) {
 	partitiontest.PartitionTest(t)
 

data/txntest/program.go

@@ -0,0 +1,51 @@
+// Copyright (C) 2019-2024 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand.  If not, see <https://www.gnu.org/licenses/>.
+
+package txntest
+
+import (
+	"fmt"
+
+	"github.com/algorand/go-algorand/data/transactions/logic"
+)
+
+// GenerateProgramOfSize return a TEAL bytecode of `size` bytes which always succeeds.
+// `size` must be at least 9 bytes
+func GenerateProgramOfSize(size uint, pragma uint) ([]byte, error) {
+	if size < 9 {
+		return nil, fmt.Errorf("size must be at least 9 bytes; got %d", size)
+	}
+	ls := fmt.Sprintf("#pragma version %d\n", pragma)
+	if size%2 == 0 {
+		ls += "int 10\npop\nint 1\npop\n"
+	} else {
+		ls += "int 1\npop\nint 1\npop\n"
+	}
+	for i := uint(11); i <= size; i += 2 {
+		ls = ls + "int 1\npop\n"
+	}
+	ls = ls + "int 1"
+	code, err := logic.AssembleString(ls)
+	if err != nil {
+		return nil, err
+	}
+	// panic if the function is not working as expected and needs to be updated
+	if len(code.Program) != int(size) {
+		panic(fmt.Sprintf("wanted to create a program of size %d but got a program of size %d",
+			size, len(code.Program)))
+	}
+	return code.Program, nil
+}

node/node_test.go

@@ -819,10 +819,10 @@ func TestMaxSizesCorrect(t *testing.T) {
 	maxCombinedTxnSize := uint64(transactions.SignedTxnMaxSize())
 	// subtract out the two smaller signature sizes (logicsig is biggest, it can *contain* the others)
 	maxCombinedTxnSize -= uint64(crypto.SignatureMaxSize() + crypto.MultisigSigMaxSize())
-	// the logicsig size is *also* an overestimate, because it thinks each
-	// logicsig arg can be big, but really the sum of the args and the program
-	// has a max size.
-	maxCombinedTxnSize -= uint64(transactions.EvalMaxArgs * config.MaxLogicSigMaxSize)
+	// the logicsig size is *also* an overestimate, because it thinks that the logicsig and
+	// the logicsig args can both be up to to MaxLogicSigMaxSize, but that's the max for
+	// them combined, so it double counts and we have to subtract one.
+	maxCombinedTxnSize -= uint64(config.MaxLogicSigMaxSize)
 
 	// maxCombinedTxnSize is still an overestimate because it assumes all txn
 	// type fields can be in the same txn.  That's not true, but it provides an