Update KZG library according to latest consensus spec changes (#49)

* Replace kzg-related data_blob.go type methods Parse, ComputeCommitment, and
Point, so we can move methods that depend on them into the kzg package.

* Remove ComputeCommitments which is unused.

* Migrate remaining EIP-4844 consensus spec code into kzg_new,
updated to include the latest consensus spec updates.

* Start implementing a bytes API with appropriate interfaces

Also fix incorrect hash type (Keccac -> Sha256) when
computing versioned hash.

* pure bytes API for VerifyKZGProof per 3097, and move kzg_new back into kzg.go now that updates are near complete

* rename verifyBlobs validateBlobTransactionWrapper per updated spec, and make implementation more closely follow it

* fix tests & type bug in kzg verification

* more consensus layer support in kzg package

* remove aggregated proof from blobs bundle

* propagate malformed blob errors when processing a sent transaction so that these get rejected

core/beacon/types.go

@@ -43,10 +43,9 @@ type payloadAttributesMarshaling struct {
 
 // BlobsBundleV1 holds the blobs of an execution payload, to be retrieved separately
 type BlobsBundleV1 struct {
-	BlockHash       common.Hash           `json:"blockHash"     gencodec:"required"`
-	KZGs            []types.KZGCommitment `json:"kzgs"      gencodec:"required"`
-	Blobs           []types.Blob          `json:"blobs"      gencodec:"required"`
-	AggregatedProof types.KZGProof        `json:"aggregatedProof" gencodec:"required"`
+	BlockHash common.Hash           `json:"blockHash"     gencodec:"required"`
+	KZGs      []types.KZGCommitment `json:"kzgs"      gencodec:"required"`
+	Blobs     []types.Blob          `json:"blobs"      gencodec:"required"`
 }
 
 //go:generate go run github.com/fjl/gencodec -type ExecutableDataV1 -field-override executableDataMarshaling -out gen_ed.go
@@ -236,11 +235,5 @@ func BlockToBlobData(block *types.Block) (*BlobsBundleV1, error) {
 			blobsBundle.KZGs = append(blobsBundle.KZGs, kzgs...)
 		}
 	}
-
-	_, _, aggregatedProof, err := types.Blobs(blobsBundle.Blobs).ComputeCommitmentsAndAggregatedProof()
-	if err != nil {
-		return nil, err
-	}
-	blobsBundle.AggregatedProof = aggregatedProof
 	return blobsBundle, nil
 }

core/types/data_blob.go

@@ -10,7 +10,6 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/crypto/kzg"
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/protolambda/go-kzg/bls"
 	"github.com/protolambda/ztyp/codec"
 	"github.com/protolambda/ztyp/tree"
 )
@@ -103,6 +102,7 @@ func (p *KZGProof) UnmarshalText(text []byte) error {
 	return hexutil.UnmarshalFixedText("KZGProof", text, p[:])
 }
 
+// BLSFieldElement is the raw bytes representation of a field element
 type BLSFieldElement [32]byte
 
 func (p BLSFieldElement) MarshalText() ([]byte, error) {
@@ -120,6 +120,16 @@ func (p *BLSFieldElement) UnmarshalText(text []byte) error {
 // Blob data
 type Blob [params.FieldElementsPerBlob]BLSFieldElement
 
+// kzg.Blob interface
+func (blob Blob) Len() int {
+	return len(blob)
+}
+
+// kzg.Blob interface
+func (blob Blob) At(i int) [32]byte {
+	return [32]byte(blob[i])
+}
+
 func (blob *Blob) Deserialize(dr *codec.DecodingReader) error {
 	if blob == nil {
 		return errors.New("cannot decode ssz into nil Blob")
@@ -156,17 +166,6 @@ func (blob *Blob) HashTreeRoot(hFn tree.HashFn) tree.Root {
 	}, params.FieldElementsPerBlob)
 }
 
-// Convert a blob to kzg.Blob
-func (blob *Blob) ToKZGBlob() (kzg.Blob, bool) {
-	frs := make([]bls.Fr, len(blob))
-	for i, elem := range blob {
-		if !bls.FrFrom32(&frs[i], elem) {
-			return []bls.Fr{}, false
-		}
-	}
-	return kzg.Blob(frs), true
-}
-
 func (blob *Blob) MarshalText() ([]byte, error) {
 	out := make([]byte, 2+params.FieldElementsPerBlob*32*2)
 	copy(out[:2], "0x")
@@ -209,6 +208,15 @@ func (blob *Blob) UnmarshalText(text []byte) error {
 
 type BlobKzgs []KZGCommitment
 
+// kzg.KZGCommitmentSequence interface
+func (bk BlobKzgs) Len() int {
+	return len(bk)
+}
+
+func (bk BlobKzgs) At(i int) kzg.KZGCommitment {
+	return kzg.KZGCommitment(bk[i])
+}
+
 func (li *BlobKzgs) Deserialize(dr *codec.DecodingReader) error {
 	return dr.List(func() codec.Deserializable {
 		i := len(*li)
@@ -227,7 +235,7 @@ func (li BlobKzgs) ByteLength() uint64 {
 	return uint64(len(li)) * 48
 }
 
-func (li *BlobKzgs) FixedLength() uint64 {
+func (li BlobKzgs) FixedLength() uint64 {
 	return 0
 }
 
@@ -245,17 +253,14 @@ func (li BlobKzgs) copy() BlobKzgs {
 
 type Blobs []Blob
 
-// Extract the crypto material underlying these blobs
-func (blobs Blobs) toKZGBlobSequence() ([][]bls.Fr, bool) {
-	out := make([][]bls.Fr, len(blobs))
-	for i, b := range blobs {
-		blob, ok := b.ToKZGBlob()
-		if !ok {
-			return nil, false
-		}
-		out[i] = blob
-	}
-	return out, true
+// kzg.BlobSequence interface
+func (blobs Blobs) Len() int {
+	return len(blobs)
+}
+
+// kzg.BlobSequence interface
+func (blobs Blobs) At(i int) kzg.Blob {
+	return blobs[i]
 }
 
 func (a *Blobs) Deserialize(dr *codec.DecodingReader) error {
@@ -301,42 +306,21 @@ func (blobs Blobs) ComputeCommitmentsAndAggregatedProof() (commitments []KZGComm
 	commitments = make([]KZGCommitment, len(blobs))
 	versionedHashes = make([]common.Hash, len(blobs))
 	for i, blob := range blobs {
-		frs, ok := blob.ToKZGBlob()
+		c, ok := kzg.BlobToKZGCommitment(blob)
 		if !ok {
-			return nil, nil, KZGProof{}, errors.New("invalid blob for commitment")
+			return nil, nil, KZGProof{}, errors.New("could not convert blob to commitment")
 		}
-		commitments[i] = KZGCommitment(kzg.BlobToKZGCommitment(frs))
-		versionedHashes[i] = common.Hash(kzg.KZGToVersionedHash(kzg.KZGCommitment(commitments[i])))
+		commitments[i] = KZGCommitment(c)
+		versionedHashes[i] = common.Hash(kzg.KZGToVersionedHash(c))
 	}
 
 	var kzgProof KZGProof
 	if len(blobs) != 0 {
-		aggregatePoly, aggregateCommitmentG1, err := computeAggregateKzgCommitment(blobs, commitments)
+		proof, err := kzg.ComputeAggregateKZGProof(blobs)
 		if err != nil {
 			return nil, nil, KZGProof{}, err
 		}
-
-		var aggregateCommitment KZGCommitment
-		copy(aggregateCommitment[:], bls.ToCompressedG1(aggregateCommitmentG1))
-
-		var aggregateBlob Blob
-		for i := range aggregatePoly {
-			aggregateBlob[i] = bls.FrTo32(&aggregatePoly[i])
-		}
-		sum, err := sszHash(&PolynomialAndCommitment{aggregateBlob, aggregateCommitment})
-		if err != nil {
-			return nil, nil, KZGProof{}, err
-		}
-		z := kzg.BytesToBLSField(sum)
-
-		var y bls.Fr
-		kzg.EvaluatePolyInEvaluationForm(&y, aggregatePoly[:], z)
-
-		aggProofG1, err := kzg.ComputeProof(aggregatePoly, z)
-		if err != nil {
-			return nil, nil, KZGProof{}, err
-		}
-		copy(kzgProof[:], bls.ToCompressedG1(aggProofG1))
+		kzgProof = KZGProof(proof)
 	}
 
 	return commitments, versionedHashes, kzgProof, nil
@@ -363,27 +347,6 @@ func (b *BlobsAndCommitments) FixedLength() uint64 {
 	return 0
 }
 
-type PolynomialAndCommitment struct {
-	b Blob
-	c KZGCommitment
-}
-
-func (p *PolynomialAndCommitment) HashTreeRoot(hFn tree.HashFn) tree.Root {
-	return hFn.HashTreeRoot(&p.b, &p.c)
-}
-
-func (p *PolynomialAndCommitment) Serialize(w *codec.EncodingWriter) error {
-	return w.Container(&p.b, &p.c)
-}
-
-func (p *PolynomialAndCommitment) ByteLength() uint64 {
-	return codec.ContainerLength(&p.b, &p.c)
-}
-
-func (p *PolynomialAndCommitment) FixedLength() uint64 {
-	return 0
-}
-
 type BlobTxWrapper struct {
 	Tx                 SignedBlobTx
 	BlobKzgs           BlobKzgs
@@ -421,19 +384,30 @@ func (b *BlobTxWrapData) sizeWrapData() common.StorageSize {
 	return common.StorageSize(4 + 4 + b.BlobKzgs.ByteLength() + b.Blobs.ByteLength() + b.KzgAggregatedProof.ByteLength())
 }
 
-func (b *BlobTxWrapData) verifyVersionedHash(inner TxData) error {
+// validateBlobTransactionWrapper implements validate_blob_transaction_wrapper from EIP-4844
+func (b *BlobTxWrapData) validateBlobTransactionWrapper(inner TxData) error {
 	blobTx, ok := inner.(*SignedBlobTx)
 	if !ok {
 		return fmt.Errorf("expected signed blob tx, got %T", inner)
 	}
-	if a, b := len(blobTx.Message.BlobVersionedHashes), params.MaxBlobsPerBlock; a > b {
-		return fmt.Errorf("too many blobs in blob tx, got %d, expected no more than %d", a, b)
+	l1 := len(b.BlobKzgs)
+	l2 := len(blobTx.Message.BlobVersionedHashes)
+	l3 := len(b.Blobs)
+	if l1 != l2 || l2 != l3 {
+		return fmt.Errorf("lengths don't match %v %v %v", l1, l2, l3)
 	}
-	if a, b := len(b.BlobKzgs), len(b.Blobs); a != b {
-		return fmt.Errorf("expected equal amount but got %d kzgs and %d blobs", a, b)
+	// the following check isn't strictly necessary as it would be caught by data gas processing
+	// (and hence it is not explicitly in the spec for this function), but it doesn't hurt to fail
+	// early in case we are getting spammed with too many blobs or there is a bug somewhere:
+	if l1 > params.MaxBlobsPerBlock {
+		return fmt.Errorf("number of blobs exceeds max: %v", l1)
 	}
-	if a, b := len(b.BlobKzgs), len(blobTx.Message.BlobVersionedHashes); a != b {
-		return fmt.Errorf("expected equal amount but got %d kzgs and %d versioned hashes", a, b)
+	ok, err := kzg.VerifyAggregateKZGProof(b.Blobs, b.BlobKzgs, kzg.KZGProof(b.KzgAggregatedProof))
+	if err != nil {
+		return fmt.Errorf("error during proof verification: %v", err)
+	}
+	if !ok {
+		return errors.New("failed to verify kzg")
 	}
 	for i, h := range blobTx.Message.BlobVersionedHashes {
 		if computed := b.BlobKzgs[i].ComputeVersionedHash(); computed != h {
@@ -443,41 +417,6 @@ func (b *BlobTxWrapData) verifyVersionedHash(inner TxData) error {
 	return nil
 }
 
-// Blob verification using KZG proofs
-func (b *BlobTxWrapData) verifyBlobs(inner TxData) error {
-	if err := b.verifyVersionedHash(inner); err != nil {
-		return err
-	}
-
-	aggregatePoly, aggregateCommitmentG1, err := computeAggregateKzgCommitment(b.Blobs, b.BlobKzgs)
-	if err != nil {
-		return fmt.Errorf("failed to compute aggregate commitment: %v", err)
-	}
-	var aggregateBlob Blob
-	for i := range aggregatePoly {
-		aggregateBlob[i] = bls.FrTo32(&aggregatePoly[i])
-	}
-	var aggregateCommitment KZGCommitment
-	copy(aggregateCommitment[:], bls.ToCompressedG1(aggregateCommitmentG1))
-	sum, err := sszHash(&PolynomialAndCommitment{aggregateBlob, aggregateCommitment})
-	if err != nil {
-		return err
-	}
-	z := kzg.BytesToBLSField(sum)
-
-	var y bls.Fr
-	kzg.EvaluatePolyInEvaluationForm(&y, aggregatePoly[:], z)
-
-	aggregateProofG1, err := bls.FromCompressedG1(b.KzgAggregatedProof[:])
-	if err != nil {
-		return fmt.Errorf("aggregate proof parse error: %v", err)
-	}
-	if !kzg.VerifyKZGProofFromPoints(aggregateCommitmentG1, z, &y, aggregateProofG1) {
-		return errors.New("failed to verify kzg")
-	}
-	return nil
-}
-
 func (b *BlobTxWrapData) copy() TxWrapData {
 	return &BlobTxWrapData{
 		BlobKzgs:           b.BlobKzgs.copy(),
@@ -514,33 +453,3 @@ func (b *BlobTxWrapData) encodeTyped(w io.Writer, txdata TxData) error {
 	}
 	return EncodeSSZ(w, &wrapped)
 }
-
-func computeAggregateKzgCommitment(blobs Blobs, commitments []KZGCommitment) ([]bls.Fr, *bls.G1Point, error) {
-	// create challenges
-	sum, err := sszHash(&BlobsAndCommitments{blobs, commitments})
-	if err != nil {
-		return nil, nil, err
-	}
-	r := kzg.BytesToBLSField(sum)
-
-	powers := kzg.ComputePowers(r, len(blobs))
-
-	commitmentsG1 := make([]bls.G1Point, len(commitments))
-	for i := 0; i < len(commitmentsG1); i++ {
-		p, _ := bls.FromCompressedG1(commitments[i][:])
-		bls.CopyG1(&commitmentsG1[i], p)
-	}
-	aggregateCommitmentG1 := bls.LinCombG1(commitmentsG1, powers)
-	var aggregateCommitment KZGCommitment
-	copy(aggregateCommitment[:], bls.ToCompressedG1(aggregateCommitmentG1))
-
-	polys, ok := blobs.toKZGBlobSequence()
-	if !ok {
-		return nil, nil, err
-	}
-	aggregatePoly, err := bls.PolyLinComb(polys, powers)
-	if err != nil {
-		return nil, nil, err
-	}
-	return aggregatePoly, aggregateCommitmentG1, nil
-}

core/types/transaction.go

@@ -98,7 +98,7 @@ type TxWrapData interface {
 	aggregatedProof() KZGProof
 	encodeTyped(w io.Writer, txdata TxData) error
 	sizeWrapData() common.StorageSize
-	verifyBlobs(inner TxData) error
+	validateBlobTransactionWrapper(inner TxData) error
 }
 
 // TxData is the underlying data of a transaction.
@@ -543,7 +543,7 @@ func (tx *Transaction) IsIncomplete() bool {
 // VerifyBlobs verifies the blob transaction
 func (tx *Transaction) VerifyBlobs() error {
 	if tx.wrapData != nil {
-		return tx.wrapData.verifyBlobs(tx.inner)
+		return tx.wrapData.validateBlobTransactionWrapper(tx.inner)
 	}
 	return nil
 }

core/types/transaction_marshalling.go

@@ -362,7 +362,7 @@ func (tx *Transaction) UnmarshalJSON(input []byte) error {
 				KzgAggregatedProof: dec.KzgAggregatedProof,
 			}
 			// Verify that versioned hashes match kzgs, and kzgs match blobs.
-			if err := tx.wrapData.verifyBlobs(&itx); err != nil {
+			if err := tx.wrapData.validateBlobTransactionWrapper(&itx); err != nil {
 				return fmt.Errorf("blob wrapping data is invalid: %v", err)
 			}
 		}

core/types/transaction_test.go

@@ -490,7 +490,7 @@ func TestTransactionCoding(t *testing.T) {
 					GasTipCap:           view.Uint256View(*uint256.NewInt(42)),
 					GasFeeCap:           view.Uint256View(*uint256.NewInt(10)),
 					AccessList:          AccessListView(accesses),
-					BlobVersionedHashes: VersionedHashesView{common.HexToHash("0x01624652859a6e98ffc1608e2af0147ca4e86e1ce27672d8d3f3c9d4ffd6ef7e")},
+					BlobVersionedHashes: VersionedHashesView{common.HexToHash("0x010657f37554c781402a22917dee2f75def7ab966d7b770905398eba3c444014")},
 				},
 			}
 			var kzgProof KZGProof

core/vm/testdata/precompiles/pointEvaluation.json

@@ -1,6 +1,6 @@
 [
   {
-    "Input": "01342233e6ebb423c766d3a0f8d183e84c453865b392f5ab1f8a8218506e89d842000000000000000000000000000000000000000000000000000000000000002b2f0b0a19cbe19b4c9dbc32af755539fec08bae3eeecbe0ec625037fe3f0a6fa3cfbde6cf9875270479e0e2290726d150412591e07b4fad36472fa1ad38c19eb232cd2ebd3738ea1d9a0a3be07764a8b2faf3776cf5fb7bea8263ab92181326b898c4dc5da95e76e6977c4e204a94f1a3fe5033e19435fa51a8c70b272c06ac",
+      "Input": "01d0db71b458e8955efa3ef62f1b6b45a2d9c8633dc59ed0b995cecf8b7bb48442000000000000000000000000000000000000000000000000000000000000003b11ebd59d5d12d6ef8e5e48f71770515e032595a0e55eaf80e906b65b2a625282bead0f31f58ee4fd81e93f796bb57acd6f8f6e4def04182c8e949c71ea00d85a44c12102bd817bc97696a6b8fd75618fcd8c2030080c9602e08e935cdf6f779d0d89e7764d855edfbaa730eddfff836fc324957db4f74d1565503bcfcaf157",
     "Expected": "",
     "Name": "pointEvaluation1",
     "Gas": 50000,