From d6bb142cb372f6d94b2e7748181da8a33a2f44bf Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sun, 1 Jul 2018 22:35:46 -0700 Subject: [PATCH] VerifyItem takes no index; Return keys/values in range; Fix 0.9.1 -- compute hash from rangeproof Require Verify() before Verify*() Review fixes from #75 First commit for generalized merkle proofs Fix imports Bump version to 0.9.3 Return nil on empty tree - bump version to 0.10.0 Add OpDecoders fix test finalize rebase fix test --- CHANGELOG.md | 6 +++ Gopkg.toml | 1 + basic_test.go | 16 ++++++-- proof.go | 3 -- proof_iavl_absence.go | 87 +++++++++++++++++++++++++++++++++++++++++++ proof_iavl_value.go | 86 ++++++++++++++++++++++++++++++++++++++++++ proof_range.go | 48 ++++++++++++------------ proof_test.go | 9 +++-- 8 files changed, 222 insertions(+), 34 deletions(-) create mode 100644 proof_iavl_absence.go create mode 100644 proof_iavl_value.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 46e3f5ca4..8e7e5f382 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,12 @@ IMPROVEMENTS - Change tendermint dep to ^v0.22.0 (#91) +## 0.10.0 (July 11, 2018) + +BREAKING CHANGES + +- getRangeProof and Get\[Versioned\]\[Range\]WithProof return nil proof/error if tree is empty. + ## 0.9.2 (July 3, 2018) IMPROVEMENTS diff --git a/Gopkg.toml b/Gopkg.toml index 39510f840..82bcbd6bc 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -5,6 +5,7 @@ [[constraint]] version = "=0.10.1" name = "github.com/tendermint/go-amino" + version = "0.10.1" [[constraint]] version = "^0.22.0" diff --git a/basic_test.go b/basic_test.go index 07e4cb9fa..8681cf138 100644 --- a/basic_test.go +++ b/basic_test.go @@ -423,10 +423,14 @@ func TestProof(t *testing.T) { func TestTreeProof(t *testing.T) { db := db.NewMemDB() tree := NewMutableTree(db, 100) + assert.Equal(t, tree.Hash(), []byte(nil)) // should get false for proof with nil root - _, _, err := tree.GetWithProof([]byte("foo")) - assert.Error(t, err) + value, proof, err := tree.GetWithProof([]byte("foo")) + assert.Nil(t, value) + assert.Nil(t, proof) + assert.Error(t, proof.Verify([]byte(nil))) + assert.NoError(t, err) // insert lots of info and store the bytes keys := make([][]byte, 200) @@ -436,9 +440,15 @@ func TestTreeProof(t *testing.T) { keys[i] = []byte(key) } + tree.SaveVersion() + // query random key fails - _, _, err = tree.GetWithProof([]byte("foo")) + value, proof, err = tree.GetWithProof([]byte("foo")) + assert.Nil(t, value) + assert.NotNil(t, proof) assert.NoError(t, err) + assert.NoError(t, proof.Verify(tree.Hash())) + assert.NoError(t, proof.VerifyAbsence([]byte("foo"))) // valid proof for real keys root := tree.WorkingHash() diff --git a/proof.go b/proof.go index b68293ccc..e3c0d0c4d 100644 --- a/proof.go +++ b/proof.go @@ -18,9 +18,6 @@ var ( // ErrInvalidRoot is returned when the root passed in does not match the proof's. ErrInvalidRoot = fmt.Errorf("invalid root") - - // ErrNilRoot is returned when the root of the tree is nil. - ErrNilRoot = fmt.Errorf("tree root is nil") ) //---------------------------------------- diff --git a/proof_iavl_absence.go b/proof_iavl_absence.go new file mode 100644 index 000000000..51edd5a5d --- /dev/null +++ b/proof_iavl_absence.go @@ -0,0 +1,87 @@ +package iavl + +import ( + "fmt" + + "github.com/tendermint/tendermint/crypto/merkle" + cmn "github.com/tendermint/tmlibs/common" +) + +const ProofOpIAVLAbsence = "iavl:a" + +// IAVLAbsenceOp takes a key as its only argument +// +// If the produced root hash matches the expected hash, the proof +// is good. +type IAVLAbsenceOp struct { + // Encoded in ProofOp.Key. + key []byte + + // To encode in ProofOp.Data. + // Proof is nil for an empty tree. + // The hash of an empty tree is nil. + Proof *RangeProof `json:"proof"` +} + +var _ merkle.ProofOperator = IAVLAbsenceOp{} + +func NewIAVLAbsenceOp(key []byte, proof *RangeProof) IAVLAbsenceOp { + return IAVLAbsenceOp{ + key: key, + Proof: proof, + } +} + +func IAVLAbsenceOpDecoder(pop merkle.ProofOp) (merkle.ProofOperator, error) { + if pop.Type != ProofOpIAVLAbsence { + return nil, cmn.NewError("unexpected ProofOp.Type; got %v, want %v", pop.Type, ProofOpIAVLAbsence) + } + var op IAVLAbsenceOp // a bit strange as we'll discard this, but it works. + err := cdc.UnmarshalBinary(pop.Data, &op) + if err != nil { + return nil, cmn.ErrorWrap(err, "decoding ProofOp.Data into IAVLAbsenceOp") + } + return NewIAVLAbsenceOp(pop.Key, op.Proof), nil +} + +func (op IAVLAbsenceOp) ProofOp() merkle.ProofOp { + bz := cdc.MustMarshalBinary(op) + return merkle.ProofOp{ + Type: ProofOpIAVLAbsence, + Key: op.key, + Data: bz, + } +} + +func (op IAVLAbsenceOp) String() string { + return fmt.Sprintf("IAVLAbsenceOp{%v}", op.GetKey()) +} + +func (op IAVLAbsenceOp) Run(args [][]byte) ([][]byte, error) { + if len(args) != 0 { + return nil, cmn.NewError("expected 0 args, got %v", len(args)) + } + // If the tree is nil, the proof is nil, and all keys are absent. + if op.Proof == nil { + return [][]byte{[]byte(nil)}, nil + } + // Compute the root hash and assume it is valid. + // The caller checks the ultimate root later. + root := op.Proof.ComputeRootHash() + err := op.Proof.Verify(root) + if err != nil { + return nil, cmn.ErrorWrap(err, "computing root hash") + } + // XXX What is the encoding for keys? + // We should decode the key depending on whether it's a string or hex, + // maybe based on quotes and 0x prefix? + err = op.Proof.VerifyAbsence([]byte(op.key)) + if err != nil { + return nil, cmn.ErrorWrap(err, "verifying absence") + } + return [][]byte{root}, nil +} + +func (op IAVLAbsenceOp) GetKey() []byte { + return op.key +} diff --git a/proof_iavl_value.go b/proof_iavl_value.go new file mode 100644 index 000000000..a2cbf122d --- /dev/null +++ b/proof_iavl_value.go @@ -0,0 +1,86 @@ +package iavl + +import ( + "fmt" + + "github.com/tendermint/tendermint/crypto/merkle" + cmn "github.com/tendermint/tmlibs/common" +) + +const ProofOpIAVLValue = "iavl:v" + +// IAVLValueOp takes a key and a single value as argument and +// produces the root hash. +// +// If the produced root hash matches the expected hash, the proof +// is good. +type IAVLValueOp struct { + // Encoded in ProofOp.Key. + key []byte + + // To encode in ProofOp.Data. + // Proof is nil for an empty tree. + // The hash of an empty tree is nil. + Proof *RangeProof `json:"proof"` +} + +var _ merkle.ProofOperator = IAVLValueOp{} + +func NewIAVLValueOp(key []byte, proof *RangeProof) IAVLValueOp { + return IAVLValueOp{ + key: key, + Proof: proof, + } +} + +func IAVLValueOpDecoder(pop merkle.ProofOp) (merkle.ProofOperator, error) { + if pop.Type != ProofOpIAVLValue { + return nil, cmn.NewError("unexpected ProofOp.Type; got %v, want %v", pop.Type, ProofOpIAVLValue) + } + var op IAVLValueOp // a bit strange as we'll discard this, but it works. + err := cdc.UnmarshalBinary(pop.Data, &op) + if err != nil { + return nil, cmn.ErrorWrap(err, "decoding ProofOp.Data into IAVLValueOp") + } + return NewIAVLValueOp(pop.Key, op.Proof), nil +} + +func (op IAVLValueOp) ProofOp() merkle.ProofOp { + bz := cdc.MustMarshalBinary(op) + return merkle.ProofOp{ + Type: ProofOpIAVLValue, + Key: op.key, + Data: bz, + } +} + +func (op IAVLValueOp) String() string { + return fmt.Sprintf("IAVLValueOp{%v}", op.GetKey()) +} + +func (op IAVLValueOp) Run(args [][]byte) ([][]byte, error) { + if len(args) != 1 { + return nil, cmn.NewError("Value size is not 1") + } + value := args[0] + + // Compute the root hash and assume it is valid. + // The caller checks the ultimate root later. + root := op.Proof.ComputeRootHash() + err := op.Proof.Verify(root) + if err != nil { + return nil, cmn.ErrorWrap(err, "computing root hash") + } + // XXX What is the encoding for keys? + // We should decode the key depending on whether it's a string or hex, + // maybe based on quotes and 0x prefix? + err = op.Proof.VerifyItem([]byte(op.key), value) + if err != nil { + return nil, cmn.ErrorWrap(err, "verifying value") + } + return [][]byte{root}, nil +} + +func (op IAVLValueOp) GetKey() []byte { + return op.key +} diff --git a/proof_range.go b/proof_range.go index ec438515e..4b5749d28 100644 --- a/proof_range.go +++ b/proof_range.go @@ -7,7 +7,7 @@ import ( "strings" "github.com/tendermint/tendermint/crypto/tmhash" - cmn "github.com/tendermint/tendermint/libs/common" + cmn "github.com/tendermint/tmlibs/common" ) type RangeProof struct { @@ -126,8 +126,9 @@ func (proof *RangeProof) VerifyAbsence(key []byte) error { if cmp < 0 { if proof.LeftPath.isLeftmost() { return nil + } else { + return cmn.NewError("absence not proved by left path") } - return cmn.NewError("absence not proved by left path") } else if cmp == 0 { return cmn.NewError("absence disproved via first item #0") } @@ -149,7 +150,7 @@ func (proof *RangeProof) VerifyAbsence(key []byte) error { } else { if i == len(proof.Leaves)-1 { // If last item, check whether - // it's the last item in the tree. + // it's the last item in teh tree. } continue @@ -164,8 +165,9 @@ func (proof *RangeProof) VerifyAbsence(key []byte) error { // It's not a valid absence proof. if len(proof.Leaves) < 2 { return cmn.NewError("absence not proved by right leaf (need another leaf?)") + } else { + return cmn.NewError("absence not proved by right leaf") } - return cmn.NewError("absence not proved by right leaf") } // Verify that proof is valid. @@ -177,7 +179,7 @@ func (proof *RangeProof) Verify(root []byte) error { return err } -func (proof *RangeProof) verify(root []byte) error { +func (proof *RangeProof) verify(root []byte) (err error) { rootHash := proof.rootHash if rootHash == nil { derivedHash, err := proof.computeRootHash() @@ -188,8 +190,9 @@ func (proof *RangeProof) verify(root []byte) error { } if !bytes.Equal(rootHash, root) { return cmn.ErrorWrap(ErrInvalidRoot, "root hash doesn't match") + } else { + proof.rootVerified = true } - proof.rootVerified = true return nil } @@ -302,13 +305,12 @@ func (proof *RangeProof) _computeRootHash() (rootHash []byte, treeEnd bool, err /////////////////////////////////////////////////////////////////////////////// // keyStart is inclusive and keyEnd is exclusive. -// Returns the range-proof and the included keys and values. // If keyStart or keyEnd don't exist, the leaf before keyStart // or after keyEnd will also be included, but not be included in values. // If keyEnd-1 exists, no later leaves will be included. // If keyStart >= keyEnd and both not nil, panics. // Limit is never exceeded. -func (t *ImmutableTree) getRangeProof(keyStart, keyEnd []byte, limit int) (*RangeProof, [][]byte, [][]byte, error) { +func (t *ImmutableTree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangeProof, keys, values [][]byte, err error) { if keyStart != nil && keyEnd != nil && bytes.Compare(keyStart, keyEnd) >= 0 { panic("if keyStart and keyEnd are present, need keyStart < keyEnd.") } @@ -316,7 +318,7 @@ func (t *ImmutableTree) getRangeProof(keyStart, keyEnd []byte, limit int) (*Rang panic("limit must be greater or equal to 0 -- 0 means no limit") } if t.root == nil { - return nil, nil, nil, cmn.ErrorWrap(ErrNilRoot, "") + return nil, nil, nil, nil } t.root.hashWithCount() // Ensure that all hashes are calculated. @@ -325,18 +327,17 @@ func (t *ImmutableTree) getRangeProof(keyStart, keyEnd []byte, limit int) (*Rang if err != nil { // Key doesn't exist, but instead we got the prev leaf (or the // first or last leaf), which provides proof of absence). - // err = nil isn't necessary as we do not use it in the returns below + err = nil } startOK := keyStart == nil || bytes.Compare(keyStart, left.key) <= 0 endOK := keyEnd == nil || bytes.Compare(left.key, keyEnd) < 0 // If left.key is in range, add it to key/values. - var keys, values [][]byte if startOK && endOK { keys = append(keys, left.key) // == keyStart values = append(values, left.value) } // Either way, add to proof leaves. - var leaves = []proofLeafNode{{ + var leaves = []proofLeafNode{proofLeafNode{ Key: left.key, ValueHash: tmhash.Sum(left.value), Version: left.version, @@ -362,7 +363,6 @@ func (t *ImmutableTree) getRangeProof(keyStart, keyEnd []byte, limit int) (*Rang // Traverse starting from afterLeft, until keyEnd or the next leaf // after keyEnd. - // nolint var innersq = []PathToLeaf(nil) var inners = PathToLeaf(nil) var leafCount = 1 // from left above. @@ -385,9 +385,9 @@ func (t *ImmutableTree) getRangeProof(keyStart, keyEnd []byte, limit int) (*Rang pn.Right != nil && !bytes.Equal(pn.Right, node.rightHash) { // We've diverged, so start appending to inners. - pathCount-- + pathCount = -1 } else { - pathCount++ + pathCount += 1 } } } @@ -403,7 +403,7 @@ func (t *ImmutableTree) getRangeProof(keyStart, keyEnd []byte, limit int) (*Rang ValueHash: tmhash.Sum(node.value), Version: node.version, }) - leafCount++ + leafCount += 1 // Maybe terminate because we found enough leaves. if limit > 0 && limit <= leafCount { return true @@ -451,16 +451,13 @@ func (t *ImmutableTree) getRangeProof(keyStart, keyEnd []byte, limit int) (*Rang // A proof of existence or absence is returned alongside the value. func (t *ImmutableTree) GetWithProof(key []byte) (value []byte, proof *RangeProof, err error) { proof, _, values, err := t.getRangeProof(key, cpIncr(key), 2) - if err == nil { - if len(values) > 0 { - if !bytes.Equal(proof.Leaves[0].Key, key) { - return nil, proof, nil - } - return values[0], proof, nil - } - return nil, proof, nil + if err != nil { + return nil, nil, cmn.ErrorWrap(err, "constructing range proof") + } + if len(values) > 0 && bytes.Equal(proof.Leaves[0].Key, key) { + return values[0], proof, nil } - return nil, nil, cmn.ErrorWrap(err, "could not construct any proof") + return nil, proof, nil } // GetRangeWithProof gets key/value pairs within the specified range and limit. @@ -477,6 +474,7 @@ func (tree *MutableTree) GetVersionedWithProof(key []byte, version int64) ([]byt if err != nil { return nil, nil, err } + return t.GetWithProof(key) } return nil, nil, cmn.ErrorWrap(ErrVersionDoesNotExist, "") diff --git a/proof_test.go b/proof_test.go index b408a0d9f..181abba22 100644 --- a/proof_test.go +++ b/proof_test.go @@ -51,9 +51,12 @@ func TestTreeKeyExistsProof(t *testing.T) { root := tree.WorkingHash() // should get false for proof with nil root - proof, _, _, err := tree.getRangeProof([]byte("foo"), nil, 1) - assert.NotNil(t, err) - assert.NotNil(t, proof.Verify(root)) + proof, keys, values, err := tree.getRangeProof([]byte("foo"), nil, 1) + assert.Nil(t, proof) + assert.Error(t, proof.Verify(root)) + assert.Nil(t, keys) + assert.Nil(t, values) + assert.NoError(t, err) // insert lots of info and store the bytes allkeys := make([][]byte, 200)