Skip to content

Commit

Permalink
refactor(block): CIDv1 and BlockPutSettings CidPrefix (#80)
Browse files Browse the repository at this point in the history 
* feat(block options): add store codec
* refactor: BlockPutSettings.CidPrefix

Removes duplicated fields and replaces them with cid.Prefix

Codec, MhType and MhLength were  already in prefix, and we already return
prefix. A lot of duplicated values and code responsible for syncing them
did not really need to exist.

* test: CIDv1 raw and dag-pb cases
* chore: release 0.7.0

Co-authored-by: Marcin Rataj <[email protected]>

This commit was moved from ipfs/interface-go-ipfs-core@a3374d9

This commit was moved from ipfs/boxo@aca3a18
  • Loading branch information
@schomatis
schomatis authored Apr 21, 2022
1 parent 9120e3f commit caa42b5
Show file tree
Hide file tree
Showing 2 changed files with 181 additions and 56 deletions.
134 changes: 86 additions & 48 deletions core/coreiface/options/block.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@ package options

import (
"fmt"

cid "github.com/ipfs/go-cid"
mc "github.com/multiformats/go-multicodec"
mh "github.com/multiformats/go-multihash"
)

type BlockPutSettings struct {
Codec string
MhType uint64
MhLength int
Pin bool
CidPrefix cid.Prefix
Pin bool
}

type BlockRmSettings struct {
Expand All @@ -20,53 +20,29 @@ type BlockRmSettings struct {
type BlockPutOption func(*BlockPutSettings) error
type BlockRmOption func(*BlockRmSettings) error

func BlockPutOptions(opts ...BlockPutOption) (*BlockPutSettings, cid.Prefix, error) {
func BlockPutOptions(opts ...BlockPutOption) (*BlockPutSettings, error) {
var cidPrefix cid.Prefix

// Baseline is CIDv1 raw sha2-255-32 (can be tweaked later via opts)
cidPrefix.Version = 1
cidPrefix.Codec = uint64(mc.Raw)
cidPrefix.MhType = mh.SHA2_256
cidPrefix.MhLength = -1 // -1 means len is to be calculated during mh.Sum()

options := &BlockPutSettings{
Codec: "",
MhType: mh.SHA2_256,
MhLength: -1,
Pin: false,
CidPrefix: cidPrefix,
Pin: false,
}

// Apply any overrides
for _, opt := range opts {
err := opt(options)
if err != nil {
return nil, cid.Prefix{}, err
}
}

var pref cid.Prefix
pref.Version = 1

if options.Codec == "" {
if options.MhType != mh.SHA2_256 || (options.MhLength != -1 && options.MhLength != 32) {
options.Codec = "protobuf"
} else {
options.Codec = "v0"
}
}

if options.Codec == "v0" && options.MhType == mh.SHA2_256 {
pref.Version = 0
}

formatval, ok := cid.Codecs[options.Codec]
if !ok {
return nil, cid.Prefix{}, fmt.Errorf("unrecognized format: %s", options.Codec)
}

if options.Codec == "v0" {
if options.MhType != mh.SHA2_256 || (options.MhLength != -1 && options.MhLength != 32) {
return nil, cid.Prefix{}, fmt.Errorf("only sha2-255-32 is allowed with CIDv0")
return nil, err
}
}

pref.Codec = formatval

pref.MhType = options.MhType
pref.MhLength = options.MhLength

return options, pref, nil
return options, nil
}

func BlockRmOptions(opts ...BlockRmOption) (*BlockRmSettings, error) {
Expand All @@ -87,22 +63,84 @@ type blockOpts struct{}

var Block blockOpts

// Format is an option for Block.Put which specifies the multicodec to use to
// serialize the object. Default is "v0"
func (blockOpts) Format(codec string) BlockPutOption {
// CidCodec is the modern option for Block.Put which specifies the multicodec to use
// in the CID returned by the Block.Put operation.
// It uses correct codes from go-multicodec and replaces the old Format now with CIDv1 as the default.
func (blockOpts) CidCodec(codecName string) BlockPutOption {
return func(settings *BlockPutSettings) error {
if codecName == "" {
return nil
}
code, err := codeFromName(codecName)
if err != nil {
return err
}
settings.CidPrefix.Codec = uint64(code)
return nil
}
}

// Map string to code from go-multicodec
func codeFromName(codecName string) (mc.Code, error) {
var cidCodec mc.Code
err := cidCodec.Set(codecName)
return cidCodec, err
}

// Format is a legacy option for Block.Put which specifies the multicodec to
// use to serialize the object.
// Provided for backward-compatibility only. Use CidCodec instead.
func (blockOpts) Format(format string) BlockPutOption {
return func(settings *BlockPutSettings) error {
settings.Codec = codec
if format == "" {
return nil
}
// Opt-in CIDv0 support for backward-compatibility
if format == "v0" {
settings.CidPrefix.Version = 0
}

// Fixup a legacy (invalid) names for dag-pb (0x70)
if format == "v0" || format == "protobuf" {
format = "dag-pb"
}

// Fixup invalid name for dag-cbor (0x71)
if format == "cbor" {
format = "dag-cbor"
}

// Set code based on name passed as "format"
code, err := codeFromName(format)
if err != nil {
return err
}
settings.CidPrefix.Codec = uint64(code)

// If CIDv0, ensure all parameters are compatible
// (in theory go-cid would validate this anyway, but we want to provide better errors)
pref := settings.CidPrefix
if pref.Version == 0 {
if pref.Codec != uint64(mc.DagPb) {
return fmt.Errorf("only dag-pb is allowed with CIDv0")
}
if pref.MhType != mh.SHA2_256 || (pref.MhLength != -1 && pref.MhLength != 32) {
return fmt.Errorf("only sha2-255-32 is allowed with CIDv0")
}
}

return nil
}

}

// Hash is an option for Block.Put which specifies the multihash settings to use
// when hashing the object. Default is mh.SHA2_256 (0x12).
// If mhLen is set to -1, default length for the hash will be used
func (blockOpts) Hash(mhType uint64, mhLen int) BlockPutOption {
return func(settings *BlockPutSettings) error {
settings.MhType = mhType
settings.MhLength = mhLen
settings.CidPrefix.MhType = mhType
settings.CidPrefix.MhLength = mhLen
return nil
}
}
Expand Down
103 changes: 95 additions & 8 deletions core/coreiface/tests/block.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,19 @@ import (
)

var (
pbCid = "QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN"
cborCid = "bafyreicnga62zhxnmnlt6ymq5hcbsg7gdhqdu6z4ehu3wpjhvqnflfy6nm"
cborKCid = "bafyr2qgsohbwdlk7ajmmbb4lhoytmest4wdbe5xnexfvtxeatuyqqmwv3fgxp3pmhpc27gwey2cct56gloqefoqwcf3yqiqzsaqb7p4jefhcw"
pbCidV0 = "QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN" // dag-pb
pbCid = "bafybeiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4" // dag-pb
rawCid = "bafkreiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4" // raw bytes
cborCid = "bafyreicnga62zhxnmnlt6ymq5hcbsg7gdhqdu6z4ehu3wpjhvqnflfy6nm" // dag-cbor
cborKCid = "bafyr2qgsohbwdlk7ajmmbb4lhoytmest4wdbe5xnexfvtxeatuyqqmwv3fgxp3pmhpc27gwey2cct56gloqefoqwcf3yqiqzsaqb7p4jefhcw" // dag-cbor keccak-512
)

// dag-pb
func pbBlock() io.Reader {
return bytes.NewReader([]byte{10, 12, 8, 2, 18, 6, 104, 101, 108, 108, 111, 10, 24, 6})
}

// dag-cbor
func cborBlock() io.Reader {
return bytes.NewReader([]byte{101, 72, 101, 108, 108, 111})
}
Expand All @@ -38,15 +42,20 @@ func (tp *TestSuite) TestBlock(t *testing.T) {
return nil
})

t.Run("TestBlockPut", tp.TestBlockPut)
t.Run("TestBlockPutFormat", tp.TestBlockPutFormat)
t.Run("TestBlockPut (get raw CIDv1)", tp.TestBlockPut)
t.Run("TestBlockPutCidCodec: dag-pb", tp.TestBlockPutCidCodecDagPb)
t.Run("TestBlockPutCidCodec: dag-cbor", tp.TestBlockPutCidCodecDagCbor)
t.Run("TestBlockPutFormat (legacy): cbor → dag-cbor", tp.TestBlockPutFormatDagCbor)
t.Run("TestBlockPutFormat (legacy): protobuf → dag-pb", tp.TestBlockPutFormatDagPb)
t.Run("TestBlockPutFormat (legacy): v0 → CIDv0", tp.TestBlockPutFormatV0)
t.Run("TestBlockPutHash", tp.TestBlockPutHash)
t.Run("TestBlockGet", tp.TestBlockGet)
t.Run("TestBlockRm", tp.TestBlockRm)
t.Run("TestBlockStat", tp.TestBlockStat)
t.Run("TestBlockPin", tp.TestBlockPin)
}

// when no opts are passed, produced CID has 'raw' codec
func (tp *TestSuite) TestBlockPut(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
Expand All @@ -60,12 +69,14 @@ func (tp *TestSuite) TestBlockPut(t *testing.T) {
t.Fatal(err)
}

if res.Path().Cid().String() != pbCid {
if res.Path().Cid().String() != rawCid {
t.Errorf("got wrong cid: %s", res.Path().Cid().String())
}
}

func (tp *TestSuite) TestBlockPutFormat(t *testing.T) {
// Format is deprecated, it used invalid codec names.
// Confirm 'cbor' gets fixed to 'dag-cbor'
func (tp *TestSuite) TestBlockPutFormatDagCbor(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
api, err := tp.makeAPI(ctx)
Expand All @@ -83,6 +94,82 @@ func (tp *TestSuite) TestBlockPutFormat(t *testing.T) {
}
}

// Format is deprecated, it used invalid codec names.
// Confirm 'protobuf' got fixed to 'dag-pb'
func (tp *TestSuite) TestBlockPutFormatDagPb(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
api, err := tp.makeAPI(ctx)
if err != nil {
t.Fatal(err)
}

res, err := api.Block().Put(ctx, pbBlock(), opt.Block.Format("protobuf"))
if err != nil {
t.Fatal(err)
}

if res.Path().Cid().String() != pbCid {
t.Errorf("got wrong cid: %s", res.Path().Cid().String())
}
}

// Format is deprecated, it used invalid codec names.
// Confirm fake codec 'v0' got fixed to CIDv0 (with implicit dag-pb codec)
func (tp *TestSuite) TestBlockPutFormatV0(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
api, err := tp.makeAPI(ctx)
if err != nil {
t.Fatal(err)
}

res, err := api.Block().Put(ctx, pbBlock(), opt.Block.Format("v0"))
if err != nil {
t.Fatal(err)
}

if res.Path().Cid().String() != pbCidV0 {
t.Errorf("got wrong cid: %s", res.Path().Cid().String())
}
}

func (tp *TestSuite) TestBlockPutCidCodecDagCbor(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
api, err := tp.makeAPI(ctx)
if err != nil {
t.Fatal(err)
}

res, err := api.Block().Put(ctx, cborBlock(), opt.Block.CidCodec("dag-cbor"))
if err != nil {
t.Fatal(err)
}

if res.Path().Cid().String() != cborCid {
t.Errorf("got wrong cid: %s", res.Path().Cid().String())
}
}

func (tp *TestSuite) TestBlockPutCidCodecDagPb(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
api, err := tp.makeAPI(ctx)
if err != nil {
t.Fatal(err)
}

res, err := api.Block().Put(ctx, pbBlock(), opt.Block.CidCodec("dag-pb"))
if err != nil {
t.Fatal(err)
}

if res.Path().Cid().String() != pbCid {
t.Errorf("got wrong cid: %s", res.Path().Cid().String())
}
}

func (tp *TestSuite) TestBlockPutHash(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
Expand All @@ -95,7 +182,7 @@ func (tp *TestSuite) TestBlockPutHash(t *testing.T) {
ctx,
cborBlock(),
opt.Block.Hash(mh.KECCAK_512, -1),
opt.Block.Format("cbor"),
opt.Block.CidCodec("dag-cbor"),
)
if err != nil {
t.Fatal(err)
Expand Down

0 comments on commit caa42b5

Please sign in to comment.