From d477ec23f96ccbb27ed09370ea8df866b61f8d0c Mon Sep 17 00:00:00 2001 From: Tristan Willy Date: Thu, 23 Feb 2023 10:14:50 -0800 Subject: [PATCH] More pkg/manifest documentation (#1854) --- private/pkg/manifest/digest.go | 1 + private/pkg/manifest/manifest.go | 32 +++++++++++++++++++++++++------- private/pkg/manifest/module.go | 4 ++-- private/pkg/manifest/storage.go | 4 ++-- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/private/pkg/manifest/digest.go b/private/pkg/manifest/digest.go index 93099a9f80..25522d543e 100644 --- a/private/pkg/manifest/digest.go +++ b/private/pkg/manifest/digest.go @@ -115,6 +115,7 @@ type shake256Digester struct { hash sha3.ShakeHash } +// NewDigester returns a digester of the requested type. func NewDigester(dtype DigestType) (Digester, error) { if dtype != DigestTypeShake256 { return nil, fmt.Errorf("not supported digest type %q", dtype) diff --git a/private/pkg/manifest/manifest.go b/private/pkg/manifest/manifest.go index f5725e0e64..1767201474 100644 --- a/private/pkg/manifest/manifest.go +++ b/private/pkg/manifest/manifest.go @@ -12,14 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Manifests are a list of paths and their hash digests, canonically ordered by -// path in increasing lexographical order. Manifests are encoded as: +// A manifest is a file containing a list of paths and their hash digests, +// canonically ordered by path in increasing lexicographical order. Manifests +// are encoded as: // // :[SP][SP][LF] // // "shake256" is the only supported digest type. The digest is 64 bytes of hex // encoded output of SHAKE256. See golang.org/x/crypto/sha3 and FIPS 202 for // details on the SHAKE hash. +// +// [Manifest] can read and write manifest files. Canonical form is produced +// when serialized ([Manifest.MarshalText]). Non-canonical form is a valid +// manifest and will not produce errors when deserializing. +// +// Interacting with a manifest is typically by path ([Manifest.Paths], +// [Manifest.DigestFor]) or by a [Digest] ([Manifest.PathsFor]). +// +// [Blob] represents file content and its digest. [BlobSet] collects related +// blobs together into a set. [NewMemoryBlob] provides an in-memory +// implementation. A manifest, being a file, is also a blob ([Manifest.Blob]). +// +// Blobs are anonymous files and a manifest gives names to anonymous files. +// It's possible to view a manifest and its associated blobs as a file system. +// [NewBucket] creates a storage bucket from a manifest and blob set. +// [NewFromBucket] does the inverse: the creation of a manifest and blob set +// from a storage bucket. package manifest import ( @@ -60,7 +78,8 @@ func New() *Manifest { } } -// NewFromReader builds a manifest from an encoded manifest reader. +// NewFromReader builds a manifest from an encoded manifest, like one produced +// by [Manifest.MarshalText]. func NewFromReader(manifest io.Reader) (*Manifest, error) { m := New() scanner := bufio.NewScanner(manifest) @@ -157,10 +176,9 @@ func (m *Manifest) MarshalText() ([]byte, error) { return coded.Bytes(), nil } -// UnmarshalText decodes a manifest from member. +// UnmarshalText decodes a manifest from text. // -// Use NewManifestFromReader if you have an io.Reader and want to avoid memory -// copying. +// See [NewFromReader] if your manifest is available in an io.Reader. func (m *Manifest) UnmarshalText(text []byte) error { newm, err := NewFromReader(bytes.NewReader(text)) if err != nil { @@ -171,7 +189,7 @@ func (m *Manifest) UnmarshalText(text []byte) error { return nil } -// Blob returns the manifest's blob. +// Blob returns the manifest as a blob. func (m *Manifest) Blob() (Blob, error) { manifestText, err := m.MarshalText() if err != nil { diff --git a/private/pkg/manifest/module.go b/private/pkg/manifest/module.go index 58ab2e5dc3..8a06100f69 100644 --- a/private/pkg/manifest/module.go +++ b/private/pkg/manifest/module.go @@ -23,7 +23,7 @@ import ( "go.uber.org/multierr" ) -// Blob is a blob with a digest and a content. +// Blob is an anonymous file associated with a digest. type Blob interface { Digest() *Digest Open(context.Context) (io.ReadCloser, error) @@ -85,7 +85,7 @@ func (b *memoryBlob) Open(context.Context) (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(b.content)), nil } -// BlobSet represents a set of deduplicated blobs, by digests. +// BlobSet represents a set of deduplicated blobs by their digests. type BlobSet struct { digestToBlob map[string]Blob } diff --git a/private/pkg/manifest/storage.go b/private/pkg/manifest/storage.go index 2c6f77ebf0..7684590f73 100644 --- a/private/pkg/manifest/storage.go +++ b/private/pkg/manifest/storage.go @@ -42,8 +42,8 @@ func (o *manifestBucketObject) ExternalPath() string { return o.path } func (o *manifestBucketObject) Read(p []byte) (int, error) { return o.file.Read(p) } func (o *manifestBucketObject) Close() error { return o.file.Close() } -// NewFromBucket creates a manifest and all its files' blobs from a storage -// bucket, with all its digests in DigestTypeShake256. +// NewFromBucket creates a manifest and blob set from the bucket's files. Blobs +// in the blob set use the [DigestTypeShake256] digest. func NewFromBucket( ctx context.Context, bucket storage.ReadBucket,