Skip to content

Commit

Permalink
Merge pull request #1171 from ccx1024cc/morgan/backport
Browse files Browse the repository at this point in the history
backport fix/feature to stable 2.2
jiangliu authored Mar 24, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
2 parents 1155252 + 3031f75 commit d2bbd82
Showing 22 changed files with 577 additions and 3,175 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/smoke.yml
Original file line number Diff line number Diff line change
@@ -2,10 +2,10 @@ name: Smoke Test

on:
push:
branches: ["*"]
branches: ["**"]
paths-ignore: [ '**.md', '**.png', '**.jpg', '**.svg', '**/docs/**' ]
pull_request:
branches: ["*"]
branches: ["**"]
paths-ignore: [ '**.md', '**.png', '**.jpg', '**.svg', '**/docs/**' ]
schedule:
# Run daily sanity check at 03:00 clock UTC
93 changes: 70 additions & 23 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

103 changes: 84 additions & 19 deletions contrib/nydus-backend-proxy/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

22 changes: 20 additions & 2 deletions contrib/nydusify/cmd/nydusify.go
Original file line number Diff line number Diff line change
@@ -340,7 +340,11 @@ func main() {
Value: false,
Usage: "Convert Docker media types to OCI media types",
EnvVars: []string{"OCI"},
Aliases: []string{"docker-v2-format"},
},
&cli.BoolFlag{
Name: "docker-v2-format",
Value: false,
Hidden: true,
},
&cli.StringFlag{
Name: "fs-version",
@@ -446,6 +450,20 @@ func main() {
}
}

docker2OCI := false
if c.Bool("docker-v2-format") {
logrus.Warn("the option `--docker-v2-format` has been deprecated, use `--oci` instead")
docker2OCI = false
} else if c.Bool("oci") {
docker2OCI = true
}

// Forcibly enable `--oci` option when `--oci-ref` be enabled.
if c.Bool("oci-ref") {
logrus.Warn("forcibly enabled `--oci` option when `--oci-ref` be enabled")
docker2OCI = true
}

opt := converter.Opt{
WorkDir: c.String("work-dir"),
NydusImagePath: c.String("nydus-image"),
@@ -469,7 +487,7 @@ func main() {

PrefetchPatterns: prefetchPatterns,
MergePlatform: c.Bool("merge-platform"),
Docker2OCI: c.Bool("oci"),
Docker2OCI: docker2OCI,
FsVersion: fsVersion,
FsAlignChunk: c.Bool("backend-aligned-chunk") || c.Bool("fs-align-chunk"),
Compressor: c.String("compressor"),
143 changes: 63 additions & 80 deletions contrib/nydusify/go.mod
Original file line number Diff line number Diff line change
@@ -3,126 +3,109 @@ module github.com/dragonflyoss/image-service/contrib/nydusify
go 1.18

require (
github.com/aliyun/aliyun-oss-go-sdk v2.1.5+incompatible
github.com/aws/aws-sdk-go-v2 v1.17.2
github.com/aws/aws-sdk-go-v2/config v1.18.4
github.com/aws/aws-sdk-go-v2/credentials v1.13.4
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.43
github.com/aws/aws-sdk-go-v2/service/s3 v1.29.5
github.com/containerd/containerd v1.6.18
github.com/docker/cli v20.10.23+incompatible
github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible
github.com/aws/aws-sdk-go-v2 v1.17.6
github.com/aws/aws-sdk-go-v2/config v1.18.16
github.com/aws/aws-sdk-go-v2/credentials v1.13.16
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.56
github.com/aws/aws-sdk-go-v2/service/s3 v1.30.6
github.com/containerd/containerd v1.7.0
github.com/docker/cli v23.0.1+incompatible
github.com/docker/distribution v2.8.1+incompatible
github.com/goharbor/acceleration-service v0.2.0
github.com/goharbor/acceleration-service v0.2.2
github.com/google/uuid v1.3.0
github.com/hashicorp/go-hclog v1.3.1
github.com/hashicorp/go-plugin v1.4.5
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799
github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b
github.com/pkg/errors v0.9.1
github.com/pkg/xattr v0.4.3
github.com/prometheus/client_golang v1.14.0
github.com/sirupsen/logrus v1.9.0
github.com/stretchr/testify v1.8.1
github.com/urfave/cli/v2 v2.24.2
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
golang.org/x/sys v0.5.0
github.com/stretchr/testify v1.8.2
github.com/urfave/cli/v2 v2.25.0
golang.org/x/sync v0.1.0
golang.org/x/sys v0.6.0
lukechampine.com/blake3 v1.1.5
)

require (
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/Microsoft/hcsshim v0.9.6 // indirect
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
github.com/astaxie/beego v1.12.2 // indirect
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect
github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221215162035-5330a85ea652 // indirect
github.com/Microsoft/go-winio v0.6.0 // indirect
github.com/Microsoft/hcsshim v0.10.0-rc.7 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.20 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.27 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.17 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.24 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.24 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.31 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.22 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.21 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.20 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.20 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.26 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.9 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.17.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.25 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.24 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.24 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.18.6 // indirect
github.com/aws/smithy-go v1.13.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.1.2 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/containerd/cgroups v1.0.4 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/containerd/cgroups v1.1.0 // indirect
github.com/containerd/continuity v0.3.0 // indirect
github.com/containerd/fifo v1.0.0 // indirect
github.com/containerd/nydus-snapshotter v0.6.1 // indirect
github.com/containerd/stargz-snapshotter v0.13.0 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.1 // indirect
github.com/containerd/ttrpc v1.1.0 // indirect
github.com/containerd/typeurl v1.0.2 // indirect
github.com/containerd/fifo v1.1.0 // indirect
github.com/containerd/nydus-snapshotter v0.7.3 // indirect
github.com/containerd/stargz-snapshotter v0.14.3 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
github.com/containerd/ttrpc v1.2.1 // indirect
github.com/containerd/typeurl/v2 v2.1.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/docker v20.10.17+incompatible // indirect
github.com/docker/docker-credential-helpers v0.6.4 // indirect
github.com/docker/docker v23.0.1+incompatible // indirect
github.com/docker/docker-credential-helpers v0.7.0 // indirect
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/felixge/httpsnoop v1.0.2 // indirect
github.com/fatih/color v1.14.1 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gocraft/work v0.5.1 // indirect
github.com/gogo/googleapis v1.4.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/goharbor/harbor/src v0.0.0-20211021012518-bc6a7f65a6fa // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/gomodule/redigo v2.0.0+incompatible // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/hashicorp/yamux v0.1.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/klauspost/compress v1.15.15 // indirect
github.com/klauspost/compress v1.16.0 // indirect
github.com/klauspost/cpuid v1.3.1 // indirect
github.com/lib/pq v1.10.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
github.com/moby/locker v1.0.1 // indirect
github.com/moby/sys/mountinfo v0.6.2 // indirect
github.com/moby/sys/signal v0.6.0 // indirect
github.com/moby/sys/sequential v0.5.0 // indirect
github.com/moby/sys/signal v0.7.0 // indirect
github.com/oklog/run v1.1.0 // indirect
github.com/opencontainers/runc v1.1.2 // indirect
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
github.com/opencontainers/selinux v1.10.1 // indirect
github.com/opencontainers/runc v1.1.4 // indirect
github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect
github.com/opencontainers/selinux v1.11.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/robfig/cron v1.0.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/vbatts/tar-split v0.11.2 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
go.opencensus.io v0.23.0 // indirect
go.opentelemetry.io/contrib v0.22.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.22.0 // indirect
go.opentelemetry.io/otel v1.3.0 // indirect
go.opentelemetry.io/otel/exporters/jaeger v1.0.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0 // indirect
go.opentelemetry.io/otel/internal/metric v0.22.0 // indirect
go.opentelemetry.io/otel/metric v0.22.0 // indirect
go.opentelemetry.io/otel/sdk v1.3.0 // indirect
go.opentelemetry.io/otel/trace v1.3.0 // indirect
go.opentelemetry.io/proto/otlp v0.11.0 // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/text v0.7.0 // indirect
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
google.golang.org/genproto v0.0.0-20220927151529-dcaddaf36704 // indirect
google.golang.org/grpc v1.50.1 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel v1.14.0 // indirect
go.opentelemetry.io/otel/trace v1.14.0 // indirect
golang.org/x/mod v0.9.0 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/text v0.8.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.7.0 // indirect
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
google.golang.org/grpc v1.53.0 // indirect
google.golang.org/protobuf v1.29.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

2,176 changes: 122 additions & 2,054 deletions contrib/nydusify/go.sum

Large diffs are not rendered by default.

6 changes: 4 additions & 2 deletions contrib/nydusify/pkg/converter/config.go
Original file line number Diff line number Diff line change
@@ -4,7 +4,9 @@

package converter

import "strconv"
import (
"strconv"
)

func getConfig(opt Opt) map[string]string {
cfg := map[string]string{}
@@ -17,7 +19,7 @@ func getConfig(opt Opt) map[string]string {
cfg["backend_force_push"] = strconv.FormatBool(opt.BackendForcePush)

cfg["chunk_dict_ref"] = opt.ChunkDictRef
cfg["docker2oci"] = strconv.FormatBool(!opt.Docker2OCI)
cfg["docker2oci"] = strconv.FormatBool(opt.Docker2OCI)
cfg["merge_manifest"] = strconv.FormatBool(opt.MergePlatform)
cfg["oci_ref"] = strconv.FormatBool(opt.OCIRef)

23 changes: 21 additions & 2 deletions contrib/nydusify/pkg/converter/converter.go
Original file line number Diff line number Diff line change
@@ -6,10 +6,12 @@ package converter

import (
"context"
"os"

"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter/provider"
"github.com/goharbor/acceleration-service/pkg/converter"
"github.com/goharbor/acceleration-service/pkg/platformutil"
"github.com/pkg/errors"
)

type Opt struct {
@@ -48,15 +50,32 @@ type Opt struct {
}

func Convert(ctx context.Context, opt Opt) error {
pvd, err := provider.New(opt.WorkDir, hosts(opt))
platformMC, err := platformutil.ParsePlatforms(opt.AllPlatforms, opt.Platforms)
if err != nil {
return err
}

platformMC, err := platformutil.ParsePlatforms(opt.AllPlatforms, opt.Platforms)
if _, err := os.Stat(opt.WorkDir); err != nil {
if errors.Is(err, os.ErrNotExist) {
if err := os.MkdirAll(opt.WorkDir, 0755); err != nil {
return errors.Wrap(err, "prepare work directory")
}
// We should only clean up when the work directory not exists
// before, otherwise it may delete user data by mistake.
defer os.RemoveAll(opt.WorkDir)
} else {
return errors.Wrap(err, "stat work directory")
}
}
tmpDir, err := os.MkdirTemp(opt.WorkDir, "nydusify-")
if err != nil {
return errors.Wrap(err, "create temp directory")
}
pvd, err := provider.New(tmpDir, hosts(opt), platformMC)
if err != nil {
return err
}
defer os.RemoveAll(tmpDir)

cvt, err := converter.New(
converter.WithProvider(pvd),
3 changes: 3 additions & 0 deletions contrib/nydusify/pkg/converter/provider/ported.go
Original file line number Diff line number Diff line change
@@ -17,6 +17,8 @@ import (
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"

// nolint:staticcheck
"github.com/containerd/containerd/remotes/docker/schema1"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -44,6 +46,7 @@ func fetch(ctx context.Context, store content.Store, rCtx *containerd.RemoteCont
limiter *semaphore.Weighted
)

// nolint:staticcheck
if desc.MediaType == images.MediaTypeDockerSchema1Manifest && rCtx.ConvertSchema1 {
schema1Converter := schema1.NewConverter(store, fetcher)

17 changes: 11 additions & 6 deletions contrib/nydusify/pkg/converter/provider/provider.go
Original file line number Diff line number Diff line change
@@ -12,6 +12,7 @@ import (
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/content/local"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/remotes"
"github.com/goharbor/acceleration-service/pkg/remote"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -23,18 +24,20 @@ type Provider struct {
images map[string]*ocispec.Descriptor
store content.Store
hosts remote.HostFunc
platformMC platforms.MatchComparer
}

func New(root string, hosts remote.HostFunc) (*Provider, error) {
func New(root string, hosts remote.HostFunc, platformMC platforms.MatchComparer) (*Provider, error) {
store, err := local.NewLabeledStore(root, newMemoryLabelStore())
if err != nil {
return nil, err
}

return &Provider{
images: make(map[string]*ocispec.Descriptor),
store: store,
hosts: hosts,
images: make(map[string]*ocispec.Descriptor),
store: store,
hosts: hosts,
platformMC: platformMC,
}, nil
}

@@ -56,7 +59,8 @@ func (pvd *Provider) Pull(ctx context.Context, ref string) error {
return err
}
rc := &containerd.RemoteContext{
Resolver: resolver,
Resolver: resolver,
PlatformMatcher: pvd.platformMC,
}

img, err := fetch(ctx, pvd.store, rc, ref, 0)
@@ -77,7 +81,8 @@ func (pvd *Provider) Push(ctx context.Context, desc ocispec.Descriptor, ref stri
return err
}
rc := &containerd.RemoteContext{
Resolver: resolver,
Resolver: resolver,
PlatformMatcher: pvd.platformMC,
}

return push(ctx, pvd.store, rc, desc, ref)
10 changes: 6 additions & 4 deletions contrib/nydusify/pkg/provider/remote.go
Original file line number Diff line number Diff line change
@@ -52,10 +52,12 @@ type withCredentialFunc = func(string) (string, string, error)
func withRemote(ref string, insecure bool, credFunc withCredentialFunc) (*remote.Remote, error) {
resolverFunc := func(retryWithHTTP bool) remotes.Resolver {
registryHosts := docker.ConfigureDefaultRegistries(
docker.WithAuthorizer(docker.NewAuthorizer(
newDefaultClient(insecure),
credFunc,
)),
docker.WithAuthorizer(
docker.NewDockerAuthorizer(
docker.WithAuthClient(newDefaultClient(insecure)),
docker.WithAuthCreds(credFunc),
),
),
docker.WithClient(newDefaultClient(insecure)),
docker.WithPlainHTTP(func(host string) (bool, error) {
return retryWithHTTP, nil
2 changes: 1 addition & 1 deletion rafs/src/fs.rs
Original file line number Diff line number Diff line change
@@ -102,7 +102,7 @@ impl Rafs {
initialized: false,
digest_validate: rafs_cfg.validate,
fs_prefetch: rafs_cfg.prefetch.enable,
amplify_io: rafs_cfg.prefetch.batch_size as u32,
amplify_io: rafs_cfg.batch_size as u32,
prefetch_all: rafs_cfg.prefetch.prefetch_all,
xattr_enabled: rafs_cfg.enable_xattr,

10 changes: 0 additions & 10 deletions rafs/src/metadata/mod.rs
Original file line number Diff line number Diff line change
@@ -699,21 +699,11 @@ impl RafsSuper {
// Old converters extracts bootstraps from data blobs with inlined bootstrap
// use blob digest as the bootstrap file name. The last blob in the blob table from
// the bootstrap has wrong blod id, so we need to fix it.
let mut fixed = false;
let blobs = rs.superblock.get_blob_infos();
for blob in blobs.iter() {
// Fix blob id for new images with old converters.
if blob.has_feature(BlobFeatures::INLINED_FS_META) {
blob.set_blob_id_from_meta_path(path.as_ref())?;
fixed = true;
}
}
if !fixed && !blob_accessible && !blobs.is_empty() {
// Fix blob id for old images with old converters.
let last = blobs.len() - 1;
let blob = &blobs[last];
if !blob.has_feature(BlobFeatures::CAP_TAR_TOC) {
rs.set_blob_id_from_meta_path(path.as_ref())?;
}
}
}
34 changes: 18 additions & 16 deletions smoke/go.mod
Original file line number Diff line number Diff line change
@@ -3,35 +3,37 @@ module github.com/dragonflyoss/image-service/smoke
go 1.18

require (
github.com/containerd/containerd v1.6.18
github.com/containerd/nydus-snapshotter v0.6.1
github.com/google/uuid v1.2.0
github.com/containerd/containerd v1.7.0-rc.1
github.com/containerd/nydus-snapshotter v0.7.3
github.com/google/uuid v1.3.0
github.com/opencontainers/go-digest v1.0.0
github.com/pkg/errors v0.9.1
github.com/pkg/xattr v0.4.9
github.com/stretchr/testify v1.8.1
golang.org/x/sys v0.4.0
github.com/stretchr/testify v1.8.2
golang.org/x/sys v0.6.0
)

require (
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/Microsoft/hcsshim v0.9.6 // indirect
github.com/containerd/cgroups v1.0.4 // indirect
github.com/Microsoft/go-winio v0.6.0 // indirect
github.com/Microsoft/hcsshim v0.10.0-rc.7 // indirect
github.com/containerd/cgroups v1.1.0 // indirect
github.com/containerd/continuity v0.3.0 // indirect
github.com/containerd/fifo v1.0.0 // indirect
github.com/containerd/fifo v1.1.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/klauspost/compress v1.15.12 // indirect
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
github.com/klauspost/compress v1.16.0 // indirect
github.com/moby/sys/sequential v0.5.0 // indirect
github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
google.golang.org/grpc v1.50.1 // indirect
go.opencensus.io v0.24.0 // indirect
golang.org/x/mod v0.9.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/tools v0.6.0 // indirect
google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 // indirect
google.golang.org/grpc v1.53.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
972 changes: 39 additions & 933 deletions smoke/go.sum

Large diffs are not rendered by default.

55 changes: 54 additions & 1 deletion smoke/tests/native_layer_test.go
Original file line number Diff line number Diff line change
@@ -78,7 +78,6 @@ func (n *NativeLayerTestSuite) TestMakeLayers() test.Generator {
}

func (n *NativeLayerTestSuite) testMakeLayers(ctx tool.Context, t *testing.T) {

packOption := converter.PackOption{
BuilderPath: ctx.Binary.Builder,
Compressor: ctx.Build.Compressor,
@@ -154,6 +153,60 @@ func (n *NativeLayerTestSuite) testMakeLayers(ctx tool.Context, t *testing.T) {
lowerLayer.Overlay(t, upperLayer)
ctx.Env.BootstrapPath = overlayBootstrap
tool.Verify(t, ctx, lowerLayer.FileTree)

// Make base layers (use as a parent bootstrap)
packOption.ChunkDictPath = ""
baseLayer1 := texture.MakeMatrixLayer(t, filepath.Join(ctx.Env.WorkDir, "source-base-1"), "1")
baseLayer1BlobDigest := baseLayer1.Pack(t, packOption, ctx.Env.BlobDir)

baseLayer2 := texture.MakeMatrixLayer(t, filepath.Join(ctx.Env.WorkDir, "source-base-2"), "2")
baseLayer2BlobDigest := baseLayer2.Pack(t, packOption, ctx.Env.BlobDir)

lowerLayer = texture.MakeLowerLayer(t, filepath.Join(ctx.Env.WorkDir, "source-lower-1"))
lowerBlobDigest = lowerLayer.Pack(t, packOption, ctx.Env.BlobDir)

upperLayer = texture.MakeUpperLayer(t, filepath.Join(ctx.Env.WorkDir, "source-upper-1"))
upperBlobDigest = upperLayer.Pack(t, packOption, ctx.Env.BlobDir)

mergeOption = converter.MergeOption{
BuilderPath: ctx.Binary.Builder,
}
baseLayerDigests, baseBootstrap := tool.MergeLayers(t, ctx, mergeOption, []converter.Layer{
{
Digest: baseLayer1BlobDigest,
},
{
Digest: baseLayer2BlobDigest,
},
})
ctx.Env.BootstrapPath = baseBootstrap
require.Equal(t, []digest.Digest{baseLayer1BlobDigest, baseLayer2BlobDigest}, baseLayerDigests)

// Test merge from a parent bootstrap
mergeOption = converter.MergeOption{
ParentBootstrapPath: baseBootstrap,
ChunkDictPath: baseBootstrap,
BuilderPath: ctx.Binary.Builder,
}
actualDigests, overlayBootstrap = tool.MergeLayers(t, ctx, mergeOption, []converter.Layer{
{
Digest: lowerBlobDigest,
},
{
Digest: upperBlobDigest,
},
})

require.Equal(t, []digest.Digest{
baseLayer1BlobDigest,
baseLayer2BlobDigest,
lowerBlobDigest,
upperBlobDigest,
}, actualDigests)

ctx.Env.BootstrapPath = overlayBootstrap
baseLayer1.Overlay(t, baseLayer2).Overlay(t, lowerLayer).Overlay(t, upperLayer)
tool.Verify(t, ctx, baseLayer1.FileTree)
}

func TestNativeLayer(t *testing.T) {
12 changes: 12 additions & 0 deletions smoke/tests/texture/layer.go
Original file line number Diff line number Diff line change
@@ -74,3 +74,15 @@ func MakeUpperLayer(t *testing.T, workDir string) *tool.Layer {

return layer
}

func MakeMatrixLayer(t *testing.T, workDir, id string) *tool.Layer {
layer := tool.NewLayer(t, workDir)

// Create regular file
file1 := fmt.Sprintf("matrix-file-%s-1", id)
file2 := fmt.Sprintf("matrix-file-%s-2", id)
layer.CreateFile(t, file1, []byte(file1))
layer.CreateFile(t, file2, []byte(file2))

return layer
}
4 changes: 3 additions & 1 deletion smoke/tests/tool/layer.go
Original file line number Diff line number Diff line change
@@ -164,7 +164,7 @@ func (l *Layer) PackRef(t *testing.T, ctx Context, blobDir string, compress bool
return ociBlobDigest, rafsBlobDigest
}

func (l *Layer) Overlay(t *testing.T, upper *Layer) {
func (l *Layer) Overlay(t *testing.T, upper *Layer) *Layer {
// Handle whiteout/opaque files
for upperName := range upper.FileTree {
name := filepath.Base(upperName)
@@ -198,6 +198,8 @@ func (l *Layer) Overlay(t *testing.T, upper *Layer) {
}
}
}

return l
}

func (l *Layer) recordFileTree(t *testing.T) {
11 changes: 10 additions & 1 deletion src/bin/nydus-image/main.rs
Original file line number Diff line number Diff line change
@@ -327,11 +327,17 @@ fn prepare_cmd_args(bti_string: &'static str) -> App {
.subcommand(
App::new("merge")
.about("Merge multiple bootstraps into a overlaid bootstrap")
.arg(
Arg::new("parent-bootstrap")
.long("parent-bootstrap")
.help("File path of the parent/referenced RAFS metadata blob (optional)")
.required(false),
)
.arg(
Arg::new("bootstrap")
.long("bootstrap")
.short('B')
.help("output path of nydus overlaid bootstrap"),
.help("Output path of nydus overlaid bootstrap"),
)
.arg(
Arg::new("blob-dir")
@@ -930,8 +936,11 @@ impl Command {
};
ctx.configuration = config.clone();

let parent_bootstrap_path = Self::get_parent_bootstrap(matches)?;

let output = Merger::merge(
&mut ctx,
parent_bootstrap_path,
source_bootstrap_paths,
blob_digests,
blob_sizes,
48 changes: 32 additions & 16 deletions src/bin/nydus-image/merge.rs
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0

use std::collections::HashMap;
use std::collections::HashSet;
use std::convert::TryFrom;
use std::ops::Deref;
@@ -61,6 +62,7 @@ impl Merger {
#[allow(clippy::too_many_arguments)]
pub fn merge(
ctx: &mut BuildContext,
parent_bootstrap_path: Option<String>,
sources: Vec<PathBuf>,
blob_digests: Option<Vec<String>>,
blob_sizes: Option<Vec<u64>>,
@@ -106,6 +108,26 @@ impl Merger {
);
}

let mut tree: Option<Tree> = None;
let mut blob_mgr = BlobManager::new(ctx.digester);

// Load parent bootstrap
let mut blob_idx_map = HashMap::new();
let mut parent_layers = 0;
if let Some(parent_bootstrap_path) = &parent_bootstrap_path {
let (rs, _) =
RafsSuper::load_from_file(parent_bootstrap_path, config_v2.clone(), false, false)
.context(format!("load parent bootstrap {:?}", parent_bootstrap_path))?;
tree = Some(Tree::from_bootstrap(&rs, &mut ())?);
let blobs = rs.superblock.get_blob_infos();
for blob in &blobs {
let blob_ctx = BlobContext::from(ctx, &blob, ChunkSource::Parent)?;
blob_idx_map.insert(blob_ctx.blob_id.clone(), blob_mgr.len());
blob_mgr.add(blob_ctx);
}
parent_layers = blobs.len();
}

// Get the blobs come from chunk dict bootstrap.
let mut chunk_dict_blobs = HashSet::new();
let mut config = None;
@@ -121,8 +143,6 @@ impl Merger {

let mut fs_version = RafsVersion::V6;
let mut chunk_size = None;
let mut tree: Option<Tree> = None;
let mut blob_mgr = BlobManager::new(ctx.digester);

for (layer_idx, bootstrap_path) in sources.iter().enumerate() {
let (rs, _) = RafsSuper::load_from_file(bootstrap_path, config_v2.clone(), true, false)
@@ -136,9 +156,9 @@ impl Merger {
ctx.digester = rs.meta.get_digester();
ctx.explicit_uidgid = rs.meta.explicit_uidgid();

let mut blob_idx_map = Vec::new();
let mut parent_blob_added = false;
for blob in rs.superblock.get_blob_infos() {
let blobs = &rs.superblock.get_blob_infos();
for blob in blobs {
let mut blob_ctx = BlobContext::from(ctx, &blob, ChunkSource::Parent)?;
if let Some(chunk_size) = chunk_size {
ensure!(
@@ -191,15 +211,8 @@ impl Merger {
}
}

let mut found = false;
for (idx, blob) in blob_mgr.get_blobs().iter().enumerate() {
if blob.blob_id == blob_ctx.blob_id {
blob_idx_map.push(idx as u32);
found = true;
}
}
if !found {
blob_idx_map.push(blob_mgr.len() as u32);
if !blob_idx_map.contains_key(&blob.blob_id()) {
blob_idx_map.insert(blob.blob_id().clone(), blob_mgr.len());
blob_mgr.add(blob_ctx);
}
}
@@ -218,16 +231,19 @@ impl Merger {
))?;
for chunk in &mut node.chunks {
let origin_blob_index = chunk.inner.blob_index() as usize;
// Set the blob index of chunk to real index in blob table of final bootstrap.
chunk.inner.set_blob_index(blob_idx_map[origin_blob_index]);
let blob_ctx = blobs[origin_blob_index].as_ref();
if let Some(blob_index) = blob_idx_map.get(&blob_ctx.blob_id()) {
// Set the blob index of chunk to real index in blob table of final bootstrap.
chunk.inner.set_blob_index(*blob_index as u32);
}
}
// Set node's layer index to distinguish same inode number (from bootstrap)
// between different layers.
node.layer_idx = u16::try_from(layer_idx).context(format!(
"too many layers {}, limited to {}",
layer_idx,
u16::MAX
))?;
))? + parent_layers as u16;
node.overlay = Overlay::UpperAddition;
match node.whiteout_type(WhiteoutSpec::Oci) {
// Insert whiteouts at the head, so they will be handled first when
2 changes: 1 addition & 1 deletion storage/Cargo.toml
Original file line number Diff line number Diff line change
@@ -23,7 +23,7 @@ leaky-bucket = "0.12.1"
libc = "0.2"
log = "0.4.8"
nix = "0.24"
reqwest = { version = "0.11.11", features = ["blocking", "json"], optional = true }
reqwest = { version = "0.11.14", features = ["blocking", "json"], optional = true }
serde = { version = "1.0.110", features = ["serde_derive", "rc"] }
serde_json = "1.0.53"
sha2 = { version = "0.10.2", optional = true }
2 changes: 1 addition & 1 deletion tests/bats/compile_nydus_snapshotter.bats
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ setup() {
}

@test "compile nydus snapshotter" {
docker run --rm -v /tmp/nydus-snapshotter:/nydus-snapshotter $compile_image bash -c 'cd /nydus-snapshotter && make clear && make'
docker run --rm -v /tmp/nydus-snapshotter:/nydus-snapshotter $compile_image bash -c 'cd /nydus-snapshotter && make clean && make'
if [ -f "/tmp/nydus-snapshotter/bin/containerd-nydus-grpc" ]; then
/usr/bin/cp -f /tmp/nydus-snapshotter/bin/containerd-nydus-grpc /usr/local/bin/
echo "nydus-snapshotter version"

0 comments on commit d2bbd82

Please sign in to comment.