From 5b88ceb0cc11c04c7793ec751a3eab2d16aa8eda Mon Sep 17 00:00:00 2001 From: Yan Song Date: Wed, 28 Aug 2024 02:34:16 +0000 Subject: [PATCH] nydusify: small improvements for mount & check subcommands - Add `--prefetch` option for enabling full image data prefetch. - Support `HTTP_PROXY` / `HTTPS_PROXY` env for enabling proxy for nydusd. - Change nydusd log level to `warn` for mount & check subcommands. Signed-off-by: Yan Song --- contrib/nydusify/cmd/nydusify.go | 13 +++-- contrib/nydusify/pkg/checker/checker.go | 1 + .../nydusify/pkg/checker/rule/filesystem.go | 26 +-------- contrib/nydusify/pkg/checker/tool/nydusd.go | 4 +- contrib/nydusify/pkg/utils/backend.go | 57 +++++++++++++++++++ contrib/nydusify/pkg/viewer/viewer.go | 20 ++++--- 6 files changed, 79 insertions(+), 42 deletions(-) create mode 100644 contrib/nydusify/pkg/utils/backend.go diff --git a/contrib/nydusify/cmd/nydusify.go b/contrib/nydusify/cmd/nydusify.go index 590ef133e05..4743d0fad8b 100644 --- a/contrib/nydusify/cmd/nydusify.go +++ b/contrib/nydusify/cmd/nydusify.go @@ -24,7 +24,6 @@ import ( "github.com/urfave/cli/v2" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker" - "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker/rule" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/chunkdict/generator" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/committer" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/converter" @@ -796,7 +795,12 @@ func main() { Usage: "Json configuration file for storage backend", EnvVars: []string{"BACKEND_CONFIG_FILE"}, }, - + &cli.BoolFlag{ + Name: "prefetch", + Value: false, + Usage: "Enable full image data prefetch", + EnvVars: []string{"PREFETCH"}, + }, &cli.StringFlag{ Name: "mount-path", Value: "./image-fs", @@ -836,13 +840,11 @@ func main() { return err } - backendConfigStruct, err := rule.NewRegistryBackendConfig(parsed) + backendConfigStruct, err := utils.NewRegistryBackendConfig(parsed, c.Bool("target-insecure")) if err != nil { return errors.Wrap(err, "parse registry backend configuration") } - backendConfigStruct.SkipVerify = c.Bool("target-insecure") - bytes, err := json.Marshal(backendConfigStruct) if err != nil { return errors.Wrap(err, "marshal registry backend configuration") @@ -865,6 +867,7 @@ func main() { BackendType: backendType, BackendConfig: backendConfig, ExpectedArch: arch, + Prefetch: c.Bool("prefetch"), }) if err != nil { return err diff --git a/contrib/nydusify/pkg/checker/checker.go b/contrib/nydusify/pkg/checker/checker.go index f8d81a40c11..58dcf92e05e 100644 --- a/contrib/nydusify/pkg/checker/checker.go +++ b/contrib/nydusify/pkg/checker/checker.go @@ -168,6 +168,7 @@ func (checker *Checker) check(ctx context.Context) error { TargetInsecure: checker.TargetInsecure, PlainHTTP: checker.targetParser.Remote.IsWithHTTP(), NydusdConfig: tool.NydusdConfig{ + EnablePrefetch: true, NydusdPath: checker.NydusdPath, BackendType: checker.BackendType, BackendConfig: checker.BackendConfig, diff --git a/contrib/nydusify/pkg/checker/rule/filesystem.go b/contrib/nydusify/pkg/checker/rule/filesystem.go index 1bb881b7ce0..898251c971a 100644 --- a/contrib/nydusify/pkg/checker/rule/filesystem.go +++ b/contrib/nydusify/pkg/checker/rule/filesystem.go @@ -6,7 +6,6 @@ package rule import ( "context" - "encoding/base64" "encoding/hex" "encoding/json" "fmt" @@ -16,7 +15,6 @@ import ( "syscall" "github.com/distribution/reference" - dockerconfig "github.com/docker/cli/cli/config" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/checker/tool" "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/parser" @@ -221,28 +219,6 @@ func (rule *FilesystemRule) mountSourceImage() (*tool.Image, error) { return image, nil } -func NewRegistryBackendConfig(parsed reference.Named) (RegistryBackendConfig, error) { - - backendConfig := RegistryBackendConfig{ - Scheme: "https", - Host: reference.Domain(parsed), - Repo: reference.Path(parsed), - } - - config := dockerconfig.LoadDefaultConfigFile(os.Stderr) - authConfig, err := config.GetAuthConfig(backendConfig.Host) - if err != nil { - return backendConfig, errors.Wrap(err, "get docker registry auth config") - } - var auth string - if authConfig.Username != "" && authConfig.Password != "" { - auth = base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", authConfig.Username, authConfig.Password))) - } - backendConfig.Auth = auth - - return backendConfig, nil -} - func (rule *FilesystemRule) mountNydusImage() (*tool.Nydusd, error) { logrus.Infof("Mounting Nydus image to %s", rule.NydusdConfig.MountPath) @@ -263,7 +239,7 @@ func (rule *FilesystemRule) mountNydusImage() (*tool.Nydusd, error) { rule.NydusdConfig.BackendType = "registry" if rule.NydusdConfig.BackendConfig == "" { - backendConfig, err := NewRegistryBackendConfig(parsed) + backendConfig, err := utils.NewRegistryBackendConfig(parsed, rule.TargetInsecure) if err != nil { return nil, errors.Wrap(err, "failed to parse backend configuration") } diff --git a/contrib/nydusify/pkg/checker/tool/nydusd.go b/contrib/nydusify/pkg/checker/tool/nydusd.go index ed8c341ae90..2f47dd0c5e1 100644 --- a/contrib/nydusify/pkg/checker/tool/nydusd.go +++ b/contrib/nydusify/pkg/checker/tool/nydusd.go @@ -76,12 +76,10 @@ func makeConfig(conf NydusdConfig) error { if conf.BackendType == "" { conf.BackendType = "localfs" conf.BackendConfig = `{"dir": "/fake"}` - conf.EnablePrefetch = false } else { if conf.BackendConfig == "" { return errors.Errorf("empty backend configuration string") } - conf.EnablePrefetch = true } if err := tpl.Execute(&ret, conf); err != nil { return errors.New("failed to prepare configuration file for Nydusd") @@ -176,7 +174,7 @@ func (nydusd *Nydusd) Mount() error { "--apisock", nydusd.APISockPath, "--log-level", - "error", + "warn", } cmd := exec.Command(nydusd.NydusdPath, args...) diff --git a/contrib/nydusify/pkg/utils/backend.go b/contrib/nydusify/pkg/utils/backend.go new file mode 100644 index 00000000000..c93690a37a7 --- /dev/null +++ b/contrib/nydusify/pkg/utils/backend.go @@ -0,0 +1,57 @@ +package utils + +import ( + "encoding/base64" + "fmt" + "os" + + "github.com/distribution/reference" + dockerconfig "github.com/docker/cli/cli/config" + "github.com/pkg/errors" +) + +type RegistryBackendConfig struct { + Scheme string `json:"scheme"` + Host string `json:"host"` + Repo string `json:"repo"` + Auth string `json:"auth,omitempty"` + SkipVerify bool `json:"skip_verify,omitempty"` + Proxy BackendProxyConfig `json:"proxy"` +} + +type BackendProxyConfig struct { + URL string `json:"url"` + Fallback bool `json:"fallback"` + PingURL string `json:"ping_url"` +} + +func NewRegistryBackendConfig(parsed reference.Named, insecure bool) (RegistryBackendConfig, error) { + proxyURL := os.Getenv("HTTP_PROXY") + if proxyURL == "" { + proxyURL = os.Getenv("HTTPS_PROXY") + } + + backendConfig := RegistryBackendConfig{ + Scheme: "https", + Host: reference.Domain(parsed), + Repo: reference.Path(parsed), + SkipVerify: insecure, + Proxy: BackendProxyConfig{ + URL: proxyURL, + Fallback: true, + }, + } + + config := dockerconfig.LoadDefaultConfigFile(os.Stderr) + authConfig, err := config.GetAuthConfig(backendConfig.Host) + if err != nil { + return backendConfig, errors.Wrap(err, "get docker registry auth config") + } + var auth string + if authConfig.Username != "" && authConfig.Password != "" { + auth = base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", authConfig.Username, authConfig.Password))) + } + backendConfig.Auth = auth + + return backendConfig, nil +} diff --git a/contrib/nydusify/pkg/viewer/viewer.go b/contrib/nydusify/pkg/viewer/viewer.go index 5f28b394864..2bea0b3f2e4 100644 --- a/contrib/nydusify/pkg/viewer/viewer.go +++ b/contrib/nydusify/pkg/viewer/viewer.go @@ -37,6 +37,7 @@ type Opt struct { BackendConfig string ExpectedArch string FsVersion string + Prefetch bool } // fsViewer provides complete view of file system in nydus image @@ -63,15 +64,16 @@ func New(opt Opt) (*FsViewer, error) { mode := "cached" nydusdConfig := tool.NydusdConfig{ - NydusdPath: opt.NydusdPath, - BackendType: opt.BackendType, - BackendConfig: opt.BackendConfig, - BootstrapPath: filepath.Join(opt.WorkDir, "nydus_bootstrap"), - ConfigPath: filepath.Join(opt.WorkDir, "fs/nydusd_config.json"), - BlobCacheDir: filepath.Join(opt.WorkDir, "fs/nydus_blobs"), - MountPath: opt.MountPath, - APISockPath: filepath.Join(opt.WorkDir, "fs/nydus_api.sock"), - Mode: mode, + EnablePrefetch: opt.Prefetch, + NydusdPath: opt.NydusdPath, + BackendType: opt.BackendType, + BackendConfig: opt.BackendConfig, + BootstrapPath: filepath.Join(opt.WorkDir, "nydus_bootstrap"), + ConfigPath: filepath.Join(opt.WorkDir, "fs/nydusd_config.json"), + BlobCacheDir: filepath.Join(opt.WorkDir, "fs/nydus_blobs"), + MountPath: opt.MountPath, + APISockPath: filepath.Join(opt.WorkDir, "fs/nydus_api.sock"), + Mode: mode, } fsViewer := &FsViewer{