-
Notifications
You must be signed in to change notification settings - Fork 208
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
nydusify: introduce optimize subcommand of nydusify
We can statically analyze the image entrypoint dependency, or use runtime dynamic analysis technologies such as ebpf, fanotify, metric, etc. to obtain the container file access pattern, and then build this part of data into an independent image layer: * preferentially fetch blob during the image startup phase to reduce network and disk IO. * avoid frequent image builds, allows for better local cache utilization. Implement optimize subcommand of nydusify to generate a new image, which references a new blob included prefetch file chunks. ``` nydusify optimize --policy separated-prefetch-blob \ --source $existed-nydus-image \ --target $new-nydus-image \ --prefetch-files /path/to/prefetch-files ``` More detailed process is as follows: 1. nydusify first downloads the source image and bootstrap, utilize nydus-image to output a new bootstrap along with an independent prefetchblob; 2. nydusify generate&push new meta layer including new bootstrap and the prefetch-files , also generates&push new manifest/config/prefetchblob, completing the incremental image build. Signed-off-by: Xing Ma <[email protected]>
- Loading branch information
Xing Ma
committed
Dec 9, 2024
1 parent
eadd808
commit 6b13369
Showing
4 changed files
with
719 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| package optimizer | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "os" | ||
| "os/exec" | ||
| "strings" | ||
| "time" | ||
|
|
||
| "github.com/pkg/errors" | ||
| "github.com/sirupsen/logrus" | ||
| ) | ||
|
|
||
| var logger = logrus.WithField("module", "optimizer") | ||
|
|
||
| func isSignalKilled(err error) bool { | ||
| return strings.Contains(err.Error(), "signal: killed") | ||
| } | ||
|
|
||
| type BuildOption struct { | ||
| BuilderPath string | ||
| PrefetchFilesPath string | ||
| BootstrapPath string | ||
| BlobDir string | ||
| OutputBootstrapPath string | ||
| OutputJSONPath string | ||
| Timeout *time.Duration | ||
| } | ||
|
|
||
| type outputJSON struct { | ||
| Blobs []string `json:"blobs"` | ||
| } | ||
|
|
||
| func Build(option BuildOption) (string, error) { | ||
| outputJSONPath := option.OutputJSONPath | ||
| args := []string{ | ||
| "optimize", | ||
| "--log-level", | ||
| "warn", | ||
| "--prefetch-files", | ||
| option.PrefetchFilesPath, | ||
| "--bootstrap", | ||
| option.BootstrapPath, | ||
| "--blob-dir", | ||
| option.BlobDir, | ||
| "--output-bootstrap", | ||
| option.OutputBootstrapPath, | ||
| "--output-json", | ||
| outputJSONPath, | ||
| } | ||
|
|
||
| ctx := context.Background() | ||
| var cancel context.CancelFunc | ||
| if option.Timeout != nil { | ||
| ctx, cancel = context.WithTimeout(ctx, *option.Timeout) | ||
| defer cancel() | ||
| } | ||
| logrus.Debugf("\tCommand: %s %s", option.BuilderPath, strings.Join(args, " ")) | ||
|
|
||
| cmd := exec.CommandContext(ctx, option.BuilderPath, args...) | ||
| cmd.Stdout = logger.Writer() | ||
| cmd.Stderr = logger.Writer() | ||
|
|
||
| if err := cmd.Run(); err != nil { | ||
| if isSignalKilled(err) && option.Timeout != nil { | ||
| logrus.WithError(err).Errorf("fail to run %v %+v, possibly due to timeout %v", option.BuilderPath, args, *option.Timeout) | ||
| } else { | ||
| logrus.WithError(err).Errorf("fail to run %v %+v", option.BuilderPath, args) | ||
| } | ||
| return "", errors.Wrap(err, "run merge command") | ||
| } | ||
|
|
||
| outputBytes, err := os.ReadFile(outputJSONPath) | ||
| if err != nil { | ||
| return "", errors.Wrapf(err, "read file %s", outputJSONPath) | ||
| } | ||
| var output outputJSON | ||
| err = json.Unmarshal(outputBytes, &output) | ||
| if err != nil { | ||
| return "", errors.Wrapf(err, "unmarshal output json file %s", outputJSONPath) | ||
| } | ||
| blobID := output.Blobs[len(output.Blobs)-1] | ||
|
|
||
| logrus.Infof("build success for prefetch blob : %s", blobID) | ||
| return blobID, nil | ||
| } |
Oops, something went wrong.