Merge branch 'master' into feat_inconsistent_fsversion

contrib/ctr-remote/go.mod

@@ -28,7 +28,7 @@ require (
 	github.com/containernetworking/plugins v1.2.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect

contrib/ctr-remote/go.sum

@@ -52,8 +52,9 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
+github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
+github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

contrib/nydusify/go.mod

@@ -61,7 +61,7 @@ require (
 	github.com/containerd/typeurl/v2 v2.1.1 // indirect
 	github.com/containers/ocicrypt v1.1.7 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/docker v23.0.3+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect

contrib/nydusify/go.sum

@@ -83,8 +83,9 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
+github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
+github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

rafs/src/metadata/cached_v5.rs

@@ -460,6 +460,26 @@ impl RafsInode for CachedInodeV5 {
         }
     }
 
+    #[inline]
+    fn is_blkdev(&self) -> bool {
+        self.i_mode & libc::S_IFMT as u32 == libc::S_IFBLK as u32
+    }
+
+    #[inline]
+    fn is_chrdev(&self) -> bool {
+        self.i_mode & libc::S_IFMT as u32 == libc::S_IFCHR as u32
+    }
+
+    #[inline]
+    fn is_sock(&self) -> bool {
+        self.i_mode & libc::S_IFMT as u32 == libc::S_IFSOCK as u32
+    }
+
+    #[inline]
+    fn is_fifo(&self) -> bool {
+        self.i_mode & libc::S_IFMT as u32 == libc::S_IFIFO as u32
+    }
+
     #[inline]
     fn is_dir(&self) -> bool {
         self.i_mode & libc::S_IFMT as u32 == libc::S_IFDIR as u32

rafs/src/metadata/direct_v5.rs

@@ -714,6 +714,10 @@ impl RafsInode for OndiskInodeWrapper {
         self
     }
 
+    impl_inode_wrapper!(is_blkdev, bool);
+    impl_inode_wrapper!(is_chrdev, bool);
+    impl_inode_wrapper!(is_sock, bool);
+    impl_inode_wrapper!(is_fifo, bool);
     impl_inode_wrapper!(is_dir, bool);
     impl_inode_wrapper!(is_reg, bool);
     impl_inode_wrapper!(is_symlink, bool);

rafs/src/metadata/direct_v6.rs

@@ -946,6 +946,26 @@ impl RafsInode for OndiskInodeWrapper {
         0
     }
 
+    #[inline]
+    fn is_blkdev(&self) -> bool {
+        self.mode_format_bits() == libc::S_IFBLK as u32
+    }
+
+    #[inline]
+    fn is_chrdev(&self) -> bool {
+        self.mode_format_bits() == libc::S_IFCHR as u32
+    }
+
+    #[inline]
+    fn is_sock(&self) -> bool {
+        self.mode_format_bits() == libc::S_IFSOCK as u32
+    }
+
+    #[inline]
+    fn is_fifo(&self) -> bool {
+        self.mode_format_bits() == libc::S_IFIFO as u32
+    }
+
     fn is_dir(&self) -> bool {
         self.mode_format_bits() == libc::S_IFDIR as u32
     }

rafs/src/metadata/inode.rs

@@ -156,7 +156,7 @@ impl InodeWrapper {
         match self {
             InodeWrapper::V5(i) => i.is_chrdev(),
             InodeWrapper::V6(i) => i.is_chrdev(),
-            InodeWrapper::Ref(_i) => unimplemented!(),
+            InodeWrapper::Ref(i) => i.as_inode().is_chrdev(),
         }
     }
 
@@ -183,7 +183,7 @@ impl InodeWrapper {
         match self {
             InodeWrapper::V5(i) => i.is_sock(),
             InodeWrapper::V6(i) => i.is_sock(),
-            InodeWrapper::Ref(_i) => unimplemented!(),
+            InodeWrapper::Ref(i) => i.as_inode().is_dir(),
         }
     }
 
@@ -322,7 +322,7 @@ impl InodeWrapper {
         match self {
             InodeWrapper::V5(i) => i.i_uid,
             InodeWrapper::V6(i) => i.i_uid,
-            InodeWrapper::Ref(_i) => unimplemented!(),
+            InodeWrapper::Ref(i) => i.as_inode().get_attr().uid,
         }
     }
 
@@ -341,7 +341,7 @@ impl InodeWrapper {
         match self {
             InodeWrapper::V5(i) => i.i_gid,
             InodeWrapper::V6(i) => i.i_gid,
-            InodeWrapper::Ref(_i) => unimplemented!(),
+            InodeWrapper::Ref(i) => i.as_inode().get_attr().gid,
         }
     }
 

rafs/src/metadata/mod.rs

@@ -177,6 +177,18 @@ pub trait RafsInode: Any {
     /// Posix: get project id associated with the inode.
     fn projid(&self) -> u32;
 
+    /// Mode: check whether the inode is a block device.
+    fn is_blkdev(&self) -> bool;
+
+    /// Mode: check whether the inode is a char device.
+    fn is_chrdev(&self) -> bool;
+
+    /// Mode: check whether the inode is a sock.
+    fn is_sock(&self) -> bool;
+
+    /// Mode: check whether the inode is a fifo.
+    fn is_fifo(&self) -> bool;
+
     /// Mode: check whether the inode is a directory.
     fn is_dir(&self) -> bool;
 

rafs/src/mock/mock_inode.rs

@@ -175,6 +175,26 @@ impl RafsInode for MockInode {
             .collect::<Vec<XattrName>>())
     }
 
+    #[inline]
+    fn is_blkdev(&self) -> bool {
+        self.i_mode & libc::S_IFMT as u32 == libc::S_IFBLK as u32
+    }
+
+    #[inline]
+    fn is_chrdev(&self) -> bool {
+        self.i_mode & libc::S_IFMT as u32 == libc::S_IFCHR as u32
+    }
+
+    #[inline]
+    fn is_sock(&self) -> bool {
+        self.i_mode & libc::S_IFMT as u32 == libc::S_IFSOCK as u32
+    }
+
+    #[inline]
+    fn is_fifo(&self) -> bool {
+        self.i_mode & libc::S_IFMT as u32 == libc::S_IFIFO as u32
+    }
+
     fn is_dir(&self) -> bool {
         self.i_mode & libc::S_IFMT as u32 == libc::S_IFDIR as u32
     }

src/bin/nydus-image/main.rs

@@ -766,8 +766,6 @@ fn main() -> Result<()> {
         result.map_err(|err| anyhow!(err))
     } else if let Some(matches) = cmd.subcommand_matches("check") {
         Command::check(matches, &build_info)
-    } else if let Some(matches) = cmd.subcommand_matches("export") {
-        Command::export(&cmd, matches, &build_info)
     } else if let Some(matches) = cmd.subcommand_matches("inspect") {
         Command::inspect(matches)
     } else if let Some(matches) = cmd.subcommand_matches("stat") {
@@ -777,8 +775,18 @@ fn main() -> Result<()> {
     } else if let Some(matches) = cmd.subcommand_matches("unpack") {
         Command::unpack(matches)
     } else {
-        println!("{}", usage);
-        Ok(())
+        #[cfg(target_os = "linux")]
+        if let Some(matches) = cmd.subcommand_matches("export") {
+            Command::export(&cmd, matches, &build_info)
+        } else {
+            println!("{}", usage);
+            Ok(())
+        }
+        #[cfg(not(target_os = "linux"))]
+        {
+            println!("{}", usage);
+            Ok(())
+        }
     }
 }
 
@@ -1310,7 +1318,16 @@ impl Command {
 
                 Some(Arc::new(local_fs))
             }
-            None => Self::get_backend(matches, "unpacker").ok(),
+            None => {
+                if let Some(backend) = &config.backend {
+                    Some(BlobFactory::new_backend(&backend, "unpacker")?)
+                } else {
+                    match Self::get_backend(matches, "unpacker") {
+                        Ok(backend) => Some(backend),
+                        Err(_) => bail!("one of `--blob`, `--blob-dir` and `--backend-config` must be specified"),
+                    }
+                }
+            }
         };
 
         OCIUnpacker::new(bootstrap, backend, output)
@@ -1718,17 +1735,6 @@ impl Command {
     }
 }
 
-#[cfg(not(target_os = "linux"))]
-impl Command {
-    fn export(
-        _args: &ArgMatches,
-        _subargs: &ArgMatches,
-        _build_info: &BuildTimeInfo,
-    ) -> Result<()> {
-        unimplemented!()
-    }
-}
-
 #[cfg(target_os = "linux")]
 impl Command {
     fn export(args: &ArgMatches, subargs: &ArgMatches, build_info: &BuildTimeInfo) -> Result<()> {

src/bin/nydus-image/unpack/mod.rs

@@ -2,6 +2,7 @@
 // Copyright (C) 2022 Alibaba Cloud. All rights reserved.
 //
 // SPDX-License-Identifier: Apache-2.0
+use std::collections::HashMap;
 use std::fs::{File, OpenOptions};
 use std::io::Read;
 use std::path::{Path, PathBuf};
@@ -115,8 +116,7 @@ impl OCITarBuilderFactory {
     ) -> Result<Box<dyn TarBuilder>> {
         let writer = self.create_writer(output_path)?;
 
-        let blob = meta.superblock.get_blob_infos().pop();
-        let builders = self.create_builders(blob, blob_backend)?;
+        let builders = self.create_builders(meta, blob_backend)?;
 
         let builder = OCITarBuilder::new(builders, writer);
 
@@ -139,7 +139,7 @@ impl OCITarBuilderFactory {
 
     fn create_builders(
         &self,
-        blob: Option<Arc<BlobInfo>>,
+        meta: &RafsSuper,
         blob_backend: &Option<Arc<dyn BlobBackend + Send + Sync>>,
     ) -> Result<Vec<Box<dyn SectionBuilder>>> {
         // PAX basic builders
@@ -155,7 +155,8 @@ impl OCITarBuilderFactory {
         let fifo_builder = OCIFifoBuilder::new(special_builder.clone());
         let char_builder = OCICharBuilder::new(special_builder.clone());
         let block_builder = OCIBlockBuilder::new(special_builder);
-        let reg_builder = self.create_reg_builder(blob, blob_backend)?;
+        let blobs = meta.superblock.get_blob_infos();
+        let reg_builder = self.create_reg_builder(blobs, blob_backend)?;
 
         // The order counts.
         let builders = vec![
@@ -174,29 +175,28 @@ impl OCITarBuilderFactory {
 
     fn create_reg_builder(
         &self,
-        blob: Option<Arc<BlobInfo>>,
+        blobs: Vec<Arc<BlobInfo>>,
         blob_backend: &Option<Arc<dyn BlobBackend + Send + Sync>>,
     ) -> Result<OCIRegBuilder> {
-        let (reader, compressor) = match blob {
-            None => (None, None),
-            Some(ref blob) => {
-                let blob_backend = blob_backend
-                    .as_deref()
-                    .with_context(|| "both blob path or blob backend config are not specified")?;
-                let reader = blob_backend
-                    .get_reader("unpacker")
-                    .map_err(|err| anyhow!("fail to get reader, error {:?}", err))?;
-
-                let compressor = blob.compressor();
-
-                (Some(reader), Some(compressor))
-            }
-        };
+        let mut readers = HashMap::new();
+        let mut compressors = HashMap::new();
+        for blob in blobs {
+            let blob_backend = blob_backend
+                .as_deref()
+                .with_context(|| "both blob path or blob backend config are not specified")?;
+            let reader = blob_backend
+                .get_reader(blob.blob_id().as_str())
+                .map_err(|err| anyhow!("fail to get reader, error {:?}", err))?;
+
+            let compressor = blob.compressor();
+            readers.insert(blob.blob_index(), reader);
+            compressors.insert(blob.blob_index(), compressor);
+        }
 
         Ok(OCIRegBuilder::new(
             Rc::new(PAXExtensionSectionBuilder::new()),
-            reader,
-            compressor,
+            readers,
+            compressors,
         ))
     }
 }