diff --git a/builder/src/compact.rs b/builder/src/compact.rs index 19fce1d0137..5847b97371d 100644 --- a/builder/src/compact.rs +++ b/builder/src/compact.rs @@ -21,6 +21,8 @@ use nydus_utils::{digest, try_round_up_4k}; use serde::{Deserialize, Serialize}; use sha2::Digest; +use crate::core::context::Artifact; + use super::core::blob::Blob; use super::core::bootstrap::Bootstrap; use super::{ diff --git a/builder/src/core/blob.rs b/builder/src/core/blob.rs index a58c1b35a5f..b579963b4d7 100644 --- a/builder/src/core/blob.rs +++ b/builder/src/core/blob.rs @@ -3,7 +3,6 @@ // SPDX-License-Identifier: Apache-2.0 use std::borrow::Cow; -use std::io::Write; use std::slice; use anyhow::{Context, Result}; @@ -16,9 +15,8 @@ use sha2::digest::Digest; use super::layout::BlobLayout; use super::node::Node; -use crate::{ - ArtifactWriter, BlobContext, BlobManager, BuildContext, ConversionType, Feature, Tree, -}; +use crate::core::context::Artifact; +use crate::{BlobContext, BlobManager, BuildContext, ConversionType, Feature, Tree}; /// Generator for RAFS data blob. pub(crate) struct Blob {} @@ -29,7 +27,7 @@ impl Blob { ctx: &BuildContext, tree: &Tree, blob_mgr: &mut BlobManager, - blob_writer: &mut ArtifactWriter, + blob_writer: &mut dyn Artifact, ) -> Result<()> { match ctx.conversion_type { ConversionType::DirectoryToRafs => { @@ -101,7 +99,7 @@ impl Blob { fn finalize_blob_data( ctx: &BuildContext, blob_mgr: &mut BlobManager, - blob_writer: &mut ArtifactWriter, + blob_writer: &mut dyn Artifact, ) -> Result<()> { // Dump buffered batch chunk data if exists. if let Some(ref batch) = ctx.blob_batch_generator { @@ -159,7 +157,7 @@ impl Blob { pub(crate) fn dump_meta_data( ctx: &BuildContext, blob_ctx: &mut BlobContext, - blob_writer: &mut ArtifactWriter, + blob_writer: &mut dyn Artifact, ) -> Result<()> { // Dump blob meta for v6 when it has chunks or bootstrap is to be inlined. if !blob_ctx.blob_meta_info_enabled || blob_ctx.uncompressed_blob_size == 0 { @@ -194,7 +192,6 @@ impl Blob { } else if ctx.blob_tar_reader.is_some() { header.set_separate_blob(true); }; - let mut compressor = Self::get_compression_algorithm_for_meta(ctx); let (compressed_data, compressed) = compress::compress(ci_data, compressor) .with_context(|| "failed to compress blob chunk info array".to_string())?; @@ -223,6 +220,9 @@ impl Blob { } blob_ctx.blob_meta_header = header; + if let Some(blob_cache) = ctx.blob_cache_generator.as_ref() { + blob_cache.write_blob_meta(ci_data, &header)?; + } let encrypted_header = crypt::encrypt_with_context(header.as_bytes(), cipher_obj, cipher_ctx, encrypt)?; let header_size = encrypted_header.len(); diff --git a/builder/src/core/context.rs b/builder/src/core/context.rs index 36f48a3c325..1c21dae4a0e 100644 --- a/builder/src/core/context.rs +++ b/builder/src/core/context.rs @@ -10,6 +10,7 @@ use std::collections::{HashMap, VecDeque}; use std::convert::TryFrom; use std::fs::{remove_file, rename, File, OpenOptions}; use std::io::{BufWriter, Cursor, Read, Seek, Write}; +use std::mem::size_of; use std::os::unix::fs::FileTypeExt; use std::path::{Display, Path, PathBuf}; use std::str::FromStr; @@ -40,7 +41,7 @@ use nydus_storage::meta::{ BlobMetaChunkArray, BlobMetaChunkInfo, ZranContextGenerator, }; use nydus_utils::digest::DigestData; -use nydus_utils::{compress, digest, div_round_up, round_down, BufReaderInfo}; +use nydus_utils::{compress, digest, div_round_up, round_down, try_round_up_4k, BufReaderInfo}; use super::node::ChunkSource; use crate::core::tree::TreeNode; @@ -193,7 +194,13 @@ impl Write for ArtifactMemoryWriter { } } -struct ArtifactFileWriter(ArtifactWriter); +struct ArtifactFileWriter(pub ArtifactWriter); + +impl ArtifactFileWriter { + pub fn finalize(&mut self, name: Option) -> Result<()> { + self.0.finalize(name) + } +} impl RafsIoWrite for ArtifactFileWriter { fn as_any(&self) -> &dyn Any { @@ -215,6 +222,12 @@ impl RafsIoWrite for ArtifactFileWriter { } } +impl ArtifactFileWriter { + pub fn set_len(&mut self, s: u64) -> std::io::Result<()> { + self.0.file.get_mut().set_len(s) + } +} + impl Seek for ArtifactFileWriter { fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { self.0.file.seek(pos) @@ -231,6 +244,37 @@ impl Write for ArtifactFileWriter { } } +pub trait Artifact: Write { + fn pos(&self) -> Result; + fn finalize(&mut self, name: Option) -> Result<()>; +} + +#[derive(Default)] +pub struct NoopArtifactWriter { + pos: usize, +} + +impl Write for NoopArtifactWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.pos += buf.len(); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl Artifact for NoopArtifactWriter { + fn pos(&self) -> Result { + Ok(self.pos as u64) + } + + fn finalize(&mut self, _name: Option) -> Result<()> { + Ok(()) + } +} + /// ArtifactWriter provides a writer to allow writing bootstrap /// or blob data to a single file or in a directory. pub struct ArtifactWriter { @@ -308,36 +352,18 @@ impl ArtifactWriter { } } } +} +impl Artifact for ArtifactWriter { /// Get the current write position. - pub fn pos(&self) -> Result { + fn pos(&self) -> Result { Ok(self.pos as u64) } - // The `inline-bootstrap` option merges the blob and bootstrap into one - // file. We need some header to index the location of the blob and bootstrap, - // write_tar_header uses tar header that arranges the data as follows: - // data | tar_header | data | tar_header - // This is a tar-like structure, except that we put the tar header after the - // data. The advantage is that we do not need to determine the size of the data - // first, so that we can write the blob data by stream without seek to improve - // the performance of the blob dump by using fifo. - fn write_tar_header(&mut self, name: &str, size: u64) -> Result
{ - let mut header = Header::new_gnu(); - header.set_path(Path::new(name))?; - header.set_entry_type(EntryType::Regular); - header.set_size(size); - // The checksum must be set to ensure that the tar reader implementation - // in golang can correctly parse the header. - header.set_cksum(); - self.write_all(header.as_bytes())?; - Ok(header) - } - /// Finalize the metadata/data blob. /// /// When `name` is None, it means that the blob is empty and should be removed. - pub fn finalize(&mut self, name: Option) -> Result<()> { + fn finalize(&mut self, name: Option) -> Result<()> { self.file.flush()?; if let Some(n) = name { @@ -367,6 +393,72 @@ impl ArtifactWriter { } } +pub struct BlobCacheGenerator { + blob_data: Mutex, + blob_meta: Mutex, +} + +impl BlobCacheGenerator { + pub fn new(storage: ArtifactStorage) -> Result { + Ok(BlobCacheGenerator { + blob_data: Mutex::new(ArtifactFileWriter(ArtifactWriter::new(storage.clone())?)), + blob_meta: Mutex::new(ArtifactFileWriter(ArtifactWriter::new(storage)?)), + }) + } + + pub fn write_blob_meta( + &self, + data: &[u8], + header: &BlobCompressionContextHeader, + ) -> Result<()> { + let mut guard = self.blob_meta.lock().unwrap(); + let aligned_uncompressed_size = try_round_up_4k(data.len() as u64).ok_or(anyhow!( + format!("invalid input {} for try_round_up_4k", data.len()) + ))?; + guard.set_len( + aligned_uncompressed_size + size_of::() as u64, + )?; + guard + .write_all(data) + .context("failed to write blob meta data")?; + guard.seek(std::io::SeekFrom::Start(aligned_uncompressed_size))?; + guard + .write_all(header.as_bytes()) + .context("failed to write blob meta header")?; + Ok(()) + } + + pub fn write_blob_data( + &self, + chunk_data: &[u8], + chunk_info: &ChunkWrapper, + aligned_d_size: u32, + ) -> Result<()> { + let mut guard = self.blob_data.lock().unwrap(); + let curr_pos = guard.seek(std::io::SeekFrom::End(0))?; + if curr_pos < chunk_info.uncompressed_offset() + aligned_d_size as u64 { + guard.set_len(chunk_info.uncompressed_offset() + aligned_d_size as u64)?; + } + + guard.seek(std::io::SeekFrom::Start(chunk_info.uncompressed_offset()))?; + guard + .write_all(&chunk_data) + .context("failed to write blob cache")?; + Ok(()) + } + + pub fn finalize(&self, name: &str) -> Result<()> { + let blob_data_name = format!("{}.blob.data", name); + let mut guard = self.blob_data.lock().unwrap(); + guard.finalize(Some(blob_data_name))?; + drop(guard); + + let blob_meta_name = format!("{}.blob.meta", name); + let mut guard = self.blob_meta.lock().unwrap(); + guard.finalize(Some(blob_meta_name)) + } +} + /// BlobContext is used to hold the blob information of a layer during build. pub struct BlobContext { /// Blob id (user specified or sha256(blob)). @@ -731,7 +823,7 @@ impl BlobContext { } /// Helper to write data to blob and update blob hash. - pub fn write_data(&mut self, blob_writer: &mut ArtifactWriter, data: &[u8]) -> Result<()> { + pub fn write_data(&mut self, blob_writer: &mut dyn Artifact, data: &[u8]) -> Result<()> { blob_writer.write_all(data)?; self.blob_hash.update(data); Ok(()) @@ -740,11 +832,28 @@ impl BlobContext { /// Helper to write a tar header to blob and update blob hash. pub fn write_tar_header( &mut self, - blob_writer: &mut ArtifactWriter, + blob_writer: &mut dyn Artifact, name: &str, size: u64, ) -> Result
{ - let header = blob_writer.write_tar_header(name, size)?; + // The `inline-bootstrap` option merges the blob and bootstrap into one + // file. We need some header to index the location of the blob and bootstrap, + // write_tar_header uses tar header that arranges the data as follows: + // data | tar_header | data | tar_header + // This is a tar-like structure, except that we put the tar header after the + // data. The advantage is that we do not need to determine the size of the data + // first, so that we can write the blob data by stream without seek to improve + // the performance of the blob dump by using fifo. + + let mut header = Header::new_gnu(); + header.set_path(Path::new(name))?; + header.set_entry_type(EntryType::Regular); + header.set_size(size); + // The checksum must be set to ensure that the tar reader implementation + // in golang can correctly parse the header. + header.set_cksum(); + + blob_writer.write_all(header.as_bytes())?; self.blob_hash.update(header.as_bytes()); Ok(header) } @@ -1182,6 +1291,8 @@ pub struct BuildContext { pub features: Features, pub configuration: Arc, + /// Generate the blob cache and blob meta + pub blob_cache_generator: Option, } impl BuildContext { @@ -1221,7 +1332,6 @@ impl BuildContext { } else { crypt::Algorithm::None }; - BuildContext { blob_id, aligned_chunk, @@ -1250,6 +1360,7 @@ impl BuildContext { features, configuration: Arc::new(ConfigV2::default()), + blob_cache_generator: None, } } @@ -1299,6 +1410,7 @@ impl Default for BuildContext { blob_inline_meta: false, features: Features::new(), configuration: Arc::new(ConfigV2::default()), + blob_cache_generator: None, } } } diff --git a/builder/src/core/node.rs b/builder/src/core/node.rs index 5357d639a28..5e100813287 100644 --- a/builder/src/core/node.rs +++ b/builder/src/core/node.rs @@ -6,7 +6,7 @@ use std::ffi::{OsStr, OsString}; use std::fmt::{self, Display, Formatter, Result as FmtResult}; use std::fs::{self, File}; -use std::io::{Read, Write}; +use std::io::Read; use std::ops::Deref; #[cfg(target_os = "linux")] use std::os::linux::fs::MetadataExt; @@ -29,9 +29,9 @@ use nydus_utils::{compress, crypt}; use nydus_utils::{div_round_up, event_tracer, root_tracer, try_round_up_4k, ByteSize}; use sha2::digest::Digest; -use crate::{ - ArtifactWriter, BlobContext, BlobManager, BuildContext, ChunkDict, ConversionType, Overlay, -}; +use crate::{BlobContext, BlobManager, BuildContext, ChunkDict, ConversionType, Overlay}; + +use super::context::Artifact; /// Filesystem root path for Unix OSs. const ROOT_PATH_NAME: &[u8] = &[b'/']; @@ -219,7 +219,7 @@ impl Node { self: &mut Node, ctx: &BuildContext, blob_mgr: &mut BlobManager, - blob_writer: &mut ArtifactWriter, + blob_writer: &mut dyn Artifact, chunk_data_buf: &mut [u8], ) -> Result { let mut reader = if self.is_reg() { @@ -243,7 +243,7 @@ impl Node { &mut self, ctx: &BuildContext, blob_mgr: &mut BlobManager, - blob_writer: &mut ArtifactWriter, + blob_writer: &mut dyn Artifact, reader: Option<&mut R>, data_buf: &mut [u8], ) -> Result { @@ -393,7 +393,7 @@ impl Node { &self, ctx: &BuildContext, blob_ctx: &mut BlobContext, - blob_writer: &mut ArtifactWriter, + blob_writer: &mut dyn Artifact, chunk_data: &[u8], chunk: &mut ChunkWrapper, ) -> Result> { @@ -462,6 +462,9 @@ impl Node { chunk.set_compressed(is_compressed); } + if let Some(blob_cache) = ctx.blob_cache_generator.as_ref() { + blob_cache.write_blob_data(chunk_data, chunk, aligned_d_size)?; + } event_tracer!("blob_uncompressed_size", +d_size); Ok(chunk_info) @@ -470,7 +473,7 @@ impl Node { pub fn write_chunk_data( ctx: &BuildContext, blob_ctx: &mut BlobContext, - blob_writer: &mut ArtifactWriter, + blob_writer: &mut dyn Artifact, chunk_data: &[u8], ) -> Result<(u64, u32, bool)> { let (compressed, is_compressed) = compress::compress(chunk_data, ctx.compressor) diff --git a/builder/src/directory.rs b/builder/src/directory.rs index 814d73e9546..7395cde9785 100644 --- a/builder/src/directory.rs +++ b/builder/src/directory.rs @@ -5,9 +5,11 @@ use std::fs; use std::fs::DirEntry; -use anyhow::{anyhow, Context, Result}; +use anyhow::{Context, Result}; use nydus_utils::{event_tracer, lazy_drop, root_tracer, timing_tracer}; +use crate::core::context::{Artifact, NoopArtifactWriter}; + use super::core::blob::Blob; use super::core::context::{ ArtifactWriter, BlobManager, BootstrapContext, BootstrapManager, BuildContext, BuildOutput, @@ -126,12 +128,10 @@ impl Builder for DirectoryBuilder { ) -> Result { let mut bootstrap_ctx = bootstrap_mgr.create_ctx()?; let layer_idx = u16::from(bootstrap_ctx.layered); - let mut blob_writer = if let Some(blob_stor) = ctx.blob_storage.clone() { - ArtifactWriter::new(blob_stor)? + let mut blob_writer: Box = if let Some(blob_stor) = ctx.blob_storage.clone() { + Box::new(ArtifactWriter::new(blob_stor)?) } else { - return Err(anyhow!( - "target blob path should always be valid for directory builder" - )); + Box::::default() }; // Scan source directory to build upper layer tree. @@ -148,13 +148,13 @@ impl Builder for DirectoryBuilder { // Dump blob file timing_tracer!( - { Blob::dump(ctx, &bootstrap.tree, blob_mgr, &mut blob_writer,) }, + { Blob::dump(ctx, &bootstrap.tree, blob_mgr, blob_writer.as_mut(),) }, "dump_blob" )?; // Dump blob meta information if let Some((_, blob_ctx)) = blob_mgr.get_current_blob() { - Blob::dump_meta_data(ctx, blob_ctx, &mut blob_writer)?; + Blob::dump_meta_data(ctx, blob_ctx, blob_writer.as_mut())?; } // Dump RAFS meta/bootstrap and finalize the data blob. @@ -167,14 +167,14 @@ impl Builder for DirectoryBuilder { &mut bootstrap_ctx, &mut bootstrap, blob_mgr, - &mut blob_writer, + blob_writer.as_mut(), ) }, "dump_bootstrap" )?; - finalize_blob(ctx, blob_mgr, &mut blob_writer)?; + finalize_blob(ctx, blob_mgr, blob_writer.as_mut())?; } else { - finalize_blob(ctx, blob_mgr, &mut blob_writer)?; + finalize_blob(ctx, blob_mgr, blob_writer.as_mut())?; timing_tracer!( { dump_bootstrap( @@ -183,7 +183,7 @@ impl Builder for DirectoryBuilder { &mut bootstrap_ctx, &mut bootstrap, blob_mgr, - &mut blob_writer, + blob_writer.as_mut(), ) }, "dump_bootstrap" diff --git a/builder/src/lib.rs b/builder/src/lib.rs index d2409a16bc0..ed933c978f2 100644 --- a/builder/src/lib.rs +++ b/builder/src/lib.rs @@ -7,6 +7,7 @@ #[macro_use] extern crate log; +use crate::core::context::Artifact; use std::ffi::OsString; use std::os::unix::ffi::OsStrExt; use std::path::{Path, PathBuf}; @@ -26,8 +27,8 @@ pub use self::compact::BlobCompactor; pub use self::core::bootstrap::Bootstrap; pub use self::core::chunk_dict::{parse_chunk_dict_arg, ChunkDict, HashChunkDict}; pub use self::core::context::{ - ArtifactStorage, ArtifactWriter, BlobContext, BlobManager, BootstrapContext, BootstrapManager, - BuildContext, BuildOutput, ConversionType, + ArtifactStorage, ArtifactWriter, BlobCacheGenerator, BlobContext, BlobManager, + BootstrapContext, BootstrapManager, BuildContext, BuildOutput, ConversionType, }; pub use self::core::feature::{Feature, Features}; pub use self::core::node::{ChunkSource, NodeChunk}; @@ -82,7 +83,7 @@ fn dump_bootstrap( bootstrap_ctx: &mut BootstrapContext, bootstrap: &mut Bootstrap, blob_mgr: &mut BlobManager, - blob_writer: &mut ArtifactWriter, + blob_writer: &mut dyn Artifact, ) -> Result<()> { // Make sure blob id is updated according to blob hash if not specified by user. if let Some((_, blob_ctx)) = blob_mgr.get_current_blob() { @@ -161,7 +162,7 @@ fn dump_bootstrap( fn dump_toc( ctx: &mut BuildContext, blob_ctx: &mut BlobContext, - blob_writer: &mut ArtifactWriter, + blob_writer: &mut dyn Artifact, ) -> Result<()> { if ctx.features.is_enabled(Feature::BlobToc) { assert_ne!(ctx.conversion_type, ConversionType::TarToTarfs); @@ -181,7 +182,7 @@ fn dump_toc( fn finalize_blob( ctx: &mut BuildContext, blob_mgr: &mut BlobManager, - blob_writer: &mut ArtifactWriter, + blob_writer: &mut dyn Artifact, ) -> Result<()> { if let Some((_, blob_ctx)) = blob_mgr.get_current_blob() { let is_tarfs = ctx.conversion_type == ConversionType::TarToTarfs; @@ -238,6 +239,10 @@ fn finalize_blob( if !is_tarfs { blob_writer.finalize(Some(blob_meta_id))?; } + + if let Some(blob_cache) = ctx.blob_cache_generator.as_ref() { + blob_cache.finalize(&blob_ctx.blob_id)?; + } } Ok(()) diff --git a/builder/src/stargz.rs b/builder/src/stargz.rs index 424e9792553..b473d6d0516 100644 --- a/builder/src/stargz.rs +++ b/builder/src/stargz.rs @@ -29,6 +29,8 @@ use nydus_utils::digest::{self, DigestData, RafsDigest}; use nydus_utils::{lazy_drop, root_tracer, timing_tracer, try_round_up_4k, ByteSize}; use serde::{Deserialize, Serialize}; +use crate::core::context::{Artifact, NoopArtifactWriter}; + use super::core::blob::Blob; use super::core::context::{ ArtifactWriter, BlobManager, BootstrapManager, BuildContext, BuildOutput, @@ -836,10 +838,10 @@ impl Builder for StargzBuilder { } else if ctx.digester != digest::Algorithm::Sha256 { bail!("stargz: invalid digest algorithm {:?}", ctx.digester); } - let mut blob_writer = if let Some(blob_stor) = ctx.blob_storage.clone() { - ArtifactWriter::new(blob_stor)? + let mut blob_writer: Box = if let Some(blob_stor) = ctx.blob_storage.clone() { + Box::new(ArtifactWriter::new(blob_stor)?) } else { - return Err(anyhow!("missing configuration for target path")); + Box::::default() }; let mut bootstrap_ctx = bootstrap_mgr.create_ctx()?; let layer_idx = u16::from(bootstrap_ctx.layered); @@ -858,13 +860,13 @@ impl Builder for StargzBuilder { // Dump blob file timing_tracer!( - { Blob::dump(ctx, &bootstrap.tree, blob_mgr, &mut blob_writer) }, + { Blob::dump(ctx, &bootstrap.tree, blob_mgr, blob_writer.as_mut()) }, "dump_blob" )?; // Dump blob meta information if let Some((_, blob_ctx)) = blob_mgr.get_current_blob() { - Blob::dump_meta_data(ctx, blob_ctx, &mut blob_writer)?; + Blob::dump_meta_data(ctx, blob_ctx, blob_writer.as_mut())?; } // Dump RAFS meta/bootstrap and finalize the data blob. @@ -877,14 +879,14 @@ impl Builder for StargzBuilder { &mut bootstrap_ctx, &mut bootstrap, blob_mgr, - &mut blob_writer, + blob_writer.as_mut(), ) }, "dump_bootstrap" )?; - finalize_blob(ctx, blob_mgr, &mut blob_writer)?; + finalize_blob(ctx, blob_mgr, blob_writer.as_mut())?; } else { - finalize_blob(ctx, blob_mgr, &mut blob_writer)?; + finalize_blob(ctx, blob_mgr, blob_writer.as_mut())?; timing_tracer!( { dump_bootstrap( @@ -893,7 +895,7 @@ impl Builder for StargzBuilder { &mut bootstrap_ctx, &mut bootstrap, blob_mgr, - &mut blob_writer, + blob_writer.as_mut(), ) }, "dump_bootstrap" diff --git a/builder/src/tarball.rs b/builder/src/tarball.rs index 6ce10aa10a7..47d054cdc62 100644 --- a/builder/src/tarball.rs +++ b/builder/src/tarball.rs @@ -39,6 +39,8 @@ use nydus_utils::compress::ZlibDecoder; use nydus_utils::digest::RafsDigest; use nydus_utils::{div_round_up, lazy_drop, root_tracer, timing_tracer, BufReaderInfo, ByteSize}; +use crate::core::context::{Artifact, NoopArtifactWriter}; + use super::core::blob::Blob; use super::core::context::{ ArtifactWriter, BlobManager, BootstrapManager, BuildContext, BuildOutput, ConversionType, @@ -99,7 +101,7 @@ struct TarballTreeBuilder<'a> { ty: ConversionType, ctx: &'a mut BuildContext, blob_mgr: &'a mut BlobManager, - blob_writer: &'a mut ArtifactWriter, + blob_writer: &'a mut dyn Artifact, buf: Vec, builder: TarBuilder, } @@ -110,7 +112,7 @@ impl<'a> TarballTreeBuilder<'a> { ty: ConversionType, ctx: &'a mut BuildContext, blob_mgr: &'a mut BlobManager, - blob_writer: &'a mut ArtifactWriter, + blob_writer: &'a mut dyn Artifact, layer_idx: u16, ) -> Self { let builder = TarBuilder::new(ctx.explicit_uidgid, layer_idx, ctx.fs_version); @@ -580,7 +582,7 @@ impl Builder for TarballBuilder { ) -> Result { let mut bootstrap_ctx = bootstrap_mgr.create_ctx()?; let layer_idx = u16::from(bootstrap_ctx.layered); - let mut blob_writer = match self.ty { + let mut blob_writer: Box = match self.ty { ConversionType::EStargzToRafs | ConversionType::EStargzToRef | ConversionType::TargzToRafs @@ -588,9 +590,9 @@ impl Builder for TarballBuilder { | ConversionType::TarToRafs | ConversionType::TarToTarfs => { if let Some(blob_stor) = ctx.blob_storage.clone() { - ArtifactWriter::new(blob_stor)? + Box::new(ArtifactWriter::new(blob_stor)?) } else { - return Err(anyhow!("tarball: missing configuration for target path")); + Box::::default() } } _ => { @@ -602,7 +604,7 @@ impl Builder for TarballBuilder { }; let mut tree_builder = - TarballTreeBuilder::new(self.ty, ctx, blob_mgr, &mut blob_writer, layer_idx); + TarballTreeBuilder::new(self.ty, ctx, blob_mgr, blob_writer.as_mut(), layer_idx); let tree = timing_tracer!({ tree_builder.build_tree() }, "build_tree")?; // Build bootstrap @@ -613,13 +615,13 @@ impl Builder for TarballBuilder { // Dump blob file timing_tracer!( - { Blob::dump(ctx, &bootstrap.tree, blob_mgr, &mut blob_writer) }, + { Blob::dump(ctx, &bootstrap.tree, blob_mgr, blob_writer.as_mut()) }, "dump_blob" )?; // Dump blob meta information if let Some((_, blob_ctx)) = blob_mgr.get_current_blob() { - Blob::dump_meta_data(ctx, blob_ctx, &mut blob_writer)?; + Blob::dump_meta_data(ctx, blob_ctx, blob_writer.as_mut())?; } // Dump RAFS meta/bootstrap and finalize the data blob. @@ -632,14 +634,14 @@ impl Builder for TarballBuilder { &mut bootstrap_ctx, &mut bootstrap, blob_mgr, - &mut blob_writer, + blob_writer.as_mut(), ) }, "dump_bootstrap" )?; - finalize_blob(ctx, blob_mgr, &mut blob_writer)?; + finalize_blob(ctx, blob_mgr, blob_writer.as_mut())?; } else { - finalize_blob(ctx, blob_mgr, &mut blob_writer)?; + finalize_blob(ctx, blob_mgr, blob_writer.as_mut())?; timing_tracer!( { dump_bootstrap( @@ -648,7 +650,7 @@ impl Builder for TarballBuilder { &mut bootstrap_ctx, &mut bootstrap, blob_mgr, - &mut blob_writer, + blob_writer.as_mut(), ) }, "dump_bootstrap" diff --git a/smoke/tests/blobcache_test.go b/smoke/tests/blobcache_test.go new file mode 100644 index 00000000000..e60dc1f9fb2 --- /dev/null +++ b/smoke/tests/blobcache_test.go @@ -0,0 +1,222 @@ +package tests + +import ( + "fmt" + "io" + "io/fs" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/containerd/containerd/log" + "github.com/dragonflyoss/image-service/smoke/tests/texture" + "github.com/dragonflyoss/image-service/smoke/tests/tool" + "github.com/dragonflyoss/image-service/smoke/tests/tool/test" + "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/require" +) + +type BlobCacheTestSuite struct { + T *testing.T +} + +func (a *BlobCacheTestSuite) compareTwoFiles(t *testing.T, left, right string) { + + lf, err := os.Open(left) + require.NoError(t, err) + defer lf.Close() + leftDigester, err := digest.FromReader(lf) + require.NoError(t, err) + + rf, err := os.Open(right) + require.NoError(t, err) + defer rf.Close() + rightDigester, err := digest.FromReader(rf) + require.NoError(t, err) + + require.Equal(t, leftDigester, rightDigester) +} + +func (a *BlobCacheTestSuite) prepareTestEnv(t *testing.T) (*tool.Context, string, digest.Digest) { + ctx := tool.DefaultContext(t) + ctx.PrepareWorkDir(t) + + rootFs := texture.MakeLowerLayer(t, filepath.Join(ctx.Env.WorkDir, "root-fs")) + + rootfsReader := rootFs.ToOCITar(t) + + ociBlobDigester := digest.Canonical.Digester() + ociBlob, err := ioutil.TempFile(ctx.Env.BlobDir, "oci-blob-") + require.NoError(t, err) + + _, err = io.Copy(io.MultiWriter(ociBlobDigester.Hash(), ociBlob), rootfsReader) + require.NoError(t, err) + + ociBlobDigest := ociBlobDigester.Digest() + err = os.Rename(ociBlob.Name(), filepath.Join(ctx.Env.BlobDir, ociBlobDigest.Hex())) + require.NoError(t, err) + + // use to generate blob.data and blob.meta + blobcacheDir := filepath.Join(ctx.Env.WorkDir, "blobcache") + err = os.MkdirAll(blobcacheDir, 0755) + require.NoError(t, err) + + ctx.Env.BootstrapPath = filepath.Join(ctx.Env.WorkDir, "bootstrap") + return ctx, blobcacheDir, ociBlobDigest +} + +func (a *BlobCacheTestSuite) TestCommandFlags(t *testing.T) { + ctx, blobcacheDir, ociBlobDigest := a.prepareTestEnv(t) + defer ctx.Destroy(t) + + testCases := []struct { + name string + conversion_type string + bootstrap string + test_args string + success bool + expected_output string + }{ + { + name: "conflict with --blob-dir", + conversion_type: "targz-ref", + bootstrap: fmt.Sprintf("--bootstrap %s", ctx.Env.BootstrapPath), + test_args: fmt.Sprintf("--blob-dir %s --blob-cache-dir %s", ctx.Env.BlobDir, blobcacheDir), + success: false, + expected_output: "The argument '--blob-dir ' cannot be used with '--blob-cache-dir '", + }, + { + name: "conflict with --blob", + conversion_type: "targz-ref", + bootstrap: fmt.Sprintf("--bootstrap %s", ctx.Env.BootstrapPath), + test_args: fmt.Sprintf("--blob %s --blob-cache-dir %s", "xxxxxx", blobcacheDir), + success: false, + expected_output: "The argument '--blob ' cannot be used with '--blob-cache-dir '", + }, + { + name: "conflict with --blob-inline-meta", + conversion_type: "targz-ref", + bootstrap: "", + test_args: fmt.Sprintf("--blob-inline-meta --blob-cache-dir %s", blobcacheDir), + success: false, + expected_output: "The argument '--blob-inline-meta' cannot be used with '--blob-cache-dir '", + }, + { + name: "conflict with --compressor", + conversion_type: "targz-ref", + bootstrap: fmt.Sprintf("--bootstrap %s", ctx.Env.BootstrapPath), + test_args: fmt.Sprintf("--compressor zstd --blob-cache-dir %s", blobcacheDir), + success: false, + expected_output: "The argument '--compressor ' cannot be used with '--blob-cache-dir '", + }, + + { + name: "conflict with tar-tarfs conversion type", + conversion_type: "tar-tarfs", + bootstrap: fmt.Sprintf("--bootstrap %s", ctx.Env.BootstrapPath), + test_args: fmt.Sprintf("--blob-cache-dir %s", blobcacheDir), + success: false, + expected_output: "conversion type `tar-tarfs` conflicts with `--blob-cache-dir`", + }, + + { + name: "conflict with estargztoc-ref conversion type", + conversion_type: "estargztoc-ref", + bootstrap: fmt.Sprintf("--bootstrap %s", ctx.Env.BootstrapPath), + test_args: fmt.Sprintf("--blob-id %s --blob-cache-dir %s", "xxxx", blobcacheDir), + success: false, + expected_output: "conversion type `estargztoc-ref` conflicts with `--blob-cache-dir`", + }, + + { + name: "conflict with estargz-rafs conversion type", + conversion_type: "estargz-rafs", + bootstrap: fmt.Sprintf("--bootstrap %s", ctx.Env.BootstrapPath), + test_args: fmt.Sprintf("--blob-cache-dir %s", blobcacheDir), + success: false, + expected_output: "conversion type `estargz-rafs` conflicts with `--blob-cache-dir`", + }, + + { + name: "conflict with estargz-ref conversion type", + conversion_type: "estargz-ref", + bootstrap: fmt.Sprintf("--bootstrap %s", ctx.Env.BootstrapPath), + test_args: fmt.Sprintf("--blob-cache-dir %s", blobcacheDir), + success: false, + expected_output: "conversion type `estargz-ref` conflicts with `--blob-cache-dir`", + }, + } + + for _, tc := range testCases { + output, err := tool.RunWithCombinedOutput(fmt.Sprintf("%s create -t %s %s %s %s", + ctx.Binary.Builder, tc.conversion_type, tc.bootstrap, tc.test_args, + filepath.Join(ctx.Env.BlobDir, ociBlobDigest.Hex()))) + + if tc.success { + require.NoError(t, err) + } else { + require.NotEqual(t, err, nil) + } + + require.Contains(t, output, tc.expected_output) + } +} + +func (a *BlobCacheTestSuite) TestGenerateBlobcache(t *testing.T) { + ctx, blobcacheDir, ociBlobDigest := a.prepareTestEnv(t) + defer ctx.Destroy(t) + + tool.Run(t, fmt.Sprintf("%s create -t targz-ref --bootstrap %s --blob-dir %s %s", + ctx.Binary.Builder, ctx.Env.BootstrapPath, ctx.Env.BlobDir, + filepath.Join(ctx.Env.BlobDir, ociBlobDigest.Hex()))) + + nydusd, err := tool.NewNydusd(tool.NydusdConfig{ + NydusdPath: ctx.Binary.Nydusd, + BootstrapPath: ctx.Env.BootstrapPath, + ConfigPath: filepath.Join(ctx.Env.WorkDir, "nydusd-config.fusedev.json"), + MountPath: ctx.Env.MountDir, + APISockPath: filepath.Join(ctx.Env.WorkDir, "nydusd-api.sock"), + BackendType: "localfs", + BackendConfig: fmt.Sprintf(`{"dir": "%s"}`, ctx.Env.BlobDir), + EnablePrefetch: ctx.Runtime.EnablePrefetch, + BlobCacheDir: ctx.Env.CacheDir, + CacheType: ctx.Runtime.CacheType, + CacheCompressed: ctx.Runtime.CacheCompressed, + RafsMode: ctx.Runtime.RafsMode, + DigestValidate: false, + }) + require.NoError(t, err) + + err = nydusd.Mount() + require.NoError(t, err) + defer func() { + if err := nydusd.Umount(); err != nil { + log.L.WithError(err).Errorf("umount") + } + }() + + // make sure blobcache ready + err = filepath.WalkDir(ctx.Env.MountDir, func(path string, entry fs.DirEntry, err error) error { + require.Nil(t, err) + if entry.Type().IsRegular() { + targetPath, err := filepath.Rel(ctx.Env.MountDir, path) + require.NoError(t, err) + _, _ = os.ReadFile(targetPath) + } + return nil + }) + require.NoError(t, err) + + // Generate blobcache + tool.Run(t, fmt.Sprintf("%s create -t targz-ref --bootstrap %s --blob-cache-dir %s %s", + ctx.Binary.Builder, ctx.Env.BootstrapPath, blobcacheDir, + filepath.Join(ctx.Env.BlobDir, ociBlobDigest.Hex()))) + + a.compareTwoFiles(t, filepath.Join(blobcacheDir, fmt.Sprintf("%s.blob.data", ociBlobDigest.Hex())), filepath.Join(ctx.Env.CacheDir, fmt.Sprintf("%s.blob.data", ociBlobDigest.Hex()))) + a.compareTwoFiles(t, filepath.Join(blobcacheDir, fmt.Sprintf("%s.blob.meta", ociBlobDigest.Hex())), filepath.Join(ctx.Env.CacheDir, fmt.Sprintf("%s.blob.meta", ociBlobDigest.Hex()))) +} + +func TestBlobCache(t *testing.T) { + test.Run(t, &BlobCacheTestSuite{T: t}) +} diff --git a/smoke/tests/tool/layer.go b/smoke/tests/tool/layer.go index 7902d5d38ae..452c6584f65 100644 --- a/smoke/tests/tool/layer.go +++ b/smoke/tests/tool/layer.go @@ -116,7 +116,7 @@ func (l *Layer) TargetPath(t *testing.T, path string) string { func (l *Layer) Pack(t *testing.T, packOption converter.PackOption, blobDir string) digest.Digest { // Output OCI tar stream - ociTar := l.toOCITar(t) + ociTar := l.ToOCITar(t) defer ociTar.Close() l.recordFileTree(t) @@ -141,7 +141,7 @@ func (l *Layer) Pack(t *testing.T, packOption converter.PackOption, blobDir stri func (l *Layer) PackRef(t *testing.T, ctx Context, blobDir string, compress bool) (digest.Digest, digest.Digest) { // Output OCI tar stream - ociTar := l.toOCITar(t) + ociTar := l.ToOCITar(t) defer ociTar.Close() l.recordFileTree(t) @@ -238,7 +238,7 @@ func (l *Layer) recordFileTree(t *testing.T) { }) } -func (l *Layer) toOCITar(t *testing.T) io.ReadCloser { +func (l *Layer) ToOCITar(t *testing.T) io.ReadCloser { return archive.Diff(context.Background(), "", l.workDir) } diff --git a/smoke/tests/tool/util.go b/smoke/tests/tool/util.go index a0cd50a836f..fc3358d7dc1 100644 --- a/smoke/tests/tool/util.go +++ b/smoke/tests/tool/util.go @@ -21,6 +21,13 @@ var defaultBinary = map[string]string{ "NYDUS_NYDUSIFY": "nydusify", } +func RunWithCombinedOutput(cmd string) (string, error) { + _cmd := exec.Command("sh", "-c", cmd) + output, err := _cmd.CombinedOutput() + + return string(output), err +} + func Run(t *testing.T, cmd string) { _cmd := exec.Command("sh", "-c", cmd) _cmd.Stdout = os.Stdout diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index 1767bd736e3..82747ec2971 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -27,9 +27,10 @@ use nix::unistd::{getegid, geteuid}; use nydus::{get_build_time_info, setup_logging}; use nydus_api::{BuildTimeInfo, ConfigV2, LocalFsConfig}; use nydus_builder::{ - parse_chunk_dict_arg, ArtifactStorage, BlobCompactor, BlobManager, BootstrapManager, - BuildContext, BuildOutput, Builder, ConversionType, DirectoryBuilder, Feature, Features, - HashChunkDict, Merger, Prefetch, PrefetchPolicy, StargzBuilder, TarballBuilder, WhiteoutSpec, + parse_chunk_dict_arg, ArtifactStorage, BlobCacheGenerator, BlobCompactor, BlobManager, + BootstrapManager, BuildContext, BuildOutput, Builder, ConversionType, DirectoryBuilder, + Feature, Features, HashChunkDict, Merger, Prefetch, PrefetchPolicy, StargzBuilder, + TarballBuilder, WhiteoutSpec, }; use nydus_rafs::metadata::{RafsSuper, RafsSuperConfig, RafsVersion}; use nydus_storage::backend::localfs::LocalFs; @@ -356,6 +357,17 @@ fn prepare_cmd_args(bti_string: &'static str) -> App { .action(ArgAction::SetTrue) .required(false) ) + .arg( + Arg::new("blob-cache-dir") + .long("blob-cache-dir") + .help("Directory path to generate blob cache files ($id.blob.meta and $id.blob.data)") + .value_parser(clap::value_parser!(PathBuf)) + .conflicts_with("blob-inline-meta") + .conflicts_with("blob") + .conflicts_with("blob-dir") + .conflicts_with("compressor") + .required(false) + ) ); let app = app.subcommand( @@ -795,12 +807,21 @@ impl Command { let prefetch = Self::get_prefetch(matches)?; let source_path = PathBuf::from(matches.get_one::("SOURCE").unwrap()); let conversion_type: ConversionType = matches.get_one::("type").unwrap().parse()?; - let blob_storage = Self::get_blob_storage(matches, conversion_type)?; let blob_inline_meta = matches.get_flag("blob-inline-meta"); let repeatable = matches.get_flag("repeatable"); let version = Self::get_fs_version(matches)?; let chunk_size = Self::get_chunk_size(matches, conversion_type)?; let batch_size = Self::get_batch_size(matches, version, conversion_type, chunk_size)?; + let blob_cache_storage = Self::get_blob_cache_storage(matches, conversion_type)?; + // blob-cacher-dir and blob-dir/blob are a set of mutually exclusive functions, + // the former is used to generate blob cache, nydusd is directly started through blob cache, + // the latter is to generate nydus blob, as nydud backend to start + let blob_storage = if blob_cache_storage.is_some() { + None + } else { + Self::get_blob_storage(matches, conversion_type)? + }; + let aligned_chunk = if version.is_v6() && conversion_type != ConversionType::TarToTarfs { true } else { @@ -833,16 +854,16 @@ impl Command { match conversion_type { ConversionType::DirectoryToRafs => { Self::ensure_directory(&source_path)?; - if blob_storage.is_none() { - bail!("both --blob and --blob-dir are missing"); + if blob_storage.is_none() && blob_cache_storage.is_none() { + bail!("both --blob and --blob-dir or --blob-cache-dir are missing"); } } ConversionType::EStargzToRafs | ConversionType::TargzToRafs | ConversionType::TarToRafs => { Self::ensure_file(&source_path)?; - if blob_storage.is_none() { - bail!("both --blob and --blob-dir are missing"); + if blob_storage.is_none() && blob_cache_storage.is_none() { + bail!("both --blob and --blob-dir or --blob-cache-dir are missing"); } } ConversionType::TarToRef @@ -867,8 +888,8 @@ impl Command { } compressor = compress::Algorithm::GZip; digester = digest::Algorithm::Sha256; - if blob_storage.is_none() { - bail!("both --blob and --blob-dir are missing"); + if blob_storage.is_none() && blob_cache_storage.is_none() { + bail!("both --blob and --blob-dir or --blob-cache-dir are missing"); } else if !prefetch.disabled && prefetch.policy == PrefetchPolicy::Blob { bail!( "conversion type {} conflicts with '--prefetch-policy blob'", @@ -914,8 +935,8 @@ impl Command { } compressor = compress::Algorithm::None; digester = digest::Algorithm::Sha256; - if blob_storage.is_none() { - bail!("both --blob and --blob-dir are missing"); + if blob_storage.is_none() && blob_cache_storage.is_none() { + bail!("both --blob and --blob-dir or --blob-cache-dir are missing"); } else if !prefetch.disabled && prefetch.policy == PrefetchPolicy::Blob { bail!( "conversion type {} conflicts with '--prefetch-policy blob'", @@ -985,9 +1006,9 @@ impl Command { } compressor = compress::Algorithm::GZip; digester = digest::Algorithm::Sha256; - if blob_storage.is_some() { + if blob_storage.is_some() || blob_cache_storage.is_some() { bail!( - "conversion type '{}' conflicts with '--blob'", + "conversion type '{}' conflicts with '--blob' and '--blob-cache-dir'", conversion_type ); } @@ -1021,6 +1042,11 @@ impl Command { bail!("`--features blob-toc` can't be used with `--version 5` "); } + if blob_cache_storage.is_some() { + // In blob cache mode, we don't need to do any compression for the original data + compressor = compress::Algorithm::None; + } + let mut build_ctx = BuildContext::new( blob_id, aligned_chunk, @@ -1041,6 +1067,12 @@ impl Command { build_ctx.set_chunk_size(chunk_size); build_ctx.set_batch_size(batch_size); + let blob_cache_generator = match blob_cache_storage { + Some(storage) => Some(BlobCacheGenerator::new(storage)?), + None => None, + }; + build_ctx.blob_cache_generator = blob_cache_generator; + let mut config = Self::get_configuration(matches)?; if let Some(cache) = Arc::get_mut(&mut config).unwrap().cache.as_mut() { cache.cache_validate = true; @@ -1479,6 +1511,31 @@ impl Command { } } + fn get_blob_cache_storage( + matches: &ArgMatches, + conversion_type: ConversionType, + ) -> Result> { + if let Some(p) = matches.get_one::("blob-cache-dir") { + if conversion_type == ConversionType::TarToTarfs + || conversion_type == ConversionType::EStargzIndexToRef + || conversion_type == ConversionType::EStargzToRafs + || conversion_type == ConversionType::EStargzToRef + { + bail!( + "conversion type `{}` conflicts with `--blob-cache-dir`", + conversion_type + ); + } + + if !p.exists() { + bail!("directory to store blob cache does not exist") + } + Ok(Some(ArtifactStorage::FileDir(p.to_owned()))) + } else { + Ok(None) + } + } + // Must specify a path to blob file. // For cli/binary interface compatibility sake, keep option `backend-config`, but // it only receives "localfs" backend type and it will be REMOVED in the future