builder: support external blob build

Signed-off-by: Yan Song <[email protected]>

Author: imeoer

builder/src/compact.rs

@@ -285,7 +285,7 @@ impl BlobCompactor {
             version,
             states: vec![Default::default(); ori_blobs_number],
             ori_blob_mgr,
-            new_blob_mgr: BlobManager::new(digester),
+            new_blob_mgr: BlobManager::new(digester, false),
             c2nodes: HashMap::new(),
             b2nodes: HashMap::new(),
             backend,
@@ -547,7 +547,8 @@ impl BlobCompactor {
                     info!("compactor: delete compacted blob {}", ori_blob_ids[idx]);
                 }
                 State::Rebuild(cs) => {
-                    let blob_storage = ArtifactStorage::FileDir(PathBuf::from(dir));
+                    let blob_storage =
+                        ArtifactStorage::FileDir((PathBuf::from(dir), String::new()));
                     let mut blob_ctx = BlobContext::new(
                         String::from(""),
                         0,
@@ -557,6 +558,7 @@ impl BlobCompactor {
                         build_ctx.cipher,
                         Default::default(),
                         None,
+                        false,
                     );
                     blob_ctx.set_meta_info_enabled(self.is_v6());
                     let blob_idx = self.new_blob_mgr.alloc_index()?;
@@ -609,6 +611,7 @@ impl BlobCompactor {
             PathBuf::from(""),
             Default::default(),
             None,
+            None,
             false,
             Features::new(),
             false,
@@ -617,7 +620,7 @@ impl BlobCompactor {
         let mut bootstrap_mgr =
             BootstrapManager::new(Some(ArtifactStorage::SingleFile(d_bootstrap)), None);
         let mut bootstrap_ctx = bootstrap_mgr.create_ctx()?;
-        let mut ori_blob_mgr = BlobManager::new(rs.meta.get_digester());
+        let mut ori_blob_mgr = BlobManager::new(rs.meta.get_digester(), false);
         ori_blob_mgr.extend_from_blob_table(&build_ctx, rs.superblock.get_blob_infos())?;
         if let Some(dict) = chunk_dict {
             ori_blob_mgr.set_chunk_dict(dict);
@@ -656,7 +659,9 @@ impl BlobCompactor {
 
         Ok(Some(BuildOutput::new(
             &compactor.new_blob_mgr,
+            None,
             &bootstrap_mgr.bootstrap_storage,
+            &None,
         )?))
     }
 }
@@ -858,6 +863,7 @@ mod tests {
             crypt::Algorithm::Aes256Xts,
             Arc::new(cipher_object),
             None,
+            false,
         );
         let ori_blob_ids = ["1".to_owned(), "2".to_owned()];
         let backend = Arc::new(MockBackend {
@@ -970,7 +976,7 @@ mod tests {
             HashChunkDict::from_commandline_arg(path, Arc::new(ConfigV2::default()), &rafs_config)
                 .unwrap();
 
-        let mut ori_blob_mgr = BlobManager::new(digest::Algorithm::Sha256);
+        let mut ori_blob_mgr = BlobManager::new(digest::Algorithm::Sha256, false);
         ori_blob_mgr.set_chunk_dict(dict);
 
         let backend = Arc::new(MockBackend {
@@ -1132,6 +1138,7 @@ mod tests {
             PathBuf::from(tmp_dir.as_path()),
             Default::default(),
             None,
+            None,
             false,
             Features::new(),
             false,
@@ -1149,6 +1156,7 @@ mod tests {
             build_ctx.cipher,
             Default::default(),
             None,
+            false,
         );
         let blob_ctx2 = BlobContext::new(
             "blob_id2".to_owned(),
@@ -1159,6 +1167,7 @@ mod tests {
             build_ctx.cipher,
             Default::default(),
             None,
+            false,
         );
         let blob_ctx3 = BlobContext::new(
             "blob_id3".to_owned(),
@@ -1169,6 +1178,7 @@ mod tests {
             build_ctx.cipher,
             Default::default(),
             None,
+            false,
         );
         let blob_ctx4 = BlobContext::new(
             "blob_id4".to_owned(),
@@ -1179,6 +1189,7 @@ mod tests {
             build_ctx.cipher,
             Default::default(),
             None,
+            false,
         );
         let blob_ctx5 = BlobContext::new(
             "blob_id5".to_owned(),
@@ -1189,6 +1200,7 @@ mod tests {
             build_ctx.cipher,
             Default::default(),
             None,
+            false,
         );
         compactor.ori_blob_mgr.add_blob(blob_ctx1);
         compactor.ori_blob_mgr.add_blob(blob_ctx2);
@@ -1230,6 +1242,7 @@ mod tests {
             PathBuf::from(tmp_dir.as_path()),
             Default::default(),
             None,
+            None,
             false,
             Features::new(),
             false,
@@ -1244,6 +1257,7 @@ mod tests {
             build_ctx.cipher,
             Default::default(),
             None,
+            false,
         );
         blob_ctx1.compressed_blob_size = 2;
         let mut blob_ctx2 = BlobContext::new(
@@ -1255,6 +1269,7 @@ mod tests {
             build_ctx.cipher,
             Default::default(),
             None,
+            false,
         );
         blob_ctx2.compressed_blob_size = 0;
         let blob_ctx3 = BlobContext::new(
@@ -1266,6 +1281,7 @@ mod tests {
             build_ctx.cipher,
             Default::default(),
             None,
+            false,
         );
         let blob_ctx4 = BlobContext::new(
             "blob_id4".to_owned(),
@@ -1276,6 +1292,7 @@ mod tests {
             build_ctx.cipher,
             Default::default(),
             None,
+            false,
         );
         let blob_ctx5 = BlobContext::new(
             "blob_id5".to_owned(),
@@ -1286,6 +1303,7 @@ mod tests {
             build_ctx.cipher,
             Default::default(),
             None,
+            false,
         );
         compactor.ori_blob_mgr.add_blob(blob_ctx1);
         compactor.ori_blob_mgr.add_blob(blob_ctx2);

builder/src/core/bootstrap.rs

@@ -75,7 +75,9 @@ impl Bootstrap {
             let digest = RafsDigest::from_buf(&bootstrap_data, digest::Algorithm::Sha256);
             let name = digest.to_string();
             bootstrap_ctx.writer.finalize(Some(name.clone()))?;
-            *bootstrap_storage = Some(ArtifactStorage::SingleFile(p.join(name)));
+            let mut path = p.0.join(name);
+            path.set_extension(&p.1);
+            *bootstrap_storage = Some(ArtifactStorage::SingleFile(path));
             Ok(())
         } else {
             bootstrap_ctx.writer.finalize(Some(String::default()))

builder/src/core/chunk_dict.rs

@@ -19,7 +19,7 @@ use nydus_utils::digest::{self, RafsDigest};
 use crate::Tree;
 
 #[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd)]
-pub struct DigestWithBlobIndex(pub RafsDigest, pub u32);
+pub struct DigestWithBlobIndex(pub RafsDigest, pub u32, pub Option<u32>);
 
 /// Trait to manage chunk cache for chunk deduplication.
 pub trait ChunkDict: Sync + Send + 'static {