storage: support external blob

Signed-off-by: Yan Song <[email protected]>

Author: imeoer

Cargo.lock

@@ -50,6 +50,7 @@ nydus-utils = { version = "0.4", path = "../utils", features = [
     "encryption",
     "zran",
 ] }
+rmp-serde = "1.1.2"
 
 [dev-dependencies]
 vmm-sys-util = "0.11"

storage/Cargo.toml

@@ -0,0 +1,56 @@
+// Copyright 2024 Nydus Developers. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use std::collections::HashMap;
+use std::io::Result;
+use std::os::unix::fs::FileExt;
+use std::path::PathBuf;
+
+use crate::backend::external::{meta::MetaMap, ExternalBlobReader};
+use crate::device::BlobChunkInfo;
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, PartialEq, Deserialize, Serialize)]
+pub struct Object {
+    #[serde(default, rename = "Path")]
+    pub path: String,
+}
+
+pub struct LocalBackend {
+    meta_map: MetaMap,
+    root: PathBuf,
+}
+
+impl LocalBackend {
+    pub fn new(meta_path: PathBuf, config: &HashMap<String, String>) -> Result<Self> {
+        let meta_map = MetaMap::new(meta_path)?;
+        let root = PathBuf::from(config.get("root").unwrap());
+        Ok(Self { meta_map, root })
+    }
+}
+
+impl ExternalBlobReader for LocalBackend {
+    fn read(&self, buf: &mut [u8], chunks: &[&dyn BlobChunkInfo]) -> Result<usize> {
+        let chunk_index = chunks[0].id();
+        let (object_bytes, chunk) = self.meta_map.get_object(chunk_index)?;
+
+        let object: Object = rmp_serde::from_slice(&object_bytes)
+            .map_err(|_e| einval!("failed to deserialize object"))?;
+
+        let path = self.root.join(&object.path);
+
+        println!(
+            "local_backend: path={:?}, object_offset={}, expected_size={}",
+            path,
+            chunk.object_offset,
+            buf.len()
+        );
+
+        let file = std::fs::File::open(path)?;
+        file.read_exact_at(buf, chunk.object_offset)?;
+
+        Ok(buf.len())
+    }
+}

storage/src/backend/external/local.rs

@@ -0,0 +1,143 @@
+// Copyright 2024 Nydus Developers. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use std::io::Result;
+use std::mem::size_of;
+use std::path::Path;
+use std::{fs::File, os::unix::fs::MetadataExt};
+
+use nydus_utils::filemap::FileMapState;
+use serde::{Deserialize, Serialize};
+
+// Layout
+//
+// header: magic | version | chunk_meta_offset | object_meta_offset
+// chunks: chunk_meta | chunk | chunk | ...
+// objects: object_meta | [object_offsets] | object | object | ...
+
+// 4096 bytes
+#[repr(C)]
+#[derive(Debug)]
+pub struct Header {
+    magic: u32,
+    version: u32,
+
+    chunk_meta_offset: u32,
+    object_meta_offset: u32,
+
+    reserved: [u8; 4080],
+}
+
+// 256 bytes
+#[repr(C)]
+#[derive(Debug)]
+pub struct ChunkMeta {
+    entry_count: u32,
+    entry_size: u32,
+
+    reserved: [u8; 248],
+}
+
+// 256 bytes
+#[repr(C)]
+#[derive(Debug)]
+pub struct ObjectMeta {
+    entry_count: u32,
+    entry_size: u32,
+
+    reserved: [u8; 248],
+}
+
+// 16 bytes
+#[repr(C)]
+#[derive(Debug)]
+pub struct Chunk {
+    pub object_index: u32,
+    reserved: [u8; 4],
+    pub object_offset: u64,
+}
+
+// 4 bytes
+pub type ObjectOffset = u32;
+
+#[derive(Debug)]
+pub struct Object {
+    entry_size: u32,
+    encoded_data: Vec<u8>,
+}
+
+#[derive(Debug, PartialEq, Deserialize, Serialize)]
+pub struct LocalObject {
+    #[serde(default, rename = "Path")]
+    pub path: String,
+}
+
+pub struct MetaMap {
+    map: FileMapState,
+}
+
+impl MetaMap {
+    pub fn new<P: AsRef<Path>>(meta_path: P) -> Result<Self> {
+        let file = File::open(meta_path)?;
+        let size = file.metadata()?.size() as usize;
+        let map = FileMapState::new(file, 0, size, false)?;
+        Ok(Self { map })
+    }
+
+    pub fn get_object(&self, chunk_index: u32) -> Result<(&[u8], &Chunk)> {
+        let header = self.map.get_ref::<Header>(0)?;
+        let chunk_meta_offset = header.chunk_meta_offset;
+        let object_meta_offset = header.object_meta_offset;
+
+        let chunk_meta = self.map.get_ref::<ChunkMeta>(chunk_meta_offset as usize)?;
+        let object_meta = self
+            .map
+            .get_ref::<ObjectMeta>(object_meta_offset as usize)?;
+
+        let chunk = self.map.get_ref::<Chunk>(
+            chunk_meta_offset as usize
+                + size_of::<ChunkMeta>()
+                + chunk_index as usize * chunk_meta.entry_size as usize,
+        )?;
+        let object_index = chunk.object_index;
+        let object_offset = if object_meta.entry_size == 0 {
+            let object_offset_offset = object_meta_offset as usize
+                + size_of::<ObjectMeta>()
+                + object_index as usize * size_of::<ObjectOffset>();
+            *self.map.get_ref::<ObjectOffset>(object_offset_offset)? as usize
+        } else {
+            object_meta_offset as usize
+                + size_of::<ObjectMeta>()
+                + object_index as usize * object_meta.entry_size as usize
+        };
+
+        let object_size = *self.map.get_ref::<u32>(object_offset)? as usize;
+
+        println!(
+            "get_object: chunk_index={}, object_index={}, object_size={}",
+            chunk_index, object_index, object_size,
+        );
+
+        let object_data: &[u8] = self
+            .map
+            .get_slice(object_offset + size_of::<u32>(), object_size)?;
+
+        Ok((object_data, chunk))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_meta_map() {
+        let meta_map = MetaMap::new("/home/imeoer/backend.meta").unwrap();
+        let (object_bytes, _) = meta_map.get_object(0).unwrap();
+        let object: LocalObject = rmp_serde::from_slice(&object_bytes)
+            .map_err(|e| einval!(format!("failed to deserialize object: {:?}", e)))
+            .unwrap();
+        println!("OBJECT {:?}", object);
+    }
+}

storage/src/backend/external/meta.rs

@@ -0,0 +1,50 @@
+// Copyright 2024 Nydus Developers. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::Result;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use crate::device::BlobChunkInfo;
+use serde::{Deserialize, Serialize};
+
+pub mod local;
+pub mod meta;
+
+pub trait ExternalBlobReader: Send + Sync {
+    fn read(&self, buf: &mut [u8], chunks: &[&dyn BlobChunkInfo]) -> Result<usize>;
+}
+
+#[derive(Debug, PartialEq, Deserialize, Serialize)]
+struct BackendConfig {
+    #[serde(rename = "type")]
+    kind: String,
+    config: HashMap<String, String>,
+}
+
+pub struct ExternalBackendFactory {}
+
+impl ExternalBackendFactory {
+    pub fn new(
+        meta_path: PathBuf,
+        backend_config_path: PathBuf,
+    ) -> Result<Arc<dyn ExternalBlobReader>> {
+        let backend_config: BackendConfig =
+            serde_json::from_reader(File::open(backend_config_path)?)?;
+        match backend_config.kind.as_str() {
+            "local" => {
+                let backend = local::LocalBackend::new(meta_path, &backend_config.config)?;
+                Ok(Arc::new(backend) as Arc<dyn ExternalBlobReader>)
+            }
+            _ => {
+                return Err(einval!(format!(
+                    "unsupported backend type: {}",
+                    backend_config.kind
+                )))
+            }
+        }
+    }
+}

storage/src/backend/external/mod.rs

@@ -27,6 +27,8 @@ use nydus_utils::{
 use crate::utils::{alloc_buf, copyv};
 use crate::StorageError;
 
+pub mod external;
+
 #[cfg(any(
     feature = "backend-oss",
     feature = "backend-registry",