Merge pull request #2304 from fermyon/spin-load-wasm

Tweak spin-loader to be able to run in Wasm

Author: rylev

crates/loader/Cargo.toml

@@ -17,7 +17,7 @@ lazy_static = "1.4.0"
 mime_guess = { version = "2.0" }
 outbound-http = { path = "../outbound-http", default-features = false }
 spin-outbound-networking = { path = "../outbound-networking" }
-path-absolutize = "3.0.11"
+path-absolutize = { version = "3.0.11", features = ["use_unix_paths_on_wasm"] }
 regex = "1.5.4"
 reqwest = "0.11.9"
 semver = "1.0"
@@ -31,7 +31,7 @@ spin-manifest = { path = "../manifest" }
 tempfile = "3.8.0"
 terminal = { path = "../terminal" }
 thiserror = "1.0.49"
-tokio = { version = "1.23", features = ["full"] }
+tokio = "1.23"
 tokio-util = "0.6"
 toml = "0.8.2"
 tracing = { workspace = true }
@@ -41,6 +41,10 @@ walkdir = "2.3.2"
 tokio = { version = "1.23", features = ["rt", "macros"] }
 ui-testing = { path = "../ui-testing" }
 
+[features]
+default = ["async-io"]
+async-io = ["tokio/fs"]
+
 [[test]]
 name = "ui"
 path = "tests/ui.rs"

crates/loader/src/cache.rs

@@ -1,10 +1,11 @@
 //! Cache for OCI registry entities.
 
 use anyhow::{ensure, Context, Result};
-use tokio::fs;
 
 use std::path::{Path, PathBuf};
 
+use crate::fs::{create_dir_all, write_file};
+
 const CONFIG_DIR: &str = "spin";
 const REGISTRY_CACHE_DIR: &str = "registry";
 const MANIFESTS_DIR: &str = "manifests";
@@ -79,13 +80,13 @@ impl Cache {
 
     /// Write the contents in the cache's wasm directory.
     pub async fn write_wasm(&self, bytes: impl AsRef<[u8]>, digest: impl AsRef<str>) -> Result<()> {
-        fs::write(self.wasm_path(digest), bytes.as_ref()).await?;
+        write_file(&self.wasm_path(digest), bytes.as_ref()).await?;
         Ok(())
     }
 
     /// Write the contents in the cache's data directory.
     pub async fn write_data(&self, bytes: impl AsRef<[u8]>, digest: impl AsRef<str>) -> Result<()> {
-        fs::write(self.data_path(digest), bytes.as_ref()).await?;
+        write_file(&self.data_path(digest), bytes.as_ref()).await?;
         Ok(())
     }
 
@@ -110,21 +111,21 @@ impl Cache {
 
         let p = root.join(MANIFESTS_DIR);
         if !p.is_dir() {
-            fs::create_dir_all(&p).await.with_context(|| {
+            create_dir_all(&p).await.with_context(|| {
                 format!("failed to create manifests directory `{}`", p.display())
             })?;
         }
 
         let p = root.join(WASM_DIR);
         if !p.is_dir() {
-            fs::create_dir_all(&p)
+            create_dir_all(&p)
                 .await
                 .with_context(|| format!("failed to create wasm directory `{}`", p.display()))?;
         }
 
         let p = root.join(DATA_DIR);
         if !p.is_dir() {
-            fs::create_dir_all(&p)
+            create_dir_all(&p)
                 .await
                 .with_context(|| format!("failed to create assets directory `{}`", p.display()))?;
         }

crates/loader/src/fs.rs

@@ -0,0 +1,50 @@
+use anyhow::Result;
+use std::path::Path;
+
+#[cfg(feature = "async-io")]
+mod io {
+    use super::*;
+
+    pub async fn write_file(path: &Path, bytes: &[u8]) -> Result<()> {
+        tokio::fs::write(path, bytes).await?;
+        Ok(())
+    }
+
+    pub async fn create_dir_all(path: &Path) -> Result<()> {
+        tokio::fs::create_dir_all(path).await?;
+        Ok(())
+    }
+
+    pub async fn copy(from: &Path, to: &Path) -> Result<u64> {
+        tokio::fs::copy(from, to).await.map_err(Into::into)
+    }
+
+    pub async fn metadata(path: &Path) -> Result<std::fs::Metadata> {
+        tokio::fs::metadata(path).await.map_err(Into::into)
+    }
+}
+
+#[cfg(not(feature = "async-io"))]
+mod io {
+    use super::*;
+
+    pub async fn write_file(path: &Path, bytes: &[u8]) -> Result<()> {
+        std::fs::write(path, bytes)?;
+        Ok(())
+    }
+
+    pub async fn create_dir_all(path: &Path) -> Result<()> {
+        std::fs::create_dir_all(path)?;
+        Ok(())
+    }
+
+    pub async fn copy(from: &Path, to: &Path) -> Result<u64> {
+        Ok(std::fs::copy(from, to)?)
+    }
+
+    pub async fn metadata(path: &Path) -> Result<std::fs::Metadata> {
+        Ok(std::fs::metadata(path)?)
+    }
+}
+
+pub use io::*;

crates/loader/src/lib.rs

@@ -12,12 +12,14 @@
 
 use std::path::{Path, PathBuf};
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 use local::LocalLoader;
 use spin_common::paths::parent_dir;
 use spin_locked_app::locked::LockedApp;
 
 pub mod cache;
+mod fs;
+#[cfg(feature = "async-io")]
 mod http;
 mod local;
 
@@ -33,7 +35,7 @@ pub async fn from_file(
     cache_root: Option<PathBuf>,
 ) -> Result<LockedApp> {
     let path = manifest_path.as_ref();
-    let app_root = parent_dir(path)?;
+    let app_root = parent_dir(path).context("manifest path has no parent directory")?;
     let loader = LocalLoader::new(&app_root, files_mount_strategy, cache_root).await?;
     loader.load_file(path).await
 }

crates/loader/src/local.rs

@@ -12,9 +12,9 @@ use spin_locked_app::{
     values::{ValuesMap, ValuesMapBuilder},
 };
 use spin_manifest::schema::v2::{self, AppManifest, KebabId, WasiFilesMount};
-use tokio::{fs, sync::Semaphore};
+use tokio::sync::Semaphore;
 
-use crate::{cache::Cache, http::verified_download, FilesMountStrategy};
+use crate::{cache::Cache, FilesMountStrategy};
 
 #[derive(Debug)]
 pub struct LocalLoader {
@@ -291,7 +291,7 @@ impl LocalLoader {
         exclude_files: &[String],
     ) -> Result<()> {
         let src_path = self.app_root.join(src);
-        let meta = fs::metadata(&src_path)
+        let meta = crate::fs::metadata(&src_path)
             .await
             .with_context(|| format!("invalid file mount source {}", quoted_path(src)))?;
         if meta.is_dir() {
@@ -366,13 +366,15 @@ impl LocalLoader {
 
         let _loading_permit = self.file_loading_permits.acquire().await?;
         let dest_parent = parent_dir(dest)?;
-        fs::create_dir_all(&dest_parent).await.with_context(|| {
-            format!(
-                "Failed to create parent directory {}",
-                quoted_path(&dest_parent)
-            )
-        })?;
-        fs::copy(src, dest).await.with_context(|| {
+        crate::fs::create_dir_all(&dest_parent)
+            .await
+            .with_context(|| {
+                format!(
+                    "Failed to create parent directory {}",
+                    quoted_path(&dest_parent)
+                )
+            })?;
+        crate::fs::copy(src, dest).await.with_context(|| {
             format!(
                 "Failed to copy {} to {}",
                 quoted_path(src),
@@ -404,15 +406,18 @@ impl LocalLoader {
     }
 }
 
-/// This canonicalizes the path in a way that works with globs. On non-Windows
-/// platforms, we can use Path::canonicalize, but on Windows platforms this
-/// expands to a UNC path, and the glob library does not work with UNC paths.
-#[cfg(not(windows))]
-fn safe_canonicalize(path: &Path) -> std::io::Result<PathBuf> {
-    path.canonicalize()
+#[cfg(feature = "async-io")]
+async fn verified_download(url: &str, digest: &str, dest: &Path) -> Result<()> {
+    crate::http::verified_download(url, digest, dest)
+        .await
+        .with_context(|| format!("Error fetching source URL {url:?}"))
+}
+
+#[cfg(not(feature = "async-io"))]
+async fn verified_download(_url: &str, _digest: &str, _dest: &Path) -> Result<()> {
+    panic!("async-io feature is required for downloading Wasm sources")
 }
 
-#[cfg(windows)]
 fn safe_canonicalize(path: &Path) -> std::io::Result<PathBuf> {
     use path_absolutize::Absolutize;
     Ok(path.absolutize()?.into_owned())
@@ -515,8 +520,7 @@ fn file_content_ref(path: impl AsRef<Path>) -> Result<ContentRef> {
 
 fn file_url(path: impl AsRef<Path>) -> Result<String> {
     let path = path.as_ref();
-    let abs_path = path
-        .canonicalize()
+    let abs_path = safe_canonicalize(path)
         .with_context(|| format!("Couldn't resolve `{}`", path.display()))?;
     Ok(Url::from_file_path(abs_path).unwrap().to_string())
 }