ref(*): second approach: unpack_archive_layer as Cache method, move a…

…rchive/unarchive helpers to spin_common

Signed-off-by: Vaughn Dice <vaughn.dice@fermyon.com>

crates/common/Cargo.toml

@@ -6,8 +6,12 @@ edition = { workspace = true }
 
 [dependencies]
 anyhow = "1.0"
+async-compression = "0.4.3"
+# Fork with nested async-std dependency bumped to satisfy Windows build; branch/revision is protected
+async-tar = { git = "https://github.com/vdice/async-tar", rev = "71e037f9652971e7a55b412a8e47a37b06f9c29d" }
 dirs = "4.0"
 sha2 = "0.10"
 tempfile = "3.5"
 tokio = { version = "1", features = ["rt", "time"] }
+tokio-util = { version = "0.7.9", features = ["compat"] }
 url = "2"

crates/oci/src/utils.rs → crates/common/src/compression.rs

@@ -1,10 +1,9 @@
-//! Utilities related to distributing Spin apps via OCI registries
+//! Utilities related to compression and archiving/unarchiving directories
 
 use anyhow::{Context, Result};
 use async_compression::tokio::bufread::GzipDecoder;
 use async_compression::tokio::write::GzipEncoder;
 use async_tar::Archive;
-use spin_common::ui::quoted_path;
 use std::path::{Path, PathBuf};
 
 /// Create a compressed archive of source, returning its path in working_dir
@@ -17,7 +16,7 @@ pub async fn archive(source: &Path, working_dir: &Path) -> Result<PathBuf> {
         .await
         .context(format!(
             "Unable to create tar archive for source {}",
-            quoted_path(source)
+            crate::ui::quoted_path(source)
         ))?;
 
     // Create encoder
@@ -33,7 +32,7 @@ pub async fn archive(source: &Path, working_dir: &Path) -> Result<PathBuf> {
         .await
         .context(format!(
             "Unable to create tar archive for source {}",
-            quoted_path(source)
+            crate::ui::quoted_path(source)
         ))?;
     // Finish writing the archive
     tar_builder.finish().await?;

crates/common/src/lib.rs

@@ -9,6 +9,7 @@
 // - Code should have at least 2 dependents
 
 pub mod arg_parser;
+pub mod compression;
 pub mod data_dir;
 pub mod paths;
 pub mod sha256;

crates/loader/src/cache.rs

@@ -2,10 +2,12 @@
 
 use anyhow::{ensure, Context, Result};
 
+use spin_common::sha256;
 use std::{
     path::PathBuf,
     sync::atomic::{AtomicBool, Ordering},
 };
+use walkdir::WalkDir;
 
 use crate::fs::{create_dir_all, write_file};
 
@@ -152,6 +154,46 @@ impl Cache {
 
         Ok(())
     }
+
+    /// Unpack contents of the provided archive layer, represented by bytes and its
+    /// corresponding digest, into the cache.
+    /// A temporary staging directory is created via tempfile::tempdir() to store
+    /// the unpacked contents prior to writing to the cache.
+    pub async fn unpack_archive_layer(
+        &self,
+        bytes: impl AsRef<[u8]>,
+        digest: impl AsRef<str>,
+    ) -> Result<()> {
+        // Write archive layer to cache as usual
+        self.write_data(&bytes, &digest).await?;
+
+        // Unpack archive into a staging dir
+        let path = self
+            .data_file(&digest)
+            .context("unable to read archive layer from cache")?;
+        let staging_dir = tempfile::tempdir()?;
+        spin_common::compression::unarchive(path.as_ref(), staging_dir.path()).await?;
+
+        // Traverse unpacked contents and if a file, write to cache by digest
+        // (if it doesn't already exist)
+        for entry in WalkDir::new(staging_dir.path()) {
+            let entry = entry?;
+            if entry.file_type().is_file() && !entry.file_type().is_dir() {
+                let bytes = tokio::fs::read(entry.path()).await?;
+                let digest = format!("sha256:{}", sha256::hex_digest_from_bytes(&bytes));
+                if self.data_file(&digest).is_ok() {
+                    tracing::debug!(
+                        "Skipping unpacked asset {:?}; file already exists",
+                        entry.path()
+                    );
+                } else {
+                    tracing::debug!("Adding unpacked asset {:?} to cache", entry.path());
+                    self.write_data(bytes, &digest).await?;
+                }
+            }
+        }
+        Ok(())
+    }
 }
 
 #[cfg(windows)]

crates/oci/Cargo.toml

@@ -6,9 +6,6 @@ edition = { workspace = true }
 
 [dependencies]
 anyhow = "1.0"
-async-compression = "0.4.3"
-# Fork with nested async-std dependency bumped to satisfy Windows build; branch/revision is protected
-async-tar = { git = "https://github.com/vdice/async-tar", rev = "71e037f9652971e7a55b412a8e47a37b06f9c29d" }
 base64 = "0.21"
 # Fork with updated auth to support ACR login
 # Ref https://github.com/camallo/dkregistry-rs/pull/263
@@ -27,7 +24,6 @@ spin-locked-app = { path = "../locked-app" }
 spin-manifest = { path = "../manifest" }
 tempfile = "3.3"
 tokio = { version = "1", features = ["fs"] }
-tokio-util = { version = "0.7.9", features = ["compat"] }
 tracing = { workspace = true }
 walkdir = "2.3"
 

crates/oci/src/client.rs

@@ -13,7 +13,6 @@ use oci_distribution::{
     token_cache::RegistryTokenType, Reference, RegistryOperation,
 };
 use reqwest::Url;
-use spin_common::sha256;
 use spin_common::ui::quoted_path;
 use spin_common::url::parse_file_url;
 use spin_loader::cache::Cache;
@@ -311,7 +310,7 @@ impl Client {
         // Only add the archive layer to the OCI manifest
         tracing::trace!("Adding archive layer for all files in source {:?}", &source);
         let working_dir = tempfile::tempdir()?;
-        let archive_path = crate::utils::archive(source, &working_dir.into_path())
+        let archive_path = spin_common::compression::archive(source, &working_dir.into_path())
             .await
             .context(format!(
                 "Unable to create compressed archive for source {:?}",
@@ -413,7 +412,9 @@ impl Client {
                             this.cache.write_wasm(&bytes, &layer.digest).await?;
                         }
                         ARCHIVE_MEDIATYPE => {
-                            unpack_archive_layer(&this.cache, &bytes, &layer.digest).await?;
+                            this.cache
+                                .unpack_archive_layer(&bytes, &layer.digest)
+                                .await?;
                         }
                         _ => {
                             this.cache.write_data(&bytes, &layer.digest).await?;
@@ -617,46 +618,6 @@ impl Client {
     }
 }
 
-/// Unpack contents of the provided archive layer, represented by bytes and its
-/// corresponding digest, into the provided cache.
-/// A temporary staging directory is created via tempfile::tempdir() to store
-/// the unpacked contents prior to writing to the cache.
-pub async fn unpack_archive_layer(
-    cache: &Cache,
-    bytes: impl AsRef<[u8]>,
-    digest: impl AsRef<str>,
-) -> Result<()> {
-    // Write archive layer to cache as usual
-    cache.write_data(&bytes, &digest).await?;
-
-    // Unpack archive into a staging dir
-    let path = cache
-        .data_file(&digest)
-        .context("unable to read archive layer from cache")?;
-    let staging_dir = tempfile::tempdir()?;
-    crate::utils::unarchive(path.as_ref(), staging_dir.path()).await?;
-
-    // Traverse unpacked contents and if a file, write to cache by digest
-    // (if it doesn't already exist)
-    for entry in WalkDir::new(staging_dir.path()) {
-        let entry = entry?;
-        if entry.file_type().is_file() && !entry.file_type().is_dir() {
-            let bytes = tokio::fs::read(entry.path()).await?;
-            let digest = format!("sha256:{}", sha256::hex_digest_from_bytes(&bytes));
-            if cache.data_file(&digest).is_ok() {
-                tracing::debug!(
-                    "Skipping unpacked asset {:?}; file already exists",
-                    entry.path()
-                );
-            } else {
-                tracing::debug!("Adding unpacked asset {:?} to cache", entry.path());
-                cache.write_data(bytes, &digest).await?;
-            }
-        }
-    }
-    Ok(())
-}
-
 fn digest_from_url(manifest_url: &str) -> Option<String> {
     // The URL is in the form "https://host/v2/refname/manifests/sha256:..."
     let manifest_url = Url::parse(manifest_url).ok()?;

crates/oci/src/lib.rs

@@ -4,7 +4,6 @@
 mod auth;
 pub mod client;
 mod loader;
-pub mod utils;
 
 pub use client::Client;
 pub use loader::OciLoader;