diff --git a/Cargo.lock b/Cargo.lock index c6c293574a..921501fe91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7723,8 +7723,11 @@ dependencies = [ "indexmap 2.7.1", "semver", "spin-app", + "spin-common", + "spin-componentize", "spin-serde", "thiserror 1.0.69", + "tokio", "wac-graph", ] @@ -8218,6 +8221,7 @@ dependencies = [ "serde", "serde_json", "spin-common", + "spin-compose", "spin-loader", "spin-locked-app", "tempfile", @@ -8225,6 +8229,9 @@ dependencies = [ "tokio-util", "tracing", "walkdir", + "wasm-encoder 0.217.0", + "wit-component 0.217.0", + "wit-parser 0.217.0", ] [[package]] @@ -8424,7 +8431,6 @@ dependencies = [ "serde_json", "spin-app", "spin-common", - "spin-componentize", "spin-compose", "spin-core", "spin-factor-key-value", @@ -11145,6 +11151,7 @@ dependencies = [ "wasm-encoder 0.217.0", "wasm-metadata 0.217.0", "wasmparser 0.217.0", + "wat", "wit-parser 0.217.0", ] diff --git a/Cargo.toml b/Cargo.toml index fc8156de74..92ebbd7dbc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -155,6 +155,12 @@ wasmtime = "29.0.1" wasmtime-wasi = "29.0.1" wasmtime-wasi-http = "29.0.1" +wasm-encoder = "0.217" +wasm-metadata = "0.217" +wasmparser = "0.217" +wit-component = "0.217" +wit-parser = "0.217" + spin-componentize = { path = "crates/componentize" } [workspace.lints.clippy] diff --git a/crates/componentize/Cargo.toml b/crates/componentize/Cargo.toml index 10a646cf9e..2a7512c191 100644 --- a/crates/componentize/Cargo.toml +++ b/crates/componentize/Cargo.toml @@ -11,11 +11,11 @@ rust-version.workspace = true [dependencies] anyhow = { workspace = true } tracing = { workspace = true } -wasm-encoder = "0.217" -wasm-metadata = "0.217" -wasmparser = "0.217" -wit-component = "0.217" -wit-parser = "0.217" +wasm-encoder = { workspace = true } +wasm-metadata = { workspace = true } +wasmparser = { workspace = true } +wit-component = { workspace = true } +wit-parser = { workspace = true } [dev-dependencies] async-trait = { workspace = true } diff --git a/crates/compose/Cargo.toml b/crates/compose/Cargo.toml index 34a910aa55..693697351b 100644 --- a/crates/compose/Cargo.toml +++ b/crates/compose/Cargo.toml @@ -14,8 +14,11 @@ async-trait = { workspace = true } indexmap = "2" semver = "1" spin-app = { path = "../app" } +spin-common = { path = "../common" } +spin-componentize = { path = "../componentize" } spin-serde = { path = "../serde" } thiserror = { workspace = true } +tokio = { workspace = true, features = ["fs"] } wac-graph = "0.6" [lints] diff --git a/crates/compose/src/lib.rs b/crates/compose/src/lib.rs index 4138799953..e06b851611 100644 --- a/crates/compose/src/lib.rs +++ b/crates/compose/src/lib.rs @@ -2,6 +2,7 @@ use anyhow::Context; use indexmap::IndexMap; use semver::Version; use spin_app::locked::{self, InheritConfiguration, LockedComponent, LockedComponentDependency}; +use spin_common::{ui::quoted_path, url::parse_file_url}; use spin_serde::{DependencyName, KebabId}; use std::collections::BTreeMap; use thiserror::Error; @@ -42,6 +43,36 @@ pub trait ComponentSourceLoader { ) -> anyhow::Result>; } +/// A ComponentSourceLoader that loads component sources from the filesystem. +pub struct ComponentSourceLoaderFs; + +#[async_trait::async_trait] +impl ComponentSourceLoader for ComponentSourceLoaderFs { + async fn load_component_source( + &self, + source: &locked::LockedComponentSource, + ) -> anyhow::Result> { + let source = source + .content + .source + .as_ref() + .context("LockedComponentSource missing source field")?; + + let path = parse_file_url(source)?; + + let bytes: Vec = tokio::fs::read(&path).await.with_context(|| { + format!( + "failed to read component source from disk at path {}", + quoted_path(&path) + ) + })?; + + let component = spin_componentize::componentize_if_necessary(&bytes)?; + + Ok(component.into()) + } +} + /// Represents an error that can occur when composing dependencies. #[derive(Debug, Error)] pub enum ComposeError { diff --git a/crates/oci/Cargo.toml b/crates/oci/Cargo.toml index 8e17099af7..affc7d0ecb 100644 --- a/crates/oci/Cargo.toml +++ b/crates/oci/Cargo.toml @@ -22,6 +22,7 @@ reqwest = "0.12" serde = { workspace = true } serde_json = { workspace = true } spin-common = { path = "../common" } +spin-compose = { path = "../compose" } spin-loader = { path = "../loader" } spin-locked-app = { path = "../locked-app" } tempfile = { workspace = true } @@ -29,3 +30,8 @@ tokio = { workspace = true, features = ["fs"] } tokio-util = { version = "0.7", features = ["compat"] } tracing = { workspace = true } walkdir = "2" + +[dev-dependencies] +wasm-encoder = { workspace = true } +wit-component = { workspace = true, features = ["dummy-module"] } +wit-parser = { workspace = true } \ No newline at end of file diff --git a/crates/oci/src/client.rs b/crates/oci/src/client.rs index 6f38255197..eab6e39ea3 100644 --- a/crates/oci/src/client.rs +++ b/crates/oci/src/client.rs @@ -16,9 +16,10 @@ use reqwest::Url; use spin_common::sha256; use spin_common::ui::quoted_path; use spin_common::url::parse_file_url; +use spin_compose::ComponentSourceLoaderFs; use spin_loader::cache::Cache; use spin_loader::FilesMountStrategy; -use spin_locked_app::locked::{ContentPath, ContentRef, LockedApp}; +use spin_locked_app::locked::{ContentPath, ContentRef, LockedApp, LockedComponent}; use tokio::fs; use walkdir::WalkDir; @@ -58,6 +59,7 @@ const DEFAULT_CONTENT_REF_INLINE_MAX_SIZE: usize = 128; const DEFAULT_TOKEN_EXPIRATION_SECS: usize = 300; /// Mode of assembly of a Spin application into an OCI image +#[derive(Copy, Clone)] enum AssemblyMode { /// Assemble the application as one layer per component and one layer for /// every static asset included with a given component @@ -67,6 +69,15 @@ enum AssemblyMode { Archive, } +/// Indicates whether to compose the components of a Spin application when pushing an image. +#[derive(Copy, Clone)] +pub enum ComposeMode { + /// Compose components before pushing the image. + All, + /// Skip composing components before pushing the image. + Skip, +} + /// Client for interacting with an OCI registry for Spin applications. pub struct Client { /// Global cache for the metadata, Wasm modules, and static assets pulled from OCI registries. @@ -119,6 +130,7 @@ impl Client { reference: impl AsRef, annotations: Option>, infer_annotations: InferPredefinedAnnotations, + compose_mode: ComposeMode, ) -> Result> { let reference: Reference = reference .as_ref() @@ -137,8 +149,15 @@ impl Client { ) .await?; - self.push_locked_core(locked, auth, reference, annotations, infer_annotations) - .await + self.push_locked_core( + locked, + auth, + reference, + annotations, + infer_annotations, + compose_mode, + ) + .await } /// Push a Spin application to an OCI registry and return the digest (or None @@ -149,6 +168,7 @@ impl Client { reference: impl AsRef, annotations: Option>, infer_annotations: InferPredefinedAnnotations, + compose_mode: ComposeMode, ) -> Result> { let reference: Reference = reference .as_ref() @@ -156,8 +176,15 @@ impl Client { .with_context(|| format!("cannot parse reference {}", reference.as_ref()))?; let auth = Self::auth(&reference).await?; - self.push_locked_core(locked, auth, reference, annotations, infer_annotations) - .await + self.push_locked_core( + locked, + auth, + reference, + annotations, + infer_annotations, + compose_mode, + ) + .await } /// Push a Spin application to an OCI registry and return the digest (or None @@ -169,10 +196,11 @@ impl Client { reference: Reference, annotations: Option>, infer_annotations: InferPredefinedAnnotations, + compose_mode: ComposeMode, ) -> Result> { let mut locked_app = locked.clone(); let mut layers = self - .assemble_layers(&mut locked_app, AssemblyMode::Simple) + .assemble_layers(&mut locked_app, AssemblyMode::Simple, compose_mode) .await .context("could not assemble layers for locked application")?; @@ -183,7 +211,7 @@ impl Client { { locked_app = locked.clone(); layers = self - .assemble_layers(&mut locked_app, AssemblyMode::Archive) + .assemble_layers(&mut locked_app, AssemblyMode::Archive, compose_mode) .await .context("could not assemble archive layers for locked application")?; } @@ -246,16 +274,43 @@ impl Client { &mut self, locked: &mut LockedApp, assembly_mode: AssemblyMode, + compose_mode: ComposeMode, ) -> Result> { - let mut layers = Vec::new(); + let (mut layers, components) = match compose_mode { + ComposeMode::All => { + self.assemble_layers_composed(assembly_mode, locked.clone()) + .await? + } + ComposeMode::Skip => { + self.assemble_layers_uncomposed(assembly_mode, locked.clone()) + .await? + } + }; + + locked.components = components; + locked.metadata.remove("origin"); + + // Deduplicate layers + layers = layers.into_iter().unique().collect(); + + Ok(layers) + } + + async fn assemble_layers_uncomposed( + &mut self, + assembly_mode: AssemblyMode, + locked: LockedApp, + ) -> Result<(Vec, Vec)> { let mut components = Vec::new(); - for mut c in locked.clone().components { + let mut layers = Vec::new(); + + for mut c in locked.components { // Add the wasm module for the component as layers. let source = c - .clone() .source .content .source + .as_ref() .context("component loaded from disk should contain a file source")?; let source = parse_file_url(source.as_str())?; @@ -284,42 +339,76 @@ impl Client { } c.dependencies = deps; - let mut files = Vec::new(); - for f in c.files { - let source = f - .content - .source - .context("file mount loaded from disk should contain a file source")?; - let source = parse_file_url(source.as_str())?; + c.files = self + .assemble_content_layers(assembly_mode, &mut layers, c.files.as_slice()) + .await?; + components.push(c); + } - match assembly_mode { - AssemblyMode::Archive => self - .push_archive_layer(&source, &mut files, &mut layers) - .await - .context(format!( - "cannot push archive layer for source {}", - quoted_path(&source) - ))?, - AssemblyMode::Simple => self - .push_file_layers(&source, &mut files, &mut layers) - .await - .context(format!( - "cannot push file layers for source {}", - quoted_path(&source) - ))?, - } - } - c.files = files; + Ok((layers, components)) + } + async fn assemble_layers_composed( + &mut self, + assembly_mode: AssemblyMode, + locked: LockedApp, + ) -> Result<(Vec, Vec)> { + let mut components = Vec::new(); + let mut layers = Vec::new(); + + for mut c in locked.components { + let composed = spin_compose::compose(&ComponentSourceLoaderFs, &c) + .await + .with_context(|| { + format!("failed to resolve dependencies for component {:?}", c.id) + })?; + let layer = ImageLayer::new(composed, WASM_LAYER_MEDIA_TYPE.to_string(), None); + c.source.content = self.content_ref_for_layer(&layer); + c.dependencies.clear(); + layers.push(layer); + + c.files = self + .assemble_content_layers(assembly_mode, &mut layers, c.files.as_slice()) + .await?; components.push(c); } - locked.components = components; - locked.metadata.remove("origin"); - // Deduplicate layers - layers = layers.into_iter().unique().collect(); + Ok((layers, components)) + } - Ok(layers) + async fn assemble_content_layers( + &mut self, + assembly_mode: AssemblyMode, + layers: &mut Vec, + contents: &[ContentPath], + ) -> Result> { + let mut files = Vec::new(); + for f in contents { + let source = f + .content + .source + .as_ref() + .context("file mount loaded from disk should contain a file source")?; + let source = parse_file_url(source.as_str())?; + + match assembly_mode { + AssemblyMode::Archive => self + .push_archive_layer(&source, &mut files, layers) + .await + .context(format!( + "cannot push archive layer for source {}", + quoted_path(&source) + ))?, + AssemblyMode::Simple => self + .push_file_layers(&source, &mut files, layers) + .await + .context(format!( + "cannot push file layers for source {}", + quoted_path(&source) + ))?, + } + } + Ok(files) } /// Archive all of the files recursively under the source directory @@ -940,6 +1029,40 @@ mod test { .await .expect("should write file contents"); + // create a component with dependencies + const TEST_WIT: &str = " + package test:test; + + interface a { + a: func(); + } + + world dep-a { + export a; + } + + world root { + import a; + } + "; + + let root = generate_dummy_component(TEST_WIT, "root"); + let dep_a = generate_dummy_component(TEST_WIT, "dep-a"); + + let mut r = tokio::fs::File::create(working_dir.path().join("root.wasm")) + .await + .expect("should create component wasm file"); + r.write_all(&root) + .await + .expect("should write component wasm contents"); + + let mut a = tokio::fs::File::create(working_dir.path().join("dep_a.wasm")) + .await + .expect("should create component wasm file"); + a.write_all(&dep_a) + .await + .expect("should write component wasm contents"); + #[derive(Clone)] struct TestCase { name: &'static str, @@ -947,6 +1070,7 @@ mod test { locked_components: Vec, expected_layer_count: usize, expected_error: Option<&'static str>, + compose_mode: ComposeMode, } let tests: Vec = [ @@ -969,6 +1093,7 @@ mod test { }}]), expected_layer_count: 2, expected_error: None, + compose_mode: ComposeMode::Skip, }, TestCase { name: "One component layer and two file layers", @@ -993,6 +1118,7 @@ mod test { }]), expected_layer_count: 3, expected_error: None, + compose_mode: ComposeMode::Skip, }, TestCase { name: "One component layer and one file with inlined content", @@ -1013,9 +1139,10 @@ mod test { }]), expected_layer_count: 1, expected_error: None, + compose_mode: ComposeMode::Skip, }, TestCase { - name: "One component layer and one dependency component layer", + name: "One component layer and one dependency component layer skipping composition", opts: Some(ClientOpts{content_ref_inline_max_size: 0}), locked_components: from_json!([{ "id": "component1", @@ -1037,6 +1164,7 @@ mod test { }]), expected_layer_count: 2, expected_error: None, + compose_mode: ComposeMode::Skip, }, TestCase { name: "Component has no source", @@ -1051,6 +1179,7 @@ mod test { }]), expected_layer_count: 0, expected_error: Some("Invalid URL: \"\""), + compose_mode: ComposeMode::Skip, }, TestCase { name: "Duplicate component sources", @@ -1071,6 +1200,7 @@ mod test { }}]), expected_layer_count: 1, expected_error: None, + compose_mode: ComposeMode::Skip, }, TestCase { name: "Duplicate file paths", @@ -1108,6 +1238,32 @@ mod test { }]), expected_layer_count: 4, expected_error: None, + compose_mode: ComposeMode::Skip, + }, + TestCase { + name: "One component layer and one dependency component layer with composition", + opts: Some(ClientOpts{content_ref_inline_max_size: 0}), + locked_components: from_json!([{ + "id": "component-with-deps", + "source": { + "content_type": "application/wasm", + "source": format!("file://{}", working_dir.path().join("root.wasm").to_str().unwrap()), + "digest": "digest", + }, + "dependencies": { + "test:test/a": { + "source": { + "content_type": "application/wasm", + "source": format!("file://{}", working_dir.path().join("dep_a.wasm").to_str().unwrap()), + "digest": "digest", + }, + "export": null, + } + } + }]), + expected_layer_count: 1, + expected_error: None, + compose_mode: ComposeMode::All, }, ] .to_vec(); @@ -1138,7 +1294,7 @@ mod test { assert_eq!( e, client - .assemble_layers(&mut locked, AssemblyMode::Simple) + .assemble_layers(&mut locked, AssemblyMode::Simple, tc.compose_mode) .await .unwrap_err() .to_string(), @@ -1150,7 +1306,7 @@ mod test { assert_eq!( tc.expected_layer_count, client - .assemble_layers(&mut locked, AssemblyMode::Simple) + .assemble_layers(&mut locked, AssemblyMode::Simple, tc.compose_mode) .await .unwrap() .len(), @@ -1162,6 +1318,29 @@ mod test { } } + fn generate_dummy_component(wit: &str, world: &str) -> Vec { + let mut resolve = wit_parser::Resolve::default(); + let package_id = resolve.push_str("test", wit).expect("should parse WIT"); + let world_id = resolve + .select_world(package_id, Some(world)) + .expect("should select world"); + + let mut wasm = wit_component::dummy_module(&resolve, world_id); + wit_component::embed_component_metadata( + &mut wasm, + &resolve, + world_id, + wit_component::StringEncoding::UTF8, + ) + .expect("should embed component metadata"); + + let encoder = wit_component::ComponentEncoder::default() + .validate(true) + .module(&wasm) + .expect("should set module"); + encoder.encode().expect("should encode component") + } + fn annotatable_app() -> LockedApp { let mut meta_builder = spin_locked_app::values::ValuesMapBuilder::new(); meta_builder diff --git a/crates/oci/src/lib.rs b/crates/oci/src/lib.rs index 6153a5cd80..db61b36025 100644 --- a/crates/oci/src/lib.rs +++ b/crates/oci/src/lib.rs @@ -6,7 +6,7 @@ pub mod client; mod loader; pub mod utils; -pub use client::Client; +pub use client::{Client, ComposeMode}; pub use loader::OciLoader; /// URL scheme used for the locked app "origin" metadata field for OCI-sourced apps. diff --git a/crates/trigger/Cargo.toml b/crates/trigger/Cargo.toml index 0f46026c33..97948ac59c 100644 --- a/crates/trigger/Cargo.toml +++ b/crates/trigger/Cargo.toml @@ -24,7 +24,6 @@ serde = { workspace = true } serde_json = { workspace = true } spin-app = { path = "../app" } spin-common = { path = "../common" } -spin-componentize = { path = "../componentize" } spin-compose = { path = "../compose" } spin-core = { path = "../core" } spin-factor-key-value = { path = "../factor-key-value" } diff --git a/crates/trigger/src/loader.rs b/crates/trigger/src/loader.rs index 8479fa5e8c..92bbb3c2fe 100644 --- a/crates/trigger/src/loader.rs +++ b/crates/trigger/src/loader.rs @@ -1,5 +1,6 @@ use anyhow::Context as _; use spin_common::{ui::quoted_path, url::parse_file_url}; +use spin_compose::ComponentSourceLoaderFs; use spin_core::{async_trait, wasmtime, Component}; use spin_factors::AppComponent; @@ -88,7 +89,7 @@ impl spin_factors_executor::ComponentLoader for ComponentLoader { .with_context(|| format!("error deserializing component from {path:?}")); } - let composed = spin_compose::compose(&ComponentSourceLoader, component.locked) + let composed = spin_compose::compose(&ComponentSourceLoaderFs, component.locked) .await .with_context(|| { format!( @@ -101,32 +102,3 @@ impl spin_factors_executor::ComponentLoader for ComponentLoader { .with_context(|| format!("failed to compile component from {}", quoted_path(&path))) } } - -struct ComponentSourceLoader; - -#[async_trait] -impl spin_compose::ComponentSourceLoader for ComponentSourceLoader { - async fn load_component_source( - &self, - source: &spin_app::locked::LockedComponentSource, - ) -> anyhow::Result> { - let source = source - .content - .source - .as_ref() - .context("LockedComponentSource missing source field")?; - - let path = parse_file_url(source)?; - - let bytes: Vec = tokio::fs::read(&path).await.with_context(|| { - format!( - "failed to read component source from disk at path {}", - quoted_path(&path) - ) - })?; - - let component = spin_componentize::componentize_if_necessary(&bytes)?; - - Ok(component.into()) - } -} diff --git a/src/commands/registry.rs b/src/commands/registry.rs index a044ba6526..39a9cc4800 100644 --- a/src/commands/registry.rs +++ b/src/commands/registry.rs @@ -3,7 +3,7 @@ use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; use indicatif::{ProgressBar, ProgressStyle}; use spin_common::arg_parser::parse_kv; -use spin_oci::{client::InferPredefinedAnnotations, Client}; +use spin_oci::{client::InferPredefinedAnnotations, Client, ComposeMode}; use std::{io::Read, path::PathBuf, time::Duration}; /// Commands for working with OCI registries to distribute applications. @@ -49,6 +49,15 @@ pub struct Push { )] pub insecure: bool, + /// Compose component dependencies before pushing the application. + /// + /// The default is to compose before pushing, which maximises compatibility with + /// different Spin runtime hosts. Turning composition off can optimise + /// bandwidth for shared dependencies, but makes the pushed image incompatible + /// with hosts that cannot carry out composition themselves. + #[clap(long = "compose", default_value_t = true)] + pub compose: bool, + /// Specifies to perform `spin build` before pushing the application. #[clap(long, takes_value = false, env = ALWAYS_BUILD_ENV)] pub build: bool, @@ -88,12 +97,19 @@ impl Push { let _spinner = create_dotted_spinner(2000, "Pushing app to the Registry".to_owned()); + let compose_mode = if self.compose { + ComposeMode::All + } else { + ComposeMode::Skip + }; + let digest = client .push( &app_file, &self.reference, annotations, InferPredefinedAnnotations::All, + compose_mode, ) .await?; match digest {