diff --git a/Cargo.lock b/Cargo.lock index 2261cd8b9b6..e49b7336aa0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4389,6 +4389,7 @@ dependencies = [ "wasm-instrument", "wasmparser 0.101.1", "wasmtime", + "zstd 0.13.2", ] [[package]] @@ -8633,7 +8634,7 @@ dependencies = [ "sha2 0.9.9", "toml 0.5.11", "windows-sys 0.36.1", - "zstd", + "zstd 0.11.2+zstd.1.5.2", ] [[package]] @@ -9523,7 +9524,16 @@ version = "0.11.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ - "zstd-safe", + "zstd-safe 5.0.2+zstd.1.5.2", +] + +[[package]] +name = "zstd" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +dependencies = [ + "zstd-safe 7.2.1", ] [[package]] @@ -9536,6 +9546,15 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "7.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" version = "2.0.13+zstd.1.5.6" diff --git a/Cargo.toml b/Cargo.toml index 2d451750178..267f48668e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -164,6 +164,7 @@ webassembly-test = "0.1.0" web-sys = "0.3.69" web-time = "1.1.0" wit-bindgen = "0.24.0" +zstd = "0.13.2" linera-base = { version = "0.12.0", path = "./linera-base" } linera-chain = { version = "0.12.0", path = "./linera-chain" } diff --git a/examples/Cargo.lock b/examples/Cargo.lock index ced82907f93..57f82f91b11 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -3441,6 +3441,7 @@ dependencies = [ "wasm-encoder 0.24.1", "wasm-instrument", "wasmparser 0.101.1", + "zstd 0.13.2", ] [[package]] @@ -6690,7 +6691,7 @@ dependencies = [ "sha2 0.9.9", "toml 0.5.11", "windows-sys 0.36.1", - "zstd", + "zstd 0.11.2+zstd.1.5.2", ] [[package]] @@ -7469,7 +7470,16 @@ version = "0.11.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ - "zstd-safe", + "zstd-safe 5.0.2+zstd.1.5.2", +] + +[[package]] +name = "zstd" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +dependencies = [ + "zstd-safe 7.2.1", ] [[package]] @@ -7482,6 +7492,15 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "7.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" version = "2.0.10+zstd.1.5.6" diff --git a/linera-core/src/client.rs b/linera-core/src/client.rs index 85a31c9ca63..f57b787a515 100644 --- a/linera-core/src/client.rs +++ b/linera-core/src/client.rs @@ -2516,8 +2516,8 @@ where service: Bytecode, ) -> Result, ChainClientError> { self.execute_operation(Operation::System(SystemOperation::PublishBytecode { - contract: contract.clone(), - service: service.clone(), + contract: contract.into(), + service: service.into(), })) .await? .try_map(|certificate| { diff --git a/linera-core/src/unit_tests/wasm_worker_tests.rs b/linera-core/src/unit_tests/wasm_worker_tests.rs index 1ab7c69f120..b9bc02ed78c 100644 --- a/linera-core/src/unit_tests/wasm_worker_tests.rs +++ b/linera-core/src/unit_tests/wasm_worker_tests.rs @@ -122,8 +122,8 @@ where // Publish some bytecode. let publish_operation = SystemOperation::PublishBytecode { - contract: contract_bytecode, - service: service_bytecode, + contract: contract_bytecode.into(), + service: service_bytecode.into(), }; let publish_message = SystemMessage::BytecodePublished { transaction_index: 0, diff --git a/linera-execution/Cargo.toml b/linera-execution/Cargo.toml index d1689f1c4c6..495acb46b48 100644 --- a/linera-execution/Cargo.toml +++ b/linera-execution/Cargo.toml @@ -60,6 +60,7 @@ wasm-encoder = { workspace = true, optional = true } wasm-instrument = { workspace = true, optional = true, features = ["sign_ext"] } wasmparser = { workspace = true, optional = true } wasmtime = { workspace = true, optional = true } +zstd.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] tokio = { workspace = true, features = ["rt-multi-thread"] } diff --git a/linera-execution/src/lib.rs b/linera-execution/src/lib.rs index 6d73a8adc00..e042162d2f6 100644 --- a/linera-execution/src/lib.rs +++ b/linera-execution/src/lib.rs @@ -21,7 +21,7 @@ mod transaction_tracker; mod util; mod wasm; -use std::{fmt, str::FromStr, sync::Arc}; +use std::{fmt, io, str::FromStr, sync::Arc}; use async_graphql::SimpleObject; use async_trait::async_trait; @@ -151,6 +151,8 @@ pub enum ExecutionError { MissingRuntimeResponse, #[error("Bytecode ID {0:?} is invalid")] InvalidBytecodeId(BytecodeId), + #[error("Bytecode could not be decompressed")] + InvalidCompressedBytecode(#[source] io::Error), #[error("Owner is None")] OwnerIsNone, #[error("Application is not authorized to perform system operations on this chain: {0:}")] @@ -1068,9 +1070,56 @@ impl AsRef<[u8]> for Bytecode { } } -impl std::fmt::Debug for Bytecode { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { - f.debug_tuple("Bytecode").finish() +impl fmt::Debug for Bytecode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Bytecode").finish_non_exhaustive() + } +} + +/// A compressed WebAssembly module's bytecode. +#[derive(Clone, Deserialize, Eq, Hash, PartialEq, Serialize)] +pub struct CompressedBytecode { + #[serde(with = "serde_bytes")] + compressed_bytes: Vec, +} + +impl From<&Bytecode> for CompressedBytecode { + fn from(bytecode: &Bytecode) -> Self { + let compressed_bytes = zstd::stream::encode_all(&*bytecode.bytes, 19) + .expect("Compressing bytes in memory should not fail"); + + CompressedBytecode { compressed_bytes } + } +} + +impl From for CompressedBytecode { + fn from(bytecode: Bytecode) -> Self { + CompressedBytecode::from(&bytecode) + } +} + +impl TryFrom<&CompressedBytecode> for Bytecode { + type Error = ExecutionError; + + fn try_from(compressed_bytecode: &CompressedBytecode) -> Result { + let bytes = zstd::stream::decode_all(&*compressed_bytecode.compressed_bytes) + .map_err(ExecutionError::InvalidCompressedBytecode)?; + + Ok(Bytecode { bytes }) + } +} + +impl TryFrom for Bytecode { + type Error = ExecutionError; + + fn try_from(compressed_bytecode: CompressedBytecode) -> Result { + Bytecode::try_from(&compressed_bytecode) + } +} + +impl fmt::Debug for CompressedBytecode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CompressedBytecode").finish_non_exhaustive() } } diff --git a/linera-execution/src/system.rs b/linera-execution/src/system.rs index 5892de15ee0..02300fef99e 100644 --- a/linera-execution/src/system.rs +++ b/linera-execution/src/system.rs @@ -37,9 +37,9 @@ use {linera_base::prometheus_util, prometheus::IntCounterVec}; use crate::test_utils::SystemExecutionState; use crate::{ committee::{Committee, Epoch}, - ApplicationRegistryView, Bytecode, BytecodeLocation, ChannelName, ChannelSubscription, - Destination, ExecutionRuntimeContext, MessageContext, MessageKind, OperationContext, - QueryContext, RawExecutionOutcome, RawOutgoingMessage, TransactionTracker, + ApplicationRegistryView, BytecodeLocation, ChannelName, ChannelSubscription, + CompressedBytecode, Destination, ExecutionRuntimeContext, MessageContext, MessageKind, + OperationContext, QueryContext, RawExecutionOutcome, RawOutgoingMessage, TransactionTracker, UserApplicationDescription, UserApplicationId, }; @@ -157,8 +157,8 @@ pub enum SystemOperation { }, /// Publishes a new application bytecode. PublishBytecode { - contract: Bytecode, - service: Bytecode, + contract: CompressedBytecode, + service: CompressedBytecode, }, /// Publishes a new blob. PublishBlob { blob_id: BlobId }, @@ -1110,7 +1110,7 @@ mod tests { use linera_views::memory::MemoryContext; use super::*; - use crate::{ExecutionOutcome, ExecutionStateView, TestExecutionRuntimeContext}; + use crate::{Bytecode, ExecutionOutcome, ExecutionStateView, TestExecutionRuntimeContext}; /// Returns an execution state view and a matching operation context, for epoch 1, with root /// chain 0 as the admin ID and one empty committee. @@ -1141,8 +1141,8 @@ mod tests { async fn bytecode_message_index() { let (mut view, context) = new_view_and_context().await; let operation = SystemOperation::PublishBytecode { - contract: Bytecode::new(vec![]), - service: Bytecode::new(vec![]), + contract: Bytecode::new(vec![]).into(), + service: Bytecode::new(vec![]).into(), }; let mut txn_tracker = TransactionTracker::default(); let new_application = view diff --git a/linera-rpc/tests/snapshots/format__format.yaml.snap b/linera-rpc/tests/snapshots/format__format.yaml.snap index ec3dc02cf95..ace2ed3fab0 100644 --- a/linera-rpc/tests/snapshots/format__format.yaml.snap +++ b/linera-rpc/tests/snapshots/format__format.yaml.snap @@ -125,9 +125,6 @@ BlockProposal: - validated_block_certificate: OPTION: TYPENAME: LiteCertificate -Bytecode: - STRUCT: - - bytes: BYTES BytecodeId: STRUCT: - message_id: @@ -340,6 +337,9 @@ Committee: TYPENAME: ValidatorState - policy: TYPENAME: ResourceControlPolicy +CompressedBytecode: + STRUCT: + - compressed_bytes: BYTES CrateVersion: STRUCT: - major: U32 @@ -1000,9 +1000,9 @@ SystemOperation: PublishBytecode: STRUCT: - contract: - TYPENAME: Bytecode + TYPENAME: CompressedBytecode - service: - TYPENAME: Bytecode + TYPENAME: CompressedBytecode 9: PublishBlob: STRUCT: diff --git a/linera-sdk/src/test/chain.rs b/linera-sdk/src/test/chain.rs index c4bf99393a2..327339ebe61 100644 --- a/linera-sdk/src/test/chain.rs +++ b/linera-sdk/src/test/chain.rs @@ -24,7 +24,7 @@ use linera_execution::{ SystemChannel, SystemExecutionError, SystemMessage, SystemOperation, CREATE_APPLICATION_MESSAGE_INDEX, PUBLISH_BYTECODE_MESSAGE_INDEX, }, - Bytecode, ExecutionError, Message, Query, Response, + Bytecode, CompressedBytecode, ExecutionError, Message, Query, Response, }; use serde::Serialize; use tokio::{fs, sync::Mutex}; @@ -219,8 +219,12 @@ impl ActiveChain { /// Searches the Cargo manifest of the crate calling this method for binaries to use as the /// contract and service bytecodes. /// - /// Returns a tuple with the loaded contract and service [`Bytecode`]s. - async fn find_bytecodes_in(&self, repository: &Path) -> (Bytecode, Bytecode) { + /// Returns a tuple with the loaded contract and service [`CompressedBytecode`]s, + /// ready to be published. + async fn find_bytecodes_in( + &self, + repository: &Path, + ) -> (CompressedBytecode, CompressedBytecode) { let manifest_path = repository.join("Cargo.toml"); let cargo_manifest = Manifest::from_path(manifest_path).expect("Failed to load Cargo.toml manifest"); @@ -252,14 +256,14 @@ impl ActiveChain { let contract_path = base_path.join(format!("{}.wasm", contract_binary)); let service_path = base_path.join(format!("{}.wasm", service_binary)); - ( - Bytecode::load_from_file(contract_path) - .await - .expect("Failed to load contract bytecode from file"), - Bytecode::load_from_file(service_path) - .await - .expect("Failed to load service bytecode from file"), - ) + let contract = Bytecode::load_from_file(contract_path) + .await + .expect("Failed to load contract bytecode from file"); + let service = Bytecode::load_from_file(service_path) + .await + .expect("Failed to load service bytecode from file"); + + (contract.into(), service.into()) } /// Searches for the directory where the built WebAssembly binaries should be. diff --git a/linera-storage/src/lib.rs b/linera-storage/src/lib.rs index 764ef2fdbc0..8ecabcfaa6f 100644 --- a/linera-storage/src/lib.rs +++ b/linera-storage/src/lib.rs @@ -327,7 +327,7 @@ pub trait Storage: Sized { unreachable!("unexpected bytecode operation"); }; Ok(Arc::new( - WasmContractModule::new(contract, wasm_runtime).await?, + WasmContractModule::new(contract.try_into()?, wasm_runtime).await?, )) } @@ -360,7 +360,7 @@ pub trait Storage: Sized { unreachable!("unexpected bytecode operation"); }; Ok(Arc::new( - WasmServiceModule::new(service, wasm_runtime).await?, + WasmServiceModule::new(service.try_into()?, wasm_runtime).await?, )) }