Compress Wasm bytecodes when publishing (#2382)

* Refactor to use imported `fmt` module

Instead of specifying the full path.

* Use `debug_struct` instead of `debug_tuple`

The type was not a tuple.

* Create a `CompressedBytecode` helper type

Use Zstandard to compress Wasm bytecodes.

* Require compressed bytecodes when publishing

Only accept bytecodes published if they are compressed.

* Update RPC schema

Use `CompressedBytecode` instead of `Bytecode`.

Cargo.toml

@@ -164,6 +164,7 @@ webassembly-test = "0.1.0"
 web-sys = "0.3.69"
 web-time = "1.1.0"
 wit-bindgen = "0.24.0"
+zstd = "0.13.2"
 
 linera-base = { version = "0.12.0", path = "./linera-base" }
 linera-chain = { version = "0.12.0", path = "./linera-chain" }

linera-core/src/client.rs

@@ -2516,8 +2516,8 @@ where
         service: Bytecode,
     ) -> Result<ClientOutcome<(BytecodeId, Certificate)>, ChainClientError> {
         self.execute_operation(Operation::System(SystemOperation::PublishBytecode {
-            contract: contract.clone(),
-            service: service.clone(),
+            contract: contract.into(),
+            service: service.into(),
         }))
         .await?
         .try_map(|certificate| {

linera-core/src/unit_tests/wasm_worker_tests.rs

@@ -122,8 +122,8 @@ where
 
     // Publish some bytecode.
     let publish_operation = SystemOperation::PublishBytecode {
-        contract: contract_bytecode,
-        service: service_bytecode,
+        contract: contract_bytecode.into(),
+        service: service_bytecode.into(),
     };
     let publish_message = SystemMessage::BytecodePublished {
         transaction_index: 0,

linera-execution/Cargo.toml

@@ -60,6 +60,7 @@ wasm-encoder = { workspace = true, optional = true }
 wasm-instrument = { workspace = true, optional = true, features = ["sign_ext"] }
 wasmparser = { workspace = true, optional = true }
 wasmtime = { workspace = true, optional = true }
+zstd.workspace = true
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 tokio = { workspace = true, features = ["rt-multi-thread"] }

linera-execution/src/lib.rs

@@ -21,7 +21,7 @@ mod transaction_tracker;
 mod util;
 mod wasm;
 
-use std::{fmt, str::FromStr, sync::Arc};
+use std::{fmt, io, str::FromStr, sync::Arc};
 
 use async_graphql::SimpleObject;
 use async_trait::async_trait;
@@ -151,6 +151,8 @@ pub enum ExecutionError {
     MissingRuntimeResponse,
     #[error("Bytecode ID {0:?} is invalid")]
     InvalidBytecodeId(BytecodeId),
+    #[error("Bytecode could not be decompressed")]
+    InvalidCompressedBytecode(#[source] io::Error),
     #[error("Owner is None")]
     OwnerIsNone,
     #[error("Application is not authorized to perform system operations on this chain: {0:}")]
@@ -1068,9 +1070,56 @@ impl AsRef<[u8]> for Bytecode {
     }
 }
 
-impl std::fmt::Debug for Bytecode {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
-        f.debug_tuple("Bytecode").finish()
+impl fmt::Debug for Bytecode {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Bytecode").finish_non_exhaustive()
+    }
+}
+
+/// A compressed WebAssembly module's bytecode.
+#[derive(Clone, Deserialize, Eq, Hash, PartialEq, Serialize)]
+pub struct CompressedBytecode {
+    #[serde(with = "serde_bytes")]
+    compressed_bytes: Vec<u8>,
+}
+
+impl From<&Bytecode> for CompressedBytecode {
+    fn from(bytecode: &Bytecode) -> Self {
+        let compressed_bytes = zstd::stream::encode_all(&*bytecode.bytes, 19)
+            .expect("Compressing bytes in memory should not fail");
+
+        CompressedBytecode { compressed_bytes }
+    }
+}
+
+impl From<Bytecode> for CompressedBytecode {
+    fn from(bytecode: Bytecode) -> Self {
+        CompressedBytecode::from(&bytecode)
+    }
+}
+
+impl TryFrom<&CompressedBytecode> for Bytecode {
+    type Error = ExecutionError;
+
+    fn try_from(compressed_bytecode: &CompressedBytecode) -> Result<Self, Self::Error> {
+        let bytes = zstd::stream::decode_all(&*compressed_bytecode.compressed_bytes)
+            .map_err(ExecutionError::InvalidCompressedBytecode)?;
+
+        Ok(Bytecode { bytes })
+    }
+}
+
+impl TryFrom<CompressedBytecode> for Bytecode {
+    type Error = ExecutionError;
+
+    fn try_from(compressed_bytecode: CompressedBytecode) -> Result<Self, Self::Error> {
+        Bytecode::try_from(&compressed_bytecode)
+    }
+}
+
+impl fmt::Debug for CompressedBytecode {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("CompressedBytecode").finish_non_exhaustive()
     }
 }
 

linera-execution/src/system.rs

@@ -37,9 +37,9 @@ use {linera_base::prometheus_util, prometheus::IntCounterVec};
 use crate::test_utils::SystemExecutionState;
 use crate::{
     committee::{Committee, Epoch},
-    ApplicationRegistryView, Bytecode, BytecodeLocation, ChannelName, ChannelSubscription,
-    Destination, ExecutionRuntimeContext, MessageContext, MessageKind, OperationContext,
-    QueryContext, RawExecutionOutcome, RawOutgoingMessage, TransactionTracker,
+    ApplicationRegistryView, BytecodeLocation, ChannelName, ChannelSubscription,
+    CompressedBytecode, Destination, ExecutionRuntimeContext, MessageContext, MessageKind,
+    OperationContext, QueryContext, RawExecutionOutcome, RawOutgoingMessage, TransactionTracker,
     UserApplicationDescription, UserApplicationId,
 };
 
@@ -157,8 +157,8 @@ pub enum SystemOperation {
     },
     /// Publishes a new application bytecode.
     PublishBytecode {
-        contract: Bytecode,
-        service: Bytecode,
+        contract: CompressedBytecode,
+        service: CompressedBytecode,
     },
     /// Publishes a new blob.
     PublishBlob { blob_id: BlobId },
@@ -1110,7 +1110,7 @@ mod tests {
     use linera_views::memory::MemoryContext;
 
     use super::*;
-    use crate::{ExecutionOutcome, ExecutionStateView, TestExecutionRuntimeContext};
+    use crate::{Bytecode, ExecutionOutcome, ExecutionStateView, TestExecutionRuntimeContext};
 
     /// Returns an execution state view and a matching operation context, for epoch 1, with root
     /// chain 0 as the admin ID and one empty committee.
@@ -1141,8 +1141,8 @@ mod tests {
     async fn bytecode_message_index() {
         let (mut view, context) = new_view_and_context().await;
         let operation = SystemOperation::PublishBytecode {
-            contract: Bytecode::new(vec![]),
-            service: Bytecode::new(vec![]),
+            contract: Bytecode::new(vec![]).into(),
+            service: Bytecode::new(vec![]).into(),
         };
         let mut txn_tracker = TransactionTracker::default();
         let new_application = view

linera-rpc/tests/snapshots/format__format.yaml.snap

@@ -125,9 +125,6 @@ BlockProposal:
     - validated_block_certificate:
         OPTION:
           TYPENAME: LiteCertificate
-Bytecode:
-  STRUCT:
-    - bytes: BYTES
 BytecodeId:
   STRUCT:
     - message_id:
@@ -340,6 +337,9 @@ Committee:
             TYPENAME: ValidatorState
     - policy:
         TYPENAME: ResourceControlPolicy
+CompressedBytecode:
+  STRUCT:
+    - compressed_bytes: BYTES
 CrateVersion:
   STRUCT:
     - major: U32
@@ -1000,9 +1000,9 @@ SystemOperation:
       PublishBytecode:
         STRUCT:
           - contract:
-              TYPENAME: Bytecode
+              TYPENAME: CompressedBytecode
           - service:
-              TYPENAME: Bytecode
+              TYPENAME: CompressedBytecode
     9:
       PublishBlob:
         STRUCT:

linera-sdk/src/test/chain.rs

@@ -24,7 +24,7 @@ use linera_execution::{
         SystemChannel, SystemExecutionError, SystemMessage, SystemOperation,
         CREATE_APPLICATION_MESSAGE_INDEX, PUBLISH_BYTECODE_MESSAGE_INDEX,
     },
-    Bytecode, ExecutionError, Message, Query, Response,
+    Bytecode, CompressedBytecode, ExecutionError, Message, Query, Response,
 };
 use serde::Serialize;
 use tokio::{fs, sync::Mutex};
@@ -219,8 +219,12 @@ impl ActiveChain {
     /// Searches the Cargo manifest of the crate calling this method for binaries to use as the
     /// contract and service bytecodes.
     ///
-    /// Returns a tuple with the loaded contract and service [`Bytecode`]s.
-    async fn find_bytecodes_in(&self, repository: &Path) -> (Bytecode, Bytecode) {
+    /// Returns a tuple with the loaded contract and service [`CompressedBytecode`]s,
+    /// ready to be published.
+    async fn find_bytecodes_in(
+        &self,
+        repository: &Path,
+    ) -> (CompressedBytecode, CompressedBytecode) {
         let manifest_path = repository.join("Cargo.toml");
         let cargo_manifest =
             Manifest::from_path(manifest_path).expect("Failed to load Cargo.toml manifest");
@@ -252,14 +256,14 @@ impl ActiveChain {
         let contract_path = base_path.join(format!("{}.wasm", contract_binary));
         let service_path = base_path.join(format!("{}.wasm", service_binary));
 
-        (
-            Bytecode::load_from_file(contract_path)
-                .await
-                .expect("Failed to load contract bytecode from file"),
-            Bytecode::load_from_file(service_path)
-                .await
-                .expect("Failed to load service bytecode from file"),
-        )
+        let contract = Bytecode::load_from_file(contract_path)
+            .await
+            .expect("Failed to load contract bytecode from file");
+        let service = Bytecode::load_from_file(service_path)
+            .await
+            .expect("Failed to load service bytecode from file");
+
+        (contract.into(), service.into())
     }
 
     /// Searches for the directory where the built WebAssembly binaries should be.

linera-storage/src/lib.rs

@@ -327,7 +327,7 @@ pub trait Storage: Sized {
             unreachable!("unexpected bytecode operation");
         };
         Ok(Arc::new(
-            WasmContractModule::new(contract, wasm_runtime).await?,
+            WasmContractModule::new(contract.try_into()?, wasm_runtime).await?,
         ))
     }
 
@@ -360,7 +360,7 @@ pub trait Storage: Sized {
             unreachable!("unexpected bytecode operation");
         };
         Ok(Arc::new(
-            WasmServiceModule::new(service, wasm_runtime).await?,
+            WasmServiceModule::new(service.try_into()?, wasm_runtime).await?,
         ))
     }