Start compiling module-linking modules (#2093)

This commit is intended to be the first of many in implementing the
module linking proposal. At this time this builds on #2059 so it
shouldn't land yet. The goal of this commit is to compile bare-bones
modules which use module linking, e.g. those with nested modules.

My hope with module linking is that almost everything in wasmtime only
needs mild refactorings to handle it. The goal is that all per-module
structures are still per-module and at the top level there's just a
`Vec` containing a bunch of modules. That's implemented currently where
`wasmtime::Module` contains `Arc<[CompiledModule]>` and an index of
which one it's pointing to. This should enable
serialization/deserialization of any module in a nested modules
scenario, no matter how you got it.

Tons of features of the module linking proposal are missing from this
commit. For example instantiation flat out doesn't work, nor does
import/export of modules or instances. That'll be coming as future
commits, but the purpose here is to start laying groundwork in Wasmtime
for handling lots of modules in lots of places.

cranelift/wasm/src/environ/spec.rs

@@ -847,4 +847,31 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment {
     fn wasm_features(&self) -> WasmFeatures {
         WasmFeatures::default()
     }
+
+    /// Indicates that this module will have `amount` submodules.
+    ///
+    /// Note that this is just child modules of this module, and each child
+    /// module may have yet more submodules.
+    fn reserve_modules(&mut self, amount: u32) {
+        drop(amount);
+    }
+
+    /// Called at the beginning of translating a module.
+    ///
+    /// The `index` argument is a monotonically increasing index which
+    /// corresponds to the nth module that's being translated. This is not the
+    /// 32-bit index in the current module's index space. For example the first
+    /// call to `module_start` will have index 0.
+    ///
+    /// Note that for nested modules this may be called multiple times.
+    fn module_start(&mut self, index: usize) {
+        drop(index);
+    }
+
+    /// Called at the end of translating a module.
+    ///
+    /// Note that for nested modules this may be called multiple times.
+    fn module_end(&mut self, index: usize) {
+        drop(index);
+    }
 }

cranelift/wasm/src/module_translator.rs

@@ -8,6 +8,7 @@ use crate::sections_translator::{
 };
 use crate::state::ModuleTranslationState;
 use cranelift_codegen::timing;
+use std::prelude::v1::*;
 use wasmparser::{NameSectionReader, Parser, Payload, Validator};
 
 /// Translate a sequence of bytes forming a valid Wasm binary into a list of valid Cranelift IR
@@ -20,14 +21,23 @@ pub fn translate_module<'data>(
     let mut module_translation_state = ModuleTranslationState::new();
     let mut validator = Validator::new();
     validator.wasm_features(environ.wasm_features());
+    let mut stack = Vec::new();
+    let mut modules = 1;
+    let mut cur_module = 0;
 
     for payload in Parser::new(0).parse_all(data) {
         match payload? {
             Payload::Version { num, range } => {
                 validator.version(num, &range)?;
+                environ.module_start(cur_module);
             }
             Payload::End => {
                 validator.end()?;
+                environ.module_end(cur_module);
+                if let Some((other, other_index)) = stack.pop() {
+                    validator = other;
+                    cur_module = other_index;
+                }
             }
 
             Payload::TypeSection(types) => {
@@ -97,7 +107,7 @@ pub fn translate_module<'data>(
 
             Payload::ModuleSection(s) => {
                 validator.module_section(&s)?;
-                unimplemented!("module linking not implemented yet")
+                environ.reserve_modules(s.get_count());
             }
             Payload::InstanceSection(s) => {
                 validator.instance_section(&s)?;
@@ -113,11 +123,14 @@ pub fn translate_module<'data>(
                 size: _,
             } => {
                 validator.module_code_section_start(count, &range)?;
-                unimplemented!("module linking not implemented yet")
             }
 
             Payload::ModuleCodeSectionEntry { .. } => {
-                unimplemented!("module linking not implemented yet")
+                let subvalidator = validator.module_code_section_entry();
+                stack.push((validator, cur_module));
+                validator = subvalidator;
+                cur_module = modules;
+                modules += 1;
             }
 
             Payload::CustomSection {

cranelift/wasm/src/sections_translator.rs

@@ -41,25 +41,43 @@ pub fn parse_type_section(
     environ.reserve_signatures(count)?;
 
     for entry in types {
-        if let Ok(TypeDef::Func(wasm_func_ty)) = entry {
-            let mut sig =
-                Signature::new(ModuleEnvironment::target_config(environ).default_call_conv);
-            sig.params.extend(wasm_func_ty.params.iter().map(|ty| {
-                let cret_arg: ir::Type = type_to_type(*ty, environ)
-                    .expect("only numeric types are supported in function signatures");
-                AbiParam::new(cret_arg)
-            }));
-            sig.returns.extend(wasm_func_ty.returns.iter().map(|ty| {
-                let cret_arg: ir::Type = type_to_type(*ty, environ)
-                    .expect("only numeric types are supported in function signatures");
-                AbiParam::new(cret_arg)
-            }));
-            environ.declare_signature(wasm_func_ty.clone().try_into()?, sig)?;
-            module_translation_state
-                .wasm_types
-                .push((wasm_func_ty.params, wasm_func_ty.returns));
-        } else {
-            unimplemented!("module linking not implemented yet")
+        match entry? {
+            TypeDef::Func(wasm_func_ty) => {
+                let mut sig =
+                    Signature::new(ModuleEnvironment::target_config(environ).default_call_conv);
+                sig.params.extend(wasm_func_ty.params.iter().map(|ty| {
+                    let cret_arg: ir::Type = type_to_type(*ty, environ)
+                        .expect("only numeric types are supported in function signatures");
+                    AbiParam::new(cret_arg)
+                }));
+                sig.returns.extend(wasm_func_ty.returns.iter().map(|ty| {
+                    let cret_arg: ir::Type = type_to_type(*ty, environ)
+                        .expect("only numeric types are supported in function signatures");
+                    AbiParam::new(cret_arg)
+                }));
+                environ.declare_signature(wasm_func_ty.clone().try_into()?, sig)?;
+                module_translation_state
+                    .wasm_types
+                    .push((wasm_func_ty.params, wasm_func_ty.returns));
+            }
+
+            // Not implemented yet for module linking. Push dummy function types
+            // though to keep the function type index space consistent. We'll
+            // want an actual implementation here that handles this eventually.
+            TypeDef::Module(_) | TypeDef::Instance(_) => {
+                let sig =
+                    Signature::new(ModuleEnvironment::target_config(environ).default_call_conv);
+                environ.declare_signature(
+                    crate::environ::WasmFuncType {
+                        params: Box::new([]),
+                        returns: Box::new([]),
+                    },
+                    sig,
+                )?;
+                module_translation_state
+                    .wasm_types
+                    .push((Box::new([]), Box::new([])));
+            }
         }
     }
     Ok(())

crates/cranelift/src/lib.rs

@@ -99,7 +99,7 @@ use std::sync::Mutex;
 use wasmtime_environ::{
     CompileError, CompiledFunction, Compiler, FunctionAddressMap, FunctionBodyData,
     InstructionAddressMap, ModuleTranslation, Relocation, RelocationTarget, StackMapInformation,
-    TrapInformation,
+    TrapInformation, Tunables,
 };
 
 mod func_environ;
@@ -356,9 +356,9 @@ impl Compiler for Cranelift {
         func_index: DefinedFuncIndex,
         mut input: FunctionBodyData<'_>,
         isa: &dyn isa::TargetIsa,
+        tunables: &Tunables,
     ) -> Result<CompiledFunction, CompileError> {
         let module = &translation.module;
-        let tunables = &translation.tunables;
         let func_index = module.func_index(func_index);
         let mut context = Context::new();
         context.func.name = get_func_name(func_index);

crates/environ/src/compilation.rs

@@ -1,7 +1,7 @@
 //! A `Compilation` contains the compiled function bodies for a WebAssembly
 //! module.
 
-use crate::{FunctionAddressMap, FunctionBodyData, ModuleTranslation};
+use crate::{FunctionAddressMap, FunctionBodyData, ModuleTranslation, Tunables};
 use cranelift_codegen::{binemit, ir, isa, isa::unwind::UnwindInfo};
 use cranelift_entity::PrimaryMap;
 use cranelift_wasm::{DefinedFuncIndex, FuncIndex, WasmError};
@@ -103,5 +103,6 @@ pub trait Compiler: Send + Sync {
         index: DefinedFuncIndex,
         data: FunctionBodyData<'_>,
         isa: &dyn isa::TargetIsa,
+        tunables: &Tunables,
     ) -> Result<CompiledFunction, CompileError>;
 }

crates/environ/src/module.rs

@@ -325,6 +325,12 @@ impl Module {
     }
 }
 
+impl Default for Module {
+    fn default() -> Module {
+        Module::new()
+    }
+}
+
 mod passive_data_serde {
     use super::{Arc, DataIndex, HashMap};
     use serde::{de::MapAccess, de::Visitor, ser::SerializeMap, Deserializer, Serializer};