Implement the local JS snippets RFC

This commit is an implementation of [RFC 6] which enables crates to
inline local JS snippets into the final output artifact of
`wasm-bindgen`. This is accompanied with a few minor breaking changes
which are intended to be relatively minor in practice:

* The `module` attribute disallows paths starting with `./` and `../`.
  It requires paths starting with `/` to actually exist on the filesystem.
* The `--browser` flag no longer emits bundler-compatible code, but
  rather emits an ES module that can be natively loaded into a browser.

Otherwise be sure to check out [the RFC][RFC 6] for more details, and
otherwise this should implement at least the MVP version of the RFC!
Notably at this time JS snippets with `--nodejs` or `--no-modules` are
not supported and will unconditionally generate an error.

[RFC 6]: rustwasm/rfcs#6

Cargo.toml

@@ -82,6 +82,7 @@ members = [
   "examples/webaudio",
   "examples/webgl",
   "examples/without-a-bundler",
+  "examples/without-a-bundler-no-modules",
   "tests/no-std",
 ]
 exclude = ['crates/typescript']

crates/backend/src/ast.rs

@@ -2,6 +2,7 @@ use proc_macro2::{Ident, Span};
 use shared;
 use syn;
 use Diagnostic;
+use std::hash::{Hash, Hasher};
 
 /// An abstract syntax tree representing a rust program. Contains
 /// extra information for joining up this rust code with javascript.
@@ -24,6 +25,8 @@ pub struct Program {
     pub dictionaries: Vec<Dictionary>,
     /// custom typescript sections to be included in the definition file
     pub typescript_custom_sections: Vec<String>,
+    /// Inline JS snippets
+    pub inline_js: Vec<String>,
 }
 
 /// A rust to js interface. Allows interaction with rust objects/functions
@@ -66,11 +69,37 @@ pub enum MethodSelf {
 #[cfg_attr(feature = "extra-traits", derive(Debug, PartialEq, Eq))]
 #[derive(Clone)]
 pub struct Import {
-    pub module: Option<String>,
+    pub module: ImportModule,
     pub js_namespace: Option<Ident>,
     pub kind: ImportKind,
 }
 
+#[cfg_attr(feature = "extra-traits", derive(Debug, PartialEq, Eq))]
+#[derive(Clone)]
+pub enum ImportModule {
+    None,
+    Named(String, Span),
+    Inline(usize, Span),
+}
+
+impl Hash for ImportModule {
+    fn hash<H: Hasher>(&self, h: &mut H) {
+        match self {
+            ImportModule::None => {
+                0u8.hash(h);
+            }
+            ImportModule::Named(name, _) => {
+                1u8.hash(h);
+                name.hash(h);
+            }
+            ImportModule::Inline(idx, _) => {
+                2u8.hash(h);
+                idx.hash(h);
+            }
+        }
+    }
+}
+
 #[cfg_attr(feature = "extra-traits", derive(Debug, PartialEq, Eq))]
 #[derive(Clone)]
 pub enum ImportKind {

crates/backend/src/codegen.rs

@@ -94,25 +94,46 @@ impl TryToTokens for ast::Program {
             shared::SCHEMA_VERSION,
             shared::version()
         );
+        let encoded = encode::encode(self)?;
         let mut bytes = Vec::new();
         bytes.push((prefix_json.len() >> 0) as u8);
         bytes.push((prefix_json.len() >> 8) as u8);
         bytes.push((prefix_json.len() >> 16) as u8);
         bytes.push((prefix_json.len() >> 24) as u8);
         bytes.extend_from_slice(prefix_json.as_bytes());
-        bytes.extend_from_slice(&encode::encode(self)?);
+        bytes.extend_from_slice(&encoded.custom_section);
 
         let generated_static_length = bytes.len();
         let generated_static_value = syn::LitByteStr::new(&bytes, Span::call_site());
 
+        // We already consumed the contents of included files when generating
+        // the custom section, but we want to make sure that updates to the
+        // generated files will cause this macro to rerun incrementally. To do
+        // that we use `include_str!` to force rustc to think it has a
+        // dependency on these files. That way when the file changes Cargo will
+        // automatically rerun rustc which will rerun this macro. Other than
+        // this we don't actually need the results of the `include_str!`, so
+        // it's just shoved into an anonymous static.
+        let file_dependencies = encoded.included_files
+            .iter()
+            .map(|file| {
+                let file = file.to_str().unwrap();
+                quote! { include_str!(#file) }
+            });
+
         (quote! {
             #[allow(non_upper_case_globals)]
             #[cfg(target_arch = "wasm32")]
             #[link_section = "__wasm_bindgen_unstable"]
             #[doc(hidden)]
             #[allow(clippy::all)]
-            pub static #generated_static_name: [u8; #generated_static_length] =
-                *#generated_static_value;
+            pub static #generated_static_name: [u8; #generated_static_length] = {
+                #[doc(hidden)]
+                static _INCLUDED_FILES: &[&str] = &[#(#file_dependencies),*];
+
+                *#generated_static_value
+            };
+
         })
         .to_tokens(tokens);
 

crates/backend/src/encode.rs

@@ -1,26 +1,50 @@
-use std::cell::RefCell;
-use std::collections::HashMap;
-
 use proc_macro2::{Ident, Span};
+use std::cell::RefCell;
+use std::collections::{HashMap, HashSet};
+use std::env;
+use std::fs;
+use std::path::PathBuf;
+use util::ShortHash;
 
 use ast;
 use Diagnostic;
 
-pub fn encode(program: &ast::Program) -> Result<Vec<u8>, Diagnostic> {
+pub struct EncodeResult {
+    pub custom_section: Vec<u8>,
+    pub included_files: Vec<PathBuf>,
+}
+
+pub fn encode(program: &ast::Program) -> Result<EncodeResult, Diagnostic> {
     let mut e = Encoder::new();
     let i = Interner::new();
     shared_program(program, &i)?.encode(&mut e);
-    Ok(e.finish())
+    let custom_section = e.finish();
+    let included_files = i.files.borrow().values().map(|p| &p.path).cloned().collect();
+    Ok(EncodeResult { custom_section, included_files })
 }
 
 struct Interner {
     map: RefCell<HashMap<Ident, String>>,
+    strings: RefCell<HashSet<String>>,
+    files: RefCell<HashMap<String, LocalFile>>,
+    root: PathBuf,
+    crate_name: String,
+}
+
+struct LocalFile {
+    path: PathBuf,
+    definition: Span,
+    new_identifier: String,
 }
 
 impl Interner {
     fn new() -> Interner {
         Interner {
             map: RefCell::new(HashMap::new()),
+            strings: RefCell::new(HashSet::new()),
+            files: RefCell::new(HashMap::new()),
+            root: env::current_dir().unwrap(),
+            crate_name: env::var("CARGO_PKG_NAME").unwrap(),
         }
     }
 
@@ -34,7 +58,45 @@ impl Interner {
     }
 
     fn intern_str(&self, s: &str) -> &str {
-        self.intern(&Ident::new(s, Span::call_site()))
+        let mut strings = self.strings.borrow_mut();
+        if let Some(s) = strings.get(s) {
+            return unsafe { &*(&**s as *const str) };
+        }
+        strings.insert(s.to_string());
+        drop(strings);
+        self.intern_str(s)
+    }
+
+    /// Given an import to a local module `id` this generates a unique module id
+    /// to assign to the contents of `id`.
+    ///
+    /// Note that repeated invocations of this function will be memoized, so the
+    /// same `id` will always return the same resulting unique `id`.
+    fn resolve_import_module(&self, id: &str, span: Span) -> Result<&str, Diagnostic> {
+        let mut files = self.files.borrow_mut();
+        if let Some(file) = files.get(id) {
+            return Ok(self.intern_str(&file.new_identifier))
+        }
+        let path = if id.starts_with("/") {
+            self.root.join(&id[1..])
+        } else if id.starts_with("./") || id.starts_with("../") {
+            let msg = "relative module paths aren't supported yet";
+            return Err(Diagnostic::span_error(span, msg))
+        } else {
+            return Ok(self.intern_str(&id))
+        };
+
+        // Generate a unique ID which is somewhat readable as well, so mix in
+        // the crate name, hash to make it unique, and then the original path.
+        let new_identifier = format!("{}-{}{}", self.crate_name, ShortHash(0), id);
+        let file = LocalFile {
+            path,
+            definition: span,
+            new_identifier,
+        };
+        files.insert(id.to_string(), file);
+        drop(files);
+        self.resolve_import_module(id, span)
     }
 }
 
@@ -64,8 +126,29 @@ fn shared_program<'a>(
             .iter()
             .map(|x| -> &'a str { &x })
             .collect(),
-        // version: shared::version(),
-        // schema_version: shared::SCHEMA_VERSION.to_string(),
+        local_modules: intern
+            .files
+            .borrow()
+            .values()
+            .map(|file| {
+                fs::read_to_string(&file.path)
+                    .map(|s| {
+                        LocalModule {
+                            identifier: intern.intern_str(&file.new_identifier),
+                            contents: intern.intern_str(&s),
+                        }
+                    })
+                    .map_err(|e| {
+                        let msg = format!("failed to read file `{}`: {}", file.path.display(), e);
+                        Diagnostic::span_error(file.definition, msg)
+                    })
+            })
+            .collect::<Result<Vec<_>, _>>()?,
+        inline_js: prog
+            .inline_js
+            .iter()
+            .map(|js| intern.intern_str(js))
+            .collect(),
     })
 }
 
@@ -111,7 +194,13 @@ fn shared_variant<'a>(v: &'a ast::Variant, intern: &'a Interner) -> EnumVariant<
 
 fn shared_import<'a>(i: &'a ast::Import, intern: &'a Interner) -> Result<Import<'a>, Diagnostic> {
     Ok(Import {
-        module: i.module.as_ref().map(|s| &**s),
+        module: match &i.module {
+            ast::ImportModule::Named(m, span) => {
+                ImportModule::Named(intern.resolve_import_module(m, *span)?)
+            }
+            ast::ImportModule::Inline(idx, _) => ImportModule::Inline(*idx as u32),
+            ast::ImportModule::None => ImportModule::None,
+        },
         js_namespace: i.js_namespace.as_ref().map(|s| intern.intern(s)),
         kind: shared_import_kind(&i.kind, intern)?,
     })

crates/backend/src/util.rs

@@ -94,7 +94,7 @@ pub fn ident_ty(ident: Ident) -> syn::Type {
 
 pub fn wrap_import_function(function: ast::ImportFunction) -> ast::Import {
     ast::Import {
-        module: None,
+        module: ast::ImportModule::None,
         js_namespace: None,
         kind: ast::ImportKind::Function(function),
     }