Add custom "emscripten_metadata" section to standalone WASM (#7815)

Currently it's not possible to execute Emscripten-generated WASM files without parsing certain data from the accompanying JS. This change adds an option EMIT_EMSCRIPTEN_METADATA to emit that data to the wasm file itself so that standalone WASM files are executable by third-party WASM runtimes.

AUTHORS

@@ -378,3 +378,4 @@ a license to everyone to use it as detailed in LICENSE.)
 * Gabriel Cuvillier <contact@gabrielcuvillier.pro>
 * Thomas Lively <tlively@google.com> (copyright owned by Google, Inc.)
 * Brandon Surmanski <b.surmanski@gmail.com>
+* Rian Hunter <rian@alum.mit.edu>

emcc.py

@@ -2575,7 +2575,14 @@ def do_binaryen(target, asm_target, options, memfile, wasm_binary_target,
     shutil.move(wso, wasm_binary_target)
     if not shared.Settings.WASM_BACKEND and not DEBUG:
       os.unlink(asm_target) # we don't need the asm.js, it can just confuse
+
+  if shared.Settings.EMIT_EMSCRIPTEN_METADATA:
+    wso = shared.WebAssembly.add_emscripten_metadata(final, wasm_binary_target)
+    shutil.move(wso, wasm_binary_target)
+
+  if shared.Settings.SIDE_MODULE:
     sys.exit(0) # and we are done.
+
   if options.opt_level >= 2:
     # minify the JS
     optimizer.do_minify() # calculate how to minify

src/settings.js

@@ -1285,3 +1285,8 @@ var ENVIRONMENT_MAY_BE_WEB_OR_WORKER = 1;
 // JS -> asm.js import names. Controlled by optimization level, enabled
 // at -O1 and higher, but disabled at -g2 and higher.
 var MINIFY_ASMJS_IMPORT_NAMES = 0;
+
+// if set to 1, then generated WASM files will contain a custom
+// "emscripten_metadata" section that contains information necessary
+// to execute the file without the accompanying JS file.
+var EMIT_EMSCRIPTEN_METADATA = 0;

tests/test_other.py

@@ -31,6 +31,7 @@
 from tools.shared import EMCC, EMXX, EMAR, EMRANLIB, PYTHON, FILE_PACKAGER, WINDOWS, MACOS, LLVM_ROOT, EMCONFIG, EM_BUILD_VERBOSE
 from tools.shared import CLANG, CLANG_CC, CLANG_CPP, LLVM_AR
 from tools.shared import COMPILER_ENGINE, NODE_JS, SPIDERMONKEY_ENGINE, JS_ENGINES, V8_ENGINE
+from tools.shared import WebAssembly
 from runner import RunnerCore, path_from_root, get_zlib_library, no_wasm_backend
 from runner import needs_dlfcn, env_modify, no_windows, chdir, with_env_modify, create_test_file
 from tools import jsrun, shared
@@ -8805,3 +8806,50 @@ def test_no_excessive_invoke_functions_are_generated_when_exceptions_are_enabled
       self.assertContained('invoke_i', output)
       self.assertNotContained('invoke_ii', output)
       self.assertNotContained('invoke_v', output)
+
+  def test_add_emscripten_metadata(self):
+    run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'),
+                 '-s', 'EMIT_EMSCRIPTEN_METADATA',
+                 '-o', 'hello_world.js'])
+    wasm = open('hello_world.wasm', 'rb').read()
+    # emscripten_metadata should be in the wasm data
+    offset = 8 # skip magic + header
+    for _ in range(100):
+      section = wasm[offset:offset + 1]
+      self.assertEqual(section, b'\0', 'No emscripten_metadata section found before standard wasm sections')
+      offset += 1
+      (section_size, offset) = WebAssembly.delebify(wasm, offset)
+      end_offset = offset + section_size
+      (name_len, offset) = WebAssembly.delebify(wasm, offset)
+      name = wasm[offset:offset + name_len]
+      if name == b'emscripten_metadata':
+        break
+      offset = end_offset
+    else:
+      self.assertFalse("No emscripten_metadata section found in first 100 custom sections")
+
+    # make sure wasm executes correctly
+    ret = run_process(NODE_JS + ['hello_world.js'], stdout=PIPE).stdout
+    self.assertTextDataIdentical('hello, world!\n', ret)
+
+  def test_add_emscripten_metadata_not_emitted(self):
+    run_process([PYTHON, EMCC, path_from_root('tests', 'hello_world.c'),
+                 '-o', 'hello_world.js'])
+    wasm = open('hello_world.wasm', 'rb').read()
+    # emscripten_metadata should be in the wasm data
+    offset = 8 # skip magic + header
+    for _ in range(100):
+      if offset >= len(wasm):
+        break
+      section = wasm[offset:offset + 1]
+      offset += 1
+      (section_size, offset) = WebAssembly.delebify(wasm, offset)
+      end_offset = offset + section_size
+      # if this is a custom section
+      if section == b'\0':
+        (name_len, offset) = WebAssembly.delebify(wasm, offset)
+        name = wasm[offset:offset + name_len]
+        self.assertNotEqual(name, b'emscripten_metadata')
+      offset = end_offset
+    else:
+      self.assertFalse("wasm file had too many sections")

tools/shared.py

@@ -507,6 +507,19 @@ def get_emscripten_version(path):
 EMSCRIPTEN_VERSION = get_emscripten_version(path_from_root('emscripten-version.txt'))
 parts = [int(x) for x in EMSCRIPTEN_VERSION.split('.')]
 EMSCRIPTEN_VERSION_MAJOR, EMSCRIPTEN_VERSION_MINOR, EMSCRIPTEN_VERSION_TINY = parts
+# For the Emscripten-specific WASM metadata section, follows semver, changes
+# whenever metadata section changes structure
+# NB: major version 0 implies no compatibility
+(EMSCRIPTEN_METADATA_MAJOR, EMSCRIPTEN_METADATA_MINOR) = (0, 0)
+# For the JS/WASM ABI, specifies the minimum ABI version required of
+# the WASM runtime implementation by the generated WASM binary. It follows
+# semver and changes whenever C types change size/signedness or
+# syscalls change signature. By semver, the maximum ABI version is
+# implied to be less than (EMSCRIPTEN_ABI_MAJOR + 1, 0). On an ABI
+# change, increment EMSCRIPTEN_ABI_MINOR if EMSCRIPTEN_ABI_MAJOR == 0
+# or the ABI change is backwards compatible, otherwise increment
+# EMSCRIPTEN_ABI_MAJOR and set EMSCRIPTEN_ABI_MINOR = 0
+(EMSCRIPTEN_ABI_MAJOR, EMSCRIPTEN_ABI_MINOR) = (0, 0)
 
 
 def generate_sanity():
@@ -2962,16 +2975,77 @@ def lebify(x):
     return bytearray(ret)
 
   @staticmethod
-  def make_shared_library(js_file, wasm_file, needed_dynlibs):
-    # a wasm shared library has a special "dylink" section, see tools-conventions repo
+  def delebify(buf, offset):
+    result = 0
+    shift = 0
+    while True:
+      byte = bytearray(buf[offset:offset + 1])[0]
+      offset += 1
+      result |= (byte & 0x7f) << shift
+      if not (byte & 0x80):
+        break
+      shift += 7
+    return (result, offset)
+
+  @staticmethod
+  def get_js_data(js_file, shared=False):
     js = open(js_file).read()
     m = re.search("var STATIC_BUMP = (\d+);", js)
     mem_size = int(m.group(1))
     m = re.search("Module\['wasmTableSize'\] = (\d+);", js)
     table_size = int(m.group(1))
-    m = re.search('gb = alignMemory\(getMemory\(\d+ \+ (\d+)\), (\d+) \|\| 1\);', js)
-    assert m.group(1) == m.group(2), 'js must contain a clear alignment for the wasm shared library'
-    mem_align = int(m.group(1))
+    if shared:
+      m = re.search('gb = alignMemory\(getMemory\(\d+ \+ (\d+)\), (\d+) \|\| 1\);', js)
+      assert m.group(1) == m.group(2), 'js must contain a clear alignment for the wasm shared library'
+      mem_align = int(m.group(1))
+    else:
+      mem_align = None
+    return (mem_size, table_size, mem_align)
+
+  @staticmethod
+  def add_emscripten_metadata(js_file, wasm_file):
+    (mem_size, table_size, _) = WebAssembly.get_js_data(js_file)
+    logger.debug('creating wasm emscripten metadata section with mem size %d, table size %d' % (mem_size, table_size,))
+    wso = js_file + '.wso'
+    wasm = open(wasm_file, 'rb').read()
+    f = open(wso, 'wb')
+    f.write(wasm[0:8]) # copy magic number and version
+    # write the special section
+    f.write(b'\0') # user section is code 0
+    # need to find the size of this section
+    name = b'\x13emscripten_metadata' # section name, including prefixed size
+    contents = (
+      # metadata section version
+      WebAssembly.lebify(EMSCRIPTEN_METADATA_MAJOR) +
+      WebAssembly.lebify(EMSCRIPTEN_METADATA_MINOR) +
+
+      # NB: The structure of the following should only be changed
+      #     if EMSCRIPTEN_METADATA_MAJOR is incremented
+      # Minimum ABI version
+      WebAssembly.lebify(EMSCRIPTEN_ABI_MAJOR) +
+      WebAssembly.lebify(EMSCRIPTEN_ABI_MINOR) +
+
+      # static bump
+      WebAssembly.lebify(mem_size) +
+
+      # table size
+      WebAssembly.lebify(table_size)
+      # NB: more data can be appended here as long as you increase
+      #     the EMSCRIPTEN_METADATA_MINOR
+    )
+
+    size = len(name) + len(contents)
+    f.write(WebAssembly.lebify(size))
+    f.write(name)
+    f.write(contents)
+    f.write(wasm[8:])
+    f.close()
+    return wso
+
+  @staticmethod
+  def make_shared_library(js_file, wasm_file, needed_dynlibs):
+    # a wasm shared library has a special "dylink" section, see tools-conventions repo
+    (mem_size, table_size, mem_align) = WebAssembly.get_js_data(js_file, True)
     mem_align = int(math.log(mem_align, 2))
     logger.debug('creating wasm dynamic library with mem size %d, table size %d, align %d' % (mem_size, table_size, mem_align))
     wso = js_file + '.wso'