Process --pre-js and --post-js files in jsifier.js (#18525)

This treats pre and post JS files more like other runtime and
library files which simplifies the code and avoids the special case
processing in emcc.py.

This also means that pre and post JS also now go through macro
preprocessor which should be useful in some cases.

ChangeLog.md

@@ -23,6 +23,9 @@ See docs/process.md for more on how version tagging works.
 - The `STACK_SIZE`, `STACK_ALIGN`, `POINTER_SIZE`, and `ASSERTIONS` JavaScript
   globals were removed by default.  In debug builds a clear error is shown if
   you try to use these. (#18503)
+- --pre-js and --post-js files are now fed through the JS preprocesor, just
+  like JS library files and the core runtime JS files.  This means they can
+  now contain #if/#else/#endif blocks and {{{ }}} macro blocks. (#18525)
 
 3.1.30 - 01/11/23
 -----------------

emcc.py

@@ -524,7 +524,11 @@ def get_all_js_syms():
 
   # We define a cache hit as when the settings and `--js-library` contents are
   # identical.
-  input_files = [json.dumps(settings.external_dict(), sort_keys=True, indent=2)]
+  # Ignore certain settings that can are no relevant to library deps.  Here we
+  # skip PRE_JS_FILES/POST_JS_FILES which don't effect the library symbol list
+  # and can contain full paths to temporary files.
+  skip_settings = {'PRE_JS_FILES', 'POST_JS_FILES'}
+  input_files = [json.dumps(settings.external_dict(skip_keys=skip_settings), sort_keys=True, indent=2)]
   for jslib in sorted(glob.glob(utils.path_from_root('src') + '/library*.js')):
     input_files.append(read_file(jslib))
   for jslib in settings.JS_LIBRARIES:
@@ -1119,7 +1123,7 @@ def package_files(options, target):
   if options.preload_files:
     # Preloading files uses --pre-js code that runs before the module is loaded.
     file_code = shared.check_call(cmd, stdout=PIPE).stdout
-    options.pre_js = js_manipulation.add_files_pre_js(options.pre_js, file_code)
+    js_manipulation.add_files_pre_js(options.pre_js, file_code)
   else:
     # Otherwise, we are embedding files, which does not require --pre-js code,
     # and instead relies on a static constrcutor to populate the filesystem.
@@ -1293,6 +1297,9 @@ def run(args):
   if len(options.preload_files) or len(options.embed_files):
     linker_arguments += package_files(options, target)
 
+  settings.PRE_JS_FILES = [os.path.abspath(f) for f in options.pre_js]
+  settings.POST_JS_FILES = [os.path.abspath(f) for f in options.post_js]
+
   if options.oformat == OFormat.OBJECT:
     logger.debug(f'link_to_object: {linker_arguments} -> {target}')
     building.link_to_object(linker_arguments, target)
@@ -1750,8 +1757,6 @@ def phase_linker_setup(options, state, newargs):
       exit_with_error('PTHREADS_PROFILING only works with ASSERTIONS enabled')
     options.post_js.append(utils.path_from_root('src/threadprofiler.js'))
 
-  options.pre_js = read_js_files(options.pre_js)
-  options.post_js = read_js_files(options.post_js)
   options.extern_pre_js = read_js_files(options.extern_pre_js)
   options.extern_post_js = read_js_files(options.extern_post_js)
 
@@ -3021,7 +3026,8 @@ def phase_post_link(options, state, in_wasm, wasm_target, target):
 
   phase_emscript(options, in_wasm, wasm_target, memfile)
 
-  phase_source_transforms(options)
+  if options.js_transform:
+    phase_source_transforms(options)
 
   if memfile and not settings.MINIMAL_RUNTIME:
     # MINIMAL_RUNTIME doesn't use `var memoryInitializer` but instead expects Module['mem'] to
@@ -3055,28 +3061,14 @@ def phase_emscript(options, in_wasm, wasm_target, memfile):
 
 @ToolchainProfiler.profile_block('source transforms')
 def phase_source_transforms(options):
-  global final_js
-
-  # Apply pre and postjs files
-  if final_js and (options.pre_js or options.post_js):
-    logger.debug('applying pre/postjses')
-    src = read_file(final_js)
-    final_js += '.pp.js'
-    with open(final_js, 'w', encoding='utf-8') as f:
-      # pre-js code goes right after the Module integration code (so it
-      # can use Module), we have a marker for it
-      f.write(do_replace(src, '// {{PRE_JSES}}', options.pre_js))
-      f.write(options.post_js)
-    save_intermediate('pre-post')
-
   # Apply a source code transformation, if requested
-  if options.js_transform:
-    safe_copy(final_js, final_js + '.tr.js')
-    final_js += '.tr.js'
-    posix = not shared.WINDOWS
-    logger.debug('applying transform: %s', options.js_transform)
-    shared.check_call(building.remove_quotes(shlex.split(options.js_transform, posix=posix) + [os.path.abspath(final_js)]))
-    save_intermediate('transformed')
+  global final_js
+  safe_copy(final_js, final_js + '.tr.js')
+  final_js += '.tr.js'
+  posix = not shared.WINDOWS
+  logger.debug('applying transform: %s', options.js_transform)
+  shared.check_call(building.remove_quotes(shlex.split(options.js_transform, posix=posix) + [os.path.abspath(final_js)]))
+  save_intermediate('transformed')
 
 
 @ToolchainProfiler.profile_block('memory initializer')

src/jsifier.js

@@ -526,6 +526,10 @@ function ${name}(${args}) {
     const postFile = MINIMAL_RUNTIME ? 'postamble_minimal.js' : 'postamble.js';
     includeFile(postFile);
 
+    for (const fileName of POST_JS_FILES) {
+      includeFile(fileName);
+    }
+
     print('//FORWARDED_DATA:' + JSON.stringify({
       librarySymbols: librarySymbols,
       warnings: warnings,

src/parseTools.js

@@ -1104,3 +1104,11 @@ function getUnsharedTextDecoderView(heap, start, end) {
   // or can use .subarray() otherwise.
   return `${heap}.buffer instanceof SharedArrayBuffer ? ${shared} : ${unshared}`;
 }
+
+function preJS() {
+  let result = '';
+  for (const fileName of PRE_JS_FILES) {
+    result += preprocess(fileName);
+  }
+  return result;
+}

src/settings_internal.js

@@ -243,3 +243,7 @@ var WEAK_IMPORTS = [];
 var STACK_FIRST = false;
 
 var HAVE_EM_ASM = true;
+
+var PRE_JS_FILES = [];
+
+var POST_JS_FILES = [];

src/shell.js

@@ -72,7 +72,7 @@ Module['ready'] = new Promise(function(resolve, reject) {
 
 // --pre-jses are emitted after the Module integration code, so that they can
 // refer to Module (if they choose; they can also define Module)
-// {{PRE_JSES}}
+{{{ preJS() }}}
 
 // Sometimes an existing Module object exists with properties
 // meant to overwrite the default module functionality. Here

src/shell_minimal.js

@@ -136,8 +136,7 @@ function ready() {
 
 // --pre-jses are emitted after the Module integration code, so that they can
 // refer to Module (if they choose; they can also define Module)
-
-// {{PRE_JSES}}
+{{{ preJS() }}}
 
 #if USE_PTHREADS
 

test/return64bit/testbindstart.js

@@ -1,3 +1 @@
-
-(function() { // Start of self-calling lambda used to avoid polluting global namespace.
-
+(function() { // Start of IIFE used to avoid polluting global namespace.

test/test_other.py

@@ -9041,6 +9041,21 @@ def test_js_preprocess(self):
     self.assertContained('JSLIB: EXIT_RUNTIME', err)
     self.assertNotContained('JSLIB: MAIN_MODULE', err)
 
+  def test_js_preprocess_pre_post(self):
+    create_file('pre.js', '''
+    #if ASSERTIONS
+    console.log('assertions enabled')
+    #else
+    console.log('assertions disabled')
+    #endif
+    ''')
+    create_file('post.js', '''
+    console.log({{{ POINTER_SIZE }}});
+    ''')
+    self.emcc_args += ['--pre-js', 'pre.js', '--post-js', 'post.js']
+    self.do_runf(test_file('hello_world.c'), 'assertions enabled\n4')
+    self.do_runf(test_file('hello_world.c'), 'assertions disabled\n4', emcc_args=['-sASSERTIONS=0'])
+
   def test_html_preprocess(self):
     src_file = test_file('module/test_stdin.c')
     output_file = 'test_stdin.html'

tools/js_manipulation.py

@@ -6,7 +6,7 @@
 import re
 
 from .settings import settings
-from . import utils
+from . import utils, shared
 
 emscripten_license = '''\
 /**
@@ -28,25 +28,34 @@
 emscripten_license_regex = r'\/\*\*?(\s*\*?\s*@license)?(\s*\*?\s*Copyright \d+ The Emscripten Authors\s*\*?\s*SPDX-License-Identifier: MIT)+\s*\*\/\s*'
 
 
-def add_files_pre_js(user_pre_js, files_pre_js):
+def add_files_pre_js(pre_js_list, files_pre_js):
   # the normal thing is to just combine the pre-js content
+  filename = shared.get_temp_files().get('.js').name
+  utils.write_file(filename, files_pre_js)
+  pre_js_list.insert(0, filename)
   if not settings.ASSERTIONS:
-    return files_pre_js + user_pre_js
+    return
 
   # if a user pre-js tramples the file code's changes to Module.preRun
   # that could be confusing. show a clear error at runtime if assertions are
   # enabled
-  return files_pre_js + '''
+  pre = shared.get_temp_files().get('.js').name
+  post = shared.get_temp_files().get('.js').name
+  utils.write_file(pre, '''
     // All the pre-js content up to here must remain later on, we need to run
     // it.
     if (Module['ENVIRONMENT_IS_PTHREAD']) Module['preRun'] = [];
     var necessaryPreJSTasks = Module['preRun'].slice();
-  ''' + user_pre_js + '''
+  ''')
+  utils.write_file(post, '''
     if (!Module['preRun']) throw 'Module.preRun should exist because file support used it; did a pre-js delete it?';
     necessaryPreJSTasks.forEach(function(task) {
       if (Module['preRun'].indexOf(task) < 0) throw 'All preRun tasks that exist before user pre-js code should remain after; did you replace Module or modify Module.preRun?';
     });
-  '''
+  ''')
+
+  pre_js_list.insert(1, pre)
+  pre_js_list.append(post)
 
 
 def handle_license(js_target):

tools/settings.py

@@ -156,8 +156,8 @@ def infer_types(self):
   def dict(self):
     return self.attrs
 
-  def external_dict(self):
-    external_settings = {k: v for k, v in self.dict().items() if k not in INTERNAL_SETTINGS}
+  def external_dict(self, skip_keys={}): # noqa
+    external_settings = {k: v for k, v in self.dict().items() if k not in INTERNAL_SETTINGS and k not in skip_keys}
     # Only the names of the legacy settings are used by the JS compiler
     # so we can reduce the size of serialized json by simplifying this
     # otherwise complex value.