Skip to content

Commit

Permalink
[Pthreads] Fix worker.js in ES6 module environments (#21041)
Browse files Browse the repository at this point in the history
This file can be an ES6 module if a package.json file indicates that all
files in the directory are.

We need to apply the same tricks as we apply to the main JS file in that
case, with createRequire etc.

We also need to emit the suffix .worker.mjs and not .js, or else node
will error.

Followup to #20939 and fixes the package.json discussion after that PR landed.
  • Loading branch information
kripken authored Jan 10, 2024
1 parent a997567 commit 49ab23a
Show file tree
Hide file tree
Showing 4 changed files with 59 additions and 18 deletions.
4 changes: 4 additions & 0 deletions ChangeLog.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ See docs/process.md for more on how version tagging works.

3.1.52 (in development)
-----------------------
- Building with `pthreads+EXPORT_ES6` will now emit the worker file as
`NAME.worker.mjs` rather than `.js`. This is a necessary breaking change to
resolve other `pthreads+EXPORT_ES6` issues in Node.js (because Node.js is
affected by the suffix in some cases). (#21041)
- Include paths added by ports (e.g. `-sUSE_SDL=2`) now use `-isystem` rather
then `-I`. This means that files in user-specified include directories will
now take precedence over port includes. (#21014)
Expand Down
16 changes: 16 additions & 0 deletions src/worker.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,16 @@ var ENVIRONMENT_IS_NODE = typeof process == 'object' && typeof process.versions
if (ENVIRONMENT_IS_NODE) {
// Create as web-worker-like an environment as we can.

// See the parallel code in shell.js, but here we don't need the condition on
// multi-environment builds, as we do not have the need to interact with the
// modularization logic as shell.js must (see link.py:node_es6_imports and
// how that is used in link.py).
#if EXPORT_ES6
const { createRequire } = await import('module');
/** @suppress{duplicate} */
var require = createRequire(import.meta.url);
#endif

var nodeWorkerThreads = require('worker_threads');

var parentPort = nodeWorkerThreads.parentPort;
Expand All @@ -32,7 +42,13 @@ if (ENVIRONMENT_IS_NODE) {
require,
Module,
location: {
// __filename is undefined in ES6 modules, and import.meta.url only in ES6
// modules.
#if EXPORT_ES6
href: typeof __filename !== 'undefined' ? __filename : import.meta.url
#else
href: __filename
#endif
},
Worker: nodeWorkerThreads.Worker,
importScripts: (f) => vm.runInThisContext(fs.readFileSync(f, 'utf8'), {filename: f}),
Expand Down
21 changes: 16 additions & 5 deletions test/test_other.py
Original file line number Diff line number Diff line change
Expand Up @@ -343,10 +343,10 @@ def test_emcc_output_worker_mjs(self, args):
test_file('hello_world.c')] + args)
src = read_file('subdir/hello_world.mjs')
self.assertContained("new URL('hello_world.wasm', import.meta.url)", src)
self.assertContained("new Worker(new URL('hello_world.worker.js', import.meta.url), {type: 'module'})", src)
self.assertContained("new Worker(new URL('hello_world.worker.mjs', import.meta.url), {type: 'module'})", src)
self.assertContained("new Worker(pthreadMainJs, {type: 'module'})", src)
self.assertContained('export default Module;', src)
src = read_file('subdir/hello_world.worker.js')
src = read_file('subdir/hello_world.worker.mjs')
self.assertContained("import('./hello_world.mjs')", src)
self.assertContained('hello, world!', self.run_js('subdir/hello_world.mjs'))

Expand All @@ -358,7 +358,7 @@ def test_emcc_output_worker_mjs_single_file(self):
test_file('hello_world.c'), '-sSINGLE_FILE'])
src = read_file('hello_world.mjs')
self.assertNotContained("new URL('data:", src)
self.assertContained("new Worker(new URL('hello_world.worker.js', import.meta.url), {type: 'module'})", src)
self.assertContained("new Worker(new URL('hello_world.worker.mjs', import.meta.url), {type: 'module'})", src)
self.assertContained("new Worker(pthreadMainJs, {type: 'module'})", src)
self.assertContained('hello, world!', self.run_js('hello_world.mjs'))

Expand Down Expand Up @@ -400,11 +400,16 @@ def test_export_es6_allows_export_in_post_js(self):
src = read_file('a.out.js')
self.assertContained('export{doNothing};', src)

@parameterized({
'': (False,),
'package_json': (True,),
})
@parameterized({
'': ([],),
'pthreads': (['-pthread'],),
# load a worker before startup to check ES6 modules there as well
'pthreads': (['-pthread', '-sPTHREAD_POOL_SIZE=1'],),
})
def test_export_es6(self, args):
def test_export_es6(self, args, package_json):
self.run_process([EMCC, test_file('hello_world.c'), '-sEXPORT_ES6',
'-o', 'hello.mjs'] + args)
# In ES6 mode we use MODULARIZE, so we must instantiate an instance of the
Expand All @@ -413,6 +418,12 @@ def test_export_es6(self, args):
import Hello from "./hello.mjs";
Hello();
''')

if package_json:
# This makes node load all files in the directory as ES6 modules,
# including the worker.js file.
create_file('package.json', '{"type":"module"}')

self.assertContained('hello, world!', self.run_js('runner.mjs'))

def test_emcc_out_file(self):
Expand Down
36 changes: 23 additions & 13 deletions tools/link.py
Original file line number Diff line number Diff line change
Expand Up @@ -513,6 +513,10 @@ def do_split_module(wasm_file, options):
building.run_binaryen_command('wasm-split', wasm_file + '.orig', outfile=wasm_file, args=args)


def get_worker_js_suffix():
return '.worker.mjs' if settings.EXPORT_ES6 else '.worker.js'


def setup_pthreads(target):
if settings.RELOCATABLE:
# phtreads + dyanmic linking has certain limitations
Expand Down Expand Up @@ -569,7 +573,7 @@ def setup_pthreads(target):
building.user_requested_exports.update(worker_imports)

# set location of worker.js
settings.PTHREAD_WORKER_FILE = unsuffixed_basename(target) + '.worker.js'
settings.PTHREAD_WORKER_FILE = unsuffixed_basename(target) + get_worker_js_suffix()

if settings.MINIMAL_RUNTIME:
building.user_requested_exports.add('exit')
Expand Down Expand Up @@ -1999,12 +2003,27 @@ def phase_memory_initializer(memfile):
final_js += '.mem.js'


# Unmangle previously mangled `import.meta` and `await import` references in
# both main code and libraries.
# See also: `preprocess` in parseTools.js.
def fix_es6_import_statements(js_file):
if not settings.EXPORT_ES6 or not settings.USE_ES6_IMPORT_META:
return

src = read_file(js_file)
write_file(js_file, src
.replace('EMSCRIPTEN$IMPORT$META', 'import.meta')
.replace('EMSCRIPTEN$AWAIT$IMPORT', 'await import'))


def create_worker_file(input_file, target_dir, output_file):
output_file = os.path.join(target_dir, output_file)
input_file = utils.path_from_root(input_file)
contents = shared.read_and_preprocess(input_file, expand_macros=True)
write_file(output_file, contents)

fix_es6_import_statements(output_file)

# Minify the worker JS file, if JS minification is enabled.
if settings.MINIFY_WHITESPACE:
contents = building.acorn_optimizer(output_file, ['minifyWhitespace'], return_output=True)
Expand Down Expand Up @@ -2045,17 +2064,8 @@ def phase_final_emitting(options, state, target, wasm_target, memfile):
# mode)
final_js = building.closure_compiler(final_js, advanced=False, extra_closure_args=options.closure_args)

# Unmangle previously mangled `import.meta` and `await import` references in
# both main code and libraries.
# See also: `preprocess` in parseTools.js.
if settings.EXPORT_ES6 and settings.USE_ES6_IMPORT_META:
src = read_file(final_js)
final_js += '.esmeta.js'
write_file(final_js, src
.replace('EMSCRIPTEN$IMPORT$META', 'import.meta')
.replace('EMSCRIPTEN$AWAIT$IMPORT', 'await import'))
shared.get_temp_files().note(final_js)
save_intermediate('es6-module')
fix_es6_import_statements(final_js)
save_intermediate('es6-module')

# Apply pre and postjs files
if options.extern_pre_js or options.extern_post_js:
Expand Down Expand Up @@ -2600,7 +2610,7 @@ def generate_worker_js(target, js_target, target_basename):
proxy_worker_filename = get_subresource_location(js_target)
else:
# compiler output goes in .worker.js file
move_file(js_target, shared.replace_suffix(js_target, '.worker.js'))
move_file(js_target, shared.replace_suffix(js_target, get_worker_js_suffix()))
worker_target_basename = target_basename + '.worker'
proxy_worker_filename = (settings.PROXY_TO_WORKER_FILENAME or worker_target_basename) + '.js'

Expand Down

0 comments on commit 49ab23a

Please sign in to comment.