emscripten renovation (#310)

* test suite fixes for link=static default

* emscripten renovation

* inherit from clang toolset
* update for 'new' fastcomp backend
* exceptions support
* dynamic linking support
* pthread support
* run-tests launcher via nodejs

src/tools/emscripten.jam

@@ -1,3 +1,4 @@
+# Copyright Nikita Kniazev
 # Copyright Rene Rivera 2016
 # Distributed under the Boost Software License, Version 1.0.
 # (See accompanying file LICENSE.txt
@@ -7,100 +8,153 @@ import feature ;
 import os ;
 import toolset ;
 import common ;
-import gcc ;
 import type ;
+import version ;
+
+feature.extend toolset : emscripten ;
 
 feature.feature embind : off on : propagated ;
 feature.feature closure : off on full : propagated ;
 feature.feature link-optimization : off on full : propagated ;
+feature.subfeature debug-symbols : source-map : on : optional propagated ;
+feature.subfeature exception-handling : demangle-support : on : optional propagated ;
+feature.subfeature exception-handling : method : js : optional propagated link-incompatible ;
+
+if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ]
+{
+    local rule .debug-configuration ( messages * )
+    {
+        ECHO "notice: [emscripten-cfg]" $(messages) ;
+    }
+}
+else
+{
+    local rule .debug-configuration ( messages * ) { }
+}
 
-rule init ( version ? :  command * : options * )
+rule init ( version ? : command * : options * )
 {
+    # On Windows 'emcc' calls emcc.bat but B2 will trip in CreateProcess if we
+    # let it call emcc because it does not work like a shell.
+    # We could make it call emcc.bat, but because gcc toolset quotes the command
+    # it will trip on cmd.exe bug, see https://github.com/bfgroup/b2/issues/309
+    if ! $(command) && [ os.name ] = NT {
+        command = [ common.get-absolute-tool-path emcc ] ;
+        if $(command) {
+            local python = [ os.environ EMSDK_PYTHON ] ;
+            python ?= py ;
+            command = $(python) $(command)\\emcc.py ;
+        }
+    }
+
     command = [ common.get-invocation-command emscripten
         : emcc
         : $(command) ] ;
 
     # Determine the version
-    if $(command)
-    {
-        local command-string = \"$(command)\" ;
-        command-string = $(command-string:J=" ") ;
-        version ?= [ MATCH "([0-9.]+)"
-            : [ SHELL "$(command-string) --version" ] ] ;
-    }
+    local command-string = [ common.make-command-string $(command) ] ;
+    version = [ MATCH "([0-9.]+)" : [ SHELL "$(command-string) --version" ] ] ;
 
     local condition = [ common.check-init-parameters emscripten
         : version $(version) ] ;
+    local conditions = [ feature.split $(condition) ] ;
 
     common.handle-options emscripten : $(condition) : $(command) : $(options) ;
-}
+    .debug-configuration $(condition) ":: emcc ::" $(command) ;
 
-feature.extend toolset : emscripten ;
+    local ar = [ feature.get-values <archiver> : $(options) ] ;
+    ar ?= $(command[1--2]) $(command[-1]:B=emar) ;
+    ar = [ common.get-invocation-command emscripten : emar : $(ar) ] ;
+    toolset.flags emscripten.archive .AR $(condition) : $(ar) ;
+    .debug-configuration $(condition) ":: archiver ::" $(ar) ;
+
+    local ProgramFiles ;
+    if [ os.name ] = NT {
+        ProgramFiles = [ os.environ "ProgramFiles" ] [ os.environ "ProgramFiles(x86)" ] ;
+    }
+    local nodejs = [ feature.get-values <nodejs> : $(options) ] ;
+    nodejs = [ common.get-invocation-command-nodefault emscripten : node : $(nodejs) : "$(ProgramFiles)\\nodejs" ] ;
+    if $(nodejs) {
+        local command-string = [ common.make-command-string $(nodejs) ] ;
+        local node-version = [ MATCH "v([0-9]+)" : [ SHELL "$(command-string) --version" ] ] ;
+        .debug-configuration $(condition) ":: nodejs version is" $(node-version:J=.) ;
+        if [ version.version-less $(node-version:E=0) : 16 ] {
+            toolset.add-defaults $(conditions:J=,)\:<exception-handling-method>js ;
+        }
+        local result = [ SHELL "$(command-string) --version --experimental-wasm-threads" : exit-status no-output ] ;
+        if $(result[2]) = 0 {
+            nodejs += --experimental-wasm-threads ;
+        }
+    }
+    nodejs ?= nodejs ;
+    import testing ;
+    toolset.flags testing LAUNCHER $(condition) : \"$(nodejs)\" : unchecked ;
+    .debug-configuration $(condition) ":: nodejs ::" $(nodejs) ;
+
+    version = [ SPLIT_BY_CHARACTERS $(version:E=0) : . ] ;
+    # The version number is a hard guess, I could not find when -pthread or -fwasm-exceptions were actually added
+    if [ version.version-less $(version[1]) : 2 ] {
+        toolset.add-requirements $(conditions:J=,)\:<exception-handling-method>js ;
+    }
+    # Workaround https://github.com/emscripten-core/emscripten/issues/19471
+    if [ version.version-less $(version) : 3 1 41 ] {
+        toolset.flags emscripten.compile OPTIONS $(condition)/<threading>multi : -sUSE_PTHREADS ;
+    }
+    if ! [ version.version-less $(version[1]) : 3 ] {
+        # bring back unsupported in v2 flags
+        toolset.flags emscripten.link FINDLIBS-ST-PFX $(condition)/<runtime-link>shared : -Wl,-Bstatic ;
+        toolset.flags emscripten.link FINDLIBS-SA-PFX $(condition)/<runtime-link>shared : -Wl,-Bdynamic ;
+    }
+}
 
-toolset.inherit-generators emscripten <toolset>emscripten
-    : gcc
-    : gcc.mingw.link gcc.mingw.link.dll gcc.compile.c.pch gcc.compile.c++.pch
-    ;
-toolset.inherit-rules emscripten : gcc ;
-toolset.inherit-flags emscripten : gcc
-        :
-        <optimization>off <optimization>speed <optimization>space
-        <optimization>minimal <optimization>debug
-        <profiling>off <profiling>on
-        <debug-symbols>off <debug-symbols>on
-        <rtti>off <rtti>on
-        ;
-
-type.set-generated-target-suffix EXE : <toolset>emscripten : "js" ;
-type.set-generated-target-suffix OBJ : <toolset>emscripten : "bc" ;
-type.set-generated-target-suffix STATIC_LIB : <toolset>emscripten : "bc" ;
-
-toolset.flags emscripten.compile OPTIONS <flags> ;
-toolset.flags emscripten.compile OPTIONS <cflags> ;
-toolset.flags emscripten.compile.c++ OPTIONS <cxxflags> ;
-
-toolset.flags emscripten.compile OPTIONS <optimization>off : -O0 ;
-toolset.flags emscripten.compile OPTIONS <optimization>speed : -O3 ;
-toolset.flags emscripten.compile OPTIONS <optimization>space : -Oz ;
-toolset.flags emscripten.link OPTIONS <optimization>off : -O0 ;
-toolset.flags emscripten.link OPTIONS <optimization>speed : -O3 ;
-toolset.flags emscripten.link OPTIONS <optimization>space : -O3 ;
+import clang-linux ;
+toolset.inherit-generators emscripten : clang-linux ;
+toolset.inherit-rules emscripten : clang-linux ;
+toolset.inherit-flags emscripten : clang-linux : :
+    # emscripten barks on them being unsupported
+    RPATH_LINK
+    RPATH_OPTION
+    SONAME_OPTION
+    # supported only in v3, we reenable them conditionally in init
+    FINDLIBS-ST-PFX
+    FINDLIBS-SA-PFX
+  ;
+
+toolset.add-defaults <toolset>emscripten:<target-os>unknown ;
+# dynamic linking is experemental and buggy
+toolset.add-defaults <toolset>emscripten:<link>static ;
+
+# Emscripten can produce different kinds of .js and .wasm outputs:
+# -o .wasm produces a standalone .wasm executable.
+# -o .js produces a .wasm executable that uses emscripten runtime and a .js launcher.
+# -o .js -sSTANDALONE_WASM produces a standalone .wasm executable and a .js launcher.
+type.set-generated-target-suffix EXE : <toolset>emscripten : js ;
+
+toolset.flags emscripten.compile.c++ OPTIONS <exception-handling>on/<exception-handling-method>js : -fexceptions ;
+toolset.flags emscripten.link        OPTIONS <exception-handling>on/<exception-handling-method>js : -fexceptions ;
+toolset.flags emscripten.compile.c++ OPTIONS <exception-handling>on/<exception-handling-method> : -fwasm-exceptions ;
+toolset.flags emscripten.link        OPTIONS <exception-handling>on/<exception-handling-method> : -fwasm-exceptions ;
+
+# Emscripten support for shared libraries is incomplete, linker embeds direct
+# dependencies into executable itself but do not embed transitive dependencies.
+# We probably could workaround that in a custom linking generator which will add
+# transitive dependencies via --preload-file/--embed-file, but it is a lot of work
+# for a niche feature which should be fixed in the linker instead.
+# There is -sNODERAWFS but LD_LIBRARY_PATH seems to only work for explicit dlopen.
+import testing ;
+# TODO: This is brittle and ugly, but currently there is no other way to specify flags for produced types by linker
+local EXE_PRODUCING_TARGET_TYPES = EXE RUN_OUTPUT RUN RUN_FAIL UNIT_TEST ;
+toolset.flags emscripten.link OPTIONS <link>shared/<main-target-type>$(EXE_PRODUCING_TARGET_TYPES) : -sMAIN_MODULE ;
+toolset.flags emscripten.link SHARED_OPTION : -sSIDE_MODULE ;
 
 toolset.flags emscripten.compile OPTIONS <profiling>on : --profiling-funcs ;
 
-toolset.flags emscripten.compile OPTIONS <inlining>off : -fno-inline ;
-toolset.flags emscripten.compile OPTIONS <inlining>on : -Wno-inline ;
-toolset.flags emscripten.compile OPTIONS <inlining>full : -Wno-inline ;
-
-toolset.flags emscripten OPTIONS <debug-symbols>off : -g0 ;
-toolset.flags emscripten OPTIONS <debug-symbols>on : -g4 -s DEMANGLE_SUPPORT=1 ;
-toolset.flags emscripten OPTIONS <rtti>off : -fno-rtti ;
+toolset.flags emscripten.link OPTIONS <exception-handling>on/<exception-handling-demangle-support>on : -sDEMANGLE_SUPPORT ;
+toolset.flags emscripten.link OPTIONS <debug-symbols>on/<debug-symbols-source-map>on : -gsource-map ;
 
 toolset.flags emscripten.link OPTIONS <embind>on : --bind ;
 toolset.flags emscripten.link OPTIONS <closure>on : --closure 1 ;
 toolset.flags emscripten.link OPTIONS <closure>full : --closure 2 ;
-toolset.flags emscripten.link OPTIONS <link-optimization>off : --llvm-lto 0 ;
+# things for old fastcomp backend which was removed in 2.0.0 (08/10/2020)
 toolset.flags emscripten.link OPTIONS <link-optimization>on : --llvm-lto 1 ;
 toolset.flags emscripten.link OPTIONS <link-optimization>full : --llvm-lto 3 ;
-
-actions compile.c
-{
-    "$(CONFIG_COMMAND)" -x c $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-actions compile.c++
-{
-    "$(CONFIG_COMMAND)" -x c++ $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
-}
-
-actions archive
-{
-    "$(CONFIG_COMMAND)" $(AROPTIONS) -r -o "$(<)" "$(>)"
-}
-
-toolset.flags emscripten.link USER_OPTIONS <linkflags> ;
-
-actions link bind LIBRARIES
-{
-    "$(CONFIG_COMMAND)" $(USER_OPTIONS) -L"$(LINKPATH)" -o "$(<)" "$(>)" "$(LIBRARIES)" $(START-GROUP) $(FINDLIBS-ST-PFX) -l$(FINDLIBS-ST) $(FINDLIBS-SA-PFX) -l$(FINDLIBS-SA) $(END-GROUP) $(OPTIONS)
-}

src/tools/features/os-feature.jam

@@ -8,6 +8,7 @@ import modules ;
 import os ;
 
 .os-names =
+    unknown
     aix android appletv bsd cygwin darwin freebsd haiku hpux iphone linux
     netbsd openbsd osf qnx qnxnto sgi solaris unix unixware windows vms vxworks
     freertos

src/tools/gcc.jam

@@ -1035,12 +1035,12 @@ rule link.dll ( targets * : sources * : properties * )
 
 actions link bind LIBRARIES
 {
-    "$(CONFIG_COMMAND)" @($(<[1]:T).rsp:O=FC:<=@":>=":E=-L"$(LINKPATH)" -Wl,$(RPATH_OPTION:E=-R)$(SPACE)-Wl,$(RPATH) -Wl,-rpath-link$(SPACE)-Wl,"$(RPATH_LINK)" -o "$(<)" $(START-GROUP) "$(>:T)" "$(LIBRARIES)" $(FINDLIBS-ST-PFX) -l$(FINDLIBS-ST) $(FINDLIBS-SA-PFX) -l$(FINDLIBS-SA) $(END-GROUP) $(OPTIONS) $(USER_OPTIONS))
+    "$(CONFIG_COMMAND)" @($(<[1]:T).rsp:O=FC:<=@":>=":E=-L"$(LINKPATH)" -Wl,$(RPATH_OPTION)$(SPACE)-Wl,$(RPATH) -Wl,-rpath-link$(SPACE)-Wl,"$(RPATH_LINK)" -o "$(<)" $(START-GROUP) "$(>:T)" "$(LIBRARIES)" $(FINDLIBS-ST-PFX) -l$(FINDLIBS-ST) $(FINDLIBS-SA-PFX) -l$(FINDLIBS-SA) $(END-GROUP) $(OPTIONS) $(USER_OPTIONS))
 }
 
 actions link.dll bind LIBRARIES
 {
-    "$(CONFIG_COMMAND)" @($(<[1]:T).rsp:O=FC:<=@":>=":E=-L"$(LINKPATH)" -Wl,$(RPATH_OPTION:E=-R)$(SPACE)-Wl,$(RPATH) -Wl,$(IMPLIB_OPTION:E=--out-implib),"$(<[2])" -o "$(<[1])" $(HAVE_SONAME)-Wl,$(SONAME_OPTION)$(SPACE)-Wl,"$(SONAME_PREFIX:E=)$(<[1]:D=)" $(SHARED_OPTION:E=-shared) $(START-GROUP) "$(>:T)" "$(LIBRARIES)" $(FINDLIBS-ST-PFX) -l$(FINDLIBS-ST) $(FINDLIBS-SA-PFX) -l$(FINDLIBS-SA) $(END-GROUP) $(OPTIONS) $(USER_OPTIONS))
+    "$(CONFIG_COMMAND)" @($(<[1]:T).rsp:O=FC:<=@":>=":E=-L"$(LINKPATH)" -Wl,$(RPATH_OPTION)$(SPACE)-Wl,$(RPATH) -Wl,$(IMPLIB_OPTION:E=--out-implib),"$(<[2])" -o "$(<[1])" $(HAVE_SONAME)-Wl,$(SONAME_OPTION)$(SPACE)-Wl,"$(SONAME_PREFIX:E=)$(<[1]:D=)" $(SHARED_OPTION:E=-shared) $(START-GROUP) "$(>:T)" "$(LIBRARIES)" $(FINDLIBS-ST-PFX) -l$(FINDLIBS-ST) $(FINDLIBS-SA-PFX) -l$(FINDLIBS-SA) $(END-GROUP) $(OPTIONS) $(USER_OPTIONS))
 }
 
 ###

test/BoostBuild.py

@@ -113,14 +113,14 @@ def get_toolset():
 
 # Detect the host OS.
 if sys.platform == "cygwin":
-    default_os = "cygwin"
+    host_os = "cygwin"
 elif sys.platform == "win32":
-    default_os = "windows"
+    host_os = "windows"
 elif hasattr(os, "uname"):
-    default_os = os.uname()[0].lower()
+    host_os = os.uname()[0].lower()
 
 
-def expand_toolset(toolset, target_os=default_os):
+def expand_toolset(toolset, target_os):
     match = re.match(r'^(clang|intel)(-[\d\.]+|)$', toolset)
     if match:
         if match.group(1) == "intel" and target_os == "windows":
@@ -133,7 +133,7 @@ def expand_toolset(toolset, target_os=default_os):
     return toolset
 
 
-def prepare_prefixes_and_suffixes(toolset, target_os=default_os):
+def prepare_prefixes_and_suffixes(toolset, target_os):
     ind = toolset.find('-')
     if ind == -1:
         rtoolset = toolset
@@ -143,7 +143,7 @@ def prepare_prefixes_and_suffixes(toolset, target_os=default_os):
     prepare_library_prefix(rtoolset, target_os)
 
 
-def prepare_suffix_map(toolset, target_os=default_os):
+def prepare_suffix_map(toolset, target_os):
     """
       Set up suffix translation performed by the Boost Build testing framework
     to accommodate different toolsets generating targets of the same type using
@@ -175,8 +175,12 @@ def prepare_suffix_map(toolset, target_os=default_os):
         if target_os == "darwin":
             suffixes[".dll"] = ".dylib"
 
+        if toolset == "emscripten":
+            suffixes[".exe"] = ".js" # or .wasm?
+            suffixes[".dll"] = ".so" # .wasn doesn't work for searched libs
 
-def prepare_library_prefix(toolset, target_os=default_os):
+
+def prepare_library_prefix(toolset, target_os):
     """
       Setup whether Boost Build is expected to automatically prepend prefixes
     to its built library targets.
@@ -282,14 +286,9 @@ def __init__(self, arguments=None, executable=None,
         self.translate_suffixes = translate_suffixes
         self.use_test_config = use_test_config
 
-        self.target_os = default_os
-        self.toolset = get_toolset()
-        self.expanded_toolset = expand_toolset(self.toolset)
-        self.pass_toolset = pass_toolset
+        self.set_toolset(get_toolset(), _pass_toolset=pass_toolset)
         self.ignore_toolset_requirements = ignore_toolset_requirements
 
-        prepare_prefixes_and_suffixes(pass_toolset and self.toolset or "gcc")
-
         use_default_bjam = "--default-bjam" in sys.argv
 
         if not use_default_bjam:
@@ -343,12 +342,14 @@ def cleanup(self):
             # this case.
             pass
 
-    def set_toolset(self, toolset, target_os=default_os):
-        self.target_os = target_os
-        self.toolset = toolset
-        self.expanded_toolset = expand_toolset(toolset, target_os)
-        self.pass_toolset = True
-        prepare_prefixes_and_suffixes(toolset, target_os)
+    def set_toolset(self, toolset, target_os=None, _pass_toolset=True):
+        self.toolset = _pass_toolset and toolset or "gcc"
+        if not target_os and self.toolset.startswith("emscripten"):
+            target_os = "unknown"
+        self.target_os = target_os or host_os
+        self.expanded_toolset = expand_toolset(self.toolset, self.target_os)
+        self.pass_toolset = _pass_toolset
+        prepare_prefixes_and_suffixes(self.toolset, self.target_os)
 
     def is_implib_expected(self):
         return self.target_os in ["windows", "cygwin"] and not re.match(r'^clang(-linux)?(-[\d.]+)?$', self.toolset)
@@ -763,6 +764,9 @@ def __ignore_junk(self):
             self.ignore("*.manifest")  # MSVC DLL manifests.
             self.ignore("bin/standalone/msvc/*/msvc-setup.bat")
 
+        # emscripten 'exe' is .js which is a laucnher for .wasm file
+        self.ignore("*.wasm")
+
         # Debug builds of bjam built with gcc produce this profiling data.
         self.ignore("gmon.out")
         self.ignore("*/gmon.out")

test/conditionals.py

@@ -23,7 +23,7 @@
 # Test conditionals in target requirements.
 t.write("jamroot.jam", "exe a : a.cpp : <link>static:<define>STATIC ;")
 t.run_build_system(["link=static"])
-t.expect_addition("bin/$toolset/debug/link-static*/a.exe")
+t.expect_addition("bin/$toolset/debug*/a.exe")
 t.rm("bin")
 
 # Test conditionals in project requirements.
@@ -32,7 +32,7 @@
 exe a : a.cpp ;
 """)
 t.run_build_system(["link=static"])
-t.expect_addition("bin/$toolset/debug/link-static*/a.exe")
+t.expect_addition("bin/$toolset/debug*/a.exe")
 t.rm("bin")
 
 # Regression test for a bug found by Ali Azarbayejani. Conditionals inside
@@ -43,6 +43,6 @@
 """)
 t.write("l.cpp", "int i;")
 t.run_build_system(["link=static"])
-t.expect_addition("bin/$toolset/debug/link-static*/a.exe")
+t.expect_addition("bin/$toolset/debug*/a.exe")
 
 t.cleanup()

test/default_build.py

@@ -29,8 +29,8 @@
 # Now check that we can specify explicit build request and default-build will be
 # combined with it.
 t.run_build_system(["optimization=space"])
-t.expect_addition("bin/$toolset/debug/optimization-space*/a.exe")
-t.expect_addition("bin/$toolset/release/optimization-space*/a.exe")
+t.expect_addition("bin/$toolset/debug*/optimization-space*/a.exe")
+t.expect_addition("bin/$toolset/release*/optimization-space*/a.exe")
 
 # Test that default-build must be identical in all alternatives. Error case.
 t.write("jamfile.jam", """\

test/example_libraries.py

@@ -13,7 +13,7 @@
 
 t.set_tree("../example/libraries")
 
-t.run_build_system()
+t.run_build_system(["link=shared"])
 
 t.expect_addition(["app/bin/$toolset/debug*/app.exe",
                    "util/foo/bin/$toolset/debug*/bar.dll"])