diff --git a/CHANGES.txt b/CHANGES.txt index 33cc7cefd5..fd316d3a80 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -35,6 +35,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Fix versioned shared library naming for MacOS platform. (Previously was libxyz.dylib.1.2.3, has been fixed to libxyz.1.2.3.dylib. Additionally the sonamed symlink had the same issue, that is now resolved as well) + - Add experimental ninja builder. (Contributed by MongoDB, Daniel Moody and many others). - Fix #3955 - _LIBDIRFLAGS leaving $( and $) in *COMSTR output. Added affect_signature flag to _concat function. If set to False, it will prepend and append $( and $). That way the various Environment variables can use that rather than "$( _concat(...) $)". diff --git a/SCons/Node/__init__.py b/SCons/Node/__init__.py index a449082a06..ec742a686b 100644 --- a/SCons/Node/__init__.py +++ b/SCons/Node/__init__.py @@ -946,6 +946,11 @@ def is_conftest(self): return False return True + def check_attributes(self, name): + """ Simple API to check if the node.attributes for name has been set""" + return getattr(getattr(self, "attributes", None), name, None) + + def alter_targets(self): """Return a list of alternate targets for this Node. """ diff --git a/SCons/SConf.py b/SCons/SConf.py index 68e890a69b..38a2b945ac 100644 --- a/SCons/SConf.py +++ b/SCons/SConf.py @@ -606,7 +606,7 @@ def TryBuild(self, builder, text=None, extension=""): f = "_".join([f, textSig, textSigCounter]) textFile = self.confdir.File(f + extension) - self._set_conftest_node(sourcetext) + self._set_conftest_node(textFile) textFileNode = self.env.SConfSourceBuilder(target=textFile, source=sourcetext) nodesToBeBuilt.extend(textFileNode) @@ -625,6 +625,7 @@ def TryBuild(self, builder, text=None, extension=""): pref = self.env.subst( builder.builder.prefix ) suff = self.env.subst( builder.builder.suffix ) target = self.confdir.File(pref + f + suff) + self._set_conftest_node(target) try: # Slide our wrapper into the construction environment as diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index 4988a8d96e..def866303f 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -39,7 +39,7 @@ diskcheck_all = SCons.Node.FS.diskcheck_types() -experimental_features = {'warp_speed', 'transporter'} +experimental_features = {'warp_speed', 'transporter', 'ninja'} def diskcheck_convert(value): @@ -749,7 +749,7 @@ def experimental_callback(option, opt, value, parser): op.add_option('--experimental', dest='experimental', action='callback', - default={}, # empty set + default=set(), # empty set type='str', # choices=experimental_options+experimental_features, callback =experimental_callback, diff --git a/SCons/Tool/ninja/Globals.py b/SCons/Tool/ninja/Globals.py new file mode 100644 index 0000000000..0dc46ea840 --- /dev/null +++ b/SCons/Tool/ninja/Globals.py @@ -0,0 +1,40 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import SCons.Action + +NINJA_RULES = "__NINJA_CUSTOM_RULES" +NINJA_POOLS = "__NINJA_CUSTOM_POOLS" +NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" +NINJA_BUILD = "NINJA_BUILD" +NINJA_WHEREIS_MEMO = {} +NINJA_STAT_MEMO = {} +__NINJA_RULE_MAPPING = {} + + +# These are the types that get_command can do something with +COMMAND_TYPES = ( + SCons.Action.CommandAction, + SCons.Action.CommandGeneratorAction, +) +ninja_builder_initialized = False \ No newline at end of file diff --git a/SCons/Tool/ninja/Methods.py b/SCons/Tool/ninja/Methods.py new file mode 100644 index 0000000000..ef8e44f6a9 --- /dev/null +++ b/SCons/Tool/ninja/Methods.py @@ -0,0 +1,268 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os +import shlex +import textwrap + +import SCons +from SCons.Tool.ninja import NINJA_CUSTOM_HANDLERS, NINJA_RULES, NINJA_POOLS +from SCons.Tool.ninja.Globals import __NINJA_RULE_MAPPING +from SCons.Tool.ninja.Utils import get_targets_sources, get_dependencies, get_order_only, get_outputs, get_inputs, \ + get_rule, get_path, generate_command, get_command_env, get_comstr + + +def register_custom_handler(env, name, handler): + """Register a custom handler for SCons function actions.""" + env[NINJA_CUSTOM_HANDLERS][name] = handler + + +def register_custom_rule_mapping(env, pre_subst_string, rule): + """Register a function to call for a given rule.""" + SCons.Tool.ninja.Globals.__NINJA_RULE_MAPPING[pre_subst_string] = rule + + +def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): + """Allows specification of Ninja rules from inside SCons files.""" + rule_obj = { + "command": command, + "description": description if description else "{} $out".format(rule), + } + + if use_depfile: + rule_obj["depfile"] = os.path.join(get_path(env['NINJA_DIR']), '$out.depfile') + + if deps is not None: + rule_obj["deps"] = deps + + if pool is not None: + rule_obj["pool"] = pool + + if use_response_file: + rule_obj["rspfile"] = "$out.rsp" + rule_obj["rspfile_content"] = response_file_content + + env[NINJA_RULES][rule] = rule_obj + + +def register_custom_pool(env, pool, size): + """Allows the creation of custom Ninja pools""" + env[NINJA_POOLS][pool] = size + + +def set_build_node_callback(env, node, callback): + if not node.is_conftest(): + node.attributes.ninja_build_callback = callback + + +def get_generic_shell_command(env, node, action, targets, sources, executor=None): + return ( + "GENERATED_CMD", + { + "cmd": generate_command(env, node, action, targets, sources, executor=executor), + "env": get_command_env(env), + }, + # Since this function is a rule mapping provider, it must return a list of dependencies, + # and usually this would be the path to a tool, such as a compiler, used for this rule. + # However this function is to generic to be able to reliably extract such deps + # from the command, so we return a placeholder empty list. It should be noted that + # generally this function will not be used solely and is more like a template to generate + # the basics for a custom provider which may have more specific options for a provider + # function for a custom NinjaRuleMapping. + [] + ) + + +def CheckNinjaCompdbExpand(env, context): + """ Configure check testing if ninja's compdb can expand response files""" + + # TODO: When would this be false? + context.Message('Checking if ninja compdb can expand response files... ') + ret, output = context.TryAction( + action='ninja -f $SOURCE -t compdb -x CMD_RSP > $TARGET', + extension='.ninja', + text=textwrap.dedent(""" + rule CMD_RSP + command = $cmd @$out.rsp > fake_output.txt + description = Building $out + rspfile = $out.rsp + rspfile_content = $rspc + build fake_output.txt: CMD_RSP fake_input.txt + cmd = echo + pool = console + rspc = "test" + """)) + result = '@fake_output.txt.rsp' not in output + context.Result(result) + return result + + +def get_command(env, node, action): # pylint: disable=too-many-branches + """Get the command to execute for node.""" + if node.env: + sub_env = node.env + else: + sub_env = env + executor = node.get_executor() + tlist, slist = get_targets_sources(node) + + # Generate a real CommandAction + if isinstance(action, SCons.Action.CommandGeneratorAction): + # pylint: disable=protected-access + action = action._generate(tlist, slist, sub_env, 1, executor=executor) + + variables = {} + + comstr = get_comstr(sub_env, action, tlist, slist) + if not comstr: + return None + + provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) + rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) + + # Get the dependencies for all targets + implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) + + # Now add in the other dependencies related to the command, + # e.g. the compiler binary. The ninja rule can be user provided so + # we must do some validation to resolve the dependency path for ninja. + for provider_dep in provider_deps: + + provider_dep = sub_env.subst(provider_dep) + if not provider_dep: + continue + + # If the tool is a node, then SCons will resolve the path later, if its not + # a node then we assume it generated from build and make sure it is existing. + if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep): + implicit.append(provider_dep) + continue + + # in some case the tool could be in the local directory and be supplied without the ext + # such as in windows, so append the executable suffix and check. + prog_suffix = sub_env.get('PROGSUFFIX', '') + provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix + if os.path.exists(provider_dep_ext): + implicit.append(provider_dep_ext) + continue + + # Many commands will assume the binary is in the path, so + # we accept this as a possible input from a given command. + + provider_dep_abspath = sub_env.WhereIs(provider_dep) or sub_env.WhereIs(provider_dep, path=os.environ["PATH"]) + if provider_dep_abspath: + implicit.append(provider_dep_abspath) + continue + + # Possibly these could be ignore and the build would still work, however it may not always + # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided + # ninja rule. + raise Exception("Could not resolve path for %s dependency on node '%s'" % (provider_dep, node)) + + ninja_build = { + "order_only": get_order_only(node), + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "implicit": implicit, + "rule": get_rule(node, rule), + "variables": variables, + } + + # Don't use sub_env here because we require that NINJA_POOL be set + # on a per-builder call basis to prevent accidental strange + # behavior like env['NINJA_POOL'] = 'console' and sub_env can be + # the global Environment object if node.env is None. + # Example: + # + # Allowed: + # + # env.Command("ls", NINJA_POOL="ls_pool") + # + # Not allowed and ignored: + # + # env["NINJA_POOL"] = "ls_pool" + # env.Command("ls") + # + # TODO: Why not alloe env['NINJA_POOL'] ? (bdbaddog) + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["NINJA_POOL"] + + return ninja_build + + +def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom_env={}): + """Generate a response file command provider for rule name.""" + + # If win32 using the environment with a response file command will cause + # ninja to fail to create the response file. Additionally since these rules + # generally are not piping through cmd.exe /c any environment variables will + # make CreateProcess fail to start. + # + # On POSIX we can still set environment variables even for compile + # commands so we do so. + use_command_env = not env["PLATFORM"] == "win32" + if "$" in tool: + tool_is_dynamic = True + + def get_response_file_command(env, node, action, targets, sources, executor=None): + if hasattr(action, "process"): + cmd_list, _, _ = action.process(targets, sources, env, executor=executor) + cmd_list = [str(c).replace("$", "$$") for c in cmd_list[0]] + else: + command = generate_command( + env, node, action, targets, sources, executor=executor + ) + cmd_list = shlex.split(command) + + if tool_is_dynamic: + tool_command = env.subst( + tool, target=targets, source=sources, executor=executor + ) + else: + tool_command = tool + + try: + # Add 1 so we always keep the actual tool inside of cmd + tool_idx = cmd_list.index(tool_command) + 1 + except ValueError: + raise Exception( + "Could not find tool {} in {} generated from {}".format( + tool, cmd_list, get_comstr(env, action, targets, sources) + ) + ) + + cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] + rsp_content = ['"' + rsp_content_item + '"' for rsp_content_item in rsp_content] + rsp_content = " ".join(rsp_content) + + variables = {"rspc": rsp_content, rule: cmd} + if use_command_env: + variables["env"] = get_command_env(env) + + for key, value in custom_env.items(): + variables["env"] += env.subst( + "export %s=%s;" % (key, value), target=targets, source=sources, executor=executor + ) + " " + return rule, variables, [tool_command] + + return get_response_file_command \ No newline at end of file diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py new file mode 100644 index 0000000000..e7d9882a1b --- /dev/null +++ b/SCons/Tool/ninja/NinjaState.py @@ -0,0 +1,711 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import io +import os +import sys +from os.path import splitext +import ninja + +import SCons +from SCons.Script import COMMAND_LINE_TARGETS +from SCons.Util import is_List +from SCons.Errors import InternalError +from .Globals import COMMAND_TYPES, NINJA_RULES, NINJA_POOLS, \ + NINJA_CUSTOM_HANDLERS +from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function +from .Utils import get_path, alias_to_ninja_build, generate_depfile, ninja_noop, get_order_only, \ + get_outputs, get_inputs, get_dependencies, get_rule, get_command_env +from .Methods import get_command + + +# pylint: disable=too-many-instance-attributes +class NinjaState: + """Maintains state of Ninja build system as it's translated from SCons.""" + + def __init__(self, env, ninja_file, writer_class): + self.env = env + self.ninja_file = ninja_file + + self.ninja_bin_path = env.get('NINJA') + if not self.ninja_bin_path: + # default to using ninja installed with python module + ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' + self.ninja_bin_path = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + ninja_bin)) + if not os.path.exists(self.ninja_bin_path): + # couldn't find it, just give the bin name and hope + # its in the path later + self.ninja_bin_path = ninja_bin + + self.writer_class = writer_class + self.__generated = False + self.translator = SConsToNinjaTranslator(env) + self.generated_suffixes = env.get("NINJA_GENERATED_SOURCE_SUFFIXES", []) + + # List of generated builds that will be written at a later stage + self.builds = dict() + + # List of targets for which we have generated a build. This + # allows us to take multiple Alias nodes as sources and to not + # fail to build if they have overlapping targets. + self.built = set() + + # SCons sets this variable to a function which knows how to do + # shell quoting on whatever platform it's run on. Here we use it + # to make the SCONS_INVOCATION variable properly quoted for things + # like CCFLAGS + escape = env.get("ESCAPE", lambda x: x) + + # if SCons was invoked from python, we expect the first arg to be the scons.py + # script, otherwise scons was invoked from the scons script + python_bin = '' + if os.path.basename(sys.argv[0]) == 'scons.py': + python_bin = escape(sys.executable) + self.variables = { + "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", + "SCONS_INVOCATION": '{} {} --disable-ninja __NINJA_NO=1 $out'.format( + python_bin, + " ".join( + [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] + ), + ), + "SCONS_INVOCATION_W_TARGETS": "{} {}".format( + python_bin, " ".join([escape(arg) for arg in sys.argv]) + ), + # This must be set to a global default per: + # https://ninja-build.org/manual.html#_deps + # English Visual Studio will have the default below, + # otherwise the user can define the variable in the first environment + # that initialized ninja tool + "msvc_deps_prefix": env.get("NINJA_MSVC_DEPS_PREFIX", "Note: including file:") + } + + self.rules = { + "CMD": { + "command": "cmd /c $env$cmd $in $out" if sys.platform == "win32" else "$env$cmd $in $out", + "description": "Building $out", + "pool": "local_pool", + }, + "GENERATED_CMD": { + "command": "cmd /c $env$cmd" if sys.platform == "win32" else "$env$cmd", + "description": "Building $out", + "pool": "local_pool", + }, + # We add the deps processing variables to this below. We + # don't pipe these through cmd.exe on Windows because we + # use this to generate a compile_commands.json database + # which can't use the shell command as it's compile + # command. + "CC": { + "command": "$env$CC @$out.rsp", + "description": "Compiling $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + }, + "CXX": { + "command": "$env$CXX @$out.rsp", + "description": "Compiling $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + }, + "LINK": { + "command": "$env$LINK @$out.rsp", + "description": "Linking $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + "pool": "local_pool", + }, + # Ninja does not automatically delete the archive before + # invoking ar. The ar utility will append to an existing archive, which + # can cause duplicate symbols if the symbols moved between object files. + # Native SCons will perform this operation so we need to force ninja + # to do the same. See related for more info: + # https://jira.mongodb.org/browse/SERVER-49457 + "AR": { + "command": "{}$env$AR @$out.rsp".format( + '' if sys.platform == "win32" else "rm -f $out && " + ), + "description": "Archiving $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + "pool": "local_pool", + }, + "SYMLINK": { + "command": ( + "cmd /c mklink $out $in" + if sys.platform == "win32" + else "ln -s $in $out" + ), + "description": "Symlink $in -> $out", + }, + "INSTALL": { + "command": "$COPY $in $out", + "description": "Install $out", + "pool": "install_pool", + # On Windows cmd.exe /c copy does not always correctly + # update the timestamp on the output file. This leads + # to a stuck constant timestamp in the Ninja database + # and needless rebuilds. + # + # Adding restat here ensures that Ninja always checks + # the copy updated the timestamp and that Ninja has + # the correct information. + "restat": 1, + }, + "TEMPLATE": { + "command": "$SCONS_INVOCATION $out", + "description": "Rendering $SCONS_INVOCATION $out", + "pool": "scons_pool", + "restat": 1, + }, + "SCONS": { + "command": "$SCONS_INVOCATION $out", + "description": "$SCONS_INVOCATION $out", + "pool": "scons_pool", + # restat + # if present, causes Ninja to re-stat the command's outputs + # after execution of the command. Each output whose + # modification time the command did not change will be + # treated as though it had never needed to be built. This + # may cause the output's reverse dependencies to be removed + # from the list of pending build actions. + # + # We use restat any time we execute SCons because + # SCons calls in Ninja typically create multiple + # targets. But since SCons is doing it's own up to + # date-ness checks it may only update say one of + # them. Restat will find out which of the multiple + # build targets did actually change then only rebuild + # those targets which depend specifically on that + # output. + "restat": 1, + }, + "REGENERATE": { + "command": "$SCONS_INVOCATION_W_TARGETS", + "description": "Regenerating $out", + "generator": 1, + "depfile": os.path.join(get_path(env['NINJA_DIR']), '$out.depfile'), + # Console pool restricts to 1 job running at a time, + # it additionally has some special handling about + # passing stdin, stdout, etc to process in this pool + # that we need for SCons to behave correctly when + # regenerating Ninja + "pool": "console", + # Again we restat in case Ninja thought the + # build.ninja should be regenerated but SCons knew + # better. + "restat": 1, + }, + } + + if env['PLATFORM'] == 'darwin' and env['AR'] == 'ar': + self.rules["AR"] = { + "command": "rm -f $out && $env$AR $rspc", + "description": "Archiving $out", + "pool": "local_pool", + } + + num_jobs = self.env.get('NINJA_MAX_JOBS', self.env.GetOption("num_jobs")) + self.pools = { + "local_pool": num_jobs, + "install_pool": num_jobs / 2, + "scons_pool": 1, + } + + for rule in ["CC", "CXX"]: + if env["PLATFORM"] == "win32": + self.rules[rule]["deps"] = "msvc" + else: + self.rules[rule]["deps"] = "gcc" + self.rules[rule]["depfile"] = "$out.d" + + def add_build(self, node): + if not node.has_builder(): + return False + + if isinstance(node, SCons.Node.Alias.Alias): + build = alias_to_ninja_build(node) + else: + build = self.translator.action_to_ninja_build(node) + + # Some things are unbuild-able or need not be built in Ninja + if build is None: + return False + + node_string = str(node) + if node_string in self.builds: + raise InternalError("Node {} added to ninja build state more than once".format(node_string)) + self.builds[node_string] = build + self.built.update(build["outputs"]) + return True + + # TODO: rely on SCons to tell us what is generated source + # or some form of user scanner maybe (Github Issue #3624) + def is_generated_source(self, output): + """Check if output ends with a known generated suffix.""" + _, suffix = splitext(output) + return suffix in self.generated_suffixes + + def has_generated_sources(self, output): + """ + Determine if output indicates this is a generated header file. + """ + for generated in output: + if self.is_generated_source(generated): + return True + return False + + # pylint: disable=too-many-branches,too-many-locals + def generate(self): + """ + Generate the build.ninja. + + This should only be called once for the lifetime of this object. + """ + if self.__generated: + return + + self.rules.update(self.env.get(NINJA_RULES, {})) + self.pools.update(self.env.get(NINJA_POOLS, {})) + + content = io.StringIO() + ninja = self.writer_class(content, width=100) + + ninja.comment("Generated by scons. DO NOT EDIT.") + + ninja.variable("builddir", get_path(self.env['NINJA_DIR'])) + + for pool_name, size in self.pools.items(): + ninja.pool(pool_name, min(self.env.get('NINJA_MAX_JOBS', size), size)) + + for var, val in self.variables.items(): + ninja.variable(var, val) + + for rule, kwargs in self.rules.items(): + if self.env.get('NINJA_MAX_JOBS') is not None and 'pool' not in kwargs: + kwargs['pool'] = 'local_pool' + ninja.rule(rule, **kwargs) + + generated_source_files = sorted({ + output + # First find builds which have header files in their outputs. + for build in self.builds.values() + if self.has_generated_sources(build["outputs"]) + for output in build["outputs"] + # Collect only the header files from the builds with them + # in their output. We do this because is_generated_source + # returns True if it finds a header in any of the outputs, + # here we need to filter so we only have the headers and + # not the other outputs. + if self.is_generated_source(output) + }) + + if generated_source_files: + ninja.build( + outputs="_generated_sources", + rule="phony", + implicit=generated_source_files + ) + + template_builders = [] + + for build in [self.builds[key] for key in sorted(self.builds.keys())]: + if build["rule"] == "TEMPLATE": + template_builders.append(build) + continue + + if "implicit" in build: + build["implicit"].sort() + + # Don't make generated sources depend on each other. We + # have to check that none of the outputs are generated + # sources and none of the direct implicit dependencies are + # generated sources or else we will create a dependency + # cycle. + if ( + generated_source_files + and not build["rule"] == "INSTALL" + and set(build["outputs"]).isdisjoint(generated_source_files) + and set(build.get("implicit", [])).isdisjoint(generated_source_files) + ): + # Make all non-generated source targets depend on + # _generated_sources. We use order_only for generated + # sources so that we don't rebuild the world if one + # generated source was rebuilt. We just need to make + # sure that all of these sources are generated before + # other builds. + order_only = build.get("order_only", []) + order_only.append("_generated_sources") + build["order_only"] = order_only + if "order_only" in build: + build["order_only"].sort() + + # When using a depfile Ninja can only have a single output + # but SCons will usually have emitted an output for every + # thing a command will create because it's caching is much + # more complex than Ninja's. This includes things like DWO + # files. Here we make sure that Ninja only ever sees one + # target when using a depfile. It will still have a command + # that will create all of the outputs but most targets don't + # depend directly on DWO files and so this assumption is safe + # to make. + rule = self.rules.get(build["rule"]) + + # Some rules like 'phony' and other builtins we don't have + # listed in self.rules so verify that we got a result + # before trying to check if it has a deps key. + # + # Anything using deps or rspfile in Ninja can only have a single + # output, but we may have a build which actually produces + # multiple outputs which other targets can depend on. Here we + # slice up the outputs so we have a single output which we will + # use for the "real" builder and multiple phony targets that + # match the file names of the remaining outputs. This way any + # build can depend on any output from any build. + # + # We assume that the first listed output is the 'key' + # output and is stably presented to us by SCons. For + # instance if -gsplit-dwarf is in play and we are + # producing foo.o and foo.dwo, we expect that outputs[0] + # from SCons will be the foo.o file and not the dwo + # file. If instead we just sorted the whole outputs array, + # we would find that the dwo file becomes the + # first_output, and this breaks, for instance, header + # dependency scanning. + if rule is not None and (rule.get("deps") or rule.get("rspfile")): + first_output, remaining_outputs = ( + build["outputs"][0], + build["outputs"][1:], + ) + + if remaining_outputs: + ninja.build( + outputs=sorted(remaining_outputs), rule="phony", implicit=first_output, + ) + + build["outputs"] = first_output + + # Optionally a rule can specify a depfile, and SCons can generate implicit + # dependencies into the depfile. This allows for dependencies to come and go + # without invalidating the ninja file. The depfile was created in ninja specifically + # for dealing with header files appearing and disappearing across rebuilds, but it can + # be repurposed for anything, as long as you have a way to regenerate the depfile. + # More specific info can be found here: https://ninja-build.org/manual.html#_depfile + if rule is not None and rule.get('depfile') and build.get('deps_files'): + path = build['outputs'] if SCons.Util.is_List(build['outputs']) else [build['outputs']] + generate_depfile(self.env, path[0], build.pop('deps_files', [])) + + if "inputs" in build: + build["inputs"].sort() + + ninja.build(**build) + + template_builds = dict() + for template_builder in template_builders: + + # Special handling for outputs and implicit since we need to + # aggregate not replace for each builder. + for agg_key in ["outputs", "implicit", "inputs"]: + new_val = template_builds.get(agg_key, []) + + # Use pop so the key is removed and so the update + # below will not overwrite our aggregated values. + cur_val = template_builder.pop(agg_key, []) + if is_List(cur_val): + new_val += cur_val + else: + new_val.append(cur_val) + template_builds[agg_key] = new_val + + # Collect all other keys + template_builds.update(template_builder) + + if template_builds.get("outputs", []): + ninja.build(**template_builds) + + # We have to glob the SCons files here to teach the ninja file + # how to regenerate itself. We'll never see ourselves in the + # DAG walk so we can't rely on action_to_ninja_build to + # generate this rule even though SCons should know we're + # dependent on SCons files. + # + # The REGENERATE rule uses depfile, so we need to generate the depfile + # in case any of the SConscripts have changed. The depfile needs to be + # path with in the build and the passed ninja file is an abspath, so + # we will use SCons to give us the path within the build. Normally + # generate_depfile should not be called like this, but instead be called + # through the use of custom rules, and filtered out in the normal + # list of build generation about. However, because the generate rule + # is hardcoded here, we need to do this generate_depfile call manually. + ninja_file_path = self.env.File(self.ninja_file).path + generate_depfile( + self.env, + ninja_file_path, + self.env['NINJA_REGENERATE_DEPS'] + ) + + ninja.build( + ninja_file_path, + rule="REGENERATE", + implicit=[__file__], + ) + + # If we ever change the name/s of the rules that include + # compile commands (i.e. something like CC) we will need to + # update this build to reflect that complete list. + ninja.build( + "compile_commands.json", + rule="CMD", + pool="console", + implicit=[str(self.ninja_file)], + variables={ + "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( + # NINJA_COMPDB_EXPAND - should only be true for ninja + # This was added to ninja's compdb tool in version 1.9.0 (merged April 2018) + # https://github.com/ninja-build/ninja/pull/1223 + # TODO: add check in generate to check version and enable this by default if it's available. + self.ninja_bin_path, str(self.ninja_file), + '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' + ) + }, + ) + + ninja.build( + "compiledb", rule="phony", implicit=["compile_commands.json"], + ) + + # Look in SCons's list of DEFAULT_TARGETS, find the ones that + # we generated a ninja build rule for. + scons_default_targets = [ + get_path(tgt) + for tgt in SCons.Script.DEFAULT_TARGETS + if get_path(tgt) in self.built + ] + + # If we found an overlap between SCons's list of default + # targets and the targets we created ninja builds for then use + # those as ninja's default as well. + if scons_default_targets: + ninja.default(" ".join(scons_default_targets)) + + with open(str(self.ninja_file), "w") as build_ninja: + build_ninja.write(content.getvalue()) + + self.__generated = True + + +class SConsToNinjaTranslator: + """Translates SCons Actions into Ninja build objects.""" + + def __init__(self, env): + self.env = env + self.func_handlers = { + # Skip conftest builders + "_createSource": ninja_noop, + # SCons has a custom FunctionAction that just makes sure the + # target isn't static. We let the commands that ninja runs do + # this check for us. + "SharedFlagChecker": ninja_noop, + # The install builder is implemented as a function action. + # TODO: use command action #3573 + "installFunc": _install_action_function, + "MkdirFunc": _mkdir_action_function, + "LibSymlinksActionFunction": _lib_symlink_action_function, + "Copy": _copy_action_function + } + + self.loaded_custom = False + + # pylint: disable=too-many-return-statements + def action_to_ninja_build(self, node, action=None): + """Generate build arguments dictionary for node.""" + + if not self.loaded_custom: + self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) + self.loaded_custom = True + + if node.builder is None: + return None + + if action is None: + action = node.builder.action + + if node.env and node.env.get("NINJA_SKIP"): + return None + + build = {} + env = node.env if node.env else self.env + + # Ideally this should never happen, and we do try to filter + # Ninja builders out of being sources of ninja builders but I + # can't fix every DAG problem so we just skip ninja_builders + # if we find one + if SCons.Tool.ninja.NINJA_STATE.ninja_file == str(node): + build = None + elif isinstance(action, SCons.Action.FunctionAction): + build = self.handle_func_action(node, action) + elif isinstance(action, SCons.Action.LazyAction): + # pylint: disable=protected-access + action = action._generate_cache(env) + build = self.action_to_ninja_build(node, action=action) + elif isinstance(action, SCons.Action.ListAction): + build = self.handle_list_action(node, action) + elif isinstance(action, COMMAND_TYPES): + build = get_command(env, node, action) + else: + raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) + + if build is not None: + build["order_only"] = get_order_only(node) + + # TODO: WPD Is this testing the filename to verify it's a configure context generated file? + if not node.is_conftest(): + node_callback = node.check_attributes("ninja_build_callback") + if callable(node_callback): + node_callback(env, node, build) + + return build + + def handle_func_action(self, node, action): + """Determine how to handle the function action.""" + name = action.function_name() + # This is the name given by the Subst/Textfile builders. So return the + # node to indicate that SCons is required. We skip sources here because + # dependencies don't really matter when we're going to shove these to + # the bottom of ninja's DAG anyway and Textfile builders can have text + # content as their source which doesn't work as an implicit dep in + # ninja. + if name == 'ninja_builder': + return None + + handler = self.func_handlers.get(name, None) + if handler is not None: + return handler(node.env if node.env else self.env, node) + elif name == "ActionCaller": + action_to_call = str(action).split('(')[0].strip() + handler = self.func_handlers.get(action_to_call, None) + if handler is not None: + return handler(node.env if node.env else self.env, node) + + SCons.Warnings.SConsWarning( + "Found unhandled function action {}, " + " generating scons command to build\n" + "Note: this is less efficient than Ninja," + " you can write your own ninja build generator for" + " this function using NinjaRegisterFunctionHandler".format(name) + ) + + return { + "rule": "TEMPLATE", + "order_only": get_order_only(node), + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "implicit": get_dependencies(node, skip_sources=True), + } + + # pylint: disable=too-many-branches + def handle_list_action(self, node, action): + """TODO write this comment""" + results = [ + self.action_to_ninja_build(node, action=act) + for act in action.list + if act is not None + ] + results = [ + result for result in results if result is not None and result["outputs"] + ] + if not results: + return None + + # No need to process the results if we only got a single result + if len(results) == 1: + return results[0] + + all_outputs = list({output for build in results for output in build["outputs"]}) + dependencies = list({dep for build in results for dep in build["implicit"]}) + + if results[0]["rule"] == "CMD" or results[0]["rule"] == "GENERATED_CMD": + cmdline = "" + for cmd in results: + + # Occasionally a command line will expand to a + # whitespace only string (i.e. ' '). Which is not a + # valid command but does not trigger the empty command + # condition if not cmdstr. So here we strip preceding + # and proceeding whitespace to make strings like the + # above become empty strings and so will be skipped. + cmdstr = cmd["variables"]["cmd"].strip() + if not cmdstr: + continue + + # Skip duplicate commands + if cmdstr in cmdline: + continue + + if cmdline: + cmdline += " && " + + cmdline += cmdstr + + # Remove all preceding and proceeding whitespace + cmdline = cmdline.strip() + + # Make sure we didn't generate an empty cmdline + if cmdline: + ninja_build = { + "outputs": all_outputs, + "rule": get_rule(node, "GENERATED_CMD"), + "variables": { + "cmd": cmdline, + "env": get_command_env(node.env if node.env else self.env), + }, + "implicit": dependencies, + } + + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["pool"] + + return ninja_build + + elif results[0]["rule"] == "phony": + return { + "outputs": all_outputs, + "rule": "phony", + "implicit": dependencies, + } + + elif results[0]["rule"] == "INSTALL": + return { + "outputs": all_outputs, + "rule": get_rule(node, "INSTALL"), + "inputs": get_inputs(node), + "implicit": dependencies, + } + + raise Exception("Unhandled list action with rule: " + results[0]["rule"]) diff --git a/SCons/Tool/ninja/Overrides.py b/SCons/Tool/ninja/Overrides.py new file mode 100644 index 0000000000..c6f12074b7 --- /dev/null +++ b/SCons/Tool/ninja/Overrides.py @@ -0,0 +1,96 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +This module is to hold logic which overrides default SCons behaviors to enable +ninja file generation +""" +import SCons + + +def ninja_hack_linkcom(env): + # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks + # without relying on the SCons hacks that SCons uses by default. + if env["PLATFORM"] == "win32": + from SCons.Tool.mslink import compositeLinkAction + + if env.get("LINKCOM", None) == compositeLinkAction: + env[ + "LINKCOM" + ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' + env[ + "SHLINKCOM" + ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' + + +def ninja_hack_arcom(env): + """ + Force ARCOM so use 's' flag on ar instead of separately running ranlib + """ + if env["PLATFORM"] != "win32" and env.get("RANLIBCOM"): + # There is no way to translate the ranlib list action into + # Ninja so add the s flag and disable ranlib. + # + # This is equivalent to Meson. + # https://github.com/mesonbuild/meson/blob/master/mesonbuild/linkers.py#L143 + old_arflags = str(env["ARFLAGS"]) + if "s" not in old_arflags: + old_arflags += "s" + + env["ARFLAGS"] = SCons.Util.CLVar([old_arflags]) + + # Disable running ranlib, since we added 's' above + env["RANLIBCOM"] = "" + + +class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): + """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" + + def __call__(self, target, source, env, for_signature): + return self.cmd + + def _print_cmd_str(*_args, **_kwargs): + """Disable this method""" + pass + + +def ninja_always_serial(self, num, taskmaster): + """Replacement for SCons.Job.Jobs constructor which always uses the Serial Job class.""" + # We still set self.num_jobs to num even though it's a lie. The + # only consumer of this attribute is the Parallel Job class AND + # the Main.py function which instantiates a Jobs class. It checks + # if Jobs.num_jobs is equal to options.num_jobs, so if the user + # provides -j12 but we set self.num_jobs = 1 they get an incorrect + # warning about this version of Python not supporting parallel + # builds. So here we lie so the Main.py will not give a false + # warning to users. + self.num_jobs = num + self.job = SCons.Job.Serial(taskmaster) + + +# pylint: disable=too-few-public-methods +class AlwaysExecAction(SCons.Action.FunctionAction): + """Override FunctionAction.__call__ to always execute.""" + + def __call__(self, *args, **kwargs): + kwargs["execute"] = 1 + return super().__call__(*args, **kwargs) \ No newline at end of file diff --git a/SCons/Tool/ninja/Rules.py b/SCons/Tool/ninja/Rules.py new file mode 100644 index 0000000000..a2f6bc5456 --- /dev/null +++ b/SCons/Tool/ninja/Rules.py @@ -0,0 +1,81 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from .Utils import get_outputs, get_rule, get_inputs, get_dependencies + + +def _install_action_function(_env, node): + """Install files using the install or copy commands""" + return { + "outputs": get_outputs(node), + "rule": get_rule(node, "INSTALL"), + "inputs": get_inputs(node), + "implicit": get_dependencies(node), + } + + +def _mkdir_action_function(env, node): + return { + "outputs": get_outputs(node), + "rule": get_rule(node, "CMD"), + # implicit explicitly omitted, we translate these so they can be + # used by anything that depends on these but commonly this is + # hit with a node that will depend on all of the fake + # srcnode's that SCons will never give us a rule for leading + # to an invalid ninja file. + "variables": { + # On Windows mkdir "-p" is always on + "cmd": "{mkdir}".format( + mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", + ), + }, + } + + +def _copy_action_function(env, node): + return { + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "rule": get_rule(node, "CMD"), + "variables": { + "cmd": "$COPY $in $out", + }, + } + + +def _lib_symlink_action_function(_env, node): + """Create shared object symlinks if any need to be created""" + symlinks = node.check_attributes("shliblinks") + + if not symlinks or symlinks is None: + return None + + outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] + inputs = [link.get_path() for link, _ in symlinks] + + return { + "outputs": outputs, + "inputs": inputs, + "rule": get_rule(node, "SYMLINK"), + "implicit": get_dependencies(node), + } diff --git a/SCons/Tool/ninja/Utils.py b/SCons/Tool/ninja/Utils.py new file mode 100644 index 0000000000..3bdefe8207 --- /dev/null +++ b/SCons/Tool/ninja/Utils.py @@ -0,0 +1,419 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +import os +import shutil +from os.path import join as joinpath + +import SCons +from SCons.Action import get_default_ENV, _string_from_cmd_list +from SCons.Script import AddOption +from SCons.Util import is_List, flatten_sequence + + +def ninja_add_command_line_options(): + """ + Add additional command line arguments to SCons specific to the ninja tool + """ + AddOption('--disable-execute-ninja', + dest='disable_execute_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + + AddOption('--disable-ninja', + dest='disable_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + + +def is_valid_dependent_node(node): + """ + Return True if node is not an alias or is an alias that has children + + This prevents us from making phony targets that depend on other + phony targets that will never have an associated ninja build + target. + + We also have to specify that it's an alias when doing the builder + check because some nodes (like src files) won't have builders but + are valid implicit dependencies. + """ + if isinstance(node, SCons.Node.Alias.Alias): + return node.children() + + return not node.get_env().get("NINJA_SKIP") + + + + +def alias_to_ninja_build(node): + """Convert an Alias node into a Ninja phony target""" + return { + "outputs": get_outputs(node), + "rule": "phony", + "implicit": [ + get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) + ], + } + + +def check_invalid_ninja_node(node): + return not isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)) + + +def filter_ninja_nodes(node_list): + ninja_nodes = [] + for node in node_list: + if isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)) and not node.get_env().get('NINJA_SKIP'): + ninja_nodes.append(node) + else: + continue + return ninja_nodes + + +def get_input_nodes(node): + if node.get_executor() is not None: + inputs = node.get_executor().get_all_sources() + else: + inputs = node.sources + return inputs + + +def invalid_ninja_nodes(node, targets): + result = False + for node_list in [node.prerequisites, get_input_nodes(node), node.children(), targets]: + if node_list: + result = result or any([check_invalid_ninja_node(node) for node in node_list]) + return result + + +def get_order_only(node): + """Return a list of order only dependencies for node.""" + if node.prerequisites is None: + return [] + return [get_path(src_file(prereq)) for prereq in filter_ninja_nodes(node.prerequisites)] + + +def get_dependencies(node, skip_sources=False): + """Return a list of dependencies for node.""" + if skip_sources: + return [ + get_path(src_file(child)) + for child in filter_ninja_nodes(node.children()) + if child not in node.sources + ] + return [get_path(src_file(child)) for child in filter_ninja_nodes(node.children())] + + +def get_inputs(node): + """Collect the Ninja inputs for node.""" + return [get_path(src_file(o)) for o in filter_ninja_nodes(get_input_nodes(node))] + + +def get_outputs(node): + """Collect the Ninja outputs for node.""" + executor = node.get_executor() + if executor is not None: + outputs = executor.get_all_targets() + else: + if hasattr(node, "target_peers"): + outputs = node.target_peers + else: + outputs = [node] + + outputs = [get_path(o) for o in filter_ninja_nodes(outputs)] + + return outputs + + +def get_targets_sources(node): + executor = node.get_executor() + if executor is not None: + tlist = executor.get_all_targets() + slist = executor.get_all_sources() + else: + if hasattr(node, "target_peers"): + tlist = node.target_peers + else: + tlist = [node] + slist = node.sources + + # Retrieve the repository file for all sources + slist = [rfile(s) for s in slist] + return tlist, slist + +def get_path(node): + """ + Return a fake path if necessary. + + As an example Aliases use this as their target name in Ninja. + """ + if hasattr(node, "get_path"): + return node.get_path() + return str(node) + + +def rfile(node): + """ + Return the repository file for node if it has one. Otherwise return node + """ + if hasattr(node, "rfile"): + return node.rfile() + return node + + +def src_file(node): + """Returns the src code file if it exists.""" + if hasattr(node, "srcnode"): + src = node.srcnode() + if src.stat() is not None: + return src + return get_path(node) + +def get_rule(node, rule): + tlist, slist = get_targets_sources(node) + if invalid_ninja_nodes(node, tlist): + return "TEMPLATE" + else: + return rule + + +def generate_depfile(env, node, dependencies): + """ + Ninja tool function for writing a depfile. The depfile should include + the node path followed by all the dependent files in a makefile format. + + dependencies arg can be a list or a subst generator which returns a list. + """ + + depfile = os.path.join(get_path(env['NINJA_DIR']), str(node) + '.depfile') + + # subst_list will take in either a raw list or a subst callable which generates + # a list, and return a list of CmdStringHolders which can be converted into raw strings. + # If a raw list was passed in, then scons_list will make a list of lists from the original + # values and even subst items in the list if they are substitutable. Flatten will flatten + # the list in that case, to ensure for either input we have a list of CmdStringHolders. + deps_list = env.Flatten(env.subst_list(dependencies)) + + # Now that we have the deps in a list as CmdStringHolders, we can convert them into raw strings + # and make sure to escape the strings to handle spaces in paths. We also will sort the result + # keep the order of the list consistent. + escaped_depends = sorted([dep.escape(env.get("ESCAPE", lambda x: x)) for dep in deps_list]) + depfile_contents = str(node) + ": " + ' '.join(escaped_depends) + + need_rewrite = False + try: + with open(depfile, 'r') as f: + need_rewrite = (f.read() != depfile_contents) + except FileNotFoundError: + need_rewrite = True + + if need_rewrite: + os.makedirs(os.path.dirname(depfile) or '.', exist_ok=True) + with open(depfile, 'w') as f: + f.write(depfile_contents) + + +def ninja_noop(*_args, **_kwargs): + """ + A general purpose no-op function. + + There are many things that happen in SCons that we don't need and + also don't return anything. We use this to disable those functions + instead of creating multiple definitions of the same thing. + """ + return None + + +def get_command_env(env): + """ + Return a string that sets the environment for any environment variables that + differ between the OS environment and the SCons command ENV. + + It will be compatible with the default shell of the operating system. + """ + try: + return env["NINJA_ENV_VAR_CACHE"] + except KeyError: + pass + + # Scan the ENV looking for any keys which do not exist in + # os.environ or differ from it. We assume if it's a new or + # differing key from the process environment then it's + # important to pass down to commands in the Ninja file. + ENV = get_default_ENV(env) + scons_specified_env = { + key: value + for key, value in ENV.items() + # TODO: Remove this filter, unless there's a good reason to keep. SCons's behavior shouldn't depend on shell's. + if key not in os.environ or os.environ.get(key, None) != value + } + + windows = env["PLATFORM"] == "win32" + command_env = "" + for key, value in scons_specified_env.items(): + # Ensure that the ENV values are all strings: + if is_List(value): + # If the value is a list, then we assume it is a + # path list, because that's a pretty common list-like + # value to stick in an environment variable: + value = flatten_sequence(value) + value = joinpath(map(str, value)) + else: + # If it isn't a string or a list, then we just coerce + # it to a string, which is the proper way to handle + # Dir and File instances and will produce something + # reasonable for just about everything else: + value = str(value) + + if windows: + command_env += "set '{}={}' && ".format(key, value) + else: + # We address here *only* the specific case that a user might have + # an environment variable which somehow gets included and has + # spaces in the value. These are escapes that Ninja handles. This + # doesn't make builds on paths with spaces (Ninja and SCons issues) + # nor expanding response file paths with spaces (Ninja issue) work. + value = value.replace(r' ', r'$ ') + command_env += "export {}='{}';".format(key, value) + + env["NINJA_ENV_VAR_CACHE"] = command_env + return command_env + + +def get_comstr(env, action, targets, sources): + """Get the un-substituted string for action.""" + # Despite being having "list" in it's name this member is not + # actually a list. It's the pre-subst'd string of the command. We + # use it to determine if the command we're about to generate needs + # to use a custom Ninja rule. By default this redirects CC, CXX, + # AR, SHLINK, and LINK commands to their respective rules but the + # user can inject custom Ninja rules and tie them to commands by + # using their pre-subst'd string. + if hasattr(action, "process"): + return action.cmd_list + + return action.genstring(targets, sources, env) + + +def generate_command(env, node, action, targets, sources, executor=None): + # Actions like CommandAction have a method called process that is + # used by SCons to generate the cmd_line they need to run. So + # check if it's a thing like CommandAction and call it if we can. + if hasattr(action, "process"): + cmd_list, _, _ = action.process(targets, sources, env, executor=executor) + cmd = _string_from_cmd_list(cmd_list[0]) + else: + # Anything else works with genstring, this is most commonly hit by + # ListActions which essentially call process on all of their + # commands and concatenate it for us. + genstring = action.genstring(targets, sources, env) + if executor is not None: + cmd = env.subst(genstring, executor=executor) + else: + cmd = env.subst(genstring, targets, sources) + + cmd = cmd.replace("\n", " && ").strip() + if cmd.endswith("&&"): + cmd = cmd[0:-2].strip() + + # Escape dollars as necessary + return cmd.replace("$", "$$") + + +def ninja_csig(original): + """Return a dummy csig""" + + def wrapper(self): + if isinstance(self, SCons.Node.Node) and self.is_sconscript(): + return original(self) + return "dummy_ninja_csig" + + return wrapper + + +def ninja_contents(original): + """Return a dummy content without doing IO""" + + def wrapper(self): + if isinstance(self, SCons.Node.Node) and self.is_sconscript(): + return original(self) + return bytes("dummy_ninja_contents", encoding="utf-8") + + return wrapper + + +def ninja_stat(_self, path): + """ + Eternally memoized stat call. + + SCons is very aggressive about clearing out cached values. For our + purposes everything should only ever call stat once since we're + running in a no_exec build the file system state should not + change. For these reasons we patch SCons.Node.FS.LocalFS.stat to + use our eternal memoized dictionary. + """ + + try: + return SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] + except KeyError: + try: + result = os.stat(path) + except os.error: + result = None + + SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] = result + return result + + +def ninja_whereis(thing, *_args, **_kwargs): + """Replace env.WhereIs with a much faster version""" + + # Optimize for success, this gets called significantly more often + # when the value is already memoized than when it's not. + try: + return SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] + except KeyError: + # TODO: Fix this to respect env['ENV']['PATH']... WPD + # We do not honor any env['ENV'] or env[*] variables in the + # generated ninja file. Ninja passes your raw shell environment + # down to it's subprocess so the only sane option is to do the + # same during generation. At some point, if and when we try to + # upstream this, I'm sure a sticking point will be respecting + # env['ENV'] variables and such but it's actually quite + # complicated. I have a naive version but making it always work + # with shell quoting is nigh impossible. So I've decided to + # cross that bridge when it's absolutely required. + path = shutil.which(thing) + SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] = path + return path + + +def ninja_print_conf_log(s, target, source, env): + """Command line print only for conftest to generate a correct conf log.""" + if target and target[0].is_conftest(): + action = SCons.Action._ActionAction() + action.print_cmd_line(s, target, source, env) \ No newline at end of file diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py new file mode 100644 index 0000000000..d940bb1636 --- /dev/null +++ b/SCons/Tool/ninja/__init__.py @@ -0,0 +1,450 @@ +# MIT License +# +# Copyright 2020 MongoDB Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +"""Generate build.ninja files from SCons aliases.""" + +import importlib +import os +import subprocess +import sys + +import SCons +import SCons.Tool.ninja.Globals +from SCons.Script import GetOption + +from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS +from .Methods import register_custom_handler, register_custom_rule_mapping, register_custom_rule, register_custom_pool, \ + set_build_node_callback, get_generic_shell_command, CheckNinjaCompdbExpand, get_command, \ + gen_get_response_file_command +from .NinjaState import NinjaState +from .Overrides import ninja_hack_linkcom, ninja_hack_arcom, NinjaNoResponseFiles, ninja_always_serial, AlwaysExecAction +from .Utils import ninja_add_command_line_options, \ + ninja_noop, ninja_print_conf_log, ninja_csig, ninja_contents, ninja_stat, ninja_whereis + +NINJA_STATE = None + + +def ninja_builder(env, target, source): + """Generate a build.ninja for source.""" + if not isinstance(source, list): + source = [source] + if not isinstance(target, list): + target = [target] + + # We have no COMSTR equivalent so print that we're generating + # here. + print("Generating:", str(target[0])) + + generated_build_ninja = target[0].get_abspath() + NINJA_STATE.generate() + + if env["PLATFORM"] == "win32": + # TODO: Is this necessary as you set env variable in the ninja build file per target? + # this is not great, its doesn't consider specific + # node environments, which means on linux the build could + # behave differently, because on linux you can set the environment + # per command in the ninja file. This is only needed if + # running ninja directly from a command line that hasn't + # had the environment setup (vcvarsall.bat) + with open('run_ninja_env.bat', 'w') as f: + for key in env['ENV']: + f.write('set {}={}\n'.format(key, env['ENV'][key])) + f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) + cmd = ['run_ninja_env.bat'] + + else: + cmd = [NINJA_STATE.ninja_bin_path, '-f', generated_build_ninja] + + if not env.get("NINJA_DISABLE_AUTO_RUN"): + print("Executing:", str(' '.join(cmd))) + + # execute the ninja build at the end of SCons, trying to + # reproduce the output like a ninja build would + def execute_ninja(): + + proc = subprocess.Popen(cmd, + stderr=sys.stderr, + stdout=subprocess.PIPE, + universal_newlines=True, + env=os.environ if env["PLATFORM"] == "win32" else env['ENV'] + ) + for stdout_line in iter(proc.stdout.readline, ""): + yield stdout_line + proc.stdout.close() + return_code = proc.wait() + if return_code: + raise subprocess.CalledProcessError(return_code, 'ninja') + + erase_previous = False + for output in execute_ninja(): + output = output.strip() + if erase_previous: + sys.stdout.write('\x1b[2K') # erase previous line + sys.stdout.write("\r") + else: + sys.stdout.write(os.linesep) + sys.stdout.write(output) + sys.stdout.flush() + # this will only erase ninjas [#/#] lines + # leaving warnings and other output, seems a bit + # prone to failure with such a simple check + erase_previous = output.startswith('[') + + +def exists(env): + """Enable if called.""" + + if 'ninja' not in GetOption('experimental'): + return False + + # This variable disables the tool when storing the SCons command in the + # generated ninja file to ensure that the ninja tool is not loaded when + # SCons should do actual work as a subprocess of a ninja build. The ninja + # tool is very invasive into the internals of SCons and so should never be + # enabled when SCons needs to build a target. + if env.get("__NINJA_NO", "0") == "1": + return False + + try: + import ninja + return ninja.__file__ + except ImportError: + SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") + return False + + +def ninja_emitter(target, source, env): + """ fix up the source/targets """ + + ninja_file = env.File(env.subst("$NINJA_FILE_NAME")) + ninja_file.attributes.ninja_file = True + + # Someone called env.Ninja('my_targetname.ninja') + if not target and len(source) == 1: + target = source + + # Default target name is $NINJA_PREFIX.$NINJA.SUFFIX + if not target: + target = [ninja_file, ] + + # No source should have been passed. Drop it. + if source: + source = [] + + return target, source + + +def generate(env): + """Generate the NINJA builders.""" + global NINJA_STATE + + if 'ninja' not in GetOption('experimental'): + return + + if not SCons.Tool.ninja.Globals.ninja_builder_initialized: + SCons.Tool.ninja.Globals.ninja_builder_initialized = True + + ninja_add_command_line_options() + + try: + import ninja # noqa: F401 + except ImportError: + SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") + return + + env["NINJA_DISABLE_AUTO_RUN"] = GetOption('disable_execute_ninja') + + env["NINJA_FILE_NAME"] = env.get("NINJA_FILE_NAME", "build.ninja") + + # Add the Ninja builder. + always_exec_ninja_action = AlwaysExecAction(ninja_builder, {}) + ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action, + emitter=ninja_emitter) + env.Append(BUILDERS={"Ninja": ninja_builder_obj}) + + env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") + env['NINJA_DIR'] = env.get("NINJA_DIR", env.Dir(".ninja").path) + + # here we allow multiple environments to construct rules and builds + # into the same ninja file + if NINJA_STATE is None: + ninja_file = env.Ninja() + env.AlwaysBuild(ninja_file) + env.Alias("$NINJA_ALIAS_NAME", ninja_file) + else: + if str(NINJA_STATE.ninja_file) != env["NINJA_FILE_NAME"]: + SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") + ninja_file = [NINJA_STATE.ninja_file] + + def ninja_generate_deps(env): + """Return a list of SConscripts + TODO: Should we also include files loaded from site_scons/*** + or even all loaded modules? https://stackoverflow.com/questions/4858100/how-to-list-imported-modules + TODO: Do we want this to be Nodes? + """ + return sorted([str(s) for s in SCons.Node.SConscriptNodes]) + + env['_NINJA_REGENERATE_DEPS_FUNC'] = ninja_generate_deps + + env['NINJA_REGENERATE_DEPS'] = env.get('NINJA_REGENERATE_DEPS', '${_NINJA_REGENERATE_DEPS_FUNC(__env__)}') + + # This adds the required flags such that the generated compile + # commands will create depfiles as appropriate in the Ninja file. + if env["PLATFORM"] == "win32": + env.Append(CCFLAGS=["/showIncludes"]) + else: + env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) + + env.AddMethod(CheckNinjaCompdbExpand, "CheckNinjaCompdbExpand") + + # Provide a way for custom rule authors to easily access command + # generation. + env.AddMethod(get_generic_shell_command, "NinjaGetGenericShellCommand") + env.AddMethod(get_command, "NinjaGetCommand") + env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider") + env.AddMethod(set_build_node_callback, "NinjaSetBuildNodeCallback") + + # Provides a way for users to handle custom FunctionActions they + # want to translate to Ninja. + env[NINJA_CUSTOM_HANDLERS] = {} + env.AddMethod(register_custom_handler, "NinjaRegisterFunctionHandler") + + # Provides a mechanism for inject custom Ninja rules which can + # then be mapped using NinjaRuleMapping. + env[NINJA_RULES] = {} + env.AddMethod(register_custom_rule, "NinjaRule") + + # Provides a mechanism for inject custom Ninja pools which can + # be used by providing the NINJA_POOL="name" as an + # OverrideEnvironment variable in a builder call. + env[NINJA_POOLS] = {} + env.AddMethod(register_custom_pool, "NinjaPool") + + # Add the ability to register custom NinjaRuleMappings for Command + # builders. We don't store this dictionary in the env to prevent + # accidental deletion of the CC/XXCOM mappings. You can still + # overwrite them if you really want to but you have to explicit + # about it this way. The reason is that if they were accidentally + # deleted you would get a very subtly incorrect Ninja file and + # might not catch it. + env.AddMethod(register_custom_rule_mapping, "NinjaRuleMapping") + + # on windows we need to change the link action + ninja_hack_linkcom(env) + + # Normally in SCons actions for the Program and *Library builders + # will return "${*COM}" as their pre-subst'd command line. However + # if a user in a SConscript overwrites those values via key access + # like env["LINKCOM"] = "$( $ICERUN $)" + env["LINKCOM"] then + # those actions no longer return the "bracketted" string and + # instead return something that looks more expanded. So to + # continue working even if a user has done this we map both the + # "bracketted" and semi-expanded versions. + def robust_rule_mapping(var, rule, tool): + provider = gen_get_response_file_command(env, rule, tool) + env.NinjaRuleMapping("${" + var + "}", provider) + env.NinjaRuleMapping(env.get(var, None), provider) + + robust_rule_mapping("CCCOM", "CC", "$CC") + robust_rule_mapping("SHCCCOM", "CC", "$CC") + robust_rule_mapping("CXXCOM", "CXX", "$CXX") + robust_rule_mapping("SHCXXCOM", "CXX", "$CXX") + robust_rule_mapping("LINKCOM", "LINK", "$LINK") + robust_rule_mapping("SHLINKCOM", "LINK", "$SHLINK") + robust_rule_mapping("ARCOM", "AR", "$AR") + + # Make SCons node walk faster by preventing unnecessary work + env.Decider("timestamp-match") + + # Used to determine if a build generates a source file. Ninja + # requires that all generated sources are added as order_only + # dependencies to any builds that *might* use them. + # TODO: switch to using SCons to help determine this (Github Issue #3624) + env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] + + # Force ARCOM so use 's' flag on ar instead of separately running ranlib + ninja_hack_arcom(env) + + if GetOption('disable_ninja'): + return env + + SCons.Warnings.SConsWarning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") + + # This is the point of no return, anything after this comment + # makes changes to SCons that are irreversible and incompatible + # with a normal SCons build. We return early if __NINJA_NO=1 has + # been given on the command line (i.e. by us in the generated + # ninja file) here to prevent these modifications from happening + # when we want SCons to do work. Everything before this was + # necessary to setup the builder and other functions so that the + # tool can be unconditionally used in the users's SCons files. + + if not exists(env): + return + + # Set a known variable that other tools can query so they can + # behave correctly during ninja generation. + env["GENERATING_NINJA"] = True + + # These methods are no-op'd because they do not work during ninja + # generation, expected to do no work, or simply fail. All of which + # are slow in SCons. So we overwrite them with no logic. + SCons.Node.FS.File.make_ready = ninja_noop + SCons.Node.FS.File.prepare = ninja_noop + SCons.Node.FS.File.push_to_cache = ninja_noop + SCons.Executor.Executor.prepare = ninja_noop + SCons.Taskmaster.Task.prepare = ninja_noop + SCons.Node.FS.File.built = ninja_noop + SCons.Node.Node.visited = ninja_noop + + # We make lstat a no-op because it is only used for SONAME + # symlinks which we're not producing. + SCons.Node.FS.LocalFS.lstat = ninja_noop + + # This is a slow method that isn't memoized. We make it a noop + # since during our generation we will never use the results of + # this or change the results. + SCons.Node.FS.is_up_to_date = ninja_noop + + # We overwrite stat and WhereIs with eternally memoized + # implementations. See the docstring of ninja_stat and + # ninja_whereis for detailed explanations. + SCons.Node.FS.LocalFS.stat = ninja_stat + SCons.Util.WhereIs = ninja_whereis + + # Monkey patch get_csig and get_contents for some classes. It + # slows down the build significantly and we don't need contents or + # content signatures calculated when generating a ninja file since + # we're not doing any SCons caching or building. + SCons.Executor.Executor.get_contents = ninja_contents( + SCons.Executor.Executor.get_contents + ) + SCons.Node.Alias.Alias.get_contents = ninja_contents( + SCons.Node.Alias.Alias.get_contents + ) + SCons.Node.FS.File.get_contents = ninja_contents(SCons.Node.FS.File.get_contents) + SCons.Node.FS.File.get_csig = ninja_csig(SCons.Node.FS.File.get_csig) + SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) + SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig) + + # Ignore CHANGED_SOURCES and CHANGED_TARGETS. We don't want those + # to have effect in a generation pass because the generator + # shouldn't generate differently depending on the current local + # state. Without this, when generating on Windows, if you already + # had a foo.obj, you would omit foo.cpp from the response file. Do the same for UNCHANGED. + SCons.Executor.Executor._get_changed_sources = SCons.Executor.Executor._get_sources + SCons.Executor.Executor._get_changed_targets = SCons.Executor.Executor._get_targets + SCons.Executor.Executor._get_unchanged_sources = SCons.Executor.Executor._get_sources + SCons.Executor.Executor._get_unchanged_targets = SCons.Executor.Executor._get_targets + + # Replace false action messages with nothing. + env["PRINT_CMD_LINE_FUNC"] = ninja_print_conf_log + + # This reduces unnecessary subst_list calls to add the compiler to + # the implicit dependencies of targets. Since we encode full paths + # in our generated commands we do not need these slow subst calls + # as executing the command will fail if the file is not found + # where we expect it. + env["IMPLICIT_COMMAND_DEPENDENCIES"] = False + + # This makes SCons more aggressively cache MD5 signatures in the + # SConsign file. + # TODO: WPD shouldn't this be set to 0? + env.SetOption("max_drift", 1) + + # The Serial job class is SIGNIFICANTLY (almost twice as) faster + # than the Parallel job class for generating Ninja files. So we + # monkey the Jobs constructor to only use the Serial Job class. + SCons.Job.Jobs.__init__ = ninja_always_serial + + ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') + + if NINJA_STATE is None: + NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) + + + # TODO: this is hacking into scons, preferable if there were a less intrusive way + # We will subvert the normal builder execute to make sure all the ninja file is dependent + # on all targets generated from any builders + SCons_Builder_BuilderBase__execute = SCons.Builder.BuilderBase._execute + + def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): + # this ensures all environments in which a builder executes from will + # not create list actions for linking on windows + ninja_hack_linkcom(env) + targets = SCons_Builder_BuilderBase__execute(self, env, target, source, overwarn=overwarn, executor_kw=executor_kw) + + if not SCons.Util.is_List(target): + target = [target] + + for target in targets: + if target.check_attributes('ninja_file') is None and not target.is_conftest(): + env.Depends(ninja_file, targets) + return targets + SCons.Builder.BuilderBase._execute = NinjaBuilderExecute + + # Here we monkey patch the Task.execute method to not do a bunch of + # unnecessary work. If a build is a regular builder (i.e not a conftest and + # not our own Ninja builder) then we add it to the NINJA_STATE. Otherwise we + # build it like normal. This skips all of the caching work that this method + # would normally do since we aren't pulling any of these targets from the + # cache. + # + # In the future we may be able to use this to actually cache the build.ninja + # file once we have the upstream support for referencing SConscripts as File + # nodes. + def ninja_execute(self): + + target = self.targets[0] + if target.get_env().get('NINJA_SKIP'): + return + if target.check_attributes('ninja_file') is None or not target.is_conftest: + NINJA_STATE.add_build(target) + else: + target.build() + + SCons.Taskmaster.Task.execute = ninja_execute + + # Make needs_execute always return true instead of determining out of + # date-ness. + SCons.Script.Main.BuildTask.needs_execute = lambda x: True + + # We will eventually need to overwrite TempFileMunge to make it + # handle persistent tempfiles or get an upstreamed change to add + # some configurability to it's behavior in regards to tempfiles. + # + # Set all three environment variables that Python's + # tempfile.mkstemp looks at as it behaves differently on different + # platforms and versions of Python. + # build_dir = env.subst("$NINJA_DIR") + # if build_dir == "": + # build_dir = "." + # os.environ["TMPDIR"] = env.Dir("{}/.response_files".format(build_dir)).get_abspath() + # os.environ["TEMP"] = os.environ["TMPDIR"] + # os.environ["TMP"] = os.environ["TMPDIR"] + # if not os.path.isdir(os.environ["TMPDIR"]): + # env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) + + env['TEMPFILEDIR'] = "$NINJA_DIR/.response_files" + env["TEMPFILE"] = NinjaNoResponseFiles diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml new file mode 100644 index 0000000000..0ec1708917 --- /dev/null +++ b/SCons/Tool/ninja/ninja.xml @@ -0,0 +1,325 @@ + + + + + %scons; + + %builders-mod; + + %functions-mod; + + %tools-mod; + + %variables-mod; + ]> + + + + + + + Sets up &b-link-Ninja; builder which generates a ninja build file, and then optionally runs &ninja;. + + + This is an experimental feature. + + This functionality is subject to change and/or removal without deprecation cycle. + + + + + NINJA_DISABLE_AUTO_RUN + NINJA_ALIAS_NAME + NINJA_DIR + NINJA_COMPDB_EXPAND + NINJA_ENV_VAR_CACHE + NINJA_FILE_NAME + NINJA_GENERATED_SOURCE_SUFFIXES + NINJA_MSVC_DEPS_PREFIX + NINJA_POOL + NINJA_REGENERATE_DEPS + NINJA_SYNTAX + _NINJA_REGENERATE_DEPS_FUNC + __NINJA_NO + IMPLICIT_COMMAND_DEPENDENCIES + + + + + + + + + + + + + AR + ARCOM + ARFLAGS + CC + CCCOM + CCFLAGS + CXX + CXXCOM + ESCAPE + LINK + LINKCOM + PLATFORM + RANLIB + RANLIBCOM + SHCCCOM + SHCXXCOM + SHLINK + SHLINKCOM + PROGSUFFIX + PRINT_CMD_LINE_FUNC + + + + + + + + + + + &b-Ninja; is a special builder which + adds a target to create a ninja build file. + The builder does not require any source files to be specified. + + + This is an experimental feature. To enable it you must use one of the following methods + + + + # On the command line + --experimental=ninja + + # Or in your SConstruct + SetOption('experimental', 'ninja') + + + This functionality is subject to change and/or removal without deprecation cycle. + + + + To use this tool you must install pypi's ninja + package. + This can be done via + pip install ninja + + + + + + If called with no arguments, + the builder will default to a target name of + ninja.build. + + + If called with a single positional argument, + &scons; will "deduce" the target name from that source + argument, giving it the same name, and then + ignore the source. + This is the usual way to call the builder if a + non-default target name is wanted. + + + If called with either the + target= + or source= keyword arguments, + the value of the argument is taken as the target name. + If called with both, the + target= + value is used and source= is ignored. + If called with multiple sources, + the source list will be ignored, + since there is no way to deduce what the intent was; + in this case the default target name will be used. + + + Available since &scons; 4.2. + + + + + + + + + The list of source file suffixes which are generated by SCons build steps. + All source files which match these suffixes will be added to the _generated_sources alias in the output + ninja.build file. + Then all other source files will be made to depend on this in the ninja.build file, forcing the + generated sources to be built first. + + + + + + + + This propagates directly into the generated ninja.build file. + From Ninja's docs + defines the string which should be stripped from msvc’s /showIncludes output + + + + + + + + This propagates directly into the generated ninja.build file. + From Ninja's docs: +
+ + builddir + A directory for some Ninja output files. ... (You can also store other build output in this + directory.) + +
+ + + + + + + + A generator function used to create a ninja depsfile which includes all the files which would require + SCons to be invoked if they change. + Or a list of said files. + + + + + + + + Boolean value (True|False) to instruct ninja to expand the command line arguments normally put into + response files. + This prevents lines in the compilation database like gcc @rsp_file and instead yields + gcc -c -o myfile.o myfile.c -Ia -DXYZ + + + Ninja's compdb tool added the -x flag in Ninja V1.9.0 + + + + + + + + A string that sets the environment for any environment variables that + differ between the OS environment and the SCons command ENV. + + It will be compatible with the default shell of the operating system. + + If not explicitly specified, SCons will generate this dynamically from the Environment()'s 'ENV' + env['ENV'] + where those values differ from the existing shell.. + + + + + + + + Set the ninja_pool for this or all targets in scope for this env var. + + + + + + + + Boolean (True|False). Default: False + When True, SCons will not run ninja automatically after creating the ninja.build file. + + + + + + + + + Internal flag. Used to tell SCons whether or not to try to import pypi's ninja python package. + This is set to True when being called by Ninja? + + + + + + + + + The filename for the generated Ninja build file defaults to + ninja.build + + + + + + + + Name of the Alias() which is will cause SCons to create the ninja.build file, and + then (optionally) run ninja. + + + + + + + + Theres also NINJA_SYNTAX which is the path to a custom ninja_syntax.py file which is used in generation. + The tool currently assumes you have ninja installed through pip, and grabs the syntax file from that + installation if none specified. + + + + + + + + Internal value used to specify the function to call with argument env to generate the list of files + which if changed would require the ninja file to be regenerated. + + + + + + diff --git a/SCons/__init__.py b/SCons/__init__.py index bc0f428f53..cd521d1f1c 100644 --- a/SCons/__init__.py +++ b/SCons/__init__.py @@ -1,9 +1,9 @@ __version__="4.1.1a" __copyright__="Copyright (c) 2001 - 2021 The SCons Foundation" __developer__="bdbaddog" -__date__="2021-04-18 22:22:37" +__date__="2021-05-30 21:33:41" __buildsys__="ProDog2020" -__revision__="1850bdd13c7668aac493b9f5b896578813e60ca9" -__build__="1850bdd13c7668aac493b9f5b896578813e60ca9" +__revision__="b4c7a0966c36d37d0c2ea125dd98bff0061e28f2" +__build__="b4c7a0966c36d37d0c2ea125dd98bff0061e28f2" # make sure compatibility is always in place import SCons.compat # noqa \ No newline at end of file diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py index baffbdcc20..a46505785c 100644 --- a/bin/SConsDoc.py +++ b/bin/SConsDoc.py @@ -364,9 +364,18 @@ def validateXml(fpath, xmlschema_context): doc.xinclude() try: TreeFactory.xmlschema.assertValid(doc) + except etree.XMLSchemaValidateError as e: + print("ERROR: %s fails to validate:" % fpath) + print(e) + print(e.error_log.last_error.message) + print("In file: [%s]" % e.error_log.last_error.filename) + print("Line : %d" % e.error_log.last_error.line) + return False + except Exception as e: print("ERROR: %s fails to validate:" % fpath) print(e) + return False return True diff --git a/doc/generated/builders.gen b/doc/generated/builders.gen index 14508ea96f..6499ef38b4 100644 --- a/doc/generated/builders.gen +++ b/doc/generated/builders.gen @@ -1221,6 +1221,69 @@ env.MSVSSolution( + + Ninja() + env.Ninja() + + &b-Ninja; is a special builder which + adds a target to create a ninja build file. + The builder does not require any source files to be specified. + + + This is an experimental feature. To enable it you must use one of the following methods + + + + # On the command line + --experimental=ninja + + # Or in your SConstruct + SetOption('experimental', 'ninja') + + + This functionality is subject to change and/or removal without deprecation cycle. + + + + To use this tool you must install pypi's ninja + package. + This can be done via + pip install ninja + + + + + + If called with no arguments, + the builder will default to a target name of + ninja.build. + + + If called with a single positional argument, + &scons; will "deduce" the target name from that source + argument, giving it the same name, and then + ignore the source. + This is the usual way to call the builder if a + non-default target name is wanted. + + + If called with either the + target= + or source= keyword arguments, + the value of the argument is taken as the target name. + If called with both, the + target= + value is used and source= is ignored. + If called with multiple sources, + the source list will be ignored, + since there is no way to deduce what the intent was; + in this case the default target name will be used. + + + Available since &scons; 4.2. + + + Object() env.Object() diff --git a/doc/generated/builders.mod b/doc/generated/builders.mod index 4835118f69..3542b99110 100644 --- a/doc/generated/builders.mod +++ b/doc/generated/builders.mod @@ -37,6 +37,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. MOFiles"> MSVSProject"> MSVSSolution"> +Ninja"> Object"> Package"> PCH"> @@ -94,6 +95,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. env.MOFiles"> env.MSVSProject"> env.MSVSSolution"> +env.Ninja"> env.Object"> env.Package"> env.PCH"> @@ -157,6 +159,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. MOFiles"> MSVSProject"> MSVSSolution"> +Ninja"> Object"> Package"> PCH"> @@ -214,6 +217,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. env.MOFiles"> env.MSVSProject"> env.MSVSSolution"> +env.Ninja"> env.Object"> env.Package"> env.PCH"> diff --git a/doc/generated/examples/caching_ex-random_1.xml b/doc/generated/examples/caching_ex-random_1.xml index a5593d302d..2990d7477d 100644 --- a/doc/generated/examples/caching_ex-random_1.xml +++ b/doc/generated/examples/caching_ex-random_1.xml @@ -1,8 +1,8 @@ % scons -Q -cc -o f2.o -c f2.c cc -o f1.o -c f1.c cc -o f3.o -c f3.c -cc -o f4.o -c f4.c cc -o f5.o -c f5.c +cc -o f4.o -c f4.c +cc -o f2.o -c f2.c cc -o prog f1.o f2.o f3.o f4.o f5.o diff --git a/doc/generated/examples/troubleshoot_Dump_1.xml b/doc/generated/examples/troubleshoot_Dump_1.xml index 8f4e624d94..2b4178bd8d 100644 --- a/doc/generated/examples/troubleshoot_Dump_1.xml +++ b/doc/generated/examples/troubleshoot_Dump_1.xml @@ -62,8 +62,8 @@ scons: Reading SConscript files ... 'TEMPFILEARGJOIN': ' ', 'TEMPFILEPREFIX': '@', 'TOOLS': ['install', 'install'], - '_CPPDEFFLAGS': '${_defines(CPPDEFPREFIX, CPPDEFINES, CPPDEFSUFFIX, ' - '__env__)}', + '_CPPDEFFLAGS': '${_defines(CPPDEFPREFIX, CPPDEFINES, CPPDEFSUFFIX, __env__, ' + 'TARGET, SOURCE)}', '_CPPINCFLAGS': '$( ${_concat(INCPREFIX, CPPPATH, INCSUFFIX, __env__, RDirs, ' 'TARGET, SOURCE)} $)', '_LIBDIRFLAGS': '$( ${_concat(LIBDIRPREFIX, LIBPATH, LIBDIRSUFFIX, __env__, ' diff --git a/doc/generated/examples/troubleshoot_Dump_2.xml b/doc/generated/examples/troubleshoot_Dump_2.xml index f2fc5e6f6f..c2fced13c1 100644 --- a/doc/generated/examples/troubleshoot_Dump_2.xml +++ b/doc/generated/examples/troubleshoot_Dump_2.xml @@ -107,8 +107,8 @@ scons: Reading SConscript files ... 'TOOLS': ['msvc', 'install', 'install'], 'VSWHERE': None, '_CCCOMCOM': '$CPPFLAGS $_CPPDEFFLAGS $_CPPINCFLAGS $CCPCHFLAGS $CCPDBFLAGS', - '_CPPDEFFLAGS': '${_defines(CPPDEFPREFIX, CPPDEFINES, CPPDEFSUFFIX, ' - '__env__)}', + '_CPPDEFFLAGS': '${_defines(CPPDEFPREFIX, CPPDEFINES, CPPDEFSUFFIX, __env__, ' + 'TARGET, SOURCE)}', '_CPPINCFLAGS': '$( ${_concat(INCPREFIX, CPPPATH, INCSUFFIX, __env__, RDirs, ' 'TARGET, SOURCE)} $)', '_LIBDIRFLAGS': '$( ${_concat(LIBDIRPREFIX, LIBPATH, LIBDIRSUFFIX, __env__, ' diff --git a/doc/generated/examples/troubleshoot_explain1_3.xml b/doc/generated/examples/troubleshoot_explain1_3.xml index 6b90e1cabe..14a105e048 100644 --- a/doc/generated/examples/troubleshoot_explain1_3.xml +++ b/doc/generated/examples/troubleshoot_explain1_3.xml @@ -2,5 +2,5 @@ cp file.in file.oout scons: warning: Cannot find target file.out after building -File "/Users/bdbaddog/devel/scons/git/as_scons/scripts/scons.py", line 96, in <module> +File "/Users/bdbaddog/devel/scons/git/scons-bugfixes-4/scripts/scons.py", line 96, in <module> diff --git a/doc/generated/examples/troubleshoot_stacktrace_2.xml b/doc/generated/examples/troubleshoot_stacktrace_2.xml index 8e8809c461..75dd6fba03 100644 --- a/doc/generated/examples/troubleshoot_stacktrace_2.xml +++ b/doc/generated/examples/troubleshoot_stacktrace_2.xml @@ -3,10 +3,10 @@ scons: *** [prog.o] Source `prog.c' not found, needed by target `prog.o'. scons: internal stack trace: File "SCons/Job.py", line 195, in start task.prepare() - File "SCons/Script/Main.py", line 179, in prepare + File "SCons/Script/Main.py", line 178, in prepare return SCons.Taskmaster.OutOfDateTask.prepare(self) File "SCons/Taskmaster.py", line 186, in prepare executor.prepare() - File "SCons/Executor.py", line 424, in prepare + File "SCons/Executor.py", line 418, in prepare raise SCons.Errors.StopError(msg % (s, self.batches[0].targets[0])) diff --git a/doc/generated/functions.gen b/doc/generated/functions.gen index a43ed1a203..5554802194 100644 --- a/doc/generated/functions.gen +++ b/doc/generated/functions.gen @@ -2124,261 +2124,242 @@ file is found. env.GetOption(name) This function provides a way to query the value of -SCons options set on scons command line -(or set using the -&f-link-SetOption; -function). -The options supported are: +options which can be set via the command line or using the +&f-link-SetOption; function. - - - - -cache_debug - - -which corresponds to ; - - - - -cache_disable - - -which corresponds to ; - - - - -cache_force - - -which corresponds to ; - - - - -cache_show - - -which corresponds to ; - - - - -clean - - -which corresponds to , -and ; - - - - -config - - -which corresponds to ; - - - - -directory - - -which corresponds to and ; - - - - -diskcheck - - -which corresponds to ; - - - - -duplicate - - -which corresponds to ; - - - - -file - - -which corresponds to , , and ; - - - - -help - - -which corresponds to and ; - - - - -ignore_errors - - -which corresponds to ; - - - - -implicit_cache - - -which corresponds to ; - - - - -implicit_deps_changed - - -which corresponds to ; - - - - -implicit_deps_unchanged - - -which corresponds to ; - - - - -interactive - - -which corresponds to and ; - - - - -keep_going - - -which corresponds to and ; - - - - -max_drift - - -which corresponds to ; - - - - -no_exec - - -which corresponds to , -, , - and ; - - - - -no_site_dir - - -which corresponds to ; - - - - -num_jobs - - -which corresponds to and ; - - - - -profile_file - - -which corresponds to ; - - - - -question - - -which corresponds to and ; - - - - -random - - -which corresponds to ; - - - - -repository - - -which corresponds to , and ; - - - - -silent - - -which corresponds to , and ; - - - - -site_dir - - -which corresponds to ; - - - - -stack_size - - -which corresponds to ; - - - - -taskmastertrace_file - -which corresponds to ; and - - - - -warn - - -which corresponds to and . - - - - +name can be an entry from the following table, +which shows the corresponding command line arguments +that could affect the value. +name can be also be the destination +variable name from a project-specific option added using the +&f-link-AddOption; function, as long as the addition +happens prior to the &f-GetOption; call in the SConscript files. + + + + + Query name + Command-line options + Notes + + + + + + cache_debug + + + + cache_disable + + , + + + + + cache_force + + , + + + + + cache_readonly + + + + cache_show + + + + clean + + , + , + + + + + climb_up + + + + + + + + + + config + + + + debug + + + + directory + , + + + diskcheck + + + + duplicate + + + + enable_virtualenv + + + + experimental + + since 4.2 + + + file + + , + , + , + + + + + hash_format + + since 4.2 + + + help + , + + + ignore_errors + , + + + ignore_virtualenv + + + + implicit_cache + + + + implicit_deps_changed + + + + implicit_deps_unchanged + + + + include_dir + , + + + install_sandbox + + Available only if the &t-link-install; tool has been called + + + keep_going + , + + + max_drift + + + + md5_chunksize + + , + + + since 4.2 + + + no_exec + + , + , + , + , + + + + + no_progress + + + + num_jobs + , + + + package_type + + Available only if the &t-link-packaging; tool has been called + + + profile_file + + + + question + , + + + random + + + + repository + + , + , + + + + + silent + + , + , + + + + + site_dir + , + + + stack_size + + + + taskmastertrace_file + + + + tree_printers + + + + warn + , + + + + + + See the documentation for the corresponding command line option for information about each specific @@ -3879,15 +3860,33 @@ if 'FOO' not in env: Sets &scons; option variable name to value. These options are all also settable via -&scons; command-line options but the variable name -may differ from the command-line option name (see table). +command-line options but the variable name +may differ from the command-line option name - +see the table for correspondences. A value set via command-line option will take precedence over one set with &f-SetOption;, which allows setting a project default in the scripts and temporarily overriding it via command line. +&f-SetOption; calls can also be placed in the +site_init.py file. + + + +See the documentation in the manpage for the +corresponding command line option for information about each specific option. +The value parameter is mandatory, +for option values which are boolean in nature +(that is, the command line option does not take an argument) +use a value +which evaluates to true (e.g. True, +1) or false (e.g. False, +0). + + + Options which affect the reading and processing of SConscript files -are not settable this way, since those files must -be read in order to find the &f-SetOption; call. +are not settable using &f-SetOption; since those files must +be read in order to find the &f-SetOption; call in the first place. @@ -3895,126 +3894,161 @@ The settable variables with their associated command-line options are: - + -VariableCommand-line options + + Settable name + Command-line options + Notes + + - -clean - -, , - - -diskcheck - - - - - -duplicate - - - - - - - - experimental - - - - - - - - -help - -, - - -implicit_cache - - - - - -max_drift - - - - -md5_chunksize - - - - -no_exec - -, , -, , - - - -no_progress - - - - -num_jobs - -, - - -random - - - - -silent - -. - - -stack_size - - - - -warn - -. - + + clean + + , + , + + + + + + diskcheck + + + + + duplicate + + + + + experimental + + since 4.2 + + + + hash_chunksize + + + Actually sets md5_chunksize. + since 4.2 + + + + + hash_format + + since 4.2 + + + + help + , + + + + implicit_cache + + + + + implicit_deps_changed + + + Also sets implicit_cache. + (settable since 4.2) + + + + + implicit_deps_unchanged + + + Also sets implicit_cache. + (settable since 4.2) + + + + + max_drift + + + + + md5_chunksize + + + + + no_exec + + , + , + , + , + + + + + + no_progress + + See + + If no_progress is set via &f-SetOption; + in an SConscript file + (but not if set in a site_init.py file) + there will still be an initial status message about + reading SConscript files since &SCons; has + to start reading them before it can see the + &f-SetOption;. + + + + + + + num_jobs + , + + + + random + + + + + silent + + , + , + + + + + + stack_size + + + + + warn + + + - -See the documentation in the manpage for the -corresponding command line option for information about each specific option. -Option values which are boolean in nature (that is, they are -either on or off) should be set to a true value (True, -1) or a false value (False, -0). - - - - -If no_progress is set via &f-SetOption; -there will still be initial progress output as &SCons; has -to start reading SConscript files before it can see the -&f-SetOption; in an SConscript file: -scons: Reading SConscript files ... - - - Example: -SetOption('max_drift', True) +SetOption('max_drift', 0) diff --git a/doc/generated/tools.gen b/doc/generated/tools.gen index dc15b6e5bc..16d13e3cfa 100644 --- a/doc/generated/tools.gen +++ b/doc/generated/tools.gen @@ -811,6 +811,19 @@ Sets construction variables for the Sets: &cv-link-AS;, &cv-link-ASCOM;, &cv-link-ASFLAGS;, &cv-link-ASPPCOM;, &cv-link-ASPPFLAGS;.Uses: &cv-link-ASCOMSTR;, &cv-link-ASPPCOMSTR;. + + ninja + + Sets up &b-link-Ninja; builder which generates a ninja build file, and then optionally runs &ninja;. + + + This is an experimental feature. + + This functionality is subject to change and/or removal without deprecation cycle. + + + Sets: &cv-link-IMPLICIT_COMMAND_DEPENDENCIES;, &cv-link-NINJA_ALIAS_NAME;, &cv-link-NINJA_COMPDB_EXPAND;, &cv-link-NINJA_DIR;, &cv-link-NINJA_DISABLE_AUTO_RUN;, &cv-link-NINJA_ENV_VAR_CACHE;, &cv-link-NINJA_FILE_NAME;, &cv-link-NINJA_GENERATED_SOURCE_SUFFIXES;, &cv-link-NINJA_MSVC_DEPS_PREFIX;, &cv-link-NINJA_POOL;, &cv-link-NINJA_REGENERATE_DEPS;, &cv-link-NINJA_SYNTAX;, &cv-link-_NINJA_REGENERATE_DEPS_FUNC;, &cv-link-__NINJA_NO;.Uses: &cv-link-AR;, &cv-link-ARCOM;, &cv-link-ARFLAGS;, &cv-link-CC;, &cv-link-CCCOM;, &cv-link-CCFLAGS;, &cv-link-CXX;, &cv-link-CXXCOM;, &cv-link-ESCAPE;, &cv-link-LINK;, &cv-link-LINKCOM;, &cv-link-PLATFORM;, &cv-link-PRINT_CMD_LINE_FUNC;, &cv-link-PROGSUFFIX;, &cv-link-RANLIB;, &cv-link-RANLIBCOM;, &cv-link-SHCCCOM;, &cv-link-SHCXXCOM;, &cv-link-SHLINK;, &cv-link-SHLINKCOM;. + packaging diff --git a/doc/generated/tools.mod b/doc/generated/tools.mod index 78aa9ef982..35eea5e2b4 100644 --- a/doc/generated/tools.mod +++ b/doc/generated/tools.mod @@ -79,6 +79,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. mwcc"> mwld"> nasm"> +ninja"> packaging"> pdf"> pdflatex"> @@ -184,6 +185,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. mwcc"> mwld"> nasm"> +ninja"> packaging"> pdf"> pdflatex"> diff --git a/doc/generated/variables.gen b/doc/generated/variables.gen index a24fd34ae9..c56217be7b 100644 --- a/doc/generated/variables.gen +++ b/doc/generated/variables.gen @@ -22,6 +22,16 @@ if &cv-link-LDMODULEVERSION; is set. Othervise it evaluates to an empty string. + + + __NINJA_NO + + + Internal flag. Used to tell SCons whether or not to try to import pypi's ninja python package. + This is set to True when being called by Ninja? + + + __SHLIBVERSIONFLAGS @@ -5156,6 +5166,150 @@ Specfies the name of the project to package. See the &b-link-Package; builder. + + + NINJA_ALIAS_NAME + + + Name of the Alias() which is will cause SCons to create the ninja.build file, and + then (optionally) run ninja. + + + + + + NINJA_COMPDB_EXPAND + + + Boolean value (True|False) to instruct ninja to expand the command line arguments normally put into + response files. + This prevents lines in the compilation database like gcc @rsp_file and instead yields + gcc -c -o myfile.o myfile.c -Ia -DXYZ + + + Ninja's compdb tool added the -x flag in Ninja V1.9.0 + + + + + + NINJA_DIR + + + This propagates directly into the generated ninja.build file. + From Ninja's docs: +
+ + builddir + A directory for some Ninja output files. ... (You can also store other build output in this + directory.) + +
+ + + + + + NINJA_DISABLE_AUTO_RUN + + + Boolean (True|False). Default: False + When True, SCons will not run ninja automatically after creating the ninja.build file. + + + + + + + NINJA_ENV_VAR_CACHE + + + A string that sets the environment for any environment variables that + differ between the OS environment and the SCons command ENV. + + It will be compatible with the default shell of the operating system. + + If not explicitly specified, SCons will generate this dynamically from the Environment()'s 'ENV' + env['ENV'] + where those values differ from the existing shell.. + + + + + + NINJA_FILE_NAME + + + The filename for the generated Ninja build file defaults to + ninja.build + + + + + + NINJA_GENERATED_SOURCE_SUFFIXES + + + The list of source file suffixes which are generated by SCons build steps. + All source files which match these suffixes will be added to the _generated_sources alias in the output + ninja.build file. + Then all other source files will be made to depend on this in the ninja.build file, forcing the + generated sources to be built first. + + + + + + NINJA_MSVC_DEPS_PREFIX + + + This propagates directly into the generated ninja.build file. + From Ninja's docs + defines the string which should be stripped from msvc’s /showIncludes output + + + + + + NINJA_POOL + + + Set the ninja_pool for this or all targets in scope for this env var. + + + + + + NINJA_REGENERATE_DEPS + + + A generator function used to create a ninja depsfile which includes all the files which would require + SCons to be invoked if they change. + Or a list of said files. + + + + + + _NINJA_REGENERATE_DEPS_FUNC + + + Internal value used to specify the function to call with argument env to generate the list of files + which if changed would require the ninja file to be regenerated. + + + + + + NINJA_SYNTAX + + + Theres also NINJA_SYNTAX which is the path to a custom ninja_syntax.py file which is used in generation. + The tool currently assumes you have ninja installed through pip, and grabs the syntax file from that + installation if none specified. + + + no_import_lib diff --git a/doc/generated/variables.mod b/doc/generated/variables.mod index a3dbc0d8f2..08aa21dbf3 100644 --- a/doc/generated/variables.mod +++ b/doc/generated/variables.mod @@ -9,6 +9,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. --> $__LDMODULEVERSIONFLAGS"> +$__NINJA_NO"> $__SHLIBVERSIONFLAGS"> $APPLELINK_COMPATIBILITY_VERSION"> $_APPLELINK_COMPATIBILITY_VERSION"> @@ -349,6 +350,18 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $MWCW_VERSION"> $MWCW_VERSIONS"> $NAME"> +$NINJA_ALIAS_NAME"> +$NINJA_COMPDB_EXPAND"> +$NINJA_DIR"> +$NINJA_DISABLE_AUTO_RUN"> +$NINJA_ENV_VAR_CACHE"> +$NINJA_FILE_NAME"> +$NINJA_GENERATED_SOURCE_SUFFIXES"> +$NINJA_MSVC_DEPS_PREFIX"> +$NINJA_POOL"> +$NINJA_REGENERATE_DEPS"> +$_NINJA_REGENERATE_DEPS_FUNC"> +$NINJA_SYNTAX"> $no_import_lib"> $OBJPREFIX"> $OBJSUFFIX"> @@ -649,6 +662,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. --> $__LDMODULEVERSIONFLAGS"> +$__NINJA_NO"> $__SHLIBVERSIONFLAGS"> $APPLELINK_COMPATIBILITY_VERSION"> $_APPLELINK_COMPATIBILITY_VERSION"> @@ -989,6 +1003,18 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $MWCW_VERSION"> $MWCW_VERSIONS"> $NAME"> +$NINJA_ALIAS_NAME"> +$NINJA_COMPDB_EXPAND"> +$NINJA_DIR"> +$NINJA_DISABLE_AUTO_RUN"> +$NINJA_ENV_VAR_CACHE"> +$NINJA_FILE_NAME"> +$NINJA_GENERATED_SOURCE_SUFFIXES"> +$NINJA_MSVC_DEPS_PREFIX"> +$NINJA_POOL"> +$NINJA_REGENERATE_DEPS"> +$_NINJA_REGENERATE_DEPS_FUNC"> +$NINJA_SYNTAX"> $no_import_lib"> $OBJPREFIX"> $OBJSUFFIX"> diff --git a/doc/scons.mod b/doc/scons.mod index 57004549f6..72adf68240 100644 --- a/doc/scons.mod +++ b/doc/scons.mod @@ -57,6 +57,7 @@ m4"> Make"> Make++"> +ninja"> pdflatex"> pdftex"> Python"> diff --git a/doc/user/external.xml b/doc/user/external.xml index 9900e93688..5f88f5adbb 100644 --- a/doc/user/external.xml +++ b/doc/user/external.xml @@ -1,286 +1,359 @@ - %scons; + + %scons; - - %builders-mod; - - %functions-mod; - - %tools-mod; - - %variables-mod; + + %builders-mod; + + %functions-mod; + + %tools-mod; + + %variables-mod; -]> + ]> -Using SCons with other build tools + Using SCons with other build tools - + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - - Sometimes a project needs to interact with other projects - in various ways. For example, many open source projects - make use of components from other open source projects, - and want to use those in their released form, not recode - their builds into &SCons;. As another example, sometimes - the flexibility and power of &SCons; is useful for managing the - overall project, but developers might like faster incremental - builds when making small changes by using a different tool. - - - - - - This chapter shows some techniques for interacting with other - projects and tools effectively from within &SCons;. - - - -
- Creating a Compilation Database + --> - Tooling to perform analysis and modification - of source code often needs to know not only the source code - itself, but also how it will be compiled, as the compilation line - affects the behavior of macros, includes, etc. &SCons; has a - record of this information once it has run, in the form of - Actions associated with the sources, and can emit this information - so tools can use it. + Sometimes a project needs to interact with other projects + in various ways. For example, many open source projects + make use of components from other open source projects, + and want to use those in their released form, not recode + their builds into &SCons;. As another example, sometimes + the flexibility and power of &SCons; is useful for managing the + overall project, but developers might like faster incremental + builds when making small changes by using a different tool. - The Clang project has defined a JSON Compilation Database. - This database is in common use as input into Clang tools - and many IDEs and editors as well. - See - - JSON Compilation Database Format Specification - - for complete information. &SCons; can emit a - compilation database in this format - by enabling the &t-link-compilation_db; tool - and calling the &b-link-CompilationDatabase; builder - (available since &scons; 4.0). - - - - - - The compilation database can be populated with - source and output files either with paths relative - to the top of the build, or using absolute paths. - This is controlled by - COMPILATIONDB_USE_ABSPATH=(True|False) - which defaults to False. - The entries in this file can be filtered by using - - COMPILATIONDB_PATH_FILTER='pattern' - where the filter pattern is a string following the Python - - fnmatch - - syntax. - This filtering can be used for outputting different - build variants to different compilation database files. - - - - - - The following example illustrates generating a compilation - database containing absolute paths: - - - - - -env = Environment(COMPILATIONDB_USE_ABSPATH=True) -env.Tool('compilation_db') -env.CompilationDatabase() -env.Program('hello.c') - - -int main( int argc, char* argv[] ) -{ - return 0; -} - - - - - scons -Q - - - compile_commands.json contains: - - -[ - { - "command": "gcc -o hello.o -c hello.c", - "directory": "/home/user/sandbox", - "file": "/home/user/sandbox/hello.c", - "output": "/home/user/sandbox/hello.o" - } -] - - - - - Notice that the generated database contains only an entry for - the hello.c/hello.o pairing, - and nothing for the generation of the final executable - hello - the transformation of - hello.o to hello - does not have any information that affects interpretation - of the source code, - so it is not interesting to the compilation database. - - - - - - Although it can be a little surprising at first glance, - a compilation database target is, like any other target, - subject to &scons; target selection rules. - This means if you set a default target (that does not - include the compilation database), or use command-line - targets, it might not be selected for building. - This can actually be an advantage, since you don't - necessarily want to regenerate the compilation database - every build. - The following example - shows selecting relative paths (the default) - for output and source, - and also giving a non-default name to the database. - In order to be able to generate the database separately from building, - an alias is set referring to the database, - which can then be used as a target - here we are only - building the compilation database target, not the code. - - - - - -env = Environment() -env.Tool('compilation_db') -cdb = env.CompilationDatabase('compile_database.json') -Alias('cdb', cdb) -env.Program('test_main.c') - - -#include "test_main.h" -int main( int argc, char* argv[] ) -{ - return 0; -} - - -/* dummy include file */ - - - - - scons -Q cdb - - - compile_database.json contains: - - -[ - { - "command": "gcc -o test_main.o -c test_main.c", - "directory": "/home/user/sandbox", - "file": "test_main.c", - "output": "test_main.o" - } -] - - - - - The following (incomplete) example shows using filtering - to separate build variants. - In the case of using variants, - you want different compilation databases for each, - since the build parameters differ, so the code analysis - needs to see the correct build lines for the 32-bit build - and 64-bit build hinted at here. - For simplicity of presentation, - the example omits the setup details of the variant directories: + This chapter shows some techniques for interacting with other + projects and tools effectively from within &SCons;. - -env = Environment() -env.Tool('compilation_db') - -env1 = env.Clone() -env1['COMPILATIONDB_PATH_FILTER'] = 'build/linux32/*' -env1.CompilationDatabase('compile_commands-linux32.json') - -env2 = env.Clone() -env2['COMPILATIONDB_PATH_FILTER'] = 'build/linux64/*' -env2.CompilationDatabase('compile_commands-linux64.json') - - - compile_commands-linux32.json contains: - - -[ - { - "command": "gcc -m32 -o build/linux32/test_main.o -c test_main.c", - "directory": "/home/user/sandbox", - "file": "test_main.c", - "output": "build/linux32/test_main.o" - } -] - - - compile_commands-linux64.json contains: - - -[ - { - "command": "gcc -m64 -o build/linux64/test_main.o -c test_main.c", - "directory": "/home/user/sandbox", - "file": "test_main.c", - "output": "build/linux64/test_main.o" - } -] - - -
+
+ Creating a Compilation Database + + + + Tooling to perform analysis and modification + of source code often needs to know not only the source code + itself, but also how it will be compiled, as the compilation line + affects the behavior of macros, includes, etc. &SCons; has a + record of this information once it has run, in the form of + Actions associated with the sources, and can emit this information + so tools can use it. + + + + + + The Clang project has defined a JSON Compilation Database. + This database is in common use as input into Clang tools + and many IDEs and editors as well. + See + + JSON Compilation Database Format Specification + + for complete information. &SCons; can emit a + compilation database in this format + by enabling the &t-link-compilation_db; tool + and calling the &b-link-CompilationDatabase; builder + (available since &scons; 4.0). + + + + + + The compilation database can be populated with + source and output files either with paths relative + to the top of the build, or using absolute paths. + This is controlled by + COMPILATIONDB_USE_ABSPATH=(True|False) + which defaults to False. + The entries in this file can be filtered by using + + COMPILATIONDB_PATH_FILTER='pattern' + where the filter pattern is a string following the Python + + fnmatch + + syntax. + This filtering can be used for outputting different + build variants to different compilation database files. + + + + + + The following example illustrates generating a compilation + database containing absolute paths: + + + + + + env = Environment(COMPILATIONDB_USE_ABSPATH=True) + env.Tool('compilation_db') + env.CompilationDatabase() + env.Program('hello.c') + + + int main( int argc, char* argv[] ) + { + return 0; + } + + + + + scons -Q + + + + compile_commands.json + contains: + + + + [ + { + "command": "gcc -o hello.o -c hello.c", + "directory": "/home/user/sandbox", + "file": "/home/user/sandbox/hello.c", + "output": "/home/user/sandbox/hello.o" + } + ] + + + + + Notice that the generated database contains only an entry for + the hello.c/hello.o pairing, + and nothing for the generation of the final executable + hello + - the transformation of + hello.o + to + hello + does not have any information that affects interpretation + of the source code, + so it is not interesting to the compilation database. + + + + + + Although it can be a little surprising at first glance, + a compilation database target is, like any other target, + subject to &scons; target selection rules. + This means if you set a default target (that does not + include the compilation database), or use command-line + targets, it might not be selected for building. + This can actually be an advantage, since you don't + necessarily want to regenerate the compilation database + every build. + The following example + shows selecting relative paths (the default) + for output and source, + and also giving a non-default name to the database. + In order to be able to generate the database separately from building, + an alias is set referring to the database, + which can then be used as a target - here we are only + building the compilation database target, not the code. + + + + + + env = Environment() + env.Tool('compilation_db') + cdb = env.CompilationDatabase('compile_database.json') + Alias('cdb', cdb) + env.Program('test_main.c') + + + #include "test_main.h" + int main( int argc, char* argv[] ) + { + return 0; + } + + + /* dummy include file */ + + + + + scons -Q cdb + + + + compile_database.json + contains: + + + + [ + { + "command": "gcc -o test_main.o -c test_main.c", + "directory": "/home/user/sandbox", + "file": "test_main.c", + "output": "test_main.o" + } + ] + + + + + The following (incomplete) example shows using filtering + to separate build variants. + In the case of using variants, + you want different compilation databases for each, + since the build parameters differ, so the code analysis + needs to see the correct build lines for the 32-bit build + and 64-bit build hinted at here. + For simplicity of presentation, + the example omits the setup details of the variant directories: + + + + + env = Environment() + env.Tool('compilation_db') + + env1 = env.Clone() + env1['COMPILATIONDB_PATH_FILTER'] = 'build/linux32/*' + env1.CompilationDatabase('compile_commands-linux32.json') + + env2 = env.Clone() + env2['COMPILATIONDB_PATH_FILTER'] = 'build/linux64/*' + env2.CompilationDatabase('compile_commands-linux64.json') + + + + compile_commands-linux32.json + contains: + + + + [ + { + "command": "gcc -m32 -o build/linux32/test_main.o -c test_main.c", + "directory": "/home/user/sandbox", + "file": "test_main.c", + "output": "build/linux32/test_main.o" + } + ] + + + + compile_commands-linux64.json + contains: + + + + [ + { + "command": "gcc -m64 -o build/linux64/test_main.o -c test_main.c", + "directory": "/home/user/sandbox", + "file": "test_main.c", + "output": "build/linux64/test_main.o" + } + ] + + +
+ +
+ Ninja Build Generator + + + + This is an experimental new feature. It is subject to change and/or removal without depreciation cycle + + + To use this tool you must install pypi's ninja + package. + This can be done via + pip install ninja + + + To enable this feature you'll need to use one of the following + + + # On the command line + --experimental=ninja + + # Or in your SConstruct + SetOption('experimental', 'ninja') + + + + + + This tool will enabled creating a ninja build file from your SCons based build system. It can then invoke + ninja to run your build. For most builds ninja will be significantly faster, but you may have to give up + some accuracy. You are NOT advised to use this for production builds. It can however significantly speed up + your build/debug/compile iterations. + + + It's not expected that the ninja builder will work for all builds at this point. It's still under active + development. If you find that your build doesn't work with ninja please bring this to the users mailing list + or devel channel on our discord server. + + + Specifically if your build has many (or even any) python function actions you may find that the ninja build + will be slower as it will run ninja, which will then run SCons for each target created by a python action. + To alleviate some of these, especially those python based actions built into SCons there is special logic to + implement those actions via shell commands in the ninja build file. + + + + Ninja Build System + + + + + + Ninja File Format Specification + + +
diff --git a/requirements.txt b/requirements.txt index 520b90bb74..4093f19278 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ sphinx<3.5.0 sphinx_rtd_theme lxml==4.6.3 rst2pdf +ninja \ No newline at end of file diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py new file mode 100644 index 0000000000..eb8eb74590 --- /dev/null +++ b/test/ninja/build_libraries.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + + +import os + +import TestSCons +from TestCmd import IS_WINDOWS, IS_MACOS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find ninja module in python") + +ninja_binary = test.where_is('ninja') +if not ninja_binary: + test.skip_test("Could not find ninja. Skipping test.") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +if IS_WINDOWS: + lib_suffix = '.lib' + staticlib_suffix = '.lib' + lib_prefix = '' + win32 = ", 'WIN32'" +else: + lib_suffix = '.so' + staticlib_suffix = '.a' + lib_prefix = 'lib' + win32 = '' + +if IS_MACOS: + lib_suffix = '.dylib' + +test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) +env = Environment() +env.Tool('ninja') +env['NINJA'] = "%(ninja_bin)s" + +shared_lib = env.SharedLibrary(target = 'test_impl', source = 'test_impl.c', CPPDEFINES=['LIBRARY_BUILD'%(win32)s]) +env.Program(target = 'test', source = 'test1.c', LIBS=['test_impl'], LIBPATH=['.'], RPATH='.') + +static_obj = env.Object(target = 'test_impl_static', source = 'test_impl.c') +static_lib = env.StaticLibrary(target = 'test_impl_static', source = static_obj) +static_obj = env.Object(target = 'test_static', source = 'test1.c') +env.Program(target = 'test_static', source = static_obj, LIBS=[static_lib], LIBPATH=['.']) +""" % locals()) +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('test'), stdout="library_function") +test.run(program=test.workpath('test_static'), stdout="library_function") + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + ('Removed %stest_impl' % lib_prefix) + lib_suffix, + 'Removed test' + _exe, + ('Removed %stest_impl_static' % lib_prefix) + staticlib_suffix, + 'Removed test_static' + _exe, + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('test')) +test.must_not_exist(test.workpath('test_static')) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program=program, stdout=None) +test.run(program=test.workpath('test'), stdout="library_function") +test.run(program=test.workpath('test_static'), stdout="library_function") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py new file mode 100644 index 0000000000..7e999b33e5 --- /dev/null +++ b/test/ninja/copy_function_command.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) +env = Environment() +env.Tool('ninja') +env.Command('foo2.c', ['foo.c'], Copy('$TARGET','$SOURCE')) +env.Program(target = 'foo', source = 'foo2.c') +""") + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('foo'), stdout="foo.c") + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed foo2.o', + 'Removed foo2.c', + 'Removed foo' + _exe, + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo')) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program=program, stdout=None) +test.run(program=test.workpath('foo'), stdout="foo.c") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py new file mode 100644 index 0000000000..91be108176 --- /dev/null +++ b/test/ninja/generate_and_build.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build', 'SConstruct') + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('foo' + _exe), stdout="foo.c") + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed foo.o', + 'Removed foo', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo' + _exe)) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program=program, stdout=None) +test.run(program=test.workpath('foo' + _exe), stdout="foo.c") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/generate_and_build_cxx.py b/test/ninja/generate_and_build_cxx.py new file mode 100644 index 0000000000..074a5cb9af --- /dev/null +++ b/test/ninja/generate_and_build_cxx.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build_cxx', + 'SConstruct') + +# generate simple build +test.run() +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('test2' + _exe), stdout="print_function") + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed test2.o', + 'Removed test2', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('test2' + _exe)) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program=program, stdout=None) +test.run(program=test.workpath('test2' + _exe), stdout="print_function") + +test.write('test2.hpp', """ +#include +#include + +class Foo +{ +public: + int print_function(); + int print_function2(){ + std::cout << "2"; + return 0; + }; +}; +""") + +# generate simple build +test.run(program=program, stdout=None) +test.run(program=test.workpath('test2' + _exe), stdout="print_function2") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py new file mode 100644 index 0000000000..8300176c2e --- /dev/null +++ b/test/ninja/generate_source.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +shell = '' if IS_WINDOWS else './' + +test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + +env = Environment() +env.Tool('ninja') +prog = env.Program(target = 'generate_source', source = 'generate_source.c') +env.Command('generated_source.c', prog, '%(shell)sgenerate_source%(_exe)s') +env.Program(target = 'generated_source', source = 'generated_source.c') +""" % locals()) + +test.write('generate_source.c', """ +#include + +int main(int argc, char *argv[]) { + FILE *fp; + + fp = fopen("generated_source.c", "w"); + fprintf(fp, "#include \\n"); + fprintf(fp, "#include \\n"); + fprintf(fp, "\\n"); + fprintf(fp, "int\\n"); + fprintf(fp, "main(int argc, char *argv[])\\n"); + fprintf(fp, "{\\n"); + fprintf(fp, " printf(\\"generated_source.c\\");\\n"); + fprintf(fp, " exit (0);\\n"); + fprintf(fp, "}\\n"); + fclose(fp); +} +""") + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('generated_source' + _exe), stdout="generated_source.c") + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed generate_source.o', + 'Removed generate_source' + _exe, + 'Removed generated_source.c', + 'Removed generated_source.o', + 'Removed generated_source' + _exe, + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('generated_source' + _exe)) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program=program, stdout=None) +test.run(program=test.workpath('generated_source' + _exe), stdout="generated_source.c") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py new file mode 100644 index 0000000000..05e372cf56 --- /dev/null +++ b/test/ninja/iterative_speedup.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os +import random +import time + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +test.write('source_0.c', """ +#include +#include +#include "source_0.h" + +int +print_function0() +{ + printf("main print"); + return 0; +} +""") + +test.write('source_0.h', """ +#include +#include + +int +print_function0(); +""") + + +def get_num_cpus(): + """ + Function to get the number of CPUs the system has. + """ + # Linux, Unix and MacOS: + if hasattr(os, "sysconf"): + if 'SC_NPROCESSORS_ONLN' in os.sysconf_names: + # Linux & Unix: + ncpus = os.sysconf("SC_NPROCESSORS_ONLN") + if isinstance(ncpus, int) and ncpus > 0: + return ncpus + # OSX: + return int(os.popen("sysctl -n hw.ncpu")[1].read()) + # Windows: + if 'NUMBER_OF_PROCESSORS' in os.environ: + ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]) + if ncpus > 0: + return ncpus + # Default + return 1 + + +def generate_source(parent_source, current_source): + test.write('source_{}.c'.format(current_source), """ + #include + #include + #include "source_%(parent_source)s.h" + #include "source_%(current_source)s.h" + + int + print_function%(current_source)s() + { + return print_function%(parent_source)s(); + } + """ % locals()) + + test.write('source_{}.h'.format(current_source), """ + #include + #include + + int + print_function%(current_source)s(); + """ % locals()) + + +def mod_source_return(test_num): + parent_source = test_num - 1 + test.write('source_{}.c'.format(test_num), """ + #include + #include + #include "source_%(parent_source)s.h" + #include "source_%(test_num)s.h" + + int + print_function%(test_num)s() + { + int test = 5 + 5; + print_function%(parent_source)s(); + return test; + } + """ % locals()) + + +def mod_source_orig(test_num): + parent_source = test_num - 1 + test.write('source_{}.c'.format(test_num), """ + #include + #include + #include "source_%(parent_source)s.h" + #include "source_%(test_num)s.h" + + int + print_function%(test_num)s() + { + return print_function%(parent_source)s(); + } + """ % locals()) + + +num_source = 200 +for i in range(1, num_source + 1): + generate_source(i - 1, i) + +test.write('main.c', """ +#include +#include +#include "source_%(num_source)s.h" +int +main() +{ + print_function%(num_source)s(); + exit(0); +} +""" % locals()) + +test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + +env = Environment() +env.Tool('ninja') +sources = ['main.c'] + env.Glob('source*.c') +env.Program(target = 'print_bin', source = sources) +""") + +test.write('SConstruct_no_ninja', """ +env = Environment() +sources = ['main.c'] + env.Glob('source*.c') +env.Program(target = 'print_bin', source = sources) +""") + +tests_mods = [] +ninja_times = [] +scons_times = [] +for _ in range(10): + tests_mods += [random.randrange(1, num_source, 1)] +jobs = '-j' + str(get_num_cpus()) + +ninja_program = [test.workpath('run_ninja_env.bat'), jobs] if IS_WINDOWS else [ninja_bin, jobs] + +start = time.perf_counter() +test.run(arguments='--disable-execute-ninja', stdout=None) +test.run(program=ninja_program, stdout=None) +stop = time.perf_counter() +ninja_times += [stop - start] +test.run(program=test.workpath('print_bin'), stdout="main print") + +for test_mod in tests_mods: + mod_source_return(test_mod) + start = time.perf_counter() + test.run(program=ninja_program, stdout=None) + stop = time.perf_counter() + ninja_times += [stop - start] + +for test_mod in tests_mods: + mod_source_orig(test_mod) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed build.ninja']) + +start = time.perf_counter() +test.run(arguments=["-f", "SConstruct_no_ninja", jobs], stdout=None) +stop = time.perf_counter() +scons_times += [stop - start] +test.run(program=test.workpath('print_bin'), stdout="main print") + +for test_mod in tests_mods: + mod_source_return(test_mod) + start = time.perf_counter() + test.run(arguments=["-f", "SConstruct_no_ninja", jobs], stdout=None) + stop = time.perf_counter() + scons_times += [stop - start] + +full_build_print = True +for ninja_time, scons_time in zip(ninja_times, scons_times): + if ninja_time > scons_time: + test.fail_test(message="Ninja was slower than SCons: SCons: {:.3f}s Ninja: {:.3f}s".format(scons_time, ninja_time)) + if full_build_print: + full_build_print = False + print("Clean build {} files - SCons: {:.3f}s Ninja: {:.3f}s".format(num_source, scons_time, ninja_time)) + else: + print("Single File Rebuild - SCons: {:.3f}s Ninja: {:.3f}s".format(scons_time, ninja_time)) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py new file mode 100644 index 0000000000..e5da6cf885 --- /dev/null +++ b/test/ninja/multi_env.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + +env = Environment() +env.Tool('ninja') +env.Program(target = 'foo', source = 'foo.c') + +env2 = Environment() +env2.Program(target = 'bar', source = 'bar.c') +""") + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('foo' + _exe), stdout="foo.c") +test.run(program=test.workpath('bar' + _exe), stdout="bar.c") + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed foo.o', + 'Removed foo' + _exe, + 'Removed bar.o', + 'Removed bar' + _exe, + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo' + _exe)) +test.must_not_exist(test.workpath('bar' + _exe)) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program=program, stdout=None) +test.run(program=test.workpath('foo' + _exe), stdout="foo.c") +test.run(program=test.workpath('bar' + _exe), stdout="bar.c") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/ninja-fixture/bar.c b/test/ninja/ninja-fixture/bar.c new file mode 100644 index 0000000000..15b2ecc46a --- /dev/null +++ b/test/ninja/ninja-fixture/bar.c @@ -0,0 +1,10 @@ +#include +#include + +int +main(int argc, char *argv[]) +{ + argv[argc++] = "--"; + printf("bar.c"); + exit (0); +} diff --git a/test/ninja/ninja-fixture/foo.c b/test/ninja/ninja-fixture/foo.c new file mode 100644 index 0000000000..ba35c687a2 --- /dev/null +++ b/test/ninja/ninja-fixture/foo.c @@ -0,0 +1,10 @@ +#include +#include + +int +main(int argc, char *argv[]) +{ + argv[argc++] = "--"; + printf("foo.c"); + exit (0); +} diff --git a/test/ninja/ninja-fixture/test1.c b/test/ninja/ninja-fixture/test1.c new file mode 100644 index 0000000000..678461f508 --- /dev/null +++ b/test/ninja/ninja-fixture/test1.c @@ -0,0 +1,21 @@ +#include +#include + +#ifdef WIN32 +#ifdef LIBRARY_BUILD +#define DLLEXPORT __declspec(dllexport) +#else +#define DLLEXPORT __declspec(dllimport) +#endif +#else +#define DLLEXPORT +#endif + +DLLEXPORT extern int library_function(void); + +int +main(int argc, char *argv[]) +{ + library_function(); + exit(0); +} diff --git a/test/ninja/ninja-fixture/test2.cpp b/test/ninja/ninja-fixture/test2.cpp new file mode 100644 index 0000000000..69b54c97f7 --- /dev/null +++ b/test/ninja/ninja-fixture/test2.cpp @@ -0,0 +1,16 @@ +#include "test2.hpp" + +int +main(int argc, char *argv[]) +{ + Foo* test = new Foo(); + test->print_function(); + test->print_function2(); + return 0; +} + +int Foo::print_function() +{ + std::cout << "print_function"; + return 0; +} \ No newline at end of file diff --git a/test/ninja/ninja-fixture/test2.hpp b/test/ninja/ninja-fixture/test2.hpp new file mode 100644 index 0000000000..f0583fc547 --- /dev/null +++ b/test/ninja/ninja-fixture/test2.hpp @@ -0,0 +1,9 @@ +#include +#include + +class Foo +{ +public: + int print_function(); + int print_function2(){return 0;}; +}; diff --git a/test/ninja/ninja-fixture/test_impl.c b/test/ninja/ninja-fixture/test_impl.c new file mode 100644 index 0000000000..ac3fd88656 --- /dev/null +++ b/test/ninja/ninja-fixture/test_impl.c @@ -0,0 +1,20 @@ +#include +#include + +#ifdef WIN32 +#ifdef LIBRARY_BUILD +#define DLLEXPORT __declspec(dllexport) +#else +#define DLLEXPORT __declspec(dllimport) +#endif +#else +#define DLLEXPORT +#endif + + +DLLEXPORT int +library_function(void) +{ + printf("library_function"); + return 0; +} diff --git a/test/ninja/ninja_test_sconscripts/sconstest.skip b/test/ninja/ninja_test_sconscripts/sconstest.skip new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build new file mode 100644 index 0000000000..81a4366755 --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build @@ -0,0 +1,6 @@ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + +env = Environment() +env.Tool('ninja') +env.Program(target='foo', source='foo.c') diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx new file mode 100644 index 0000000000..f7137dfb6e --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx @@ -0,0 +1,5 @@ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) +env = Environment() +env.Tool('ninja') +env.Program(target = 'test2', source = 'test2.cpp') diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py new file mode 100644 index 0000000000..a6926c778d --- /dev/null +++ b/test/ninja/shell_command.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +shell = '' if IS_WINDOWS else './' + +test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + +env = Environment() +env.Tool('ninja') +prog = env.Program(target = 'foo', source = 'foo.c') +env.Command('foo.out', prog, '%(shell)sfoo%(_exe)s > foo.out') +""" % locals()) + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.must_match('foo.out', 'foo.c') + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed foo.o', + 'Removed foo%(_exe)s' % locals(), + 'Removed foo.out', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo.out')) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program=program, stdout=None) +test.must_match('foo.out', 'foo.c') + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/option/option--experimental.py b/test/option/option--experimental.py index 0f0e9ec677..2e06dc3777 100644 --- a/test/option/option--experimental.py +++ b/test/option/option--experimental.py @@ -35,12 +35,13 @@ tests = [ ('.', []), - ('--experimental=all', ['transporter', 'warp_speed']), + ('--experimental=ninja', ['ninja']), + ('--experimental=all', ['ninja', 'transporter', 'warp_speed']), ('--experimental=none', []), ] for args, exper in tests: - read_string = """All Features=transporter,warp_speed + read_string = """All Features=ninja,transporter,warp_speed Experimental=%s """ % (exper) test.run(arguments=args, @@ -49,7 +50,7 @@ test.run(arguments='--experimental=warp_drive', stderr="""usage: scons [OPTION] [TARGET] ... -SCons Error: option --experimental: invalid choice: 'warp_drive' (choose from 'all','none','transporter','warp_speed') +SCons Error: option --experimental: invalid choice: 'warp_drive' (choose from 'all','none','ninja','transporter','warp_speed') """, status=2) diff --git a/testing/framework/TestCmd.py b/testing/framework/TestCmd.py index 333901c6a9..903bee693a 100644 --- a/testing/framework/TestCmd.py +++ b/testing/framework/TestCmd.py @@ -314,6 +314,7 @@ IS_WINDOWS = sys.platform == 'win32' +IS_MACOS = sys.platform == 'darwin' IS_64_BIT = sys.maxsize > 2**32 IS_PYPY = hasattr(sys, 'pypy_translation_info')