From cc6a52fe7b78c868d18f0b1757899d92cd8b9573 Mon Sep 17 00:00:00 2001 From: Mathew Robinson Date: Fri, 24 Jan 2020 17:25:31 -0500 Subject: [PATCH 001/163] [WIP] write a tool to generate build.ninja files from SCons --- SCons/Script/SConscript.py | 3 + SCons/Script/__init__.py | 1 + src/engine/SCons/Tool/ninja.py | 1396 ++++++++++++++++++++++++++++++++ 3 files changed, 1400 insertions(+) create mode 100644 src/engine/SCons/Tool/ninja.py diff --git a/SCons/Script/SConscript.py b/SCons/Script/SConscript.py index 596fca0463..ded0fcfef9 100644 --- a/SCons/Script/SConscript.py +++ b/SCons/Script/SConscript.py @@ -203,9 +203,11 @@ def _SConscript(fs, *files, **kw): if f.rexists(): actual = f.rfile() _file_ = open(actual.get_abspath(), "rb") + SCons.Script.LOADED_SCONSCRIPTS.append(actual.get_abspath()) elif f.srcnode().rexists(): actual = f.srcnode().rfile() _file_ = open(actual.get_abspath(), "rb") + SCons.Script.LOADED_SCONSCRIPTS.append(actual.get_abspath()) elif f.has_src_builder(): # The SConscript file apparently exists in a source # code management system. Build it, but then clear @@ -214,6 +216,7 @@ def _SConscript(fs, *files, **kw): f.build() f.built() f.builder_set(None) + SCons.Script.LOADED_SCONSCRIPTS.append(f.get_abspath()) if f.exists(): _file_ = open(f.get_abspath(), "rb") if _file_: diff --git a/SCons/Script/__init__.py b/SCons/Script/__init__.py index dff15673b1..c7f6a22a93 100644 --- a/SCons/Script/__init__.py +++ b/SCons/Script/__init__.py @@ -187,6 +187,7 @@ def _clear(self): BUILD_TARGETS = TargetList() COMMAND_LINE_TARGETS = [] DEFAULT_TARGETS = [] +LOADED_SCONSCRIPTS = [] # BUILD_TARGETS can be modified in the SConscript files. If so, we # want to treat the modified BUILD_TARGETS list as if they specified diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py new file mode 100644 index 0000000000..b34759e401 --- /dev/null +++ b/src/engine/SCons/Tool/ninja.py @@ -0,0 +1,1396 @@ +# Copyright 2019 MongoDB Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Generate build.ninja files from SCons aliases.""" + +import sys +import os +import importlib +import io +import shutil + +from threading import Lock +from os.path import join as joinpath +from os.path import splitext + +import SCons +from SCons.Action import _string_from_cmd_list, get_default_ENV +from SCons.Util import is_String, is_List +from SCons.Script import COMMAND_LINE_TARGETS, LOADED_SCONSCRIPTS + +NINJA_SYNTAX = "NINJA_SYNTAX" +NINJA_RULES = "__NINJA_CUSTOM_RULES" +NINJA_POOLS = "__NINJA_CUSTOM_POOLS" +NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" +NINJA_BUILD = "NINJA_BUILD" +NINJA_WHEREIS_MEMO = {} +NINJA_STAT_MEMO = {} +MEMO_LOCK = Lock() + +__NINJA_RULE_MAPPING = {} + +# These are the types that get_command can do something with +COMMAND_TYPES = ( + SCons.Action.CommandAction, + SCons.Action.CommandGeneratorAction, +) + + +def _install_action_function(_env, node): + """Install files using the install or copy commands""" + return { + "outputs": get_outputs(node), + "rule": "INSTALL", + "pool": "install_pool", + "inputs": [get_path(src_file(s)) for s in node.sources], + "implicit": get_dependencies(node), + } + + +def _lib_symlink_action_function(_env, node): + """Create shared object symlinks if any need to be created""" + symlinks = getattr(getattr(node, "attributes", None), "shliblinks", None) + + if not symlinks or symlinks is None: + return None + + outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] + inputs = [link.get_path() for link, _ in symlinks] + + return { + "outputs": outputs, + "inputs": inputs, + "rule": "SYMLINK", + "implicit": get_dependencies(node), + } + + +def is_valid_dependent_node(node): + """ + Return True if node is not an alias or is an alias that has children + + This prevents us from making phony targets that depend on other + phony targets that will never have an associated ninja build + target. + + We also have to specify that it's an alias when doing the builder + check because some nodes (like src files) won't have builders but + are valid implicit dependencies. + """ + return not isinstance(node, SCons.Node.Alias.Alias) or node.children() + + +def alias_to_ninja_build(node): + """Convert an Alias node into a Ninja phony target""" + return { + "outputs": get_outputs(node), + "rule": "phony", + "implicit": [ + get_path(n) for n in node.children() if is_valid_dependent_node(n) + ], + } + + +def get_dependencies(node): + """Return a list of dependencies for node.""" + return [get_path(src_file(child)) for child in node.children()] + + +def get_inputs(node): + """Collect the Ninja inputs for node.""" + executor = node.get_executor() + if executor is not None: + inputs = executor.get_all_sources() + else: + inputs = node.sources + + inputs = [get_path(src_file(o)) for o in inputs] + return inputs + + +def get_outputs(node): + """Collect the Ninja outputs for node.""" + executor = node.get_executor() + if executor is not None: + outputs = executor.get_all_targets() + else: + if hasattr(node, "target_peers"): + outputs = node.target_peers + else: + outputs = [node] + + outputs = [get_path(o) for o in outputs] + return outputs + + +class SConsToNinjaTranslator: + """Translates SCons Actions into Ninja build objects.""" + + def __init__(self, env): + self.env = env + self.func_handlers = { + # Skip conftest builders + "_createSource": ninja_noop, + # SCons has a custom FunctionAction that just makes sure the + # target isn't static. We let the commands that ninja runs do + # this check for us. + "SharedFlagChecker": ninja_noop, + # The install builder is implemented as a function action. + "installFunc": _install_action_function, + "LibSymlinksActionFunction": _lib_symlink_action_function, + } + + self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) + + # pylint: disable=too-many-return-statements + def action_to_ninja_build(self, node, action=None): + """Generate build arguments dictionary for node.""" + # Use False since None is a valid value for this Attribute + build = getattr(node.attributes, NINJA_BUILD, False) + if build is not False: + return build + + if node.builder is None: + return None + + if action is None: + action = node.builder.action + + # Ideally this should never happen, and we do try to filter + # Ninja builders out of being sources of ninja builders but I + # can't fix every DAG problem so we just skip ninja_builders + # if we find one + if node.builder == self.env["BUILDERS"]["Ninja"]: + return None + + if isinstance(action, SCons.Action.FunctionAction): + return self.handle_func_action(node, action) + + if isinstance(action, SCons.Action.LazyAction): + # pylint: disable=protected-access + action = action._generate_cache(node.env if node.env else self.env) + return self.action_to_ninja_build(node, action=action) + + if isinstance(action, SCons.Action.ListAction): + return self.handle_list_action(node, action) + + if isinstance(action, COMMAND_TYPES): + return get_command(node.env if node.env else self.env, node, action) + + # Return the node to indicate that SCons is required + return { + "rule": "SCONS", + "outputs": get_outputs(node), + "implicit": get_dependencies(node), + } + + def handle_func_action(self, node, action): + """Determine how to handle the function action.""" + name = action.function_name() + # This is the name given by the Subst/Textfile builders. So return the + # node to indicate that SCons is required + if name == "_action": + return { + "rule": "TEMPLATE", + "outputs": get_outputs(node), + "implicit": get_dependencies(node), + } + + handler = self.func_handlers.get(name, None) + if handler is not None: + return handler(node.env if node.env else self.env, node) + + print( + "Found unhandled function action {}, " + " generating scons command to build\n" + "Note: this is less efficient than Ninja," + " you can write your own ninja build generator for" + " this function using NinjaRegisterFunctionHandler".format(name) + ) + + return { + "rule": "SCONS", + "outputs": get_outputs(node), + "implicit": get_dependencies(node), + } + + # pylint: disable=too-many-branches + def handle_list_action(self, node, action): + """ + Attempt to translate list actions to Ninja. + + List actions are tricky to move to ninja. First we translate + each individual action in the action list to a Ninja + build. Then we process the resulting ninja builds to see if + they are all the same ninja rule. If they are not all the same + rule we cannot make them a single resulting ninja build, so + instead we make them a single SCons invocation to build all of + the targets. + + If they are all the same rule and the rule is CMD we attempt + to combine the cmdlines together using ' && ' which we then + combine into a single ninja build. + + If they are all phony targets we simple combine the outputs + and dependencies. + + If they are all INSTALL rules we combine all of the sources + and outputs. + + If they are all SCONS rules we do the same as if they are not + the same rule and make a build that will use SCons to generate + them. + + If they're all one rule and None of the above rules we throw an Exception. + """ + + results = [ + self.action_to_ninja_build(node, action=act) + for act in action.list + if act is not None + ] + results = [result for result in results if result is not None] + if not results: + return None + + # No need to process the results if we only got a single result + if len(results) == 1: + return results[0] + + all_outputs = list({output for build in results for output in build["outputs"]}) + # If we have no outputs we're done + if not all_outputs: + return None + + # Used to verify if all rules are the same + all_one_rule = len( + [ + r + for r in results + if isinstance(r, dict) and r["rule"] == results[0]["rule"] + ] + ) == len(results) + dependencies = get_dependencies(node) + + if not all_one_rule: + # If they aren't all the same rule use scons to generate these + # outputs. At this time nothing hits this case. + return { + "outputs": all_outputs, + "rule": "SCONS", + "implicit": dependencies, + } + + if results[0]["rule"] == "CMD": + cmdline = "" + for cmd in results: + + # Occasionally a command line will expand to a + # whitespace only string (i.e. ' '). Which is not a + # valid command but does not trigger the empty command + # condition if not cmdstr. So here we strip preceding + # and proceeding whitespace to make strings like the + # above become empty strings and so will be skipped. + cmdstr = cmd["variables"]["cmd"].strip() + if not cmdstr: + continue + + # Skip duplicate commands + if cmdstr in cmdline: + continue + + if cmdline: + cmdline += " && " + + cmdline += cmdstr + + # Remove all preceding and proceeding whitespace + cmdline = cmdline.strip() + + # Make sure we didn't generate an empty cmdline + if cmdline: + ninja_build = { + "outputs": all_outputs, + "rule": "CMD", + "variables": {"cmd": cmdline}, + "implicit": dependencies, + } + + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["pool"] + + return ninja_build + + elif results[0]["rule"] == "phony": + return { + "outputs": all_outputs, + "rule": "phony", + "implicit": dependencies, + } + + elif results[0]["rule"] == "INSTALL": + return { + "outputs": all_outputs, + "rule": "INSTALL", + "pool": "install_pool", + "inputs": [get_path(src_file(s)) for s in node.sources], + "implicit": dependencies, + } + + elif results[0]["rule"] == "SCONS": + return { + "outputs": all_outputs, + "rule": "SCONS", + "inputs": dependencies, + } + + raise Exception("Unhandled list action with rule: " + results[0]["rule"]) + + +# pylint: disable=too-many-instance-attributes +class NinjaState: + """Maintains state of Ninja build system as it's translated from SCons.""" + + def __init__(self, env, writer_class): + self.env = env + self.writer_class = writer_class + self.__generated = False + self.translator = SConsToNinjaTranslator(env) + self.generated_suffixes = env.get("NINJA_GENERATED_SOURCE_SUFFIXES", []) + + # List of generated builds that will be written at a later stage + self.builds = list() + + # List of targets for which we have generated a build. This + # allows us to take multiple Alias nodes as sources and to not + # fail to build if they have overlapping targets. + self.built = set() + + # SCons sets this variable to a function which knows how to do + # shell quoting on whatever platform it's run on. Here we use it + # to make the SCONS_INVOCATION variable properly quoted for things + # like CCFLAGS + escape = env.get("ESCAPE", lambda x: x) + + self.variables = { + "COPY": "cmd.exe /c copy" if sys.platform == "win32" else "cp", + "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format( + sys.executable, + " ".join( + [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] + ), + ), + "SCONS_INVOCATION_W_TARGETS": "{} {}".format( + sys.executable, " ".join([escape(arg) for arg in sys.argv]) + ), + # This must be set to a global default per: + # https://ninja-build.org/manual.html + # + # (The deps section) + "msvc_deps_prefix": "Note: including file:", + } + + self.rules = { + "CMD": { + "command": "cmd /c $cmd" if sys.platform == "win32" else "$cmd", + "description": "Building $out", + }, + # We add the deps processing variables to this below. We + # don't pipe this through cmd.exe on Windows because we + # use this to generate a compile_commands.json database + # which can't use the shell command as it's compile + # command. This does mean that we assume anything using + # CMD_W_DEPS is a straight up compile which is true today. + "CMD_W_DEPS": {"command": "$cmd", "description": "Building $out"}, + "SYMLINK": { + "command": ( + "cmd /c mklink $out $in" + if sys.platform == "win32" + else "ln -s $in $out" + ), + "description": "Symlink $in -> $out", + }, + "INSTALL": { + "command": "$COPY $in $out", + "description": "Install $out", + # On Windows cmd.exe /c copy does not always correctly + # update the timestamp on the output file. This leads + # to a stuck constant timestamp in the Ninja database + # and needless rebuilds. + # + # Adding restat here ensures that Ninja always checks + # the copy updated the timestamp and that Ninja has + # the correct information. + "restat": 1, + }, + "TEMPLATE": { + "command": "$SCONS_INVOCATION $out", + "description": "Rendering $out", + "pool": "scons_pool", + "restat": 1, + }, + "SCONS": { + "command": "$SCONS_INVOCATION $out", + "description": "SCons $out", + "pool": "scons_pool", + # restat + # if present, causes Ninja to re-stat the command's outputs + # after execution of the command. Each output whose + # modification time the command did not change will be + # treated as though it had never needed to be built. This + # may cause the output's reverse dependencies to be removed + # from the list of pending build actions. + # + # We use restat any time we execute SCons because + # SCons calls in Ninja typically create multiple + # targets. But since SCons is doing it's own up to + # date-ness checks it may only update say one of + # them. Restat will find out which of the multiple + # build targets did actually change then only rebuild + # those targets which depend specifically on that + # output. + "restat": 1, + }, + "REGENERATE": { + "command": "$SCONS_INVOCATION_W_TARGETS", + "description": "Regenerating $out", + "generator": 1, + # Console pool restricts to 1 job running at a time, + # it additionally has some special handling about + # passing stdin, stdout, etc to process in this pool + # that we need for SCons to behave correctly when + # regenerating Ninja + "pool": "console", + # Again we restat in case Ninja thought the + # build.ninja should be regenerated but SCons knew + # better. + "restat": 1, + }, + } + + self.pools = { + "install_pool": self.env.GetOption("num_jobs") / 2, + "scons_pool": 1, + } + + if env["PLATFORM"] == "win32": + self.rules["CMD_W_DEPS"]["deps"] = "msvc" + else: + self.rules["CMD_W_DEPS"]["deps"] = "gcc" + self.rules["CMD_W_DEPS"]["depfile"] = "$out.d" + + self.rules.update(env.get(NINJA_RULES, {})) + self.pools.update(env.get(NINJA_POOLS, {})) + + def generate_builds(self, node): + """Generate a ninja build rule for node and it's children.""" + # Filter out nodes with no builder. They are likely source files + # and so no work needs to be done, it will be used in the + # generation for some real target. + # + # Note that all nodes have a builder attribute but it is sometimes + # set to None. So we cannot use a simpler hasattr check here. + if getattr(node, "builder", None) is None: + return + + stack = [[node]] + while stack: + frame = stack.pop() + for child in frame: + outputs = set(get_outputs(child)) + # Check if all the outputs are in self.built, if they + # are we've already seen this node and it's children. + if not outputs.isdisjoint(self.built): + continue + + self.built = self.built.union(outputs) + stack.append(child.children()) + + if isinstance(child, SCons.Node.Alias.Alias): + build = alias_to_ninja_build(child) + elif node.builder is not None: + # Use False since None is a valid value for this attribute + build = getattr(child.attributes, NINJA_BUILD, False) + if build is False: + build = self.translator.action_to_ninja_build(child) + setattr(child.attributes, NINJA_BUILD, build) + else: + build = None + + # Some things are unbuild-able or need not be built in Ninja + if build is None or build == 0: + continue + + self.builds.append(build) + + def is_generated_source(self, output): + """Check if output ends with a known generated suffix.""" + _, suffix = splitext(output) + return suffix in self.generated_suffixes + + def has_generated_sources(self, output): + """ + Determine if output indicates this is a generated header file. + """ + for generated in output: + if self.is_generated_source(generated): + return True + return False + + # pylint: disable=too-many-branches,too-many-locals + def generate(self, ninja_file, fallback_default_target=None): + """ + Generate the build.ninja. + + This should only be called once for the lifetime of this object. + """ + if self.__generated: + return + + content = io.StringIO() + ninja = self.writer_class(content, width=100) + + ninja.comment("Generated by scons. DO NOT EDIT.") + + for pool_name, size in self.pools.items(): + ninja.pool(pool_name, size) + + for var, val in self.variables.items(): + ninja.variable(var, val) + + for rule, kwargs in self.rules.items(): + ninja.rule(rule, **kwargs) + + generated_source_files = { + output + # First find builds which have header files in their outputs. + for build in self.builds + if self.has_generated_sources(build["outputs"]) + for output in build["outputs"] + # Collect only the header files from the builds with them + # in their output. We do this because is_generated_source + # returns True if it finds a header in any of the outputs, + # here we need to filter so we only have the headers and + # not the other outputs. + if self.is_generated_source(output) + } + + if generated_source_files: + ninja.build( + outputs="_generated_sources", + rule="phony", + implicit=list(generated_source_files), + ) + + template_builders = [] + + for build in self.builds: + if build["rule"] == "TEMPLATE": + template_builders.append(build) + continue + + implicit = build.get("implicit", []) + implicit.append(ninja_file) + build["implicit"] = implicit + + # Don't make generated sources depend on each other. We + # have to check that none of the outputs are generated + # sources and none of the direct implicit dependencies are + # generated sources or else we will create a dependency + # cycle. + if ( + generated_source_files + and not build["rule"] == "INSTALL" + and set(build["outputs"]).isdisjoint(generated_source_files) + and set(implicit).isdisjoint(generated_source_files) + ): + + # Make all non-generated source targets depend on + # _generated_sources. We use order_only for generated + # sources so that we don't rebuild the world if one + # generated source was rebuilt. We just need to make + # sure that all of these sources are generated before + # other builds. + build["order_only"] = "_generated_sources" + + # When using a depfile Ninja can only have a single output + # but SCons will usually have emitted an output for every + # thing a command will create because it's caching is much + # more complex than Ninja's. This includes things like DWO + # files. Here we make sure that Ninja only ever sees one + # target when using a depfile. It will still have a command + # that will create all of the outputs but most targets don't + # depend direclty on DWO files and so this assumption is safe + # to make. + rule = self.rules.get(build["rule"]) + + # Some rules like 'phony' and other builtins we don't have + # listed in self.rules so verify that we got a result + # before trying to check if it has a deps key. + if rule is not None and rule.get("deps"): + + # Anything using deps in Ninja can only have a single + # output, but we may have a build which actually + # produces multiple outputs which other targets can + # depend on. Here we slice up the outputs so we have a + # single output which we will use for the "real" + # builder and multiple phony targets that match the + # file names of the remaining outputs. This way any + # build can depend on any output from any build. + first_output, remaining_outputs = build["outputs"][0], build["outputs"][1:] + if remaining_outputs: + ninja.build( + outputs=remaining_outputs, + rule="phony", + implicit=first_output, + ) + + build["outputs"] = first_output + + ninja.build(**build) + + template_builds = dict() + for template_builder in template_builders: + + # Special handling for outputs and implicit since we need to + # aggregate not replace for each builder. + for agg_key in ["outputs", "implicit"]: + new_val = template_builds.get(agg_key, []) + + # Use pop so the key is removed and so the update + # below will not overwrite our aggregated values. + cur_val = template_builder.pop(agg_key, []) + if isinstance(cur_val, list): + new_val += cur_val + else: + new_val.append(cur_val) + template_builds[agg_key] = new_val + + # Collect all other keys + template_builds.update(template_builder) + + if template_builds.get("outputs", []): + ninja.build(**template_builds) + + # Here to teach the ninja file how to regenerate itself. We'll + # never see ourselves in the DAG walk so we can't rely on + # action_to_ninja_build to generate this rule + ninja.build( + ninja_file, + rule="REGENERATE", + implicit=[ + self.env.File("#SConstruct").get_abspath(), + os.path.abspath(__file__), + ] + + LOADED_SCONSCRIPTS, + ) + + ninja.build( + "scons-invocation", + rule="CMD", + pool="console", + variables={"cmd": "echo $SCONS_INVOCATION_W_TARGETS"}, + ) + + # Note the use of CMD_W_DEPS below. CMD_W_DEPS are always + # compile commands in this generator. If we ever change the + # name/s of the rules that include compile commands + # (i.e. something like CC/CXX) we will need to update this + # build to reflect that complete list. + ninja.build( + "compile_commands.json", + rule="CMD", + pool="console", + variables={ + "cmd": "ninja -f {} -t compdb CMD_W_DEPS > compile_commands.json".format( + ninja_file + ) + }, + ) + + ninja.build( + "compiledb", rule="phony", implicit=["compile_commands.json"], + ) + + # Look in SCons's list of DEFAULT_TARGETS, find the ones that + # we generated a ninja build rule for. + scons_default_targets = [ + get_path(tgt) + for tgt in SCons.Script.DEFAULT_TARGETS + if get_path(tgt) in self.built + ] + + # If we found an overlap between SCons's list of default + # targets and the targets we created ninja builds for then use + # those as ninja's default as well. + if scons_default_targets: + ninja.default(" ".join(scons_default_targets)) + + # If not then set the default to the fallback_default_target we were given. + # Otherwise we won't create a default ninja target. + elif fallback_default_target is not None: + ninja.default(fallback_default_target) + + with open(ninja_file, "w") as build_ninja: + build_ninja.write(content.getvalue()) + + self.__generated = True + + +def get_path(node): + """ + Return a fake path if necessary. + + As an example Aliases use this as their target name in Ninja. + """ + if hasattr(node, "get_path"): + return node.get_path() + return str(node) + + +def rfile(node): + """ + Return the repository file for node if it has one. Otherwise return node + """ + if hasattr(node, "rfile"): + return node.rfile() + return node + + +def src_file(node): + """Returns the src code file if it exists.""" + if hasattr(node, "srcnode"): + src = node.srcnode() + if src.stat() is not None: + return src + return get_path(node) + + +# TODO: Make the Rules smarter. Instead of just using a "cmd" rule +# everywhere we should be smarter about generating CC, CXX, LINK, +# etc. rules +def get_command(env, node, action): # pylint: disable=too-many-branches + """Get the command to execute for node.""" + if node.env: + sub_env = node.env + else: + sub_env = env + + executor = node.get_executor() + if executor is not None: + tlist = executor.get_all_targets() + slist = executor.get_all_sources() + else: + if hasattr(node, "target_peers"): + tlist = node.target_peers + else: + tlist = [node] + slist = node.sources + + # Retrieve the repository file for all sources + slist = [rfile(s) for s in slist] + + # Get the dependencies for all targets + implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) + + # Generate a real CommandAction + if isinstance(action, SCons.Action.CommandGeneratorAction): + # pylint: disable=protected-access + action = action._generate(tlist, slist, sub_env, 1, executor=executor) + + rule = "CMD" + + # Actions like CommandAction have a method called process that is + # used by SCons to generate the cmd_line they need to run. So + # check if it's a thing like CommandAction and call it if we can. + if hasattr(action, "process"): + cmd_list, _, _ = action.process(tlist, slist, sub_env, executor=executor) + + # Despite being having "list" in it's name this member is not + # actually a list. It's the pre-subst'd string of the command. We + # use it to determine if the command we generated needs to use a + # custom Ninja rule. By default this redirects CC/CXX commands to + # CMD_W_DEPS but the user can inject custom Ninja rules and tie + # them to commands by using their pre-subst'd string. + rule = __NINJA_RULE_MAPPING.get(action.cmd_list, "CMD") + + cmd = _string_from_cmd_list(cmd_list[0]) + else: + # Anything else works with genstring, this is most commonly hit by + # ListActions which essentially call process on all of their + # commands and concatenate it for us. + genstring = action.genstring(tlist, slist, sub_env) + + # Detect if we have a custom rule for this + # "ListActionCommandAction" type thing. + rule = __NINJA_RULE_MAPPING.get(genstring, "CMD") + + if executor is not None: + cmd = sub_env.subst(genstring, executor=executor) + else: + cmd = sub_env.subst(genstring, target=tlist, source=slist) + + # Since we're only enabling Ninja for developer builds right + # now we skip all Manifest related work on Windows as it's not + # necessary. We shouldn't have gotten here but on Windows + # SCons has a ListAction which shows as a + # CommandGeneratorAction for linking. That ListAction ends + # with a FunctionAction (embedManifestExeCheck, + # embedManifestDllCheck) that simply say "does + # target[0].manifest exist?" if so execute the real command + # action underlying me, otherwise do nothing. + # + # Eventually we'll want to find a way to translate this to + # Ninja but for now, and partially because the existing Ninja + # generator does so, we just disable it all together. + cmd = cmd.replace("\n", " && ").strip() + if env["PLATFORM"] == "win32" and ( + "embedManifestExeCheck" in cmd or "embedManifestDllCheck" in cmd + ): + cmd = " && ".join(cmd.split(" && ")[0:-1]) + + if cmd.endswith("&&"): + cmd = cmd[0:-2].strip() + + outputs = get_outputs(node) + command_env = "" + windows = env["PLATFORM"] == "win32" + + # If win32 and rule == CMD_W_DEPS then we don't want to calculate + # an environment for this command. It's a compile command and + # compiledb doesn't support shell syntax on Windows. We need the + # shell syntax to use environment variables on Windows so we just + # skip this platform / rule combination to keep the compiledb + # working. + # + # On POSIX we can still set environment variables even for compile + # commands so we do so. + if not (windows and rule == "CMD_W_DEPS"): + + # Scan the ENV looking for any keys which do not exist in + # os.environ or differ from it. We assume if it's a new or + # differing key from the process environment then it's + # important to pass down to commands in the Ninja file. + ENV = get_default_ENV(sub_env) + scons_specified_env = { + key: value + for key, value in ENV.items() + if key not in os.environ or os.environ.get(key, None) != value + } + + for key, value in scons_specified_env.items(): + # Ensure that the ENV values are all strings: + if is_List(value): + # If the value is a list, then we assume it is a + # path list, because that's a pretty common list-like + # value to stick in an environment variable: + value = flatten_sequence(value) + value = joinpath(map(str, value)) + else: + # If it isn't a string or a list, then we just coerce + # it to a string, which is the proper way to handle + # Dir and File instances and will produce something + # reasonable for just about everything else: + value = str(value) + + if windows: + command_env += "set '{}={}' && ".format(key, value) + else: + command_env += "{}={} ".format(key, value) + + variables = {"cmd": command_env + cmd} + extra_vars = getattr(node.attributes, "NINJA_EXTRA_VARS", {}) + if extra_vars: + variables.update(extra_vars) + + ninja_build = { + "outputs": outputs, + "inputs": get_inputs(node), + "implicit": implicit, + "rule": rule, + "variables": variables, + } + + # Don't use sub_env here because we require that NINJA_POOL be set + # on a per-builder call basis to prevent accidental strange + # behavior like env['NINJA_POOL'] = 'console' and sub_env can be + # the global Environment object if node.env is None. + # Example: + # + # Allowed: + # + # env.Command("ls", NINJA_POOL="ls_pool") + # + # Not allowed and ignored: + # + # env["NINJA_POOL"] = "ls_pool" + # env.Command("ls") + # + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["NINJA_POOL"] + + return ninja_build + + +def ninja_builder(env, target, source): + """Generate a build.ninja for source.""" + if not isinstance(source, list): + source = [source] + if not isinstance(target, list): + target = [target] + + # We have no COMSTR equivalent so print that we're generating + # here. + print("Generating:", str(target[0])) + + # The environment variable NINJA_SYNTAX points to the + # ninja_syntax.py module from the ninja sources found here: + # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py + # + # This should be vendored into the build sources and it's location + # set in NINJA_SYNTAX. This code block loads the location from + # that variable, gets the absolute path to the vendored file, gets + # it's parent directory then uses importlib to import the module + # dynamically. + ninja_syntax_file = env[NINJA_SYNTAX] + if isinstance(ninja_syntax_file, str): + ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() + ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) + sys.path.append(ninja_syntax_mod_dir) + ninja_syntax_mod_name = os.path.basename(ninja_syntax_file) + ninja_syntax = importlib.import_module(ninja_syntax_mod_name.replace(".py", "")) + + suffix = env.get("NINJA_SUFFIX", "") + if suffix and not suffix[0] == ".": + suffix = "." + suffix + + generated_build_ninja = target[0].get_abspath() + suffix + ninja_state = NinjaState(env, ninja_syntax.Writer) + + for src in source: + ninja_state.generate_builds(src) + + ninja_state.generate(generated_build_ninja, str(source[0])) + + return 0 + + +# pylint: disable=too-few-public-methods +class AlwaysExecAction(SCons.Action.FunctionAction): + """Override FunctionAction.__call__ to always execute.""" + + def __call__(self, *args, **kwargs): + kwargs["execute"] = 1 + return super().__call__(*args, **kwargs) + + +def ninja_print(_cmd, target, _source, env): + """Tag targets with the commands to build them.""" + if target: + for tgt in target: + if ( + tgt.has_builder() + # Use 'is False' because not would still trigger on + # None's which we don't want to regenerate + and getattr(tgt.attributes, NINJA_BUILD, False) is False + and isinstance(tgt.builder.action, COMMAND_TYPES) + ): + ninja_action = get_command(env, tgt, tgt.builder.action) + setattr(tgt.attributes, NINJA_BUILD, ninja_action) + # Preload the attributes dependencies while we're still running + # multithreaded + get_dependencies(tgt) + return 0 + + +def register_custom_handler(env, name, handler): + """Register a custom handler for SCons function actions.""" + env[NINJA_CUSTOM_HANDLERS][name] = handler + + +def register_custom_rule_mapping(env, pre_subst_string, rule): + """Register a custom handler for SCons function actions.""" + global __NINJA_RULE_MAPPING + __NINJA_RULE_MAPPING[pre_subst_string] = rule + + +def register_custom_rule(env, rule, command, description="", deps=None): + """Allows specification of Ninja rules from inside SCons files.""" + rule_obj = { + "command": command, + "description": description if description else "{} $out".format(rule), + } + + if deps is not None: + rule_obj["deps"] = deps + + env[NINJA_RULES][rule] = rule_obj + + +def register_custom_pool(env, pool, size): + """Allows the creation of custom Ninja pools""" + env[NINJA_POOLS][pool] = size + + +def ninja_csig(original): + """Return a dummy csig""" + + def wrapper(self): + name = str(self) + if "SConscript" in name or "SConstruct" in name: + return original(self) + return "dummy_ninja_csig" + + return wrapper + + +def ninja_contents(original): + """Return a dummy content without doing IO""" + + def wrapper(self): + name = str(self) + if "SConscript" in name or "SConstruct" in name: + return original(self) + return bytes("dummy_ninja_contents", encoding="utf-8") + + return wrapper + + +def ninja_stat(_self, path): + """ + Eternally memoized stat call. + + SCons is very aggressive about clearing out cached values. For our + purposes everything should only ever call stat once since we're + running in a no_exec build the file system state should not + change. For these reasons we patch SCons.Node.FS.LocalFS.stat to + use our eternal memoized dictionary. + + Since this is happening during the Node walk it's being run while + threaded, we have to protect adding to the memoized dictionary + with a threading.Lock otherwise many targets miss the memoization + due to racing. + """ + global NINJA_STAT_MEMO + + try: + return NINJA_STAT_MEMO[path] + except KeyError: + try: + result = os.stat(path) + except os.error: + result = None + + with MEMO_LOCK: + NINJA_STAT_MEMO[path] = result + + return result + + +def ninja_noop(*_args, **_kwargs): + """ + A general purpose no-op function. + + There are many things that happen in SCons that we don't need and + also don't return anything. We use this to disable those functions + instead of creating multiple definitions of the same thing. + """ + return None + + +def ninja_whereis(thing, *_args, **_kwargs): + """Replace env.WhereIs with a much faster version""" + global NINJA_WHEREIS_MEMO + + # Optimize for success, this gets called significantly more often + # when the value is already memoized than when it's not. + try: + return NINJA_WHEREIS_MEMO[thing] + except KeyError: + # We do not honor any env['ENV'] or env[*] variables in the + # generated ninja ile. Ninja passes your raw shell environment + # down to it's subprocess so the only sane option is to do the + # same during generation. At some point, if and when we try to + # upstream this, I'm sure a sticking point will be respecting + # env['ENV'] variables and such but it's actually quite + # complicated. I have a naive version but making it always work + # with shell quoting is nigh impossible. So I've decided to + # cross that bridge when it's absolutely required. + path = shutil.which(thing) + NINJA_WHEREIS_MEMO[thing] = path + return path + + +def ninja_always_serial(self, num, taskmaster): + """Replacement for SCons.Job.Jobs constructor which always uses the Serial Job class.""" + # We still set self.num_jobs to num even though it's a lie. The + # only consumer of this attribute is the Parallel Job class AND + # the Main.py function which instantiates a Jobs class. It checks + # if Jobs.num_jobs is equal to options.num_jobs, so if the user + # provides -j12 but we set self.num_jobs = 1 they get an incorrect + # warning about this version of Python not supporting parallel + # builds. So here we lie so the Main.py will not give a false + # warning to users. + self.num_jobs = num + self.job = SCons.Job.Serial(taskmaster) + + +class NinjaEternalTempFile(SCons.Platform.TempFileMunge): + """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" + + def __call__(self, target, source, env, for_signature): + if for_signature: + return self.cmd + + node = target[0] if SCons.Util.is_List(target) else target + if node is not None: + cmdlist = getattr(node.attributes, "tempfile_cmdlist", None) + if cmdlist is not None: + return cmdlist + + cmd = super().__call__(target, source, env, for_signature) + + # If TempFileMunge.__call__ returns a string it means that no + # response file was needed. No processing required so just + # return the command. + if isinstance(cmd, str): + return cmd + + # Strip the removal commands from the command list. + # + # SCons' TempFileMunge class has some very strange + # behavior where it, as part of the command line, tries to + # delete the response file after executing the link + # command. We want to keep those response files since + # Ninja will keep using them over and over. The + # TempFileMunge class creates a cmdlist to do this, a + # common SCons convention for executing commands see: + # https://github.com/SCons/scons/blob/master/src/engine/SCons/Action.py#L949 + # + # This deletion behavior is not configurable. So we wanted + # to remove the deletion command from the command list by + # simply slicing it out here. Unfortunately for some + # strange reason TempFileMunge doesn't make the "rm" + # command it's own list element. It appends it to the + # tempfile argument to cmd[0] (which is CC/CXX) and then + # adds the tempfile again as it's own element. + # + # So we just kind of skip that middle element. Since the + # tempfile is in the command list on it's own at the end we + # can cut it out entirely. This is what I would call + # "likely to break" in future SCons updates. Hopefully it + # breaks because they start doing the right thing and not + # weirdly splitting these arguments up. For reference a + # command list that we get back from the OG TempFileMunge + # looks like this: + # + # [ + # 'g++', + # '@/mats/tempfiles/random_string.lnk\nrm', + # '/mats/tempfiles/random_string.lnk', + # ] + # + # Note the weird newline and rm command in the middle + # element and the lack of TEMPFILEPREFIX on the last + # element. + prefix = env.subst("$TEMPFILEPREFIX") + if not prefix: + prefix = "@" + + new_cmdlist = [cmd[0], prefix + cmd[-1]] + setattr(node.attributes, "tempfile_cmdlist", new_cmdlist) + return new_cmdlist + + def _print_cmd_str(*_args, **_kwargs): + """Disable this method""" + pass + + +def exists(env): + """Enable if called.""" + + # This variable disables the tool when storing the SCons command in the + # generated ninja file to ensure that the ninja tool is not loaded when + # SCons should do actual work as a subprocess of a ninja build. The ninja + # tool is very invasive into the internals of SCons and so should never be + # enabled when SCons needs to build a target. + if env.get("__NINJA_NO", "0") == "1": + return False + + return True + + +def generate(env): + """Generate the NINJA builders.""" + env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") + + # Add the Ninja builder. + always_exec_ninja_action = AlwaysExecAction(ninja_builder, {}) + ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action) + env.Append(BUILDERS={"Ninja": ninja_builder_obj}) + + # This adds the required flags such that the generated compile + # commands will create depfiles as appropriate in the Ninja file. + if env["PLATFORM"] == "win32": + env.Append(CCFLAGS=["/showIncludes"]) + else: + env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) + + # Provides a way for users to handle custom FunctionActions they + # want to translate to Ninja. + env[NINJA_CUSTOM_HANDLERS] = {} + env.AddMethod(register_custom_handler, "NinjaRegisterFunctionHandler") + + # Provides a mechanism for inject custom Ninja rules which can + # then be mapped using NinjaRuleMapping. + env[NINJA_RULES] = {} + env.AddMethod(register_custom_rule, "NinjaRule") + + # Provides a mechanism for inject custom Ninja pools which can + # be used by providing the NINJA_POOL="name" as an + # OverrideEnvironment variable in a builder call. + env[NINJA_POOLS] = {} + env.AddMethod(register_custom_pool, "NinjaPool") + + # Add the ability to register custom NinjaRuleMappings for Command + # builders. We don't store this dictionary in the env to prevent + # accidental deletion of the CC/XXCOM mappings. You can still + # overwrite them if you really want to but you have to explicit + # about it this way. The reason is that if they were accidentally + # deleted you would get a very subtly incorrect Ninja file and + # might not catch it. + env.AddMethod(register_custom_rule_mapping, "NinjaRuleMapping") + env.NinjaRuleMapping("${CCCOM}", "CMD_W_DEPS") + env.NinjaRuleMapping("${CXXCOM}", "CMD_W_DEPS") + + # Make SCons node walk faster by preventing unnecessary work + env.Decider("timestamp-match") + + # Used to determine if a build generates a source file. Ninja + # requires that all generated sources are added as order_only + # dependencies to any builds that *might* use them. + env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] + + # This is the point of no return, anything after this comment + # makes changes to SCons that are irreversible and incompatible + # with a normal SCons build. We return early if __NINJA_NO=1 has + # been given on the command line (i.e. by us in the generated + # ninja file) here to prevent these modifications from happening + # when we want SCons to do work. Everything before this was + # necessary to setup the builder and other functions so that the + # tool can be unconditionally used in the users's SCons files. + + if not exists(env): + return + + # Set a known variable that other tools can query so they can + # behave correctly during ninja generation. + env["GENERATING_NINJA"] = True + + # These methods are no-op'd because they do not work during ninja + # generation, expected to do no work, or simply fail. All of which + # are slow in SCons. So we overwrite them with no logic. + SCons.Node.FS.File.make_ready = ninja_noop + SCons.Node.FS.File.prepare = ninja_noop + SCons.Node.FS.File.push_to_cache = ninja_noop + SCons.Executor.Executor.prepare = ninja_noop + SCons.Node.FS.File.built = ninja_noop + + # We make lstat a no-op because it is only used for SONAME + # symlinks which we're not producing. + SCons.Node.FS.LocalFS.lstat = ninja_noop + + # This is a slow method that isn't memoized. We make it a noop + # since during our generation we will never use the results of + # this or change the results. + SCons.Node.FS.is_up_to_date = ninja_noop + + # We overwrite stat and WhereIs with eternally memoized + # implementations. See the docstring of ninja_stat and + # ninja_whereis for detailed explanations. + SCons.Node.FS.LocalFS.stat = ninja_stat + SCons.Util.WhereIs = ninja_whereis + + # Monkey patch get_csig and get_contents for some classes. It + # slows down the build significantly and we don't need contents or + # content signatures calculated when generating a ninja file since + # we're not doing any SCons caching or building. + SCons.Executor.Executor.get_contents = ninja_contents( + SCons.Executor.Executor.get_contents + ) + SCons.Node.Alias.Alias.get_contents = ninja_contents( + SCons.Node.Alias.Alias.get_contents + ) + SCons.Node.FS.File.get_contents = ninja_contents(SCons.Node.FS.File.get_contents) + SCons.Node.FS.File.get_csig = ninja_csig(SCons.Node.FS.File.get_csig) + SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) + SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig) + + # Replace false Compiling* messages with a more accurate output + # + # We also use this to tag all Nodes with Builders using + # CommandActions with the final command that was used to compile + # it for passing to Ninja. If we don't inject this behavior at + # this stage in the build too much state is lost to generate the + # command at the actual ninja_builder execution time for most + # commands. + # + # We do attempt command generation again in ninja_builder if it + # hasn't been tagged and it seems to work for anything that + # doesn't represent as a non-FunctionAction during the print_func + # call. + env["PRINT_CMD_LINE_FUNC"] = ninja_print + + # This reduces unnecessary subst_list calls to add the compiler to + # the implicit dependencies of targets. Since we encode full paths + # in our generated commands we do not need these slow subst calls + # as executing the command will fail if the file is not found + # where we expect it. + env["IMPLICIT_COMMAND_DEPENDENCIES"] = False + + # Set build to no_exec, our sublcass of FunctionAction will force + # an execution for ninja_builder so this simply effects all other + # Builders. + env.SetOption("no_exec", True) + + # This makes SCons more aggressively cache MD5 signatures in the + # SConsign file. + env.SetOption("max_drift", 1) + + # The Serial job class is SIGNIFICANTLY (almost twice as) faster + # than the Parallel job class for generating Ninja files. So we + # monkey the Jobs constructor to only use the Serial Job class. + SCons.Job.Jobs.__init__ = ninja_always_serial + + # We will eventually need to overwrite TempFileMunge to make it + # handle persistent tempfiles or get an upstreamed change to add + # some configurability to it's behavior in regards to tempfiles. + # + # Set all three environment variables that Python's + # tempfile.mkstemp looks at as it behaves differently on different + # platforms and versions of Python. + os.environ["TMPDIR"] = env.Dir("$BUILD_DIR/response_files").get_abspath() + os.environ["TEMP"] = os.environ["TMPDIR"] + os.environ["TMP"] = os.environ["TMPDIR"] + if not os.path.isdir(os.environ["TMPDIR"]): + env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) + + env["TEMPFILE"] = NinjaEternalTempFile + + # Force the SConsign to be written, we benefit from SCons caching of + # implicit dependencies and conftests. Unfortunately, we have to do this + # using an atexit handler because SCons will not write the file when in a + # no_exec build. + import atexit + + atexit.register(SCons.SConsign.write) From d61f256d355d31459b416ea24b76c65590a3684d Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 5 May 2020 23:44:01 -0400 Subject: [PATCH 002/163] updated to ninja-next, added some small fixes, and added simple test --- src/engine/SCons/Tool/ninja.py | 1071 +++++++++++++++++------------- test/ninja/CC.py | 66 ++ test/ninja/ninja-fixture/bar.c | 10 + test/ninja/ninja-fixture/foo.c | 10 + test/ninja/ninja-fixture/test1.c | 3 + test/ninja/ninja-fixture/test2.C | 3 + 6 files changed, 691 insertions(+), 472 deletions(-) create mode 100644 test/ninja/CC.py create mode 100644 test/ninja/ninja-fixture/bar.c create mode 100644 test/ninja/ninja-fixture/foo.c create mode 100644 test/ninja/ninja-fixture/test1.c create mode 100644 test/ninja/ninja-fixture/test2.C diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index b34759e401..d1cbafa495 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -1,16 +1,25 @@ -# Copyright 2019 MongoDB Inc. +# Copyright 2020 MongoDB Inc. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: # -# http://www.apache.org/licenses/LICENSE-2.0 +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + """Generate build.ninja files from SCons aliases.""" import sys @@ -18,16 +27,19 @@ import importlib import io import shutil +import shlex +import subprocess -from threading import Lock +from glob import glob from os.path import join as joinpath from os.path import splitext import SCons from SCons.Action import _string_from_cmd_list, get_default_ENV -from SCons.Util import is_String, is_List -from SCons.Script import COMMAND_LINE_TARGETS, LOADED_SCONSCRIPTS +from SCons.Util import is_List, flatten_sequence +from SCons.Script import COMMAND_LINE_TARGETS +NINJA_STATE = None NINJA_SYNTAX = "NINJA_SYNTAX" NINJA_RULES = "__NINJA_CUSTOM_RULES" NINJA_POOLS = "__NINJA_CUSTOM_POOLS" @@ -35,7 +47,6 @@ NINJA_BUILD = "NINJA_BUILD" NINJA_WHEREIS_MEMO = {} NINJA_STAT_MEMO = {} -MEMO_LOCK = Lock() __NINJA_RULE_MAPPING = {} @@ -51,11 +62,43 @@ def _install_action_function(_env, node): return { "outputs": get_outputs(node), "rule": "INSTALL", - "pool": "install_pool", "inputs": [get_path(src_file(s)) for s in node.sources], "implicit": get_dependencies(node), } +def _mkdir_action_function(env, node): + return { + "outputs": get_outputs(node), + "rule": "CMD", + # implicit explicitly omitted, we translate these so they can be + # used by anything that depends on these but commonly this is + # hit with a node that will depend on all of the fake + # srcnode's that SCons will never give us a rule for leading + # to an invalid ninja file. + "variables": { + # On Windows mkdir "-p" is always on + "cmd": "{mkdir} $out".format( + mkdir="mkdir" if env["PLATFORM"] == "win32" else "mkdir -p", + ), + }, + } + +def _copy_action_function(env, node): + return { + "outputs": get_outputs(node), + "inputs": [get_path(src_file(s)) for s in node.sources], + "rule": "CMD", + # implicit explicitly omitted, we translate these so they can be + # used by anything that depends on these but commonly this is + # hit with a node that will depend on all of the fake + # srcnode's that SCons will never give us a rule for leading + # to an invalid ninja file. + "variables": { + # On Windows mkdir "-p" is always on + "cmd": "$COPY $in $out", + }, + } + def _lib_symlink_action_function(_env, node): """Create shared object symlinks if any need to be created""" @@ -87,7 +130,13 @@ def is_valid_dependent_node(node): check because some nodes (like src files) won't have builders but are valid implicit dependencies. """ - return not isinstance(node, SCons.Node.Alias.Alias) or node.children() + if isinstance(node, SCons.Node.Alias.Alias): + return node.children() + + if not node.env: + return True + + return not node.env.get("NINJA_SKIP") def alias_to_ninja_build(node): @@ -96,13 +145,26 @@ def alias_to_ninja_build(node): "outputs": get_outputs(node), "rule": "phony", "implicit": [ - get_path(n) for n in node.children() if is_valid_dependent_node(n) + get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) ], } -def get_dependencies(node): +def get_order_only(node): + """Return a list of order only dependencies for node.""" + if node.prerequisites is None: + return [] + return [get_path(src_file(prereq)) for prereq in node.prerequisites] + + +def get_dependencies(node, skip_sources=False): """Return a list of dependencies for node.""" + if skip_sources: + return [ + get_path(src_file(child)) + for child in node.children() + if child not in node.sources + ] return [get_path(src_file(child)) for child in node.children()] @@ -130,6 +192,7 @@ def get_outputs(node): outputs = [node] outputs = [get_path(o) for o in outputs] + return outputs @@ -147,18 +210,19 @@ def __init__(self, env): "SharedFlagChecker": ninja_noop, # The install builder is implemented as a function action. "installFunc": _install_action_function, + "MkdirFunc": _mkdir_action_function, "LibSymlinksActionFunction": _lib_symlink_action_function, + "Copy" : _copy_action_function } - self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) + self.loaded_custom = False # pylint: disable=too-many-return-statements def action_to_ninja_build(self, node, action=None): """Generate build arguments dictionary for node.""" - # Use False since None is a valid value for this Attribute - build = getattr(node.attributes, NINJA_BUILD, False) - if build is not False: - return build + if not self.loaded_custom: + self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) + self.loaded_custom = True if node.builder is None: return None @@ -166,51 +230,60 @@ def action_to_ninja_build(self, node, action=None): if action is None: action = node.builder.action + if node.env and node.env.get("NINJA_SKIP"): + return None + + build = {} + # Ideally this should never happen, and we do try to filter # Ninja builders out of being sources of ninja builders but I # can't fix every DAG problem so we just skip ninja_builders # if we find one if node.builder == self.env["BUILDERS"]["Ninja"]: - return None - - if isinstance(action, SCons.Action.FunctionAction): - return self.handle_func_action(node, action) - - if isinstance(action, SCons.Action.LazyAction): + build = None + elif isinstance(action, SCons.Action.FunctionAction): + build = self.handle_func_action(node, action) + elif isinstance(action, SCons.Action.LazyAction): # pylint: disable=protected-access action = action._generate_cache(node.env if node.env else self.env) - return self.action_to_ninja_build(node, action=action) - - if isinstance(action, SCons.Action.ListAction): - return self.handle_list_action(node, action) + build = self.action_to_ninja_build(node, action=action) + elif isinstance(action, SCons.Action.ListAction): + build = self.handle_list_action(node, action) + elif isinstance(action, COMMAND_TYPES): + build = get_command(node.env if node.env else self.env, node, action) + else: + raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) - if isinstance(action, COMMAND_TYPES): - return get_command(node.env if node.env else self.env, node, action) + if build is not None: + build["order_only"] = get_order_only(node) - # Return the node to indicate that SCons is required - return { - "rule": "SCONS", - "outputs": get_outputs(node), - "implicit": get_dependencies(node), - } + return build def handle_func_action(self, node, action): """Determine how to handle the function action.""" name = action.function_name() # This is the name given by the Subst/Textfile builders. So return the - # node to indicate that SCons is required + # node to indicate that SCons is required. We skip sources here because + # dependencies don't really matter when we're going to shove these to + # the bottom of ninja's DAG anyway and Textfile builders can have text + # content as their source which doesn't work as an implicit dep in + # ninja. if name == "_action": return { "rule": "TEMPLATE", "outputs": get_outputs(node), - "implicit": get_dependencies(node), + "implicit": get_dependencies(node, skip_sources=True), } - handler = self.func_handlers.get(name, None) if handler is not None: return handler(node.env if node.env else self.env, node) + elif name == "ActionCaller": + action_to_call = str(action).split('(')[0].strip() + handler = self.func_handlers.get(action_to_call, None) + if handler is not None: + return handler(node.env if node.env else self.env, node) - print( + raise Exception( "Found unhandled function action {}, " " generating scons command to build\n" "Note: this is less efficient than Ninja," @@ -218,48 +291,17 @@ def handle_func_action(self, node, action): " this function using NinjaRegisterFunctionHandler".format(name) ) - return { - "rule": "SCONS", - "outputs": get_outputs(node), - "implicit": get_dependencies(node), - } - # pylint: disable=too-many-branches def handle_list_action(self, node, action): - """ - Attempt to translate list actions to Ninja. - - List actions are tricky to move to ninja. First we translate - each individual action in the action list to a Ninja - build. Then we process the resulting ninja builds to see if - they are all the same ninja rule. If they are not all the same - rule we cannot make them a single resulting ninja build, so - instead we make them a single SCons invocation to build all of - the targets. - - If they are all the same rule and the rule is CMD we attempt - to combine the cmdlines together using ' && ' which we then - combine into a single ninja build. - - If they are all phony targets we simple combine the outputs - and dependencies. - - If they are all INSTALL rules we combine all of the sources - and outputs. - - If they are all SCONS rules we do the same as if they are not - the same rule and make a build that will use SCons to generate - them. - - If they're all one rule and None of the above rules we throw an Exception. - """ - + """TODO write this comment""" results = [ self.action_to_ninja_build(node, action=act) for act in action.list if act is not None ] - results = [result for result in results if result is not None] + results = [ + result for result in results if result is not None and result["outputs"] + ] if not results: return None @@ -268,28 +310,7 @@ def handle_list_action(self, node, action): return results[0] all_outputs = list({output for build in results for output in build["outputs"]}) - # If we have no outputs we're done - if not all_outputs: - return None - - # Used to verify if all rules are the same - all_one_rule = len( - [ - r - for r in results - if isinstance(r, dict) and r["rule"] == results[0]["rule"] - ] - ) == len(results) - dependencies = get_dependencies(node) - - if not all_one_rule: - # If they aren't all the same rule use scons to generate these - # outputs. At this time nothing hits this case. - return { - "outputs": all_outputs, - "rule": "SCONS", - "implicit": dependencies, - } + dependencies = list({dep for build in results for dep in build["implicit"]}) if results[0]["rule"] == "CMD": cmdline = "" @@ -322,7 +343,10 @@ def handle_list_action(self, node, action): ninja_build = { "outputs": all_outputs, "rule": "CMD", - "variables": {"cmd": cmdline}, + "variables": { + "cmd": cmdline, + "env": get_command_env(node.env if node.env else self.env), + }, "implicit": dependencies, } @@ -342,18 +366,10 @@ def handle_list_action(self, node, action): return { "outputs": all_outputs, "rule": "INSTALL", - "pool": "install_pool", "inputs": [get_path(src_file(s)) for s in node.sources], "implicit": dependencies, } - elif results[0]["rule"] == "SCONS": - return { - "outputs": all_outputs, - "rule": "SCONS", - "inputs": dependencies, - } - raise Exception("Unhandled list action with rule: " + results[0]["rule"]) @@ -369,7 +385,7 @@ def __init__(self, env, writer_class): self.generated_suffixes = env.get("NINJA_GENERATED_SOURCE_SUFFIXES", []) # List of generated builds that will be written at a later stage - self.builds = list() + self.builds = dict() # List of targets for which we have generated a build. This # allows us to take multiple Alias nodes as sources and to not @@ -383,7 +399,7 @@ def __init__(self, env, writer_class): escape = env.get("ESCAPE", lambda x: x) self.variables = { - "COPY": "cmd.exe /c copy" if sys.platform == "win32" else "cp", + "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format( sys.executable, " ".join( @@ -402,16 +418,46 @@ def __init__(self, env, writer_class): self.rules = { "CMD": { - "command": "cmd /c $cmd" if sys.platform == "win32" else "$cmd", + "command": "cmd /c $env$cmd $in $out" if sys.platform == "win32" else "$env$cmd $in $out", + "description": "Building $out", + "pool": "local_pool", + }, + "GENERATED_CMD": { + "command": "cmd /c $env$cmd" if sys.platform == "win32" else "$env$cmd", "description": "Building $out", + "pool": "local_pool", }, # We add the deps processing variables to this below. We - # don't pipe this through cmd.exe on Windows because we + # don't pipe these through cmd.exe on Windows because we # use this to generate a compile_commands.json database # which can't use the shell command as it's compile - # command. This does mean that we assume anything using - # CMD_W_DEPS is a straight up compile which is true today. - "CMD_W_DEPS": {"command": "$cmd", "description": "Building $out"}, + # command. + "CC": { + "command": "$env$CC @$out.rsp", + "description": "Compiling $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + }, + "CXX": { + "command": "$env$CXX @$out.rsp", + "description": "Compiling $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + }, + "LINK": { + "command": "$env$LINK @$out.rsp", + "description": "Linking $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + "pool": "local_pool", + }, + "AR": { + "command": "$env$AR @$out.rsp", + "description": "Archiving $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + "pool": "local_pool", + }, "SYMLINK": { "command": ( "cmd /c mklink $out $in" @@ -423,6 +469,7 @@ def __init__(self, env, writer_class): "INSTALL": { "command": "$COPY $in $out", "description": "Install $out", + "pool": "install_pool", # On Windows cmd.exe /c copy does not always correctly # update the timestamp on the output file. This leads # to a stuck constant timestamp in the Ninja database @@ -479,65 +526,43 @@ def __init__(self, env, writer_class): } self.pools = { + "local_pool": self.env.GetOption("num_jobs"), "install_pool": self.env.GetOption("num_jobs") / 2, "scons_pool": 1, } - if env["PLATFORM"] == "win32": - self.rules["CMD_W_DEPS"]["deps"] = "msvc" - else: - self.rules["CMD_W_DEPS"]["deps"] = "gcc" - self.rules["CMD_W_DEPS"]["depfile"] = "$out.d" - - self.rules.update(env.get(NINJA_RULES, {})) - self.pools.update(env.get(NINJA_POOLS, {})) - - def generate_builds(self, node): - """Generate a ninja build rule for node and it's children.""" - # Filter out nodes with no builder. They are likely source files - # and so no work needs to be done, it will be used in the - # generation for some real target. - # - # Note that all nodes have a builder attribute but it is sometimes - # set to None. So we cannot use a simpler hasattr check here. - if getattr(node, "builder", None) is None: - return + for rule in ["CC", "CXX"]: + if env["PLATFORM"] == "win32": + self.rules[rule]["deps"] = "msvc" + else: + self.rules[rule]["deps"] = "gcc" + self.rules[rule]["depfile"] = "$out.d" - stack = [[node]] - while stack: - frame = stack.pop() - for child in frame: - outputs = set(get_outputs(child)) - # Check if all the outputs are in self.built, if they - # are we've already seen this node and it's children. - if not outputs.isdisjoint(self.built): - continue + def add_build(self, node): + if not node.has_builder(): + return False - self.built = self.built.union(outputs) - stack.append(child.children()) - - if isinstance(child, SCons.Node.Alias.Alias): - build = alias_to_ninja_build(child) - elif node.builder is not None: - # Use False since None is a valid value for this attribute - build = getattr(child.attributes, NINJA_BUILD, False) - if build is False: - build = self.translator.action_to_ninja_build(child) - setattr(child.attributes, NINJA_BUILD, build) - else: - build = None + if isinstance(node, SCons.Node.Alias.Alias): + build = alias_to_ninja_build(node) + else: + build = self.translator.action_to_ninja_build(node) - # Some things are unbuild-able or need not be built in Ninja - if build is None or build == 0: - continue + # Some things are unbuild-able or need not be built in Ninja + if build is None: + return False - self.builds.append(build) + node_string = str(node) + if node_string in self.builds: + raise Exception("Node {} added to ninja build state more than once".format(node_string)) + self.builds[node_string] = build + self.built.update(build["outputs"]) + return True def is_generated_source(self, output): """Check if output ends with a known generated suffix.""" _, suffix = splitext(output) return suffix in self.generated_suffixes - + def has_generated_sources(self, output): """ Determine if output indicates this is a generated header file. @@ -548,7 +573,7 @@ def has_generated_sources(self, output): return False # pylint: disable=too-many-branches,too-many-locals - def generate(self, ninja_file, fallback_default_target=None): + def generate(self, ninja_file): """ Generate the build.ninja. @@ -557,6 +582,9 @@ def generate(self, ninja_file, fallback_default_target=None): if self.__generated: return + self.rules.update(self.env.get(NINJA_RULES, {})) + self.pools.update(self.env.get(NINJA_POOLS, {})) + content = io.StringIO() ninja = self.writer_class(content, width=100) @@ -571,10 +599,10 @@ def generate(self, ninja_file, fallback_default_target=None): for rule, kwargs in self.rules.items(): ninja.rule(rule, **kwargs) - generated_source_files = { + generated_source_files = sorted({ output # First find builds which have header files in their outputs. - for build in self.builds + for build in self.builds.values() if self.has_generated_sources(build["outputs"]) for output in build["outputs"] # Collect only the header files from the builds with them @@ -583,25 +611,24 @@ def generate(self, ninja_file, fallback_default_target=None): # here we need to filter so we only have the headers and # not the other outputs. if self.is_generated_source(output) - } + }) if generated_source_files: ninja.build( outputs="_generated_sources", rule="phony", - implicit=list(generated_source_files), + implicit=generated_source_files ) template_builders = [] - for build in self.builds: + for build in [self.builds[key] for key in sorted(self.builds.keys())]: if build["rule"] == "TEMPLATE": template_builders.append(build) continue - implicit = build.get("implicit", []) - implicit.append(ninja_file) - build["implicit"] = implicit + if "implicit" in build: + build["implicit"].sort() # Don't make generated sources depend on each other. We # have to check that none of the outputs are generated @@ -612,7 +639,7 @@ def generate(self, ninja_file, fallback_default_target=None): generated_source_files and not build["rule"] == "INSTALL" and set(build["outputs"]).isdisjoint(generated_source_files) - and set(implicit).isdisjoint(generated_source_files) + and set(build.get("implicit", [])).isdisjoint(generated_source_files) ): # Make all non-generated source targets depend on @@ -621,7 +648,11 @@ def generate(self, ninja_file, fallback_default_target=None): # generated source was rebuilt. We just need to make # sure that all of these sources are generated before # other builds. - build["order_only"] = "_generated_sources" + order_only = build.get("order_only", []) + order_only.append("_generated_sources") + build["order_only"] = order_only + if "order_only" in build: + build["order_only"].sort() # When using a depfile Ninja can only have a single output # but SCons will usually have emitted an output for every @@ -637,26 +668,31 @@ def generate(self, ninja_file, fallback_default_target=None): # Some rules like 'phony' and other builtins we don't have # listed in self.rules so verify that we got a result # before trying to check if it has a deps key. - if rule is not None and rule.get("deps"): - - # Anything using deps in Ninja can only have a single - # output, but we may have a build which actually - # produces multiple outputs which other targets can - # depend on. Here we slice up the outputs so we have a - # single output which we will use for the "real" - # builder and multiple phony targets that match the - # file names of the remaining outputs. This way any - # build can depend on any output from any build. - first_output, remaining_outputs = build["outputs"][0], build["outputs"][1:] + # + # Anything using deps or rspfile in Ninja can only have a single + # output, but we may have a build which actually produces + # multiple outputs which other targets can depend on. Here we + # slice up the outputs so we have a single output which we will + # use for the "real" builder and multiple phony targets that + # match the file names of the remaining outputs. This way any + # build can depend on any output from any build. + build["outputs"].sort() + if rule is not None and (rule.get("deps") or rule.get("rspfile")): + first_output, remaining_outputs = ( + build["outputs"][0], + build["outputs"][1:], + ) + if remaining_outputs: ninja.build( - outputs=remaining_outputs, - rule="phony", - implicit=first_output, + outputs=remaining_outputs, rule="phony", implicit=first_output, ) build["outputs"] = first_output + if "inputs" in build: + build["inputs"].sort() + ninja.build(**build) template_builds = dict() @@ -682,37 +718,37 @@ def generate(self, ninja_file, fallback_default_target=None): if template_builds.get("outputs", []): ninja.build(**template_builds) - # Here to teach the ninja file how to regenerate itself. We'll - # never see ourselves in the DAG walk so we can't rely on - # action_to_ninja_build to generate this rule + # We have to glob the SCons files here to teach the ninja file + # how to regenerate itself. We'll never see ourselves in the + # DAG walk so we can't rely on action_to_ninja_build to + # generate this rule even though SCons should know we're + # dependent on SCons files. + # + # TODO: We're working on getting an API into SCons that will + # allow us to query the actual SConscripts used. Right now + # this glob method has deficiencies like skipping + # jstests/SConscript and being specific to the MongoDB + # repository layout. ninja.build( - ninja_file, + self.env.File(ninja_file).path, rule="REGENERATE", implicit=[ - self.env.File("#SConstruct").get_abspath(), - os.path.abspath(__file__), + self.env.File("#SConstruct").path, + __file__, ] - + LOADED_SCONSCRIPTS, + + sorted(glob("src/**/SConscript", recursive=True)), ) - ninja.build( - "scons-invocation", - rule="CMD", - pool="console", - variables={"cmd": "echo $SCONS_INVOCATION_W_TARGETS"}, - ) - - # Note the use of CMD_W_DEPS below. CMD_W_DEPS are always - # compile commands in this generator. If we ever change the - # name/s of the rules that include compile commands - # (i.e. something like CC/CXX) we will need to update this - # build to reflect that complete list. + # If we ever change the name/s of the rules that include + # compile commands (i.e. something like CC) we will need to + # update this build to reflect that complete list. ninja.build( "compile_commands.json", rule="CMD", pool="console", + implicit=[ninja_file], variables={ - "cmd": "ninja -f {} -t compdb CMD_W_DEPS > compile_commands.json".format( + "cmd": "ninja -f {} -t compdb CC CXX > compile_commands.json".format( ninja_file ) }, @@ -736,11 +772,6 @@ def generate(self, ninja_file, fallback_default_target=None): if scons_default_targets: ninja.default(" ".join(scons_default_targets)) - # If not then set the default to the fallback_default_target we were given. - # Otherwise we won't create a default ninja target. - elif fallback_default_target is not None: - ninja.default(fallback_default_target) - with open(ninja_file, "w") as build_ninja: build_ninja.write(content.getvalue()) @@ -776,9 +807,158 @@ def src_file(node): return get_path(node) -# TODO: Make the Rules smarter. Instead of just using a "cmd" rule -# everywhere we should be smarter about generating CC, CXX, LINK, -# etc. rules +def get_comstr(env, action, targets, sources): + """Get the un-substituted string for action.""" + # Despite being having "list" in it's name this member is not + # actually a list. It's the pre-subst'd string of the command. We + # use it to determine if the command we're about to generate needs + # to use a custom Ninja rule. By default this redirects CC, CXX, + # AR, SHLINK, and LINK commands to their respective rules but the + # user can inject custom Ninja rules and tie them to commands by + # using their pre-subst'd string. + if hasattr(action, "process"): + return action.cmd_list + + return action.genstring(targets, sources, env) + + +def get_command_env(env): + """ + Return a string that sets the enrivonment for any environment variables that + differ between the OS environment and the SCons command ENV. + + It will be compatible with the default shell of the operating system. + """ + try: + return env["NINJA_ENV_VAR_CACHE"] + except KeyError: + pass + + # Scan the ENV looking for any keys which do not exist in + # os.environ or differ from it. We assume if it's a new or + # differing key from the process environment then it's + # important to pass down to commands in the Ninja file. + ENV = get_default_ENV(env) + scons_specified_env = { + key: value + for key, value in ENV.items() + if key not in os.environ or os.environ.get(key, None) != value + } + + windows = env["PLATFORM"] == "win32" + command_env = "" + for key, value in scons_specified_env.items(): + # Ensure that the ENV values are all strings: + if is_List(value): + # If the value is a list, then we assume it is a + # path list, because that's a pretty common list-like + # value to stick in an environment variable: + value = flatten_sequence(value) + value = joinpath(map(str, value)) + else: + # If it isn't a string or a list, then we just coerce + # it to a string, which is the proper way to handle + # Dir and File instances and will produce something + # reasonable for just about everything else: + value = str(value) + + if windows: + command_env += "set '{}={}' && ".format(key, value) + else: + command_env += "{}={} ".format(key, value) + + env["NINJA_ENV_VAR_CACHE"] = command_env + return command_env + + +def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False): + """Generate a response file command provider for rule name.""" + + # If win32 using the environment with a response file command will cause + # ninja to fail to create the response file. Additionally since these rules + # generally are not piping through cmd.exe /c any environment variables will + # make CreateProcess fail to start. + # + # On POSIX we can still set environment variables even for compile + # commands so we do so. + use_command_env = not env["PLATFORM"] == "win32" + if "$" in tool: + tool_is_dynamic = True + + def get_response_file_command(env, node, action, targets, sources, executor=None): + if hasattr(action, "process"): + cmd_list, _, _ = action.process(targets, sources, env, executor=executor) + cmd_list = [str(c).replace("$", "$$") for c in cmd_list[0]] + else: + command = generate_command( + env, node, action, targets, sources, executor=executor + ) + cmd_list = shlex.split(command) + + if tool_is_dynamic: + tool_command = env.subst( + tool, target=targets, source=sources, executor=executor + ) + else: + tool_command = tool + + try: + # Add 1 so we always keep the actual tool inside of cmd + tool_idx = cmd_list.index(tool_command) + 1 + except ValueError: + raise Exception( + "Could not find tool {} in {} generated from {}".format( + tool_command, cmd_list, get_comstr(env, action, targets, sources) + ) + ) + + cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] + rsp_content = " ".join(rsp_content) + + variables = {"rspc": rsp_content} + variables[rule] = cmd + if use_command_env: + variables["env"] = get_command_env(env) + return rule, variables + + return get_response_file_command + + +def generate_command(env, node, action, targets, sources, executor=None): + # Actions like CommandAction have a method called process that is + # used by SCons to generate the cmd_line they need to run. So + # check if it's a thing like CommandAction and call it if we can. + if hasattr(action, "process"): + cmd_list, _, _ = action.process(targets, sources, env, executor=executor) + cmd = _string_from_cmd_list(cmd_list[0]) + else: + # Anything else works with genstring, this is most commonly hit by + # ListActions which essentially call process on all of their + # commands and concatenate it for us. + genstring = action.genstring(targets, sources, env) + if executor is not None: + cmd = env.subst(genstring, executor=executor) + else: + cmd = env.subst(genstring, targets, sources) + + cmd = cmd.replace("\n", " && ").strip() + if cmd.endswith("&&"): + cmd = cmd[0:-2].strip() + + # Escape dollars as necessary + return cmd.replace("$", "$$") + + +def get_shell_command(env, node, action, targets, sources, executor=None): + return ( + "GENERATED_CMD", + { + "cmd": generate_command(env, node, action, targets, sources, executor=None), + "env": get_command_env(env), + }, + ) + + def get_command(env, node, action): # pylint: disable=too-many-branches """Get the command to execute for node.""" if node.env: @@ -800,121 +980,26 @@ def get_command(env, node, action): # pylint: disable=too-many-branches # Retrieve the repository file for all sources slist = [rfile(s) for s in slist] - # Get the dependencies for all targets - implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) - # Generate a real CommandAction if isinstance(action, SCons.Action.CommandGeneratorAction): # pylint: disable=protected-access action = action._generate(tlist, slist, sub_env, 1, executor=executor) - rule = "CMD" - - # Actions like CommandAction have a method called process that is - # used by SCons to generate the cmd_line they need to run. So - # check if it's a thing like CommandAction and call it if we can. - if hasattr(action, "process"): - cmd_list, _, _ = action.process(tlist, slist, sub_env, executor=executor) - - # Despite being having "list" in it's name this member is not - # actually a list. It's the pre-subst'd string of the command. We - # use it to determine if the command we generated needs to use a - # custom Ninja rule. By default this redirects CC/CXX commands to - # CMD_W_DEPS but the user can inject custom Ninja rules and tie - # them to commands by using their pre-subst'd string. - rule = __NINJA_RULE_MAPPING.get(action.cmd_list, "CMD") - - cmd = _string_from_cmd_list(cmd_list[0]) - else: - # Anything else works with genstring, this is most commonly hit by - # ListActions which essentially call process on all of their - # commands and concatenate it for us. - genstring = action.genstring(tlist, slist, sub_env) + variables = {} - # Detect if we have a custom rule for this - # "ListActionCommandAction" type thing. - rule = __NINJA_RULE_MAPPING.get(genstring, "CMD") - - if executor is not None: - cmd = sub_env.subst(genstring, executor=executor) - else: - cmd = sub_env.subst(genstring, target=tlist, source=slist) - - # Since we're only enabling Ninja for developer builds right - # now we skip all Manifest related work on Windows as it's not - # necessary. We shouldn't have gotten here but on Windows - # SCons has a ListAction which shows as a - # CommandGeneratorAction for linking. That ListAction ends - # with a FunctionAction (embedManifestExeCheck, - # embedManifestDllCheck) that simply say "does - # target[0].manifest exist?" if so execute the real command - # action underlying me, otherwise do nothing. - # - # Eventually we'll want to find a way to translate this to - # Ninja but for now, and partially because the existing Ninja - # generator does so, we just disable it all together. - cmd = cmd.replace("\n", " && ").strip() - if env["PLATFORM"] == "win32" and ( - "embedManifestExeCheck" in cmd or "embedManifestDllCheck" in cmd - ): - cmd = " && ".join(cmd.split(" && ")[0:-1]) - - if cmd.endswith("&&"): - cmd = cmd[0:-2].strip() - - outputs = get_outputs(node) - command_env = "" - windows = env["PLATFORM"] == "win32" - - # If win32 and rule == CMD_W_DEPS then we don't want to calculate - # an environment for this command. It's a compile command and - # compiledb doesn't support shell syntax on Windows. We need the - # shell syntax to use environment variables on Windows so we just - # skip this platform / rule combination to keep the compiledb - # working. - # - # On POSIX we can still set environment variables even for compile - # commands so we do so. - if not (windows and rule == "CMD_W_DEPS"): - - # Scan the ENV looking for any keys which do not exist in - # os.environ or differ from it. We assume if it's a new or - # differing key from the process environment then it's - # important to pass down to commands in the Ninja file. - ENV = get_default_ENV(sub_env) - scons_specified_env = { - key: value - for key, value in ENV.items() - if key not in os.environ or os.environ.get(key, None) != value - } + comstr = get_comstr(sub_env, action, tlist, slist) + if not comstr: + return None - for key, value in scons_specified_env.items(): - # Ensure that the ENV values are all strings: - if is_List(value): - # If the value is a list, then we assume it is a - # path list, because that's a pretty common list-like - # value to stick in an environment variable: - value = flatten_sequence(value) - value = joinpath(map(str, value)) - else: - # If it isn't a string or a list, then we just coerce - # it to a string, which is the proper way to handle - # Dir and File instances and will produce something - # reasonable for just about everything else: - value = str(value) - - if windows: - command_env += "set '{}={}' && ".format(key, value) - else: - command_env += "{}={} ".format(key, value) + provider = __NINJA_RULE_MAPPING.get(comstr, get_shell_command) + rule, variables = provider(sub_env, node, action, tlist, slist, executor=executor) - variables = {"cmd": command_env + cmd} - extra_vars = getattr(node.attributes, "NINJA_EXTRA_VARS", {}) - if extra_vars: - variables.update(extra_vars) + # Get the dependencies for all targets + implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) ninja_build = { - "outputs": outputs, + "order_only": get_order_only(node), + "outputs": get_outputs(node), "inputs": get_inputs(node), "implicit": implicit, "rule": rule, @@ -953,37 +1038,30 @@ def ninja_builder(env, target, source): # here. print("Generating:", str(target[0])) - # The environment variable NINJA_SYNTAX points to the - # ninja_syntax.py module from the ninja sources found here: - # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py - # - # This should be vendored into the build sources and it's location - # set in NINJA_SYNTAX. This code block loads the location from - # that variable, gets the absolute path to the vendored file, gets - # it's parent directory then uses importlib to import the module - # dynamically. - ninja_syntax_file = env[NINJA_SYNTAX] - if isinstance(ninja_syntax_file, str): - ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() - ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) - sys.path.append(ninja_syntax_mod_dir) - ninja_syntax_mod_name = os.path.basename(ninja_syntax_file) - ninja_syntax = importlib.import_module(ninja_syntax_mod_name.replace(".py", "")) - - suffix = env.get("NINJA_SUFFIX", "") - if suffix and not suffix[0] == ".": - suffix = "." + suffix + generated_build_ninja = target[0].get_abspath() + NINJA_STATE.generate(generated_build_ninja) + if env.get("DISABLE_AUTO_NINJA") != True: + print("Executing:", str(target[0])) - generated_build_ninja = target[0].get_abspath() + suffix - ninja_state = NinjaState(env, ninja_syntax.Writer) - - for src in source: - ninja_state.generate_builds(src) - - ninja_state.generate(generated_build_ninja, str(source[0])) - - return 0 + def execute_ninja(): + proc = subprocess.Popen( ['ninja', '-f', generated_build_ninja], + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + universal_newlines=True + ) + for stdout_line in iter(proc.stdout.readline, ""): + yield stdout_line + proc.stdout.close() + return_code = proc.wait() + if return_code: + raise subprocess.CalledProcessError(return_code, 'ninja') + + for output in execute_ninja(): + output = output.strip() + sys.stdout.write('\x1b[2K') # erase previous line + sys.stdout.write(output + "\r") + sys.stdout.flush() # pylint: disable=too-few-public-methods class AlwaysExecAction(SCons.Action.FunctionAction): @@ -994,25 +1072,6 @@ def __call__(self, *args, **kwargs): return super().__call__(*args, **kwargs) -def ninja_print(_cmd, target, _source, env): - """Tag targets with the commands to build them.""" - if target: - for tgt in target: - if ( - tgt.has_builder() - # Use 'is False' because not would still trigger on - # None's which we don't want to regenerate - and getattr(tgt.attributes, NINJA_BUILD, False) is False - and isinstance(tgt.builder.action, COMMAND_TYPES) - ): - ninja_action = get_command(env, tgt, tgt.builder.action) - setattr(tgt.attributes, NINJA_BUILD, ninja_action) - # Preload the attributes dependencies while we're still running - # multithreaded - get_dependencies(tgt) - return 0 - - def register_custom_handler(env, name, handler): """Register a custom handler for SCons function actions.""" env[NINJA_CUSTOM_HANDLERS][name] = handler @@ -1024,7 +1083,7 @@ def register_custom_rule_mapping(env, pre_subst_string, rule): __NINJA_RULE_MAPPING[pre_subst_string] = rule -def register_custom_rule(env, rule, command, description="", deps=None): +def register_custom_rule(env, rule, command, description="", deps=None, pool=None): """Allows specification of Ninja rules from inside SCons files.""" rule_obj = { "command": command, @@ -1034,6 +1093,9 @@ def register_custom_rule(env, rule, command, description="", deps=None): if deps is not None: rule_obj["deps"] = deps + if pool is not None: + rule_obj["pool"] = pool + env[NINJA_RULES][rule] = rule_obj @@ -1075,11 +1137,6 @@ def ninja_stat(_self, path): running in a no_exec build the file system state should not change. For these reasons we patch SCons.Node.FS.LocalFS.stat to use our eternal memoized dictionary. - - Since this is happening during the Node walk it's being run while - threaded, we have to protect adding to the memoized dictionary - with a threading.Lock otherwise many targets miss the memoization - due to racing. """ global NINJA_STAT_MEMO @@ -1091,9 +1148,7 @@ def ninja_stat(_self, path): except os.error: result = None - with MEMO_LOCK: - NINJA_STAT_MEMO[path] = result - + NINJA_STAT_MEMO[path] = result return result @@ -1145,71 +1200,11 @@ def ninja_always_serial(self, num, taskmaster): self.job = SCons.Job.Serial(taskmaster) -class NinjaEternalTempFile(SCons.Platform.TempFileMunge): +class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" def __call__(self, target, source, env, for_signature): - if for_signature: - return self.cmd - - node = target[0] if SCons.Util.is_List(target) else target - if node is not None: - cmdlist = getattr(node.attributes, "tempfile_cmdlist", None) - if cmdlist is not None: - return cmdlist - - cmd = super().__call__(target, source, env, for_signature) - - # If TempFileMunge.__call__ returns a string it means that no - # response file was needed. No processing required so just - # return the command. - if isinstance(cmd, str): - return cmd - - # Strip the removal commands from the command list. - # - # SCons' TempFileMunge class has some very strange - # behavior where it, as part of the command line, tries to - # delete the response file after executing the link - # command. We want to keep those response files since - # Ninja will keep using them over and over. The - # TempFileMunge class creates a cmdlist to do this, a - # common SCons convention for executing commands see: - # https://github.com/SCons/scons/blob/master/src/engine/SCons/Action.py#L949 - # - # This deletion behavior is not configurable. So we wanted - # to remove the deletion command from the command list by - # simply slicing it out here. Unfortunately for some - # strange reason TempFileMunge doesn't make the "rm" - # command it's own list element. It appends it to the - # tempfile argument to cmd[0] (which is CC/CXX) and then - # adds the tempfile again as it's own element. - # - # So we just kind of skip that middle element. Since the - # tempfile is in the command list on it's own at the end we - # can cut it out entirely. This is what I would call - # "likely to break" in future SCons updates. Hopefully it - # breaks because they start doing the right thing and not - # weirdly splitting these arguments up. For reference a - # command list that we get back from the OG TempFileMunge - # looks like this: - # - # [ - # 'g++', - # '@/mats/tempfiles/random_string.lnk\nrm', - # '/mats/tempfiles/random_string.lnk', - # ] - # - # Note the weird newline and rm command in the middle - # element and the lack of TEMPFILEPREFIX on the last - # element. - prefix = env.subst("$TEMPFILEPREFIX") - if not prefix: - prefix = "@" - - new_cmdlist = [cmd[0], prefix + cmd[-1]] - setattr(node.attributes, "tempfile_cmdlist", new_cmdlist) - return new_cmdlist + return self.cmd def _print_cmd_str(*_args, **_kwargs): """Disable this method""" @@ -1229,9 +1224,22 @@ def exists(env): return True +added = None def generate(env): """Generate the NINJA builders.""" + from SCons.Script import AddOption, GetOption + global added + if not added: + added = 1 + AddOption('--disable-auto-ninja', + dest='disable_auto_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + env["DISABLE_AUTO_NINJA"] = GetOption('disable_auto_ninja') + env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") # Add the Ninja builder. @@ -1239,6 +1247,15 @@ def generate(env): ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action) env.Append(BUILDERS={"Ninja": ninja_builder_obj}) + env["NINJA_PREFIX"] = env.get("NINJA_PREFIX", "build") + env["NINJA_SUFFIX"] = env.get("NINJA_SUFFIX", "ninja") + env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") + + ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") + ninja_file = env.Ninja(target=ninja_file_name, source=[]) + env.AlwaysBuild(ninja_file) + env.Alias("$NINJA_ALIAS_NAME", ninja_file) + # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. if env["PLATFORM"] == "win32": @@ -1246,6 +1263,11 @@ def generate(env): else: env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) + # Provide a way for custom rule authors to easily access command + # generation. + env.AddMethod(get_shell_command, "NinjaGetShellCommand") + env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider") + # Provides a way for users to handle custom FunctionActions they # want to translate to Ninja. env[NINJA_CUSTOM_HANDLERS] = {} @@ -1270,8 +1292,40 @@ def generate(env): # deleted you would get a very subtly incorrect Ninja file and # might not catch it. env.AddMethod(register_custom_rule_mapping, "NinjaRuleMapping") - env.NinjaRuleMapping("${CCCOM}", "CMD_W_DEPS") - env.NinjaRuleMapping("${CXXCOM}", "CMD_W_DEPS") + + # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks + # without relying on the SCons hacks that SCons uses by default. + if env["PLATFORM"] == "win32": + from SCons.Tool.mslink import compositeLinkAction + + if env["LINKCOM"] == compositeLinkAction: + env[ + "LINKCOM" + ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' + env[ + "SHLINKCOM" + ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' + + # Normally in SCons actions for the Program and *Library builders + # will return "${*COM}" as their pre-subst'd command line. However + # if a user in a SConscript overwrites those values via key access + # like env["LINKCOM"] = "$( $ICERUN $)" + env["LINKCOM"] then + # those actions no longer return the "bracketted" string and + # instead return something that looks more expanded. So to + # continue working even if a user has done this we map both the + # "bracketted" and semi-expanded versions. + def robust_rule_mapping(var, rule, tool): + provider = gen_get_response_file_command(env, rule, tool) + env.NinjaRuleMapping("${" + var + "}", provider) + env.NinjaRuleMapping(env[var], provider) + + robust_rule_mapping("CCCOM", "CC", "$CC") + robust_rule_mapping("SHCCCOM", "CC", "$CC") + robust_rule_mapping("CXXCOM", "CXX", "$CXX") + robust_rule_mapping("SHCXXCOM", "CXX", "$CXX") + robust_rule_mapping("LINKCOM", "LINK", "$LINK") + robust_rule_mapping("SHLINKCOM", "LINK", "$SHLINK") + robust_rule_mapping("ARCOM", "AR", "$AR") # Make SCons node walk faster by preventing unnecessary work env.Decider("timestamp-match") @@ -1281,6 +1335,21 @@ def generate(env): # dependencies to any builds that *might* use them. env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] + if env["PLATFORM"] != "win32" and env.get("RANLIBCOM"): + # There is no way to translate the ranlib list action into + # Ninja so add the s flag and disable ranlib. + # + # This is equivalent to Meson. + # https://github.com/mesonbuild/meson/blob/master/mesonbuild/linkers.py#L143 + old_arflags = str(env["ARFLAGS"]) + if "s" not in old_arflags: + old_arflags += "s" + + env["ARFLAGS"] = SCons.Util.CLVar([old_arflags]) + + # Disable running ranlib, since we added 's' above + env["RANLIBCOM"] = "" + # This is the point of no return, anything after this comment # makes changes to SCons that are irreversible and incompatible # with a normal SCons build. We return early if __NINJA_NO=1 has @@ -1304,7 +1373,9 @@ def generate(env): SCons.Node.FS.File.prepare = ninja_noop SCons.Node.FS.File.push_to_cache = ninja_noop SCons.Executor.Executor.prepare = ninja_noop + SCons.Taskmaster.Task.prepare = ninja_noop SCons.Node.FS.File.built = ninja_noop + SCons.Node.Node.visited = ninja_noop # We make lstat a no-op because it is only used for SONAME # symlinks which we're not producing. @@ -1336,20 +1407,8 @@ def generate(env): SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig) - # Replace false Compiling* messages with a more accurate output - # - # We also use this to tag all Nodes with Builders using - # CommandActions with the final command that was used to compile - # it for passing to Ninja. If we don't inject this behavior at - # this stage in the build too much state is lost to generate the - # command at the actual ninja_builder execution time for most - # commands. - # - # We do attempt command generation again in ninja_builder if it - # hasn't been tagged and it seems to work for anything that - # doesn't represent as a non-FunctionAction during the print_func - # call. - env["PRINT_CMD_LINE_FUNC"] = ninja_print + # Replace false action messages with nothing. + env["PRINT_CMD_LINE_FUNC"] = ninja_noop # This reduces unnecessary subst_list calls to add the compiler to # the implicit dependencies of targets. Since we encode full paths @@ -1358,11 +1417,6 @@ def generate(env): # where we expect it. env["IMPLICIT_COMMAND_DEPENDENCIES"] = False - # Set build to no_exec, our sublcass of FunctionAction will force - # an execution for ninja_builder so this simply effects all other - # Builders. - env.SetOption("no_exec", True) - # This makes SCons more aggressively cache MD5 signatures in the # SConsign file. env.SetOption("max_drift", 1) @@ -1372,6 +1426,84 @@ def generate(env): # monkey the Jobs constructor to only use the Serial Job class. SCons.Job.Jobs.__init__ = ninja_always_serial + # The environment variable NINJA_SYNTAX points to the + # ninja_syntax.py module from the ninja sources found here: + # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py + # + # This should be vendored into the build sources and it's location + # set in NINJA_SYNTAX. This code block loads the location from + # that variable, gets the absolute path to the vendored file, gets + # it's parent directory then uses importlib to import the module + # dynamically. + ninja_syntax_file = env[NINJA_SYNTAX] + + if os.path.exists(ninja_syntax_file): + if isinstance(ninja_syntax_file, str): + ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() + ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) + sys.path.append(ninja_syntax_mod_dir) + ninja_syntax_mod_name = os.path.basename(ninja_syntax_file).replace(".py", "") + ninja_syntax = importlib.import_module(ninja_syntax_mod_name) + else: + ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') + + global NINJA_STATE + NINJA_STATE = NinjaState(env, ninja_syntax.Writer) + + # Here we will force every builder to use an emitter which makes the ninja + # file depend on it's target. This forces the ninja file to the bottom of + # the DAG which is required so that we walk every target, and therefore add + # it to the global NINJA_STATE, before we try to write the ninja file. + def ninja_file_depends_on_all(target, source, env): + if not any("conftest" in str(t) for t in target): + env.Depends(ninja_file, target) + return target, source + + # The "Alias Builder" isn't in the BUILDERS map so we have to + # modify it directly. + SCons.Environment.AliasBuilder.emitter = ninja_file_depends_on_all + + for _, builder in env["BUILDERS"].items(): + try: + emitter = builder.emitter + if emitter is not None: + builder.emitter = SCons.Builder.ListEmitter( + [emitter, ninja_file_depends_on_all] + ) + else: + builder.emitter = ninja_file_depends_on_all + # Users can inject whatever they want into the BUILDERS + # dictionary so if the thing doesn't have an emitter we'll + # just ignore it. + except AttributeError: + pass + + # Here we monkey patch the Task.execute method to not do a bunch of + # unnecessary work. If a build is a regular builder (i.e not a conftest and + # not our own Ninja builder) then we add it to the NINJA_STATE. Otherwise we + # build it like normal. This skips all of the caching work that this method + # would normally do since we aren't pulling any of these targets from the + # cache. + # + # In the future we may be able to use this to actually cache the build.ninja + # file once we have the upstream support for referencing SConscripts as File + # nodes. + def ninja_execute(self): + global NINJA_STATE + + target = self.targets[0] + target_name = str(target) + if target_name != ninja_file_name and "conftest" not in target_name: + NINJA_STATE.add_build(target) + else: + target.build() + + SCons.Taskmaster.Task.execute = ninja_execute + + # Make needs_execute always return true instead of determining out of + # date-ness. + SCons.Script.Main.BuildTask.needs_execute = lambda x: True + # We will eventually need to overwrite TempFileMunge to make it # handle persistent tempfiles or get an upstreamed change to add # some configurability to it's behavior in regards to tempfiles. @@ -1379,18 +1511,13 @@ def generate(env): # Set all three environment variables that Python's # tempfile.mkstemp looks at as it behaves differently on different # platforms and versions of Python. - os.environ["TMPDIR"] = env.Dir("$BUILD_DIR/response_files").get_abspath() + build_dir = env.subst("$BUILD_DIR") + if build_dir == "": + build_dir = "." + os.environ["TMPDIR"] = env.Dir("{}/.response_files".format(build_dir)).get_abspath() os.environ["TEMP"] = os.environ["TMPDIR"] os.environ["TMP"] = os.environ["TMPDIR"] if not os.path.isdir(os.environ["TMPDIR"]): env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) - env["TEMPFILE"] = NinjaEternalTempFile - - # Force the SConsign to be written, we benefit from SCons caching of - # implicit dependencies and conftests. Unfortunately, we have to do this - # using an atexit handler because SCons will not write the file when in a - # no_exec build. - import atexit - - atexit.register(SCons.SConsign.write) + env["TEMPFILE"] = NinjaNoResponseFiles \ No newline at end of file diff --git a/test/ninja/CC.py b/test/ninja/CC.py new file mode 100644 index 0000000000..fe18721b9d --- /dev/null +++ b/test/ninja/CC.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import sys +import TestSCons + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') +env.Program(target = 'foo', source = 'foo.c') +""" % locals()) + +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) + +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed foo.o', + 'Removed foo', + 'Removed build.ninja']) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/ninja-fixture/bar.c b/test/ninja/ninja-fixture/bar.c new file mode 100644 index 0000000000..de1e6e50b6 --- /dev/null +++ b/test/ninja/ninja-fixture/bar.c @@ -0,0 +1,10 @@ +#include +#include + +int +main(int argc, char *argv[]) +{ + argv[argc++] = "--"; + printf("foo.c\n"); + exit (0); +} diff --git a/test/ninja/ninja-fixture/foo.c b/test/ninja/ninja-fixture/foo.c new file mode 100644 index 0000000000..de1e6e50b6 --- /dev/null +++ b/test/ninja/ninja-fixture/foo.c @@ -0,0 +1,10 @@ +#include +#include + +int +main(int argc, char *argv[]) +{ + argv[argc++] = "--"; + printf("foo.c\n"); + exit (0); +} diff --git a/test/ninja/ninja-fixture/test1.c b/test/ninja/ninja-fixture/test1.c new file mode 100644 index 0000000000..7535b0aa56 --- /dev/null +++ b/test/ninja/ninja-fixture/test1.c @@ -0,0 +1,3 @@ +This is a .c file. +/*cc*/ +/*link*/ diff --git a/test/ninja/ninja-fixture/test2.C b/test/ninja/ninja-fixture/test2.C new file mode 100644 index 0000000000..a1ee9e32b9 --- /dev/null +++ b/test/ninja/ninja-fixture/test2.C @@ -0,0 +1,3 @@ +This is a .C file. +/*cc*/ +/*link*/ From d6ea93a6ff3bf546e940049c8e583593ae52770e Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 6 May 2020 13:40:11 -0400 Subject: [PATCH 003/163] added some more test and update ninja tool to handle commands --- src/engine/SCons/Tool/ninja.py | 15 +++- test/ninja/copy_function_command.py | 81 ++++++++++++++++++++ test/ninja/{CC.py => generate_and_build.py} | 15 +++- test/ninja/shell_command.py | 83 +++++++++++++++++++++ 4 files changed, 191 insertions(+), 3 deletions(-) create mode 100644 test/ninja/copy_function_command.py rename test/ninja/{CC.py => generate_and_build.py} (85%) create mode 100644 test/ninja/shell_command.py diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index d1cbafa495..e0129d8bcd 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -692,7 +692,7 @@ def generate(self, ninja_file): if "inputs" in build: build["inputs"].sort() - + ninja.build(**build) template_builds = dict() @@ -990,7 +990,7 @@ def get_command(env, node, action): # pylint: disable=too-many-branches comstr = get_comstr(sub_env, action, tlist, slist) if not comstr: return None - + provider = __NINJA_RULE_MAPPING.get(comstr, get_shell_command) rule, variables = provider(sub_env, node, action, tlist, slist, executor=executor) @@ -1478,6 +1478,17 @@ def ninja_file_depends_on_all(target, source, env): except AttributeError: pass + # We will subvert the normal Command to make sure all targets generated + # from commands will be linked to the ninja file + SconsCommand = SCons.Environment.Environment.Command + + def NinjaCommand(self, target, source, action, **kw): + targets = SconsCommand(env, target, source, action, **kw) + env.Depends(ninja_file, targets) + return targets + + SCons.Environment.Environment.Command = NinjaCommand + # Here we monkey patch the Task.execute method to not do a bunch of # unnecessary work. If a build is a regular builder (i.e not a conftest and # not our own Ninja builder) then we add it to the NINJA_STATE. Otherwise we diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py new file mode 100644 index 0000000000..f86f717e6f --- /dev/null +++ b/test/ninja/copy_function_command.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import sys +import TestSCons + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') +env.Command('foo2.c', ['foo.c'], Copy('$TARGET','$SOURCE')) +env.Program(target = 'foo', source = 'foo2.c') +""") + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed foo2.o', + 'Removed foo2.c', + 'Removed foo', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) + +# run ninja independently +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/CC.py b/test/ninja/generate_and_build.py similarity index 85% rename from test/ninja/CC.py rename to test/ninja/generate_and_build.py index fe18721b9d..fd111a29be 100644 --- a/test/ninja/CC.py +++ b/test/ninja/generate_and_build.py @@ -35,28 +35,41 @@ test.dir_fixture('ninja-fixture') +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + test.write('SConstruct', """ env = Environment() env.Tool('ninja') env.Program(target = 'foo', source = 'foo.c') -""" % locals()) +""") +# generate simple build test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +# clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo.o', 'Removed foo', 'Removed build.ninja']) + +# only generate the ninja file test.run(arguments='--disable-auto-ninja', stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_not_contain_any_line(test.stdout(), ['Executing: build.ninja']) +# run ninja independently +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) + test.pass_test() # Local Variables: diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py new file mode 100644 index 0000000000..fd0e35f8de --- /dev/null +++ b/test/ninja/shell_command.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import sys +import TestSCons + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') +env.Program(target = 'foo', source = 'foo.c') +env.Command('foo.out', ['foo'], './foo > foo.out') +""") + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_match('foo.out', 'foo.c' + os.linesep) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed foo.o', + 'Removed foo', + 'Removed foo.out', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) + +# run ninja independently +test.run(program = ninja, stdout=None) +test.must_match('foo.out', 'foo.c' + os.linesep) + + + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: From 76437269647a1a58c57c16d6008f2e2605653b44 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 6 May 2020 17:23:53 -0400 Subject: [PATCH 004/163] update to pass import.py test and support multiple environments --- src/engine/SCons/Tool/ninja.py | 41 ++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index e0129d8bcd..63ba243c61 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -274,6 +274,9 @@ def handle_func_action(self, node, action): "outputs": get_outputs(node), "implicit": get_dependencies(node, skip_sources=True), } + if name == 'ninja_builder': + return None + handler = self.func_handlers.get(name, None) if handler is not None: return handler(node.env if node.env else self.env, node) @@ -377,8 +380,9 @@ def handle_list_action(self, node, action): class NinjaState: """Maintains state of Ninja build system as it's translated from SCons.""" - def __init__(self, env, writer_class): + def __init__(self, env, ninja_file, writer_class): self.env = env + self.ninja_file = ninja_file self.writer_class = writer_class self.__generated = False self.translator = SConsToNinjaTranslator(env) @@ -573,7 +577,7 @@ def has_generated_sources(self, output): return False # pylint: disable=too-many-branches,too-many-locals - def generate(self, ninja_file): + def generate(self): """ Generate the build.ninja. @@ -730,7 +734,7 @@ def generate(self, ninja_file): # jstests/SConscript and being specific to the MongoDB # repository layout. ninja.build( - self.env.File(ninja_file).path, + self.ninja_file.path, rule="REGENERATE", implicit=[ self.env.File("#SConstruct").path, @@ -746,10 +750,10 @@ def generate(self, ninja_file): "compile_commands.json", rule="CMD", pool="console", - implicit=[ninja_file], + implicit=[str(self.ninja_file)], variables={ "cmd": "ninja -f {} -t compdb CC CXX > compile_commands.json".format( - ninja_file + str(self.ninja_file) ) }, ) @@ -772,7 +776,7 @@ def generate(self, ninja_file): if scons_default_targets: ninja.default(" ".join(scons_default_targets)) - with open(ninja_file, "w") as build_ninja: + with open(str(self.ninja_file), "w") as build_ninja: build_ninja.write(content.getvalue()) self.__generated = True @@ -1039,7 +1043,7 @@ def ninja_builder(env, target, source): print("Generating:", str(target[0])) generated_build_ninja = target[0].get_abspath() - NINJA_STATE.generate(generated_build_ninja) + NINJA_STATE.generate() if env.get("DISABLE_AUTO_NINJA") != True: print("Executing:", str(target[0])) @@ -1240,6 +1244,7 @@ def generate(env): help='Disable ninja automatically building after scons') env["DISABLE_AUTO_NINJA"] = GetOption('disable_auto_ninja') + global NINJA_STATE env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") # Add the Ninja builder. @@ -1252,10 +1257,18 @@ def generate(env): env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") - ninja_file = env.Ninja(target=ninja_file_name, source=[]) - env.AlwaysBuild(ninja_file) - env.Alias("$NINJA_ALIAS_NAME", ninja_file) - + # here we allow multiple environments to construct rules and builds + # into the same ninja file + if NINJA_STATE is None: + ninja_file = env.Ninja(target=ninja_file_name, source=[]) + env.AlwaysBuild(ninja_file) + env.Alias("$NINJA_ALIAS_NAME", ninja_file) + else: + if str(NINJA_STATE.ninja_file) != ninja_file_name: + raise Exception("Generating multiple ninja files not supported.") + else: + ninja_file = [NINJA_STATE.ninja_file] + # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. if env["PLATFORM"] == "win32": @@ -1317,7 +1330,7 @@ def generate(env): def robust_rule_mapping(var, rule, tool): provider = gen_get_response_file_command(env, rule, tool) env.NinjaRuleMapping("${" + var + "}", provider) - env.NinjaRuleMapping(env[var], provider) + env.NinjaRuleMapping(env.get(var, None), provider) robust_rule_mapping("CCCOM", "CC", "$CC") robust_rule_mapping("SHCCCOM", "CC", "$CC") @@ -1447,8 +1460,8 @@ def robust_rule_mapping(var, rule, tool): else: ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') - global NINJA_STATE - NINJA_STATE = NinjaState(env, ninja_syntax.Writer) + if NINJA_STATE is None: + NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) # Here we will force every builder to use an emitter which makes the ninja # file depend on it's target. This forces the ninja file to the bottom of From d7626f28eadfcf4b6ec4a23f51fe37617bb68ff5 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 7 May 2020 00:25:15 -0400 Subject: [PATCH 005/163] added more test, including ninja speed test --- test/ninja/build_libraries.py | 92 +++++++++++ test/ninja/generate_source.py | 102 ++++++++++++ test/ninja/iterative_speedup.py | 226 +++++++++++++++++++++++++++ test/ninja/multi_env.py | 89 +++++++++++ test/ninja/ninja-fixture/bar.c | 2 +- test/ninja/ninja-fixture/test1.c | 13 +- test/ninja/ninja-fixture/test2.C | 3 - test/ninja/ninja-fixture/test_impl.c | 8 + 8 files changed, 528 insertions(+), 7 deletions(-) create mode 100644 test/ninja/build_libraries.py create mode 100644 test/ninja/generate_source.py create mode 100644 test/ninja/iterative_speedup.py create mode 100644 test/ninja/multi_env.py delete mode 100644 test/ninja/ninja-fixture/test2.C create mode 100644 test/ninja/ninja-fixture/test_impl.c diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py new file mode 100644 index 0000000000..662c584fe8 --- /dev/null +++ b/test/ninja/build_libraries.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import sys +import TestSCons + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') + +shared_lib = env.SharedLibrary(target = 'test_impl', source = 'test_impl.c') +env.Program(target = 'test', source = 'test1.c', LIBS=[shared_lib], LIBPATH=['.'], RPATH='.') + +static_lib = env.StaticLibrary(target = 'test_impl_static', source = 'test_impl.c') +static_obj = env.Object(target = 'test_static.o', source = 'test1.c') +env.Program(target = 'test_static', source = static_obj, LIBS=[static_lib], LIBPATH=['.']) +""") +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('test'), stdout="library_function" + os.linesep) +test.run(program = test.workpath('test_static'), stdout="library_function" + os.linesep) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed test_impl.os', + 'Removed libtest_impl.so', + 'Removed test1.o', + 'Removed test', + 'Removed test_impl.o', + 'Removed libtest_impl_static.a', + 'Removed test_static.o', + 'Removed test_static', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) + +# run ninja independently +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('test'), stdout="library_function" + os.linesep) +test.run(program = test.workpath('test_static'), stdout="library_function" + os.linesep) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py new file mode 100644 index 0000000000..8ae0a80c39 --- /dev/null +++ b/test/ninja/generate_source.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import sys +import TestSCons + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') +env.Program(target = 'generate_source', source = 'generate_source.c') +env.Command('generated_source.c', ['generate_source'], './generate_source') +env.Program(target = 'generated_source', source = 'generated_source.c') +""") + +test.write('generate_source.c', """ +#include + +int main(int argc, char *argv[]) { + FILE *fp; + + fp = fopen("generated_source.c", "w"); + fprintf(fp, "#include \\n"); + fprintf(fp, "#include \\n"); + fprintf(fp, "\\n"); + fprintf(fp, "int\\n"); + fprintf(fp, "main(int argc, char *argv[])\\n"); + fprintf(fp, "{\\n"); + fprintf(fp, " printf(\\"generated_source.c\\\\n\\");\\n"); + fprintf(fp, " exit (0);\\n"); + fprintf(fp, "}\\n"); + fclose(fp); +} +""") + +# generate simple build +test.run(stdout=None) +test.run(program = test.workpath('generated_source'), stdout="generated_source.c" + os.linesep) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed generate_source.o', + 'Removed generate_source', + 'Removed generated_source.c', + 'Removed generated_source.o', + 'Removed generated_source', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) + +# run ninja independently +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('generated_source'), stdout="generated_source.c" + os.linesep) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py new file mode 100644 index 0000000000..018ba7beda --- /dev/null +++ b/test/ninja/iterative_speedup.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import sys +import time +import random +import TestSCons + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + +test.write('source_0.c', """ +#include +#include +#include "source_0.h" + +int +print_function0() +{ + printf("main print\\n"); +} +""") + +test.write('source_0.h', """ +#include +#include + +int +print_function0(); +""") + +def get_num_cpus(): + """ + Function to get the number of CPUs the system has. + """ + # Linux, Unix and MacOS: + if hasattr(os, "sysconf"): + if 'SC_NPROCESSORS_ONLN' in os.sysconf_names: + # Linux & Unix: + ncpus = os.sysconf("SC_NPROCESSORS_ONLN") + if isinstance(ncpus, int) and ncpus > 0: + return ncpus + # OSX: + return int(os.popen("sysctl -n hw.ncpu")[1].read()) + # Windows: + if 'NUMBER_OF_PROCESSORS' in os.environ: + ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]) + if ncpus > 0: + return ncpus + # Default + return 1 + +def generate_source(parent_source, current_source): + test.write('source_{}.c'.format(current_source), """ + #include + #include + #include "source_%(parent_source)s.h" + #include "source_%(current_source)s.h" + + int + print_function%(current_source)s() + { + print_function%(parent_source)s(); + } + """ % locals()) + + test.write('source_{}.h'.format(current_source), """ + #include + #include + + int + print_function%(current_source)s(); + """ % locals()) + +def mod_source_return(test_num): + parent_source = test_num-1 + test.write('source_{}.c'.format(test_num), """ + #include + #include + #include "source_%(parent_source)s.h" + #include "source_%(test_num)s.h" + + int + print_function%(test_num)s() + { + int test = 5 + 5; + print_function%(parent_source)s(); + return test; + } + """ % locals()) + +def mod_source_orig(test_num): + parent_source = test_num-1 + test.write('source_{}.c'.format(test_num), """ + #include + #include + #include "source_%(parent_source)s.h" + #include "source_%(test_num)s.h" + + int + print_function%(test_num)s() + { + print_function%(parent_source)s(); + } + """ % locals()) + +num_source = 200 +for i in range(1, num_source+1): + generate_source(i-1, i) + +test.write('main.c', """ +#include +#include +#include "source_%(num_source)s.h" +int +main() +{ + print_function%(num_source)s(); +} +""" % locals()) + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') +sources = ['main.c'] + env.Glob('source*.c') +env.Program(target = 'print_bin', source = sources) +""") + +test.write('SConstruct_no_ninja', """ +env = Environment() +sources = ['main.c'] + env.Glob('source*.c') +env.Program(target = 'print_bin', source = sources) +""") + +tests_mods = [] +ninja_times = [] +scons_times = [] +for _ in range(10): + tests_mods += [random.randrange(1, num_source, 1)] +jobs = '-j' + str(get_num_cpus()) + +start = time.perf_counter() +test.run(arguments='--disable-auto-ninja', stdout=None) +test.run(program = ninja, arguments=[jobs], stdout=None) +stop = time.perf_counter() +ninja_times += [stop - start] +test.run(program = test.workpath('print_bin'), stdout="main print" + os.linesep) + +for test_mod in tests_mods: + mod_source_return(test_mod) + start = time.perf_counter() + test.run(program = ninja, arguments=[jobs], stdout=None) + stop = time.perf_counter() + ninja_times += [stop - start] + +for test_mod in tests_mods: + mod_source_orig(test_mod) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed build.ninja']) + +start = time.perf_counter() +test.run(arguments = ["-f", "SConstruct_no_ninja", jobs], stdout=None) +stop = time.perf_counter() +scons_times += [stop - start] +test.run(program = test.workpath('print_bin'), stdout="main print" + os.linesep) + +for test_mod in tests_mods: + mod_source_return(test_mod) + start = time.perf_counter() + test.run(arguments = ["-f", "SConstruct_no_ninja", jobs], stdout=None) + stop = time.perf_counter() + scons_times += [stop - start] + +full_build_print = True +for ninja, scons in zip(ninja_times, scons_times): + if ninja > scons: + test.fail_test() + if full_build_print: + full_build_print = False + print("Clean build {} files - SCons: {:.3f}s Ninja: {:.3f}s".format(num_source, scons, ninja)) + else: + print("Single File Rebuild - SCons: {:.3f}s Ninja: {:.3f}s".format(scons, ninja)) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py new file mode 100644 index 0000000000..3612d7b84a --- /dev/null +++ b/test/ninja/multi_env.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import sys +import TestSCons + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') +env.Program(target = 'foo', source = 'foo.c') + +env2 = Environment() +env2.Tool('ninja') +env2.Program(target = 'bar', source = 'bar.c') +""") + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +test.run(program = test.workpath('bar'), stdout="bar.c" + os.linesep) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed foo.o', + 'Removed foo', + 'Removed bar.o', + 'Removed bar', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) + +# run ninja independently +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +test.run(program = test.workpath('bar'), stdout="bar.c" + os.linesep) + + + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/ninja-fixture/bar.c b/test/ninja/ninja-fixture/bar.c index de1e6e50b6..3767857b00 100644 --- a/test/ninja/ninja-fixture/bar.c +++ b/test/ninja/ninja-fixture/bar.c @@ -5,6 +5,6 @@ int main(int argc, char *argv[]) { argv[argc++] = "--"; - printf("foo.c\n"); + printf("bar.c\n"); exit (0); } diff --git a/test/ninja/ninja-fixture/test1.c b/test/ninja/ninja-fixture/test1.c index 7535b0aa56..c53f54ac85 100644 --- a/test/ninja/ninja-fixture/test1.c +++ b/test/ninja/ninja-fixture/test1.c @@ -1,3 +1,10 @@ -This is a .c file. -/*cc*/ -/*link*/ +#include +#include + +extern int library_function(void); + +int +main(int argc, char *argv[]) +{ + library_function(); +} diff --git a/test/ninja/ninja-fixture/test2.C b/test/ninja/ninja-fixture/test2.C deleted file mode 100644 index a1ee9e32b9..0000000000 --- a/test/ninja/ninja-fixture/test2.C +++ /dev/null @@ -1,3 +0,0 @@ -This is a .C file. -/*cc*/ -/*link*/ diff --git a/test/ninja/ninja-fixture/test_impl.c b/test/ninja/ninja-fixture/test_impl.c new file mode 100644 index 0000000000..ae5effc965 --- /dev/null +++ b/test/ninja/ninja-fixture/test_impl.c @@ -0,0 +1,8 @@ +#include +#include + +int +library_function(void) +{ + printf("library_function\n"); +} From ec8c313126709a4bca17f97ee53a3619ca23ef5d Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 7 May 2020 00:31:28 -0400 Subject: [PATCH 006/163] fixed sider issues --- src/engine/SCons/Tool/ninja.py | 4 ++-- test/ninja/build_libraries.py | 1 - test/ninja/copy_function_command.py | 1 - test/ninja/generate_and_build.py | 1 - test/ninja/generate_source.py | 1 - test/ninja/iterative_speedup.py | 1 - test/ninja/multi_env.py | 1 - test/ninja/shell_command.py | 1 - 8 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 63ba243c61..bf399e2592 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -665,7 +665,7 @@ def generate(self): # files. Here we make sure that Ninja only ever sees one # target when using a depfile. It will still have a command # that will create all of the outputs but most targets don't - # depend direclty on DWO files and so this assumption is safe + # depend directly on DWO files and so this assumption is safe # to make. rule = self.rules.get(build["rule"]) @@ -1044,7 +1044,7 @@ def ninja_builder(env, target, source): generated_build_ninja = target[0].get_abspath() NINJA_STATE.generate() - if env.get("DISABLE_AUTO_NINJA") != True: + if not env.get("DISABLE_AUTO_NINJA"): print("Executing:", str(target[0])) def execute_ninja(): diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 662c584fe8..7e0ec2365a 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index f86f717e6f..a1e72b7181 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index fd111a29be..82aab5e53b 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index 8ae0a80c39..d1bfe34a6b 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index 018ba7beda..cf999d8568 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import time import random import TestSCons diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 3612d7b84a..5360fd215c 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index fd0e35f8de..b5c8323f43 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ From 6eb1818c7b9b85e4645cd87e9e78c70f35fed35d Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 8 May 2020 12:44:27 -0500 Subject: [PATCH 007/163] update tests to work on windows, added some environment support for windows and msvc --- src/engine/SCons/Tool/ninja.py | 47 +++++++++++++++++++++------- test/ninja/build_libraries.py | 41 +++++++++++++----------- test/ninja/copy_function_command.py | 10 +++--- test/ninja/generate_and_build.py | 9 ++++-- test/ninja/generate_source.py | 22 +++++++------ test/ninja/iterative_speedup.py | 19 ++++++----- test/ninja/multi_env.py | 18 +++++------ test/ninja/ninja-fixture/bar.c | 2 +- test/ninja/ninja-fixture/foo.c | 2 +- test/ninja/ninja-fixture/test1.c | 13 +++++++- test/ninja/ninja-fixture/test_impl.c | 15 +++++++-- test/ninja/shell_command.py | 20 ++++++------ 12 files changed, 142 insertions(+), 76 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index bf399e2592..10e27f3423 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -77,8 +77,8 @@ def _mkdir_action_function(env, node): # to an invalid ninja file. "variables": { # On Windows mkdir "-p" is always on - "cmd": "{mkdir} $out".format( - mkdir="mkdir" if env["PLATFORM"] == "win32" else "mkdir -p", + "cmd": "{mkdir}".format( + mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", ), }, } @@ -383,6 +383,7 @@ class NinjaState: def __init__(self, env, ninja_file, writer_class): self.env = env self.ninja_file = ninja_file + self.ninja_bin_path = '' self.writer_class = writer_class self.__generated = False self.translator = SConsToNinjaTranslator(env) @@ -752,7 +753,7 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "ninja -f {} -t compdb CC CXX > compile_commands.json".format( + "cmd": "{}/ninja -f {} -t compdb CC CXX > compile_commands.json".format(self.ninja_bin_path, str(self.ninja_file) ) }, @@ -1044,15 +1045,32 @@ def ninja_builder(env, target, source): generated_build_ninja = target[0].get_abspath() NINJA_STATE.generate() + + if env["PLATFORM"] == "win32": + # this is not great, it executes everytime + # and its doesn't consider specific node environments + # also a bit quirky to use, but usually MSVC is not + # setup system wide for command line use so this is needed + # on the standard MSVC setup, this is only needed if + # running ninja directly from a command line that hasn't + # had the environment setup (vcvarsall.bat) + # todo: hook this into a command so that it only regnerates + # the .bat if the env['ENV'] changes + with open('ninja_env.bat', 'w') as f: + for key in env['ENV']: + f.write('set {}={}\n'.format(key, env['ENV'][key])) + if not env.get("DISABLE_AUTO_NINJA"): print("Executing:", str(target[0])) def execute_ninja(): - proc = subprocess.Popen( ['ninja', '-f', generated_build_ninja], - stderr=subprocess.STDOUT, + env.AppendENVPath('PATH', NINJA_STATE.ninja_bin_path) + proc = subprocess.Popen(['ninja', '-f', generated_build_ninja], + stderr=sys.stderr, stdout=subprocess.PIPE, - universal_newlines=True + universal_newlines=True, + env=env['ENV'] ) for stdout_line in iter(proc.stdout.readline, ""): yield stdout_line @@ -1060,12 +1078,17 @@ def execute_ninja(): return_code = proc.wait() if return_code: raise subprocess.CalledProcessError(return_code, 'ninja') - + erase_previous = False for output in execute_ninja(): output = output.strip() - sys.stdout.write('\x1b[2K') # erase previous line - sys.stdout.write(output + "\r") + if erase_previous: + sys.stdout.write('\x1b[2K') # erase previous line + sys.stdout.write("\r") + else: + sys.stdout.write(os.linesep) + sys.stdout.write(output) sys.stdout.flush() + erase_previous = output.startswith('[') # pylint: disable=too-few-public-methods class AlwaysExecAction(SCons.Action.FunctionAction): @@ -1311,7 +1334,7 @@ def generate(env): if env["PLATFORM"] == "win32": from SCons.Tool.mslink import compositeLinkAction - if env["LINKCOM"] == compositeLinkAction: + if env.get("LINKCOM", None) == compositeLinkAction: env[ "LINKCOM" ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' @@ -1459,10 +1482,10 @@ def robust_rule_mapping(var, rule, tool): ninja_syntax = importlib.import_module(ninja_syntax_mod_name) else: ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') - + if NINJA_STATE is None: NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) - + NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join(ninja_syntax.__file__, os.pardir, 'data', 'bin')) # Here we will force every builder to use an emitter which makes the ninja # file depend on it's target. This forces the ninja file to the bottom of # the DAG which is required so that we walk every target, and therefore add diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 7e0ec2365a..5e491cc90b 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -39,35 +40,38 @@ if not ninja: test.skip_test("Could not find ninja in environment") +lib_suffix = '.lib' if IS_WINDOWS else '.so' +staticlib_suffix = '.lib' if IS_WINDOWS else '.a' +lib_prefix = '' if IS_WINDOWS else 'lib' + +win32 = ", 'WIN32'" if IS_WINDOWS else '' + test.write('SConstruct', """ env = Environment() env.Tool('ninja') -shared_lib = env.SharedLibrary(target = 'test_impl', source = 'test_impl.c') -env.Program(target = 'test', source = 'test1.c', LIBS=[shared_lib], LIBPATH=['.'], RPATH='.') +shared_lib = env.SharedLibrary(target = 'test_impl', source = 'test_impl.c', CPPDEFINES=['LIBRARY_BUILD'%(win32)s]) +env.Program(target = 'test', source = 'test1.c', LIBS=['test_impl'], LIBPATH=['.'], RPATH='.') -static_lib = env.StaticLibrary(target = 'test_impl_static', source = 'test_impl.c') -static_obj = env.Object(target = 'test_static.o', source = 'test1.c') +static_obj = env.Object(target = 'test_impl_static', source = 'test_impl.c') +static_lib = env.StaticLibrary(target = 'test_impl_static', source = static_obj) +static_obj = env.Object(target = 'test_static', source = 'test1.c') env.Program(target = 'test_static', source = static_obj, LIBS=[static_lib], LIBPATH=['.']) -""") +""" % locals()) # generate simple build test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('test'), stdout="library_function" + os.linesep) -test.run(program = test.workpath('test_static'), stdout="library_function" + os.linesep) +test.run(program = test.workpath('test'), stdout="library_function") +test.run(program = test.workpath('test_static'), stdout="library_function") # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ - 'Removed test_impl.os', - 'Removed libtest_impl.so', - 'Removed test1.o', - 'Removed test', - 'Removed test_impl.o', - 'Removed libtest_impl_static.a', - 'Removed test_static.o', - 'Removed test_static', + ('Removed %stest_impl' % lib_prefix) + lib_suffix, + 'Removed test' + _exe, + ('Removed %stest_impl_static' % lib_prefix) + staticlib_suffix, + 'Removed test_static' + _exe, 'Removed build.ninja']) # only generate the ninja file @@ -78,9 +82,10 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('test'), stdout="library_function" + os.linesep) -test.run(program = test.workpath('test_static'), stdout="library_function" + os.linesep) +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('test'), stdout="library_function") +test.run(program = test.workpath('test_static'), stdout="library_function") test.pass_test() diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index a1e72b7181..06991d375c 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -50,14 +51,14 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +test.run(program = test.workpath('foo'), stdout="foo.c") # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo2.o', 'Removed foo2.c', - 'Removed foo', + 'Removed foo' + _exe, 'Removed build.ninja']) # only generate the ninja file @@ -68,8 +69,9 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('foo'), stdout="foo.c") test.pass_test() diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index 82aab5e53b..904e46a3cd 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -45,11 +46,12 @@ env.Program(target = 'foo', source = 'foo.c') """) + # generate simple build test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +test.run(program = test.workpath('foo' + _exe), stdout="foo.c") # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -66,8 +68,9 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('foo' + _exe), stdout="foo.c") test.pass_test() diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index d1bfe34a6b..298d227fea 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -39,13 +40,15 @@ if not ninja: test.skip_test("Could not find ninja in environment") +shell = '' if IS_WINDOWS else './' + test.write('SConstruct', """ env = Environment() env.Tool('ninja') -env.Program(target = 'generate_source', source = 'generate_source.c') -env.Command('generated_source.c', ['generate_source'], './generate_source') +prog = env.Program(target = 'generate_source', source = 'generate_source.c') +env.Command('generated_source.c', prog, '%(shell)sgenerate_source%(_exe)s') env.Program(target = 'generated_source', source = 'generated_source.c') -""") +""" % locals()) test.write('generate_source.c', """ #include @@ -60,7 +63,7 @@ fprintf(fp, "int\\n"); fprintf(fp, "main(int argc, char *argv[])\\n"); fprintf(fp, "{\\n"); - fprintf(fp, " printf(\\"generated_source.c\\\\n\\");\\n"); + fprintf(fp, " printf(\\"generated_source.c\\");\\n"); fprintf(fp, " exit (0);\\n"); fprintf(fp, "}\\n"); fclose(fp); @@ -69,16 +72,16 @@ # generate simple build test.run(stdout=None) -test.run(program = test.workpath('generated_source'), stdout="generated_source.c" + os.linesep) +test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed generate_source.o', - 'Removed generate_source', + 'Removed generate_source' + _exe, 'Removed generated_source.c', 'Removed generated_source.o', - 'Removed generated_source', + 'Removed generated_source' + _exe, 'Removed build.ninja']) # only generate the ninja file @@ -89,8 +92,9 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('generated_source'), stdout="generated_source.c" + os.linesep) +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") test.pass_test() diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index cf999d8568..6675bf3350 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -28,6 +28,7 @@ import time import random import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -49,7 +50,8 @@ int print_function0() { - printf("main print\\n"); + printf("main print"); + return 0; } """) @@ -92,7 +94,7 @@ def generate_source(parent_source, current_source): int print_function%(current_source)s() { - print_function%(parent_source)s(); + return print_function%(parent_source)s(); } """ % locals()) @@ -132,7 +134,7 @@ def mod_source_orig(test_num): int print_function%(test_num)s() { - print_function%(parent_source)s(); + return print_function%(parent_source)s(); } """ % locals()) @@ -148,6 +150,7 @@ def mod_source_orig(test_num): main() { print_function%(num_source)s(); + exit(0); } """ % locals()) @@ -171,17 +174,19 @@ def mod_source_orig(test_num): tests_mods += [random.randrange(1, num_source, 1)] jobs = '-j' + str(get_num_cpus()) +ninja_program = [test.workpath('ninja_env.bat'), '&', ninja, jobs] if IS_WINDOWS else [ninja, jobs] + start = time.perf_counter() test.run(arguments='--disable-auto-ninja', stdout=None) -test.run(program = ninja, arguments=[jobs], stdout=None) +test.run(program = ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] -test.run(program = test.workpath('print_bin'), stdout="main print" + os.linesep) +test.run(program = test.workpath('print_bin'), stdout="main print") for test_mod in tests_mods: mod_source_return(test_mod) start = time.perf_counter() - test.run(program = ninja, arguments=[jobs], stdout=None) + test.run(program = ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] @@ -197,7 +202,7 @@ def mod_source_orig(test_num): test.run(arguments = ["-f", "SConstruct_no_ninja", jobs], stdout=None) stop = time.perf_counter() scons_times += [stop - start] -test.run(program = test.workpath('print_bin'), stdout="main print" + os.linesep) +test.run(program = test.workpath('print_bin'), stdout="main print") for test_mod in tests_mods: mod_source_return(test_mod) diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 5360fd215c..087d392b52 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -53,16 +54,16 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) -test.run(program = test.workpath('bar'), stdout="bar.c" + os.linesep) +test.run(program = test.workpath('foo' + _exe), stdout="foo.c") +test.run(program = test.workpath('bar' + _exe), stdout="bar.c") # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo.o', - 'Removed foo', + 'Removed foo' + _exe, 'Removed bar.o', - 'Removed bar', + 'Removed bar' + _exe, 'Removed build.ninja']) # only generate the ninja file @@ -73,11 +74,10 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) -test.run(program = test.workpath('bar'), stdout="bar.c" + os.linesep) - - +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('foo' + _exe), stdout="foo.c") +test.run(program = test.workpath('bar' + _exe), stdout="bar.c") test.pass_test() diff --git a/test/ninja/ninja-fixture/bar.c b/test/ninja/ninja-fixture/bar.c index 3767857b00..15b2ecc46a 100644 --- a/test/ninja/ninja-fixture/bar.c +++ b/test/ninja/ninja-fixture/bar.c @@ -5,6 +5,6 @@ int main(int argc, char *argv[]) { argv[argc++] = "--"; - printf("bar.c\n"); + printf("bar.c"); exit (0); } diff --git a/test/ninja/ninja-fixture/foo.c b/test/ninja/ninja-fixture/foo.c index de1e6e50b6..ba35c687a2 100644 --- a/test/ninja/ninja-fixture/foo.c +++ b/test/ninja/ninja-fixture/foo.c @@ -5,6 +5,6 @@ int main(int argc, char *argv[]) { argv[argc++] = "--"; - printf("foo.c\n"); + printf("foo.c"); exit (0); } diff --git a/test/ninja/ninja-fixture/test1.c b/test/ninja/ninja-fixture/test1.c index c53f54ac85..678461f508 100644 --- a/test/ninja/ninja-fixture/test1.c +++ b/test/ninja/ninja-fixture/test1.c @@ -1,10 +1,21 @@ #include #include -extern int library_function(void); +#ifdef WIN32 +#ifdef LIBRARY_BUILD +#define DLLEXPORT __declspec(dllexport) +#else +#define DLLEXPORT __declspec(dllimport) +#endif +#else +#define DLLEXPORT +#endif + +DLLEXPORT extern int library_function(void); int main(int argc, char *argv[]) { library_function(); + exit(0); } diff --git a/test/ninja/ninja-fixture/test_impl.c b/test/ninja/ninja-fixture/test_impl.c index ae5effc965..89c26ede6f 100644 --- a/test/ninja/ninja-fixture/test_impl.c +++ b/test/ninja/ninja-fixture/test_impl.c @@ -1,8 +1,19 @@ #include #include -int +#ifdef WIN32 +#ifdef LIBRARY_BUILD +#define DLLEXPORT __declspec(dllexport) +#else +#define DLLEXPORT __declspec(dllimport) +#endif +#else +#define DLLEXPORT +#endif + + +DLLEXPORT int library_function(void) { - printf("library_function\n"); + printf("library_function"); } diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index b5c8323f43..5d491645c6 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -39,24 +40,26 @@ if not ninja: test.skip_test("Could not find ninja in environment") +shell = '' if IS_WINDOWS else './' + test.write('SConstruct', """ env = Environment() env.Tool('ninja') -env.Program(target = 'foo', source = 'foo.c') -env.Command('foo.out', ['foo'], './foo > foo.out') -""") +prog = env.Program(target = 'foo', source = 'foo.c') +env.Command('foo.out', prog, '%(shell)sfoo%(_exe)s > foo.out') +""" % locals()) # generate simple build test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.must_match('foo.out', 'foo.c' + os.linesep) +test.must_match('foo.out', 'foo.c') # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo.o', - 'Removed foo', + 'Removed foo%(_exe)s' % locals(), 'Removed foo.out', 'Removed build.ninja']) @@ -68,10 +71,9 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.must_match('foo.out', 'foo.c' + os.linesep) - - +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.must_match('foo.out', 'foo.c') test.pass_test() From 234143dc0bd975e1ee9d0132a9462049a6b4ad6b Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 14 May 2020 02:52:16 -0400 Subject: [PATCH 008/163] used different method for pushing ninja file to bottom of DAG, use import ninja to get ninja_syntax and ninja bin, and added some more basic testing. --- src/engine/SCons/Tool/ninja.py | 196 +++++++++++++-------------- test/ninja/build_libraries.py | 34 +++-- test/ninja/copy_function_command.py | 33 +++-- test/ninja/generate_and_build.py | 33 +++-- test/ninja/generate_and_build_cxx.py | 106 +++++++++++++++ test/ninja/generate_source.py | 31 +++-- test/ninja/iterative_speedup.py | 22 ++- test/ninja/multi_env.py | 35 +++-- test/ninja/ninja-fixture/test2.cpp | 16 +++ test/ninja/ninja-fixture/test2.hpp | 9 ++ test/ninja/shell_command.py | 33 +++-- 11 files changed, 364 insertions(+), 184 deletions(-) create mode 100644 test/ninja/generate_and_build_cxx.py create mode 100644 test/ninja/ninja-fixture/test2.cpp create mode 100644 test/ninja/ninja-fixture/test2.hpp diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 10e27f3423..bc505b8937 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -88,13 +88,7 @@ def _copy_action_function(env, node): "outputs": get_outputs(node), "inputs": [get_path(src_file(s)) for s in node.sources], "rule": "CMD", - # implicit explicitly omitted, we translate these so they can be - # used by anything that depends on these but commonly this is - # hit with a node that will depend on all of the fake - # srcnode's that SCons will never give us a rule for leading - # to an invalid ninja file. "variables": { - # On Windows mkdir "-p" is always on "cmd": "$COPY $in $out", }, } @@ -753,7 +747,7 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "{}/ninja -f {} -t compdb CC CXX > compile_commands.json".format(self.ninja_bin_path, + "cmd": "{} -f {} -t compdb CC CXX > compile_commands.json".format(self.ninja_bin_path, str(self.ninja_file) ) }, @@ -1047,30 +1041,30 @@ def ninja_builder(env, target, source): NINJA_STATE.generate() if env["PLATFORM"] == "win32": - # this is not great, it executes everytime - # and its doesn't consider specific node environments - # also a bit quirky to use, but usually MSVC is not - # setup system wide for command line use so this is needed - # on the standard MSVC setup, this is only needed if + # this is not great, its doesn't consider specific + # node environments, which means on linux the build could + # behave differently, becuase on linux you can set the environment + # per command in the ninja file. This is only needed if # running ninja directly from a command line that hasn't # had the environment setup (vcvarsall.bat) - # todo: hook this into a command so that it only regnerates - # the .bat if the env['ENV'] changes - with open('ninja_env.bat', 'w') as f: + with open('run_ninja_env.bat', 'w') as f: for key in env['ENV']: f.write('set {}={}\n'.format(key, env['ENV'][key])) + f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) if not env.get("DISABLE_AUTO_NINJA"): - print("Executing:", str(target[0])) + cmd = [NINJA_STATE.ninja_bin_path, '-f', generated_build_ninja] + print("Executing:", str(' '.join(cmd))) + # execute the ninja build at the end of SCons, trying to + # reproduce the output like a ninja build would def execute_ninja(): - env.AppendENVPath('PATH', NINJA_STATE.ninja_bin_path) - proc = subprocess.Popen(['ninja', '-f', generated_build_ninja], + proc = subprocess.Popen(cmd, stderr=sys.stderr, stdout=subprocess.PIPE, universal_newlines=True, - env=env['ENV'] + env=env['ENV'] # ninja build items won't consider node env on win32 ) for stdout_line in iter(proc.stdout.readline, ""): yield stdout_line @@ -1078,6 +1072,7 @@ def execute_ninja(): return_code = proc.wait() if return_code: raise subprocess.CalledProcessError(return_code, 'ninja') + erase_previous = False for output in execute_ninja(): output = output.strip() @@ -1088,6 +1083,9 @@ def execute_ninja(): sys.stdout.write(os.linesep) sys.stdout.write(output) sys.stdout.flush() + # this will only erase ninjas [#/#] lines + # leaving warnings and other output, seems a bit + # prone to failure with such a simple check erase_previous = output.startswith('[') # pylint: disable=too-few-public-methods @@ -1226,6 +1224,20 @@ def ninja_always_serial(self, num, taskmaster): self.num_jobs = num self.job = SCons.Job.Serial(taskmaster) +def ninja_hack_linkcom(env): + # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks + # without relying on the SCons hacks that SCons uses by default. + if env["PLATFORM"] == "win32": + from SCons.Tool.mslink import compositeLinkAction + + if env.get("LINKCOM", None) == compositeLinkAction: + env[ + "LINKCOM" + ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' + env[ + "SHLINKCOM" + ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' + class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" @@ -1259,13 +1271,31 @@ def generate(env): global added if not added: added = 1 - AddOption('--disable-auto-ninja', - dest='disable_auto_ninja', - metavar='BOOL', - action="store_true", - default=False, - help='Disable ninja automatically building after scons') - env["DISABLE_AUTO_NINJA"] = GetOption('disable_auto_ninja') + + AddOption('--disable-execute-ninja', + dest='disable_execute_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + + AddOption('--disable-ninja', + dest='disable_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + + if GetOption('disable_ninja'): + return env + + try: + import ninja + except ImportError: + SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") + return + + env["DISABLE_AUTO_NINJA"] = GetOption('disable_execute_ninja') global NINJA_STATE env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") @@ -1288,16 +1318,15 @@ def generate(env): env.Alias("$NINJA_ALIAS_NAME", ninja_file) else: if str(NINJA_STATE.ninja_file) != ninja_file_name: - raise Exception("Generating multiple ninja files not supported.") - else: - ninja_file = [NINJA_STATE.ninja_file] + SCons.Warnings.Warning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") + ninja_file = [NINJA_STATE.ninja_file] # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. if env["PLATFORM"] == "win32": env.Append(CCFLAGS=["/showIncludes"]) else: - env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) + env.Append(CCFLAGS=["-MD", "-MF", "${TARGET}.d"]) # Provide a way for custom rule authors to easily access command # generation. @@ -1329,18 +1358,8 @@ def generate(env): # might not catch it. env.AddMethod(register_custom_rule_mapping, "NinjaRuleMapping") - # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks - # without relying on the SCons hacks that SCons uses by default. - if env["PLATFORM"] == "win32": - from SCons.Tool.mslink import compositeLinkAction - - if env.get("LINKCOM", None) == compositeLinkAction: - env[ - "LINKCOM" - ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' - env[ - "SHLINKCOM" - ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' + # on windows we need to change the link action + ninja_hack_linkcom(env) # Normally in SCons actions for the Program and *Library builders # will return "${*COM}" as their pre-subst'd command line. However @@ -1386,6 +1405,8 @@ def robust_rule_mapping(var, rule, tool): # Disable running ranlib, since we added 's' above env["RANLIBCOM"] = "" + SCons.Warnings.Warning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") + # This is the point of no return, anything after this comment # makes changes to SCons that are irreversible and incompatible # with a normal SCons build. We return early if __NINJA_NO=1 has @@ -1462,68 +1483,43 @@ def robust_rule_mapping(var, rule, tool): # monkey the Jobs constructor to only use the Serial Job class. SCons.Job.Jobs.__init__ = ninja_always_serial - # The environment variable NINJA_SYNTAX points to the - # ninja_syntax.py module from the ninja sources found here: - # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py - # - # This should be vendored into the build sources and it's location - # set in NINJA_SYNTAX. This code block loads the location from - # that variable, gets the absolute path to the vendored file, gets - # it's parent directory then uses importlib to import the module - # dynamically. - ninja_syntax_file = env[NINJA_SYNTAX] - - if os.path.exists(ninja_syntax_file): - if isinstance(ninja_syntax_file, str): - ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() - ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) - sys.path.append(ninja_syntax_mod_dir) - ninja_syntax_mod_name = os.path.basename(ninja_syntax_file).replace(".py", "") - ninja_syntax = importlib.import_module(ninja_syntax_mod_name) - else: - ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') + ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') if NINJA_STATE is None: NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) - NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join(ninja_syntax.__file__, os.pardir, 'data', 'bin')) - # Here we will force every builder to use an emitter which makes the ninja - # file depend on it's target. This forces the ninja file to the bottom of - # the DAG which is required so that we walk every target, and therefore add - # it to the global NINJA_STATE, before we try to write the ninja file. - def ninja_file_depends_on_all(target, source, env): - if not any("conftest" in str(t) for t in target): - env.Depends(ninja_file, target) - return target, source - - # The "Alias Builder" isn't in the BUILDERS map so we have to - # modify it directly. - SCons.Environment.AliasBuilder.emitter = ninja_file_depends_on_all - - for _, builder in env["BUILDERS"].items(): - try: - emitter = builder.emitter - if emitter is not None: - builder.emitter = SCons.Builder.ListEmitter( - [emitter, ninja_file_depends_on_all] - ) - else: - builder.emitter = ninja_file_depends_on_all - # Users can inject whatever they want into the BUILDERS - # dictionary so if the thing doesn't have an emitter we'll - # just ignore it. - except AttributeError: - pass - - # We will subvert the normal Command to make sure all targets generated - # from commands will be linked to the ninja file - SconsCommand = SCons.Environment.Environment.Command - - def NinjaCommand(self, target, source, action, **kw): - targets = SconsCommand(env, target, source, action, **kw) - env.Depends(ninja_file, targets) + NINJA_STATE.ninja_bin_path = env.get('NINJA_BIN') + if not NINJA_STATE.ninja_bin_path: + # default to using ninja installed with python module + ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' + NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( + ninja_syntax.__file__, + os.pardir, + 'data', + 'bin', + ninja_bin)) + if not os.path.exists(NINJA_STATE.ninja_bin_path): + # couldn't find it, just give the bin name and hope + # its in the path later + NINJA_STATE.ninja_bin_path = ninja_bin + + # TODO: this is hacking into scons, preferable if there were a less intrusive way + # We will subvert the normal builder execute to make sure all the ninja file is dependent + # on all targets generated from any builders + SCons_Builder_BuilderBase__execute = SCons.Builder.BuilderBase._execute + def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): + # this ensures all environments in which a builder executes from will + # not create list actions for linking on windows + ninja_hack_linkcom(env) + targets = SCons_Builder_BuilderBase__execute(self, env, target, source, overwarn=overwarn, executor_kw=executor_kw) + + if not SCons.Util.is_List(target): + target = [target] + + for target in targets: + if str(target) != ninja_file_name and "conftest" not in str(target): + env.Depends(ninja_file, targets) return targets - - SCons.Environment.Environment.Command = NinjaCommand + SCons.Builder.BuilderBase._execute = NinjaBuilderExecute # Here we monkey patch the Task.execute method to not do a bunch of # unnecessary work. If a build is a regular builder (i.e not a conftest and diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 5e491cc90b..347d63902c 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - lib_suffix = '.lib' if IS_WINDOWS else '.so' staticlib_suffix = '.lib' if IS_WINDOWS else '.a' lib_prefix = '' if IS_WINDOWS else 'lib' @@ -60,8 +68,9 @@ """ % locals()) # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('test'), stdout="library_function") test.run(program = test.workpath('test_static'), stdout="library_function") @@ -75,14 +84,13 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('test')) +test.must_not_exist(test.workpath('test_static')) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('test'), stdout="library_function") test.run(program = test.workpath('test_static'), stdout="library_function") diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index 06991d375c..0beb8de11c 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - test.write('SConstruct', """ env = Environment() env.Tool('ninja') @@ -49,8 +57,9 @@ # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('foo'), stdout="foo.c") # clean build and ninja files @@ -62,14 +71,12 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo')) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('foo'), stdout="foo.c") diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index 904e46a3cd..73c71f1b40 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - test.write('SConstruct', """ env = Environment() env.Tool('ninja') @@ -49,8 +57,9 @@ # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('foo' + _exe), stdout="foo.c") # clean build and ninja files @@ -61,14 +70,12 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo' + _exe)) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('foo' + _exe), stdout="foo.c") diff --git a/test/ninja/generate_and_build_cxx.py b/test/ninja/generate_and_build_cxx.py new file mode 100644 index 0000000000..ac0f55444f --- /dev/null +++ b/test/ninja/generate_and_build_cxx.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import importlib +import TestSCons +from TestCmd import IS_WINDOWS + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') +env.Program(target = 'test2', source = 'test2.cpp') +""") + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) +test.run(program = test.workpath('test2' + _exe), stdout="print_function") + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed test2.o', + 'Removed test2', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('test2' + _exe)) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program = program, stdout=None) +test.run(program = test.workpath('test2' + _exe), stdout="print_function") + +test.write('test2.hpp', """ +#include +#include + +class Foo +{ +public: + int print_function(); + int print_function2(){ + std::cout << "2"; + return 0; + }; +}; +""") + +# generate simple build +test.run(program = program, stdout=None) +test.run(program = test.workpath('test2' + _exe), stdout="print_function2") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index 298d227fea..d9b9c4ed59 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - shell = '' if IS_WINDOWS else './' test.write('SConstruct', """ @@ -72,6 +80,9 @@ # generate simple build test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") # clean build and ninja files @@ -85,14 +96,12 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('generated_source' + _exe)) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index 6675bf3350..b03e9cbcb0 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -27,21 +27,29 @@ import os import time import random +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - test.write('source_0.c', """ #include #include @@ -174,10 +182,10 @@ def mod_source_orig(test_num): tests_mods += [random.randrange(1, num_source, 1)] jobs = '-j' + str(get_num_cpus()) -ninja_program = [test.workpath('ninja_env.bat'), '&', ninja, jobs] if IS_WINDOWS else [ninja, jobs] +ninja_program = [test.workpath('run_ninja_env.bat'), jobs] if IS_WINDOWS else [ninja_bin, jobs] start = time.perf_counter() -test.run(arguments='--disable-auto-ninja', stdout=None) +test.run(arguments='--disable-execute-ninja', stdout=None) test.run(program = ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 087d392b52..c9b21b7d70 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -25,35 +25,43 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - test.write('SConstruct', """ env = Environment() env.Tool('ninja') env.Program(target = 'foo', source = 'foo.c') env2 = Environment() -env2.Tool('ninja') env2.Program(target = 'bar', source = 'bar.c') """) # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('foo' + _exe), stdout="foo.c") test.run(program = test.workpath('bar' + _exe), stdout="bar.c") @@ -67,14 +75,13 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo' + _exe)) +test.must_not_exist(test.workpath('bar' + _exe)) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('foo' + _exe), stdout="foo.c") test.run(program = test.workpath('bar' + _exe), stdout="bar.c") diff --git a/test/ninja/ninja-fixture/test2.cpp b/test/ninja/ninja-fixture/test2.cpp new file mode 100644 index 0000000000..69b54c97f7 --- /dev/null +++ b/test/ninja/ninja-fixture/test2.cpp @@ -0,0 +1,16 @@ +#include "test2.hpp" + +int +main(int argc, char *argv[]) +{ + Foo* test = new Foo(); + test->print_function(); + test->print_function2(); + return 0; +} + +int Foo::print_function() +{ + std::cout << "print_function"; + return 0; +} \ No newline at end of file diff --git a/test/ninja/ninja-fixture/test2.hpp b/test/ninja/ninja-fixture/test2.hpp new file mode 100644 index 0000000000..f0583fc547 --- /dev/null +++ b/test/ninja/ninja-fixture/test2.hpp @@ -0,0 +1,9 @@ +#include +#include + +class Foo +{ +public: + int print_function(); + int print_function2(){return 0;}; +}; diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index 5d491645c6..b35a52b6c0 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - shell = '' if IS_WINDOWS else './' test.write('SConstruct', """ @@ -51,8 +59,9 @@ # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.must_match('foo.out', 'foo.c') # clean build and ninja files @@ -64,14 +73,12 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo.out')) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.must_match('foo.out', 'foo.c') From c111cefdde56a94d98d6261680c1b7fa01777dd2 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 14 May 2020 12:28:17 -0400 Subject: [PATCH 009/163] is link should use the base nodes lstat instead of local fs stat builder is not garunteed to be in the environment, so check if the node is the ninja_file fix sider issues --- src/engine/SCons/Tool/ninja.py | 9 +++++---- test/ninja/build_libraries.py | 7 +++---- test/ninja/copy_function_command.py | 7 +++---- test/ninja/generate_and_build.py | 7 +++---- test/ninja/generate_and_build_cxx.py | 7 +++---- test/ninja/generate_source.py | 7 +++---- test/ninja/iterative_speedup.py | 15 +++++++-------- test/ninja/multi_env.py | 5 +++-- test/ninja/shell_command.py | 7 +++---- 9 files changed, 33 insertions(+), 38 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index bc505b8937..a68655ef1d 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -233,9 +233,10 @@ def action_to_ninja_build(self, node, action=None): # Ninja builders out of being sources of ninja builders but I # can't fix every DAG problem so we just skip ninja_builders # if we find one - if node.builder == self.env["BUILDERS"]["Ninja"]: + global NINJA_STATE + if NINJA_STATE.ninja_file == str(node): build = None - elif isinstance(action, SCons.Action.FunctionAction): + if isinstance(action, SCons.Action.FunctionAction): build = self.handle_func_action(node, action) elif isinstance(action, SCons.Action.LazyAction): # pylint: disable=protected-access @@ -1043,7 +1044,7 @@ def ninja_builder(env, target, source): if env["PLATFORM"] == "win32": # this is not great, its doesn't consider specific # node environments, which means on linux the build could - # behave differently, becuase on linux you can set the environment + # behave differently, because on linux you can set the environment # per command in the ninja file. This is only needed if # running ninja directly from a command line that hasn't # had the environment setup (vcvarsall.bat) @@ -1492,7 +1493,7 @@ def robust_rule_mapping(var, rule, tool): # default to using ninja installed with python module ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( - ninja_syntax.__file__, + ninja.__file__, os.pardir, 'data', 'bin', diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 347d63902c..40404152fa 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') lib_suffix = '.lib' if IS_WINDOWS else '.so' diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index 0beb8de11c..8e7acff7b7 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') test.write('SConstruct', """ diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index 73c71f1b40..faf395a1ef 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') test.write('SConstruct', """ diff --git a/test/ninja/generate_and_build_cxx.py b/test/ninja/generate_and_build_cxx.py index ac0f55444f..663282bd92 100644 --- a/test/ninja/generate_and_build_cxx.py +++ b/test/ninja/generate_and_build_cxx.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') test.write('SConstruct', """ diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index d9b9c4ed59..76c79bb7da 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') shell = '' if IS_WINDOWS else './' diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index b03e9cbcb0..ff50f502a3 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -27,10 +27,11 @@ import os import time import random -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -40,14 +41,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') test.write('source_0.c', """ @@ -220,14 +219,14 @@ def mod_source_orig(test_num): scons_times += [stop - start] full_build_print = True -for ninja, scons in zip(ninja_times, scons_times): - if ninja > scons: +for ninja_time, scons_time in zip(ninja_times, scons_times): + if ninja_time > scons_time: test.fail_test() if full_build_print: full_build_print = False - print("Clean build {} files - SCons: {:.3f}s Ninja: {:.3f}s".format(num_source, scons, ninja)) + print("Clean build {} files - SCons: {:.3f}s Ninja: {:.3f}s".format(num_source, scons_time, ninja_time)) else: - print("Single File Rebuild - SCons: {:.3f}s Ninja: {:.3f}s".format(scons, ninja)) + print("Single File Rebuild - SCons: {:.3f}s Ninja: {:.3f}s".format(scons_time, ninja_time)) test.pass_test() diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index c9b21b7d70..18ca3cbc69 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,7 +39,7 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index b35a52b6c0..5d7f97e215 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') shell = '' if IS_WINDOWS else './' From 82395cbf33f909e1118dec1bb9eb66768d63ba1d Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 21 May 2020 16:23:13 -0400 Subject: [PATCH 010/163] removed NINJA_SYNTAX completely --- src/engine/SCons/Tool/ninja.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index a68655ef1d..5515a35413 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -40,7 +40,6 @@ from SCons.Script import COMMAND_LINE_TARGETS NINJA_STATE = None -NINJA_SYNTAX = "NINJA_SYNTAX" NINJA_RULES = "__NINJA_CUSTOM_RULES" NINJA_POOLS = "__NINJA_CUSTOM_POOLS" NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" @@ -1299,7 +1298,6 @@ def generate(env): env["DISABLE_AUTO_NINJA"] = GetOption('disable_execute_ninja') global NINJA_STATE - env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") # Add the Ninja builder. always_exec_ninja_action = AlwaysExecAction(ninja_builder, {}) From e50f0801fabfe24f0b1d264b0692eea6f15dc3dc Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 22 May 2020 00:49:03 -0400 Subject: [PATCH 011/163] removed old sconscript changes --- SCons/Script/SConscript.py | 3 --- SCons/Script/__init__.py | 1 - src/engine/SCons/Tool/ninja.py | 2 +- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/SCons/Script/SConscript.py b/SCons/Script/SConscript.py index ded0fcfef9..596fca0463 100644 --- a/SCons/Script/SConscript.py +++ b/SCons/Script/SConscript.py @@ -203,11 +203,9 @@ def _SConscript(fs, *files, **kw): if f.rexists(): actual = f.rfile() _file_ = open(actual.get_abspath(), "rb") - SCons.Script.LOADED_SCONSCRIPTS.append(actual.get_abspath()) elif f.srcnode().rexists(): actual = f.srcnode().rfile() _file_ = open(actual.get_abspath(), "rb") - SCons.Script.LOADED_SCONSCRIPTS.append(actual.get_abspath()) elif f.has_src_builder(): # The SConscript file apparently exists in a source # code management system. Build it, but then clear @@ -216,7 +214,6 @@ def _SConscript(fs, *files, **kw): f.build() f.built() f.builder_set(None) - SCons.Script.LOADED_SCONSCRIPTS.append(f.get_abspath()) if f.exists(): _file_ = open(f.get_abspath(), "rb") if _file_: diff --git a/SCons/Script/__init__.py b/SCons/Script/__init__.py index c7f6a22a93..dff15673b1 100644 --- a/SCons/Script/__init__.py +++ b/SCons/Script/__init__.py @@ -187,7 +187,6 @@ def _clear(self): BUILD_TARGETS = TargetList() COMMAND_LINE_TARGETS = [] DEFAULT_TARGETS = [] -LOADED_SCONSCRIPTS = [] # BUILD_TARGETS can be modified in the SConscript files. If so, we # want to treat the modified BUILD_TARGETS list as if they specified diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 5515a35413..205bfa79a9 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -727,7 +727,7 @@ def generate(self): # allow us to query the actual SConscripts used. Right now # this glob method has deficiencies like skipping # jstests/SConscript and being specific to the MongoDB - # repository layout. + # repository layout. (github issue #3625) ninja.build( self.ninja_file.path, rule="REGENERATE", From 5d4b1698531c15aac2c3df8b8db31d9202cdf721 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 3 Jun 2020 10:47:54 -0400 Subject: [PATCH 012/163] merge commit a7541c60e5904e7deafdedf5bb040cc8924ac7d3 from https://github.com/mongodb/mongo --- src/engine/SCons/Tool/ninja.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 205bfa79a9..689f2ee262 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -1463,6 +1463,16 @@ def robust_rule_mapping(var, rule, tool): SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig) + # Ignore CHANGED_SOURCES and CHANGED_TARGETS. We don't want those + # to have effect in a generation pass because the generator + # shouldn't generate differently depending on the current local + # state. Without this, when generating on Windows, if you already + # had a foo.obj, you would omit foo.cpp from the response file. Do the same for UNCHANGED. + SCons.Executor.Executor._get_changed_sources = SCons.Executor.Executor._get_sources + SCons.Executor.Executor._get_changed_targets = SCons.Executor.Executor._get_targets + SCons.Executor.Executor._get_unchanged_sources = SCons.Executor.Executor._get_sources + SCons.Executor.Executor._get_unchanged_targets = SCons.Executor.Executor._get_targets + # Replace false action messages with nothing. env["PRINT_CMD_LINE_FUNC"] = ninja_noop From 2cd3d8fa6e2da334fb1ee4699c223123ca97a2a8 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 3 Jun 2020 10:49:15 -0400 Subject: [PATCH 013/163] merge commit 18cbf0d581162b2d15d66577b1fe08fe22006699 from https://github.com/mongodb/mongo --- src/engine/SCons/Tool/ninja.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 689f2ee262..3a1034fea2 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -675,7 +675,16 @@ def generate(self): # use for the "real" builder and multiple phony targets that # match the file names of the remaining outputs. This way any # build can depend on any output from any build. - build["outputs"].sort() + # + # We assume that the first listed output is the 'key' + # output and is stably presented to us by SCons. For + # instance if -gsplit-dwarf is in play and we are + # producing foo.o and foo.dwo, we expect that outputs[0] + # from SCons will be the foo.o file and not the dwo + # file. If instead we just sorted the whole outputs array, + # we would find that the dwo file becomes the + # first_output, and this breaks, for instance, header + # dependency scanning. if rule is not None and (rule.get("deps") or rule.get("rspfile")): first_output, remaining_outputs = ( build["outputs"][0], @@ -684,7 +693,7 @@ def generate(self): if remaining_outputs: ninja.build( - outputs=remaining_outputs, rule="phony", implicit=first_output, + outputs=sorted(remaining_outputs), rule="phony", implicit=first_output, ) build["outputs"] = first_output From 71a64aab3fe1e4b018d3c9c0b84aad216d66b3b3 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 8 Jun 2020 21:43:59 -0400 Subject: [PATCH 014/163] update to build godot reinvoke scons for unhandled actions Ignore Python.Values (need fix) escape rsp content check is_sconscript fix sider issues --- src/engine/SCons/Tool/ninja.py | 98 +++++++++++++++++++--------------- 1 file changed, 56 insertions(+), 42 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 3a1034fea2..3fdadf9d77 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -30,7 +30,6 @@ import shlex import subprocess -from glob import glob from os.path import join as joinpath from os.path import splitext @@ -38,6 +37,7 @@ from SCons.Action import _string_from_cmd_list, get_default_ENV from SCons.Util import is_List, flatten_sequence from SCons.Script import COMMAND_LINE_TARGETS +from SCons.Node import SConscriptNodes NINJA_STATE = None NINJA_RULES = "__NINJA_CUSTOM_RULES" @@ -58,10 +58,11 @@ def _install_action_function(_env, node): """Install files using the install or copy commands""" + #TODO: handle Python.Value nodes return { "outputs": get_outputs(node), "rule": "INSTALL", - "inputs": [get_path(src_file(s)) for s in node.sources], + "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], "implicit": get_dependencies(node), } @@ -83,9 +84,10 @@ def _mkdir_action_function(env, node): } def _copy_action_function(env, node): + #TODO: handle Python.Value nodes return { "outputs": get_outputs(node), - "inputs": [get_path(src_file(s)) for s in node.sources], + "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], "rule": "CMD", "variables": { "cmd": "$COPY $in $out", @@ -134,11 +136,12 @@ def is_valid_dependent_node(node): def alias_to_ninja_build(node): """Convert an Alias node into a Ninja phony target""" + # TODO: handle Python.Values return { "outputs": get_outputs(node), "rule": "phony", "implicit": [ - get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) + get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) and not isinstance(n, SCons.Node.Python.Value) ], } @@ -147,18 +150,20 @@ def get_order_only(node): """Return a list of order only dependencies for node.""" if node.prerequisites is None: return [] - return [get_path(src_file(prereq)) for prereq in node.prerequisites] + #TODO: handle Python.Value nodes + return [get_path(src_file(prereq)) for prereq in node.prerequisites if not isinstance(prereq, SCons.Node.Python.Value)] def get_dependencies(node, skip_sources=False): """Return a list of dependencies for node.""" + #TODO: handle Python.Value nodes if skip_sources: return [ get_path(src_file(child)) for child in node.children() - if child not in node.sources + if child not in node.sources and not isinstance(child, SCons.Node.Python.Value) ] - return [get_path(src_file(child)) for child in node.children()] + return [get_path(src_file(child)) for child in node.children() if not isinstance(child, SCons.Node.Python.Value)] def get_inputs(node): @@ -168,8 +173,8 @@ def get_inputs(node): inputs = executor.get_all_sources() else: inputs = node.sources - - inputs = [get_path(src_file(o)) for o in inputs] + #TODO: handle Python.Value nodes + inputs = [get_path(src_file(o)) for o in inputs if not isinstance(o, SCons.Node.Python.Value)] return inputs @@ -202,6 +207,7 @@ def __init__(self, env): # this check for us. "SharedFlagChecker": ninja_noop, # The install builder is implemented as a function action. + # TODO: use command action #3573 "installFunc": _install_action_function, "MkdirFunc": _mkdir_action_function, "LibSymlinksActionFunction": _lib_symlink_action_function, @@ -262,12 +268,6 @@ def handle_func_action(self, node, action): # the bottom of ninja's DAG anyway and Textfile builders can have text # content as their source which doesn't work as an implicit dep in # ninja. - if name == "_action": - return { - "rule": "TEMPLATE", - "outputs": get_outputs(node), - "implicit": get_dependencies(node, skip_sources=True), - } if name == 'ninja_builder': return None @@ -280,7 +280,7 @@ def handle_func_action(self, node, action): if handler is not None: return handler(node.env if node.env else self.env, node) - raise Exception( + SCons.Warnings.Warning( "Found unhandled function action {}, " " generating scons command to build\n" "Note: this is less efficient than Ninja," @@ -288,6 +288,12 @@ def handle_func_action(self, node, action): " this function using NinjaRegisterFunctionHandler".format(name) ) + return { + "rule": "TEMPLATE", + "outputs": get_outputs(node), + "implicit": get_dependencies(node, skip_sources=True), + } + # pylint: disable=too-many-branches def handle_list_action(self, node, action): """TODO write this comment""" @@ -309,7 +315,7 @@ def handle_list_action(self, node, action): all_outputs = list({output for build in results for output in build["outputs"]}) dependencies = list({dep for build in results for dep in build["implicit"]}) - if results[0]["rule"] == "CMD": + if results[0]["rule"] == "CMD" or results[0]["rule"] == "GENERATED_CMD": cmdline = "" for cmd in results: @@ -339,7 +345,7 @@ def handle_list_action(self, node, action): if cmdline: ninja_build = { "outputs": all_outputs, - "rule": "CMD", + "rule": "GENERATED_CMD", "variables": { "cmd": cmdline, "env": get_command_env(node.env if node.env else self.env), @@ -360,10 +366,11 @@ def handle_list_action(self, node, action): } elif results[0]["rule"] == "INSTALL": + #TODO: handle Python.Value nodes return { "outputs": all_outputs, "rule": "INSTALL", - "inputs": [get_path(src_file(s)) for s in node.sources], + "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], "implicit": dependencies, } @@ -397,22 +404,28 @@ def __init__(self, env, ninja_file, writer_class): # like CCFLAGS escape = env.get("ESCAPE", lambda x: x) + # if SCons was invoked from python, we expect the first arg to be the scons.py + # script, otherwise scons was invoked from the scons script + python_bin = '' + if os.path.basename(sys.argv[0]) == 'scons.py': + python_bin = escape(sys.executable) self.variables = { "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", - "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format( - sys.executable, + "SCONS_INVOCATION": '{} {} --disable-ninja __NINJA_NO=1 $out'.format( + python_bin, " ".join( [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] ), ), - "SCONS_INVOCATION_W_TARGETS": "{} {}".format( - sys.executable, " ".join([escape(arg) for arg in sys.argv]) + "SCONS_INVOCATION_W_TARGETS": "{} {} --disable-ninja".format( + python_bin, " ".join([escape(arg) for arg in sys.argv]) ), # This must be set to a global default per: - # https://ninja-build.org/manual.html - # - # (The deps section) - "msvc_deps_prefix": "Note: including file:", + # https://ninja-build.org/manual.html#_deps + # English Visual Studio will have the default below, + # otherwise the user can define the variable in the first environment + # that initialized ninja tool + "msvc_deps_prefix": env.get("NINJA_MSVC_DEPS_PREFIX", "Note: including file:") } self.rules = { @@ -481,13 +494,13 @@ def __init__(self, env, ninja_file, writer_class): }, "TEMPLATE": { "command": "$SCONS_INVOCATION $out", - "description": "Rendering $out", + "description": "Rendering $SCONS_INVOCATION $out", "pool": "scons_pool", "restat": 1, }, "SCONS": { "command": "$SCONS_INVOCATION $out", - "description": "SCons $out", + "description": "$SCONS_INVOCATION $out", "pool": "scons_pool", # restat # if present, causes Ninja to re-stat the command's outputs @@ -557,6 +570,8 @@ def add_build(self, node): self.built.update(build["outputs"]) return True + # TODO: rely on SCons to tell us what is generated source + # or some form of user scanner maybe (Github Issue #3624) def is_generated_source(self, output): """Check if output ends with a known generated suffix.""" _, suffix = splitext(output) @@ -740,11 +755,7 @@ def generate(self): ninja.build( self.ninja_file.path, rule="REGENERATE", - implicit=[ - self.env.File("#SConstruct").path, - __file__, - ] - + sorted(glob("src/**/SConscript", recursive=True)), + implicit=[__file__] + [str(node) for node in SConscriptNodes], ) # If we ever change the name/s of the rules that include @@ -921,6 +932,7 @@ def get_response_file_command(env, node, action, targets, sources, executor=None ) cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] + rsp_content = ['"' + rsp_content_item + '"' for rsp_content_item in rsp_content] rsp_content = " ".join(rsp_content) variables = {"rspc": rsp_content} @@ -1060,20 +1072,23 @@ def ninja_builder(env, target, source): for key in env['ENV']: f.write('set {}={}\n'.format(key, env['ENV'][key])) f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) - - if not env.get("DISABLE_AUTO_NINJA"): + cmd = ['run_ninja_env.bat'] + + else: cmd = [NINJA_STATE.ninja_bin_path, '-f', generated_build_ninja] + + if not env.get("DISABLE_AUTO_NINJA"): print("Executing:", str(' '.join(cmd))) # execute the ninja build at the end of SCons, trying to # reproduce the output like a ninja build would def execute_ninja(): - + proc = subprocess.Popen(cmd, stderr=sys.stderr, stdout=subprocess.PIPE, universal_newlines=True, - env=env['ENV'] # ninja build items won't consider node env on win32 + env=os.environ if env["PLATFORM"] == "win32" else env['ENV'] ) for stdout_line in iter(proc.stdout.readline, ""): yield stdout_line @@ -1142,8 +1157,7 @@ def ninja_csig(original): """Return a dummy csig""" def wrapper(self): - name = str(self) - if "SConscript" in name or "SConstruct" in name: + if isinstance(self, SCons.Node.Node) and self.is_sconscript(): return original(self) return "dummy_ninja_csig" @@ -1154,8 +1168,7 @@ def ninja_contents(original): """Return a dummy content without doing IO""" def wrapper(self): - name = str(self) - if "SConscript" in name or "SConstruct" in name: + if isinstance(self, SCons.Node.Node) and self.is_sconscript(): return original(self) return bytes("dummy_ninja_contents", encoding="utf-8") @@ -1396,6 +1409,7 @@ def robust_rule_mapping(var, rule, tool): # Used to determine if a build generates a source file. Ninja # requires that all generated sources are added as order_only # dependencies to any builds that *might* use them. + # TODO: switch to using SCons to help determine this (Github Issue #3624) env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] if env["PLATFORM"] != "win32" and env.get("RANLIBCOM"): From 017d73ac8912272b159db5300550ac9ea2492759 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 22 Jun 2020 15:21:34 -0400 Subject: [PATCH 015/163] revert ninja install requirement expand response file in ninja comdb output --- src/engine/SCons/Tool/ninja.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 3fdadf9d77..ad36758788 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -767,7 +767,7 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "{} -f {} -t compdb CC CXX > compile_commands.json".format(self.ninja_bin_path, + "cmd": "{} -f {} -t compdb -x CC CXX > compile_commands.json".format(self.ninja_bin_path, str(self.ninja_file) ) }, From 6dae3890f788ba52b7dcacb242bea479b6bce976 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 25 Jun 2020 23:05:28 -0400 Subject: [PATCH 016/163] handle files which are not file or alias by reinvoking scons --- src/engine/SCons/Tool/ninja.py | 228 +++++++++++++++++++-------------- 1 file changed, 134 insertions(+), 94 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index ad36758788..50a9518aae 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -56,63 +56,6 @@ ) -def _install_action_function(_env, node): - """Install files using the install or copy commands""" - #TODO: handle Python.Value nodes - return { - "outputs": get_outputs(node), - "rule": "INSTALL", - "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], - "implicit": get_dependencies(node), - } - -def _mkdir_action_function(env, node): - return { - "outputs": get_outputs(node), - "rule": "CMD", - # implicit explicitly omitted, we translate these so they can be - # used by anything that depends on these but commonly this is - # hit with a node that will depend on all of the fake - # srcnode's that SCons will never give us a rule for leading - # to an invalid ninja file. - "variables": { - # On Windows mkdir "-p" is always on - "cmd": "{mkdir}".format( - mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", - ), - }, - } - -def _copy_action_function(env, node): - #TODO: handle Python.Value nodes - return { - "outputs": get_outputs(node), - "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], - "rule": "CMD", - "variables": { - "cmd": "$COPY $in $out", - }, - } - - -def _lib_symlink_action_function(_env, node): - """Create shared object symlinks if any need to be created""" - symlinks = getattr(getattr(node, "attributes", None), "shliblinks", None) - - if not symlinks or symlinks is None: - return None - - outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] - inputs = [link.get_path() for link, _ in symlinks] - - return { - "outputs": outputs, - "inputs": inputs, - "rule": "SYMLINK", - "implicit": get_dependencies(node), - } - - def is_valid_dependent_node(node): """ Return True if node is not an alias or is an alias that has children @@ -136,46 +79,65 @@ def is_valid_dependent_node(node): def alias_to_ninja_build(node): """Convert an Alias node into a Ninja phony target""" - # TODO: handle Python.Values return { "outputs": get_outputs(node), "rule": "phony", "implicit": [ - get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) and not isinstance(n, SCons.Node.Python.Value) + get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) ], } +def check_invalid_ninja_node(node): + return not isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)) + + +def filter_ninja_nodes(node_list): + ninja_nodes = [] + for node in node_list: + if isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)): + ninja_nodes.append(node) + else: + continue + return ninja_nodes + +def get_input_nodes(node): + if node.get_executor() is not None: + inputs = node.get_executor().get_all_sources() + else: + inputs = node.sources + return inputs + + +def invalid_ninja_nodes(node, targets): + result = False + for node_list in [node.prerequisites, get_input_nodes(node), node.children(), targets]: + if node_list: + result = result or any([check_invalid_ninja_node(node) for node in node_list]) + return result + + def get_order_only(node): """Return a list of order only dependencies for node.""" if node.prerequisites is None: return [] - #TODO: handle Python.Value nodes - return [get_path(src_file(prereq)) for prereq in node.prerequisites if not isinstance(prereq, SCons.Node.Python.Value)] + return [get_path(src_file(prereq)) for prereq in filter_ninja_nodes(node.prerequisites)] def get_dependencies(node, skip_sources=False): """Return a list of dependencies for node.""" - #TODO: handle Python.Value nodes if skip_sources: return [ get_path(src_file(child)) - for child in node.children() - if child not in node.sources and not isinstance(child, SCons.Node.Python.Value) + for child in filter_ninja_nodes(node.children()) + if child not in node.sources ] - return [get_path(src_file(child)) for child in node.children() if not isinstance(child, SCons.Node.Python.Value)] + return [get_path(src_file(child)) for child in filter_ninja_nodes(node.children())] def get_inputs(node): """Collect the Ninja inputs for node.""" - executor = node.get_executor() - if executor is not None: - inputs = executor.get_all_sources() - else: - inputs = node.sources - #TODO: handle Python.Value nodes - inputs = [get_path(src_file(o)) for o in inputs if not isinstance(o, SCons.Node.Python.Value)] - return inputs + return [get_path(src_file(o)) for o in filter_ninja_nodes(get_input_nodes(node))] def get_outputs(node): @@ -189,11 +151,93 @@ def get_outputs(node): else: outputs = [node] - outputs = [get_path(o) for o in outputs] + outputs = [get_path(o) for o in filter_ninja_nodes(outputs)] return outputs +def get_targets_sources(node): + executor = node.get_executor() + if executor is not None: + tlist = executor.get_all_targets() + slist = executor.get_all_sources() + else: + if hasattr(node, "target_peers"): + tlist = node.target_peers + else: + tlist = [node] + slist = node.sources + + # Retrieve the repository file for all sources + slist = [rfile(s) for s in slist] + return tlist, slist + + +def get_rule(node, rule): + tlist, slist = get_targets_sources(node) + if invalid_ninja_nodes(node, tlist): + return "TEMPLATE" + else: + return rule + + +def _install_action_function(_env, node): + """Install files using the install or copy commands""" + return { + "outputs": get_outputs(node), + "rule": get_rule(node, "INSTALL"), + "inputs": get_inputs(node), + "implicit": get_dependencies(node), + } + + +def _mkdir_action_function(env, node): + return { + "outputs": get_outputs(node), + "rule": get_rule(node, "CMD"), + # implicit explicitly omitted, we translate these so they can be + # used by anything that depends on these but commonly this is + # hit with a node that will depend on all of the fake + # srcnode's that SCons will never give us a rule for leading + # to an invalid ninja file. + "variables": { + # On Windows mkdir "-p" is always on + "cmd": "{mkdir}".format( + mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", + ), + }, + } + + +def _copy_action_function(env, node): + return { + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "rule": get_rule(node, "CMD"), + "variables": { + "cmd": "$COPY $in $out", + }, + } + + +def _lib_symlink_action_function(_env, node): + """Create shared object symlinks if any need to be created""" + symlinks = getattr(getattr(node, "attributes", None), "shliblinks", None) + + if not symlinks or symlinks is None: + return None + + outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] + inputs = [link.get_path() for link, _ in symlinks] + + return { + "outputs": outputs, + "inputs": inputs, + "rule": get_rule(node, "SYMLINK"), + "implicit": get_dependencies(node), + } + + class SConsToNinjaTranslator: """Translates SCons Actions into Ninja build objects.""" @@ -290,7 +334,9 @@ def handle_func_action(self, node, action): return { "rule": "TEMPLATE", + "order_only": get_order_only(node), "outputs": get_outputs(node), + "inputs": get_inputs(node), "implicit": get_dependencies(node, skip_sources=True), } @@ -345,7 +391,7 @@ def handle_list_action(self, node, action): if cmdline: ninja_build = { "outputs": all_outputs, - "rule": "GENERATED_CMD", + "rule": get_rule(node, "GENERATED_CMD"), "variables": { "cmd": cmdline, "env": get_command_env(node.env if node.env else self.env), @@ -366,11 +412,10 @@ def handle_list_action(self, node, action): } elif results[0]["rule"] == "INSTALL": - #TODO: handle Python.Value nodes return { "outputs": all_outputs, - "rule": "INSTALL", - "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], + "rule": get_rule(node, "INSTALL"), + "inputs": get_inputs(node), "implicit": dependencies, } @@ -985,20 +1030,8 @@ def get_command(env, node, action): # pylint: disable=too-many-branches sub_env = node.env else: sub_env = env - executor = node.get_executor() - if executor is not None: - tlist = executor.get_all_targets() - slist = executor.get_all_sources() - else: - if hasattr(node, "target_peers"): - tlist = node.target_peers - else: - tlist = [node] - slist = node.sources - - # Retrieve the repository file for all sources - slist = [rfile(s) for s in slist] + tlist, slist = get_targets_sources(node) # Generate a real CommandAction if isinstance(action, SCons.Action.CommandGeneratorAction): @@ -1022,10 +1055,11 @@ def get_command(env, node, action): # pylint: disable=too-many-branches "outputs": get_outputs(node), "inputs": get_inputs(node), "implicit": implicit, - "rule": rule, + "rule": get_rule(node, rule), "variables": variables, } + # Don't use sub_env here because we require that NINJA_POOL be set # on a per-builder call basis to prevent accidental strange # behavior like env['NINJA_POOL'] = 'console' and sub_env can be @@ -1283,6 +1317,11 @@ def exists(env): if env.get("__NINJA_NO", "0") == "1": return False + try: + import ninja + except ImportError: + SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") + return False return True added = None @@ -1308,8 +1347,6 @@ def generate(env): default=False, help='Disable ninja automatically building after scons') - if GetOption('disable_ninja'): - return env try: import ninja @@ -1427,6 +1464,9 @@ def robust_rule_mapping(var, rule, tool): # Disable running ranlib, since we added 's' above env["RANLIBCOM"] = "" + if GetOption('disable_ninja'): + return env + SCons.Warnings.Warning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") # This is the point of no return, anything after this comment From 5eb70dd8ee2833f9ad9bce9c9d0cc80988992dca Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 20 Jul 2020 17:31:56 +0000 Subject: [PATCH 017/163] updated with some changes from latest mongodb version: 21075112a999e252a22e9c9bd64e403cec892df3 5fe923a0aa312044062df044eb4eaa47951f70ec c7348f391124e681d9c62aceb0e13e0d07fca8bc --- src/engine/SCons/Tool/ninja.py | 40 +++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 50a9518aae..bf7c45add0 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -29,6 +29,7 @@ import shutil import shlex import subprocess +import textwrap from os.path import join as joinpath from os.path import splitext @@ -768,13 +769,13 @@ def generate(self): # Special handling for outputs and implicit since we need to # aggregate not replace for each builder. - for agg_key in ["outputs", "implicit"]: + for agg_key in ["outputs", "implicit", 'inputs']: new_val = template_builds.get(agg_key, []) # Use pop so the key is removed and so the update # below will not overwrite our aggregated values. cur_val = template_builder.pop(agg_key, []) - if isinstance(cur_val, list): + if is_List(cur_val): new_val += cur_val else: new_val.append(cur_val) @@ -812,8 +813,8 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "{} -f {} -t compdb -x CC CXX > compile_commands.json".format(self.ninja_bin_path, - str(self.ninja_file) + "cmd": "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format( + ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' ) }, ) @@ -929,7 +930,13 @@ def get_command_env(env): if windows: command_env += "set '{}={}' && ".format(key, value) else: - command_env += "{}={} ".format(key, value) + # We address here *only* the specific case that a user might have + # an environment variable which somehow gets included and has + # spaces in the value. These are escapes that Ninja handles. This + # doesn't make builds on paths with spaces (Ninja and SCons issues) + # nor expanding response file paths with spaces (Ninja issue) work. + value = value.replace(r' ', r'$ ') + command_env += "{}='{}' ".format(key, value) env["NINJA_ENV_VAR_CACHE"] = command_env return command_env @@ -1208,6 +1215,27 @@ def wrapper(self): return wrapper +def CheckNinjaCompdbExpand(env, context): + """ Configure check testing if ninja's compdb can expand response files""" + + context.Message('Checking if ninja compdb can expand response files... ') + ret, output = context.TryAction( + action='ninja -f $SOURCE -t compdb -x CMD_RSP > $TARGET', + extension='.ninja', + text=textwrap.dedent(""" + rule CMD_RSP + command = $cmd @$out.rsp > fake_output.txt + description = Building $out + rspfile = $out.rsp + rspfile_content = $rspc + build fake_output.txt: CMD_RSP fake_input.txt + cmd = echo + pool = console + rspc = "test" + """)) + result = '@fake_output.txt.rsp' not in output + context.Result(result) + return result def ninja_stat(_self, path): """ @@ -1386,6 +1414,8 @@ def generate(env): else: env.Append(CCFLAGS=["-MD", "-MF", "${TARGET}.d"]) + env.AddMethod(CheckNinjaCompdbExpand, "CheckNinjaCompdbExpand") + # Provide a way for custom rule authors to easily access command # generation. env.AddMethod(get_shell_command, "NinjaGetShellCommand") From d2ddf4aab1ff294cfb4e4637b7f930374c9165bf Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 20 Jul 2020 17:43:48 +0000 Subject: [PATCH 018/163] fixed sider issues --- src/engine/SCons/Tool/ninja.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index bf7c45add0..153a8943b5 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -813,8 +813,8 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format( - ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' + "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( + self.ninja_bin_path, str(self.ninja_file), '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' ) }, ) @@ -1347,10 +1347,11 @@ def exists(env): try: import ninja + return ninja.__file__ except ImportError: SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") return False - return True + added = None From 142adbe59e9c24f9597e606deca69857412355f8 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 20 Jul 2020 18:25:39 +0000 Subject: [PATCH 019/163] updated warning to the latest API --- src/engine/SCons/Tool/ninja.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 153a8943b5..502d365014 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -325,7 +325,7 @@ def handle_func_action(self, node, action): if handler is not None: return handler(node.env if node.env else self.env, node) - SCons.Warnings.Warning( + SCons.Warnings.SConsWarning( "Found unhandled function action {}, " " generating scons command to build\n" "Note: this is less efficient than Ninja," @@ -1349,7 +1349,7 @@ def exists(env): import ninja return ninja.__file__ except ImportError: - SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") + SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return False @@ -1380,7 +1380,7 @@ def generate(env): try: import ninja except ImportError: - SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") + SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return env["DISABLE_AUTO_NINJA"] = GetOption('disable_execute_ninja') @@ -1405,7 +1405,7 @@ def generate(env): env.Alias("$NINJA_ALIAS_NAME", ninja_file) else: if str(NINJA_STATE.ninja_file) != ninja_file_name: - SCons.Warnings.Warning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") + SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] # This adds the required flags such that the generated compile @@ -1498,7 +1498,7 @@ def robust_rule_mapping(var, rule, tool): if GetOption('disable_ninja'): return env - SCons.Warnings.Warning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") + SCons.Warnings.SConsWarning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") # This is the point of no return, anything after this comment # makes changes to SCons that are irreversible and incompatible From 3471412452092a51c5199dfcc0ec9c71c443607b Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 31 Dec 2020 15:58:31 +0000 Subject: [PATCH 020/163] Sync with mongo ninja file --- src/engine/SCons/Tool/ninja.py | 230 ++++++++++++++++++++++++++------- 1 file changed, 186 insertions(+), 44 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 502d365014..ea01d590ee 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -31,6 +31,7 @@ import subprocess import textwrap +from glob import glob from os.path import join as joinpath from os.path import splitext @@ -156,6 +157,41 @@ def get_outputs(node): return outputs +def generate_depfile(env, node, dependencies): + """ + Ninja tool function for writing a depfile. The depfile should include + the node path followed by all the dependent files in a makefile format. + + dependencies arg can be a list or a subst generator which returns a list. + """ + + depfile = os.path.join(get_path(env['NINJA_BUILDDIR']), str(node) + '.depfile') + + # subst_list will take in either a raw list or a subst callable which generates + # a list, and return a list of CmdStringHolders which can be converted into raw strings. + # If a raw list was passed in, then scons_list will make a list of lists from the original + # values and even subst items in the list if they are substitutable. Flatten will flatten + # the list in that case, to ensure for either input we have a list of CmdStringHolders. + deps_list = env.Flatten(env.subst_list(dependencies)) + + # Now that we have the deps in a list as CmdStringHolders, we can convert them into raw strings + # and make sure to escape the strings to handle spaces in paths. We also will sort the result + # keep the order of the list consistent. + escaped_depends = sorted([dep.escape(env.get("ESCAPE", lambda x: x)) for dep in deps_list]) + depfile_contents = str(node) + ": " + ' '.join(escaped_depends) + + need_rewrite = False + try: + with open(depfile, 'r') as f: + need_rewrite = (f.read() != depfile_contents) + except FileNotFoundError: + need_rewrite = True + + if need_rewrite: + os.makedirs(os.path.dirname(depfile) or '.', exist_ok=True) + with open(depfile, 'w') as f: + f.write(depfile_contents) + def get_targets_sources(node): executor = node.get_executor() @@ -278,6 +314,7 @@ def action_to_ninja_build(self, node, action=None): return None build = {} + env = node.env if node.env else self.env # Ideally this should never happen, and we do try to filter # Ninja builders out of being sources of ninja builders but I @@ -286,22 +323,27 @@ def action_to_ninja_build(self, node, action=None): global NINJA_STATE if NINJA_STATE.ninja_file == str(node): build = None - if isinstance(action, SCons.Action.FunctionAction): + elif isinstance(action, SCons.Action.FunctionAction): build = self.handle_func_action(node, action) elif isinstance(action, SCons.Action.LazyAction): # pylint: disable=protected-access - action = action._generate_cache(node.env if node.env else self.env) + action = action._generate_cache(env) build = self.action_to_ninja_build(node, action=action) elif isinstance(action, SCons.Action.ListAction): build = self.handle_list_action(node, action) elif isinstance(action, COMMAND_TYPES): - build = get_command(node.env if node.env else self.env, node, action) + build = get_command(env, node, action) else: raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) if build is not None: build["order_only"] = get_order_only(node) + if 'conftest' not in str(node): + node_callback = getattr(node.attributes, "ninja_build_callback", None) + if callable(node_callback): + node_callback(env, node, build) + return build def handle_func_action(self, node, action): @@ -509,8 +551,16 @@ def __init__(self, env, ninja_file, writer_class): "rspfile_content": "$rspc", "pool": "local_pool", }, + # Ninja does not automatically delete the archive before + # invoking ar. The ar utility will append to an existing archive, which + # can cause duplicate symbols if the symbols moved between object files. + # Native SCons will perform this operation so we need to force ninja + # to do the same. See related for more info: + # https://jira.mongodb.org/browse/SERVER-49457 "AR": { - "command": "$env$AR @$out.rsp", + "command": "{}$env$AR @$out.rsp".format( + '' if sys.platform == "win32" else "rm -f $out && " + ), "description": "Archiving $out", "rspfile": "$out.rsp", "rspfile_content": "$rspc", @@ -540,7 +590,7 @@ def __init__(self, env, ninja_file, writer_class): }, "TEMPLATE": { "command": "$SCONS_INVOCATION $out", - "description": "Rendering $SCONS_INVOCATION $out", + "description": "Rendering $SCONS_INVOCATION $out", "pool": "scons_pool", "restat": 1, }, @@ -570,6 +620,7 @@ def __init__(self, env, ninja_file, writer_class): "command": "$SCONS_INVOCATION_W_TARGETS", "description": "Regenerating $out", "generator": 1, + "depfile": os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile'), # Console pool restricts to 1 job running at a time, # it additionally has some special handling about # passing stdin, stdout, etc to process in this pool @@ -650,6 +701,8 @@ def generate(self): ninja.comment("Generated by scons. DO NOT EDIT.") + ninja.variable("builddir", get_path(self.env['NINJA_BUILDDIR'])) + for pool_name, size in self.pools.items(): ninja.pool(pool_name, size) @@ -759,9 +812,19 @@ def generate(self): build["outputs"] = first_output + # Optionally a rule can specify a depfile, and SCons can generate implicit + # dependencies into the depfile. This allows for dependencies to come and go + # without invalidating the ninja file. The depfile was created in ninja specifically + # for dealing with header files appearing and disappearing across rebuilds, but it can + # be repurposed for anything, as long as you have a way to regenerate the depfile. + # More specific info can be found here: https://ninja-build.org/manual.html#_depfile + if rule is not None and rule.get('depfile') and build.get('deps_files'): + path = build['outputs'] if SCons.Util.is_List(build['outputs']) else [build['outputs']] + generate_depfile(self.env, path[0], build.pop('deps_files', [])) + if "inputs" in build: build["inputs"].sort() - + ninja.build(**build) template_builds = dict() @@ -769,7 +832,7 @@ def generate(self): # Special handling for outputs and implicit since we need to # aggregate not replace for each builder. - for agg_key in ["outputs", "implicit", 'inputs']: + for agg_key in ["outputs", "implicit", "inputs"]: new_val = template_builds.get(agg_key, []) # Use pop so the key is removed and so the update @@ -793,15 +856,25 @@ def generate(self): # generate this rule even though SCons should know we're # dependent on SCons files. # - # TODO: We're working on getting an API into SCons that will - # allow us to query the actual SConscripts used. Right now - # this glob method has deficiencies like skipping - # jstests/SConscript and being specific to the MongoDB - # repository layout. (github issue #3625) + # The REGENERATE rule uses depfile, so we need to generate the depfile + # in case any of the SConscripts have changed. The depfile needs to be + # path with in the build and the passed ninja file is an abspath, so + # we will use SCons to give us the path within the build. Normally + # generate_depfile should not be called like this, but instead be called + # through the use of custom rules, and filtered out in the normal + # list of build generation about. However, because the generate rule + # is hardcoded here, we need to do this generate_depfile call manually. + ninja_file_path = self.env.File(self.ninja_file).path + generate_depfile( + self.env, + ninja_file_path, + self.env['NINJA_REGENERATE_DEPS'] + ) + ninja.build( - self.ninja_file.path, + ninja_file_path, rule="REGENERATE", - implicit=[__file__] + [str(node) for node in SConscriptNodes], + implicit=[__file__], ) # If we ever change the name/s of the rules that include @@ -936,13 +1009,13 @@ def get_command_env(env): # doesn't make builds on paths with spaces (Ninja and SCons issues) # nor expanding response file paths with spaces (Ninja issue) work. value = value.replace(r' ', r'$ ') - command_env += "{}='{}' ".format(key, value) + command_env += "export {}='{}';".format(key, value) env["NINJA_ENV_VAR_CACHE"] = command_env return command_env -def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False): +def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom_env={}): """Generate a response file command provider for rule name.""" # If win32 using the environment with a response file command will cause @@ -979,7 +1052,7 @@ def get_response_file_command(env, node, action, targets, sources, executor=None except ValueError: raise Exception( "Could not find tool {} in {} generated from {}".format( - tool_command, cmd_list, get_comstr(env, action, targets, sources) + tool, cmd_list, get_comstr(env, action, targets, sources) ) ) @@ -991,7 +1064,12 @@ def get_response_file_command(env, node, action, targets, sources, executor=None variables[rule] = cmd if use_command_env: variables["env"] = get_command_env(env) - return rule, variables + + for key, value in custom_env.items(): + variables["env"] += env.subst( + f"export {key}={value};", target=targets, source=sources, executor=executor + ) + " " + return rule, variables, [tool_command] return get_response_file_command @@ -1021,13 +1099,21 @@ def generate_command(env, node, action, targets, sources, executor=None): return cmd.replace("$", "$$") -def get_shell_command(env, node, action, targets, sources, executor=None): +def get_generic_shell_command(env, node, action, targets, sources, executor=None): return ( - "GENERATED_CMD", + "CMD", { "cmd": generate_command(env, node, action, targets, sources, executor=None), "env": get_command_env(env), }, + # Since this function is a rule mapping provider, it must return a list of dependencies, + # and usually this would be the path to a tool, such as a compiler, used for this rule. + # However this function is to generic to be able to reliably extract such deps + # from the command, so we return a placeholder empty list. It should be noted that + # generally this function will not be used soley and is more like a template to generate + # the basics for a custom provider which may have more specific options for a provier + # function for a custom NinjaRuleMapping. + [] ) @@ -1050,13 +1136,49 @@ def get_command(env, node, action): # pylint: disable=too-many-branches comstr = get_comstr(sub_env, action, tlist, slist) if not comstr: return None - - provider = __NINJA_RULE_MAPPING.get(comstr, get_shell_command) - rule, variables = provider(sub_env, node, action, tlist, slist, executor=executor) + + provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) + rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) # Get the dependencies for all targets implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) + # Now add in the other dependencies related to the command, + # e.g. the compiler binary. The ninja rule can be user provided so + # we must do some validation to resolve the dependency path for ninja. + for provider_dep in provider_deps: + + provider_dep = sub_env.subst(provider_dep) + if not provider_dep: + continue + + # If the tool is a node, then SCons will resolve the path later, if its not + # a node then we assume it generated from build and make sure it is existing. + if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep): + implicit.append(provider_dep) + continue + + # in some case the tool could be in the local directory and be suppled without the ext + # such as in windows, so append the executable suffix and check. + prog_suffix = sub_env.get('PROGSUFFIX', '') + provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix + if os.path.exists(provider_dep_ext): + implicit.append(provider_dep_ext) + continue + + # Many commands will assume the binary is in the path, so + # we accept this as a possible input from a given command. + + provider_dep_abspath = sub_env.WhereIs(provider_dep) or sub_env.WhereIs(provider_dep, path=os.environ["PATH"]) + if provider_dep_abspath: + implicit.append(provider_dep_abspath) + continue + + # Possibly these could be ignore and the build would still work, however it may not always + # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided + # ninja rule. + raise Exception(f"Could not resolve path for {provider_dep} dependency on node '{node}'") + ninja_build = { "order_only": get_order_only(node), "outputs": get_outputs(node), @@ -1066,7 +1188,6 @@ def get_command(env, node, action): # pylint: disable=too-many-branches "variables": variables, } - # Don't use sub_env here because we require that NINJA_POOL be set # on a per-builder call basis to prevent accidental strange # behavior like env['NINJA_POOL'] = 'console' and sub_env can be @@ -1103,28 +1224,28 @@ def ninja_builder(env, target, source): NINJA_STATE.generate() if env["PLATFORM"] == "win32": - # this is not great, its doesn't consider specific + # this is not great, its doesn't consider specific # node environments, which means on linux the build could # behave differently, because on linux you can set the environment - # per command in the ninja file. This is only needed if + # per command in the ninja file. This is only needed if # running ninja directly from a command line that hasn't # had the environment setup (vcvarsall.bat) with open('run_ninja_env.bat', 'w') as f: for key in env['ENV']: f.write('set {}={}\n'.format(key, env['ENV'][key])) - f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) - cmd = ['run_ninja_env.bat'] + f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) + cmd = ['run_ninja_env.bat'] else: cmd = [NINJA_STATE.ninja_bin_path, '-f', generated_build_ninja] - + if not env.get("DISABLE_AUTO_NINJA"): print("Executing:", str(' '.join(cmd))) # execute the ninja build at the end of SCons, trying to # reproduce the output like a ninja build would def execute_ninja(): - + proc = subprocess.Popen(cmd, stderr=sys.stderr, stdout=subprocess.PIPE, @@ -1137,7 +1258,7 @@ def execute_ninja(): return_code = proc.wait() if return_code: raise subprocess.CalledProcessError(return_code, 'ninja') - + erase_previous = False for output in execute_ninja(): output = output.strip() @@ -1168,24 +1289,31 @@ def register_custom_handler(env, name, handler): def register_custom_rule_mapping(env, pre_subst_string, rule): - """Register a custom handler for SCons function actions.""" + """Register a function to call for a given rule.""" global __NINJA_RULE_MAPPING __NINJA_RULE_MAPPING[pre_subst_string] = rule -def register_custom_rule(env, rule, command, description="", deps=None, pool=None): +def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): """Allows specification of Ninja rules from inside SCons files.""" rule_obj = { "command": command, "description": description if description else "{} $out".format(rule), } + if use_depfile: + rule_obj["depfile"] = os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile') + if deps is not None: rule_obj["deps"] = deps if pool is not None: rule_obj["pool"] = pool + if use_response_file: + rule_obj["rspfile"] = "$out.rsp" + rule_obj["rspfile_content"] = response_file_content + env[NINJA_RULES][rule] = rule_obj @@ -1193,6 +1321,9 @@ def register_custom_pool(env, pool, size): """Allows the creation of custom Ninja pools""" env[NINJA_POOLS][pool] = size +def set_build_node_callback(env, node, callback): + if 'conftest' not in str(node): + setattr(node.attributes, "ninja_build_callback", callback) def ninja_csig(original): """Return a dummy csig""" @@ -1395,7 +1526,7 @@ def generate(env): env["NINJA_PREFIX"] = env.get("NINJA_PREFIX", "build") env["NINJA_SUFFIX"] = env.get("NINJA_SUFFIX", "ninja") env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") - + env['NINJA_BUILDDIR'] = env.get("NINJA_BUILDDIR", env.Dir(".ninja").path) ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") # here we allow multiple environments to construct rules and builds # into the same ninja file @@ -1407,20 +1538,31 @@ def generate(env): if str(NINJA_STATE.ninja_file) != ninja_file_name: SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] - + + + # TODO: API for getting the SConscripts programmatically + # exists upstream: https://github.com/SCons/scons/issues/3625 + def ninja_generate_deps(env): + return sorted([env.File("#SConstruct").path] + glob("**/SConscript", recursive=True)) + env['_NINJA_REGENERATE_DEPS_FUNC'] = ninja_generate_deps + + env['NINJA_REGENERATE_DEPS'] = env.get('NINJA_REGENERATE_DEPS', '${_NINJA_REGENERATE_DEPS_FUNC(__env__)}') + # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. if env["PLATFORM"] == "win32": env.Append(CCFLAGS=["/showIncludes"]) else: - env.Append(CCFLAGS=["-MD", "-MF", "${TARGET}.d"]) + env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) env.AddMethod(CheckNinjaCompdbExpand, "CheckNinjaCompdbExpand") # Provide a way for custom rule authors to easily access command # generation. - env.AddMethod(get_shell_command, "NinjaGetShellCommand") + env.AddMethod(get_generic_shell_command, "NinjaGetGenericShellCommand") + env.AddMethod(get_command, "NinjaGetCommand") env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider") + env.AddMethod(set_build_node_callback, "NinjaSetBuildNodeCallback") # Provides a way for users to handle custom FunctionActions they # want to translate to Ninja. @@ -1587,7 +1729,7 @@ def robust_rule_mapping(var, rule, tool): SCons.Job.Jobs.__init__ = ninja_always_serial ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') - + if NINJA_STATE is None: NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) NINJA_STATE.ninja_bin_path = env.get('NINJA_BIN') @@ -1595,10 +1737,10 @@ def robust_rule_mapping(var, rule, tool): # default to using ninja installed with python module ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', + ninja.__file__, + os.pardir, + 'data', + 'bin', ninja_bin)) if not os.path.exists(NINJA_STATE.ninja_bin_path): # couldn't find it, just give the bin name and hope @@ -1607,7 +1749,7 @@ def robust_rule_mapping(var, rule, tool): # TODO: this is hacking into scons, preferable if there were a less intrusive way # We will subvert the normal builder execute to make sure all the ninja file is dependent - # on all targets generated from any builders + # on all targets generated from any builders SCons_Builder_BuilderBase__execute = SCons.Builder.BuilderBase._execute def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): # this ensures all environments in which a builder executes from will @@ -1666,4 +1808,4 @@ def ninja_execute(self): if not os.path.isdir(os.environ["TMPDIR"]): env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) - env["TEMPFILE"] = NinjaNoResponseFiles \ No newline at end of file + env["TEMPFILE"] = NinjaNoResponseFiles From 4b8406b5f62a755d003e92bbd6fc44af77ed554e Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 31 Dec 2020 16:06:53 +0000 Subject: [PATCH 021/163] Update ninja to new scons layout --- {src/engine/SCons => SCons}/Tool/ninja.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {src/engine/SCons => SCons}/Tool/ninja.py (100%) diff --git a/src/engine/SCons/Tool/ninja.py b/SCons/Tool/ninja.py similarity index 100% rename from src/engine/SCons/Tool/ninja.py rename to SCons/Tool/ninja.py From 3f66125b68bd1335517f416f66202cc2d0741669 Mon Sep 17 00:00:00 2001 From: Mathew Robinson Date: Fri, 24 Jan 2020 17:25:31 -0500 Subject: [PATCH 022/163] [WIP] write a tool to generate build.ninja files from SCons --- SCons/Script/SConscript.py | 3 + SCons/Script/__init__.py | 1 + src/engine/SCons/Tool/ninja.py | 1396 ++++++++++++++++++++++++++++++++ 3 files changed, 1400 insertions(+) create mode 100644 src/engine/SCons/Tool/ninja.py diff --git a/SCons/Script/SConscript.py b/SCons/Script/SConscript.py index 596fca0463..ded0fcfef9 100644 --- a/SCons/Script/SConscript.py +++ b/SCons/Script/SConscript.py @@ -203,9 +203,11 @@ def _SConscript(fs, *files, **kw): if f.rexists(): actual = f.rfile() _file_ = open(actual.get_abspath(), "rb") + SCons.Script.LOADED_SCONSCRIPTS.append(actual.get_abspath()) elif f.srcnode().rexists(): actual = f.srcnode().rfile() _file_ = open(actual.get_abspath(), "rb") + SCons.Script.LOADED_SCONSCRIPTS.append(actual.get_abspath()) elif f.has_src_builder(): # The SConscript file apparently exists in a source # code management system. Build it, but then clear @@ -214,6 +216,7 @@ def _SConscript(fs, *files, **kw): f.build() f.built() f.builder_set(None) + SCons.Script.LOADED_SCONSCRIPTS.append(f.get_abspath()) if f.exists(): _file_ = open(f.get_abspath(), "rb") if _file_: diff --git a/SCons/Script/__init__.py b/SCons/Script/__init__.py index dff15673b1..c7f6a22a93 100644 --- a/SCons/Script/__init__.py +++ b/SCons/Script/__init__.py @@ -187,6 +187,7 @@ def _clear(self): BUILD_TARGETS = TargetList() COMMAND_LINE_TARGETS = [] DEFAULT_TARGETS = [] +LOADED_SCONSCRIPTS = [] # BUILD_TARGETS can be modified in the SConscript files. If so, we # want to treat the modified BUILD_TARGETS list as if they specified diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py new file mode 100644 index 0000000000..b34759e401 --- /dev/null +++ b/src/engine/SCons/Tool/ninja.py @@ -0,0 +1,1396 @@ +# Copyright 2019 MongoDB Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Generate build.ninja files from SCons aliases.""" + +import sys +import os +import importlib +import io +import shutil + +from threading import Lock +from os.path import join as joinpath +from os.path import splitext + +import SCons +from SCons.Action import _string_from_cmd_list, get_default_ENV +from SCons.Util import is_String, is_List +from SCons.Script import COMMAND_LINE_TARGETS, LOADED_SCONSCRIPTS + +NINJA_SYNTAX = "NINJA_SYNTAX" +NINJA_RULES = "__NINJA_CUSTOM_RULES" +NINJA_POOLS = "__NINJA_CUSTOM_POOLS" +NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" +NINJA_BUILD = "NINJA_BUILD" +NINJA_WHEREIS_MEMO = {} +NINJA_STAT_MEMO = {} +MEMO_LOCK = Lock() + +__NINJA_RULE_MAPPING = {} + +# These are the types that get_command can do something with +COMMAND_TYPES = ( + SCons.Action.CommandAction, + SCons.Action.CommandGeneratorAction, +) + + +def _install_action_function(_env, node): + """Install files using the install or copy commands""" + return { + "outputs": get_outputs(node), + "rule": "INSTALL", + "pool": "install_pool", + "inputs": [get_path(src_file(s)) for s in node.sources], + "implicit": get_dependencies(node), + } + + +def _lib_symlink_action_function(_env, node): + """Create shared object symlinks if any need to be created""" + symlinks = getattr(getattr(node, "attributes", None), "shliblinks", None) + + if not symlinks or symlinks is None: + return None + + outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] + inputs = [link.get_path() for link, _ in symlinks] + + return { + "outputs": outputs, + "inputs": inputs, + "rule": "SYMLINK", + "implicit": get_dependencies(node), + } + + +def is_valid_dependent_node(node): + """ + Return True if node is not an alias or is an alias that has children + + This prevents us from making phony targets that depend on other + phony targets that will never have an associated ninja build + target. + + We also have to specify that it's an alias when doing the builder + check because some nodes (like src files) won't have builders but + are valid implicit dependencies. + """ + return not isinstance(node, SCons.Node.Alias.Alias) or node.children() + + +def alias_to_ninja_build(node): + """Convert an Alias node into a Ninja phony target""" + return { + "outputs": get_outputs(node), + "rule": "phony", + "implicit": [ + get_path(n) for n in node.children() if is_valid_dependent_node(n) + ], + } + + +def get_dependencies(node): + """Return a list of dependencies for node.""" + return [get_path(src_file(child)) for child in node.children()] + + +def get_inputs(node): + """Collect the Ninja inputs for node.""" + executor = node.get_executor() + if executor is not None: + inputs = executor.get_all_sources() + else: + inputs = node.sources + + inputs = [get_path(src_file(o)) for o in inputs] + return inputs + + +def get_outputs(node): + """Collect the Ninja outputs for node.""" + executor = node.get_executor() + if executor is not None: + outputs = executor.get_all_targets() + else: + if hasattr(node, "target_peers"): + outputs = node.target_peers + else: + outputs = [node] + + outputs = [get_path(o) for o in outputs] + return outputs + + +class SConsToNinjaTranslator: + """Translates SCons Actions into Ninja build objects.""" + + def __init__(self, env): + self.env = env + self.func_handlers = { + # Skip conftest builders + "_createSource": ninja_noop, + # SCons has a custom FunctionAction that just makes sure the + # target isn't static. We let the commands that ninja runs do + # this check for us. + "SharedFlagChecker": ninja_noop, + # The install builder is implemented as a function action. + "installFunc": _install_action_function, + "LibSymlinksActionFunction": _lib_symlink_action_function, + } + + self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) + + # pylint: disable=too-many-return-statements + def action_to_ninja_build(self, node, action=None): + """Generate build arguments dictionary for node.""" + # Use False since None is a valid value for this Attribute + build = getattr(node.attributes, NINJA_BUILD, False) + if build is not False: + return build + + if node.builder is None: + return None + + if action is None: + action = node.builder.action + + # Ideally this should never happen, and we do try to filter + # Ninja builders out of being sources of ninja builders but I + # can't fix every DAG problem so we just skip ninja_builders + # if we find one + if node.builder == self.env["BUILDERS"]["Ninja"]: + return None + + if isinstance(action, SCons.Action.FunctionAction): + return self.handle_func_action(node, action) + + if isinstance(action, SCons.Action.LazyAction): + # pylint: disable=protected-access + action = action._generate_cache(node.env if node.env else self.env) + return self.action_to_ninja_build(node, action=action) + + if isinstance(action, SCons.Action.ListAction): + return self.handle_list_action(node, action) + + if isinstance(action, COMMAND_TYPES): + return get_command(node.env if node.env else self.env, node, action) + + # Return the node to indicate that SCons is required + return { + "rule": "SCONS", + "outputs": get_outputs(node), + "implicit": get_dependencies(node), + } + + def handle_func_action(self, node, action): + """Determine how to handle the function action.""" + name = action.function_name() + # This is the name given by the Subst/Textfile builders. So return the + # node to indicate that SCons is required + if name == "_action": + return { + "rule": "TEMPLATE", + "outputs": get_outputs(node), + "implicit": get_dependencies(node), + } + + handler = self.func_handlers.get(name, None) + if handler is not None: + return handler(node.env if node.env else self.env, node) + + print( + "Found unhandled function action {}, " + " generating scons command to build\n" + "Note: this is less efficient than Ninja," + " you can write your own ninja build generator for" + " this function using NinjaRegisterFunctionHandler".format(name) + ) + + return { + "rule": "SCONS", + "outputs": get_outputs(node), + "implicit": get_dependencies(node), + } + + # pylint: disable=too-many-branches + def handle_list_action(self, node, action): + """ + Attempt to translate list actions to Ninja. + + List actions are tricky to move to ninja. First we translate + each individual action in the action list to a Ninja + build. Then we process the resulting ninja builds to see if + they are all the same ninja rule. If they are not all the same + rule we cannot make them a single resulting ninja build, so + instead we make them a single SCons invocation to build all of + the targets. + + If they are all the same rule and the rule is CMD we attempt + to combine the cmdlines together using ' && ' which we then + combine into a single ninja build. + + If they are all phony targets we simple combine the outputs + and dependencies. + + If they are all INSTALL rules we combine all of the sources + and outputs. + + If they are all SCONS rules we do the same as if they are not + the same rule and make a build that will use SCons to generate + them. + + If they're all one rule and None of the above rules we throw an Exception. + """ + + results = [ + self.action_to_ninja_build(node, action=act) + for act in action.list + if act is not None + ] + results = [result for result in results if result is not None] + if not results: + return None + + # No need to process the results if we only got a single result + if len(results) == 1: + return results[0] + + all_outputs = list({output for build in results for output in build["outputs"]}) + # If we have no outputs we're done + if not all_outputs: + return None + + # Used to verify if all rules are the same + all_one_rule = len( + [ + r + for r in results + if isinstance(r, dict) and r["rule"] == results[0]["rule"] + ] + ) == len(results) + dependencies = get_dependencies(node) + + if not all_one_rule: + # If they aren't all the same rule use scons to generate these + # outputs. At this time nothing hits this case. + return { + "outputs": all_outputs, + "rule": "SCONS", + "implicit": dependencies, + } + + if results[0]["rule"] == "CMD": + cmdline = "" + for cmd in results: + + # Occasionally a command line will expand to a + # whitespace only string (i.e. ' '). Which is not a + # valid command but does not trigger the empty command + # condition if not cmdstr. So here we strip preceding + # and proceeding whitespace to make strings like the + # above become empty strings and so will be skipped. + cmdstr = cmd["variables"]["cmd"].strip() + if not cmdstr: + continue + + # Skip duplicate commands + if cmdstr in cmdline: + continue + + if cmdline: + cmdline += " && " + + cmdline += cmdstr + + # Remove all preceding and proceeding whitespace + cmdline = cmdline.strip() + + # Make sure we didn't generate an empty cmdline + if cmdline: + ninja_build = { + "outputs": all_outputs, + "rule": "CMD", + "variables": {"cmd": cmdline}, + "implicit": dependencies, + } + + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["pool"] + + return ninja_build + + elif results[0]["rule"] == "phony": + return { + "outputs": all_outputs, + "rule": "phony", + "implicit": dependencies, + } + + elif results[0]["rule"] == "INSTALL": + return { + "outputs": all_outputs, + "rule": "INSTALL", + "pool": "install_pool", + "inputs": [get_path(src_file(s)) for s in node.sources], + "implicit": dependencies, + } + + elif results[0]["rule"] == "SCONS": + return { + "outputs": all_outputs, + "rule": "SCONS", + "inputs": dependencies, + } + + raise Exception("Unhandled list action with rule: " + results[0]["rule"]) + + +# pylint: disable=too-many-instance-attributes +class NinjaState: + """Maintains state of Ninja build system as it's translated from SCons.""" + + def __init__(self, env, writer_class): + self.env = env + self.writer_class = writer_class + self.__generated = False + self.translator = SConsToNinjaTranslator(env) + self.generated_suffixes = env.get("NINJA_GENERATED_SOURCE_SUFFIXES", []) + + # List of generated builds that will be written at a later stage + self.builds = list() + + # List of targets for which we have generated a build. This + # allows us to take multiple Alias nodes as sources and to not + # fail to build if they have overlapping targets. + self.built = set() + + # SCons sets this variable to a function which knows how to do + # shell quoting on whatever platform it's run on. Here we use it + # to make the SCONS_INVOCATION variable properly quoted for things + # like CCFLAGS + escape = env.get("ESCAPE", lambda x: x) + + self.variables = { + "COPY": "cmd.exe /c copy" if sys.platform == "win32" else "cp", + "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format( + sys.executable, + " ".join( + [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] + ), + ), + "SCONS_INVOCATION_W_TARGETS": "{} {}".format( + sys.executable, " ".join([escape(arg) for arg in sys.argv]) + ), + # This must be set to a global default per: + # https://ninja-build.org/manual.html + # + # (The deps section) + "msvc_deps_prefix": "Note: including file:", + } + + self.rules = { + "CMD": { + "command": "cmd /c $cmd" if sys.platform == "win32" else "$cmd", + "description": "Building $out", + }, + # We add the deps processing variables to this below. We + # don't pipe this through cmd.exe on Windows because we + # use this to generate a compile_commands.json database + # which can't use the shell command as it's compile + # command. This does mean that we assume anything using + # CMD_W_DEPS is a straight up compile which is true today. + "CMD_W_DEPS": {"command": "$cmd", "description": "Building $out"}, + "SYMLINK": { + "command": ( + "cmd /c mklink $out $in" + if sys.platform == "win32" + else "ln -s $in $out" + ), + "description": "Symlink $in -> $out", + }, + "INSTALL": { + "command": "$COPY $in $out", + "description": "Install $out", + # On Windows cmd.exe /c copy does not always correctly + # update the timestamp on the output file. This leads + # to a stuck constant timestamp in the Ninja database + # and needless rebuilds. + # + # Adding restat here ensures that Ninja always checks + # the copy updated the timestamp and that Ninja has + # the correct information. + "restat": 1, + }, + "TEMPLATE": { + "command": "$SCONS_INVOCATION $out", + "description": "Rendering $out", + "pool": "scons_pool", + "restat": 1, + }, + "SCONS": { + "command": "$SCONS_INVOCATION $out", + "description": "SCons $out", + "pool": "scons_pool", + # restat + # if present, causes Ninja to re-stat the command's outputs + # after execution of the command. Each output whose + # modification time the command did not change will be + # treated as though it had never needed to be built. This + # may cause the output's reverse dependencies to be removed + # from the list of pending build actions. + # + # We use restat any time we execute SCons because + # SCons calls in Ninja typically create multiple + # targets. But since SCons is doing it's own up to + # date-ness checks it may only update say one of + # them. Restat will find out which of the multiple + # build targets did actually change then only rebuild + # those targets which depend specifically on that + # output. + "restat": 1, + }, + "REGENERATE": { + "command": "$SCONS_INVOCATION_W_TARGETS", + "description": "Regenerating $out", + "generator": 1, + # Console pool restricts to 1 job running at a time, + # it additionally has some special handling about + # passing stdin, stdout, etc to process in this pool + # that we need for SCons to behave correctly when + # regenerating Ninja + "pool": "console", + # Again we restat in case Ninja thought the + # build.ninja should be regenerated but SCons knew + # better. + "restat": 1, + }, + } + + self.pools = { + "install_pool": self.env.GetOption("num_jobs") / 2, + "scons_pool": 1, + } + + if env["PLATFORM"] == "win32": + self.rules["CMD_W_DEPS"]["deps"] = "msvc" + else: + self.rules["CMD_W_DEPS"]["deps"] = "gcc" + self.rules["CMD_W_DEPS"]["depfile"] = "$out.d" + + self.rules.update(env.get(NINJA_RULES, {})) + self.pools.update(env.get(NINJA_POOLS, {})) + + def generate_builds(self, node): + """Generate a ninja build rule for node and it's children.""" + # Filter out nodes with no builder. They are likely source files + # and so no work needs to be done, it will be used in the + # generation for some real target. + # + # Note that all nodes have a builder attribute but it is sometimes + # set to None. So we cannot use a simpler hasattr check here. + if getattr(node, "builder", None) is None: + return + + stack = [[node]] + while stack: + frame = stack.pop() + for child in frame: + outputs = set(get_outputs(child)) + # Check if all the outputs are in self.built, if they + # are we've already seen this node and it's children. + if not outputs.isdisjoint(self.built): + continue + + self.built = self.built.union(outputs) + stack.append(child.children()) + + if isinstance(child, SCons.Node.Alias.Alias): + build = alias_to_ninja_build(child) + elif node.builder is not None: + # Use False since None is a valid value for this attribute + build = getattr(child.attributes, NINJA_BUILD, False) + if build is False: + build = self.translator.action_to_ninja_build(child) + setattr(child.attributes, NINJA_BUILD, build) + else: + build = None + + # Some things are unbuild-able or need not be built in Ninja + if build is None or build == 0: + continue + + self.builds.append(build) + + def is_generated_source(self, output): + """Check if output ends with a known generated suffix.""" + _, suffix = splitext(output) + return suffix in self.generated_suffixes + + def has_generated_sources(self, output): + """ + Determine if output indicates this is a generated header file. + """ + for generated in output: + if self.is_generated_source(generated): + return True + return False + + # pylint: disable=too-many-branches,too-many-locals + def generate(self, ninja_file, fallback_default_target=None): + """ + Generate the build.ninja. + + This should only be called once for the lifetime of this object. + """ + if self.__generated: + return + + content = io.StringIO() + ninja = self.writer_class(content, width=100) + + ninja.comment("Generated by scons. DO NOT EDIT.") + + for pool_name, size in self.pools.items(): + ninja.pool(pool_name, size) + + for var, val in self.variables.items(): + ninja.variable(var, val) + + for rule, kwargs in self.rules.items(): + ninja.rule(rule, **kwargs) + + generated_source_files = { + output + # First find builds which have header files in their outputs. + for build in self.builds + if self.has_generated_sources(build["outputs"]) + for output in build["outputs"] + # Collect only the header files from the builds with them + # in their output. We do this because is_generated_source + # returns True if it finds a header in any of the outputs, + # here we need to filter so we only have the headers and + # not the other outputs. + if self.is_generated_source(output) + } + + if generated_source_files: + ninja.build( + outputs="_generated_sources", + rule="phony", + implicit=list(generated_source_files), + ) + + template_builders = [] + + for build in self.builds: + if build["rule"] == "TEMPLATE": + template_builders.append(build) + continue + + implicit = build.get("implicit", []) + implicit.append(ninja_file) + build["implicit"] = implicit + + # Don't make generated sources depend on each other. We + # have to check that none of the outputs are generated + # sources and none of the direct implicit dependencies are + # generated sources or else we will create a dependency + # cycle. + if ( + generated_source_files + and not build["rule"] == "INSTALL" + and set(build["outputs"]).isdisjoint(generated_source_files) + and set(implicit).isdisjoint(generated_source_files) + ): + + # Make all non-generated source targets depend on + # _generated_sources. We use order_only for generated + # sources so that we don't rebuild the world if one + # generated source was rebuilt. We just need to make + # sure that all of these sources are generated before + # other builds. + build["order_only"] = "_generated_sources" + + # When using a depfile Ninja can only have a single output + # but SCons will usually have emitted an output for every + # thing a command will create because it's caching is much + # more complex than Ninja's. This includes things like DWO + # files. Here we make sure that Ninja only ever sees one + # target when using a depfile. It will still have a command + # that will create all of the outputs but most targets don't + # depend direclty on DWO files and so this assumption is safe + # to make. + rule = self.rules.get(build["rule"]) + + # Some rules like 'phony' and other builtins we don't have + # listed in self.rules so verify that we got a result + # before trying to check if it has a deps key. + if rule is not None and rule.get("deps"): + + # Anything using deps in Ninja can only have a single + # output, but we may have a build which actually + # produces multiple outputs which other targets can + # depend on. Here we slice up the outputs so we have a + # single output which we will use for the "real" + # builder and multiple phony targets that match the + # file names of the remaining outputs. This way any + # build can depend on any output from any build. + first_output, remaining_outputs = build["outputs"][0], build["outputs"][1:] + if remaining_outputs: + ninja.build( + outputs=remaining_outputs, + rule="phony", + implicit=first_output, + ) + + build["outputs"] = first_output + + ninja.build(**build) + + template_builds = dict() + for template_builder in template_builders: + + # Special handling for outputs and implicit since we need to + # aggregate not replace for each builder. + for agg_key in ["outputs", "implicit"]: + new_val = template_builds.get(agg_key, []) + + # Use pop so the key is removed and so the update + # below will not overwrite our aggregated values. + cur_val = template_builder.pop(agg_key, []) + if isinstance(cur_val, list): + new_val += cur_val + else: + new_val.append(cur_val) + template_builds[agg_key] = new_val + + # Collect all other keys + template_builds.update(template_builder) + + if template_builds.get("outputs", []): + ninja.build(**template_builds) + + # Here to teach the ninja file how to regenerate itself. We'll + # never see ourselves in the DAG walk so we can't rely on + # action_to_ninja_build to generate this rule + ninja.build( + ninja_file, + rule="REGENERATE", + implicit=[ + self.env.File("#SConstruct").get_abspath(), + os.path.abspath(__file__), + ] + + LOADED_SCONSCRIPTS, + ) + + ninja.build( + "scons-invocation", + rule="CMD", + pool="console", + variables={"cmd": "echo $SCONS_INVOCATION_W_TARGETS"}, + ) + + # Note the use of CMD_W_DEPS below. CMD_W_DEPS are always + # compile commands in this generator. If we ever change the + # name/s of the rules that include compile commands + # (i.e. something like CC/CXX) we will need to update this + # build to reflect that complete list. + ninja.build( + "compile_commands.json", + rule="CMD", + pool="console", + variables={ + "cmd": "ninja -f {} -t compdb CMD_W_DEPS > compile_commands.json".format( + ninja_file + ) + }, + ) + + ninja.build( + "compiledb", rule="phony", implicit=["compile_commands.json"], + ) + + # Look in SCons's list of DEFAULT_TARGETS, find the ones that + # we generated a ninja build rule for. + scons_default_targets = [ + get_path(tgt) + for tgt in SCons.Script.DEFAULT_TARGETS + if get_path(tgt) in self.built + ] + + # If we found an overlap between SCons's list of default + # targets and the targets we created ninja builds for then use + # those as ninja's default as well. + if scons_default_targets: + ninja.default(" ".join(scons_default_targets)) + + # If not then set the default to the fallback_default_target we were given. + # Otherwise we won't create a default ninja target. + elif fallback_default_target is not None: + ninja.default(fallback_default_target) + + with open(ninja_file, "w") as build_ninja: + build_ninja.write(content.getvalue()) + + self.__generated = True + + +def get_path(node): + """ + Return a fake path if necessary. + + As an example Aliases use this as their target name in Ninja. + """ + if hasattr(node, "get_path"): + return node.get_path() + return str(node) + + +def rfile(node): + """ + Return the repository file for node if it has one. Otherwise return node + """ + if hasattr(node, "rfile"): + return node.rfile() + return node + + +def src_file(node): + """Returns the src code file if it exists.""" + if hasattr(node, "srcnode"): + src = node.srcnode() + if src.stat() is not None: + return src + return get_path(node) + + +# TODO: Make the Rules smarter. Instead of just using a "cmd" rule +# everywhere we should be smarter about generating CC, CXX, LINK, +# etc. rules +def get_command(env, node, action): # pylint: disable=too-many-branches + """Get the command to execute for node.""" + if node.env: + sub_env = node.env + else: + sub_env = env + + executor = node.get_executor() + if executor is not None: + tlist = executor.get_all_targets() + slist = executor.get_all_sources() + else: + if hasattr(node, "target_peers"): + tlist = node.target_peers + else: + tlist = [node] + slist = node.sources + + # Retrieve the repository file for all sources + slist = [rfile(s) for s in slist] + + # Get the dependencies for all targets + implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) + + # Generate a real CommandAction + if isinstance(action, SCons.Action.CommandGeneratorAction): + # pylint: disable=protected-access + action = action._generate(tlist, slist, sub_env, 1, executor=executor) + + rule = "CMD" + + # Actions like CommandAction have a method called process that is + # used by SCons to generate the cmd_line they need to run. So + # check if it's a thing like CommandAction and call it if we can. + if hasattr(action, "process"): + cmd_list, _, _ = action.process(tlist, slist, sub_env, executor=executor) + + # Despite being having "list" in it's name this member is not + # actually a list. It's the pre-subst'd string of the command. We + # use it to determine if the command we generated needs to use a + # custom Ninja rule. By default this redirects CC/CXX commands to + # CMD_W_DEPS but the user can inject custom Ninja rules and tie + # them to commands by using their pre-subst'd string. + rule = __NINJA_RULE_MAPPING.get(action.cmd_list, "CMD") + + cmd = _string_from_cmd_list(cmd_list[0]) + else: + # Anything else works with genstring, this is most commonly hit by + # ListActions which essentially call process on all of their + # commands and concatenate it for us. + genstring = action.genstring(tlist, slist, sub_env) + + # Detect if we have a custom rule for this + # "ListActionCommandAction" type thing. + rule = __NINJA_RULE_MAPPING.get(genstring, "CMD") + + if executor is not None: + cmd = sub_env.subst(genstring, executor=executor) + else: + cmd = sub_env.subst(genstring, target=tlist, source=slist) + + # Since we're only enabling Ninja for developer builds right + # now we skip all Manifest related work on Windows as it's not + # necessary. We shouldn't have gotten here but on Windows + # SCons has a ListAction which shows as a + # CommandGeneratorAction for linking. That ListAction ends + # with a FunctionAction (embedManifestExeCheck, + # embedManifestDllCheck) that simply say "does + # target[0].manifest exist?" if so execute the real command + # action underlying me, otherwise do nothing. + # + # Eventually we'll want to find a way to translate this to + # Ninja but for now, and partially because the existing Ninja + # generator does so, we just disable it all together. + cmd = cmd.replace("\n", " && ").strip() + if env["PLATFORM"] == "win32" and ( + "embedManifestExeCheck" in cmd or "embedManifestDllCheck" in cmd + ): + cmd = " && ".join(cmd.split(" && ")[0:-1]) + + if cmd.endswith("&&"): + cmd = cmd[0:-2].strip() + + outputs = get_outputs(node) + command_env = "" + windows = env["PLATFORM"] == "win32" + + # If win32 and rule == CMD_W_DEPS then we don't want to calculate + # an environment for this command. It's a compile command and + # compiledb doesn't support shell syntax on Windows. We need the + # shell syntax to use environment variables on Windows so we just + # skip this platform / rule combination to keep the compiledb + # working. + # + # On POSIX we can still set environment variables even for compile + # commands so we do so. + if not (windows and rule == "CMD_W_DEPS"): + + # Scan the ENV looking for any keys which do not exist in + # os.environ or differ from it. We assume if it's a new or + # differing key from the process environment then it's + # important to pass down to commands in the Ninja file. + ENV = get_default_ENV(sub_env) + scons_specified_env = { + key: value + for key, value in ENV.items() + if key not in os.environ or os.environ.get(key, None) != value + } + + for key, value in scons_specified_env.items(): + # Ensure that the ENV values are all strings: + if is_List(value): + # If the value is a list, then we assume it is a + # path list, because that's a pretty common list-like + # value to stick in an environment variable: + value = flatten_sequence(value) + value = joinpath(map(str, value)) + else: + # If it isn't a string or a list, then we just coerce + # it to a string, which is the proper way to handle + # Dir and File instances and will produce something + # reasonable for just about everything else: + value = str(value) + + if windows: + command_env += "set '{}={}' && ".format(key, value) + else: + command_env += "{}={} ".format(key, value) + + variables = {"cmd": command_env + cmd} + extra_vars = getattr(node.attributes, "NINJA_EXTRA_VARS", {}) + if extra_vars: + variables.update(extra_vars) + + ninja_build = { + "outputs": outputs, + "inputs": get_inputs(node), + "implicit": implicit, + "rule": rule, + "variables": variables, + } + + # Don't use sub_env here because we require that NINJA_POOL be set + # on a per-builder call basis to prevent accidental strange + # behavior like env['NINJA_POOL'] = 'console' and sub_env can be + # the global Environment object if node.env is None. + # Example: + # + # Allowed: + # + # env.Command("ls", NINJA_POOL="ls_pool") + # + # Not allowed and ignored: + # + # env["NINJA_POOL"] = "ls_pool" + # env.Command("ls") + # + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["NINJA_POOL"] + + return ninja_build + + +def ninja_builder(env, target, source): + """Generate a build.ninja for source.""" + if not isinstance(source, list): + source = [source] + if not isinstance(target, list): + target = [target] + + # We have no COMSTR equivalent so print that we're generating + # here. + print("Generating:", str(target[0])) + + # The environment variable NINJA_SYNTAX points to the + # ninja_syntax.py module from the ninja sources found here: + # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py + # + # This should be vendored into the build sources and it's location + # set in NINJA_SYNTAX. This code block loads the location from + # that variable, gets the absolute path to the vendored file, gets + # it's parent directory then uses importlib to import the module + # dynamically. + ninja_syntax_file = env[NINJA_SYNTAX] + if isinstance(ninja_syntax_file, str): + ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() + ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) + sys.path.append(ninja_syntax_mod_dir) + ninja_syntax_mod_name = os.path.basename(ninja_syntax_file) + ninja_syntax = importlib.import_module(ninja_syntax_mod_name.replace(".py", "")) + + suffix = env.get("NINJA_SUFFIX", "") + if suffix and not suffix[0] == ".": + suffix = "." + suffix + + generated_build_ninja = target[0].get_abspath() + suffix + ninja_state = NinjaState(env, ninja_syntax.Writer) + + for src in source: + ninja_state.generate_builds(src) + + ninja_state.generate(generated_build_ninja, str(source[0])) + + return 0 + + +# pylint: disable=too-few-public-methods +class AlwaysExecAction(SCons.Action.FunctionAction): + """Override FunctionAction.__call__ to always execute.""" + + def __call__(self, *args, **kwargs): + kwargs["execute"] = 1 + return super().__call__(*args, **kwargs) + + +def ninja_print(_cmd, target, _source, env): + """Tag targets with the commands to build them.""" + if target: + for tgt in target: + if ( + tgt.has_builder() + # Use 'is False' because not would still trigger on + # None's which we don't want to regenerate + and getattr(tgt.attributes, NINJA_BUILD, False) is False + and isinstance(tgt.builder.action, COMMAND_TYPES) + ): + ninja_action = get_command(env, tgt, tgt.builder.action) + setattr(tgt.attributes, NINJA_BUILD, ninja_action) + # Preload the attributes dependencies while we're still running + # multithreaded + get_dependencies(tgt) + return 0 + + +def register_custom_handler(env, name, handler): + """Register a custom handler for SCons function actions.""" + env[NINJA_CUSTOM_HANDLERS][name] = handler + + +def register_custom_rule_mapping(env, pre_subst_string, rule): + """Register a custom handler for SCons function actions.""" + global __NINJA_RULE_MAPPING + __NINJA_RULE_MAPPING[pre_subst_string] = rule + + +def register_custom_rule(env, rule, command, description="", deps=None): + """Allows specification of Ninja rules from inside SCons files.""" + rule_obj = { + "command": command, + "description": description if description else "{} $out".format(rule), + } + + if deps is not None: + rule_obj["deps"] = deps + + env[NINJA_RULES][rule] = rule_obj + + +def register_custom_pool(env, pool, size): + """Allows the creation of custom Ninja pools""" + env[NINJA_POOLS][pool] = size + + +def ninja_csig(original): + """Return a dummy csig""" + + def wrapper(self): + name = str(self) + if "SConscript" in name or "SConstruct" in name: + return original(self) + return "dummy_ninja_csig" + + return wrapper + + +def ninja_contents(original): + """Return a dummy content without doing IO""" + + def wrapper(self): + name = str(self) + if "SConscript" in name or "SConstruct" in name: + return original(self) + return bytes("dummy_ninja_contents", encoding="utf-8") + + return wrapper + + +def ninja_stat(_self, path): + """ + Eternally memoized stat call. + + SCons is very aggressive about clearing out cached values. For our + purposes everything should only ever call stat once since we're + running in a no_exec build the file system state should not + change. For these reasons we patch SCons.Node.FS.LocalFS.stat to + use our eternal memoized dictionary. + + Since this is happening during the Node walk it's being run while + threaded, we have to protect adding to the memoized dictionary + with a threading.Lock otherwise many targets miss the memoization + due to racing. + """ + global NINJA_STAT_MEMO + + try: + return NINJA_STAT_MEMO[path] + except KeyError: + try: + result = os.stat(path) + except os.error: + result = None + + with MEMO_LOCK: + NINJA_STAT_MEMO[path] = result + + return result + + +def ninja_noop(*_args, **_kwargs): + """ + A general purpose no-op function. + + There are many things that happen in SCons that we don't need and + also don't return anything. We use this to disable those functions + instead of creating multiple definitions of the same thing. + """ + return None + + +def ninja_whereis(thing, *_args, **_kwargs): + """Replace env.WhereIs with a much faster version""" + global NINJA_WHEREIS_MEMO + + # Optimize for success, this gets called significantly more often + # when the value is already memoized than when it's not. + try: + return NINJA_WHEREIS_MEMO[thing] + except KeyError: + # We do not honor any env['ENV'] or env[*] variables in the + # generated ninja ile. Ninja passes your raw shell environment + # down to it's subprocess so the only sane option is to do the + # same during generation. At some point, if and when we try to + # upstream this, I'm sure a sticking point will be respecting + # env['ENV'] variables and such but it's actually quite + # complicated. I have a naive version but making it always work + # with shell quoting is nigh impossible. So I've decided to + # cross that bridge when it's absolutely required. + path = shutil.which(thing) + NINJA_WHEREIS_MEMO[thing] = path + return path + + +def ninja_always_serial(self, num, taskmaster): + """Replacement for SCons.Job.Jobs constructor which always uses the Serial Job class.""" + # We still set self.num_jobs to num even though it's a lie. The + # only consumer of this attribute is the Parallel Job class AND + # the Main.py function which instantiates a Jobs class. It checks + # if Jobs.num_jobs is equal to options.num_jobs, so if the user + # provides -j12 but we set self.num_jobs = 1 they get an incorrect + # warning about this version of Python not supporting parallel + # builds. So here we lie so the Main.py will not give a false + # warning to users. + self.num_jobs = num + self.job = SCons.Job.Serial(taskmaster) + + +class NinjaEternalTempFile(SCons.Platform.TempFileMunge): + """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" + + def __call__(self, target, source, env, for_signature): + if for_signature: + return self.cmd + + node = target[0] if SCons.Util.is_List(target) else target + if node is not None: + cmdlist = getattr(node.attributes, "tempfile_cmdlist", None) + if cmdlist is not None: + return cmdlist + + cmd = super().__call__(target, source, env, for_signature) + + # If TempFileMunge.__call__ returns a string it means that no + # response file was needed. No processing required so just + # return the command. + if isinstance(cmd, str): + return cmd + + # Strip the removal commands from the command list. + # + # SCons' TempFileMunge class has some very strange + # behavior where it, as part of the command line, tries to + # delete the response file after executing the link + # command. We want to keep those response files since + # Ninja will keep using them over and over. The + # TempFileMunge class creates a cmdlist to do this, a + # common SCons convention for executing commands see: + # https://github.com/SCons/scons/blob/master/src/engine/SCons/Action.py#L949 + # + # This deletion behavior is not configurable. So we wanted + # to remove the deletion command from the command list by + # simply slicing it out here. Unfortunately for some + # strange reason TempFileMunge doesn't make the "rm" + # command it's own list element. It appends it to the + # tempfile argument to cmd[0] (which is CC/CXX) and then + # adds the tempfile again as it's own element. + # + # So we just kind of skip that middle element. Since the + # tempfile is in the command list on it's own at the end we + # can cut it out entirely. This is what I would call + # "likely to break" in future SCons updates. Hopefully it + # breaks because they start doing the right thing and not + # weirdly splitting these arguments up. For reference a + # command list that we get back from the OG TempFileMunge + # looks like this: + # + # [ + # 'g++', + # '@/mats/tempfiles/random_string.lnk\nrm', + # '/mats/tempfiles/random_string.lnk', + # ] + # + # Note the weird newline and rm command in the middle + # element and the lack of TEMPFILEPREFIX on the last + # element. + prefix = env.subst("$TEMPFILEPREFIX") + if not prefix: + prefix = "@" + + new_cmdlist = [cmd[0], prefix + cmd[-1]] + setattr(node.attributes, "tempfile_cmdlist", new_cmdlist) + return new_cmdlist + + def _print_cmd_str(*_args, **_kwargs): + """Disable this method""" + pass + + +def exists(env): + """Enable if called.""" + + # This variable disables the tool when storing the SCons command in the + # generated ninja file to ensure that the ninja tool is not loaded when + # SCons should do actual work as a subprocess of a ninja build. The ninja + # tool is very invasive into the internals of SCons and so should never be + # enabled when SCons needs to build a target. + if env.get("__NINJA_NO", "0") == "1": + return False + + return True + + +def generate(env): + """Generate the NINJA builders.""" + env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") + + # Add the Ninja builder. + always_exec_ninja_action = AlwaysExecAction(ninja_builder, {}) + ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action) + env.Append(BUILDERS={"Ninja": ninja_builder_obj}) + + # This adds the required flags such that the generated compile + # commands will create depfiles as appropriate in the Ninja file. + if env["PLATFORM"] == "win32": + env.Append(CCFLAGS=["/showIncludes"]) + else: + env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) + + # Provides a way for users to handle custom FunctionActions they + # want to translate to Ninja. + env[NINJA_CUSTOM_HANDLERS] = {} + env.AddMethod(register_custom_handler, "NinjaRegisterFunctionHandler") + + # Provides a mechanism for inject custom Ninja rules which can + # then be mapped using NinjaRuleMapping. + env[NINJA_RULES] = {} + env.AddMethod(register_custom_rule, "NinjaRule") + + # Provides a mechanism for inject custom Ninja pools which can + # be used by providing the NINJA_POOL="name" as an + # OverrideEnvironment variable in a builder call. + env[NINJA_POOLS] = {} + env.AddMethod(register_custom_pool, "NinjaPool") + + # Add the ability to register custom NinjaRuleMappings for Command + # builders. We don't store this dictionary in the env to prevent + # accidental deletion of the CC/XXCOM mappings. You can still + # overwrite them if you really want to but you have to explicit + # about it this way. The reason is that if they were accidentally + # deleted you would get a very subtly incorrect Ninja file and + # might not catch it. + env.AddMethod(register_custom_rule_mapping, "NinjaRuleMapping") + env.NinjaRuleMapping("${CCCOM}", "CMD_W_DEPS") + env.NinjaRuleMapping("${CXXCOM}", "CMD_W_DEPS") + + # Make SCons node walk faster by preventing unnecessary work + env.Decider("timestamp-match") + + # Used to determine if a build generates a source file. Ninja + # requires that all generated sources are added as order_only + # dependencies to any builds that *might* use them. + env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] + + # This is the point of no return, anything after this comment + # makes changes to SCons that are irreversible and incompatible + # with a normal SCons build. We return early if __NINJA_NO=1 has + # been given on the command line (i.e. by us in the generated + # ninja file) here to prevent these modifications from happening + # when we want SCons to do work. Everything before this was + # necessary to setup the builder and other functions so that the + # tool can be unconditionally used in the users's SCons files. + + if not exists(env): + return + + # Set a known variable that other tools can query so they can + # behave correctly during ninja generation. + env["GENERATING_NINJA"] = True + + # These methods are no-op'd because they do not work during ninja + # generation, expected to do no work, or simply fail. All of which + # are slow in SCons. So we overwrite them with no logic. + SCons.Node.FS.File.make_ready = ninja_noop + SCons.Node.FS.File.prepare = ninja_noop + SCons.Node.FS.File.push_to_cache = ninja_noop + SCons.Executor.Executor.prepare = ninja_noop + SCons.Node.FS.File.built = ninja_noop + + # We make lstat a no-op because it is only used for SONAME + # symlinks which we're not producing. + SCons.Node.FS.LocalFS.lstat = ninja_noop + + # This is a slow method that isn't memoized. We make it a noop + # since during our generation we will never use the results of + # this or change the results. + SCons.Node.FS.is_up_to_date = ninja_noop + + # We overwrite stat and WhereIs with eternally memoized + # implementations. See the docstring of ninja_stat and + # ninja_whereis for detailed explanations. + SCons.Node.FS.LocalFS.stat = ninja_stat + SCons.Util.WhereIs = ninja_whereis + + # Monkey patch get_csig and get_contents for some classes. It + # slows down the build significantly and we don't need contents or + # content signatures calculated when generating a ninja file since + # we're not doing any SCons caching or building. + SCons.Executor.Executor.get_contents = ninja_contents( + SCons.Executor.Executor.get_contents + ) + SCons.Node.Alias.Alias.get_contents = ninja_contents( + SCons.Node.Alias.Alias.get_contents + ) + SCons.Node.FS.File.get_contents = ninja_contents(SCons.Node.FS.File.get_contents) + SCons.Node.FS.File.get_csig = ninja_csig(SCons.Node.FS.File.get_csig) + SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) + SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig) + + # Replace false Compiling* messages with a more accurate output + # + # We also use this to tag all Nodes with Builders using + # CommandActions with the final command that was used to compile + # it for passing to Ninja. If we don't inject this behavior at + # this stage in the build too much state is lost to generate the + # command at the actual ninja_builder execution time for most + # commands. + # + # We do attempt command generation again in ninja_builder if it + # hasn't been tagged and it seems to work for anything that + # doesn't represent as a non-FunctionAction during the print_func + # call. + env["PRINT_CMD_LINE_FUNC"] = ninja_print + + # This reduces unnecessary subst_list calls to add the compiler to + # the implicit dependencies of targets. Since we encode full paths + # in our generated commands we do not need these slow subst calls + # as executing the command will fail if the file is not found + # where we expect it. + env["IMPLICIT_COMMAND_DEPENDENCIES"] = False + + # Set build to no_exec, our sublcass of FunctionAction will force + # an execution for ninja_builder so this simply effects all other + # Builders. + env.SetOption("no_exec", True) + + # This makes SCons more aggressively cache MD5 signatures in the + # SConsign file. + env.SetOption("max_drift", 1) + + # The Serial job class is SIGNIFICANTLY (almost twice as) faster + # than the Parallel job class for generating Ninja files. So we + # monkey the Jobs constructor to only use the Serial Job class. + SCons.Job.Jobs.__init__ = ninja_always_serial + + # We will eventually need to overwrite TempFileMunge to make it + # handle persistent tempfiles or get an upstreamed change to add + # some configurability to it's behavior in regards to tempfiles. + # + # Set all three environment variables that Python's + # tempfile.mkstemp looks at as it behaves differently on different + # platforms and versions of Python. + os.environ["TMPDIR"] = env.Dir("$BUILD_DIR/response_files").get_abspath() + os.environ["TEMP"] = os.environ["TMPDIR"] + os.environ["TMP"] = os.environ["TMPDIR"] + if not os.path.isdir(os.environ["TMPDIR"]): + env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) + + env["TEMPFILE"] = NinjaEternalTempFile + + # Force the SConsign to be written, we benefit from SCons caching of + # implicit dependencies and conftests. Unfortunately, we have to do this + # using an atexit handler because SCons will not write the file when in a + # no_exec build. + import atexit + + atexit.register(SCons.SConsign.write) From b5e71b179e98a4805a8cbe7b0f1a6da951389252 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 5 May 2020 23:44:01 -0400 Subject: [PATCH 023/163] updated to ninja-next, added some small fixes, and added simple test --- src/engine/SCons/Tool/ninja.py | 1071 +++++++++++++++++------------- test/ninja/CC.py | 66 ++ test/ninja/ninja-fixture/test2.C | 3 + 3 files changed, 668 insertions(+), 472 deletions(-) create mode 100644 test/ninja/CC.py create mode 100644 test/ninja/ninja-fixture/test2.C diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index b34759e401..d1cbafa495 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -1,16 +1,25 @@ -# Copyright 2019 MongoDB Inc. +# Copyright 2020 MongoDB Inc. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: # -# http://www.apache.org/licenses/LICENSE-2.0 +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + """Generate build.ninja files from SCons aliases.""" import sys @@ -18,16 +27,19 @@ import importlib import io import shutil +import shlex +import subprocess -from threading import Lock +from glob import glob from os.path import join as joinpath from os.path import splitext import SCons from SCons.Action import _string_from_cmd_list, get_default_ENV -from SCons.Util import is_String, is_List -from SCons.Script import COMMAND_LINE_TARGETS, LOADED_SCONSCRIPTS +from SCons.Util import is_List, flatten_sequence +from SCons.Script import COMMAND_LINE_TARGETS +NINJA_STATE = None NINJA_SYNTAX = "NINJA_SYNTAX" NINJA_RULES = "__NINJA_CUSTOM_RULES" NINJA_POOLS = "__NINJA_CUSTOM_POOLS" @@ -35,7 +47,6 @@ NINJA_BUILD = "NINJA_BUILD" NINJA_WHEREIS_MEMO = {} NINJA_STAT_MEMO = {} -MEMO_LOCK = Lock() __NINJA_RULE_MAPPING = {} @@ -51,11 +62,43 @@ def _install_action_function(_env, node): return { "outputs": get_outputs(node), "rule": "INSTALL", - "pool": "install_pool", "inputs": [get_path(src_file(s)) for s in node.sources], "implicit": get_dependencies(node), } +def _mkdir_action_function(env, node): + return { + "outputs": get_outputs(node), + "rule": "CMD", + # implicit explicitly omitted, we translate these so they can be + # used by anything that depends on these but commonly this is + # hit with a node that will depend on all of the fake + # srcnode's that SCons will never give us a rule for leading + # to an invalid ninja file. + "variables": { + # On Windows mkdir "-p" is always on + "cmd": "{mkdir} $out".format( + mkdir="mkdir" if env["PLATFORM"] == "win32" else "mkdir -p", + ), + }, + } + +def _copy_action_function(env, node): + return { + "outputs": get_outputs(node), + "inputs": [get_path(src_file(s)) for s in node.sources], + "rule": "CMD", + # implicit explicitly omitted, we translate these so they can be + # used by anything that depends on these but commonly this is + # hit with a node that will depend on all of the fake + # srcnode's that SCons will never give us a rule for leading + # to an invalid ninja file. + "variables": { + # On Windows mkdir "-p" is always on + "cmd": "$COPY $in $out", + }, + } + def _lib_symlink_action_function(_env, node): """Create shared object symlinks if any need to be created""" @@ -87,7 +130,13 @@ def is_valid_dependent_node(node): check because some nodes (like src files) won't have builders but are valid implicit dependencies. """ - return not isinstance(node, SCons.Node.Alias.Alias) or node.children() + if isinstance(node, SCons.Node.Alias.Alias): + return node.children() + + if not node.env: + return True + + return not node.env.get("NINJA_SKIP") def alias_to_ninja_build(node): @@ -96,13 +145,26 @@ def alias_to_ninja_build(node): "outputs": get_outputs(node), "rule": "phony", "implicit": [ - get_path(n) for n in node.children() if is_valid_dependent_node(n) + get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) ], } -def get_dependencies(node): +def get_order_only(node): + """Return a list of order only dependencies for node.""" + if node.prerequisites is None: + return [] + return [get_path(src_file(prereq)) for prereq in node.prerequisites] + + +def get_dependencies(node, skip_sources=False): """Return a list of dependencies for node.""" + if skip_sources: + return [ + get_path(src_file(child)) + for child in node.children() + if child not in node.sources + ] return [get_path(src_file(child)) for child in node.children()] @@ -130,6 +192,7 @@ def get_outputs(node): outputs = [node] outputs = [get_path(o) for o in outputs] + return outputs @@ -147,18 +210,19 @@ def __init__(self, env): "SharedFlagChecker": ninja_noop, # The install builder is implemented as a function action. "installFunc": _install_action_function, + "MkdirFunc": _mkdir_action_function, "LibSymlinksActionFunction": _lib_symlink_action_function, + "Copy" : _copy_action_function } - self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) + self.loaded_custom = False # pylint: disable=too-many-return-statements def action_to_ninja_build(self, node, action=None): """Generate build arguments dictionary for node.""" - # Use False since None is a valid value for this Attribute - build = getattr(node.attributes, NINJA_BUILD, False) - if build is not False: - return build + if not self.loaded_custom: + self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) + self.loaded_custom = True if node.builder is None: return None @@ -166,51 +230,60 @@ def action_to_ninja_build(self, node, action=None): if action is None: action = node.builder.action + if node.env and node.env.get("NINJA_SKIP"): + return None + + build = {} + # Ideally this should never happen, and we do try to filter # Ninja builders out of being sources of ninja builders but I # can't fix every DAG problem so we just skip ninja_builders # if we find one if node.builder == self.env["BUILDERS"]["Ninja"]: - return None - - if isinstance(action, SCons.Action.FunctionAction): - return self.handle_func_action(node, action) - - if isinstance(action, SCons.Action.LazyAction): + build = None + elif isinstance(action, SCons.Action.FunctionAction): + build = self.handle_func_action(node, action) + elif isinstance(action, SCons.Action.LazyAction): # pylint: disable=protected-access action = action._generate_cache(node.env if node.env else self.env) - return self.action_to_ninja_build(node, action=action) - - if isinstance(action, SCons.Action.ListAction): - return self.handle_list_action(node, action) + build = self.action_to_ninja_build(node, action=action) + elif isinstance(action, SCons.Action.ListAction): + build = self.handle_list_action(node, action) + elif isinstance(action, COMMAND_TYPES): + build = get_command(node.env if node.env else self.env, node, action) + else: + raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) - if isinstance(action, COMMAND_TYPES): - return get_command(node.env if node.env else self.env, node, action) + if build is not None: + build["order_only"] = get_order_only(node) - # Return the node to indicate that SCons is required - return { - "rule": "SCONS", - "outputs": get_outputs(node), - "implicit": get_dependencies(node), - } + return build def handle_func_action(self, node, action): """Determine how to handle the function action.""" name = action.function_name() # This is the name given by the Subst/Textfile builders. So return the - # node to indicate that SCons is required + # node to indicate that SCons is required. We skip sources here because + # dependencies don't really matter when we're going to shove these to + # the bottom of ninja's DAG anyway and Textfile builders can have text + # content as their source which doesn't work as an implicit dep in + # ninja. if name == "_action": return { "rule": "TEMPLATE", "outputs": get_outputs(node), - "implicit": get_dependencies(node), + "implicit": get_dependencies(node, skip_sources=True), } - handler = self.func_handlers.get(name, None) if handler is not None: return handler(node.env if node.env else self.env, node) + elif name == "ActionCaller": + action_to_call = str(action).split('(')[0].strip() + handler = self.func_handlers.get(action_to_call, None) + if handler is not None: + return handler(node.env if node.env else self.env, node) - print( + raise Exception( "Found unhandled function action {}, " " generating scons command to build\n" "Note: this is less efficient than Ninja," @@ -218,48 +291,17 @@ def handle_func_action(self, node, action): " this function using NinjaRegisterFunctionHandler".format(name) ) - return { - "rule": "SCONS", - "outputs": get_outputs(node), - "implicit": get_dependencies(node), - } - # pylint: disable=too-many-branches def handle_list_action(self, node, action): - """ - Attempt to translate list actions to Ninja. - - List actions are tricky to move to ninja. First we translate - each individual action in the action list to a Ninja - build. Then we process the resulting ninja builds to see if - they are all the same ninja rule. If they are not all the same - rule we cannot make them a single resulting ninja build, so - instead we make them a single SCons invocation to build all of - the targets. - - If they are all the same rule and the rule is CMD we attempt - to combine the cmdlines together using ' && ' which we then - combine into a single ninja build. - - If they are all phony targets we simple combine the outputs - and dependencies. - - If they are all INSTALL rules we combine all of the sources - and outputs. - - If they are all SCONS rules we do the same as if they are not - the same rule and make a build that will use SCons to generate - them. - - If they're all one rule and None of the above rules we throw an Exception. - """ - + """TODO write this comment""" results = [ self.action_to_ninja_build(node, action=act) for act in action.list if act is not None ] - results = [result for result in results if result is not None] + results = [ + result for result in results if result is not None and result["outputs"] + ] if not results: return None @@ -268,28 +310,7 @@ def handle_list_action(self, node, action): return results[0] all_outputs = list({output for build in results for output in build["outputs"]}) - # If we have no outputs we're done - if not all_outputs: - return None - - # Used to verify if all rules are the same - all_one_rule = len( - [ - r - for r in results - if isinstance(r, dict) and r["rule"] == results[0]["rule"] - ] - ) == len(results) - dependencies = get_dependencies(node) - - if not all_one_rule: - # If they aren't all the same rule use scons to generate these - # outputs. At this time nothing hits this case. - return { - "outputs": all_outputs, - "rule": "SCONS", - "implicit": dependencies, - } + dependencies = list({dep for build in results for dep in build["implicit"]}) if results[0]["rule"] == "CMD": cmdline = "" @@ -322,7 +343,10 @@ def handle_list_action(self, node, action): ninja_build = { "outputs": all_outputs, "rule": "CMD", - "variables": {"cmd": cmdline}, + "variables": { + "cmd": cmdline, + "env": get_command_env(node.env if node.env else self.env), + }, "implicit": dependencies, } @@ -342,18 +366,10 @@ def handle_list_action(self, node, action): return { "outputs": all_outputs, "rule": "INSTALL", - "pool": "install_pool", "inputs": [get_path(src_file(s)) for s in node.sources], "implicit": dependencies, } - elif results[0]["rule"] == "SCONS": - return { - "outputs": all_outputs, - "rule": "SCONS", - "inputs": dependencies, - } - raise Exception("Unhandled list action with rule: " + results[0]["rule"]) @@ -369,7 +385,7 @@ def __init__(self, env, writer_class): self.generated_suffixes = env.get("NINJA_GENERATED_SOURCE_SUFFIXES", []) # List of generated builds that will be written at a later stage - self.builds = list() + self.builds = dict() # List of targets for which we have generated a build. This # allows us to take multiple Alias nodes as sources and to not @@ -383,7 +399,7 @@ def __init__(self, env, writer_class): escape = env.get("ESCAPE", lambda x: x) self.variables = { - "COPY": "cmd.exe /c copy" if sys.platform == "win32" else "cp", + "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format( sys.executable, " ".join( @@ -402,16 +418,46 @@ def __init__(self, env, writer_class): self.rules = { "CMD": { - "command": "cmd /c $cmd" if sys.platform == "win32" else "$cmd", + "command": "cmd /c $env$cmd $in $out" if sys.platform == "win32" else "$env$cmd $in $out", + "description": "Building $out", + "pool": "local_pool", + }, + "GENERATED_CMD": { + "command": "cmd /c $env$cmd" if sys.platform == "win32" else "$env$cmd", "description": "Building $out", + "pool": "local_pool", }, # We add the deps processing variables to this below. We - # don't pipe this through cmd.exe on Windows because we + # don't pipe these through cmd.exe on Windows because we # use this to generate a compile_commands.json database # which can't use the shell command as it's compile - # command. This does mean that we assume anything using - # CMD_W_DEPS is a straight up compile which is true today. - "CMD_W_DEPS": {"command": "$cmd", "description": "Building $out"}, + # command. + "CC": { + "command": "$env$CC @$out.rsp", + "description": "Compiling $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + }, + "CXX": { + "command": "$env$CXX @$out.rsp", + "description": "Compiling $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + }, + "LINK": { + "command": "$env$LINK @$out.rsp", + "description": "Linking $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + "pool": "local_pool", + }, + "AR": { + "command": "$env$AR @$out.rsp", + "description": "Archiving $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + "pool": "local_pool", + }, "SYMLINK": { "command": ( "cmd /c mklink $out $in" @@ -423,6 +469,7 @@ def __init__(self, env, writer_class): "INSTALL": { "command": "$COPY $in $out", "description": "Install $out", + "pool": "install_pool", # On Windows cmd.exe /c copy does not always correctly # update the timestamp on the output file. This leads # to a stuck constant timestamp in the Ninja database @@ -479,65 +526,43 @@ def __init__(self, env, writer_class): } self.pools = { + "local_pool": self.env.GetOption("num_jobs"), "install_pool": self.env.GetOption("num_jobs") / 2, "scons_pool": 1, } - if env["PLATFORM"] == "win32": - self.rules["CMD_W_DEPS"]["deps"] = "msvc" - else: - self.rules["CMD_W_DEPS"]["deps"] = "gcc" - self.rules["CMD_W_DEPS"]["depfile"] = "$out.d" - - self.rules.update(env.get(NINJA_RULES, {})) - self.pools.update(env.get(NINJA_POOLS, {})) - - def generate_builds(self, node): - """Generate a ninja build rule for node and it's children.""" - # Filter out nodes with no builder. They are likely source files - # and so no work needs to be done, it will be used in the - # generation for some real target. - # - # Note that all nodes have a builder attribute but it is sometimes - # set to None. So we cannot use a simpler hasattr check here. - if getattr(node, "builder", None) is None: - return + for rule in ["CC", "CXX"]: + if env["PLATFORM"] == "win32": + self.rules[rule]["deps"] = "msvc" + else: + self.rules[rule]["deps"] = "gcc" + self.rules[rule]["depfile"] = "$out.d" - stack = [[node]] - while stack: - frame = stack.pop() - for child in frame: - outputs = set(get_outputs(child)) - # Check if all the outputs are in self.built, if they - # are we've already seen this node and it's children. - if not outputs.isdisjoint(self.built): - continue + def add_build(self, node): + if not node.has_builder(): + return False - self.built = self.built.union(outputs) - stack.append(child.children()) - - if isinstance(child, SCons.Node.Alias.Alias): - build = alias_to_ninja_build(child) - elif node.builder is not None: - # Use False since None is a valid value for this attribute - build = getattr(child.attributes, NINJA_BUILD, False) - if build is False: - build = self.translator.action_to_ninja_build(child) - setattr(child.attributes, NINJA_BUILD, build) - else: - build = None + if isinstance(node, SCons.Node.Alias.Alias): + build = alias_to_ninja_build(node) + else: + build = self.translator.action_to_ninja_build(node) - # Some things are unbuild-able or need not be built in Ninja - if build is None or build == 0: - continue + # Some things are unbuild-able or need not be built in Ninja + if build is None: + return False - self.builds.append(build) + node_string = str(node) + if node_string in self.builds: + raise Exception("Node {} added to ninja build state more than once".format(node_string)) + self.builds[node_string] = build + self.built.update(build["outputs"]) + return True def is_generated_source(self, output): """Check if output ends with a known generated suffix.""" _, suffix = splitext(output) return suffix in self.generated_suffixes - + def has_generated_sources(self, output): """ Determine if output indicates this is a generated header file. @@ -548,7 +573,7 @@ def has_generated_sources(self, output): return False # pylint: disable=too-many-branches,too-many-locals - def generate(self, ninja_file, fallback_default_target=None): + def generate(self, ninja_file): """ Generate the build.ninja. @@ -557,6 +582,9 @@ def generate(self, ninja_file, fallback_default_target=None): if self.__generated: return + self.rules.update(self.env.get(NINJA_RULES, {})) + self.pools.update(self.env.get(NINJA_POOLS, {})) + content = io.StringIO() ninja = self.writer_class(content, width=100) @@ -571,10 +599,10 @@ def generate(self, ninja_file, fallback_default_target=None): for rule, kwargs in self.rules.items(): ninja.rule(rule, **kwargs) - generated_source_files = { + generated_source_files = sorted({ output # First find builds which have header files in their outputs. - for build in self.builds + for build in self.builds.values() if self.has_generated_sources(build["outputs"]) for output in build["outputs"] # Collect only the header files from the builds with them @@ -583,25 +611,24 @@ def generate(self, ninja_file, fallback_default_target=None): # here we need to filter so we only have the headers and # not the other outputs. if self.is_generated_source(output) - } + }) if generated_source_files: ninja.build( outputs="_generated_sources", rule="phony", - implicit=list(generated_source_files), + implicit=generated_source_files ) template_builders = [] - for build in self.builds: + for build in [self.builds[key] for key in sorted(self.builds.keys())]: if build["rule"] == "TEMPLATE": template_builders.append(build) continue - implicit = build.get("implicit", []) - implicit.append(ninja_file) - build["implicit"] = implicit + if "implicit" in build: + build["implicit"].sort() # Don't make generated sources depend on each other. We # have to check that none of the outputs are generated @@ -612,7 +639,7 @@ def generate(self, ninja_file, fallback_default_target=None): generated_source_files and not build["rule"] == "INSTALL" and set(build["outputs"]).isdisjoint(generated_source_files) - and set(implicit).isdisjoint(generated_source_files) + and set(build.get("implicit", [])).isdisjoint(generated_source_files) ): # Make all non-generated source targets depend on @@ -621,7 +648,11 @@ def generate(self, ninja_file, fallback_default_target=None): # generated source was rebuilt. We just need to make # sure that all of these sources are generated before # other builds. - build["order_only"] = "_generated_sources" + order_only = build.get("order_only", []) + order_only.append("_generated_sources") + build["order_only"] = order_only + if "order_only" in build: + build["order_only"].sort() # When using a depfile Ninja can only have a single output # but SCons will usually have emitted an output for every @@ -637,26 +668,31 @@ def generate(self, ninja_file, fallback_default_target=None): # Some rules like 'phony' and other builtins we don't have # listed in self.rules so verify that we got a result # before trying to check if it has a deps key. - if rule is not None and rule.get("deps"): - - # Anything using deps in Ninja can only have a single - # output, but we may have a build which actually - # produces multiple outputs which other targets can - # depend on. Here we slice up the outputs so we have a - # single output which we will use for the "real" - # builder and multiple phony targets that match the - # file names of the remaining outputs. This way any - # build can depend on any output from any build. - first_output, remaining_outputs = build["outputs"][0], build["outputs"][1:] + # + # Anything using deps or rspfile in Ninja can only have a single + # output, but we may have a build which actually produces + # multiple outputs which other targets can depend on. Here we + # slice up the outputs so we have a single output which we will + # use for the "real" builder and multiple phony targets that + # match the file names of the remaining outputs. This way any + # build can depend on any output from any build. + build["outputs"].sort() + if rule is not None and (rule.get("deps") or rule.get("rspfile")): + first_output, remaining_outputs = ( + build["outputs"][0], + build["outputs"][1:], + ) + if remaining_outputs: ninja.build( - outputs=remaining_outputs, - rule="phony", - implicit=first_output, + outputs=remaining_outputs, rule="phony", implicit=first_output, ) build["outputs"] = first_output + if "inputs" in build: + build["inputs"].sort() + ninja.build(**build) template_builds = dict() @@ -682,37 +718,37 @@ def generate(self, ninja_file, fallback_default_target=None): if template_builds.get("outputs", []): ninja.build(**template_builds) - # Here to teach the ninja file how to regenerate itself. We'll - # never see ourselves in the DAG walk so we can't rely on - # action_to_ninja_build to generate this rule + # We have to glob the SCons files here to teach the ninja file + # how to regenerate itself. We'll never see ourselves in the + # DAG walk so we can't rely on action_to_ninja_build to + # generate this rule even though SCons should know we're + # dependent on SCons files. + # + # TODO: We're working on getting an API into SCons that will + # allow us to query the actual SConscripts used. Right now + # this glob method has deficiencies like skipping + # jstests/SConscript and being specific to the MongoDB + # repository layout. ninja.build( - ninja_file, + self.env.File(ninja_file).path, rule="REGENERATE", implicit=[ - self.env.File("#SConstruct").get_abspath(), - os.path.abspath(__file__), + self.env.File("#SConstruct").path, + __file__, ] - + LOADED_SCONSCRIPTS, + + sorted(glob("src/**/SConscript", recursive=True)), ) - ninja.build( - "scons-invocation", - rule="CMD", - pool="console", - variables={"cmd": "echo $SCONS_INVOCATION_W_TARGETS"}, - ) - - # Note the use of CMD_W_DEPS below. CMD_W_DEPS are always - # compile commands in this generator. If we ever change the - # name/s of the rules that include compile commands - # (i.e. something like CC/CXX) we will need to update this - # build to reflect that complete list. + # If we ever change the name/s of the rules that include + # compile commands (i.e. something like CC) we will need to + # update this build to reflect that complete list. ninja.build( "compile_commands.json", rule="CMD", pool="console", + implicit=[ninja_file], variables={ - "cmd": "ninja -f {} -t compdb CMD_W_DEPS > compile_commands.json".format( + "cmd": "ninja -f {} -t compdb CC CXX > compile_commands.json".format( ninja_file ) }, @@ -736,11 +772,6 @@ def generate(self, ninja_file, fallback_default_target=None): if scons_default_targets: ninja.default(" ".join(scons_default_targets)) - # If not then set the default to the fallback_default_target we were given. - # Otherwise we won't create a default ninja target. - elif fallback_default_target is not None: - ninja.default(fallback_default_target) - with open(ninja_file, "w") as build_ninja: build_ninja.write(content.getvalue()) @@ -776,9 +807,158 @@ def src_file(node): return get_path(node) -# TODO: Make the Rules smarter. Instead of just using a "cmd" rule -# everywhere we should be smarter about generating CC, CXX, LINK, -# etc. rules +def get_comstr(env, action, targets, sources): + """Get the un-substituted string for action.""" + # Despite being having "list" in it's name this member is not + # actually a list. It's the pre-subst'd string of the command. We + # use it to determine if the command we're about to generate needs + # to use a custom Ninja rule. By default this redirects CC, CXX, + # AR, SHLINK, and LINK commands to their respective rules but the + # user can inject custom Ninja rules and tie them to commands by + # using their pre-subst'd string. + if hasattr(action, "process"): + return action.cmd_list + + return action.genstring(targets, sources, env) + + +def get_command_env(env): + """ + Return a string that sets the enrivonment for any environment variables that + differ between the OS environment and the SCons command ENV. + + It will be compatible with the default shell of the operating system. + """ + try: + return env["NINJA_ENV_VAR_CACHE"] + except KeyError: + pass + + # Scan the ENV looking for any keys which do not exist in + # os.environ or differ from it. We assume if it's a new or + # differing key from the process environment then it's + # important to pass down to commands in the Ninja file. + ENV = get_default_ENV(env) + scons_specified_env = { + key: value + for key, value in ENV.items() + if key not in os.environ or os.environ.get(key, None) != value + } + + windows = env["PLATFORM"] == "win32" + command_env = "" + for key, value in scons_specified_env.items(): + # Ensure that the ENV values are all strings: + if is_List(value): + # If the value is a list, then we assume it is a + # path list, because that's a pretty common list-like + # value to stick in an environment variable: + value = flatten_sequence(value) + value = joinpath(map(str, value)) + else: + # If it isn't a string or a list, then we just coerce + # it to a string, which is the proper way to handle + # Dir and File instances and will produce something + # reasonable for just about everything else: + value = str(value) + + if windows: + command_env += "set '{}={}' && ".format(key, value) + else: + command_env += "{}={} ".format(key, value) + + env["NINJA_ENV_VAR_CACHE"] = command_env + return command_env + + +def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False): + """Generate a response file command provider for rule name.""" + + # If win32 using the environment with a response file command will cause + # ninja to fail to create the response file. Additionally since these rules + # generally are not piping through cmd.exe /c any environment variables will + # make CreateProcess fail to start. + # + # On POSIX we can still set environment variables even for compile + # commands so we do so. + use_command_env = not env["PLATFORM"] == "win32" + if "$" in tool: + tool_is_dynamic = True + + def get_response_file_command(env, node, action, targets, sources, executor=None): + if hasattr(action, "process"): + cmd_list, _, _ = action.process(targets, sources, env, executor=executor) + cmd_list = [str(c).replace("$", "$$") for c in cmd_list[0]] + else: + command = generate_command( + env, node, action, targets, sources, executor=executor + ) + cmd_list = shlex.split(command) + + if tool_is_dynamic: + tool_command = env.subst( + tool, target=targets, source=sources, executor=executor + ) + else: + tool_command = tool + + try: + # Add 1 so we always keep the actual tool inside of cmd + tool_idx = cmd_list.index(tool_command) + 1 + except ValueError: + raise Exception( + "Could not find tool {} in {} generated from {}".format( + tool_command, cmd_list, get_comstr(env, action, targets, sources) + ) + ) + + cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] + rsp_content = " ".join(rsp_content) + + variables = {"rspc": rsp_content} + variables[rule] = cmd + if use_command_env: + variables["env"] = get_command_env(env) + return rule, variables + + return get_response_file_command + + +def generate_command(env, node, action, targets, sources, executor=None): + # Actions like CommandAction have a method called process that is + # used by SCons to generate the cmd_line they need to run. So + # check if it's a thing like CommandAction and call it if we can. + if hasattr(action, "process"): + cmd_list, _, _ = action.process(targets, sources, env, executor=executor) + cmd = _string_from_cmd_list(cmd_list[0]) + else: + # Anything else works with genstring, this is most commonly hit by + # ListActions which essentially call process on all of their + # commands and concatenate it for us. + genstring = action.genstring(targets, sources, env) + if executor is not None: + cmd = env.subst(genstring, executor=executor) + else: + cmd = env.subst(genstring, targets, sources) + + cmd = cmd.replace("\n", " && ").strip() + if cmd.endswith("&&"): + cmd = cmd[0:-2].strip() + + # Escape dollars as necessary + return cmd.replace("$", "$$") + + +def get_shell_command(env, node, action, targets, sources, executor=None): + return ( + "GENERATED_CMD", + { + "cmd": generate_command(env, node, action, targets, sources, executor=None), + "env": get_command_env(env), + }, + ) + + def get_command(env, node, action): # pylint: disable=too-many-branches """Get the command to execute for node.""" if node.env: @@ -800,121 +980,26 @@ def get_command(env, node, action): # pylint: disable=too-many-branches # Retrieve the repository file for all sources slist = [rfile(s) for s in slist] - # Get the dependencies for all targets - implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) - # Generate a real CommandAction if isinstance(action, SCons.Action.CommandGeneratorAction): # pylint: disable=protected-access action = action._generate(tlist, slist, sub_env, 1, executor=executor) - rule = "CMD" - - # Actions like CommandAction have a method called process that is - # used by SCons to generate the cmd_line they need to run. So - # check if it's a thing like CommandAction and call it if we can. - if hasattr(action, "process"): - cmd_list, _, _ = action.process(tlist, slist, sub_env, executor=executor) - - # Despite being having "list" in it's name this member is not - # actually a list. It's the pre-subst'd string of the command. We - # use it to determine if the command we generated needs to use a - # custom Ninja rule. By default this redirects CC/CXX commands to - # CMD_W_DEPS but the user can inject custom Ninja rules and tie - # them to commands by using their pre-subst'd string. - rule = __NINJA_RULE_MAPPING.get(action.cmd_list, "CMD") - - cmd = _string_from_cmd_list(cmd_list[0]) - else: - # Anything else works with genstring, this is most commonly hit by - # ListActions which essentially call process on all of their - # commands and concatenate it for us. - genstring = action.genstring(tlist, slist, sub_env) + variables = {} - # Detect if we have a custom rule for this - # "ListActionCommandAction" type thing. - rule = __NINJA_RULE_MAPPING.get(genstring, "CMD") - - if executor is not None: - cmd = sub_env.subst(genstring, executor=executor) - else: - cmd = sub_env.subst(genstring, target=tlist, source=slist) - - # Since we're only enabling Ninja for developer builds right - # now we skip all Manifest related work on Windows as it's not - # necessary. We shouldn't have gotten here but on Windows - # SCons has a ListAction which shows as a - # CommandGeneratorAction for linking. That ListAction ends - # with a FunctionAction (embedManifestExeCheck, - # embedManifestDllCheck) that simply say "does - # target[0].manifest exist?" if so execute the real command - # action underlying me, otherwise do nothing. - # - # Eventually we'll want to find a way to translate this to - # Ninja but for now, and partially because the existing Ninja - # generator does so, we just disable it all together. - cmd = cmd.replace("\n", " && ").strip() - if env["PLATFORM"] == "win32" and ( - "embedManifestExeCheck" in cmd or "embedManifestDllCheck" in cmd - ): - cmd = " && ".join(cmd.split(" && ")[0:-1]) - - if cmd.endswith("&&"): - cmd = cmd[0:-2].strip() - - outputs = get_outputs(node) - command_env = "" - windows = env["PLATFORM"] == "win32" - - # If win32 and rule == CMD_W_DEPS then we don't want to calculate - # an environment for this command. It's a compile command and - # compiledb doesn't support shell syntax on Windows. We need the - # shell syntax to use environment variables on Windows so we just - # skip this platform / rule combination to keep the compiledb - # working. - # - # On POSIX we can still set environment variables even for compile - # commands so we do so. - if not (windows and rule == "CMD_W_DEPS"): - - # Scan the ENV looking for any keys which do not exist in - # os.environ or differ from it. We assume if it's a new or - # differing key from the process environment then it's - # important to pass down to commands in the Ninja file. - ENV = get_default_ENV(sub_env) - scons_specified_env = { - key: value - for key, value in ENV.items() - if key not in os.environ or os.environ.get(key, None) != value - } + comstr = get_comstr(sub_env, action, tlist, slist) + if not comstr: + return None - for key, value in scons_specified_env.items(): - # Ensure that the ENV values are all strings: - if is_List(value): - # If the value is a list, then we assume it is a - # path list, because that's a pretty common list-like - # value to stick in an environment variable: - value = flatten_sequence(value) - value = joinpath(map(str, value)) - else: - # If it isn't a string or a list, then we just coerce - # it to a string, which is the proper way to handle - # Dir and File instances and will produce something - # reasonable for just about everything else: - value = str(value) - - if windows: - command_env += "set '{}={}' && ".format(key, value) - else: - command_env += "{}={} ".format(key, value) + provider = __NINJA_RULE_MAPPING.get(comstr, get_shell_command) + rule, variables = provider(sub_env, node, action, tlist, slist, executor=executor) - variables = {"cmd": command_env + cmd} - extra_vars = getattr(node.attributes, "NINJA_EXTRA_VARS", {}) - if extra_vars: - variables.update(extra_vars) + # Get the dependencies for all targets + implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) ninja_build = { - "outputs": outputs, + "order_only": get_order_only(node), + "outputs": get_outputs(node), "inputs": get_inputs(node), "implicit": implicit, "rule": rule, @@ -953,37 +1038,30 @@ def ninja_builder(env, target, source): # here. print("Generating:", str(target[0])) - # The environment variable NINJA_SYNTAX points to the - # ninja_syntax.py module from the ninja sources found here: - # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py - # - # This should be vendored into the build sources and it's location - # set in NINJA_SYNTAX. This code block loads the location from - # that variable, gets the absolute path to the vendored file, gets - # it's parent directory then uses importlib to import the module - # dynamically. - ninja_syntax_file = env[NINJA_SYNTAX] - if isinstance(ninja_syntax_file, str): - ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() - ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) - sys.path.append(ninja_syntax_mod_dir) - ninja_syntax_mod_name = os.path.basename(ninja_syntax_file) - ninja_syntax = importlib.import_module(ninja_syntax_mod_name.replace(".py", "")) - - suffix = env.get("NINJA_SUFFIX", "") - if suffix and not suffix[0] == ".": - suffix = "." + suffix + generated_build_ninja = target[0].get_abspath() + NINJA_STATE.generate(generated_build_ninja) + if env.get("DISABLE_AUTO_NINJA") != True: + print("Executing:", str(target[0])) - generated_build_ninja = target[0].get_abspath() + suffix - ninja_state = NinjaState(env, ninja_syntax.Writer) - - for src in source: - ninja_state.generate_builds(src) - - ninja_state.generate(generated_build_ninja, str(source[0])) - - return 0 + def execute_ninja(): + proc = subprocess.Popen( ['ninja', '-f', generated_build_ninja], + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + universal_newlines=True + ) + for stdout_line in iter(proc.stdout.readline, ""): + yield stdout_line + proc.stdout.close() + return_code = proc.wait() + if return_code: + raise subprocess.CalledProcessError(return_code, 'ninja') + + for output in execute_ninja(): + output = output.strip() + sys.stdout.write('\x1b[2K') # erase previous line + sys.stdout.write(output + "\r") + sys.stdout.flush() # pylint: disable=too-few-public-methods class AlwaysExecAction(SCons.Action.FunctionAction): @@ -994,25 +1072,6 @@ def __call__(self, *args, **kwargs): return super().__call__(*args, **kwargs) -def ninja_print(_cmd, target, _source, env): - """Tag targets with the commands to build them.""" - if target: - for tgt in target: - if ( - tgt.has_builder() - # Use 'is False' because not would still trigger on - # None's which we don't want to regenerate - and getattr(tgt.attributes, NINJA_BUILD, False) is False - and isinstance(tgt.builder.action, COMMAND_TYPES) - ): - ninja_action = get_command(env, tgt, tgt.builder.action) - setattr(tgt.attributes, NINJA_BUILD, ninja_action) - # Preload the attributes dependencies while we're still running - # multithreaded - get_dependencies(tgt) - return 0 - - def register_custom_handler(env, name, handler): """Register a custom handler for SCons function actions.""" env[NINJA_CUSTOM_HANDLERS][name] = handler @@ -1024,7 +1083,7 @@ def register_custom_rule_mapping(env, pre_subst_string, rule): __NINJA_RULE_MAPPING[pre_subst_string] = rule -def register_custom_rule(env, rule, command, description="", deps=None): +def register_custom_rule(env, rule, command, description="", deps=None, pool=None): """Allows specification of Ninja rules from inside SCons files.""" rule_obj = { "command": command, @@ -1034,6 +1093,9 @@ def register_custom_rule(env, rule, command, description="", deps=None): if deps is not None: rule_obj["deps"] = deps + if pool is not None: + rule_obj["pool"] = pool + env[NINJA_RULES][rule] = rule_obj @@ -1075,11 +1137,6 @@ def ninja_stat(_self, path): running in a no_exec build the file system state should not change. For these reasons we patch SCons.Node.FS.LocalFS.stat to use our eternal memoized dictionary. - - Since this is happening during the Node walk it's being run while - threaded, we have to protect adding to the memoized dictionary - with a threading.Lock otherwise many targets miss the memoization - due to racing. """ global NINJA_STAT_MEMO @@ -1091,9 +1148,7 @@ def ninja_stat(_self, path): except os.error: result = None - with MEMO_LOCK: - NINJA_STAT_MEMO[path] = result - + NINJA_STAT_MEMO[path] = result return result @@ -1145,71 +1200,11 @@ def ninja_always_serial(self, num, taskmaster): self.job = SCons.Job.Serial(taskmaster) -class NinjaEternalTempFile(SCons.Platform.TempFileMunge): +class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" def __call__(self, target, source, env, for_signature): - if for_signature: - return self.cmd - - node = target[0] if SCons.Util.is_List(target) else target - if node is not None: - cmdlist = getattr(node.attributes, "tempfile_cmdlist", None) - if cmdlist is not None: - return cmdlist - - cmd = super().__call__(target, source, env, for_signature) - - # If TempFileMunge.__call__ returns a string it means that no - # response file was needed. No processing required so just - # return the command. - if isinstance(cmd, str): - return cmd - - # Strip the removal commands from the command list. - # - # SCons' TempFileMunge class has some very strange - # behavior where it, as part of the command line, tries to - # delete the response file after executing the link - # command. We want to keep those response files since - # Ninja will keep using them over and over. The - # TempFileMunge class creates a cmdlist to do this, a - # common SCons convention for executing commands see: - # https://github.com/SCons/scons/blob/master/src/engine/SCons/Action.py#L949 - # - # This deletion behavior is not configurable. So we wanted - # to remove the deletion command from the command list by - # simply slicing it out here. Unfortunately for some - # strange reason TempFileMunge doesn't make the "rm" - # command it's own list element. It appends it to the - # tempfile argument to cmd[0] (which is CC/CXX) and then - # adds the tempfile again as it's own element. - # - # So we just kind of skip that middle element. Since the - # tempfile is in the command list on it's own at the end we - # can cut it out entirely. This is what I would call - # "likely to break" in future SCons updates. Hopefully it - # breaks because they start doing the right thing and not - # weirdly splitting these arguments up. For reference a - # command list that we get back from the OG TempFileMunge - # looks like this: - # - # [ - # 'g++', - # '@/mats/tempfiles/random_string.lnk\nrm', - # '/mats/tempfiles/random_string.lnk', - # ] - # - # Note the weird newline and rm command in the middle - # element and the lack of TEMPFILEPREFIX on the last - # element. - prefix = env.subst("$TEMPFILEPREFIX") - if not prefix: - prefix = "@" - - new_cmdlist = [cmd[0], prefix + cmd[-1]] - setattr(node.attributes, "tempfile_cmdlist", new_cmdlist) - return new_cmdlist + return self.cmd def _print_cmd_str(*_args, **_kwargs): """Disable this method""" @@ -1229,9 +1224,22 @@ def exists(env): return True +added = None def generate(env): """Generate the NINJA builders.""" + from SCons.Script import AddOption, GetOption + global added + if not added: + added = 1 + AddOption('--disable-auto-ninja', + dest='disable_auto_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + env["DISABLE_AUTO_NINJA"] = GetOption('disable_auto_ninja') + env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") # Add the Ninja builder. @@ -1239,6 +1247,15 @@ def generate(env): ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action) env.Append(BUILDERS={"Ninja": ninja_builder_obj}) + env["NINJA_PREFIX"] = env.get("NINJA_PREFIX", "build") + env["NINJA_SUFFIX"] = env.get("NINJA_SUFFIX", "ninja") + env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") + + ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") + ninja_file = env.Ninja(target=ninja_file_name, source=[]) + env.AlwaysBuild(ninja_file) + env.Alias("$NINJA_ALIAS_NAME", ninja_file) + # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. if env["PLATFORM"] == "win32": @@ -1246,6 +1263,11 @@ def generate(env): else: env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) + # Provide a way for custom rule authors to easily access command + # generation. + env.AddMethod(get_shell_command, "NinjaGetShellCommand") + env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider") + # Provides a way for users to handle custom FunctionActions they # want to translate to Ninja. env[NINJA_CUSTOM_HANDLERS] = {} @@ -1270,8 +1292,40 @@ def generate(env): # deleted you would get a very subtly incorrect Ninja file and # might not catch it. env.AddMethod(register_custom_rule_mapping, "NinjaRuleMapping") - env.NinjaRuleMapping("${CCCOM}", "CMD_W_DEPS") - env.NinjaRuleMapping("${CXXCOM}", "CMD_W_DEPS") + + # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks + # without relying on the SCons hacks that SCons uses by default. + if env["PLATFORM"] == "win32": + from SCons.Tool.mslink import compositeLinkAction + + if env["LINKCOM"] == compositeLinkAction: + env[ + "LINKCOM" + ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' + env[ + "SHLINKCOM" + ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' + + # Normally in SCons actions for the Program and *Library builders + # will return "${*COM}" as their pre-subst'd command line. However + # if a user in a SConscript overwrites those values via key access + # like env["LINKCOM"] = "$( $ICERUN $)" + env["LINKCOM"] then + # those actions no longer return the "bracketted" string and + # instead return something that looks more expanded. So to + # continue working even if a user has done this we map both the + # "bracketted" and semi-expanded versions. + def robust_rule_mapping(var, rule, tool): + provider = gen_get_response_file_command(env, rule, tool) + env.NinjaRuleMapping("${" + var + "}", provider) + env.NinjaRuleMapping(env[var], provider) + + robust_rule_mapping("CCCOM", "CC", "$CC") + robust_rule_mapping("SHCCCOM", "CC", "$CC") + robust_rule_mapping("CXXCOM", "CXX", "$CXX") + robust_rule_mapping("SHCXXCOM", "CXX", "$CXX") + robust_rule_mapping("LINKCOM", "LINK", "$LINK") + robust_rule_mapping("SHLINKCOM", "LINK", "$SHLINK") + robust_rule_mapping("ARCOM", "AR", "$AR") # Make SCons node walk faster by preventing unnecessary work env.Decider("timestamp-match") @@ -1281,6 +1335,21 @@ def generate(env): # dependencies to any builds that *might* use them. env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] + if env["PLATFORM"] != "win32" and env.get("RANLIBCOM"): + # There is no way to translate the ranlib list action into + # Ninja so add the s flag and disable ranlib. + # + # This is equivalent to Meson. + # https://github.com/mesonbuild/meson/blob/master/mesonbuild/linkers.py#L143 + old_arflags = str(env["ARFLAGS"]) + if "s" not in old_arflags: + old_arflags += "s" + + env["ARFLAGS"] = SCons.Util.CLVar([old_arflags]) + + # Disable running ranlib, since we added 's' above + env["RANLIBCOM"] = "" + # This is the point of no return, anything after this comment # makes changes to SCons that are irreversible and incompatible # with a normal SCons build. We return early if __NINJA_NO=1 has @@ -1304,7 +1373,9 @@ def generate(env): SCons.Node.FS.File.prepare = ninja_noop SCons.Node.FS.File.push_to_cache = ninja_noop SCons.Executor.Executor.prepare = ninja_noop + SCons.Taskmaster.Task.prepare = ninja_noop SCons.Node.FS.File.built = ninja_noop + SCons.Node.Node.visited = ninja_noop # We make lstat a no-op because it is only used for SONAME # symlinks which we're not producing. @@ -1336,20 +1407,8 @@ def generate(env): SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig) - # Replace false Compiling* messages with a more accurate output - # - # We also use this to tag all Nodes with Builders using - # CommandActions with the final command that was used to compile - # it for passing to Ninja. If we don't inject this behavior at - # this stage in the build too much state is lost to generate the - # command at the actual ninja_builder execution time for most - # commands. - # - # We do attempt command generation again in ninja_builder if it - # hasn't been tagged and it seems to work for anything that - # doesn't represent as a non-FunctionAction during the print_func - # call. - env["PRINT_CMD_LINE_FUNC"] = ninja_print + # Replace false action messages with nothing. + env["PRINT_CMD_LINE_FUNC"] = ninja_noop # This reduces unnecessary subst_list calls to add the compiler to # the implicit dependencies of targets. Since we encode full paths @@ -1358,11 +1417,6 @@ def generate(env): # where we expect it. env["IMPLICIT_COMMAND_DEPENDENCIES"] = False - # Set build to no_exec, our sublcass of FunctionAction will force - # an execution for ninja_builder so this simply effects all other - # Builders. - env.SetOption("no_exec", True) - # This makes SCons more aggressively cache MD5 signatures in the # SConsign file. env.SetOption("max_drift", 1) @@ -1372,6 +1426,84 @@ def generate(env): # monkey the Jobs constructor to only use the Serial Job class. SCons.Job.Jobs.__init__ = ninja_always_serial + # The environment variable NINJA_SYNTAX points to the + # ninja_syntax.py module from the ninja sources found here: + # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py + # + # This should be vendored into the build sources and it's location + # set in NINJA_SYNTAX. This code block loads the location from + # that variable, gets the absolute path to the vendored file, gets + # it's parent directory then uses importlib to import the module + # dynamically. + ninja_syntax_file = env[NINJA_SYNTAX] + + if os.path.exists(ninja_syntax_file): + if isinstance(ninja_syntax_file, str): + ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() + ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) + sys.path.append(ninja_syntax_mod_dir) + ninja_syntax_mod_name = os.path.basename(ninja_syntax_file).replace(".py", "") + ninja_syntax = importlib.import_module(ninja_syntax_mod_name) + else: + ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') + + global NINJA_STATE + NINJA_STATE = NinjaState(env, ninja_syntax.Writer) + + # Here we will force every builder to use an emitter which makes the ninja + # file depend on it's target. This forces the ninja file to the bottom of + # the DAG which is required so that we walk every target, and therefore add + # it to the global NINJA_STATE, before we try to write the ninja file. + def ninja_file_depends_on_all(target, source, env): + if not any("conftest" in str(t) for t in target): + env.Depends(ninja_file, target) + return target, source + + # The "Alias Builder" isn't in the BUILDERS map so we have to + # modify it directly. + SCons.Environment.AliasBuilder.emitter = ninja_file_depends_on_all + + for _, builder in env["BUILDERS"].items(): + try: + emitter = builder.emitter + if emitter is not None: + builder.emitter = SCons.Builder.ListEmitter( + [emitter, ninja_file_depends_on_all] + ) + else: + builder.emitter = ninja_file_depends_on_all + # Users can inject whatever they want into the BUILDERS + # dictionary so if the thing doesn't have an emitter we'll + # just ignore it. + except AttributeError: + pass + + # Here we monkey patch the Task.execute method to not do a bunch of + # unnecessary work. If a build is a regular builder (i.e not a conftest and + # not our own Ninja builder) then we add it to the NINJA_STATE. Otherwise we + # build it like normal. This skips all of the caching work that this method + # would normally do since we aren't pulling any of these targets from the + # cache. + # + # In the future we may be able to use this to actually cache the build.ninja + # file once we have the upstream support for referencing SConscripts as File + # nodes. + def ninja_execute(self): + global NINJA_STATE + + target = self.targets[0] + target_name = str(target) + if target_name != ninja_file_name and "conftest" not in target_name: + NINJA_STATE.add_build(target) + else: + target.build() + + SCons.Taskmaster.Task.execute = ninja_execute + + # Make needs_execute always return true instead of determining out of + # date-ness. + SCons.Script.Main.BuildTask.needs_execute = lambda x: True + # We will eventually need to overwrite TempFileMunge to make it # handle persistent tempfiles or get an upstreamed change to add # some configurability to it's behavior in regards to tempfiles. @@ -1379,18 +1511,13 @@ def generate(env): # Set all three environment variables that Python's # tempfile.mkstemp looks at as it behaves differently on different # platforms and versions of Python. - os.environ["TMPDIR"] = env.Dir("$BUILD_DIR/response_files").get_abspath() + build_dir = env.subst("$BUILD_DIR") + if build_dir == "": + build_dir = "." + os.environ["TMPDIR"] = env.Dir("{}/.response_files".format(build_dir)).get_abspath() os.environ["TEMP"] = os.environ["TMPDIR"] os.environ["TMP"] = os.environ["TMPDIR"] if not os.path.isdir(os.environ["TMPDIR"]): env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) - env["TEMPFILE"] = NinjaEternalTempFile - - # Force the SConsign to be written, we benefit from SCons caching of - # implicit dependencies and conftests. Unfortunately, we have to do this - # using an atexit handler because SCons will not write the file when in a - # no_exec build. - import atexit - - atexit.register(SCons.SConsign.write) + env["TEMPFILE"] = NinjaNoResponseFiles \ No newline at end of file diff --git a/test/ninja/CC.py b/test/ninja/CC.py new file mode 100644 index 0000000000..fe18721b9d --- /dev/null +++ b/test/ninja/CC.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import sys +import TestSCons + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') +env.Program(target = 'foo', source = 'foo.c') +""" % locals()) + +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) + +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed foo.o', + 'Removed foo', + 'Removed build.ninja']) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/ninja-fixture/test2.C b/test/ninja/ninja-fixture/test2.C new file mode 100644 index 0000000000..a1ee9e32b9 --- /dev/null +++ b/test/ninja/ninja-fixture/test2.C @@ -0,0 +1,3 @@ +This is a .C file. +/*cc*/ +/*link*/ From 237a2e650ffd666a1b304b3e19a56dadcfd5848a Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 6 May 2020 13:40:11 -0400 Subject: [PATCH 024/163] added some more test and update ninja tool to handle commands --- src/engine/SCons/Tool/ninja.py | 15 ++++++- test/ninja/CC.py | 66 ----------------------------- test/ninja/copy_function_command.py | 43 ++++++++----------- test/ninja/generate_and_build.py | 42 ++++++++---------- test/ninja/shell_command.py | 51 ++++++++++------------ 5 files changed, 70 insertions(+), 147 deletions(-) delete mode 100644 test/ninja/CC.py diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index d1cbafa495..e0129d8bcd 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -692,7 +692,7 @@ def generate(self, ninja_file): if "inputs" in build: build["inputs"].sort() - + ninja.build(**build) template_builds = dict() @@ -990,7 +990,7 @@ def get_command(env, node, action): # pylint: disable=too-many-branches comstr = get_comstr(sub_env, action, tlist, slist) if not comstr: return None - + provider = __NINJA_RULE_MAPPING.get(comstr, get_shell_command) rule, variables = provider(sub_env, node, action, tlist, slist, executor=executor) @@ -1478,6 +1478,17 @@ def ninja_file_depends_on_all(target, source, env): except AttributeError: pass + # We will subvert the normal Command to make sure all targets generated + # from commands will be linked to the ninja file + SconsCommand = SCons.Environment.Environment.Command + + def NinjaCommand(self, target, source, action, **kw): + targets = SconsCommand(env, target, source, action, **kw) + env.Depends(ninja_file, targets) + return targets + + SCons.Environment.Environment.Command = NinjaCommand + # Here we monkey patch the Task.execute method to not do a bunch of # unnecessary work. If a build is a regular builder (i.e not a conftest and # not our own Ninja builder) then we add it to the NINJA_STATE. Otherwise we diff --git a/test/ninja/CC.py b/test/ninja/CC.py deleted file mode 100644 index fe18721b9d..0000000000 --- a/test/ninja/CC.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -import os -import sys -import TestSCons - -_python_ = TestSCons._python_ -_exe = TestSCons._exe - -test = TestSCons.TestSCons() - -test.dir_fixture('ninja-fixture') - -test.write('SConstruct', """ -env = Environment() -env.Tool('ninja') -env.Program(target = 'foo', source = 'foo.c') -""" % locals()) - -test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) - -test.run(arguments='-c', stdout=None) -test.must_contain_all_lines(test.stdout(), [ - 'Removed foo.o', - 'Removed foo', - 'Removed build.ninja']) -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) - -test.pass_test() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index 8e7acff7b7..f86f717e6f 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -25,28 +25,21 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import sys import TestSCons -from TestCmd import IS_WINDOWS - -test = TestSCons.TestSCons() - -try: - import ninja -except ImportError: - test.skip_test("Could not find module in python") _python_ = TestSCons._python_ _exe = TestSCons._exe -ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - 'ninja' + _exe)) +test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + test.write('SConstruct', """ env = Environment() env.Tool('ninja') @@ -56,28 +49,28 @@ # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('foo'), stdout="foo.c") +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo2.o', 'Removed foo2.c', - 'Removed foo' + _exe, + 'Removed foo', 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-execute-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_not_exist(test.workpath('foo')) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) # run ninja independently -program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('foo'), stdout="foo.c") +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) test.pass_test() diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index faf395a1ef..fd111a29be 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -25,41 +25,32 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import sys import TestSCons -from TestCmd import IS_WINDOWS - -test = TestSCons.TestSCons() - -try: - import ninja -except ImportError: - test.skip_test("Could not find module in python") _python_ = TestSCons._python_ _exe = TestSCons._exe -ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - 'ninja' + _exe)) +test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + test.write('SConstruct', """ env = Environment() env.Tool('ninja') env.Program(target = 'foo', source = 'foo.c') """) - # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('foo' + _exe), stdout="foo.c") +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -69,14 +60,15 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-execute-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_not_exist(test.workpath('foo' + _exe)) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) # run ninja independently -program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('foo' + _exe), stdout="foo.c") +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) test.pass_test() diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index 5d7f97e215..fd0e35f8de 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -25,61 +25,54 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import sys import TestSCons -from TestCmd import IS_WINDOWS - -test = TestSCons.TestSCons() - -try: - import ninja -except ImportError: - test.skip_test("Could not find module in python") _python_ = TestSCons._python_ _exe = TestSCons._exe -ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - 'ninja' + _exe)) +test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -shell = '' if IS_WINDOWS else './' +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") test.write('SConstruct', """ env = Environment() env.Tool('ninja') -prog = env.Program(target = 'foo', source = 'foo.c') -env.Command('foo.out', prog, '%(shell)sfoo%(_exe)s > foo.out') -""" % locals()) +env.Program(target = 'foo', source = 'foo.c') +env.Command('foo.out', ['foo'], './foo > foo.out') +""") # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.must_match('foo.out', 'foo.c') +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_match('foo.out', 'foo.c' + os.linesep) # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo.o', - 'Removed foo%(_exe)s' % locals(), + 'Removed foo', 'Removed foo.out', 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-execute-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_not_exist(test.workpath('foo.out')) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) # run ninja independently -program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.must_match('foo.out', 'foo.c') +test.run(program = ninja, stdout=None) +test.must_match('foo.out', 'foo.c' + os.linesep) + + test.pass_test() From 9a8adf17133b4c76baaa83c5804ac4685c06fe5f Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 6 May 2020 17:23:53 -0400 Subject: [PATCH 025/163] update to pass import.py test and support multiple environments --- src/engine/SCons/Tool/ninja.py | 41 ++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index e0129d8bcd..63ba243c61 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -274,6 +274,9 @@ def handle_func_action(self, node, action): "outputs": get_outputs(node), "implicit": get_dependencies(node, skip_sources=True), } + if name == 'ninja_builder': + return None + handler = self.func_handlers.get(name, None) if handler is not None: return handler(node.env if node.env else self.env, node) @@ -377,8 +380,9 @@ def handle_list_action(self, node, action): class NinjaState: """Maintains state of Ninja build system as it's translated from SCons.""" - def __init__(self, env, writer_class): + def __init__(self, env, ninja_file, writer_class): self.env = env + self.ninja_file = ninja_file self.writer_class = writer_class self.__generated = False self.translator = SConsToNinjaTranslator(env) @@ -573,7 +577,7 @@ def has_generated_sources(self, output): return False # pylint: disable=too-many-branches,too-many-locals - def generate(self, ninja_file): + def generate(self): """ Generate the build.ninja. @@ -730,7 +734,7 @@ def generate(self, ninja_file): # jstests/SConscript and being specific to the MongoDB # repository layout. ninja.build( - self.env.File(ninja_file).path, + self.ninja_file.path, rule="REGENERATE", implicit=[ self.env.File("#SConstruct").path, @@ -746,10 +750,10 @@ def generate(self, ninja_file): "compile_commands.json", rule="CMD", pool="console", - implicit=[ninja_file], + implicit=[str(self.ninja_file)], variables={ "cmd": "ninja -f {} -t compdb CC CXX > compile_commands.json".format( - ninja_file + str(self.ninja_file) ) }, ) @@ -772,7 +776,7 @@ def generate(self, ninja_file): if scons_default_targets: ninja.default(" ".join(scons_default_targets)) - with open(ninja_file, "w") as build_ninja: + with open(str(self.ninja_file), "w") as build_ninja: build_ninja.write(content.getvalue()) self.__generated = True @@ -1039,7 +1043,7 @@ def ninja_builder(env, target, source): print("Generating:", str(target[0])) generated_build_ninja = target[0].get_abspath() - NINJA_STATE.generate(generated_build_ninja) + NINJA_STATE.generate() if env.get("DISABLE_AUTO_NINJA") != True: print("Executing:", str(target[0])) @@ -1240,6 +1244,7 @@ def generate(env): help='Disable ninja automatically building after scons') env["DISABLE_AUTO_NINJA"] = GetOption('disable_auto_ninja') + global NINJA_STATE env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") # Add the Ninja builder. @@ -1252,10 +1257,18 @@ def generate(env): env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") - ninja_file = env.Ninja(target=ninja_file_name, source=[]) - env.AlwaysBuild(ninja_file) - env.Alias("$NINJA_ALIAS_NAME", ninja_file) - + # here we allow multiple environments to construct rules and builds + # into the same ninja file + if NINJA_STATE is None: + ninja_file = env.Ninja(target=ninja_file_name, source=[]) + env.AlwaysBuild(ninja_file) + env.Alias("$NINJA_ALIAS_NAME", ninja_file) + else: + if str(NINJA_STATE.ninja_file) != ninja_file_name: + raise Exception("Generating multiple ninja files not supported.") + else: + ninja_file = [NINJA_STATE.ninja_file] + # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. if env["PLATFORM"] == "win32": @@ -1317,7 +1330,7 @@ def generate(env): def robust_rule_mapping(var, rule, tool): provider = gen_get_response_file_command(env, rule, tool) env.NinjaRuleMapping("${" + var + "}", provider) - env.NinjaRuleMapping(env[var], provider) + env.NinjaRuleMapping(env.get(var, None), provider) robust_rule_mapping("CCCOM", "CC", "$CC") robust_rule_mapping("SHCCCOM", "CC", "$CC") @@ -1447,8 +1460,8 @@ def robust_rule_mapping(var, rule, tool): else: ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') - global NINJA_STATE - NINJA_STATE = NinjaState(env, ninja_syntax.Writer) + if NINJA_STATE is None: + NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) # Here we will force every builder to use an emitter which makes the ninja # file depend on it's target. This forces the ninja file to the bottom of From b9a2711779bb8a2826e60ff7dcc665d640cd9b4f Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 6 May 2020 17:59:10 -0400 Subject: [PATCH 026/163] update CI to install ninja pypi package --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 520b90bb74..4093f19278 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ sphinx<3.5.0 sphinx_rtd_theme lxml==4.6.3 rst2pdf +ninja \ No newline at end of file From 30f7c82cef0eef2b5b5ccd8b48167de18cb62967 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 7 May 2020 00:25:15 -0400 Subject: [PATCH 027/163] added more test, including ninja speed test --- test/ninja/build_libraries.py | 71 ++++++++++++---------------- test/ninja/generate_source.py | 51 ++++++++------------ test/ninja/iterative_speedup.py | 49 ++++++++----------- test/ninja/multi_env.py | 53 +++++++++------------ test/ninja/ninja-fixture/bar.c | 2 +- test/ninja/ninja-fixture/test1.c | 13 +---- test/ninja/ninja-fixture/test2.C | 3 -- test/ninja/ninja-fixture/test_impl.c | 15 +----- 8 files changed, 96 insertions(+), 161 deletions(-) delete mode 100644 test/ninja/ninja-fixture/test2.C diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 40404152fa..662c584fe8 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -25,74 +25,63 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import sys import TestSCons -from TestCmd import IS_WINDOWS - -test = TestSCons.TestSCons() - -try: - import ninja -except ImportError: - test.skip_test("Could not find module in python") _python_ = TestSCons._python_ _exe = TestSCons._exe -ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - 'ninja' + _exe)) +test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -lib_suffix = '.lib' if IS_WINDOWS else '.so' -staticlib_suffix = '.lib' if IS_WINDOWS else '.a' -lib_prefix = '' if IS_WINDOWS else 'lib' +ninja = test.where_is('ninja', os.environ['PATH']) -win32 = ", 'WIN32'" if IS_WINDOWS else '' +if not ninja: + test.skip_test("Could not find ninja in environment") test.write('SConstruct', """ env = Environment() env.Tool('ninja') -shared_lib = env.SharedLibrary(target = 'test_impl', source = 'test_impl.c', CPPDEFINES=['LIBRARY_BUILD'%(win32)s]) -env.Program(target = 'test', source = 'test1.c', LIBS=['test_impl'], LIBPATH=['.'], RPATH='.') +shared_lib = env.SharedLibrary(target = 'test_impl', source = 'test_impl.c') +env.Program(target = 'test', source = 'test1.c', LIBS=[shared_lib], LIBPATH=['.'], RPATH='.') -static_obj = env.Object(target = 'test_impl_static', source = 'test_impl.c') -static_lib = env.StaticLibrary(target = 'test_impl_static', source = static_obj) -static_obj = env.Object(target = 'test_static', source = 'test1.c') +static_lib = env.StaticLibrary(target = 'test_impl_static', source = 'test_impl.c') +static_obj = env.Object(target = 'test_static.o', source = 'test1.c') env.Program(target = 'test_static', source = static_obj, LIBS=[static_lib], LIBPATH=['.']) -""" % locals()) +""") # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('test'), stdout="library_function") -test.run(program = test.workpath('test_static'), stdout="library_function") +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('test'), stdout="library_function" + os.linesep) +test.run(program = test.workpath('test_static'), stdout="library_function" + os.linesep) # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ - ('Removed %stest_impl' % lib_prefix) + lib_suffix, - 'Removed test' + _exe, - ('Removed %stest_impl_static' % lib_prefix) + staticlib_suffix, - 'Removed test_static' + _exe, + 'Removed test_impl.os', + 'Removed libtest_impl.so', + 'Removed test1.o', + 'Removed test', + 'Removed test_impl.o', + 'Removed libtest_impl_static.a', + 'Removed test_static.o', + 'Removed test_static', 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-execute-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_not_exist(test.workpath('test')) -test.must_not_exist(test.workpath('test_static')) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) # run ninja independently -program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('test'), stdout="library_function") -test.run(program = test.workpath('test_static'), stdout="library_function") +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('test'), stdout="library_function" + os.linesep) +test.run(program = test.workpath('test_static'), stdout="library_function" + os.linesep) test.pass_test() diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index 76c79bb7da..8ae0a80c39 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -25,37 +25,28 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import sys import TestSCons -from TestCmd import IS_WINDOWS - -test = TestSCons.TestSCons() - -try: - import ninja -except ImportError: - test.skip_test("Could not find module in python") _python_ = TestSCons._python_ _exe = TestSCons._exe -ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - 'ninja' + _exe)) +test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -shell = '' if IS_WINDOWS else './' +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") test.write('SConstruct', """ env = Environment() env.Tool('ninja') -prog = env.Program(target = 'generate_source', source = 'generate_source.c') -env.Command('generated_source.c', prog, '%(shell)sgenerate_source%(_exe)s') +env.Program(target = 'generate_source', source = 'generate_source.c') +env.Command('generated_source.c', ['generate_source'], './generate_source') env.Program(target = 'generated_source', source = 'generated_source.c') -""" % locals()) +""") test.write('generate_source.c', """ #include @@ -70,7 +61,7 @@ fprintf(fp, "int\\n"); fprintf(fp, "main(int argc, char *argv[])\\n"); fprintf(fp, "{\\n"); - fprintf(fp, " printf(\\"generated_source.c\\");\\n"); + fprintf(fp, " printf(\\"generated_source.c\\\\n\\");\\n"); fprintf(fp, " exit (0);\\n"); fprintf(fp, "}\\n"); fclose(fp); @@ -79,30 +70,28 @@ # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") +test.run(program = test.workpath('generated_source'), stdout="generated_source.c" + os.linesep) # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed generate_source.o', - 'Removed generate_source' + _exe, + 'Removed generate_source', 'Removed generated_source.c', 'Removed generated_source.o', - 'Removed generated_source' + _exe, + 'Removed generated_source', 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-execute-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_not_exist(test.workpath('generated_source' + _exe)) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) # run ninja independently -program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('generated_source'), stdout="generated_source.c" + os.linesep) test.pass_test() diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index ff50f502a3..018ba7beda 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -25,30 +25,23 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import sys import time import random import TestSCons -from TestCmd import IS_WINDOWS - -test = TestSCons.TestSCons() - -try: - import ninja -except ImportError: - test.skip_test("Could not find module in python") _python_ = TestSCons._python_ _exe = TestSCons._exe -ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - 'ninja' + _exe)) +test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + test.write('source_0.c', """ #include #include @@ -57,8 +50,7 @@ int print_function0() { - printf("main print"); - return 0; + printf("main print\\n"); } """) @@ -101,7 +93,7 @@ def generate_source(parent_source, current_source): int print_function%(current_source)s() { - return print_function%(parent_source)s(); + print_function%(parent_source)s(); } """ % locals()) @@ -141,7 +133,7 @@ def mod_source_orig(test_num): int print_function%(test_num)s() { - return print_function%(parent_source)s(); + print_function%(parent_source)s(); } """ % locals()) @@ -157,7 +149,6 @@ def mod_source_orig(test_num): main() { print_function%(num_source)s(); - exit(0); } """ % locals()) @@ -181,19 +172,17 @@ def mod_source_orig(test_num): tests_mods += [random.randrange(1, num_source, 1)] jobs = '-j' + str(get_num_cpus()) -ninja_program = [test.workpath('run_ninja_env.bat'), jobs] if IS_WINDOWS else [ninja_bin, jobs] - start = time.perf_counter() -test.run(arguments='--disable-execute-ninja', stdout=None) -test.run(program = ninja_program, stdout=None) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.run(program = ninja, arguments=[jobs], stdout=None) stop = time.perf_counter() ninja_times += [stop - start] -test.run(program = test.workpath('print_bin'), stdout="main print") +test.run(program = test.workpath('print_bin'), stdout="main print" + os.linesep) for test_mod in tests_mods: mod_source_return(test_mod) start = time.perf_counter() - test.run(program = ninja_program, stdout=None) + test.run(program = ninja, arguments=[jobs], stdout=None) stop = time.perf_counter() ninja_times += [stop - start] @@ -209,7 +198,7 @@ def mod_source_orig(test_num): test.run(arguments = ["-f", "SConstruct_no_ninja", jobs], stdout=None) stop = time.perf_counter() scons_times += [stop - start] -test.run(program = test.workpath('print_bin'), stdout="main print") +test.run(program = test.workpath('print_bin'), stdout="main print" + os.linesep) for test_mod in tests_mods: mod_source_return(test_mod) @@ -219,14 +208,14 @@ def mod_source_orig(test_num): scons_times += [stop - start] full_build_print = True -for ninja_time, scons_time in zip(ninja_times, scons_times): - if ninja_time > scons_time: +for ninja, scons in zip(ninja_times, scons_times): + if ninja > scons: test.fail_test() if full_build_print: full_build_print = False - print("Clean build {} files - SCons: {:.3f}s Ninja: {:.3f}s".format(num_source, scons_time, ninja_time)) + print("Clean build {} files - SCons: {:.3f}s Ninja: {:.3f}s".format(num_source, scons, ninja)) else: - print("Single File Rebuild - SCons: {:.3f}s Ninja: {:.3f}s".format(scons_time, ninja_time)) + print("Single File Rebuild - SCons: {:.3f}s Ninja: {:.3f}s".format(scons, ninja)) test.pass_test() diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 18ca3cbc69..3612d7b84a 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -25,67 +25,60 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import sys import TestSCons -from TestCmd import IS_WINDOWS - -test = TestSCons.TestSCons() - -try: - import ninja -except ImportError: - test.skip_test("Could not find module in python") _python_ = TestSCons._python_ _exe = TestSCons._exe -ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - 'ninja' + _exe)) - test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + test.write('SConstruct', """ env = Environment() env.Tool('ninja') env.Program(target = 'foo', source = 'foo.c') env2 = Environment() +env2.Tool('ninja') env2.Program(target = 'bar', source = 'bar.c') """) # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('foo' + _exe), stdout="foo.c") -test.run(program = test.workpath('bar' + _exe), stdout="bar.c") +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +test.run(program = test.workpath('bar'), stdout="bar.c" + os.linesep) # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo.o', - 'Removed foo' + _exe, + 'Removed foo', 'Removed bar.o', - 'Removed bar' + _exe, + 'Removed bar', 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-execute-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_not_exist(test.workpath('foo' + _exe)) -test.must_not_exist(test.workpath('bar' + _exe)) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) # run ninja independently -program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('foo' + _exe), stdout="foo.c") -test.run(program = test.workpath('bar' + _exe), stdout="bar.c") +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +test.run(program = test.workpath('bar'), stdout="bar.c" + os.linesep) + + test.pass_test() diff --git a/test/ninja/ninja-fixture/bar.c b/test/ninja/ninja-fixture/bar.c index 15b2ecc46a..3767857b00 100644 --- a/test/ninja/ninja-fixture/bar.c +++ b/test/ninja/ninja-fixture/bar.c @@ -5,6 +5,6 @@ int main(int argc, char *argv[]) { argv[argc++] = "--"; - printf("bar.c"); + printf("bar.c\n"); exit (0); } diff --git a/test/ninja/ninja-fixture/test1.c b/test/ninja/ninja-fixture/test1.c index 678461f508..c53f54ac85 100644 --- a/test/ninja/ninja-fixture/test1.c +++ b/test/ninja/ninja-fixture/test1.c @@ -1,21 +1,10 @@ #include #include -#ifdef WIN32 -#ifdef LIBRARY_BUILD -#define DLLEXPORT __declspec(dllexport) -#else -#define DLLEXPORT __declspec(dllimport) -#endif -#else -#define DLLEXPORT -#endif - -DLLEXPORT extern int library_function(void); +extern int library_function(void); int main(int argc, char *argv[]) { library_function(); - exit(0); } diff --git a/test/ninja/ninja-fixture/test2.C b/test/ninja/ninja-fixture/test2.C deleted file mode 100644 index a1ee9e32b9..0000000000 --- a/test/ninja/ninja-fixture/test2.C +++ /dev/null @@ -1,3 +0,0 @@ -This is a .C file. -/*cc*/ -/*link*/ diff --git a/test/ninja/ninja-fixture/test_impl.c b/test/ninja/ninja-fixture/test_impl.c index 89c26ede6f..ae5effc965 100644 --- a/test/ninja/ninja-fixture/test_impl.c +++ b/test/ninja/ninja-fixture/test_impl.c @@ -1,19 +1,8 @@ #include #include -#ifdef WIN32 -#ifdef LIBRARY_BUILD -#define DLLEXPORT __declspec(dllexport) -#else -#define DLLEXPORT __declspec(dllimport) -#endif -#else -#define DLLEXPORT -#endif - - -DLLEXPORT int +int library_function(void) { - printf("library_function"); + printf("library_function\n"); } From 4f377862f64cb8d67fdc6493b0917c77fe3d2e00 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 7 May 2020 00:31:28 -0400 Subject: [PATCH 028/163] fixed sider issues --- src/engine/SCons/Tool/ninja.py | 4 ++-- test/ninja/build_libraries.py | 1 - test/ninja/copy_function_command.py | 1 - test/ninja/generate_and_build.py | 1 - test/ninja/generate_source.py | 1 - test/ninja/iterative_speedup.py | 1 - test/ninja/multi_env.py | 1 - test/ninja/shell_command.py | 1 - 8 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 63ba243c61..bf399e2592 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -665,7 +665,7 @@ def generate(self): # files. Here we make sure that Ninja only ever sees one # target when using a depfile. It will still have a command # that will create all of the outputs but most targets don't - # depend direclty on DWO files and so this assumption is safe + # depend directly on DWO files and so this assumption is safe # to make. rule = self.rules.get(build["rule"]) @@ -1044,7 +1044,7 @@ def ninja_builder(env, target, source): generated_build_ninja = target[0].get_abspath() NINJA_STATE.generate() - if env.get("DISABLE_AUTO_NINJA") != True: + if not env.get("DISABLE_AUTO_NINJA"): print("Executing:", str(target[0])) def execute_ninja(): diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 662c584fe8..7e0ec2365a 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index f86f717e6f..a1e72b7181 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index fd111a29be..82aab5e53b 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index 8ae0a80c39..d1bfe34a6b 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index 018ba7beda..cf999d8568 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import time import random import TestSCons diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 3612d7b84a..5360fd215c 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index fd0e35f8de..b5c8323f43 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ From ae9161b868bec1ecf106bace8366d318ab2498a8 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 8 May 2020 12:44:27 -0500 Subject: [PATCH 029/163] update tests to work on windows, added some environment support for windows and msvc --- src/engine/SCons/Tool/ninja.py | 47 +++++++++++++++++++++------- test/ninja/build_libraries.py | 41 +++++++++++++----------- test/ninja/copy_function_command.py | 10 +++--- test/ninja/generate_and_build.py | 9 ++++-- test/ninja/generate_source.py | 22 +++++++------ test/ninja/iterative_speedup.py | 19 ++++++----- test/ninja/multi_env.py | 18 +++++------ test/ninja/ninja-fixture/bar.c | 2 +- test/ninja/ninja-fixture/test1.c | 13 +++++++- test/ninja/ninja-fixture/test_impl.c | 15 +++++++-- test/ninja/shell_command.py | 20 ++++++------ 11 files changed, 141 insertions(+), 75 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index bf399e2592..10e27f3423 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -77,8 +77,8 @@ def _mkdir_action_function(env, node): # to an invalid ninja file. "variables": { # On Windows mkdir "-p" is always on - "cmd": "{mkdir} $out".format( - mkdir="mkdir" if env["PLATFORM"] == "win32" else "mkdir -p", + "cmd": "{mkdir}".format( + mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", ), }, } @@ -383,6 +383,7 @@ class NinjaState: def __init__(self, env, ninja_file, writer_class): self.env = env self.ninja_file = ninja_file + self.ninja_bin_path = '' self.writer_class = writer_class self.__generated = False self.translator = SConsToNinjaTranslator(env) @@ -752,7 +753,7 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "ninja -f {} -t compdb CC CXX > compile_commands.json".format( + "cmd": "{}/ninja -f {} -t compdb CC CXX > compile_commands.json".format(self.ninja_bin_path, str(self.ninja_file) ) }, @@ -1044,15 +1045,32 @@ def ninja_builder(env, target, source): generated_build_ninja = target[0].get_abspath() NINJA_STATE.generate() + + if env["PLATFORM"] == "win32": + # this is not great, it executes everytime + # and its doesn't consider specific node environments + # also a bit quirky to use, but usually MSVC is not + # setup system wide for command line use so this is needed + # on the standard MSVC setup, this is only needed if + # running ninja directly from a command line that hasn't + # had the environment setup (vcvarsall.bat) + # todo: hook this into a command so that it only regnerates + # the .bat if the env['ENV'] changes + with open('ninja_env.bat', 'w') as f: + for key in env['ENV']: + f.write('set {}={}\n'.format(key, env['ENV'][key])) + if not env.get("DISABLE_AUTO_NINJA"): print("Executing:", str(target[0])) def execute_ninja(): - proc = subprocess.Popen( ['ninja', '-f', generated_build_ninja], - stderr=subprocess.STDOUT, + env.AppendENVPath('PATH', NINJA_STATE.ninja_bin_path) + proc = subprocess.Popen(['ninja', '-f', generated_build_ninja], + stderr=sys.stderr, stdout=subprocess.PIPE, - universal_newlines=True + universal_newlines=True, + env=env['ENV'] ) for stdout_line in iter(proc.stdout.readline, ""): yield stdout_line @@ -1060,12 +1078,17 @@ def execute_ninja(): return_code = proc.wait() if return_code: raise subprocess.CalledProcessError(return_code, 'ninja') - + erase_previous = False for output in execute_ninja(): output = output.strip() - sys.stdout.write('\x1b[2K') # erase previous line - sys.stdout.write(output + "\r") + if erase_previous: + sys.stdout.write('\x1b[2K') # erase previous line + sys.stdout.write("\r") + else: + sys.stdout.write(os.linesep) + sys.stdout.write(output) sys.stdout.flush() + erase_previous = output.startswith('[') # pylint: disable=too-few-public-methods class AlwaysExecAction(SCons.Action.FunctionAction): @@ -1311,7 +1334,7 @@ def generate(env): if env["PLATFORM"] == "win32": from SCons.Tool.mslink import compositeLinkAction - if env["LINKCOM"] == compositeLinkAction: + if env.get("LINKCOM", None) == compositeLinkAction: env[ "LINKCOM" ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' @@ -1459,10 +1482,10 @@ def robust_rule_mapping(var, rule, tool): ninja_syntax = importlib.import_module(ninja_syntax_mod_name) else: ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') - + if NINJA_STATE is None: NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) - + NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join(ninja_syntax.__file__, os.pardir, 'data', 'bin')) # Here we will force every builder to use an emitter which makes the ninja # file depend on it's target. This forces the ninja file to the bottom of # the DAG which is required so that we walk every target, and therefore add diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 7e0ec2365a..5e491cc90b 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -39,35 +40,38 @@ if not ninja: test.skip_test("Could not find ninja in environment") +lib_suffix = '.lib' if IS_WINDOWS else '.so' +staticlib_suffix = '.lib' if IS_WINDOWS else '.a' +lib_prefix = '' if IS_WINDOWS else 'lib' + +win32 = ", 'WIN32'" if IS_WINDOWS else '' + test.write('SConstruct', """ env = Environment() env.Tool('ninja') -shared_lib = env.SharedLibrary(target = 'test_impl', source = 'test_impl.c') -env.Program(target = 'test', source = 'test1.c', LIBS=[shared_lib], LIBPATH=['.'], RPATH='.') +shared_lib = env.SharedLibrary(target = 'test_impl', source = 'test_impl.c', CPPDEFINES=['LIBRARY_BUILD'%(win32)s]) +env.Program(target = 'test', source = 'test1.c', LIBS=['test_impl'], LIBPATH=['.'], RPATH='.') -static_lib = env.StaticLibrary(target = 'test_impl_static', source = 'test_impl.c') -static_obj = env.Object(target = 'test_static.o', source = 'test1.c') +static_obj = env.Object(target = 'test_impl_static', source = 'test_impl.c') +static_lib = env.StaticLibrary(target = 'test_impl_static', source = static_obj) +static_obj = env.Object(target = 'test_static', source = 'test1.c') env.Program(target = 'test_static', source = static_obj, LIBS=[static_lib], LIBPATH=['.']) -""") +""" % locals()) # generate simple build test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('test'), stdout="library_function" + os.linesep) -test.run(program = test.workpath('test_static'), stdout="library_function" + os.linesep) +test.run(program = test.workpath('test'), stdout="library_function") +test.run(program = test.workpath('test_static'), stdout="library_function") # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ - 'Removed test_impl.os', - 'Removed libtest_impl.so', - 'Removed test1.o', - 'Removed test', - 'Removed test_impl.o', - 'Removed libtest_impl_static.a', - 'Removed test_static.o', - 'Removed test_static', + ('Removed %stest_impl' % lib_prefix) + lib_suffix, + 'Removed test' + _exe, + ('Removed %stest_impl_static' % lib_prefix) + staticlib_suffix, + 'Removed test_static' + _exe, 'Removed build.ninja']) # only generate the ninja file @@ -78,9 +82,10 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('test'), stdout="library_function" + os.linesep) -test.run(program = test.workpath('test_static'), stdout="library_function" + os.linesep) +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('test'), stdout="library_function") +test.run(program = test.workpath('test_static'), stdout="library_function") test.pass_test() diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index a1e72b7181..06991d375c 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -50,14 +51,14 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +test.run(program = test.workpath('foo'), stdout="foo.c") # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo2.o', 'Removed foo2.c', - 'Removed foo', + 'Removed foo' + _exe, 'Removed build.ninja']) # only generate the ninja file @@ -68,8 +69,9 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('foo'), stdout="foo.c") test.pass_test() diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index 82aab5e53b..904e46a3cd 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -45,11 +46,12 @@ env.Program(target = 'foo', source = 'foo.c') """) + # generate simple build test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +test.run(program = test.workpath('foo' + _exe), stdout="foo.c") # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -66,8 +68,9 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('foo' + _exe), stdout="foo.c") test.pass_test() diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index d1bfe34a6b..298d227fea 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -39,13 +40,15 @@ if not ninja: test.skip_test("Could not find ninja in environment") +shell = '' if IS_WINDOWS else './' + test.write('SConstruct', """ env = Environment() env.Tool('ninja') -env.Program(target = 'generate_source', source = 'generate_source.c') -env.Command('generated_source.c', ['generate_source'], './generate_source') +prog = env.Program(target = 'generate_source', source = 'generate_source.c') +env.Command('generated_source.c', prog, '%(shell)sgenerate_source%(_exe)s') env.Program(target = 'generated_source', source = 'generated_source.c') -""") +""" % locals()) test.write('generate_source.c', """ #include @@ -60,7 +63,7 @@ fprintf(fp, "int\\n"); fprintf(fp, "main(int argc, char *argv[])\\n"); fprintf(fp, "{\\n"); - fprintf(fp, " printf(\\"generated_source.c\\\\n\\");\\n"); + fprintf(fp, " printf(\\"generated_source.c\\");\\n"); fprintf(fp, " exit (0);\\n"); fprintf(fp, "}\\n"); fclose(fp); @@ -69,16 +72,16 @@ # generate simple build test.run(stdout=None) -test.run(program = test.workpath('generated_source'), stdout="generated_source.c" + os.linesep) +test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed generate_source.o', - 'Removed generate_source', + 'Removed generate_source' + _exe, 'Removed generated_source.c', 'Removed generated_source.o', - 'Removed generated_source', + 'Removed generated_source' + _exe, 'Removed build.ninja']) # only generate the ninja file @@ -89,8 +92,9 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('generated_source'), stdout="generated_source.c" + os.linesep) +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") test.pass_test() diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index cf999d8568..6675bf3350 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -28,6 +28,7 @@ import time import random import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -49,7 +50,8 @@ int print_function0() { - printf("main print\\n"); + printf("main print"); + return 0; } """) @@ -92,7 +94,7 @@ def generate_source(parent_source, current_source): int print_function%(current_source)s() { - print_function%(parent_source)s(); + return print_function%(parent_source)s(); } """ % locals()) @@ -132,7 +134,7 @@ def mod_source_orig(test_num): int print_function%(test_num)s() { - print_function%(parent_source)s(); + return print_function%(parent_source)s(); } """ % locals()) @@ -148,6 +150,7 @@ def mod_source_orig(test_num): main() { print_function%(num_source)s(); + exit(0); } """ % locals()) @@ -171,17 +174,19 @@ def mod_source_orig(test_num): tests_mods += [random.randrange(1, num_source, 1)] jobs = '-j' + str(get_num_cpus()) +ninja_program = [test.workpath('ninja_env.bat'), '&', ninja, jobs] if IS_WINDOWS else [ninja, jobs] + start = time.perf_counter() test.run(arguments='--disable-auto-ninja', stdout=None) -test.run(program = ninja, arguments=[jobs], stdout=None) +test.run(program = ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] -test.run(program = test.workpath('print_bin'), stdout="main print" + os.linesep) +test.run(program = test.workpath('print_bin'), stdout="main print") for test_mod in tests_mods: mod_source_return(test_mod) start = time.perf_counter() - test.run(program = ninja, arguments=[jobs], stdout=None) + test.run(program = ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] @@ -197,7 +202,7 @@ def mod_source_orig(test_num): test.run(arguments = ["-f", "SConstruct_no_ninja", jobs], stdout=None) stop = time.perf_counter() scons_times += [stop - start] -test.run(program = test.workpath('print_bin'), stdout="main print" + os.linesep) +test.run(program = test.workpath('print_bin'), stdout="main print") for test_mod in tests_mods: mod_source_return(test_mod) diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 5360fd215c..087d392b52 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -53,16 +54,16 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) -test.run(program = test.workpath('bar'), stdout="bar.c" + os.linesep) +test.run(program = test.workpath('foo' + _exe), stdout="foo.c") +test.run(program = test.workpath('bar' + _exe), stdout="bar.c") # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo.o', - 'Removed foo', + 'Removed foo' + _exe, 'Removed bar.o', - 'Removed bar', + 'Removed bar' + _exe, 'Removed build.ninja']) # only generate the ninja file @@ -73,11 +74,10 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) -test.run(program = test.workpath('bar'), stdout="bar.c" + os.linesep) - - +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('foo' + _exe), stdout="foo.c") +test.run(program = test.workpath('bar' + _exe), stdout="bar.c") test.pass_test() diff --git a/test/ninja/ninja-fixture/bar.c b/test/ninja/ninja-fixture/bar.c index 3767857b00..15b2ecc46a 100644 --- a/test/ninja/ninja-fixture/bar.c +++ b/test/ninja/ninja-fixture/bar.c @@ -5,6 +5,6 @@ int main(int argc, char *argv[]) { argv[argc++] = "--"; - printf("bar.c\n"); + printf("bar.c"); exit (0); } diff --git a/test/ninja/ninja-fixture/test1.c b/test/ninja/ninja-fixture/test1.c index c53f54ac85..678461f508 100644 --- a/test/ninja/ninja-fixture/test1.c +++ b/test/ninja/ninja-fixture/test1.c @@ -1,10 +1,21 @@ #include #include -extern int library_function(void); +#ifdef WIN32 +#ifdef LIBRARY_BUILD +#define DLLEXPORT __declspec(dllexport) +#else +#define DLLEXPORT __declspec(dllimport) +#endif +#else +#define DLLEXPORT +#endif + +DLLEXPORT extern int library_function(void); int main(int argc, char *argv[]) { library_function(); + exit(0); } diff --git a/test/ninja/ninja-fixture/test_impl.c b/test/ninja/ninja-fixture/test_impl.c index ae5effc965..89c26ede6f 100644 --- a/test/ninja/ninja-fixture/test_impl.c +++ b/test/ninja/ninja-fixture/test_impl.c @@ -1,8 +1,19 @@ #include #include -int +#ifdef WIN32 +#ifdef LIBRARY_BUILD +#define DLLEXPORT __declspec(dllexport) +#else +#define DLLEXPORT __declspec(dllimport) +#endif +#else +#define DLLEXPORT +#endif + + +DLLEXPORT int library_function(void) { - printf("library_function\n"); + printf("library_function"); } diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index b5c8323f43..5d491645c6 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -39,24 +40,26 @@ if not ninja: test.skip_test("Could not find ninja in environment") +shell = '' if IS_WINDOWS else './' + test.write('SConstruct', """ env = Environment() env.Tool('ninja') -env.Program(target = 'foo', source = 'foo.c') -env.Command('foo.out', ['foo'], './foo > foo.out') -""") +prog = env.Program(target = 'foo', source = 'foo.c') +env.Command('foo.out', prog, '%(shell)sfoo%(_exe)s > foo.out') +""" % locals()) # generate simple build test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.must_match('foo.out', 'foo.c' + os.linesep) +test.must_match('foo.out', 'foo.c') # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo.o', - 'Removed foo', + 'Removed foo%(_exe)s' % locals(), 'Removed foo.out', 'Removed build.ninja']) @@ -68,10 +71,9 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.must_match('foo.out', 'foo.c' + os.linesep) - - +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.must_match('foo.out', 'foo.c') test.pass_test() From d8a4fed3db0426c21a73886f25c1ce2584ec0f24 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 14 May 2020 02:52:16 -0400 Subject: [PATCH 030/163] used different method for pushing ninja file to bottom of DAG, use import ninja to get ninja_syntax and ninja bin, and added some more basic testing. --- src/engine/SCons/Tool/ninja.py | 196 +++++++++++++-------------- test/ninja/build_libraries.py | 34 +++-- test/ninja/copy_function_command.py | 33 +++-- test/ninja/generate_and_build.py | 33 +++-- test/ninja/generate_and_build_cxx.py | 7 +- test/ninja/generate_source.py | 31 +++-- test/ninja/iterative_speedup.py | 22 ++- test/ninja/multi_env.py | 35 +++-- test/ninja/shell_command.py | 33 +++-- 9 files changed, 237 insertions(+), 187 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 10e27f3423..bc505b8937 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -88,13 +88,7 @@ def _copy_action_function(env, node): "outputs": get_outputs(node), "inputs": [get_path(src_file(s)) for s in node.sources], "rule": "CMD", - # implicit explicitly omitted, we translate these so they can be - # used by anything that depends on these but commonly this is - # hit with a node that will depend on all of the fake - # srcnode's that SCons will never give us a rule for leading - # to an invalid ninja file. "variables": { - # On Windows mkdir "-p" is always on "cmd": "$COPY $in $out", }, } @@ -753,7 +747,7 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "{}/ninja -f {} -t compdb CC CXX > compile_commands.json".format(self.ninja_bin_path, + "cmd": "{} -f {} -t compdb CC CXX > compile_commands.json".format(self.ninja_bin_path, str(self.ninja_file) ) }, @@ -1047,30 +1041,30 @@ def ninja_builder(env, target, source): NINJA_STATE.generate() if env["PLATFORM"] == "win32": - # this is not great, it executes everytime - # and its doesn't consider specific node environments - # also a bit quirky to use, but usually MSVC is not - # setup system wide for command line use so this is needed - # on the standard MSVC setup, this is only needed if + # this is not great, its doesn't consider specific + # node environments, which means on linux the build could + # behave differently, becuase on linux you can set the environment + # per command in the ninja file. This is only needed if # running ninja directly from a command line that hasn't # had the environment setup (vcvarsall.bat) - # todo: hook this into a command so that it only regnerates - # the .bat if the env['ENV'] changes - with open('ninja_env.bat', 'w') as f: + with open('run_ninja_env.bat', 'w') as f: for key in env['ENV']: f.write('set {}={}\n'.format(key, env['ENV'][key])) + f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) if not env.get("DISABLE_AUTO_NINJA"): - print("Executing:", str(target[0])) + cmd = [NINJA_STATE.ninja_bin_path, '-f', generated_build_ninja] + print("Executing:", str(' '.join(cmd))) + # execute the ninja build at the end of SCons, trying to + # reproduce the output like a ninja build would def execute_ninja(): - env.AppendENVPath('PATH', NINJA_STATE.ninja_bin_path) - proc = subprocess.Popen(['ninja', '-f', generated_build_ninja], + proc = subprocess.Popen(cmd, stderr=sys.stderr, stdout=subprocess.PIPE, universal_newlines=True, - env=env['ENV'] + env=env['ENV'] # ninja build items won't consider node env on win32 ) for stdout_line in iter(proc.stdout.readline, ""): yield stdout_line @@ -1078,6 +1072,7 @@ def execute_ninja(): return_code = proc.wait() if return_code: raise subprocess.CalledProcessError(return_code, 'ninja') + erase_previous = False for output in execute_ninja(): output = output.strip() @@ -1088,6 +1083,9 @@ def execute_ninja(): sys.stdout.write(os.linesep) sys.stdout.write(output) sys.stdout.flush() + # this will only erase ninjas [#/#] lines + # leaving warnings and other output, seems a bit + # prone to failure with such a simple check erase_previous = output.startswith('[') # pylint: disable=too-few-public-methods @@ -1226,6 +1224,20 @@ def ninja_always_serial(self, num, taskmaster): self.num_jobs = num self.job = SCons.Job.Serial(taskmaster) +def ninja_hack_linkcom(env): + # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks + # without relying on the SCons hacks that SCons uses by default. + if env["PLATFORM"] == "win32": + from SCons.Tool.mslink import compositeLinkAction + + if env.get("LINKCOM", None) == compositeLinkAction: + env[ + "LINKCOM" + ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' + env[ + "SHLINKCOM" + ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' + class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" @@ -1259,13 +1271,31 @@ def generate(env): global added if not added: added = 1 - AddOption('--disable-auto-ninja', - dest='disable_auto_ninja', - metavar='BOOL', - action="store_true", - default=False, - help='Disable ninja automatically building after scons') - env["DISABLE_AUTO_NINJA"] = GetOption('disable_auto_ninja') + + AddOption('--disable-execute-ninja', + dest='disable_execute_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + + AddOption('--disable-ninja', + dest='disable_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + + if GetOption('disable_ninja'): + return env + + try: + import ninja + except ImportError: + SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") + return + + env["DISABLE_AUTO_NINJA"] = GetOption('disable_execute_ninja') global NINJA_STATE env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") @@ -1288,16 +1318,15 @@ def generate(env): env.Alias("$NINJA_ALIAS_NAME", ninja_file) else: if str(NINJA_STATE.ninja_file) != ninja_file_name: - raise Exception("Generating multiple ninja files not supported.") - else: - ninja_file = [NINJA_STATE.ninja_file] + SCons.Warnings.Warning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") + ninja_file = [NINJA_STATE.ninja_file] # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. if env["PLATFORM"] == "win32": env.Append(CCFLAGS=["/showIncludes"]) else: - env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) + env.Append(CCFLAGS=["-MD", "-MF", "${TARGET}.d"]) # Provide a way for custom rule authors to easily access command # generation. @@ -1329,18 +1358,8 @@ def generate(env): # might not catch it. env.AddMethod(register_custom_rule_mapping, "NinjaRuleMapping") - # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks - # without relying on the SCons hacks that SCons uses by default. - if env["PLATFORM"] == "win32": - from SCons.Tool.mslink import compositeLinkAction - - if env.get("LINKCOM", None) == compositeLinkAction: - env[ - "LINKCOM" - ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' - env[ - "SHLINKCOM" - ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' + # on windows we need to change the link action + ninja_hack_linkcom(env) # Normally in SCons actions for the Program and *Library builders # will return "${*COM}" as their pre-subst'd command line. However @@ -1386,6 +1405,8 @@ def robust_rule_mapping(var, rule, tool): # Disable running ranlib, since we added 's' above env["RANLIBCOM"] = "" + SCons.Warnings.Warning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") + # This is the point of no return, anything after this comment # makes changes to SCons that are irreversible and incompatible # with a normal SCons build. We return early if __NINJA_NO=1 has @@ -1462,68 +1483,43 @@ def robust_rule_mapping(var, rule, tool): # monkey the Jobs constructor to only use the Serial Job class. SCons.Job.Jobs.__init__ = ninja_always_serial - # The environment variable NINJA_SYNTAX points to the - # ninja_syntax.py module from the ninja sources found here: - # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py - # - # This should be vendored into the build sources and it's location - # set in NINJA_SYNTAX. This code block loads the location from - # that variable, gets the absolute path to the vendored file, gets - # it's parent directory then uses importlib to import the module - # dynamically. - ninja_syntax_file = env[NINJA_SYNTAX] - - if os.path.exists(ninja_syntax_file): - if isinstance(ninja_syntax_file, str): - ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() - ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) - sys.path.append(ninja_syntax_mod_dir) - ninja_syntax_mod_name = os.path.basename(ninja_syntax_file).replace(".py", "") - ninja_syntax = importlib.import_module(ninja_syntax_mod_name) - else: - ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') + ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') if NINJA_STATE is None: NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) - NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join(ninja_syntax.__file__, os.pardir, 'data', 'bin')) - # Here we will force every builder to use an emitter which makes the ninja - # file depend on it's target. This forces the ninja file to the bottom of - # the DAG which is required so that we walk every target, and therefore add - # it to the global NINJA_STATE, before we try to write the ninja file. - def ninja_file_depends_on_all(target, source, env): - if not any("conftest" in str(t) for t in target): - env.Depends(ninja_file, target) - return target, source - - # The "Alias Builder" isn't in the BUILDERS map so we have to - # modify it directly. - SCons.Environment.AliasBuilder.emitter = ninja_file_depends_on_all - - for _, builder in env["BUILDERS"].items(): - try: - emitter = builder.emitter - if emitter is not None: - builder.emitter = SCons.Builder.ListEmitter( - [emitter, ninja_file_depends_on_all] - ) - else: - builder.emitter = ninja_file_depends_on_all - # Users can inject whatever they want into the BUILDERS - # dictionary so if the thing doesn't have an emitter we'll - # just ignore it. - except AttributeError: - pass - - # We will subvert the normal Command to make sure all targets generated - # from commands will be linked to the ninja file - SconsCommand = SCons.Environment.Environment.Command - - def NinjaCommand(self, target, source, action, **kw): - targets = SconsCommand(env, target, source, action, **kw) - env.Depends(ninja_file, targets) + NINJA_STATE.ninja_bin_path = env.get('NINJA_BIN') + if not NINJA_STATE.ninja_bin_path: + # default to using ninja installed with python module + ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' + NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( + ninja_syntax.__file__, + os.pardir, + 'data', + 'bin', + ninja_bin)) + if not os.path.exists(NINJA_STATE.ninja_bin_path): + # couldn't find it, just give the bin name and hope + # its in the path later + NINJA_STATE.ninja_bin_path = ninja_bin + + # TODO: this is hacking into scons, preferable if there were a less intrusive way + # We will subvert the normal builder execute to make sure all the ninja file is dependent + # on all targets generated from any builders + SCons_Builder_BuilderBase__execute = SCons.Builder.BuilderBase._execute + def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): + # this ensures all environments in which a builder executes from will + # not create list actions for linking on windows + ninja_hack_linkcom(env) + targets = SCons_Builder_BuilderBase__execute(self, env, target, source, overwarn=overwarn, executor_kw=executor_kw) + + if not SCons.Util.is_List(target): + target = [target] + + for target in targets: + if str(target) != ninja_file_name and "conftest" not in str(target): + env.Depends(ninja_file, targets) return targets - - SCons.Environment.Environment.Command = NinjaCommand + SCons.Builder.BuilderBase._execute = NinjaBuilderExecute # Here we monkey patch the Task.execute method to not do a bunch of # unnecessary work. If a build is a regular builder (i.e not a conftest and diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 5e491cc90b..347d63902c 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - lib_suffix = '.lib' if IS_WINDOWS else '.so' staticlib_suffix = '.lib' if IS_WINDOWS else '.a' lib_prefix = '' if IS_WINDOWS else 'lib' @@ -60,8 +68,9 @@ """ % locals()) # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('test'), stdout="library_function") test.run(program = test.workpath('test_static'), stdout="library_function") @@ -75,14 +84,13 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('test')) +test.must_not_exist(test.workpath('test_static')) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('test'), stdout="library_function") test.run(program = test.workpath('test_static'), stdout="library_function") diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index 06991d375c..0beb8de11c 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - test.write('SConstruct', """ env = Environment() env.Tool('ninja') @@ -49,8 +57,9 @@ # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('foo'), stdout="foo.c") # clean build and ninja files @@ -62,14 +71,12 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo')) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('foo'), stdout="foo.c") diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index 904e46a3cd..73c71f1b40 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - test.write('SConstruct', """ env = Environment() env.Tool('ninja') @@ -49,8 +57,9 @@ # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('foo' + _exe), stdout="foo.c") # clean build and ninja files @@ -61,14 +70,12 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo' + _exe)) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('foo' + _exe), stdout="foo.c") diff --git a/test/ninja/generate_and_build_cxx.py b/test/ninja/generate_and_build_cxx.py index 663282bd92..ac0f55444f 100644 --- a/test/ninja/generate_and_build_cxx.py +++ b/test/ninja/generate_and_build_cxx.py @@ -25,11 +25,10 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS -test = TestSCons.TestSCons() - try: import ninja except ImportError: @@ -39,12 +38,14 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, + importlib.import_module(".ninja_syntax", package='ninja').__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) +test = TestSCons.TestSCons() + test.dir_fixture('ninja-fixture') test.write('SConstruct', """ diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index 298d227fea..d9b9c4ed59 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - shell = '' if IS_WINDOWS else './' test.write('SConstruct', """ @@ -72,6 +80,9 @@ # generate simple build test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") # clean build and ninja files @@ -85,14 +96,12 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('generated_source' + _exe)) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index 6675bf3350..b03e9cbcb0 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -27,21 +27,29 @@ import os import time import random +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - test.write('source_0.c', """ #include #include @@ -174,10 +182,10 @@ def mod_source_orig(test_num): tests_mods += [random.randrange(1, num_source, 1)] jobs = '-j' + str(get_num_cpus()) -ninja_program = [test.workpath('ninja_env.bat'), '&', ninja, jobs] if IS_WINDOWS else [ninja, jobs] +ninja_program = [test.workpath('run_ninja_env.bat'), jobs] if IS_WINDOWS else [ninja_bin, jobs] start = time.perf_counter() -test.run(arguments='--disable-auto-ninja', stdout=None) +test.run(arguments='--disable-execute-ninja', stdout=None) test.run(program = ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 087d392b52..c9b21b7d70 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -25,35 +25,43 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - test.write('SConstruct', """ env = Environment() env.Tool('ninja') env.Program(target = 'foo', source = 'foo.c') env2 = Environment() -env2.Tool('ninja') env2.Program(target = 'bar', source = 'bar.c') """) # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('foo' + _exe), stdout="foo.c") test.run(program = test.workpath('bar' + _exe), stdout="bar.c") @@ -67,14 +75,13 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo' + _exe)) +test.must_not_exist(test.workpath('bar' + _exe)) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('foo' + _exe), stdout="foo.c") test.run(program = test.workpath('bar' + _exe), stdout="bar.c") diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index 5d491645c6..b35a52b6c0 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - shell = '' if IS_WINDOWS else './' test.write('SConstruct', """ @@ -51,8 +59,9 @@ # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.must_match('foo.out', 'foo.c') # clean build and ninja files @@ -64,14 +73,12 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo.out')) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.must_match('foo.out', 'foo.c') From 80d6c25e6bc8b4054ca2aaaafce9a17e7e8f8465 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 14 May 2020 12:28:17 -0400 Subject: [PATCH 031/163] is link should use the base nodes lstat instead of local fs stat builder is not garunteed to be in the environment, so check if the node is the ninja_file fix sider issues --- src/engine/SCons/Tool/ninja.py | 9 +++++---- test/ninja/build_libraries.py | 7 +++---- test/ninja/copy_function_command.py | 7 +++---- test/ninja/generate_and_build.py | 7 +++---- test/ninja/generate_and_build_cxx.py | 7 +++---- test/ninja/generate_source.py | 7 +++---- test/ninja/iterative_speedup.py | 15 +++++++-------- test/ninja/multi_env.py | 5 +++-- test/ninja/shell_command.py | 7 +++---- 9 files changed, 33 insertions(+), 38 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index bc505b8937..a68655ef1d 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -233,9 +233,10 @@ def action_to_ninja_build(self, node, action=None): # Ninja builders out of being sources of ninja builders but I # can't fix every DAG problem so we just skip ninja_builders # if we find one - if node.builder == self.env["BUILDERS"]["Ninja"]: + global NINJA_STATE + if NINJA_STATE.ninja_file == str(node): build = None - elif isinstance(action, SCons.Action.FunctionAction): + if isinstance(action, SCons.Action.FunctionAction): build = self.handle_func_action(node, action) elif isinstance(action, SCons.Action.LazyAction): # pylint: disable=protected-access @@ -1043,7 +1044,7 @@ def ninja_builder(env, target, source): if env["PLATFORM"] == "win32": # this is not great, its doesn't consider specific # node environments, which means on linux the build could - # behave differently, becuase on linux you can set the environment + # behave differently, because on linux you can set the environment # per command in the ninja file. This is only needed if # running ninja directly from a command line that hasn't # had the environment setup (vcvarsall.bat) @@ -1492,7 +1493,7 @@ def robust_rule_mapping(var, rule, tool): # default to using ninja installed with python module ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( - ninja_syntax.__file__, + ninja.__file__, os.pardir, 'data', 'bin', diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 347d63902c..40404152fa 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') lib_suffix = '.lib' if IS_WINDOWS else '.so' diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index 0beb8de11c..8e7acff7b7 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') test.write('SConstruct', """ diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index 73c71f1b40..faf395a1ef 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') test.write('SConstruct', """ diff --git a/test/ninja/generate_and_build_cxx.py b/test/ninja/generate_and_build_cxx.py index ac0f55444f..663282bd92 100644 --- a/test/ninja/generate_and_build_cxx.py +++ b/test/ninja/generate_and_build_cxx.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') test.write('SConstruct', """ diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index d9b9c4ed59..76c79bb7da 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') shell = '' if IS_WINDOWS else './' diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index b03e9cbcb0..ff50f502a3 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -27,10 +27,11 @@ import os import time import random -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -40,14 +41,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') test.write('source_0.c', """ @@ -220,14 +219,14 @@ def mod_source_orig(test_num): scons_times += [stop - start] full_build_print = True -for ninja, scons in zip(ninja_times, scons_times): - if ninja > scons: +for ninja_time, scons_time in zip(ninja_times, scons_times): + if ninja_time > scons_time: test.fail_test() if full_build_print: full_build_print = False - print("Clean build {} files - SCons: {:.3f}s Ninja: {:.3f}s".format(num_source, scons, ninja)) + print("Clean build {} files - SCons: {:.3f}s Ninja: {:.3f}s".format(num_source, scons_time, ninja_time)) else: - print("Single File Rebuild - SCons: {:.3f}s Ninja: {:.3f}s".format(scons, ninja)) + print("Single File Rebuild - SCons: {:.3f}s Ninja: {:.3f}s".format(scons_time, ninja_time)) test.pass_test() diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index c9b21b7d70..18ca3cbc69 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,7 +39,7 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index b35a52b6c0..5d7f97e215 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') shell = '' if IS_WINDOWS else './' From 3fa490204dc9dc9c641b2317d1c446cf5f9209bd Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 21 May 2020 16:23:13 -0400 Subject: [PATCH 032/163] removed NINJA_SYNTAX completely --- src/engine/SCons/Tool/ninja.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index a68655ef1d..5515a35413 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -40,7 +40,6 @@ from SCons.Script import COMMAND_LINE_TARGETS NINJA_STATE = None -NINJA_SYNTAX = "NINJA_SYNTAX" NINJA_RULES = "__NINJA_CUSTOM_RULES" NINJA_POOLS = "__NINJA_CUSTOM_POOLS" NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" @@ -1299,7 +1298,6 @@ def generate(env): env["DISABLE_AUTO_NINJA"] = GetOption('disable_execute_ninja') global NINJA_STATE - env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") # Add the Ninja builder. always_exec_ninja_action = AlwaysExecAction(ninja_builder, {}) From ac7b13332a4d6455b694ef079f83d0386e715fa4 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 22 May 2020 00:49:03 -0400 Subject: [PATCH 033/163] removed old sconscript changes --- SCons/Script/SConscript.py | 3 --- SCons/Script/__init__.py | 1 - src/engine/SCons/Tool/ninja.py | 2 +- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/SCons/Script/SConscript.py b/SCons/Script/SConscript.py index ded0fcfef9..596fca0463 100644 --- a/SCons/Script/SConscript.py +++ b/SCons/Script/SConscript.py @@ -203,11 +203,9 @@ def _SConscript(fs, *files, **kw): if f.rexists(): actual = f.rfile() _file_ = open(actual.get_abspath(), "rb") - SCons.Script.LOADED_SCONSCRIPTS.append(actual.get_abspath()) elif f.srcnode().rexists(): actual = f.srcnode().rfile() _file_ = open(actual.get_abspath(), "rb") - SCons.Script.LOADED_SCONSCRIPTS.append(actual.get_abspath()) elif f.has_src_builder(): # The SConscript file apparently exists in a source # code management system. Build it, but then clear @@ -216,7 +214,6 @@ def _SConscript(fs, *files, **kw): f.build() f.built() f.builder_set(None) - SCons.Script.LOADED_SCONSCRIPTS.append(f.get_abspath()) if f.exists(): _file_ = open(f.get_abspath(), "rb") if _file_: diff --git a/SCons/Script/__init__.py b/SCons/Script/__init__.py index c7f6a22a93..dff15673b1 100644 --- a/SCons/Script/__init__.py +++ b/SCons/Script/__init__.py @@ -187,7 +187,6 @@ def _clear(self): BUILD_TARGETS = TargetList() COMMAND_LINE_TARGETS = [] DEFAULT_TARGETS = [] -LOADED_SCONSCRIPTS = [] # BUILD_TARGETS can be modified in the SConscript files. If so, we # want to treat the modified BUILD_TARGETS list as if they specified diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 5515a35413..205bfa79a9 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -727,7 +727,7 @@ def generate(self): # allow us to query the actual SConscripts used. Right now # this glob method has deficiencies like skipping # jstests/SConscript and being specific to the MongoDB - # repository layout. + # repository layout. (github issue #3625) ninja.build( self.ninja_file.path, rule="REGENERATE", From 57a67bb4bcf624c12d98354bea8d49513b1a8822 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 3 Jun 2020 10:47:54 -0400 Subject: [PATCH 034/163] merge commit a7541c60e5904e7deafdedf5bb040cc8924ac7d3 from https://github.com/mongodb/mongo --- src/engine/SCons/Tool/ninja.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 205bfa79a9..689f2ee262 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -1463,6 +1463,16 @@ def robust_rule_mapping(var, rule, tool): SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig) + # Ignore CHANGED_SOURCES and CHANGED_TARGETS. We don't want those + # to have effect in a generation pass because the generator + # shouldn't generate differently depending on the current local + # state. Without this, when generating on Windows, if you already + # had a foo.obj, you would omit foo.cpp from the response file. Do the same for UNCHANGED. + SCons.Executor.Executor._get_changed_sources = SCons.Executor.Executor._get_sources + SCons.Executor.Executor._get_changed_targets = SCons.Executor.Executor._get_targets + SCons.Executor.Executor._get_unchanged_sources = SCons.Executor.Executor._get_sources + SCons.Executor.Executor._get_unchanged_targets = SCons.Executor.Executor._get_targets + # Replace false action messages with nothing. env["PRINT_CMD_LINE_FUNC"] = ninja_noop From 97cb0e6c16e6270e12a5a56f8bad72505b4f808c Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 3 Jun 2020 10:49:15 -0400 Subject: [PATCH 035/163] merge commit 18cbf0d581162b2d15d66577b1fe08fe22006699 from https://github.com/mongodb/mongo --- src/engine/SCons/Tool/ninja.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 689f2ee262..3a1034fea2 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -675,7 +675,16 @@ def generate(self): # use for the "real" builder and multiple phony targets that # match the file names of the remaining outputs. This way any # build can depend on any output from any build. - build["outputs"].sort() + # + # We assume that the first listed output is the 'key' + # output and is stably presented to us by SCons. For + # instance if -gsplit-dwarf is in play and we are + # producing foo.o and foo.dwo, we expect that outputs[0] + # from SCons will be the foo.o file and not the dwo + # file. If instead we just sorted the whole outputs array, + # we would find that the dwo file becomes the + # first_output, and this breaks, for instance, header + # dependency scanning. if rule is not None and (rule.get("deps") or rule.get("rspfile")): first_output, remaining_outputs = ( build["outputs"][0], @@ -684,7 +693,7 @@ def generate(self): if remaining_outputs: ninja.build( - outputs=remaining_outputs, rule="phony", implicit=first_output, + outputs=sorted(remaining_outputs), rule="phony", implicit=first_output, ) build["outputs"] = first_output From 4fc4d2eb205f59e8aa355fb8acb0d186c6819f6e Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 8 Jun 2020 21:43:59 -0400 Subject: [PATCH 036/163] update to build godot reinvoke scons for unhandled actions Ignore Python.Values (need fix) escape rsp content check is_sconscript fix sider issues --- src/engine/SCons/Tool/ninja.py | 98 +++++++++++++++++++--------------- 1 file changed, 56 insertions(+), 42 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 3a1034fea2..3fdadf9d77 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -30,7 +30,6 @@ import shlex import subprocess -from glob import glob from os.path import join as joinpath from os.path import splitext @@ -38,6 +37,7 @@ from SCons.Action import _string_from_cmd_list, get_default_ENV from SCons.Util import is_List, flatten_sequence from SCons.Script import COMMAND_LINE_TARGETS +from SCons.Node import SConscriptNodes NINJA_STATE = None NINJA_RULES = "__NINJA_CUSTOM_RULES" @@ -58,10 +58,11 @@ def _install_action_function(_env, node): """Install files using the install or copy commands""" + #TODO: handle Python.Value nodes return { "outputs": get_outputs(node), "rule": "INSTALL", - "inputs": [get_path(src_file(s)) for s in node.sources], + "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], "implicit": get_dependencies(node), } @@ -83,9 +84,10 @@ def _mkdir_action_function(env, node): } def _copy_action_function(env, node): + #TODO: handle Python.Value nodes return { "outputs": get_outputs(node), - "inputs": [get_path(src_file(s)) for s in node.sources], + "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], "rule": "CMD", "variables": { "cmd": "$COPY $in $out", @@ -134,11 +136,12 @@ def is_valid_dependent_node(node): def alias_to_ninja_build(node): """Convert an Alias node into a Ninja phony target""" + # TODO: handle Python.Values return { "outputs": get_outputs(node), "rule": "phony", "implicit": [ - get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) + get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) and not isinstance(n, SCons.Node.Python.Value) ], } @@ -147,18 +150,20 @@ def get_order_only(node): """Return a list of order only dependencies for node.""" if node.prerequisites is None: return [] - return [get_path(src_file(prereq)) for prereq in node.prerequisites] + #TODO: handle Python.Value nodes + return [get_path(src_file(prereq)) for prereq in node.prerequisites if not isinstance(prereq, SCons.Node.Python.Value)] def get_dependencies(node, skip_sources=False): """Return a list of dependencies for node.""" + #TODO: handle Python.Value nodes if skip_sources: return [ get_path(src_file(child)) for child in node.children() - if child not in node.sources + if child not in node.sources and not isinstance(child, SCons.Node.Python.Value) ] - return [get_path(src_file(child)) for child in node.children()] + return [get_path(src_file(child)) for child in node.children() if not isinstance(child, SCons.Node.Python.Value)] def get_inputs(node): @@ -168,8 +173,8 @@ def get_inputs(node): inputs = executor.get_all_sources() else: inputs = node.sources - - inputs = [get_path(src_file(o)) for o in inputs] + #TODO: handle Python.Value nodes + inputs = [get_path(src_file(o)) for o in inputs if not isinstance(o, SCons.Node.Python.Value)] return inputs @@ -202,6 +207,7 @@ def __init__(self, env): # this check for us. "SharedFlagChecker": ninja_noop, # The install builder is implemented as a function action. + # TODO: use command action #3573 "installFunc": _install_action_function, "MkdirFunc": _mkdir_action_function, "LibSymlinksActionFunction": _lib_symlink_action_function, @@ -262,12 +268,6 @@ def handle_func_action(self, node, action): # the bottom of ninja's DAG anyway and Textfile builders can have text # content as their source which doesn't work as an implicit dep in # ninja. - if name == "_action": - return { - "rule": "TEMPLATE", - "outputs": get_outputs(node), - "implicit": get_dependencies(node, skip_sources=True), - } if name == 'ninja_builder': return None @@ -280,7 +280,7 @@ def handle_func_action(self, node, action): if handler is not None: return handler(node.env if node.env else self.env, node) - raise Exception( + SCons.Warnings.Warning( "Found unhandled function action {}, " " generating scons command to build\n" "Note: this is less efficient than Ninja," @@ -288,6 +288,12 @@ def handle_func_action(self, node, action): " this function using NinjaRegisterFunctionHandler".format(name) ) + return { + "rule": "TEMPLATE", + "outputs": get_outputs(node), + "implicit": get_dependencies(node, skip_sources=True), + } + # pylint: disable=too-many-branches def handle_list_action(self, node, action): """TODO write this comment""" @@ -309,7 +315,7 @@ def handle_list_action(self, node, action): all_outputs = list({output for build in results for output in build["outputs"]}) dependencies = list({dep for build in results for dep in build["implicit"]}) - if results[0]["rule"] == "CMD": + if results[0]["rule"] == "CMD" or results[0]["rule"] == "GENERATED_CMD": cmdline = "" for cmd in results: @@ -339,7 +345,7 @@ def handle_list_action(self, node, action): if cmdline: ninja_build = { "outputs": all_outputs, - "rule": "CMD", + "rule": "GENERATED_CMD", "variables": { "cmd": cmdline, "env": get_command_env(node.env if node.env else self.env), @@ -360,10 +366,11 @@ def handle_list_action(self, node, action): } elif results[0]["rule"] == "INSTALL": + #TODO: handle Python.Value nodes return { "outputs": all_outputs, "rule": "INSTALL", - "inputs": [get_path(src_file(s)) for s in node.sources], + "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], "implicit": dependencies, } @@ -397,22 +404,28 @@ def __init__(self, env, ninja_file, writer_class): # like CCFLAGS escape = env.get("ESCAPE", lambda x: x) + # if SCons was invoked from python, we expect the first arg to be the scons.py + # script, otherwise scons was invoked from the scons script + python_bin = '' + if os.path.basename(sys.argv[0]) == 'scons.py': + python_bin = escape(sys.executable) self.variables = { "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", - "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format( - sys.executable, + "SCONS_INVOCATION": '{} {} --disable-ninja __NINJA_NO=1 $out'.format( + python_bin, " ".join( [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] ), ), - "SCONS_INVOCATION_W_TARGETS": "{} {}".format( - sys.executable, " ".join([escape(arg) for arg in sys.argv]) + "SCONS_INVOCATION_W_TARGETS": "{} {} --disable-ninja".format( + python_bin, " ".join([escape(arg) for arg in sys.argv]) ), # This must be set to a global default per: - # https://ninja-build.org/manual.html - # - # (The deps section) - "msvc_deps_prefix": "Note: including file:", + # https://ninja-build.org/manual.html#_deps + # English Visual Studio will have the default below, + # otherwise the user can define the variable in the first environment + # that initialized ninja tool + "msvc_deps_prefix": env.get("NINJA_MSVC_DEPS_PREFIX", "Note: including file:") } self.rules = { @@ -481,13 +494,13 @@ def __init__(self, env, ninja_file, writer_class): }, "TEMPLATE": { "command": "$SCONS_INVOCATION $out", - "description": "Rendering $out", + "description": "Rendering $SCONS_INVOCATION $out", "pool": "scons_pool", "restat": 1, }, "SCONS": { "command": "$SCONS_INVOCATION $out", - "description": "SCons $out", + "description": "$SCONS_INVOCATION $out", "pool": "scons_pool", # restat # if present, causes Ninja to re-stat the command's outputs @@ -557,6 +570,8 @@ def add_build(self, node): self.built.update(build["outputs"]) return True + # TODO: rely on SCons to tell us what is generated source + # or some form of user scanner maybe (Github Issue #3624) def is_generated_source(self, output): """Check if output ends with a known generated suffix.""" _, suffix = splitext(output) @@ -740,11 +755,7 @@ def generate(self): ninja.build( self.ninja_file.path, rule="REGENERATE", - implicit=[ - self.env.File("#SConstruct").path, - __file__, - ] - + sorted(glob("src/**/SConscript", recursive=True)), + implicit=[__file__] + [str(node) for node in SConscriptNodes], ) # If we ever change the name/s of the rules that include @@ -921,6 +932,7 @@ def get_response_file_command(env, node, action, targets, sources, executor=None ) cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] + rsp_content = ['"' + rsp_content_item + '"' for rsp_content_item in rsp_content] rsp_content = " ".join(rsp_content) variables = {"rspc": rsp_content} @@ -1060,20 +1072,23 @@ def ninja_builder(env, target, source): for key in env['ENV']: f.write('set {}={}\n'.format(key, env['ENV'][key])) f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) - - if not env.get("DISABLE_AUTO_NINJA"): + cmd = ['run_ninja_env.bat'] + + else: cmd = [NINJA_STATE.ninja_bin_path, '-f', generated_build_ninja] + + if not env.get("DISABLE_AUTO_NINJA"): print("Executing:", str(' '.join(cmd))) # execute the ninja build at the end of SCons, trying to # reproduce the output like a ninja build would def execute_ninja(): - + proc = subprocess.Popen(cmd, stderr=sys.stderr, stdout=subprocess.PIPE, universal_newlines=True, - env=env['ENV'] # ninja build items won't consider node env on win32 + env=os.environ if env["PLATFORM"] == "win32" else env['ENV'] ) for stdout_line in iter(proc.stdout.readline, ""): yield stdout_line @@ -1142,8 +1157,7 @@ def ninja_csig(original): """Return a dummy csig""" def wrapper(self): - name = str(self) - if "SConscript" in name or "SConstruct" in name: + if isinstance(self, SCons.Node.Node) and self.is_sconscript(): return original(self) return "dummy_ninja_csig" @@ -1154,8 +1168,7 @@ def ninja_contents(original): """Return a dummy content without doing IO""" def wrapper(self): - name = str(self) - if "SConscript" in name or "SConstruct" in name: + if isinstance(self, SCons.Node.Node) and self.is_sconscript(): return original(self) return bytes("dummy_ninja_contents", encoding="utf-8") @@ -1396,6 +1409,7 @@ def robust_rule_mapping(var, rule, tool): # Used to determine if a build generates a source file. Ninja # requires that all generated sources are added as order_only # dependencies to any builds that *might* use them. + # TODO: switch to using SCons to help determine this (Github Issue #3624) env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] if env["PLATFORM"] != "win32" and env.get("RANLIBCOM"): From 03e57a501644e67e5dac857c25ee4956c995aef0 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 22 Jun 2020 15:21:34 -0400 Subject: [PATCH 037/163] revert ninja install requirement expand response file in ninja comdb output --- src/engine/SCons/Tool/ninja.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 3fdadf9d77..ad36758788 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -767,7 +767,7 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "{} -f {} -t compdb CC CXX > compile_commands.json".format(self.ninja_bin_path, + "cmd": "{} -f {} -t compdb -x CC CXX > compile_commands.json".format(self.ninja_bin_path, str(self.ninja_file) ) }, From 4b958617d8db6072a99b6e81990c9b13890687cf Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 25 Jun 2020 23:05:28 -0400 Subject: [PATCH 038/163] handle files which are not file or alias by reinvoking scons --- src/engine/SCons/Tool/ninja.py | 228 +++++++++++++++++++-------------- 1 file changed, 134 insertions(+), 94 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index ad36758788..50a9518aae 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -56,63 +56,6 @@ ) -def _install_action_function(_env, node): - """Install files using the install or copy commands""" - #TODO: handle Python.Value nodes - return { - "outputs": get_outputs(node), - "rule": "INSTALL", - "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], - "implicit": get_dependencies(node), - } - -def _mkdir_action_function(env, node): - return { - "outputs": get_outputs(node), - "rule": "CMD", - # implicit explicitly omitted, we translate these so they can be - # used by anything that depends on these but commonly this is - # hit with a node that will depend on all of the fake - # srcnode's that SCons will never give us a rule for leading - # to an invalid ninja file. - "variables": { - # On Windows mkdir "-p" is always on - "cmd": "{mkdir}".format( - mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", - ), - }, - } - -def _copy_action_function(env, node): - #TODO: handle Python.Value nodes - return { - "outputs": get_outputs(node), - "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], - "rule": "CMD", - "variables": { - "cmd": "$COPY $in $out", - }, - } - - -def _lib_symlink_action_function(_env, node): - """Create shared object symlinks if any need to be created""" - symlinks = getattr(getattr(node, "attributes", None), "shliblinks", None) - - if not symlinks or symlinks is None: - return None - - outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] - inputs = [link.get_path() for link, _ in symlinks] - - return { - "outputs": outputs, - "inputs": inputs, - "rule": "SYMLINK", - "implicit": get_dependencies(node), - } - - def is_valid_dependent_node(node): """ Return True if node is not an alias or is an alias that has children @@ -136,46 +79,65 @@ def is_valid_dependent_node(node): def alias_to_ninja_build(node): """Convert an Alias node into a Ninja phony target""" - # TODO: handle Python.Values return { "outputs": get_outputs(node), "rule": "phony", "implicit": [ - get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) and not isinstance(n, SCons.Node.Python.Value) + get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) ], } +def check_invalid_ninja_node(node): + return not isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)) + + +def filter_ninja_nodes(node_list): + ninja_nodes = [] + for node in node_list: + if isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)): + ninja_nodes.append(node) + else: + continue + return ninja_nodes + +def get_input_nodes(node): + if node.get_executor() is not None: + inputs = node.get_executor().get_all_sources() + else: + inputs = node.sources + return inputs + + +def invalid_ninja_nodes(node, targets): + result = False + for node_list in [node.prerequisites, get_input_nodes(node), node.children(), targets]: + if node_list: + result = result or any([check_invalid_ninja_node(node) for node in node_list]) + return result + + def get_order_only(node): """Return a list of order only dependencies for node.""" if node.prerequisites is None: return [] - #TODO: handle Python.Value nodes - return [get_path(src_file(prereq)) for prereq in node.prerequisites if not isinstance(prereq, SCons.Node.Python.Value)] + return [get_path(src_file(prereq)) for prereq in filter_ninja_nodes(node.prerequisites)] def get_dependencies(node, skip_sources=False): """Return a list of dependencies for node.""" - #TODO: handle Python.Value nodes if skip_sources: return [ get_path(src_file(child)) - for child in node.children() - if child not in node.sources and not isinstance(child, SCons.Node.Python.Value) + for child in filter_ninja_nodes(node.children()) + if child not in node.sources ] - return [get_path(src_file(child)) for child in node.children() if not isinstance(child, SCons.Node.Python.Value)] + return [get_path(src_file(child)) for child in filter_ninja_nodes(node.children())] def get_inputs(node): """Collect the Ninja inputs for node.""" - executor = node.get_executor() - if executor is not None: - inputs = executor.get_all_sources() - else: - inputs = node.sources - #TODO: handle Python.Value nodes - inputs = [get_path(src_file(o)) for o in inputs if not isinstance(o, SCons.Node.Python.Value)] - return inputs + return [get_path(src_file(o)) for o in filter_ninja_nodes(get_input_nodes(node))] def get_outputs(node): @@ -189,11 +151,93 @@ def get_outputs(node): else: outputs = [node] - outputs = [get_path(o) for o in outputs] + outputs = [get_path(o) for o in filter_ninja_nodes(outputs)] return outputs +def get_targets_sources(node): + executor = node.get_executor() + if executor is not None: + tlist = executor.get_all_targets() + slist = executor.get_all_sources() + else: + if hasattr(node, "target_peers"): + tlist = node.target_peers + else: + tlist = [node] + slist = node.sources + + # Retrieve the repository file for all sources + slist = [rfile(s) for s in slist] + return tlist, slist + + +def get_rule(node, rule): + tlist, slist = get_targets_sources(node) + if invalid_ninja_nodes(node, tlist): + return "TEMPLATE" + else: + return rule + + +def _install_action_function(_env, node): + """Install files using the install or copy commands""" + return { + "outputs": get_outputs(node), + "rule": get_rule(node, "INSTALL"), + "inputs": get_inputs(node), + "implicit": get_dependencies(node), + } + + +def _mkdir_action_function(env, node): + return { + "outputs": get_outputs(node), + "rule": get_rule(node, "CMD"), + # implicit explicitly omitted, we translate these so they can be + # used by anything that depends on these but commonly this is + # hit with a node that will depend on all of the fake + # srcnode's that SCons will never give us a rule for leading + # to an invalid ninja file. + "variables": { + # On Windows mkdir "-p" is always on + "cmd": "{mkdir}".format( + mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", + ), + }, + } + + +def _copy_action_function(env, node): + return { + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "rule": get_rule(node, "CMD"), + "variables": { + "cmd": "$COPY $in $out", + }, + } + + +def _lib_symlink_action_function(_env, node): + """Create shared object symlinks if any need to be created""" + symlinks = getattr(getattr(node, "attributes", None), "shliblinks", None) + + if not symlinks or symlinks is None: + return None + + outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] + inputs = [link.get_path() for link, _ in symlinks] + + return { + "outputs": outputs, + "inputs": inputs, + "rule": get_rule(node, "SYMLINK"), + "implicit": get_dependencies(node), + } + + class SConsToNinjaTranslator: """Translates SCons Actions into Ninja build objects.""" @@ -290,7 +334,9 @@ def handle_func_action(self, node, action): return { "rule": "TEMPLATE", + "order_only": get_order_only(node), "outputs": get_outputs(node), + "inputs": get_inputs(node), "implicit": get_dependencies(node, skip_sources=True), } @@ -345,7 +391,7 @@ def handle_list_action(self, node, action): if cmdline: ninja_build = { "outputs": all_outputs, - "rule": "GENERATED_CMD", + "rule": get_rule(node, "GENERATED_CMD"), "variables": { "cmd": cmdline, "env": get_command_env(node.env if node.env else self.env), @@ -366,11 +412,10 @@ def handle_list_action(self, node, action): } elif results[0]["rule"] == "INSTALL": - #TODO: handle Python.Value nodes return { "outputs": all_outputs, - "rule": "INSTALL", - "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], + "rule": get_rule(node, "INSTALL"), + "inputs": get_inputs(node), "implicit": dependencies, } @@ -985,20 +1030,8 @@ def get_command(env, node, action): # pylint: disable=too-many-branches sub_env = node.env else: sub_env = env - executor = node.get_executor() - if executor is not None: - tlist = executor.get_all_targets() - slist = executor.get_all_sources() - else: - if hasattr(node, "target_peers"): - tlist = node.target_peers - else: - tlist = [node] - slist = node.sources - - # Retrieve the repository file for all sources - slist = [rfile(s) for s in slist] + tlist, slist = get_targets_sources(node) # Generate a real CommandAction if isinstance(action, SCons.Action.CommandGeneratorAction): @@ -1022,10 +1055,11 @@ def get_command(env, node, action): # pylint: disable=too-many-branches "outputs": get_outputs(node), "inputs": get_inputs(node), "implicit": implicit, - "rule": rule, + "rule": get_rule(node, rule), "variables": variables, } + # Don't use sub_env here because we require that NINJA_POOL be set # on a per-builder call basis to prevent accidental strange # behavior like env['NINJA_POOL'] = 'console' and sub_env can be @@ -1283,6 +1317,11 @@ def exists(env): if env.get("__NINJA_NO", "0") == "1": return False + try: + import ninja + except ImportError: + SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") + return False return True added = None @@ -1308,8 +1347,6 @@ def generate(env): default=False, help='Disable ninja automatically building after scons') - if GetOption('disable_ninja'): - return env try: import ninja @@ -1427,6 +1464,9 @@ def robust_rule_mapping(var, rule, tool): # Disable running ranlib, since we added 's' above env["RANLIBCOM"] = "" + if GetOption('disable_ninja'): + return env + SCons.Warnings.Warning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") # This is the point of no return, anything after this comment From 51c10b6ddd9e48aad52720e972c8b990a19a8072 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 20 Jul 2020 17:31:56 +0000 Subject: [PATCH 039/163] updated with some changes from latest mongodb version: 21075112a999e252a22e9c9bd64e403cec892df3 5fe923a0aa312044062df044eb4eaa47951f70ec c7348f391124e681d9c62aceb0e13e0d07fca8bc --- src/engine/SCons/Tool/ninja.py | 40 +++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 50a9518aae..bf7c45add0 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -29,6 +29,7 @@ import shutil import shlex import subprocess +import textwrap from os.path import join as joinpath from os.path import splitext @@ -768,13 +769,13 @@ def generate(self): # Special handling for outputs and implicit since we need to # aggregate not replace for each builder. - for agg_key in ["outputs", "implicit"]: + for agg_key in ["outputs", "implicit", 'inputs']: new_val = template_builds.get(agg_key, []) # Use pop so the key is removed and so the update # below will not overwrite our aggregated values. cur_val = template_builder.pop(agg_key, []) - if isinstance(cur_val, list): + if is_List(cur_val): new_val += cur_val else: new_val.append(cur_val) @@ -812,8 +813,8 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "{} -f {} -t compdb -x CC CXX > compile_commands.json".format(self.ninja_bin_path, - str(self.ninja_file) + "cmd": "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format( + ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' ) }, ) @@ -929,7 +930,13 @@ def get_command_env(env): if windows: command_env += "set '{}={}' && ".format(key, value) else: - command_env += "{}={} ".format(key, value) + # We address here *only* the specific case that a user might have + # an environment variable which somehow gets included and has + # spaces in the value. These are escapes that Ninja handles. This + # doesn't make builds on paths with spaces (Ninja and SCons issues) + # nor expanding response file paths with spaces (Ninja issue) work. + value = value.replace(r' ', r'$ ') + command_env += "{}='{}' ".format(key, value) env["NINJA_ENV_VAR_CACHE"] = command_env return command_env @@ -1208,6 +1215,27 @@ def wrapper(self): return wrapper +def CheckNinjaCompdbExpand(env, context): + """ Configure check testing if ninja's compdb can expand response files""" + + context.Message('Checking if ninja compdb can expand response files... ') + ret, output = context.TryAction( + action='ninja -f $SOURCE -t compdb -x CMD_RSP > $TARGET', + extension='.ninja', + text=textwrap.dedent(""" + rule CMD_RSP + command = $cmd @$out.rsp > fake_output.txt + description = Building $out + rspfile = $out.rsp + rspfile_content = $rspc + build fake_output.txt: CMD_RSP fake_input.txt + cmd = echo + pool = console + rspc = "test" + """)) + result = '@fake_output.txt.rsp' not in output + context.Result(result) + return result def ninja_stat(_self, path): """ @@ -1386,6 +1414,8 @@ def generate(env): else: env.Append(CCFLAGS=["-MD", "-MF", "${TARGET}.d"]) + env.AddMethod(CheckNinjaCompdbExpand, "CheckNinjaCompdbExpand") + # Provide a way for custom rule authors to easily access command # generation. env.AddMethod(get_shell_command, "NinjaGetShellCommand") From b683911bb512300b6a6123ce92f4ba8566b3b3ce Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 20 Jul 2020 17:43:48 +0000 Subject: [PATCH 040/163] fixed sider issues --- src/engine/SCons/Tool/ninja.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index bf7c45add0..153a8943b5 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -813,8 +813,8 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format( - ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' + "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( + self.ninja_bin_path, str(self.ninja_file), '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' ) }, ) @@ -1347,10 +1347,11 @@ def exists(env): try: import ninja + return ninja.__file__ except ImportError: SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") return False - return True + added = None From cc1ce35f62e214e3599afcdc4d413396677e2ea9 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 20 Jul 2020 18:25:39 +0000 Subject: [PATCH 041/163] updated warning to the latest API --- src/engine/SCons/Tool/ninja.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 153a8943b5..502d365014 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -325,7 +325,7 @@ def handle_func_action(self, node, action): if handler is not None: return handler(node.env if node.env else self.env, node) - SCons.Warnings.Warning( + SCons.Warnings.SConsWarning( "Found unhandled function action {}, " " generating scons command to build\n" "Note: this is less efficient than Ninja," @@ -1349,7 +1349,7 @@ def exists(env): import ninja return ninja.__file__ except ImportError: - SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") + SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return False @@ -1380,7 +1380,7 @@ def generate(env): try: import ninja except ImportError: - SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") + SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return env["DISABLE_AUTO_NINJA"] = GetOption('disable_execute_ninja') @@ -1405,7 +1405,7 @@ def generate(env): env.Alias("$NINJA_ALIAS_NAME", ninja_file) else: if str(NINJA_STATE.ninja_file) != ninja_file_name: - SCons.Warnings.Warning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") + SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] # This adds the required flags such that the generated compile @@ -1498,7 +1498,7 @@ def robust_rule_mapping(var, rule, tool): if GetOption('disable_ninja'): return env - SCons.Warnings.Warning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") + SCons.Warnings.SConsWarning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") # This is the point of no return, anything after this comment # makes changes to SCons that are irreversible and incompatible From 7f72e3ae6d988fd1fa430d10465d76aa1caaf722 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 31 Dec 2020 15:58:31 +0000 Subject: [PATCH 042/163] Sync with mongo ninja file --- src/engine/SCons/Tool/ninja.py | 230 ++++++++++++++++++++++++++------- 1 file changed, 186 insertions(+), 44 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 502d365014..ea01d590ee 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -31,6 +31,7 @@ import subprocess import textwrap +from glob import glob from os.path import join as joinpath from os.path import splitext @@ -156,6 +157,41 @@ def get_outputs(node): return outputs +def generate_depfile(env, node, dependencies): + """ + Ninja tool function for writing a depfile. The depfile should include + the node path followed by all the dependent files in a makefile format. + + dependencies arg can be a list or a subst generator which returns a list. + """ + + depfile = os.path.join(get_path(env['NINJA_BUILDDIR']), str(node) + '.depfile') + + # subst_list will take in either a raw list or a subst callable which generates + # a list, and return a list of CmdStringHolders which can be converted into raw strings. + # If a raw list was passed in, then scons_list will make a list of lists from the original + # values and even subst items in the list if they are substitutable. Flatten will flatten + # the list in that case, to ensure for either input we have a list of CmdStringHolders. + deps_list = env.Flatten(env.subst_list(dependencies)) + + # Now that we have the deps in a list as CmdStringHolders, we can convert them into raw strings + # and make sure to escape the strings to handle spaces in paths. We also will sort the result + # keep the order of the list consistent. + escaped_depends = sorted([dep.escape(env.get("ESCAPE", lambda x: x)) for dep in deps_list]) + depfile_contents = str(node) + ": " + ' '.join(escaped_depends) + + need_rewrite = False + try: + with open(depfile, 'r') as f: + need_rewrite = (f.read() != depfile_contents) + except FileNotFoundError: + need_rewrite = True + + if need_rewrite: + os.makedirs(os.path.dirname(depfile) or '.', exist_ok=True) + with open(depfile, 'w') as f: + f.write(depfile_contents) + def get_targets_sources(node): executor = node.get_executor() @@ -278,6 +314,7 @@ def action_to_ninja_build(self, node, action=None): return None build = {} + env = node.env if node.env else self.env # Ideally this should never happen, and we do try to filter # Ninja builders out of being sources of ninja builders but I @@ -286,22 +323,27 @@ def action_to_ninja_build(self, node, action=None): global NINJA_STATE if NINJA_STATE.ninja_file == str(node): build = None - if isinstance(action, SCons.Action.FunctionAction): + elif isinstance(action, SCons.Action.FunctionAction): build = self.handle_func_action(node, action) elif isinstance(action, SCons.Action.LazyAction): # pylint: disable=protected-access - action = action._generate_cache(node.env if node.env else self.env) + action = action._generate_cache(env) build = self.action_to_ninja_build(node, action=action) elif isinstance(action, SCons.Action.ListAction): build = self.handle_list_action(node, action) elif isinstance(action, COMMAND_TYPES): - build = get_command(node.env if node.env else self.env, node, action) + build = get_command(env, node, action) else: raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) if build is not None: build["order_only"] = get_order_only(node) + if 'conftest' not in str(node): + node_callback = getattr(node.attributes, "ninja_build_callback", None) + if callable(node_callback): + node_callback(env, node, build) + return build def handle_func_action(self, node, action): @@ -509,8 +551,16 @@ def __init__(self, env, ninja_file, writer_class): "rspfile_content": "$rspc", "pool": "local_pool", }, + # Ninja does not automatically delete the archive before + # invoking ar. The ar utility will append to an existing archive, which + # can cause duplicate symbols if the symbols moved between object files. + # Native SCons will perform this operation so we need to force ninja + # to do the same. See related for more info: + # https://jira.mongodb.org/browse/SERVER-49457 "AR": { - "command": "$env$AR @$out.rsp", + "command": "{}$env$AR @$out.rsp".format( + '' if sys.platform == "win32" else "rm -f $out && " + ), "description": "Archiving $out", "rspfile": "$out.rsp", "rspfile_content": "$rspc", @@ -540,7 +590,7 @@ def __init__(self, env, ninja_file, writer_class): }, "TEMPLATE": { "command": "$SCONS_INVOCATION $out", - "description": "Rendering $SCONS_INVOCATION $out", + "description": "Rendering $SCONS_INVOCATION $out", "pool": "scons_pool", "restat": 1, }, @@ -570,6 +620,7 @@ def __init__(self, env, ninja_file, writer_class): "command": "$SCONS_INVOCATION_W_TARGETS", "description": "Regenerating $out", "generator": 1, + "depfile": os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile'), # Console pool restricts to 1 job running at a time, # it additionally has some special handling about # passing stdin, stdout, etc to process in this pool @@ -650,6 +701,8 @@ def generate(self): ninja.comment("Generated by scons. DO NOT EDIT.") + ninja.variable("builddir", get_path(self.env['NINJA_BUILDDIR'])) + for pool_name, size in self.pools.items(): ninja.pool(pool_name, size) @@ -759,9 +812,19 @@ def generate(self): build["outputs"] = first_output + # Optionally a rule can specify a depfile, and SCons can generate implicit + # dependencies into the depfile. This allows for dependencies to come and go + # without invalidating the ninja file. The depfile was created in ninja specifically + # for dealing with header files appearing and disappearing across rebuilds, but it can + # be repurposed for anything, as long as you have a way to regenerate the depfile. + # More specific info can be found here: https://ninja-build.org/manual.html#_depfile + if rule is not None and rule.get('depfile') and build.get('deps_files'): + path = build['outputs'] if SCons.Util.is_List(build['outputs']) else [build['outputs']] + generate_depfile(self.env, path[0], build.pop('deps_files', [])) + if "inputs" in build: build["inputs"].sort() - + ninja.build(**build) template_builds = dict() @@ -769,7 +832,7 @@ def generate(self): # Special handling for outputs and implicit since we need to # aggregate not replace for each builder. - for agg_key in ["outputs", "implicit", 'inputs']: + for agg_key in ["outputs", "implicit", "inputs"]: new_val = template_builds.get(agg_key, []) # Use pop so the key is removed and so the update @@ -793,15 +856,25 @@ def generate(self): # generate this rule even though SCons should know we're # dependent on SCons files. # - # TODO: We're working on getting an API into SCons that will - # allow us to query the actual SConscripts used. Right now - # this glob method has deficiencies like skipping - # jstests/SConscript and being specific to the MongoDB - # repository layout. (github issue #3625) + # The REGENERATE rule uses depfile, so we need to generate the depfile + # in case any of the SConscripts have changed. The depfile needs to be + # path with in the build and the passed ninja file is an abspath, so + # we will use SCons to give us the path within the build. Normally + # generate_depfile should not be called like this, but instead be called + # through the use of custom rules, and filtered out in the normal + # list of build generation about. However, because the generate rule + # is hardcoded here, we need to do this generate_depfile call manually. + ninja_file_path = self.env.File(self.ninja_file).path + generate_depfile( + self.env, + ninja_file_path, + self.env['NINJA_REGENERATE_DEPS'] + ) + ninja.build( - self.ninja_file.path, + ninja_file_path, rule="REGENERATE", - implicit=[__file__] + [str(node) for node in SConscriptNodes], + implicit=[__file__], ) # If we ever change the name/s of the rules that include @@ -936,13 +1009,13 @@ def get_command_env(env): # doesn't make builds on paths with spaces (Ninja and SCons issues) # nor expanding response file paths with spaces (Ninja issue) work. value = value.replace(r' ', r'$ ') - command_env += "{}='{}' ".format(key, value) + command_env += "export {}='{}';".format(key, value) env["NINJA_ENV_VAR_CACHE"] = command_env return command_env -def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False): +def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom_env={}): """Generate a response file command provider for rule name.""" # If win32 using the environment with a response file command will cause @@ -979,7 +1052,7 @@ def get_response_file_command(env, node, action, targets, sources, executor=None except ValueError: raise Exception( "Could not find tool {} in {} generated from {}".format( - tool_command, cmd_list, get_comstr(env, action, targets, sources) + tool, cmd_list, get_comstr(env, action, targets, sources) ) ) @@ -991,7 +1064,12 @@ def get_response_file_command(env, node, action, targets, sources, executor=None variables[rule] = cmd if use_command_env: variables["env"] = get_command_env(env) - return rule, variables + + for key, value in custom_env.items(): + variables["env"] += env.subst( + f"export {key}={value};", target=targets, source=sources, executor=executor + ) + " " + return rule, variables, [tool_command] return get_response_file_command @@ -1021,13 +1099,21 @@ def generate_command(env, node, action, targets, sources, executor=None): return cmd.replace("$", "$$") -def get_shell_command(env, node, action, targets, sources, executor=None): +def get_generic_shell_command(env, node, action, targets, sources, executor=None): return ( - "GENERATED_CMD", + "CMD", { "cmd": generate_command(env, node, action, targets, sources, executor=None), "env": get_command_env(env), }, + # Since this function is a rule mapping provider, it must return a list of dependencies, + # and usually this would be the path to a tool, such as a compiler, used for this rule. + # However this function is to generic to be able to reliably extract such deps + # from the command, so we return a placeholder empty list. It should be noted that + # generally this function will not be used soley and is more like a template to generate + # the basics for a custom provider which may have more specific options for a provier + # function for a custom NinjaRuleMapping. + [] ) @@ -1050,13 +1136,49 @@ def get_command(env, node, action): # pylint: disable=too-many-branches comstr = get_comstr(sub_env, action, tlist, slist) if not comstr: return None - - provider = __NINJA_RULE_MAPPING.get(comstr, get_shell_command) - rule, variables = provider(sub_env, node, action, tlist, slist, executor=executor) + + provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) + rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) # Get the dependencies for all targets implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) + # Now add in the other dependencies related to the command, + # e.g. the compiler binary. The ninja rule can be user provided so + # we must do some validation to resolve the dependency path for ninja. + for provider_dep in provider_deps: + + provider_dep = sub_env.subst(provider_dep) + if not provider_dep: + continue + + # If the tool is a node, then SCons will resolve the path later, if its not + # a node then we assume it generated from build and make sure it is existing. + if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep): + implicit.append(provider_dep) + continue + + # in some case the tool could be in the local directory and be suppled without the ext + # such as in windows, so append the executable suffix and check. + prog_suffix = sub_env.get('PROGSUFFIX', '') + provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix + if os.path.exists(provider_dep_ext): + implicit.append(provider_dep_ext) + continue + + # Many commands will assume the binary is in the path, so + # we accept this as a possible input from a given command. + + provider_dep_abspath = sub_env.WhereIs(provider_dep) or sub_env.WhereIs(provider_dep, path=os.environ["PATH"]) + if provider_dep_abspath: + implicit.append(provider_dep_abspath) + continue + + # Possibly these could be ignore and the build would still work, however it may not always + # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided + # ninja rule. + raise Exception(f"Could not resolve path for {provider_dep} dependency on node '{node}'") + ninja_build = { "order_only": get_order_only(node), "outputs": get_outputs(node), @@ -1066,7 +1188,6 @@ def get_command(env, node, action): # pylint: disable=too-many-branches "variables": variables, } - # Don't use sub_env here because we require that NINJA_POOL be set # on a per-builder call basis to prevent accidental strange # behavior like env['NINJA_POOL'] = 'console' and sub_env can be @@ -1103,28 +1224,28 @@ def ninja_builder(env, target, source): NINJA_STATE.generate() if env["PLATFORM"] == "win32": - # this is not great, its doesn't consider specific + # this is not great, its doesn't consider specific # node environments, which means on linux the build could # behave differently, because on linux you can set the environment - # per command in the ninja file. This is only needed if + # per command in the ninja file. This is only needed if # running ninja directly from a command line that hasn't # had the environment setup (vcvarsall.bat) with open('run_ninja_env.bat', 'w') as f: for key in env['ENV']: f.write('set {}={}\n'.format(key, env['ENV'][key])) - f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) - cmd = ['run_ninja_env.bat'] + f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) + cmd = ['run_ninja_env.bat'] else: cmd = [NINJA_STATE.ninja_bin_path, '-f', generated_build_ninja] - + if not env.get("DISABLE_AUTO_NINJA"): print("Executing:", str(' '.join(cmd))) # execute the ninja build at the end of SCons, trying to # reproduce the output like a ninja build would def execute_ninja(): - + proc = subprocess.Popen(cmd, stderr=sys.stderr, stdout=subprocess.PIPE, @@ -1137,7 +1258,7 @@ def execute_ninja(): return_code = proc.wait() if return_code: raise subprocess.CalledProcessError(return_code, 'ninja') - + erase_previous = False for output in execute_ninja(): output = output.strip() @@ -1168,24 +1289,31 @@ def register_custom_handler(env, name, handler): def register_custom_rule_mapping(env, pre_subst_string, rule): - """Register a custom handler for SCons function actions.""" + """Register a function to call for a given rule.""" global __NINJA_RULE_MAPPING __NINJA_RULE_MAPPING[pre_subst_string] = rule -def register_custom_rule(env, rule, command, description="", deps=None, pool=None): +def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): """Allows specification of Ninja rules from inside SCons files.""" rule_obj = { "command": command, "description": description if description else "{} $out".format(rule), } + if use_depfile: + rule_obj["depfile"] = os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile') + if deps is not None: rule_obj["deps"] = deps if pool is not None: rule_obj["pool"] = pool + if use_response_file: + rule_obj["rspfile"] = "$out.rsp" + rule_obj["rspfile_content"] = response_file_content + env[NINJA_RULES][rule] = rule_obj @@ -1193,6 +1321,9 @@ def register_custom_pool(env, pool, size): """Allows the creation of custom Ninja pools""" env[NINJA_POOLS][pool] = size +def set_build_node_callback(env, node, callback): + if 'conftest' not in str(node): + setattr(node.attributes, "ninja_build_callback", callback) def ninja_csig(original): """Return a dummy csig""" @@ -1395,7 +1526,7 @@ def generate(env): env["NINJA_PREFIX"] = env.get("NINJA_PREFIX", "build") env["NINJA_SUFFIX"] = env.get("NINJA_SUFFIX", "ninja") env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") - + env['NINJA_BUILDDIR'] = env.get("NINJA_BUILDDIR", env.Dir(".ninja").path) ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") # here we allow multiple environments to construct rules and builds # into the same ninja file @@ -1407,20 +1538,31 @@ def generate(env): if str(NINJA_STATE.ninja_file) != ninja_file_name: SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] - + + + # TODO: API for getting the SConscripts programmatically + # exists upstream: https://github.com/SCons/scons/issues/3625 + def ninja_generate_deps(env): + return sorted([env.File("#SConstruct").path] + glob("**/SConscript", recursive=True)) + env['_NINJA_REGENERATE_DEPS_FUNC'] = ninja_generate_deps + + env['NINJA_REGENERATE_DEPS'] = env.get('NINJA_REGENERATE_DEPS', '${_NINJA_REGENERATE_DEPS_FUNC(__env__)}') + # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. if env["PLATFORM"] == "win32": env.Append(CCFLAGS=["/showIncludes"]) else: - env.Append(CCFLAGS=["-MD", "-MF", "${TARGET}.d"]) + env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) env.AddMethod(CheckNinjaCompdbExpand, "CheckNinjaCompdbExpand") # Provide a way for custom rule authors to easily access command # generation. - env.AddMethod(get_shell_command, "NinjaGetShellCommand") + env.AddMethod(get_generic_shell_command, "NinjaGetGenericShellCommand") + env.AddMethod(get_command, "NinjaGetCommand") env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider") + env.AddMethod(set_build_node_callback, "NinjaSetBuildNodeCallback") # Provides a way for users to handle custom FunctionActions they # want to translate to Ninja. @@ -1587,7 +1729,7 @@ def robust_rule_mapping(var, rule, tool): SCons.Job.Jobs.__init__ = ninja_always_serial ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') - + if NINJA_STATE is None: NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) NINJA_STATE.ninja_bin_path = env.get('NINJA_BIN') @@ -1595,10 +1737,10 @@ def robust_rule_mapping(var, rule, tool): # default to using ninja installed with python module ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', + ninja.__file__, + os.pardir, + 'data', + 'bin', ninja_bin)) if not os.path.exists(NINJA_STATE.ninja_bin_path): # couldn't find it, just give the bin name and hope @@ -1607,7 +1749,7 @@ def robust_rule_mapping(var, rule, tool): # TODO: this is hacking into scons, preferable if there were a less intrusive way # We will subvert the normal builder execute to make sure all the ninja file is dependent - # on all targets generated from any builders + # on all targets generated from any builders SCons_Builder_BuilderBase__execute = SCons.Builder.BuilderBase._execute def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): # this ensures all environments in which a builder executes from will @@ -1666,4 +1808,4 @@ def ninja_execute(self): if not os.path.isdir(os.environ["TMPDIR"]): env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) - env["TEMPFILE"] = NinjaNoResponseFiles \ No newline at end of file + env["TEMPFILE"] = NinjaNoResponseFiles From b6bd8808b5f7970c803a71e4ae9095185a592136 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 31 Dec 2020 16:06:53 +0000 Subject: [PATCH 043/163] Update ninja to new scons layout --- src/engine/SCons/Tool/ninja.py | 1811 -------------------------------- 1 file changed, 1811 deletions(-) delete mode 100644 src/engine/SCons/Tool/ninja.py diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py deleted file mode 100644 index ea01d590ee..0000000000 --- a/src/engine/SCons/Tool/ninja.py +++ /dev/null @@ -1,1811 +0,0 @@ -# Copyright 2020 MongoDB Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -"""Generate build.ninja files from SCons aliases.""" - -import sys -import os -import importlib -import io -import shutil -import shlex -import subprocess -import textwrap - -from glob import glob -from os.path import join as joinpath -from os.path import splitext - -import SCons -from SCons.Action import _string_from_cmd_list, get_default_ENV -from SCons.Util import is_List, flatten_sequence -from SCons.Script import COMMAND_LINE_TARGETS -from SCons.Node import SConscriptNodes - -NINJA_STATE = None -NINJA_RULES = "__NINJA_CUSTOM_RULES" -NINJA_POOLS = "__NINJA_CUSTOM_POOLS" -NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" -NINJA_BUILD = "NINJA_BUILD" -NINJA_WHEREIS_MEMO = {} -NINJA_STAT_MEMO = {} - -__NINJA_RULE_MAPPING = {} - -# These are the types that get_command can do something with -COMMAND_TYPES = ( - SCons.Action.CommandAction, - SCons.Action.CommandGeneratorAction, -) - - -def is_valid_dependent_node(node): - """ - Return True if node is not an alias or is an alias that has children - - This prevents us from making phony targets that depend on other - phony targets that will never have an associated ninja build - target. - - We also have to specify that it's an alias when doing the builder - check because some nodes (like src files) won't have builders but - are valid implicit dependencies. - """ - if isinstance(node, SCons.Node.Alias.Alias): - return node.children() - - if not node.env: - return True - - return not node.env.get("NINJA_SKIP") - - -def alias_to_ninja_build(node): - """Convert an Alias node into a Ninja phony target""" - return { - "outputs": get_outputs(node), - "rule": "phony", - "implicit": [ - get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) - ], - } - - -def check_invalid_ninja_node(node): - return not isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)) - - -def filter_ninja_nodes(node_list): - ninja_nodes = [] - for node in node_list: - if isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)): - ninja_nodes.append(node) - else: - continue - return ninja_nodes - -def get_input_nodes(node): - if node.get_executor() is not None: - inputs = node.get_executor().get_all_sources() - else: - inputs = node.sources - return inputs - - -def invalid_ninja_nodes(node, targets): - result = False - for node_list in [node.prerequisites, get_input_nodes(node), node.children(), targets]: - if node_list: - result = result or any([check_invalid_ninja_node(node) for node in node_list]) - return result - - -def get_order_only(node): - """Return a list of order only dependencies for node.""" - if node.prerequisites is None: - return [] - return [get_path(src_file(prereq)) for prereq in filter_ninja_nodes(node.prerequisites)] - - -def get_dependencies(node, skip_sources=False): - """Return a list of dependencies for node.""" - if skip_sources: - return [ - get_path(src_file(child)) - for child in filter_ninja_nodes(node.children()) - if child not in node.sources - ] - return [get_path(src_file(child)) for child in filter_ninja_nodes(node.children())] - - -def get_inputs(node): - """Collect the Ninja inputs for node.""" - return [get_path(src_file(o)) for o in filter_ninja_nodes(get_input_nodes(node))] - - -def get_outputs(node): - """Collect the Ninja outputs for node.""" - executor = node.get_executor() - if executor is not None: - outputs = executor.get_all_targets() - else: - if hasattr(node, "target_peers"): - outputs = node.target_peers - else: - outputs = [node] - - outputs = [get_path(o) for o in filter_ninja_nodes(outputs)] - - return outputs - -def generate_depfile(env, node, dependencies): - """ - Ninja tool function for writing a depfile. The depfile should include - the node path followed by all the dependent files in a makefile format. - - dependencies arg can be a list or a subst generator which returns a list. - """ - - depfile = os.path.join(get_path(env['NINJA_BUILDDIR']), str(node) + '.depfile') - - # subst_list will take in either a raw list or a subst callable which generates - # a list, and return a list of CmdStringHolders which can be converted into raw strings. - # If a raw list was passed in, then scons_list will make a list of lists from the original - # values and even subst items in the list if they are substitutable. Flatten will flatten - # the list in that case, to ensure for either input we have a list of CmdStringHolders. - deps_list = env.Flatten(env.subst_list(dependencies)) - - # Now that we have the deps in a list as CmdStringHolders, we can convert them into raw strings - # and make sure to escape the strings to handle spaces in paths. We also will sort the result - # keep the order of the list consistent. - escaped_depends = sorted([dep.escape(env.get("ESCAPE", lambda x: x)) for dep in deps_list]) - depfile_contents = str(node) + ": " + ' '.join(escaped_depends) - - need_rewrite = False - try: - with open(depfile, 'r') as f: - need_rewrite = (f.read() != depfile_contents) - except FileNotFoundError: - need_rewrite = True - - if need_rewrite: - os.makedirs(os.path.dirname(depfile) or '.', exist_ok=True) - with open(depfile, 'w') as f: - f.write(depfile_contents) - - -def get_targets_sources(node): - executor = node.get_executor() - if executor is not None: - tlist = executor.get_all_targets() - slist = executor.get_all_sources() - else: - if hasattr(node, "target_peers"): - tlist = node.target_peers - else: - tlist = [node] - slist = node.sources - - # Retrieve the repository file for all sources - slist = [rfile(s) for s in slist] - return tlist, slist - - -def get_rule(node, rule): - tlist, slist = get_targets_sources(node) - if invalid_ninja_nodes(node, tlist): - return "TEMPLATE" - else: - return rule - - -def _install_action_function(_env, node): - """Install files using the install or copy commands""" - return { - "outputs": get_outputs(node), - "rule": get_rule(node, "INSTALL"), - "inputs": get_inputs(node), - "implicit": get_dependencies(node), - } - - -def _mkdir_action_function(env, node): - return { - "outputs": get_outputs(node), - "rule": get_rule(node, "CMD"), - # implicit explicitly omitted, we translate these so they can be - # used by anything that depends on these but commonly this is - # hit with a node that will depend on all of the fake - # srcnode's that SCons will never give us a rule for leading - # to an invalid ninja file. - "variables": { - # On Windows mkdir "-p" is always on - "cmd": "{mkdir}".format( - mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", - ), - }, - } - - -def _copy_action_function(env, node): - return { - "outputs": get_outputs(node), - "inputs": get_inputs(node), - "rule": get_rule(node, "CMD"), - "variables": { - "cmd": "$COPY $in $out", - }, - } - - -def _lib_symlink_action_function(_env, node): - """Create shared object symlinks if any need to be created""" - symlinks = getattr(getattr(node, "attributes", None), "shliblinks", None) - - if not symlinks or symlinks is None: - return None - - outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] - inputs = [link.get_path() for link, _ in symlinks] - - return { - "outputs": outputs, - "inputs": inputs, - "rule": get_rule(node, "SYMLINK"), - "implicit": get_dependencies(node), - } - - -class SConsToNinjaTranslator: - """Translates SCons Actions into Ninja build objects.""" - - def __init__(self, env): - self.env = env - self.func_handlers = { - # Skip conftest builders - "_createSource": ninja_noop, - # SCons has a custom FunctionAction that just makes sure the - # target isn't static. We let the commands that ninja runs do - # this check for us. - "SharedFlagChecker": ninja_noop, - # The install builder is implemented as a function action. - # TODO: use command action #3573 - "installFunc": _install_action_function, - "MkdirFunc": _mkdir_action_function, - "LibSymlinksActionFunction": _lib_symlink_action_function, - "Copy" : _copy_action_function - } - - self.loaded_custom = False - - # pylint: disable=too-many-return-statements - def action_to_ninja_build(self, node, action=None): - """Generate build arguments dictionary for node.""" - if not self.loaded_custom: - self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) - self.loaded_custom = True - - if node.builder is None: - return None - - if action is None: - action = node.builder.action - - if node.env and node.env.get("NINJA_SKIP"): - return None - - build = {} - env = node.env if node.env else self.env - - # Ideally this should never happen, and we do try to filter - # Ninja builders out of being sources of ninja builders but I - # can't fix every DAG problem so we just skip ninja_builders - # if we find one - global NINJA_STATE - if NINJA_STATE.ninja_file == str(node): - build = None - elif isinstance(action, SCons.Action.FunctionAction): - build = self.handle_func_action(node, action) - elif isinstance(action, SCons.Action.LazyAction): - # pylint: disable=protected-access - action = action._generate_cache(env) - build = self.action_to_ninja_build(node, action=action) - elif isinstance(action, SCons.Action.ListAction): - build = self.handle_list_action(node, action) - elif isinstance(action, COMMAND_TYPES): - build = get_command(env, node, action) - else: - raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) - - if build is not None: - build["order_only"] = get_order_only(node) - - if 'conftest' not in str(node): - node_callback = getattr(node.attributes, "ninja_build_callback", None) - if callable(node_callback): - node_callback(env, node, build) - - return build - - def handle_func_action(self, node, action): - """Determine how to handle the function action.""" - name = action.function_name() - # This is the name given by the Subst/Textfile builders. So return the - # node to indicate that SCons is required. We skip sources here because - # dependencies don't really matter when we're going to shove these to - # the bottom of ninja's DAG anyway and Textfile builders can have text - # content as their source which doesn't work as an implicit dep in - # ninja. - if name == 'ninja_builder': - return None - - handler = self.func_handlers.get(name, None) - if handler is not None: - return handler(node.env if node.env else self.env, node) - elif name == "ActionCaller": - action_to_call = str(action).split('(')[0].strip() - handler = self.func_handlers.get(action_to_call, None) - if handler is not None: - return handler(node.env if node.env else self.env, node) - - SCons.Warnings.SConsWarning( - "Found unhandled function action {}, " - " generating scons command to build\n" - "Note: this is less efficient than Ninja," - " you can write your own ninja build generator for" - " this function using NinjaRegisterFunctionHandler".format(name) - ) - - return { - "rule": "TEMPLATE", - "order_only": get_order_only(node), - "outputs": get_outputs(node), - "inputs": get_inputs(node), - "implicit": get_dependencies(node, skip_sources=True), - } - - # pylint: disable=too-many-branches - def handle_list_action(self, node, action): - """TODO write this comment""" - results = [ - self.action_to_ninja_build(node, action=act) - for act in action.list - if act is not None - ] - results = [ - result for result in results if result is not None and result["outputs"] - ] - if not results: - return None - - # No need to process the results if we only got a single result - if len(results) == 1: - return results[0] - - all_outputs = list({output for build in results for output in build["outputs"]}) - dependencies = list({dep for build in results for dep in build["implicit"]}) - - if results[0]["rule"] == "CMD" or results[0]["rule"] == "GENERATED_CMD": - cmdline = "" - for cmd in results: - - # Occasionally a command line will expand to a - # whitespace only string (i.e. ' '). Which is not a - # valid command but does not trigger the empty command - # condition if not cmdstr. So here we strip preceding - # and proceeding whitespace to make strings like the - # above become empty strings and so will be skipped. - cmdstr = cmd["variables"]["cmd"].strip() - if not cmdstr: - continue - - # Skip duplicate commands - if cmdstr in cmdline: - continue - - if cmdline: - cmdline += " && " - - cmdline += cmdstr - - # Remove all preceding and proceeding whitespace - cmdline = cmdline.strip() - - # Make sure we didn't generate an empty cmdline - if cmdline: - ninja_build = { - "outputs": all_outputs, - "rule": get_rule(node, "GENERATED_CMD"), - "variables": { - "cmd": cmdline, - "env": get_command_env(node.env if node.env else self.env), - }, - "implicit": dependencies, - } - - if node.env and node.env.get("NINJA_POOL", None) is not None: - ninja_build["pool"] = node.env["pool"] - - return ninja_build - - elif results[0]["rule"] == "phony": - return { - "outputs": all_outputs, - "rule": "phony", - "implicit": dependencies, - } - - elif results[0]["rule"] == "INSTALL": - return { - "outputs": all_outputs, - "rule": get_rule(node, "INSTALL"), - "inputs": get_inputs(node), - "implicit": dependencies, - } - - raise Exception("Unhandled list action with rule: " + results[0]["rule"]) - - -# pylint: disable=too-many-instance-attributes -class NinjaState: - """Maintains state of Ninja build system as it's translated from SCons.""" - - def __init__(self, env, ninja_file, writer_class): - self.env = env - self.ninja_file = ninja_file - self.ninja_bin_path = '' - self.writer_class = writer_class - self.__generated = False - self.translator = SConsToNinjaTranslator(env) - self.generated_suffixes = env.get("NINJA_GENERATED_SOURCE_SUFFIXES", []) - - # List of generated builds that will be written at a later stage - self.builds = dict() - - # List of targets for which we have generated a build. This - # allows us to take multiple Alias nodes as sources and to not - # fail to build if they have overlapping targets. - self.built = set() - - # SCons sets this variable to a function which knows how to do - # shell quoting on whatever platform it's run on. Here we use it - # to make the SCONS_INVOCATION variable properly quoted for things - # like CCFLAGS - escape = env.get("ESCAPE", lambda x: x) - - # if SCons was invoked from python, we expect the first arg to be the scons.py - # script, otherwise scons was invoked from the scons script - python_bin = '' - if os.path.basename(sys.argv[0]) == 'scons.py': - python_bin = escape(sys.executable) - self.variables = { - "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", - "SCONS_INVOCATION": '{} {} --disable-ninja __NINJA_NO=1 $out'.format( - python_bin, - " ".join( - [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] - ), - ), - "SCONS_INVOCATION_W_TARGETS": "{} {} --disable-ninja".format( - python_bin, " ".join([escape(arg) for arg in sys.argv]) - ), - # This must be set to a global default per: - # https://ninja-build.org/manual.html#_deps - # English Visual Studio will have the default below, - # otherwise the user can define the variable in the first environment - # that initialized ninja tool - "msvc_deps_prefix": env.get("NINJA_MSVC_DEPS_PREFIX", "Note: including file:") - } - - self.rules = { - "CMD": { - "command": "cmd /c $env$cmd $in $out" if sys.platform == "win32" else "$env$cmd $in $out", - "description": "Building $out", - "pool": "local_pool", - }, - "GENERATED_CMD": { - "command": "cmd /c $env$cmd" if sys.platform == "win32" else "$env$cmd", - "description": "Building $out", - "pool": "local_pool", - }, - # We add the deps processing variables to this below. We - # don't pipe these through cmd.exe on Windows because we - # use this to generate a compile_commands.json database - # which can't use the shell command as it's compile - # command. - "CC": { - "command": "$env$CC @$out.rsp", - "description": "Compiling $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - }, - "CXX": { - "command": "$env$CXX @$out.rsp", - "description": "Compiling $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - }, - "LINK": { - "command": "$env$LINK @$out.rsp", - "description": "Linking $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - "pool": "local_pool", - }, - # Ninja does not automatically delete the archive before - # invoking ar. The ar utility will append to an existing archive, which - # can cause duplicate symbols if the symbols moved between object files. - # Native SCons will perform this operation so we need to force ninja - # to do the same. See related for more info: - # https://jira.mongodb.org/browse/SERVER-49457 - "AR": { - "command": "{}$env$AR @$out.rsp".format( - '' if sys.platform == "win32" else "rm -f $out && " - ), - "description": "Archiving $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - "pool": "local_pool", - }, - "SYMLINK": { - "command": ( - "cmd /c mklink $out $in" - if sys.platform == "win32" - else "ln -s $in $out" - ), - "description": "Symlink $in -> $out", - }, - "INSTALL": { - "command": "$COPY $in $out", - "description": "Install $out", - "pool": "install_pool", - # On Windows cmd.exe /c copy does not always correctly - # update the timestamp on the output file. This leads - # to a stuck constant timestamp in the Ninja database - # and needless rebuilds. - # - # Adding restat here ensures that Ninja always checks - # the copy updated the timestamp and that Ninja has - # the correct information. - "restat": 1, - }, - "TEMPLATE": { - "command": "$SCONS_INVOCATION $out", - "description": "Rendering $SCONS_INVOCATION $out", - "pool": "scons_pool", - "restat": 1, - }, - "SCONS": { - "command": "$SCONS_INVOCATION $out", - "description": "$SCONS_INVOCATION $out", - "pool": "scons_pool", - # restat - # if present, causes Ninja to re-stat the command's outputs - # after execution of the command. Each output whose - # modification time the command did not change will be - # treated as though it had never needed to be built. This - # may cause the output's reverse dependencies to be removed - # from the list of pending build actions. - # - # We use restat any time we execute SCons because - # SCons calls in Ninja typically create multiple - # targets. But since SCons is doing it's own up to - # date-ness checks it may only update say one of - # them. Restat will find out which of the multiple - # build targets did actually change then only rebuild - # those targets which depend specifically on that - # output. - "restat": 1, - }, - "REGENERATE": { - "command": "$SCONS_INVOCATION_W_TARGETS", - "description": "Regenerating $out", - "generator": 1, - "depfile": os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile'), - # Console pool restricts to 1 job running at a time, - # it additionally has some special handling about - # passing stdin, stdout, etc to process in this pool - # that we need for SCons to behave correctly when - # regenerating Ninja - "pool": "console", - # Again we restat in case Ninja thought the - # build.ninja should be regenerated but SCons knew - # better. - "restat": 1, - }, - } - - self.pools = { - "local_pool": self.env.GetOption("num_jobs"), - "install_pool": self.env.GetOption("num_jobs") / 2, - "scons_pool": 1, - } - - for rule in ["CC", "CXX"]: - if env["PLATFORM"] == "win32": - self.rules[rule]["deps"] = "msvc" - else: - self.rules[rule]["deps"] = "gcc" - self.rules[rule]["depfile"] = "$out.d" - - def add_build(self, node): - if not node.has_builder(): - return False - - if isinstance(node, SCons.Node.Alias.Alias): - build = alias_to_ninja_build(node) - else: - build = self.translator.action_to_ninja_build(node) - - # Some things are unbuild-able or need not be built in Ninja - if build is None: - return False - - node_string = str(node) - if node_string in self.builds: - raise Exception("Node {} added to ninja build state more than once".format(node_string)) - self.builds[node_string] = build - self.built.update(build["outputs"]) - return True - - # TODO: rely on SCons to tell us what is generated source - # or some form of user scanner maybe (Github Issue #3624) - def is_generated_source(self, output): - """Check if output ends with a known generated suffix.""" - _, suffix = splitext(output) - return suffix in self.generated_suffixes - - def has_generated_sources(self, output): - """ - Determine if output indicates this is a generated header file. - """ - for generated in output: - if self.is_generated_source(generated): - return True - return False - - # pylint: disable=too-many-branches,too-many-locals - def generate(self): - """ - Generate the build.ninja. - - This should only be called once for the lifetime of this object. - """ - if self.__generated: - return - - self.rules.update(self.env.get(NINJA_RULES, {})) - self.pools.update(self.env.get(NINJA_POOLS, {})) - - content = io.StringIO() - ninja = self.writer_class(content, width=100) - - ninja.comment("Generated by scons. DO NOT EDIT.") - - ninja.variable("builddir", get_path(self.env['NINJA_BUILDDIR'])) - - for pool_name, size in self.pools.items(): - ninja.pool(pool_name, size) - - for var, val in self.variables.items(): - ninja.variable(var, val) - - for rule, kwargs in self.rules.items(): - ninja.rule(rule, **kwargs) - - generated_source_files = sorted({ - output - # First find builds which have header files in their outputs. - for build in self.builds.values() - if self.has_generated_sources(build["outputs"]) - for output in build["outputs"] - # Collect only the header files from the builds with them - # in their output. We do this because is_generated_source - # returns True if it finds a header in any of the outputs, - # here we need to filter so we only have the headers and - # not the other outputs. - if self.is_generated_source(output) - }) - - if generated_source_files: - ninja.build( - outputs="_generated_sources", - rule="phony", - implicit=generated_source_files - ) - - template_builders = [] - - for build in [self.builds[key] for key in sorted(self.builds.keys())]: - if build["rule"] == "TEMPLATE": - template_builders.append(build) - continue - - if "implicit" in build: - build["implicit"].sort() - - # Don't make generated sources depend on each other. We - # have to check that none of the outputs are generated - # sources and none of the direct implicit dependencies are - # generated sources or else we will create a dependency - # cycle. - if ( - generated_source_files - and not build["rule"] == "INSTALL" - and set(build["outputs"]).isdisjoint(generated_source_files) - and set(build.get("implicit", [])).isdisjoint(generated_source_files) - ): - - # Make all non-generated source targets depend on - # _generated_sources. We use order_only for generated - # sources so that we don't rebuild the world if one - # generated source was rebuilt. We just need to make - # sure that all of these sources are generated before - # other builds. - order_only = build.get("order_only", []) - order_only.append("_generated_sources") - build["order_only"] = order_only - if "order_only" in build: - build["order_only"].sort() - - # When using a depfile Ninja can only have a single output - # but SCons will usually have emitted an output for every - # thing a command will create because it's caching is much - # more complex than Ninja's. This includes things like DWO - # files. Here we make sure that Ninja only ever sees one - # target when using a depfile. It will still have a command - # that will create all of the outputs but most targets don't - # depend directly on DWO files and so this assumption is safe - # to make. - rule = self.rules.get(build["rule"]) - - # Some rules like 'phony' and other builtins we don't have - # listed in self.rules so verify that we got a result - # before trying to check if it has a deps key. - # - # Anything using deps or rspfile in Ninja can only have a single - # output, but we may have a build which actually produces - # multiple outputs which other targets can depend on. Here we - # slice up the outputs so we have a single output which we will - # use for the "real" builder and multiple phony targets that - # match the file names of the remaining outputs. This way any - # build can depend on any output from any build. - # - # We assume that the first listed output is the 'key' - # output and is stably presented to us by SCons. For - # instance if -gsplit-dwarf is in play and we are - # producing foo.o and foo.dwo, we expect that outputs[0] - # from SCons will be the foo.o file and not the dwo - # file. If instead we just sorted the whole outputs array, - # we would find that the dwo file becomes the - # first_output, and this breaks, for instance, header - # dependency scanning. - if rule is not None and (rule.get("deps") or rule.get("rspfile")): - first_output, remaining_outputs = ( - build["outputs"][0], - build["outputs"][1:], - ) - - if remaining_outputs: - ninja.build( - outputs=sorted(remaining_outputs), rule="phony", implicit=first_output, - ) - - build["outputs"] = first_output - - # Optionally a rule can specify a depfile, and SCons can generate implicit - # dependencies into the depfile. This allows for dependencies to come and go - # without invalidating the ninja file. The depfile was created in ninja specifically - # for dealing with header files appearing and disappearing across rebuilds, but it can - # be repurposed for anything, as long as you have a way to regenerate the depfile. - # More specific info can be found here: https://ninja-build.org/manual.html#_depfile - if rule is not None and rule.get('depfile') and build.get('deps_files'): - path = build['outputs'] if SCons.Util.is_List(build['outputs']) else [build['outputs']] - generate_depfile(self.env, path[0], build.pop('deps_files', [])) - - if "inputs" in build: - build["inputs"].sort() - - ninja.build(**build) - - template_builds = dict() - for template_builder in template_builders: - - # Special handling for outputs and implicit since we need to - # aggregate not replace for each builder. - for agg_key in ["outputs", "implicit", "inputs"]: - new_val = template_builds.get(agg_key, []) - - # Use pop so the key is removed and so the update - # below will not overwrite our aggregated values. - cur_val = template_builder.pop(agg_key, []) - if is_List(cur_val): - new_val += cur_val - else: - new_val.append(cur_val) - template_builds[agg_key] = new_val - - # Collect all other keys - template_builds.update(template_builder) - - if template_builds.get("outputs", []): - ninja.build(**template_builds) - - # We have to glob the SCons files here to teach the ninja file - # how to regenerate itself. We'll never see ourselves in the - # DAG walk so we can't rely on action_to_ninja_build to - # generate this rule even though SCons should know we're - # dependent on SCons files. - # - # The REGENERATE rule uses depfile, so we need to generate the depfile - # in case any of the SConscripts have changed. The depfile needs to be - # path with in the build and the passed ninja file is an abspath, so - # we will use SCons to give us the path within the build. Normally - # generate_depfile should not be called like this, but instead be called - # through the use of custom rules, and filtered out in the normal - # list of build generation about. However, because the generate rule - # is hardcoded here, we need to do this generate_depfile call manually. - ninja_file_path = self.env.File(self.ninja_file).path - generate_depfile( - self.env, - ninja_file_path, - self.env['NINJA_REGENERATE_DEPS'] - ) - - ninja.build( - ninja_file_path, - rule="REGENERATE", - implicit=[__file__], - ) - - # If we ever change the name/s of the rules that include - # compile commands (i.e. something like CC) we will need to - # update this build to reflect that complete list. - ninja.build( - "compile_commands.json", - rule="CMD", - pool="console", - implicit=[str(self.ninja_file)], - variables={ - "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( - self.ninja_bin_path, str(self.ninja_file), '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' - ) - }, - ) - - ninja.build( - "compiledb", rule="phony", implicit=["compile_commands.json"], - ) - - # Look in SCons's list of DEFAULT_TARGETS, find the ones that - # we generated a ninja build rule for. - scons_default_targets = [ - get_path(tgt) - for tgt in SCons.Script.DEFAULT_TARGETS - if get_path(tgt) in self.built - ] - - # If we found an overlap between SCons's list of default - # targets and the targets we created ninja builds for then use - # those as ninja's default as well. - if scons_default_targets: - ninja.default(" ".join(scons_default_targets)) - - with open(str(self.ninja_file), "w") as build_ninja: - build_ninja.write(content.getvalue()) - - self.__generated = True - - -def get_path(node): - """ - Return a fake path if necessary. - - As an example Aliases use this as their target name in Ninja. - """ - if hasattr(node, "get_path"): - return node.get_path() - return str(node) - - -def rfile(node): - """ - Return the repository file for node if it has one. Otherwise return node - """ - if hasattr(node, "rfile"): - return node.rfile() - return node - - -def src_file(node): - """Returns the src code file if it exists.""" - if hasattr(node, "srcnode"): - src = node.srcnode() - if src.stat() is not None: - return src - return get_path(node) - - -def get_comstr(env, action, targets, sources): - """Get the un-substituted string for action.""" - # Despite being having "list" in it's name this member is not - # actually a list. It's the pre-subst'd string of the command. We - # use it to determine if the command we're about to generate needs - # to use a custom Ninja rule. By default this redirects CC, CXX, - # AR, SHLINK, and LINK commands to their respective rules but the - # user can inject custom Ninja rules and tie them to commands by - # using their pre-subst'd string. - if hasattr(action, "process"): - return action.cmd_list - - return action.genstring(targets, sources, env) - - -def get_command_env(env): - """ - Return a string that sets the enrivonment for any environment variables that - differ between the OS environment and the SCons command ENV. - - It will be compatible with the default shell of the operating system. - """ - try: - return env["NINJA_ENV_VAR_CACHE"] - except KeyError: - pass - - # Scan the ENV looking for any keys which do not exist in - # os.environ or differ from it. We assume if it's a new or - # differing key from the process environment then it's - # important to pass down to commands in the Ninja file. - ENV = get_default_ENV(env) - scons_specified_env = { - key: value - for key, value in ENV.items() - if key not in os.environ or os.environ.get(key, None) != value - } - - windows = env["PLATFORM"] == "win32" - command_env = "" - for key, value in scons_specified_env.items(): - # Ensure that the ENV values are all strings: - if is_List(value): - # If the value is a list, then we assume it is a - # path list, because that's a pretty common list-like - # value to stick in an environment variable: - value = flatten_sequence(value) - value = joinpath(map(str, value)) - else: - # If it isn't a string or a list, then we just coerce - # it to a string, which is the proper way to handle - # Dir and File instances and will produce something - # reasonable for just about everything else: - value = str(value) - - if windows: - command_env += "set '{}={}' && ".format(key, value) - else: - # We address here *only* the specific case that a user might have - # an environment variable which somehow gets included and has - # spaces in the value. These are escapes that Ninja handles. This - # doesn't make builds on paths with spaces (Ninja and SCons issues) - # nor expanding response file paths with spaces (Ninja issue) work. - value = value.replace(r' ', r'$ ') - command_env += "export {}='{}';".format(key, value) - - env["NINJA_ENV_VAR_CACHE"] = command_env - return command_env - - -def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom_env={}): - """Generate a response file command provider for rule name.""" - - # If win32 using the environment with a response file command will cause - # ninja to fail to create the response file. Additionally since these rules - # generally are not piping through cmd.exe /c any environment variables will - # make CreateProcess fail to start. - # - # On POSIX we can still set environment variables even for compile - # commands so we do so. - use_command_env = not env["PLATFORM"] == "win32" - if "$" in tool: - tool_is_dynamic = True - - def get_response_file_command(env, node, action, targets, sources, executor=None): - if hasattr(action, "process"): - cmd_list, _, _ = action.process(targets, sources, env, executor=executor) - cmd_list = [str(c).replace("$", "$$") for c in cmd_list[0]] - else: - command = generate_command( - env, node, action, targets, sources, executor=executor - ) - cmd_list = shlex.split(command) - - if tool_is_dynamic: - tool_command = env.subst( - tool, target=targets, source=sources, executor=executor - ) - else: - tool_command = tool - - try: - # Add 1 so we always keep the actual tool inside of cmd - tool_idx = cmd_list.index(tool_command) + 1 - except ValueError: - raise Exception( - "Could not find tool {} in {} generated from {}".format( - tool, cmd_list, get_comstr(env, action, targets, sources) - ) - ) - - cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] - rsp_content = ['"' + rsp_content_item + '"' for rsp_content_item in rsp_content] - rsp_content = " ".join(rsp_content) - - variables = {"rspc": rsp_content} - variables[rule] = cmd - if use_command_env: - variables["env"] = get_command_env(env) - - for key, value in custom_env.items(): - variables["env"] += env.subst( - f"export {key}={value};", target=targets, source=sources, executor=executor - ) + " " - return rule, variables, [tool_command] - - return get_response_file_command - - -def generate_command(env, node, action, targets, sources, executor=None): - # Actions like CommandAction have a method called process that is - # used by SCons to generate the cmd_line they need to run. So - # check if it's a thing like CommandAction and call it if we can. - if hasattr(action, "process"): - cmd_list, _, _ = action.process(targets, sources, env, executor=executor) - cmd = _string_from_cmd_list(cmd_list[0]) - else: - # Anything else works with genstring, this is most commonly hit by - # ListActions which essentially call process on all of their - # commands and concatenate it for us. - genstring = action.genstring(targets, sources, env) - if executor is not None: - cmd = env.subst(genstring, executor=executor) - else: - cmd = env.subst(genstring, targets, sources) - - cmd = cmd.replace("\n", " && ").strip() - if cmd.endswith("&&"): - cmd = cmd[0:-2].strip() - - # Escape dollars as necessary - return cmd.replace("$", "$$") - - -def get_generic_shell_command(env, node, action, targets, sources, executor=None): - return ( - "CMD", - { - "cmd": generate_command(env, node, action, targets, sources, executor=None), - "env": get_command_env(env), - }, - # Since this function is a rule mapping provider, it must return a list of dependencies, - # and usually this would be the path to a tool, such as a compiler, used for this rule. - # However this function is to generic to be able to reliably extract such deps - # from the command, so we return a placeholder empty list. It should be noted that - # generally this function will not be used soley and is more like a template to generate - # the basics for a custom provider which may have more specific options for a provier - # function for a custom NinjaRuleMapping. - [] - ) - - -def get_command(env, node, action): # pylint: disable=too-many-branches - """Get the command to execute for node.""" - if node.env: - sub_env = node.env - else: - sub_env = env - executor = node.get_executor() - tlist, slist = get_targets_sources(node) - - # Generate a real CommandAction - if isinstance(action, SCons.Action.CommandGeneratorAction): - # pylint: disable=protected-access - action = action._generate(tlist, slist, sub_env, 1, executor=executor) - - variables = {} - - comstr = get_comstr(sub_env, action, tlist, slist) - if not comstr: - return None - - provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) - rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) - - # Get the dependencies for all targets - implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) - - # Now add in the other dependencies related to the command, - # e.g. the compiler binary. The ninja rule can be user provided so - # we must do some validation to resolve the dependency path for ninja. - for provider_dep in provider_deps: - - provider_dep = sub_env.subst(provider_dep) - if not provider_dep: - continue - - # If the tool is a node, then SCons will resolve the path later, if its not - # a node then we assume it generated from build and make sure it is existing. - if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep): - implicit.append(provider_dep) - continue - - # in some case the tool could be in the local directory and be suppled without the ext - # such as in windows, so append the executable suffix and check. - prog_suffix = sub_env.get('PROGSUFFIX', '') - provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix - if os.path.exists(provider_dep_ext): - implicit.append(provider_dep_ext) - continue - - # Many commands will assume the binary is in the path, so - # we accept this as a possible input from a given command. - - provider_dep_abspath = sub_env.WhereIs(provider_dep) or sub_env.WhereIs(provider_dep, path=os.environ["PATH"]) - if provider_dep_abspath: - implicit.append(provider_dep_abspath) - continue - - # Possibly these could be ignore and the build would still work, however it may not always - # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided - # ninja rule. - raise Exception(f"Could not resolve path for {provider_dep} dependency on node '{node}'") - - ninja_build = { - "order_only": get_order_only(node), - "outputs": get_outputs(node), - "inputs": get_inputs(node), - "implicit": implicit, - "rule": get_rule(node, rule), - "variables": variables, - } - - # Don't use sub_env here because we require that NINJA_POOL be set - # on a per-builder call basis to prevent accidental strange - # behavior like env['NINJA_POOL'] = 'console' and sub_env can be - # the global Environment object if node.env is None. - # Example: - # - # Allowed: - # - # env.Command("ls", NINJA_POOL="ls_pool") - # - # Not allowed and ignored: - # - # env["NINJA_POOL"] = "ls_pool" - # env.Command("ls") - # - if node.env and node.env.get("NINJA_POOL", None) is not None: - ninja_build["pool"] = node.env["NINJA_POOL"] - - return ninja_build - - -def ninja_builder(env, target, source): - """Generate a build.ninja for source.""" - if not isinstance(source, list): - source = [source] - if not isinstance(target, list): - target = [target] - - # We have no COMSTR equivalent so print that we're generating - # here. - print("Generating:", str(target[0])) - - generated_build_ninja = target[0].get_abspath() - NINJA_STATE.generate() - - if env["PLATFORM"] == "win32": - # this is not great, its doesn't consider specific - # node environments, which means on linux the build could - # behave differently, because on linux you can set the environment - # per command in the ninja file. This is only needed if - # running ninja directly from a command line that hasn't - # had the environment setup (vcvarsall.bat) - with open('run_ninja_env.bat', 'w') as f: - for key in env['ENV']: - f.write('set {}={}\n'.format(key, env['ENV'][key])) - f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) - cmd = ['run_ninja_env.bat'] - - else: - cmd = [NINJA_STATE.ninja_bin_path, '-f', generated_build_ninja] - - if not env.get("DISABLE_AUTO_NINJA"): - print("Executing:", str(' '.join(cmd))) - - # execute the ninja build at the end of SCons, trying to - # reproduce the output like a ninja build would - def execute_ninja(): - - proc = subprocess.Popen(cmd, - stderr=sys.stderr, - stdout=subprocess.PIPE, - universal_newlines=True, - env=os.environ if env["PLATFORM"] == "win32" else env['ENV'] - ) - for stdout_line in iter(proc.stdout.readline, ""): - yield stdout_line - proc.stdout.close() - return_code = proc.wait() - if return_code: - raise subprocess.CalledProcessError(return_code, 'ninja') - - erase_previous = False - for output in execute_ninja(): - output = output.strip() - if erase_previous: - sys.stdout.write('\x1b[2K') # erase previous line - sys.stdout.write("\r") - else: - sys.stdout.write(os.linesep) - sys.stdout.write(output) - sys.stdout.flush() - # this will only erase ninjas [#/#] lines - # leaving warnings and other output, seems a bit - # prone to failure with such a simple check - erase_previous = output.startswith('[') - -# pylint: disable=too-few-public-methods -class AlwaysExecAction(SCons.Action.FunctionAction): - """Override FunctionAction.__call__ to always execute.""" - - def __call__(self, *args, **kwargs): - kwargs["execute"] = 1 - return super().__call__(*args, **kwargs) - - -def register_custom_handler(env, name, handler): - """Register a custom handler for SCons function actions.""" - env[NINJA_CUSTOM_HANDLERS][name] = handler - - -def register_custom_rule_mapping(env, pre_subst_string, rule): - """Register a function to call for a given rule.""" - global __NINJA_RULE_MAPPING - __NINJA_RULE_MAPPING[pre_subst_string] = rule - - -def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): - """Allows specification of Ninja rules from inside SCons files.""" - rule_obj = { - "command": command, - "description": description if description else "{} $out".format(rule), - } - - if use_depfile: - rule_obj["depfile"] = os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile') - - if deps is not None: - rule_obj["deps"] = deps - - if pool is not None: - rule_obj["pool"] = pool - - if use_response_file: - rule_obj["rspfile"] = "$out.rsp" - rule_obj["rspfile_content"] = response_file_content - - env[NINJA_RULES][rule] = rule_obj - - -def register_custom_pool(env, pool, size): - """Allows the creation of custom Ninja pools""" - env[NINJA_POOLS][pool] = size - -def set_build_node_callback(env, node, callback): - if 'conftest' not in str(node): - setattr(node.attributes, "ninja_build_callback", callback) - -def ninja_csig(original): - """Return a dummy csig""" - - def wrapper(self): - if isinstance(self, SCons.Node.Node) and self.is_sconscript(): - return original(self) - return "dummy_ninja_csig" - - return wrapper - - -def ninja_contents(original): - """Return a dummy content without doing IO""" - - def wrapper(self): - if isinstance(self, SCons.Node.Node) and self.is_sconscript(): - return original(self) - return bytes("dummy_ninja_contents", encoding="utf-8") - - return wrapper - -def CheckNinjaCompdbExpand(env, context): - """ Configure check testing if ninja's compdb can expand response files""" - - context.Message('Checking if ninja compdb can expand response files... ') - ret, output = context.TryAction( - action='ninja -f $SOURCE -t compdb -x CMD_RSP > $TARGET', - extension='.ninja', - text=textwrap.dedent(""" - rule CMD_RSP - command = $cmd @$out.rsp > fake_output.txt - description = Building $out - rspfile = $out.rsp - rspfile_content = $rspc - build fake_output.txt: CMD_RSP fake_input.txt - cmd = echo - pool = console - rspc = "test" - """)) - result = '@fake_output.txt.rsp' not in output - context.Result(result) - return result - -def ninja_stat(_self, path): - """ - Eternally memoized stat call. - - SCons is very aggressive about clearing out cached values. For our - purposes everything should only ever call stat once since we're - running in a no_exec build the file system state should not - change. For these reasons we patch SCons.Node.FS.LocalFS.stat to - use our eternal memoized dictionary. - """ - global NINJA_STAT_MEMO - - try: - return NINJA_STAT_MEMO[path] - except KeyError: - try: - result = os.stat(path) - except os.error: - result = None - - NINJA_STAT_MEMO[path] = result - return result - - -def ninja_noop(*_args, **_kwargs): - """ - A general purpose no-op function. - - There are many things that happen in SCons that we don't need and - also don't return anything. We use this to disable those functions - instead of creating multiple definitions of the same thing. - """ - return None - - -def ninja_whereis(thing, *_args, **_kwargs): - """Replace env.WhereIs with a much faster version""" - global NINJA_WHEREIS_MEMO - - # Optimize for success, this gets called significantly more often - # when the value is already memoized than when it's not. - try: - return NINJA_WHEREIS_MEMO[thing] - except KeyError: - # We do not honor any env['ENV'] or env[*] variables in the - # generated ninja ile. Ninja passes your raw shell environment - # down to it's subprocess so the only sane option is to do the - # same during generation. At some point, if and when we try to - # upstream this, I'm sure a sticking point will be respecting - # env['ENV'] variables and such but it's actually quite - # complicated. I have a naive version but making it always work - # with shell quoting is nigh impossible. So I've decided to - # cross that bridge when it's absolutely required. - path = shutil.which(thing) - NINJA_WHEREIS_MEMO[thing] = path - return path - - -def ninja_always_serial(self, num, taskmaster): - """Replacement for SCons.Job.Jobs constructor which always uses the Serial Job class.""" - # We still set self.num_jobs to num even though it's a lie. The - # only consumer of this attribute is the Parallel Job class AND - # the Main.py function which instantiates a Jobs class. It checks - # if Jobs.num_jobs is equal to options.num_jobs, so if the user - # provides -j12 but we set self.num_jobs = 1 they get an incorrect - # warning about this version of Python not supporting parallel - # builds. So here we lie so the Main.py will not give a false - # warning to users. - self.num_jobs = num - self.job = SCons.Job.Serial(taskmaster) - -def ninja_hack_linkcom(env): - # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks - # without relying on the SCons hacks that SCons uses by default. - if env["PLATFORM"] == "win32": - from SCons.Tool.mslink import compositeLinkAction - - if env.get("LINKCOM", None) == compositeLinkAction: - env[ - "LINKCOM" - ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' - env[ - "SHLINKCOM" - ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' - - -class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): - """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" - - def __call__(self, target, source, env, for_signature): - return self.cmd - - def _print_cmd_str(*_args, **_kwargs): - """Disable this method""" - pass - - -def exists(env): - """Enable if called.""" - - # This variable disables the tool when storing the SCons command in the - # generated ninja file to ensure that the ninja tool is not loaded when - # SCons should do actual work as a subprocess of a ninja build. The ninja - # tool is very invasive into the internals of SCons and so should never be - # enabled when SCons needs to build a target. - if env.get("__NINJA_NO", "0") == "1": - return False - - try: - import ninja - return ninja.__file__ - except ImportError: - SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") - return False - - -added = None - -def generate(env): - """Generate the NINJA builders.""" - from SCons.Script import AddOption, GetOption - global added - if not added: - added = 1 - - AddOption('--disable-execute-ninja', - dest='disable_execute_ninja', - metavar='BOOL', - action="store_true", - default=False, - help='Disable ninja automatically building after scons') - - AddOption('--disable-ninja', - dest='disable_ninja', - metavar='BOOL', - action="store_true", - default=False, - help='Disable ninja automatically building after scons') - - - try: - import ninja - except ImportError: - SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") - return - - env["DISABLE_AUTO_NINJA"] = GetOption('disable_execute_ninja') - - global NINJA_STATE - - # Add the Ninja builder. - always_exec_ninja_action = AlwaysExecAction(ninja_builder, {}) - ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action) - env.Append(BUILDERS={"Ninja": ninja_builder_obj}) - - env["NINJA_PREFIX"] = env.get("NINJA_PREFIX", "build") - env["NINJA_SUFFIX"] = env.get("NINJA_SUFFIX", "ninja") - env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") - env['NINJA_BUILDDIR'] = env.get("NINJA_BUILDDIR", env.Dir(".ninja").path) - ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") - # here we allow multiple environments to construct rules and builds - # into the same ninja file - if NINJA_STATE is None: - ninja_file = env.Ninja(target=ninja_file_name, source=[]) - env.AlwaysBuild(ninja_file) - env.Alias("$NINJA_ALIAS_NAME", ninja_file) - else: - if str(NINJA_STATE.ninja_file) != ninja_file_name: - SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") - ninja_file = [NINJA_STATE.ninja_file] - - - # TODO: API for getting the SConscripts programmatically - # exists upstream: https://github.com/SCons/scons/issues/3625 - def ninja_generate_deps(env): - return sorted([env.File("#SConstruct").path] + glob("**/SConscript", recursive=True)) - env['_NINJA_REGENERATE_DEPS_FUNC'] = ninja_generate_deps - - env['NINJA_REGENERATE_DEPS'] = env.get('NINJA_REGENERATE_DEPS', '${_NINJA_REGENERATE_DEPS_FUNC(__env__)}') - - # This adds the required flags such that the generated compile - # commands will create depfiles as appropriate in the Ninja file. - if env["PLATFORM"] == "win32": - env.Append(CCFLAGS=["/showIncludes"]) - else: - env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) - - env.AddMethod(CheckNinjaCompdbExpand, "CheckNinjaCompdbExpand") - - # Provide a way for custom rule authors to easily access command - # generation. - env.AddMethod(get_generic_shell_command, "NinjaGetGenericShellCommand") - env.AddMethod(get_command, "NinjaGetCommand") - env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider") - env.AddMethod(set_build_node_callback, "NinjaSetBuildNodeCallback") - - # Provides a way for users to handle custom FunctionActions they - # want to translate to Ninja. - env[NINJA_CUSTOM_HANDLERS] = {} - env.AddMethod(register_custom_handler, "NinjaRegisterFunctionHandler") - - # Provides a mechanism for inject custom Ninja rules which can - # then be mapped using NinjaRuleMapping. - env[NINJA_RULES] = {} - env.AddMethod(register_custom_rule, "NinjaRule") - - # Provides a mechanism for inject custom Ninja pools which can - # be used by providing the NINJA_POOL="name" as an - # OverrideEnvironment variable in a builder call. - env[NINJA_POOLS] = {} - env.AddMethod(register_custom_pool, "NinjaPool") - - # Add the ability to register custom NinjaRuleMappings for Command - # builders. We don't store this dictionary in the env to prevent - # accidental deletion of the CC/XXCOM mappings. You can still - # overwrite them if you really want to but you have to explicit - # about it this way. The reason is that if they were accidentally - # deleted you would get a very subtly incorrect Ninja file and - # might not catch it. - env.AddMethod(register_custom_rule_mapping, "NinjaRuleMapping") - - # on windows we need to change the link action - ninja_hack_linkcom(env) - - # Normally in SCons actions for the Program and *Library builders - # will return "${*COM}" as their pre-subst'd command line. However - # if a user in a SConscript overwrites those values via key access - # like env["LINKCOM"] = "$( $ICERUN $)" + env["LINKCOM"] then - # those actions no longer return the "bracketted" string and - # instead return something that looks more expanded. So to - # continue working even if a user has done this we map both the - # "bracketted" and semi-expanded versions. - def robust_rule_mapping(var, rule, tool): - provider = gen_get_response_file_command(env, rule, tool) - env.NinjaRuleMapping("${" + var + "}", provider) - env.NinjaRuleMapping(env.get(var, None), provider) - - robust_rule_mapping("CCCOM", "CC", "$CC") - robust_rule_mapping("SHCCCOM", "CC", "$CC") - robust_rule_mapping("CXXCOM", "CXX", "$CXX") - robust_rule_mapping("SHCXXCOM", "CXX", "$CXX") - robust_rule_mapping("LINKCOM", "LINK", "$LINK") - robust_rule_mapping("SHLINKCOM", "LINK", "$SHLINK") - robust_rule_mapping("ARCOM", "AR", "$AR") - - # Make SCons node walk faster by preventing unnecessary work - env.Decider("timestamp-match") - - # Used to determine if a build generates a source file. Ninja - # requires that all generated sources are added as order_only - # dependencies to any builds that *might* use them. - # TODO: switch to using SCons to help determine this (Github Issue #3624) - env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] - - if env["PLATFORM"] != "win32" and env.get("RANLIBCOM"): - # There is no way to translate the ranlib list action into - # Ninja so add the s flag and disable ranlib. - # - # This is equivalent to Meson. - # https://github.com/mesonbuild/meson/blob/master/mesonbuild/linkers.py#L143 - old_arflags = str(env["ARFLAGS"]) - if "s" not in old_arflags: - old_arflags += "s" - - env["ARFLAGS"] = SCons.Util.CLVar([old_arflags]) - - # Disable running ranlib, since we added 's' above - env["RANLIBCOM"] = "" - - if GetOption('disable_ninja'): - return env - - SCons.Warnings.SConsWarning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") - - # This is the point of no return, anything after this comment - # makes changes to SCons that are irreversible and incompatible - # with a normal SCons build. We return early if __NINJA_NO=1 has - # been given on the command line (i.e. by us in the generated - # ninja file) here to prevent these modifications from happening - # when we want SCons to do work. Everything before this was - # necessary to setup the builder and other functions so that the - # tool can be unconditionally used in the users's SCons files. - - if not exists(env): - return - - # Set a known variable that other tools can query so they can - # behave correctly during ninja generation. - env["GENERATING_NINJA"] = True - - # These methods are no-op'd because they do not work during ninja - # generation, expected to do no work, or simply fail. All of which - # are slow in SCons. So we overwrite them with no logic. - SCons.Node.FS.File.make_ready = ninja_noop - SCons.Node.FS.File.prepare = ninja_noop - SCons.Node.FS.File.push_to_cache = ninja_noop - SCons.Executor.Executor.prepare = ninja_noop - SCons.Taskmaster.Task.prepare = ninja_noop - SCons.Node.FS.File.built = ninja_noop - SCons.Node.Node.visited = ninja_noop - - # We make lstat a no-op because it is only used for SONAME - # symlinks which we're not producing. - SCons.Node.FS.LocalFS.lstat = ninja_noop - - # This is a slow method that isn't memoized. We make it a noop - # since during our generation we will never use the results of - # this or change the results. - SCons.Node.FS.is_up_to_date = ninja_noop - - # We overwrite stat and WhereIs with eternally memoized - # implementations. See the docstring of ninja_stat and - # ninja_whereis for detailed explanations. - SCons.Node.FS.LocalFS.stat = ninja_stat - SCons.Util.WhereIs = ninja_whereis - - # Monkey patch get_csig and get_contents for some classes. It - # slows down the build significantly and we don't need contents or - # content signatures calculated when generating a ninja file since - # we're not doing any SCons caching or building. - SCons.Executor.Executor.get_contents = ninja_contents( - SCons.Executor.Executor.get_contents - ) - SCons.Node.Alias.Alias.get_contents = ninja_contents( - SCons.Node.Alias.Alias.get_contents - ) - SCons.Node.FS.File.get_contents = ninja_contents(SCons.Node.FS.File.get_contents) - SCons.Node.FS.File.get_csig = ninja_csig(SCons.Node.FS.File.get_csig) - SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) - SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig) - - # Ignore CHANGED_SOURCES and CHANGED_TARGETS. We don't want those - # to have effect in a generation pass because the generator - # shouldn't generate differently depending on the current local - # state. Without this, when generating on Windows, if you already - # had a foo.obj, you would omit foo.cpp from the response file. Do the same for UNCHANGED. - SCons.Executor.Executor._get_changed_sources = SCons.Executor.Executor._get_sources - SCons.Executor.Executor._get_changed_targets = SCons.Executor.Executor._get_targets - SCons.Executor.Executor._get_unchanged_sources = SCons.Executor.Executor._get_sources - SCons.Executor.Executor._get_unchanged_targets = SCons.Executor.Executor._get_targets - - # Replace false action messages with nothing. - env["PRINT_CMD_LINE_FUNC"] = ninja_noop - - # This reduces unnecessary subst_list calls to add the compiler to - # the implicit dependencies of targets. Since we encode full paths - # in our generated commands we do not need these slow subst calls - # as executing the command will fail if the file is not found - # where we expect it. - env["IMPLICIT_COMMAND_DEPENDENCIES"] = False - - # This makes SCons more aggressively cache MD5 signatures in the - # SConsign file. - env.SetOption("max_drift", 1) - - # The Serial job class is SIGNIFICANTLY (almost twice as) faster - # than the Parallel job class for generating Ninja files. So we - # monkey the Jobs constructor to only use the Serial Job class. - SCons.Job.Jobs.__init__ = ninja_always_serial - - ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') - - if NINJA_STATE is None: - NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) - NINJA_STATE.ninja_bin_path = env.get('NINJA_BIN') - if not NINJA_STATE.ninja_bin_path: - # default to using ninja installed with python module - ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' - NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - ninja_bin)) - if not os.path.exists(NINJA_STATE.ninja_bin_path): - # couldn't find it, just give the bin name and hope - # its in the path later - NINJA_STATE.ninja_bin_path = ninja_bin - - # TODO: this is hacking into scons, preferable if there were a less intrusive way - # We will subvert the normal builder execute to make sure all the ninja file is dependent - # on all targets generated from any builders - SCons_Builder_BuilderBase__execute = SCons.Builder.BuilderBase._execute - def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): - # this ensures all environments in which a builder executes from will - # not create list actions for linking on windows - ninja_hack_linkcom(env) - targets = SCons_Builder_BuilderBase__execute(self, env, target, source, overwarn=overwarn, executor_kw=executor_kw) - - if not SCons.Util.is_List(target): - target = [target] - - for target in targets: - if str(target) != ninja_file_name and "conftest" not in str(target): - env.Depends(ninja_file, targets) - return targets - SCons.Builder.BuilderBase._execute = NinjaBuilderExecute - - # Here we monkey patch the Task.execute method to not do a bunch of - # unnecessary work. If a build is a regular builder (i.e not a conftest and - # not our own Ninja builder) then we add it to the NINJA_STATE. Otherwise we - # build it like normal. This skips all of the caching work that this method - # would normally do since we aren't pulling any of these targets from the - # cache. - # - # In the future we may be able to use this to actually cache the build.ninja - # file once we have the upstream support for referencing SConscripts as File - # nodes. - def ninja_execute(self): - global NINJA_STATE - - target = self.targets[0] - target_name = str(target) - if target_name != ninja_file_name and "conftest" not in target_name: - NINJA_STATE.add_build(target) - else: - target.build() - - SCons.Taskmaster.Task.execute = ninja_execute - - # Make needs_execute always return true instead of determining out of - # date-ness. - SCons.Script.Main.BuildTask.needs_execute = lambda x: True - - # We will eventually need to overwrite TempFileMunge to make it - # handle persistent tempfiles or get an upstreamed change to add - # some configurability to it's behavior in regards to tempfiles. - # - # Set all three environment variables that Python's - # tempfile.mkstemp looks at as it behaves differently on different - # platforms and versions of Python. - build_dir = env.subst("$BUILD_DIR") - if build_dir == "": - build_dir = "." - os.environ["TMPDIR"] = env.Dir("{}/.response_files".format(build_dir)).get_abspath() - os.environ["TEMP"] = os.environ["TMPDIR"] - os.environ["TMP"] = os.environ["TMPDIR"] - if not os.path.isdir(os.environ["TMPDIR"]): - env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) - - env["TEMPFILE"] = NinjaNoResponseFiles From c8875994db59461e5dde8e2ae96352238840a766 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 27 Jan 2021 15:28:05 -0800 Subject: [PATCH 044/163] Fix int function not returning int value --- test/ninja/ninja-fixture/test_impl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/test/ninja/ninja-fixture/test_impl.c b/test/ninja/ninja-fixture/test_impl.c index 89c26ede6f..ac3fd88656 100644 --- a/test/ninja/ninja-fixture/test_impl.c +++ b/test/ninja/ninja-fixture/test_impl.c @@ -16,4 +16,5 @@ DLLEXPORT int library_function(void) { printf("library_function"); + return 0; } From 3fc1e82daaaa43bfd844a0bbde2ba305f0a7f6a2 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 27 Jan 2021 15:29:09 -0800 Subject: [PATCH 045/163] Refactor ninja -> module, change NINJA_BIN -> NINJA --- SCons/Tool/{ninja.py => ninja/__init__.py} | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) rename SCons/Tool/{ninja.py => ninja/__init__.py} (99%) diff --git a/SCons/Tool/ninja.py b/SCons/Tool/ninja/__init__.py similarity index 99% rename from SCons/Tool/ninja.py rename to SCons/Tool/ninja/__init__.py index ea01d590ee..213b12ac8f 100644 --- a/SCons/Tool/ninja.py +++ b/SCons/Tool/ninja/__init__.py @@ -339,6 +339,7 @@ def action_to_ninja_build(self, node, action=None): if build is not None: build["order_only"] = get_order_only(node) + #TODO: WPD Is this testing the filename to verify it's a configure context generated file? if 'conftest' not in str(node): node_callback = getattr(node.attributes, "ninja_build_callback", None) if callable(node_callback): @@ -1477,8 +1478,8 @@ def exists(env): return False try: - import ninja - return ninja.__file__ + import __init__ + return __init__.__file__ except ImportError: SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return False @@ -1509,7 +1510,7 @@ def generate(env): try: - import ninja + import __init__ except ImportError: SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return @@ -1721,6 +1722,7 @@ def robust_rule_mapping(var, rule, tool): # This makes SCons more aggressively cache MD5 signatures in the # SConsign file. + # TODO: WPD shouldn't this be set to 0? env.SetOption("max_drift", 1) # The Serial job class is SIGNIFICANTLY (almost twice as) faster @@ -1732,12 +1734,12 @@ def robust_rule_mapping(var, rule, tool): if NINJA_STATE is None: NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) - NINJA_STATE.ninja_bin_path = env.get('NINJA_BIN') + NINJA_STATE.ninja_bin_path = env.get('NINJA') if not NINJA_STATE.ninja_bin_path: # default to using ninja installed with python module ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( - ninja.__file__, + __init__.__file__, os.pardir, 'data', 'bin', @@ -1801,7 +1803,7 @@ def ninja_execute(self): # platforms and versions of Python. build_dir = env.subst("$BUILD_DIR") if build_dir == "": - build_dir = "." + build_dir = ".." os.environ["TMPDIR"] = env.Dir("{}/.response_files".format(build_dir)).get_abspath() os.environ["TEMP"] = os.environ["TMPDIR"] os.environ["TMP"] = os.environ["TMPDIR"] From 9edbe3bdd615769b7708ff617a75d0bc54aed6d5 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 27 Jan 2021 15:29:31 -0800 Subject: [PATCH 046/163] refactor test a bit, explicitly use ninja found by test framework --- test/ninja/build_libraries.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 40404152fa..a1c3282475 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -33,7 +33,13 @@ try: import ninja except ImportError: - test.skip_test("Could not find module in python") + test.skip_test("Could not find ninja module in python") + + +ninja_binary = test.where_is('ninja') +if not ninja_binary: + test.skip_test("Could not find ninja. Skipping test.") + _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -47,15 +53,21 @@ test.dir_fixture('ninja-fixture') -lib_suffix = '.lib' if IS_WINDOWS else '.so' -staticlib_suffix = '.lib' if IS_WINDOWS else '.a' -lib_prefix = '' if IS_WINDOWS else 'lib' - -win32 = ", 'WIN32'" if IS_WINDOWS else '' +if IS_WINDOWS: + lib_suffix = '.lib' + staticlib_suffix = '.lib' + lib_prefix = '' + win32 = ", 'WIN32'" +else: + lib_suffix = '.so' + staticlib_suffix = '.a' + lib_prefix = 'lib' + win32 = '' test.write('SConstruct', """ env = Environment() env.Tool('ninja') +env['NINJA'] = "%(ninja_bin)s" shared_lib = env.SharedLibrary(target = 'test_impl', source = 'test_impl.c', CPPDEFINES=['LIBRARY_BUILD'%(win32)s]) env.Program(target = 'test', source = 'test1.c', LIBS=['test_impl'], LIBPATH=['.'], RPATH='.') From 8ca8f7185468bc41e95943e1a53d7c3c99207d69 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 12 Feb 2021 11:27:57 -0800 Subject: [PATCH 047/163] remove f-string to retain py3.5 compatibility --- SCons/Tool/ninja/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 213b12ac8f..fe5e084dd1 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -1068,7 +1068,7 @@ def get_response_file_command(env, node, action, targets, sources, executor=None for key, value in custom_env.items(): variables["env"] += env.subst( - f"export {key}={value};", target=targets, source=sources, executor=executor + "export %s=%s;"%(key, value), target=targets, source=sources, executor=executor ) + " " return rule, variables, [tool_command] @@ -1178,7 +1178,7 @@ def get_command(env, node, action): # pylint: disable=too-many-branches # Possibly these could be ignore and the build would still work, however it may not always # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided # ninja rule. - raise Exception(f"Could not resolve path for {provider_dep} dependency on node '{node}'") + raise Exception("Could not resolve path for %s dependency on node '%s'"%(provider_dep, node)) ninja_build = { "order_only": get_order_only(node), From 785438ed527613f09f9275a16cc369e05a28e782 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 27 Jan 2021 21:21:27 -0800 Subject: [PATCH 048/163] can't name module ninja as it will conflict with the non-SCons ninja module --- SCons/Tool/ninja.py | 26 +++++++++++++++++++ SCons/Tool/{ninja => ninjaCommon}/__init__.py | 12 +++++---- 2 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 SCons/Tool/ninja.py rename SCons/Tool/{ninja => ninjaCommon}/__init__.py (99%) diff --git a/SCons/Tool/ninja.py b/SCons/Tool/ninja.py new file mode 100644 index 0000000000..e597971d2d --- /dev/null +++ b/SCons/Tool/ninja.py @@ -0,0 +1,26 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + + +from .ninjaCommon import generate, exists diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninjaCommon/__init__.py similarity index 99% rename from SCons/Tool/ninja/__init__.py rename to SCons/Tool/ninjaCommon/__init__.py index fe5e084dd1..9b330d3387 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -1,3 +1,5 @@ +# MIT License +# # Copyright 2020 MongoDB Inc. # # Permission is hereby granted, free of charge, to any person obtaining @@ -1478,8 +1480,8 @@ def exists(env): return False try: - import __init__ - return __init__.__file__ + import ninja + return ninja.__file__ except ImportError: SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return False @@ -1510,7 +1512,7 @@ def generate(env): try: - import __init__ + import ninja except ImportError: SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return @@ -1739,7 +1741,7 @@ def robust_rule_mapping(var, rule, tool): # default to using ninja installed with python module ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( - __init__.__file__, + ninja.__file__, os.pardir, 'data', 'bin', @@ -1803,7 +1805,7 @@ def ninja_execute(self): # platforms and versions of Python. build_dir = env.subst("$BUILD_DIR") if build_dir == "": - build_dir = ".." + build_dir = "." os.environ["TMPDIR"] = env.Dir("{}/.response_files".format(build_dir)).get_abspath() os.environ["TEMP"] = os.environ["TMPDIR"] os.environ["TMP"] = os.environ["TMPDIR"] From cd738f88e2cef1aac653226bce6860733b8fbb44 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 12 Feb 2021 13:59:09 -0800 Subject: [PATCH 049/163] refactor initiazation state variable to be more module specific --- SCons/Tool/ninjaCommon/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index 9b330d3387..090d3d8d1b 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -1487,14 +1487,15 @@ def exists(env): return False -added = None +ninja_builder_initialized = False + def generate(env): """Generate the NINJA builders.""" from SCons.Script import AddOption, GetOption - global added - if not added: - added = 1 + global ninja_builder_initialized + if not ninja_builder_initialized: + ninja_builder_initialized = 1 AddOption('--disable-execute-ninja', dest='disable_execute_ninja', From 046e37f25cd64f30e0fea7ac12bb59e89e390d31 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 12 Feb 2021 14:28:53 -0800 Subject: [PATCH 050/163] Fix ninja tool rules for macos/ar for static libs to skip response files for now. Also fix build_libraries to have proper shlib suffix --- SCons/Tool/ninjaCommon/__init__.py | 7 +++++++ test/ninja/build_libraries.py | 5 ++++- testing/framework/TestCmd.py | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index 090d3d8d1b..d46d692669 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -637,6 +637,13 @@ def __init__(self, env, ninja_file, writer_class): }, } + if env['PLATFORM'] == 'darwin' and env['AR'] == 'ar': + self.rules["AR"] = { + "command": "rm -f $out && $env$AR $rspc", + "description": "Archiving $out", + "pool": "local_pool", + } + self.pools = { "local_pool": self.env.GetOption("num_jobs"), "install_pool": self.env.GetOption("num_jobs") / 2, diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index a1c3282475..b3c7b54611 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -26,7 +26,7 @@ import os import TestSCons -from TestCmd import IS_WINDOWS +from TestCmd import IS_WINDOWS, IS_MACOS test = TestSCons.TestSCons() @@ -64,6 +64,9 @@ lib_prefix = 'lib' win32 = '' +if IS_MACOS: + lib_suffix = '.dylib' + test.write('SConstruct', """ env = Environment() env.Tool('ninja') diff --git a/testing/framework/TestCmd.py b/testing/framework/TestCmd.py index 333901c6a9..903bee693a 100644 --- a/testing/framework/TestCmd.py +++ b/testing/framework/TestCmd.py @@ -314,6 +314,7 @@ IS_WINDOWS = sys.platform == 'win32' +IS_MACOS = sys.platform == 'darwin' IS_64_BIT = sys.maxsize > 2**32 IS_PYPY = hasattr(sys, 'pypy_translation_info') From b805e2db88b75bd902eff3cc1ba89e2faa78a1a9 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 16 Feb 2021 11:51:47 -0800 Subject: [PATCH 051/163] [ci skip] Initial doc structure added --- SCons/Tool/ninjaCommon/__init__.py | 2 +- SCons/Tool/ninjaCommon/ninja.xml | 203 +++++++++++++++++++++++++++++ doc/user/external.xml | 56 +++++--- 3 files changed, 240 insertions(+), 21 deletions(-) create mode 100644 SCons/Tool/ninjaCommon/ninja.xml diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index d46d692669..b1bbb68779 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -972,7 +972,7 @@ def get_comstr(env, action, targets, sources): def get_command_env(env): """ - Return a string that sets the enrivonment for any environment variables that + Return a string that sets the environment for any environment variables that differ between the OS environment and the SCons command ENV. It will be compatible with the default shell of the operating system. diff --git a/SCons/Tool/ninjaCommon/ninja.xml b/SCons/Tool/ninjaCommon/ninja.xml new file mode 100644 index 0000000000..c7f693f350 --- /dev/null +++ b/SCons/Tool/ninjaCommon/ninja.xml @@ -0,0 +1,203 @@ + + + + + %scons; + + %builders-mod; + + %functions-mod; + + %tools-mod; + + %variables-mod; + ]> + + + + + + + Generate ninja build files and run ninja to build the bulk of your targets. + + + + NINJA_GENERATED_SOURCE_SUFFIXES + NINJA_MSVC_DEPS_PREFIX + NINJA_BUILDDIR + NINJA_REGENERATE_DEPS + NINJA_COMPDB_EXPAND + NINJA_ENV_VAR_CACHE + NINJA_POOL + DISABLE_AUTO_NINJA + __NINJA_NO + + NINJA_PREFIX + NINJA_SUFFIX + NINJA_ALIAS_NAME + + _NINJA_REGENERATE_DEPS_FUNC + + + + CCCOM + CXXCOM + SHCXXCOM + CC + CXX + + LINKCOM + LINK + + SHLINKCOM + SHLINK + + ARCOM + AR + RANLIB + ARFLAGS + RANLIBCOM + PLATFORM + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + + + + The C compiler. + + + + + + + + + The C compiler. + + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + diff --git a/doc/user/external.xml b/doc/user/external.xml index 9900e93688..8c466dcdb2 100644 --- a/doc/user/external.xml +++ b/doc/user/external.xml @@ -22,26 +22,28 @@ @@ -283,4 +285,18 @@ env2.CompilationDatabase('compile_commands-linux64.json') + +
+ Ninja Build Generator + + + Ninja Build System + + + + + Ninja File Format Specification + + +
From 8cefb500e1e53efc9bc8d33afe50536b6441c2d9 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 16 Feb 2021 15:04:42 -0800 Subject: [PATCH 052/163] Incremental checkin. Fleshing out the various env vars used by ninja builders. Also annotated code with some TODO: questions --- SCons/Tool/ninjaCommon/__init__.py | 8 ++++ SCons/Tool/ninjaCommon/ninja.xml | 72 ++++++++++++++++++++++++------ doc/user/external.xml | 2 +- 3 files changed, 68 insertions(+), 14 deletions(-) diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index b1bbb68779..288e3eaaad 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -897,6 +897,10 @@ def generate(self): implicit=[str(self.ninja_file)], variables={ "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( + # NINJA_COMPDB_EXPAND - should only be true for ninja + # This was added to ninja's compdb tool in version 1.9.0 (merged April 2018) + # https://github.com/ninja-build/ninja/pull/1223 + # TODO: add check in generate to check version and enable this by default if it's available. self.ninja_bin_path, str(self.ninja_file), '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' ) }, @@ -990,6 +994,7 @@ def get_command_env(env): scons_specified_env = { key: value for key, value in ENV.items() + # TODO: Remove this filter, unless there's a good reason to keep. SCons's behavior shouldn't depend on shell's. if key not in os.environ or os.environ.get(key, None) != value } @@ -1213,6 +1218,7 @@ def get_command(env, node, action): # pylint: disable=too-many-branches # env["NINJA_POOL"] = "ls_pool" # env.Command("ls") # + # TODO: Why not alloe env['NINJA_POOL'] ? (bdbaddog) if node.env and node.env.get("NINJA_POOL", None) is not None: ninja_build["pool"] = node.env["NINJA_POOL"] @@ -1534,8 +1540,10 @@ def generate(env): ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action) env.Append(BUILDERS={"Ninja": ninja_builder_obj}) + # TODO: Can't we simplify this to just be NINJA_FILENAME ? (bdbaddog) env["NINJA_PREFIX"] = env.get("NINJA_PREFIX", "build") env["NINJA_SUFFIX"] = env.get("NINJA_SUFFIX", "ninja") + env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") env['NINJA_BUILDDIR'] = env.get("NINJA_BUILDDIR", env.Dir(".ninja").path) ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") diff --git a/SCons/Tool/ninjaCommon/ninja.xml b/SCons/Tool/ninjaCommon/ninja.xml index c7f693f350..9eb7febef3 100644 --- a/SCons/Tool/ninjaCommon/ninja.xml +++ b/SCons/Tool/ninjaCommon/ninja.xml @@ -65,6 +65,7 @@ See its __doc__ string for a discussion of the format. NINJA_PREFIX NINJA_SUFFIX NINJA_ALIAS_NAME + NINJA_SYNTAX _NINJA_REGENERATE_DEPS_FUNC @@ -88,6 +89,7 @@ See its __doc__ string for a discussion of the format. ARFLAGS RANLIBCOM PLATFORM + ESCAPE @@ -95,7 +97,11 @@ See its __doc__ string for a discussion of the format. - The C compiler. + The list of source file suffixes which are generated by SCons build steps. + All source files which match these suffixes will be added to the _generated_sources alias in the output + ninja.build file. + Then all other source files will be made to depend on this in the Ninja.build file, forcing the + generated sources to be built first. @@ -103,7 +109,9 @@ See its __doc__ string for a discussion of the format. - The C compiler. + This propagates directly into the generated Ninja.build file. + From Ninja's docs + defines the string which should be stripped from msvc’s /showIncludes output @@ -111,7 +119,13 @@ See its __doc__ string for a discussion of the format. - The C compiler. + This propagates directly into the generated Ninja.build file. + From Ninja's docs: +
+ builddir + A directory for some Ninja output files. ... (You can also store other build output in this + directory.) +
@@ -119,7 +133,9 @@ See its __doc__ string for a discussion of the format. - The C compiler. + A generator function used to create a ninja depsfile which includes all the files which would require + SCons to be invoked if they change. + Or a list of said files. @@ -127,7 +143,13 @@ See its __doc__ string for a discussion of the format. - The C compiler. + Boolean value (True|False) to instruct ninja to expand the command line arguments normally put into + response files. + This prevents lines in the compilation database like gcc @rsp_file and instead yields + gcc -c -o myfile.o myfile.c -Ia -DXYZ + + + Ninja's compdb tool added the -x flag in Ninja V1.9.0 @@ -135,7 +157,14 @@ See its __doc__ string for a discussion of the format. - The C compiler. + A string that sets the environment for any environment variables that + differ between the OS environment and the SCons command ENV. + + It will be compatible with the default shell of the operating system. + + If not explicitly specified, SCons will generate this dynamically from the Environment()'s 'ENV' + env['ENV'] + where those values differ from the existing shell.. @@ -143,7 +172,7 @@ See its __doc__ string for a discussion of the format. - The C compiler. + Set the ninja_pool for this or all targets in scope for this env var. @@ -151,7 +180,9 @@ See its __doc__ string for a discussion of the format. - The C compiler. + Boolean (True|False). Default: False + When True, SCons will not run ninja automatically after creating the ninja.build file. + @@ -159,7 +190,8 @@ See its __doc__ string for a discussion of the format. - The C compiler. + Internal flag. Used to tell SCons whether or not to try to import pypi's ninja python package. + This is set to True when being called by Ninja? @@ -168,7 +200,8 @@ See its __doc__ string for a discussion of the format. - The C compiler. + The basename for the ninja.build file (so in that case just + ninja @@ -177,7 +210,8 @@ See its __doc__ string for a discussion of the format. - The C compiler. + The suffix for the ninja.build file (so in that case just + build @@ -186,7 +220,18 @@ See its __doc__ string for a discussion of the format. - The C compiler. + Name of the Alias() which is will cause SCons to create the ninja.build file, and + then (optionally) run ninja. + + + + + + + + Theres also NINJA_SYNTAX which is the path to a custom ninja_syntax.py file which is used in generation. + The tool currently assumes you have ninja installed through pip, and grabs the syntax file from that + installation if none specified. @@ -194,7 +239,8 @@ See its __doc__ string for a discussion of the format. - The C compiler. + Internal value used to specify the function to call with argument env to generate the list of files + which if changed would require the ninja file to be regenerated. diff --git a/doc/user/external.xml b/doc/user/external.xml index 8c466dcdb2..5d24845c51 100644 --- a/doc/user/external.xml +++ b/doc/user/external.xml @@ -292,7 +292,7 @@ env2.CompilationDatabase('compile_commands-linux64.json') Ninja Build System - + Ninja File Format Specification From 49590e2acc34d1d599d8aa9cc21bdc516b4cef21 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 18 Feb 2021 10:47:01 -0800 Subject: [PATCH 053/163] Fix path to SCons/Docbooks style files. Add some content. Still a WIP --- SCons/Tool/ninjaCommon/ninja.xml | 86 ++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 22 deletions(-) diff --git a/SCons/Tool/ninjaCommon/ninja.xml b/SCons/Tool/ninjaCommon/ninja.xml index 9eb7febef3..8f7de362bf 100644 --- a/SCons/Tool/ninjaCommon/ninja.xml +++ b/SCons/Tool/ninjaCommon/ninja.xml @@ -29,15 +29,15 @@ See its __doc__ string for a discussion of the format. --> + %scons; - + %builders-mod; - + %functions-mod; - + %tools-mod; - + %variables-mod; ]> @@ -48,7 +48,7 @@ See its __doc__ string for a discussion of the format. - Generate ninja build files and run ninja to build the bulk of your targets. + Sets up &b-link-Ninja; builder which generates a ninja build file, and then optionally runs ninja. @@ -62,8 +62,8 @@ See its __doc__ string for a discussion of the format. DISABLE_AUTO_NINJA __NINJA_NO - NINJA_PREFIX - NINJA_SUFFIX + NINJA_FILE_NAME + NINJA_ALIAS_NAME NINJA_SYNTAX @@ -94,6 +94,60 @@ See its __doc__ string for a discussion of the format. + + + + &b-Ninja; is a special builder which + adds a target to create a ninja build file. + The builder does not require any source files to be specified, + + + + If called with no arguments, + the builder will default to a target name of + ninja.build. + + + If called with a single positional argument, + &scons; will "deduce" the target name from that source + argument, giving it the same name, and then + ignore the source. + This is the usual way to call the builder if a + non-default target name is wanted. + + + If called with either the + target= + or source= keyword arguments, + the value of the argument is taken as the target name. + If called with both, the + target= + value is used and source= is ignored. + If called with multiple sources, + the source list will be ignored, + since there is no way to deduce what the intent was; + in this case the default target name will be used. + + + + You must load the &t-ninja; tool prior to specifying + any part of your build or some source/output + files will not show up in the compilation database. + + + To use this tool you must install pypi's ninja + package. + This can be done via + pip install ninja + + + + Available since &scons; 4.2. + + + + + @@ -197,26 +251,14 @@ See its __doc__ string for a discussion of the format. - - - - The basename for the ninja.build file (so in that case just - ninja - - - - - - + - The suffix for the ninja.build file (so in that case just - build + The filename for the generated Ninja build file defaults to ninja.build - From b6fc9597f1fc0a407276e1b341bdb975d5a70161 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 14 Mar 2021 17:24:33 -0700 Subject: [PATCH 054/163] [ci skip] fix copyright header --- test/ninja/generate_and_build_cxx.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/ninja/generate_and_build_cxx.py b/test/ninja/generate_and_build_cxx.py index 663282bd92..1d149a9822 100644 --- a/test/ninja/generate_and_build_cxx.py +++ b/test/ninja/generate_and_build_cxx.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -22,8 +22,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - import os import TestSCons from TestCmd import IS_WINDOWS From 5092eef9eea013fb4501539ce7ff5458a309f2f4 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 14 Mar 2021 17:58:31 -0700 Subject: [PATCH 055/163] Add API to see if a node has a given attribute stored in node.attributes --- SCons/Node/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/SCons/Node/__init__.py b/SCons/Node/__init__.py index ad177a7029..4e488f45b9 100644 --- a/SCons/Node/__init__.py +++ b/SCons/Node/__init__.py @@ -951,6 +951,11 @@ def is_conftest(self): return False return True + def check_attributes(self, name): + """ Simple API to check if the node.attributes for name has been set""" + return getattr(getattr(self,"attributes", None), name, None) + + def alter_targets(self): """Return a list of alternate targets for this Node. """ From fb554f61935db54b25533c20fce39fd6eeb3cfb4 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 15 Mar 2021 08:32:33 -0700 Subject: [PATCH 056/163] Incremental update with code reorganization. Currently broken. Some debug code enabled --- SCons/Tool/ninjaCommon/Rules.py | 58 ++++ SCons/Tool/ninjaCommon/Util.py | 173 +++++++++++ SCons/Tool/ninjaCommon/__init__.py | 289 +++--------------- test/ninja/generate_and_build.py | 4 +- test/ninja/generate_and_build_cxx.py | 13 +- .../sconstruct_generate_and_build_cxx.py | 4 + 6 files changed, 293 insertions(+), 248 deletions(-) create mode 100644 SCons/Tool/ninjaCommon/Rules.py create mode 100644 SCons/Tool/ninjaCommon/Util.py create mode 100644 test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py diff --git a/SCons/Tool/ninjaCommon/Rules.py b/SCons/Tool/ninjaCommon/Rules.py new file mode 100644 index 0000000000..99b11e4c8e --- /dev/null +++ b/SCons/Tool/ninjaCommon/Rules.py @@ -0,0 +1,58 @@ +from .Util import get_outputs, get_rule, get_inputs, get_dependencies + + +def _install_action_function(_env, node): + """Install files using the install or copy commands""" + return { + "outputs": get_outputs(node), + "rule": get_rule(node, "INSTALL"), + "inputs": get_inputs(node), + "implicit": get_dependencies(node), + } + + +def _mkdir_action_function(env, node): + return { + "outputs": get_outputs(node), + "rule": get_rule(node, "CMD"), + # implicit explicitly omitted, we translate these so they can be + # used by anything that depends on these but commonly this is + # hit with a node that will depend on all of the fake + # srcnode's that SCons will never give us a rule for leading + # to an invalid ninja file. + "variables": { + # On Windows mkdir "-p" is always on + "cmd": "{mkdir}".format( + mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", + ), + }, + } + + +def _copy_action_function(env, node): + return { + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "rule": get_rule(node, "CMD"), + "variables": { + "cmd": "$COPY $in $out", + }, + } + + +def _lib_symlink_action_function(_env, node): + """Create shared object symlinks if any need to be created""" + symlinks = node.check_attributes("shliblinks") + + if not symlinks or symlinks is None: + return None + + outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] + inputs = [link.get_path() for link, _ in symlinks] + + return { + "outputs": outputs, + "inputs": inputs, + "rule": get_rule(node, "SYMLINK"), + "implicit": get_dependencies(node), + } \ No newline at end of file diff --git a/SCons/Tool/ninjaCommon/Util.py b/SCons/Tool/ninjaCommon/Util.py new file mode 100644 index 0000000000..5f14a96060 --- /dev/null +++ b/SCons/Tool/ninjaCommon/Util.py @@ -0,0 +1,173 @@ +import SCons +from SCons.Script import AddOption + +def ninja_add_command_line_options(): + """ + Add additional command line arguments to SCons specific to the ninja tool + """ + AddOption('--disable-execute-ninja', + dest='disable_execute_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + + AddOption('--disable-ninja', + dest='disable_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + + +def is_valid_dependent_node(node): + """ + Return True if node is not an alias or is an alias that has children + + This prevents us from making phony targets that depend on other + phony targets that will never have an associated ninja build + target. + + We also have to specify that it's an alias when doing the builder + check because some nodes (like src files) won't have builders but + are valid implicit dependencies. + """ + if isinstance(node, SCons.Node.Alias.Alias): + return node.children() + + if not node.env: + return True + + return not node.env.get("NINJA_SKIP") + + +def alias_to_ninja_build(node): + """Convert an Alias node into a Ninja phony target""" + return { + "outputs": get_outputs(node), + "rule": "phony", + "implicit": [ + get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) + ], + } + + +def check_invalid_ninja_node(node): + return not isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)) + + +def filter_ninja_nodes(node_list): + ninja_nodes = [] + for node in node_list: + if isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)): + ninja_nodes.append(node) + else: + continue + return ninja_nodes + + +def get_input_nodes(node): + if node.get_executor() is not None: + inputs = node.get_executor().get_all_sources() + else: + inputs = node.sources + return inputs + + +def invalid_ninja_nodes(node, targets): + result = False + for node_list in [node.prerequisites, get_input_nodes(node), node.children(), targets]: + if node_list: + result = result or any([check_invalid_ninja_node(node) for node in node_list]) + return result + + +def get_order_only(node): + """Return a list of order only dependencies for node.""" + if node.prerequisites is None: + return [] + return [get_path(src_file(prereq)) for prereq in filter_ninja_nodes(node.prerequisites)] + + +def get_dependencies(node, skip_sources=False): + """Return a list of dependencies for node.""" + if skip_sources: + return [ + get_path(src_file(child)) + for child in filter_ninja_nodes(node.children()) + if child not in node.sources + ] + return [get_path(src_file(child)) for child in filter_ninja_nodes(node.children())] + + +def get_inputs(node): + """Collect the Ninja inputs for node.""" + return [get_path(src_file(o)) for o in filter_ninja_nodes(get_input_nodes(node))] + + +def get_outputs(node): + """Collect the Ninja outputs for node.""" + executor = node.get_executor() + if executor is not None: + outputs = executor.get_all_targets() + else: + if hasattr(node, "target_peers"): + outputs = node.target_peers + else: + outputs = [node] + + outputs = [get_path(o) for o in filter_ninja_nodes(outputs)] + + return outputs + + +def get_targets_sources(node): + executor = node.get_executor() + if executor is not None: + tlist = executor.get_all_targets() + slist = executor.get_all_sources() + else: + if hasattr(node, "target_peers"): + tlist = node.target_peers + else: + tlist = [node] + slist = node.sources + + # Retrieve the repository file for all sources + slist = [rfile(s) for s in slist] + return tlist, slist + +def get_path(node): + """ + Return a fake path if necessary. + + As an example Aliases use this as their target name in Ninja. + """ + if hasattr(node, "get_path"): + return node.get_path() + return str(node) + + +def rfile(node): + """ + Return the repository file for node if it has one. Otherwise return node + """ + if hasattr(node, "rfile"): + return node.rfile() + return node + + +def src_file(node): + """Returns the src code file if it exists.""" + if hasattr(node, "srcnode"): + src = node.srcnode() + if src.stat() is not None: + return src + return get_path(node) + +def get_rule(node, rule): + tlist, slist = get_targets_sources(node) + if invalid_ninja_nodes(node, tlist): + return "TEMPLATE" + else: + return rule \ No newline at end of file diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index 288e3eaaad..bb74845a90 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -24,24 +24,30 @@ """Generate build.ninja files from SCons aliases.""" -import sys -import os import importlib import io -import shutil +import os import shlex +import shutil import subprocess +import sys import textwrap - from glob import glob from os.path import join as joinpath from os.path import splitext import SCons from SCons.Action import _string_from_cmd_list, get_default_ENV -from SCons.Util import is_List, flatten_sequence -from SCons.Script import COMMAND_LINE_TARGETS from SCons.Node import SConscriptNodes +from SCons.Script import AddOption, GetOption +from SCons.Script import COMMAND_LINE_TARGETS + +from .Util import ninja_add_command_line_options, is_valid_dependent_node, alias_to_ninja_build, \ + filter_ninja_nodes, get_input_nodes, invalid_ninja_nodes, get_order_only, get_dependencies, get_inputs, get_outputs, \ + get_targets_sources, get_path +from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function + +from SCons.Util import is_List, flatten_sequence NINJA_STATE = None NINJA_RULES = "__NINJA_CUSTOM_RULES" @@ -59,105 +65,8 @@ SCons.Action.CommandGeneratorAction, ) +ninja_builder_initialized = False -def is_valid_dependent_node(node): - """ - Return True if node is not an alias or is an alias that has children - - This prevents us from making phony targets that depend on other - phony targets that will never have an associated ninja build - target. - - We also have to specify that it's an alias when doing the builder - check because some nodes (like src files) won't have builders but - are valid implicit dependencies. - """ - if isinstance(node, SCons.Node.Alias.Alias): - return node.children() - - if not node.env: - return True - - return not node.env.get("NINJA_SKIP") - - -def alias_to_ninja_build(node): - """Convert an Alias node into a Ninja phony target""" - return { - "outputs": get_outputs(node), - "rule": "phony", - "implicit": [ - get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) - ], - } - - -def check_invalid_ninja_node(node): - return not isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)) - - -def filter_ninja_nodes(node_list): - ninja_nodes = [] - for node in node_list: - if isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)): - ninja_nodes.append(node) - else: - continue - return ninja_nodes - -def get_input_nodes(node): - if node.get_executor() is not None: - inputs = node.get_executor().get_all_sources() - else: - inputs = node.sources - return inputs - - -def invalid_ninja_nodes(node, targets): - result = False - for node_list in [node.prerequisites, get_input_nodes(node), node.children(), targets]: - if node_list: - result = result or any([check_invalid_ninja_node(node) for node in node_list]) - return result - - -def get_order_only(node): - """Return a list of order only dependencies for node.""" - if node.prerequisites is None: - return [] - return [get_path(src_file(prereq)) for prereq in filter_ninja_nodes(node.prerequisites)] - - -def get_dependencies(node, skip_sources=False): - """Return a list of dependencies for node.""" - if skip_sources: - return [ - get_path(src_file(child)) - for child in filter_ninja_nodes(node.children()) - if child not in node.sources - ] - return [get_path(src_file(child)) for child in filter_ninja_nodes(node.children())] - - -def get_inputs(node): - """Collect the Ninja inputs for node.""" - return [get_path(src_file(o)) for o in filter_ninja_nodes(get_input_nodes(node))] - - -def get_outputs(node): - """Collect the Ninja outputs for node.""" - executor = node.get_executor() - if executor is not None: - outputs = executor.get_all_targets() - else: - if hasattr(node, "target_peers"): - outputs = node.target_peers - else: - outputs = [node] - - outputs = [get_path(o) for o in filter_ninja_nodes(outputs)] - - return outputs def generate_depfile(env, node, dependencies): """ @@ -195,87 +104,8 @@ def generate_depfile(env, node, dependencies): f.write(depfile_contents) -def get_targets_sources(node): - executor = node.get_executor() - if executor is not None: - tlist = executor.get_all_targets() - slist = executor.get_all_sources() - else: - if hasattr(node, "target_peers"): - tlist = node.target_peers - else: - tlist = [node] - slist = node.sources - - # Retrieve the repository file for all sources - slist = [rfile(s) for s in slist] - return tlist, slist - - -def get_rule(node, rule): - tlist, slist = get_targets_sources(node) - if invalid_ninja_nodes(node, tlist): - return "TEMPLATE" - else: - return rule - - -def _install_action_function(_env, node): - """Install files using the install or copy commands""" - return { - "outputs": get_outputs(node), - "rule": get_rule(node, "INSTALL"), - "inputs": get_inputs(node), - "implicit": get_dependencies(node), - } - - -def _mkdir_action_function(env, node): - return { - "outputs": get_outputs(node), - "rule": get_rule(node, "CMD"), - # implicit explicitly omitted, we translate these so they can be - # used by anything that depends on these but commonly this is - # hit with a node that will depend on all of the fake - # srcnode's that SCons will never give us a rule for leading - # to an invalid ninja file. - "variables": { - # On Windows mkdir "-p" is always on - "cmd": "{mkdir}".format( - mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", - ), - }, - } -def _copy_action_function(env, node): - return { - "outputs": get_outputs(node), - "inputs": get_inputs(node), - "rule": get_rule(node, "CMD"), - "variables": { - "cmd": "$COPY $in $out", - }, - } - - -def _lib_symlink_action_function(_env, node): - """Create shared object symlinks if any need to be created""" - symlinks = getattr(getattr(node, "attributes", None), "shliblinks", None) - - if not symlinks or symlinks is None: - return None - - outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] - inputs = [link.get_path() for link, _ in symlinks] - - return { - "outputs": outputs, - "inputs": inputs, - "rule": get_rule(node, "SYMLINK"), - "implicit": get_dependencies(node), - } - class SConsToNinjaTranslator: """Translates SCons Actions into Ninja build objects.""" @@ -342,8 +172,8 @@ def action_to_ninja_build(self, node, action=None): build["order_only"] = get_order_only(node) #TODO: WPD Is this testing the filename to verify it's a configure context generated file? - if 'conftest' not in str(node): - node_callback = getattr(node.attributes, "ninja_build_callback", None) + if not node.is_conftest(): + node_callback = node.check_attr("ninja_build_callback") if callable(node_callback): node_callback(env, node, build) @@ -930,33 +760,7 @@ def generate(self): self.__generated = True -def get_path(node): - """ - Return a fake path if necessary. - As an example Aliases use this as their target name in Ninja. - """ - if hasattr(node, "get_path"): - return node.get_path() - return str(node) - - -def rfile(node): - """ - Return the repository file for node if it has one. Otherwise return node - """ - if hasattr(node, "rfile"): - return node.rfile() - return node - - -def src_file(node): - """Returns the src code file if it exists.""" - if hasattr(node, "srcnode"): - src = node.srcnode() - if src.stat() is not None: - return src - return get_path(node) def get_comstr(env, action, targets, sources): @@ -1118,6 +922,7 @@ def get_generic_shell_command(env, node, action, targets, sources, executor=None return ( "CMD", { + # TODO: Why is executor passed in and then ignored below? (bdbaddog) "cmd": generate_command(env, node, action, targets, sources, executor=None), "env": get_command_env(env), }, @@ -1337,9 +1142,11 @@ def register_custom_pool(env, pool, size): """Allows the creation of custom Ninja pools""" env[NINJA_POOLS][pool] = size + def set_build_node_callback(env, node, callback): - if 'conftest' not in str(node): - setattr(node.attributes, "ninja_build_callback", callback) + if not node.is_conftest(): + node.attributes.ninja_build_callback = callback + def ninja_csig(original): """Return a dummy csig""" @@ -1362,6 +1169,7 @@ def wrapper(self): return wrapper + def CheckNinjaCompdbExpand(env, context): """ Configure check testing if ninja's compdb can expand response files""" @@ -1500,30 +1308,34 @@ def exists(env): return False -ninja_builder_initialized = False +def ninja_emitter(target, source, env): + """ fix up the source/targets """ + + ninja_file = env.File(env.subst("$NINJA_FILE_NAME")) + ninja_file.attributes.ninja_file = True + + # Someone called env.Ninja('my_targetname.ninja') + if not target and len(source) == 1: + target = source + + # Default target name is $NINJA_PREFIX.$NINJA.SUFFIX + if not target: + target = [ninja_file, ] + + # No source should have been passed. Drop it. + if source: + source = [] + + return target, source def generate(env): """Generate the NINJA builders.""" - from SCons.Script import AddOption, GetOption global ninja_builder_initialized if not ninja_builder_initialized: - ninja_builder_initialized = 1 - - AddOption('--disable-execute-ninja', - dest='disable_execute_ninja', - metavar='BOOL', - action="store_true", - default=False, - help='Disable ninja automatically building after scons') - - AddOption('--disable-ninja', - dest='disable_ninja', - metavar='BOOL', - action="store_true", - default=False, - help='Disable ninja automatically building after scons') + ninja_builder_initialized = True + ninja_add_command_line_options() try: import ninja @@ -1535,22 +1347,22 @@ def generate(env): global NINJA_STATE + env["NINJA_FILE_NAME"] = env.get("NINJA_FILE_NAME", "build.ninja") + # Add the Ninja builder. always_exec_ninja_action = AlwaysExecAction(ninja_builder, {}) - ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action) + ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action, + emitter=ninja_emitter) env.Append(BUILDERS={"Ninja": ninja_builder_obj}) - # TODO: Can't we simplify this to just be NINJA_FILENAME ? (bdbaddog) - env["NINJA_PREFIX"] = env.get("NINJA_PREFIX", "build") - env["NINJA_SUFFIX"] = env.get("NINJA_SUFFIX", "ninja") env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") env['NINJA_BUILDDIR'] = env.get("NINJA_BUILDDIR", env.Dir(".ninja").path) - ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") + # here we allow multiple environments to construct rules and builds # into the same ninja file if NINJA_STATE is None: - ninja_file = env.Ninja(target=ninja_file_name, source=[]) + ninja_file = env.Ninja() env.AlwaysBuild(ninja_file) env.Alias("$NINJA_ALIAS_NAME", ninja_file) else: @@ -1558,7 +1370,6 @@ def generate(env): SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] - # TODO: API for getting the SConscripts programmatically # exists upstream: https://github.com/SCons/scons/issues/3625 def ninja_generate_deps(env): @@ -1781,7 +1592,7 @@ def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): target = [target] for target in targets: - if str(target) != ninja_file_name and "conftest" not in str(target): + if target.check_attributes('ninja_file') is None and not target.is_conftest(): env.Depends(ninja_file, targets) return targets SCons.Builder.BuilderBase._execute = NinjaBuilderExecute @@ -1801,7 +1612,7 @@ def ninja_execute(self): target = self.targets[0] target_name = str(target) - if target_name != ninja_file_name and "conftest" not in target_name: + if target.check_attributes('ninja_file') is None and not target.is_conftest: NINJA_STATE.add_build(target) else: target.build() diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index faf395a1ef..147ba3a006 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -22,8 +22,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - import os import TestSCons from TestCmd import IS_WINDOWS diff --git a/test/ninja/generate_and_build_cxx.py b/test/ninja/generate_and_build_cxx.py index 1d149a9822..ee6e0bd8e3 100644 --- a/test/ninja/generate_and_build_cxx.py +++ b/test/ninja/generate_and_build_cxx.py @@ -45,14 +45,15 @@ test.dir_fixture('ninja-fixture') -test.write('SConstruct', """ -env = Environment() -env.Tool('ninja') -env.Program(target = 'test2', source = 'test2.cpp') -""") +# test.write('SConstruct', """ +# env = Environment() +# env.Tool('ninja') +# env.Program(target = 'test2', source = 'test2.cpp') +# """) +test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py','SConstruct') # generate simple build -test.run(stdout=None) +test.run() test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py new file mode 100644 index 0000000000..0b16d819c9 --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py @@ -0,0 +1,4 @@ +# DefaultEnvironment(tools=[]) +env = Environment() +env.Tool('ninja') +env.Program(target = 'test2', source = 'test2.cpp') From f5315e8287649208b8dda0da06bbb9e7d18542dd Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 24 Mar 2021 17:27:36 -0700 Subject: [PATCH 057/163] add sconstest.skip to file fixture dir for ninja system tests --- test/ninja/ninja_test_sconscripts/sconstest.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 test/ninja/ninja_test_sconscripts/sconstest.skip diff --git a/test/ninja/ninja_test_sconscripts/sconstest.skip b/test/ninja/ninja_test_sconscripts/sconstest.skip new file mode 100644 index 0000000000..e69de29bb2 From 5972f959fc82802bc0109de361c01ee164da9876 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 27 Mar 2021 20:35:22 -0700 Subject: [PATCH 058/163] fix broken target check in ninja_execute. It was improperly adding all nodes to NINJA_STATE.add_build(). Instead of only ones which weren't ninja files, nor conftest files --- SCons/Node/__init__.py | 2 +- SCons/Tool/ninjaCommon/__init__.py | 7 ++++--- doc/user/external.xml | 6 ++++++ test/ninja/generate_and_build.py | 18 +++++++----------- .../sconstruct_generate_and_build.py | 5 +++++ .../sconstruct_generate_and_build_cxx.py | 2 +- 6 files changed, 24 insertions(+), 16 deletions(-) create mode 100644 test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py diff --git a/SCons/Node/__init__.py b/SCons/Node/__init__.py index 4e488f45b9..0744be4b25 100644 --- a/SCons/Node/__init__.py +++ b/SCons/Node/__init__.py @@ -953,7 +953,7 @@ def is_conftest(self): def check_attributes(self, name): """ Simple API to check if the node.attributes for name has been set""" - return getattr(getattr(self,"attributes", None), name, None) + return getattr(getattr(self, "attributes", None), name, None) def alter_targets(self): diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index bb74845a90..e6eb9421e1 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -44,7 +44,7 @@ from .Util import ninja_add_command_line_options, is_valid_dependent_node, alias_to_ninja_build, \ filter_ninja_nodes, get_input_nodes, invalid_ninja_nodes, get_order_only, get_dependencies, get_inputs, get_outputs, \ - get_targets_sources, get_path + get_targets_sources, get_path, get_rule from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function from SCons.Util import is_List, flatten_sequence @@ -173,7 +173,7 @@ def action_to_ninja_build(self, node, action=None): #TODO: WPD Is this testing the filename to verify it's a configure context generated file? if not node.is_conftest(): - node_callback = node.check_attr("ninja_build_callback") + node_callback = node.check_attributes("ninja_build_callback") if callable(node_callback): node_callback(env, node, build) @@ -1612,7 +1612,8 @@ def ninja_execute(self): target = self.targets[0] target_name = str(target) - if target.check_attributes('ninja_file') is None and not target.is_conftest: + # print("File:%s -> %s"%(str(target), target.check_attributes('ninja_file'))) + if target.check_attributes('ninja_file') is None or not target.is_conftest: NINJA_STATE.add_build(target) else: target.build() diff --git a/doc/user/external.xml b/doc/user/external.xml index 5d24845c51..f580d3ff46 100644 --- a/doc/user/external.xml +++ b/doc/user/external.xml @@ -289,6 +289,12 @@ env2.CompilationDatabase('compile_commands-linux64.json')
Ninja Build Generator + + This is an experimental new feature. + + + Using the + Ninja Build System diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index 147ba3a006..6972a54aa5 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -23,6 +23,7 @@ # import os + import TestSCons from TestCmd import IS_WINDOWS @@ -34,7 +35,7 @@ test.skip_test("Could not find module in python") _python_ = TestSCons._python_ -_exe = TestSCons._exe +_exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( ninja.__file__, @@ -45,19 +46,14 @@ test.dir_fixture('ninja-fixture') -test.write('SConstruct', """ -env = Environment() -env.Tool('ninja') -env.Program(target = 'foo', source = 'foo.c') -""") - +test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build.py', 'SConstruct') # generate simple build test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('foo' + _exe), stdout="foo.c") +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('foo' + _exe), stdout="foo.c") # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -73,8 +69,8 @@ # run ninja independently program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('foo' + _exe), stdout="foo.c") +test.run(program=program, stdout=None) +test.run(program=test.workpath('foo' + _exe), stdout="foo.c") test.pass_test() diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py new file mode 100644 index 0000000000..242eb76265 --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py @@ -0,0 +1,5 @@ +DefaultEnvironment(tools=[]) + +env = Environment() +env.Tool('ninja') +env.Program(target='foo', source='foo.c') diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py index 0b16d819c9..51ca6c356d 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py @@ -1,4 +1,4 @@ -# DefaultEnvironment(tools=[]) +DefaultEnvironment(tools=[]) env = Environment() env.Tool('ninja') env.Program(target = 'test2', source = 'test2.cpp') From 0d77bc29cb1a31edcb89c5f2fc1c9a717c619521 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 27 Mar 2021 21:10:10 -0700 Subject: [PATCH 059/163] address sider issues. Also update copyright text to current version --- SCons/Tool/ninja.py | 2 +- SCons/Tool/ninjaCommon/__init__.py | 39 +++++++++---------- test/ninja/build_libraries.py | 3 +- test/ninja/copy_function_command.py | 4 +- test/ninja/generate_source.py | 4 +- test/ninja/iterative_speedup.py | 4 +- test/ninja/multi_env.py | 4 +- .../sconstruct_generate_and_build.py | 1 + .../sconstruct_generate_and_build_cxx.py | 2 + test/ninja/shell_command.py | 4 +- 10 files changed, 28 insertions(+), 39 deletions(-) diff --git a/SCons/Tool/ninja.py b/SCons/Tool/ninja.py index e597971d2d..36daec6ae8 100644 --- a/SCons/Tool/ninja.py +++ b/SCons/Tool/ninja.py @@ -22,5 +22,5 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # - +# noqa: F401 from .ninjaCommon import generate, exists diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index e6eb9421e1..0af6c21b3e 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -38,16 +38,14 @@ import SCons from SCons.Action import _string_from_cmd_list, get_default_ENV -from SCons.Node import SConscriptNodes -from SCons.Script import AddOption, GetOption +# from SCons.Node import SConscriptNodes +from SCons.Script import GetOption from SCons.Script import COMMAND_LINE_TARGETS - -from .Util import ninja_add_command_line_options, is_valid_dependent_node, alias_to_ninja_build, \ - filter_ninja_nodes, get_input_nodes, invalid_ninja_nodes, get_order_only, get_dependencies, get_inputs, get_outputs, \ - get_targets_sources, get_path, get_rule -from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function - from SCons.Util import is_List, flatten_sequence +from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function +from .Util import ninja_add_command_line_options, alias_to_ninja_build, \ + get_order_only, get_dependencies, get_inputs, get_outputs, \ + get_targets_sources, get_path, get_rule NINJA_STATE = None NINJA_RULES = "__NINJA_CUSTOM_RULES" @@ -886,7 +884,7 @@ def get_response_file_command(env, node, action, targets, sources, executor=None for key, value in custom_env.items(): variables["env"] += env.subst( - "export %s=%s;"%(key, value), target=targets, source=sources, executor=executor + "export %s=%s;" % (key, value), target=targets, source=sources, executor=executor ) + " " return rule, variables, [tool_command] @@ -930,8 +928,8 @@ def get_generic_shell_command(env, node, action, targets, sources, executor=None # and usually this would be the path to a tool, such as a compiler, used for this rule. # However this function is to generic to be able to reliably extract such deps # from the command, so we return a placeholder empty list. It should be noted that - # generally this function will not be used soley and is more like a template to generate - # the basics for a custom provider which may have more specific options for a provier + # generally this function will not be used solely and is more like a template to generate + # the basics for a custom provider which may have more specific options for a provider # function for a custom NinjaRuleMapping. [] ) @@ -997,7 +995,7 @@ def get_command(env, node, action): # pylint: disable=too-many-branches # Possibly these could be ignore and the build would still work, however it may not always # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided # ninja rule. - raise Exception("Could not resolve path for %s dependency on node '%s'"%(provider_dep, node)) + raise Exception("Could not resolve path for %s dependency on node '%s'" % (provider_dep, node)) ninja_build = { "order_only": get_order_only(node), @@ -1068,11 +1066,11 @@ def ninja_builder(env, target, source): def execute_ninja(): proc = subprocess.Popen(cmd, - stderr=sys.stderr, - stdout=subprocess.PIPE, - universal_newlines=True, - env=os.environ if env["PLATFORM"] == "win32" else env['ENV'] - ) + stderr=sys.stderr, + stdout=subprocess.PIPE, + universal_newlines=True, + env=os.environ if env["PLATFORM"] == "win32" else env['ENV'] + ) for stdout_line in iter(proc.stdout.readline, ""): yield stdout_line proc.stdout.close() @@ -1084,7 +1082,7 @@ def execute_ninja(): for output in execute_ninja(): output = output.strip() if erase_previous: - sys.stdout.write('\x1b[2K') # erase previous line + sys.stdout.write('\x1b[2K') # erase previous line sys.stdout.write("\r") else: sys.stdout.write(os.linesep) @@ -1366,7 +1364,7 @@ def generate(env): env.AlwaysBuild(ninja_file) env.Alias("$NINJA_ALIAS_NAME", ninja_file) else: - if str(NINJA_STATE.ninja_file) != ninja_file_name: + if str(NINJA_STATE.ninja_file) != env["NINJA_FILE_NAME"]: SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] @@ -1582,6 +1580,7 @@ def robust_rule_mapping(var, rule, tool): # We will subvert the normal builder execute to make sure all the ninja file is dependent # on all targets generated from any builders SCons_Builder_BuilderBase__execute = SCons.Builder.BuilderBase._execute + def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): # this ensures all environments in which a builder executes from will # not create list actions for linking on windows @@ -1611,8 +1610,6 @@ def ninja_execute(self): global NINJA_STATE target = self.targets[0] - target_name = str(target) - # print("File:%s -> %s"%(str(target), target.check_attributes('ninja_file'))) if target.check_attributes('ninja_file') is None or not target.is_conftest: NINJA_STATE.add_build(target) else: diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index b3c7b54611..d1e7c13dc2 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -22,7 +22,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os import TestSCons diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index 8e7acff7b7..08e8274622 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -22,8 +22,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - import os import TestSCons from TestCmd import IS_WINDOWS diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index 76c79bb7da..141b19c24d 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -22,8 +22,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - import os import TestSCons from TestCmd import IS_WINDOWS diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index ff50f502a3..5fd4bc5aa2 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -22,8 +22,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - import os import time import random diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 18ca3cbc69..53ec83b505 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -22,8 +22,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - import os import TestSCons from TestCmd import IS_WINDOWS diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py index 242eb76265..4123fd87b0 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py @@ -1,3 +1,4 @@ +# noqa: f821 DefaultEnvironment(tools=[]) env = Environment() diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py index 51ca6c356d..ab304fabe1 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py @@ -1,3 +1,5 @@ +# noqa: f821 + DefaultEnvironment(tools=[]) env = Environment() env.Tool('ninja') diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index 5d7f97e215..f477505293 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -22,8 +22,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - import os import TestSCons from TestCmd import IS_WINDOWS From edf9643d4c0ced3884d47c04fe62f47ad8567bd0 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 27 Mar 2021 21:22:43 -0700 Subject: [PATCH 060/163] Continue fixing sider complaints. --- test/ninja/build_libraries.py | 17 +++++------ test/ninja/copy_function_command.py | 11 +++---- test/ninja/generate_and_build.py | 2 +- test/ninja/generate_and_build_cxx.py | 3 +- test/ninja/iterative_speedup.py | 30 +++++++++++-------- test/ninja/multi_env.py | 15 +++++----- ...build.py => sconstruct_generate_and_build} | 1 - ...x.py => sconstruct_generate_and_build_cxx} | 2 -- 8 files changed, 43 insertions(+), 38 deletions(-) rename test/ninja/ninja_test_sconscripts/{sconstruct_generate_and_build.py => sconstruct_generate_and_build} (89%) rename test/ninja/ninja_test_sconscripts/{sconstruct_generate_and_build_cxx.py => sconstruct_generate_and_build_cxx} (89%) diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index d1e7c13dc2..198dcced99 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -24,6 +24,7 @@ import os + import TestSCons from TestCmd import IS_WINDOWS, IS_MACOS @@ -34,14 +35,12 @@ except ImportError: test.skip_test("Could not find ninja module in python") - ninja_binary = test.where_is('ninja') if not ninja_binary: test.skip_test("Could not find ninja. Skipping test.") - _python_ = TestSCons._python_ -_exe = TestSCons._exe +_exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( ninja.__file__, @@ -83,9 +82,9 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('test'), stdout="library_function") -test.run(program = test.workpath('test_static'), stdout="library_function") +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('test'), stdout="library_function") +test.run(program=test.workpath('test_static'), stdout="library_function") # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -104,9 +103,9 @@ # run ninja independently program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('test'), stdout="library_function") -test.run(program = test.workpath('test_static'), stdout="library_function") +test.run(program=program, stdout=None) +test.run(program=test.workpath('test'), stdout="library_function") +test.run(program=test.workpath('test_static'), stdout="library_function") test.pass_test() diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index 08e8274622..37af62f52e 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -23,6 +23,7 @@ # import os + import TestSCons from TestCmd import IS_WINDOWS @@ -34,7 +35,7 @@ test.skip_test("Could not find module in python") _python_ = TestSCons._python_ -_exe = TestSCons._exe +_exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( ninja.__file__, @@ -56,8 +57,8 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('foo'), stdout="foo.c") +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('foo'), stdout="foo.c") # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -74,8 +75,8 @@ # run ninja independently program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('foo'), stdout="foo.c") +test.run(program=program, stdout=None) +test.run(program=test.workpath('foo'), stdout="foo.c") test.pass_test() diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index 6972a54aa5..91be108176 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -46,7 +46,7 @@ test.dir_fixture('ninja-fixture') -test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build.py', 'SConstruct') +test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build', 'SConstruct') # generate simple build test.run(stdout=None) diff --git a/test/ninja/generate_and_build_cxx.py b/test/ninja/generate_and_build_cxx.py index ee6e0bd8e3..481a7e51b1 100644 --- a/test/ninja/generate_and_build_cxx.py +++ b/test/ninja/generate_and_build_cxx.py @@ -23,6 +23,7 @@ # import os + import TestSCons from TestCmd import IS_WINDOWS @@ -50,7 +51,7 @@ # env.Tool('ninja') # env.Program(target = 'test2', source = 'test2.cpp') # """) -test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py','SConstruct') +test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build_cxx', 'SConstruct') # generate simple build test.run() diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index 5fd4bc5aa2..df010f4d5b 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -23,8 +23,9 @@ # import os -import time import random +import time + import TestSCons from TestCmd import IS_WINDOWS @@ -36,7 +37,7 @@ test.skip_test("Could not find module in python") _python_ = TestSCons._python_ -_exe = TestSCons._exe +_exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( ninja.__file__, @@ -68,6 +69,7 @@ print_function0(); """) + def get_num_cpus(): """ Function to get the number of CPUs the system has. @@ -89,6 +91,7 @@ def get_num_cpus(): # Default return 1 + def generate_source(parent_source, current_source): test.write('source_{}.c'.format(current_source), """ #include @@ -111,8 +114,9 @@ def generate_source(parent_source, current_source): print_function%(current_source)s(); """ % locals()) + def mod_source_return(test_num): - parent_source = test_num-1 + parent_source = test_num - 1 test.write('source_{}.c'.format(test_num), """ #include #include @@ -128,8 +132,9 @@ def mod_source_return(test_num): } """ % locals()) + def mod_source_orig(test_num): - parent_source = test_num-1 + parent_source = test_num - 1 test.write('source_{}.c'.format(test_num), """ #include #include @@ -143,9 +148,10 @@ def mod_source_orig(test_num): } """ % locals()) + num_source = 200 -for i in range(1, num_source+1): - generate_source(i-1, i) +for i in range(1, num_source + 1): + generate_source(i - 1, i) test.write('main.c', """ #include @@ -183,15 +189,15 @@ def mod_source_orig(test_num): start = time.perf_counter() test.run(arguments='--disable-execute-ninja', stdout=None) -test.run(program = ninja_program, stdout=None) +test.run(program=ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] -test.run(program = test.workpath('print_bin'), stdout="main print") +test.run(program=test.workpath('print_bin'), stdout="main print") for test_mod in tests_mods: mod_source_return(test_mod) start = time.perf_counter() - test.run(program = ninja_program, stdout=None) + test.run(program=ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] @@ -204,15 +210,15 @@ def mod_source_orig(test_num): 'Removed build.ninja']) start = time.perf_counter() -test.run(arguments = ["-f", "SConstruct_no_ninja", jobs], stdout=None) +test.run(arguments=["-f", "SConstruct_no_ninja", jobs], stdout=None) stop = time.perf_counter() scons_times += [stop - start] -test.run(program = test.workpath('print_bin'), stdout="main print") +test.run(program=test.workpath('print_bin'), stdout="main print") for test_mod in tests_mods: mod_source_return(test_mod) start = time.perf_counter() - test.run(arguments = ["-f", "SConstruct_no_ninja", jobs], stdout=None) + test.run(arguments=["-f", "SConstruct_no_ninja", jobs], stdout=None) stop = time.perf_counter() scons_times += [stop - start] diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 53ec83b505..d14588876b 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -23,6 +23,7 @@ # import os + import TestSCons from TestCmd import IS_WINDOWS @@ -34,7 +35,7 @@ test.skip_test("Could not find module in python") _python_ = TestSCons._python_ -_exe = TestSCons._exe +_exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( ninja.__file__, @@ -60,9 +61,9 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('foo' + _exe), stdout="foo.c") -test.run(program = test.workpath('bar' + _exe), stdout="bar.c") +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('foo' + _exe), stdout="foo.c") +test.run(program=test.workpath('bar' + _exe), stdout="bar.c") # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -81,9 +82,9 @@ # run ninja independently program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('foo' + _exe), stdout="foo.c") -test.run(program = test.workpath('bar' + _exe), stdout="bar.c") +test.run(program=program, stdout=None) +test.run(program=test.workpath('foo' + _exe), stdout="foo.c") +test.run(program=test.workpath('bar' + _exe), stdout="bar.c") test.pass_test() diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build similarity index 89% rename from test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py rename to test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build index 4123fd87b0..242eb76265 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build @@ -1,4 +1,3 @@ -# noqa: f821 DefaultEnvironment(tools=[]) env = Environment() diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx similarity index 89% rename from test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py rename to test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx index ab304fabe1..51ca6c356d 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx @@ -1,5 +1,3 @@ -# noqa: f821 - DefaultEnvironment(tools=[]) env = Environment() env.Tool('ninja') From 93ad3b1b41b98e4eaf5a74b2694c21835038cf31 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 27 Mar 2021 21:27:52 -0700 Subject: [PATCH 061/163] more sider fixes --- SCons/Tool/ninja.py | 4 ++-- test/ninja/generate_and_build_cxx.py | 22 +++++++++------------- test/ninja/generate_source.py | 11 ++++++----- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/SCons/Tool/ninja.py b/SCons/Tool/ninja.py index 36daec6ae8..88b7d48f09 100644 --- a/SCons/Tool/ninja.py +++ b/SCons/Tool/ninja.py @@ -22,5 +22,5 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -# noqa: F401 -from .ninjaCommon import generate, exists + +from .ninjaCommon import generate, exists # noqa: F401 diff --git a/test/ninja/generate_and_build_cxx.py b/test/ninja/generate_and_build_cxx.py index 481a7e51b1..074a5cb9af 100644 --- a/test/ninja/generate_and_build_cxx.py +++ b/test/ninja/generate_and_build_cxx.py @@ -35,7 +35,7 @@ test.skip_test("Could not find module in python") _python_ = TestSCons._python_ -_exe = TestSCons._exe +_exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( ninja.__file__, @@ -46,19 +46,15 @@ test.dir_fixture('ninja-fixture') -# test.write('SConstruct', """ -# env = Environment() -# env.Tool('ninja') -# env.Program(target = 'test2', source = 'test2.cpp') -# """) -test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build_cxx', 'SConstruct') +test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build_cxx', + 'SConstruct') # generate simple build test.run() test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('test2' + _exe), stdout="print_function") +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('test2' + _exe), stdout="print_function") # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -74,8 +70,8 @@ # run ninja independently program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('test2' + _exe), stdout="print_function") +test.run(program=program, stdout=None) +test.run(program=test.workpath('test2' + _exe), stdout="print_function") test.write('test2.hpp', """ #include @@ -93,8 +89,8 @@ class Foo """) # generate simple build -test.run(program = program, stdout=None) -test.run(program = test.workpath('test2' + _exe), stdout="print_function2") +test.run(program=program, stdout=None) +test.run(program=test.workpath('test2' + _exe), stdout="print_function2") test.pass_test() diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index 141b19c24d..18b53f249b 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -23,6 +23,7 @@ # import os + import TestSCons from TestCmd import IS_WINDOWS @@ -34,7 +35,7 @@ test.skip_test("Could not find module in python") _python_ = TestSCons._python_ -_exe = TestSCons._exe +_exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( ninja.__file__, @@ -79,8 +80,8 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('generated_source' + _exe), stdout="generated_source.c") # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -99,8 +100,8 @@ # run ninja independently program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") +test.run(program=program, stdout=None) +test.run(program=test.workpath('generated_source' + _exe), stdout="generated_source.c") test.pass_test() From 6cf598e21dbdc5450cf8a325f807c2926d5c022c Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 27 Mar 2021 21:33:01 -0700 Subject: [PATCH 062/163] fix sider issue --- test/ninja/shell_command.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index f477505293..f0450d9cef 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -23,6 +23,7 @@ # import os + import TestSCons from TestCmd import IS_WINDOWS @@ -34,7 +35,7 @@ test.skip_test("Could not find module in python") _python_ = TestSCons._python_ -_exe = TestSCons._exe +_exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( ninja.__file__, @@ -58,7 +59,7 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) test.must_match('foo.out', 'foo.c') # clean build and ninja files @@ -76,7 +77,7 @@ # run ninja independently program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) +test.run(program=program, stdout=None) test.must_match('foo.out', 'foo.c') test.pass_test() From adb3cb21f13ca8e381c6e003459acdb88af17035 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 10 Apr 2021 16:50:43 -0700 Subject: [PATCH 063/163] Incremental checkin. Functionality restored after refactor broke it. --- SCons/Tool/ninjaCommon/Globals.py | 40 ++ SCons/Tool/ninjaCommon/NinjaState.py | 707 +++++++++++++++++++ SCons/Tool/ninjaCommon/Overrides.py | 0 SCons/Tool/ninjaCommon/Rules.py | 25 +- SCons/Tool/ninjaCommon/Util.py | 286 +++++++- SCons/Tool/ninjaCommon/__init__.py | 990 +-------------------------- 6 files changed, 1077 insertions(+), 971 deletions(-) create mode 100644 SCons/Tool/ninjaCommon/Globals.py create mode 100644 SCons/Tool/ninjaCommon/NinjaState.py create mode 100644 SCons/Tool/ninjaCommon/Overrides.py diff --git a/SCons/Tool/ninjaCommon/Globals.py b/SCons/Tool/ninjaCommon/Globals.py new file mode 100644 index 0000000000..0dc46ea840 --- /dev/null +++ b/SCons/Tool/ninjaCommon/Globals.py @@ -0,0 +1,40 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import SCons.Action + +NINJA_RULES = "__NINJA_CUSTOM_RULES" +NINJA_POOLS = "__NINJA_CUSTOM_POOLS" +NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" +NINJA_BUILD = "NINJA_BUILD" +NINJA_WHEREIS_MEMO = {} +NINJA_STAT_MEMO = {} +__NINJA_RULE_MAPPING = {} + + +# These are the types that get_command can do something with +COMMAND_TYPES = ( + SCons.Action.CommandAction, + SCons.Action.CommandGeneratorAction, +) +ninja_builder_initialized = False \ No newline at end of file diff --git a/SCons/Tool/ninjaCommon/NinjaState.py b/SCons/Tool/ninjaCommon/NinjaState.py new file mode 100644 index 0000000000..4e41934aa3 --- /dev/null +++ b/SCons/Tool/ninjaCommon/NinjaState.py @@ -0,0 +1,707 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import io +import os +import sys +from os.path import splitext +import ninja + +import SCons +from SCons.Script import COMMAND_LINE_TARGETS +from SCons.Util import is_List +import SCons.Tool.ninjaCommon.Globals +from .Globals import COMMAND_TYPES, NINJA_RULES, NINJA_POOLS, \ + NINJA_CUSTOM_HANDLERS +from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function +from .Util import get_path, alias_to_ninja_build, generate_depfile, ninja_noop, get_command, get_order_only, \ + get_outputs, get_inputs, get_dependencies, get_rule, get_command_env + + + +# pylint: disable=too-many-instance-attributes +class NinjaState: + """Maintains state of Ninja build system as it's translated from SCons.""" + + def __init__(self, env, ninja_file, writer_class): + self.env = env + self.ninja_file = ninja_file + + self.ninja_bin_path = env.get('NINJA') + if not self.ninja_bin_path: + # default to using ninja installed with python module + ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' + self.ninja_bin_path = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + ninja_bin)) + if not os.path.exists(self.ninja_bin_path): + # couldn't find it, just give the bin name and hope + # its in the path later + self.ninja_bin_path = ninja_bin + + self.writer_class = writer_class + self.__generated = False + self.translator = SConsToNinjaTranslator(env) + self.generated_suffixes = env.get("NINJA_GENERATED_SOURCE_SUFFIXES", []) + + # List of generated builds that will be written at a later stage + self.builds = dict() + + # List of targets for which we have generated a build. This + # allows us to take multiple Alias nodes as sources and to not + # fail to build if they have overlapping targets. + self.built = set() + + # SCons sets this variable to a function which knows how to do + # shell quoting on whatever platform it's run on. Here we use it + # to make the SCONS_INVOCATION variable properly quoted for things + # like CCFLAGS + escape = env.get("ESCAPE", lambda x: x) + + # if SCons was invoked from python, we expect the first arg to be the scons.py + # script, otherwise scons was invoked from the scons script + python_bin = '' + if os.path.basename(sys.argv[0]) == 'scons.py': + python_bin = escape(sys.executable) + self.variables = { + "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", + "SCONS_INVOCATION": '{} {} --disable-ninja __NINJA_NO=1 $out'.format( + python_bin, + " ".join( + [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] + ), + ), + "SCONS_INVOCATION_W_TARGETS": "{} {} --disable-ninja".format( + python_bin, " ".join([escape(arg) for arg in sys.argv]) + ), + # This must be set to a global default per: + # https://ninja-build.org/manual.html#_deps + # English Visual Studio will have the default below, + # otherwise the user can define the variable in the first environment + # that initialized ninja tool + "msvc_deps_prefix": env.get("NINJA_MSVC_DEPS_PREFIX", "Note: including file:") + } + + self.rules = { + "CMD": { + "command": "cmd /c $env$cmd $in $out" if sys.platform == "win32" else "$env$cmd $in $out", + "description": "Building $out", + "pool": "local_pool", + }, + "GENERATED_CMD": { + "command": "cmd /c $env$cmd" if sys.platform == "win32" else "$env$cmd", + "description": "Building $out", + "pool": "local_pool", + }, + # We add the deps processing variables to this below. We + # don't pipe these through cmd.exe on Windows because we + # use this to generate a compile_commands.json database + # which can't use the shell command as it's compile + # command. + "CC": { + "command": "$env$CC @$out.rsp", + "description": "Compiling $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + }, + "CXX": { + "command": "$env$CXX @$out.rsp", + "description": "Compiling $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + }, + "LINK": { + "command": "$env$LINK @$out.rsp", + "description": "Linking $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + "pool": "local_pool", + }, + # Ninja does not automatically delete the archive before + # invoking ar. The ar utility will append to an existing archive, which + # can cause duplicate symbols if the symbols moved between object files. + # Native SCons will perform this operation so we need to force ninja + # to do the same. See related for more info: + # https://jira.mongodb.org/browse/SERVER-49457 + "AR": { + "command": "{}$env$AR @$out.rsp".format( + '' if sys.platform == "win32" else "rm -f $out && " + ), + "description": "Archiving $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + "pool": "local_pool", + }, + "SYMLINK": { + "command": ( + "cmd /c mklink $out $in" + if sys.platform == "win32" + else "ln -s $in $out" + ), + "description": "Symlink $in -> $out", + }, + "INSTALL": { + "command": "$COPY $in $out", + "description": "Install $out", + "pool": "install_pool", + # On Windows cmd.exe /c copy does not always correctly + # update the timestamp on the output file. This leads + # to a stuck constant timestamp in the Ninja database + # and needless rebuilds. + # + # Adding restat here ensures that Ninja always checks + # the copy updated the timestamp and that Ninja has + # the correct information. + "restat": 1, + }, + "TEMPLATE": { + "command": "$SCONS_INVOCATION $out", + "description": "Rendering $SCONS_INVOCATION $out", + "pool": "scons_pool", + "restat": 1, + }, + "SCONS": { + "command": "$SCONS_INVOCATION $out", + "description": "$SCONS_INVOCATION $out", + "pool": "scons_pool", + # restat + # if present, causes Ninja to re-stat the command's outputs + # after execution of the command. Each output whose + # modification time the command did not change will be + # treated as though it had never needed to be built. This + # may cause the output's reverse dependencies to be removed + # from the list of pending build actions. + # + # We use restat any time we execute SCons because + # SCons calls in Ninja typically create multiple + # targets. But since SCons is doing it's own up to + # date-ness checks it may only update say one of + # them. Restat will find out which of the multiple + # build targets did actually change then only rebuild + # those targets which depend specifically on that + # output. + "restat": 1, + }, + "REGENERATE": { + "command": "$SCONS_INVOCATION_W_TARGETS", + "description": "Regenerating $out", + "generator": 1, + "depfile": os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile'), + # Console pool restricts to 1 job running at a time, + # it additionally has some special handling about + # passing stdin, stdout, etc to process in this pool + # that we need for SCons to behave correctly when + # regenerating Ninja + "pool": "console", + # Again we restat in case Ninja thought the + # build.ninja should be regenerated but SCons knew + # better. + "restat": 1, + }, + } + + if env['PLATFORM'] == 'darwin' and env['AR'] == 'ar': + self.rules["AR"] = { + "command": "rm -f $out && $env$AR $rspc", + "description": "Archiving $out", + "pool": "local_pool", + } + + self.pools = { + "local_pool": self.env.GetOption("num_jobs"), + "install_pool": self.env.GetOption("num_jobs") / 2, + "scons_pool": 1, + } + + for rule in ["CC", "CXX"]: + if env["PLATFORM"] == "win32": + self.rules[rule]["deps"] = "msvc" + else: + self.rules[rule]["deps"] = "gcc" + self.rules[rule]["depfile"] = "$out.d" + + def add_build(self, node): + if not node.has_builder(): + return False + + if isinstance(node, SCons.Node.Alias.Alias): + build = alias_to_ninja_build(node) + else: + build = self.translator.action_to_ninja_build(node) + + # Some things are unbuild-able or need not be built in Ninja + if build is None: + return False + + node_string = str(node) + if node_string in self.builds: + raise Exception("Node {} added to ninja build state more than once".format(node_string)) + self.builds[node_string] = build + self.built.update(build["outputs"]) + return True + + # TODO: rely on SCons to tell us what is generated source + # or some form of user scanner maybe (Github Issue #3624) + def is_generated_source(self, output): + """Check if output ends with a known generated suffix.""" + _, suffix = splitext(output) + return suffix in self.generated_suffixes + + def has_generated_sources(self, output): + """ + Determine if output indicates this is a generated header file. + """ + for generated in output: + if self.is_generated_source(generated): + return True + return False + + # pylint: disable=too-many-branches,too-many-locals + def generate(self): + """ + Generate the build.ninja. + + This should only be called once for the lifetime of this object. + """ + if self.__generated: + return + + self.rules.update(self.env.get(NINJA_RULES, {})) + self.pools.update(self.env.get(NINJA_POOLS, {})) + + content = io.StringIO() + ninja = self.writer_class(content, width=100) + + ninja.comment("Generated by scons. DO NOT EDIT.") + + ninja.variable("builddir", get_path(self.env['NINJA_BUILDDIR'])) + + for pool_name, size in self.pools.items(): + ninja.pool(pool_name, size) + + for var, val in self.variables.items(): + ninja.variable(var, val) + + for rule, kwargs in self.rules.items(): + ninja.rule(rule, **kwargs) + + generated_source_files = sorted({ + output + # First find builds which have header files in their outputs. + for build in self.builds.values() + if self.has_generated_sources(build["outputs"]) + for output in build["outputs"] + # Collect only the header files from the builds with them + # in their output. We do this because is_generated_source + # returns True if it finds a header in any of the outputs, + # here we need to filter so we only have the headers and + # not the other outputs. + if self.is_generated_source(output) + }) + + if generated_source_files: + ninja.build( + outputs="_generated_sources", + rule="phony", + implicit=generated_source_files + ) + + template_builders = [] + + for build in [self.builds[key] for key in sorted(self.builds.keys())]: + if build["rule"] == "TEMPLATE": + template_builders.append(build) + continue + + if "implicit" in build: + build["implicit"].sort() + + # Don't make generated sources depend on each other. We + # have to check that none of the outputs are generated + # sources and none of the direct implicit dependencies are + # generated sources or else we will create a dependency + # cycle. + if ( + generated_source_files + and not build["rule"] == "INSTALL" + and set(build["outputs"]).isdisjoint(generated_source_files) + and set(build.get("implicit", [])).isdisjoint(generated_source_files) + ): + # Make all non-generated source targets depend on + # _generated_sources. We use order_only for generated + # sources so that we don't rebuild the world if one + # generated source was rebuilt. We just need to make + # sure that all of these sources are generated before + # other builds. + order_only = build.get("order_only", []) + order_only.append("_generated_sources") + build["order_only"] = order_only + if "order_only" in build: + build["order_only"].sort() + + # When using a depfile Ninja can only have a single output + # but SCons will usually have emitted an output for every + # thing a command will create because it's caching is much + # more complex than Ninja's. This includes things like DWO + # files. Here we make sure that Ninja only ever sees one + # target when using a depfile. It will still have a command + # that will create all of the outputs but most targets don't + # depend directly on DWO files and so this assumption is safe + # to make. + rule = self.rules.get(build["rule"]) + + # Some rules like 'phony' and other builtins we don't have + # listed in self.rules so verify that we got a result + # before trying to check if it has a deps key. + # + # Anything using deps or rspfile in Ninja can only have a single + # output, but we may have a build which actually produces + # multiple outputs which other targets can depend on. Here we + # slice up the outputs so we have a single output which we will + # use for the "real" builder and multiple phony targets that + # match the file names of the remaining outputs. This way any + # build can depend on any output from any build. + # + # We assume that the first listed output is the 'key' + # output and is stably presented to us by SCons. For + # instance if -gsplit-dwarf is in play and we are + # producing foo.o and foo.dwo, we expect that outputs[0] + # from SCons will be the foo.o file and not the dwo + # file. If instead we just sorted the whole outputs array, + # we would find that the dwo file becomes the + # first_output, and this breaks, for instance, header + # dependency scanning. + if rule is not None and (rule.get("deps") or rule.get("rspfile")): + first_output, remaining_outputs = ( + build["outputs"][0], + build["outputs"][1:], + ) + + if remaining_outputs: + ninja.build( + outputs=sorted(remaining_outputs), rule="phony", implicit=first_output, + ) + + build["outputs"] = first_output + + # Optionally a rule can specify a depfile, and SCons can generate implicit + # dependencies into the depfile. This allows for dependencies to come and go + # without invalidating the ninja file. The depfile was created in ninja specifically + # for dealing with header files appearing and disappearing across rebuilds, but it can + # be repurposed for anything, as long as you have a way to regenerate the depfile. + # More specific info can be found here: https://ninja-build.org/manual.html#_depfile + if rule is not None and rule.get('depfile') and build.get('deps_files'): + path = build['outputs'] if SCons.Util.is_List(build['outputs']) else [build['outputs']] + generate_depfile(self.env, path[0], build.pop('deps_files', [])) + + if "inputs" in build: + build["inputs"].sort() + + ninja.build(**build) + + template_builds = dict() + for template_builder in template_builders: + + # Special handling for outputs and implicit since we need to + # aggregate not replace for each builder. + for agg_key in ["outputs", "implicit", "inputs"]: + new_val = template_builds.get(agg_key, []) + + # Use pop so the key is removed and so the update + # below will not overwrite our aggregated values. + cur_val = template_builder.pop(agg_key, []) + if is_List(cur_val): + new_val += cur_val + else: + new_val.append(cur_val) + template_builds[agg_key] = new_val + + # Collect all other keys + template_builds.update(template_builder) + + if template_builds.get("outputs", []): + ninja.build(**template_builds) + + # We have to glob the SCons files here to teach the ninja file + # how to regenerate itself. We'll never see ourselves in the + # DAG walk so we can't rely on action_to_ninja_build to + # generate this rule even though SCons should know we're + # dependent on SCons files. + # + # The REGENERATE rule uses depfile, so we need to generate the depfile + # in case any of the SConscripts have changed. The depfile needs to be + # path with in the build and the passed ninja file is an abspath, so + # we will use SCons to give us the path within the build. Normally + # generate_depfile should not be called like this, but instead be called + # through the use of custom rules, and filtered out in the normal + # list of build generation about. However, because the generate rule + # is hardcoded here, we need to do this generate_depfile call manually. + ninja_file_path = self.env.File(self.ninja_file).path + generate_depfile( + self.env, + ninja_file_path, + self.env['NINJA_REGENERATE_DEPS'] + ) + + ninja.build( + ninja_file_path, + rule="REGENERATE", + implicit=[__file__], + ) + + # If we ever change the name/s of the rules that include + # compile commands (i.e. something like CC) we will need to + # update this build to reflect that complete list. + ninja.build( + "compile_commands.json", + rule="CMD", + pool="console", + implicit=[str(self.ninja_file)], + variables={ + "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( + # NINJA_COMPDB_EXPAND - should only be true for ninja + # This was added to ninja's compdb tool in version 1.9.0 (merged April 2018) + # https://github.com/ninja-build/ninja/pull/1223 + # TODO: add check in generate to check version and enable this by default if it's available. + self.ninja_bin_path, str(self.ninja_file), + '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' + ) + }, + ) + + ninja.build( + "compiledb", rule="phony", implicit=["compile_commands.json"], + ) + + # Look in SCons's list of DEFAULT_TARGETS, find the ones that + # we generated a ninja build rule for. + scons_default_targets = [ + get_path(tgt) + for tgt in SCons.Script.DEFAULT_TARGETS + if get_path(tgt) in self.built + ] + + # If we found an overlap between SCons's list of default + # targets and the targets we created ninja builds for then use + # those as ninja's default as well. + if scons_default_targets: + ninja.default(" ".join(scons_default_targets)) + + with open(str(self.ninja_file), "w") as build_ninja: + build_ninja.write(content.getvalue()) + + self.__generated = True + + +class SConsToNinjaTranslator: + """Translates SCons Actions into Ninja build objects.""" + + def __init__(self, env): + self.env = env + self.func_handlers = { + # Skip conftest builders + "_createSource": ninja_noop, + # SCons has a custom FunctionAction that just makes sure the + # target isn't static. We let the commands that ninja runs do + # this check for us. + "SharedFlagChecker": ninja_noop, + # The install builder is implemented as a function action. + # TODO: use command action #3573 + "installFunc": _install_action_function, + "MkdirFunc": _mkdir_action_function, + "LibSymlinksActionFunction": _lib_symlink_action_function, + "Copy": _copy_action_function + } + + self.loaded_custom = False + + # pylint: disable=too-many-return-statements + def action_to_ninja_build(self, node, action=None): + """Generate build arguments dictionary for node.""" + + if not self.loaded_custom: + self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) + self.loaded_custom = True + + if node.builder is None: + return None + + if action is None: + action = node.builder.action + + if node.env and node.env.get("NINJA_SKIP"): + return None + + build = {} + env = node.env if node.env else self.env + + # Ideally this should never happen, and we do try to filter + # Ninja builders out of being sources of ninja builders but I + # can't fix every DAG problem so we just skip ninja_builders + # if we find one + if SCons.Tool.ninjaCommon.NINJA_STATE.ninja_file == str(node): + build = None + elif isinstance(action, SCons.Action.FunctionAction): + build = self.handle_func_action(node, action) + elif isinstance(action, SCons.Action.LazyAction): + # pylint: disable=protected-access + action = action._generate_cache(env) + build = self.action_to_ninja_build(node, action=action) + elif isinstance(action, SCons.Action.ListAction): + build = self.handle_list_action(node, action) + elif isinstance(action, COMMAND_TYPES): + build = get_command(env, node, action) + else: + raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) + + if build is not None: + build["order_only"] = get_order_only(node) + + # TODO: WPD Is this testing the filename to verify it's a configure context generated file? + if not node.is_conftest(): + node_callback = node.check_attributes("ninja_build_callback") + if callable(node_callback): + node_callback(env, node, build) + + return build + + def handle_func_action(self, node, action): + """Determine how to handle the function action.""" + name = action.function_name() + # This is the name given by the Subst/Textfile builders. So return the + # node to indicate that SCons is required. We skip sources here because + # dependencies don't really matter when we're going to shove these to + # the bottom of ninja's DAG anyway and Textfile builders can have text + # content as their source which doesn't work as an implicit dep in + # ninja. + if name == 'ninja_builder': + return None + + handler = self.func_handlers.get(name, None) + if handler is not None: + return handler(node.env if node.env else self.env, node) + elif name == "ActionCaller": + action_to_call = str(action).split('(')[0].strip() + handler = self.func_handlers.get(action_to_call, None) + if handler is not None: + return handler(node.env if node.env else self.env, node) + + SCons.Warnings.SConsWarning( + "Found unhandled function action {}, " + " generating scons command to build\n" + "Note: this is less efficient than Ninja," + " you can write your own ninja build generator for" + " this function using NinjaRegisterFunctionHandler".format(name) + ) + + return { + "rule": "TEMPLATE", + "order_only": get_order_only(node), + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "implicit": get_dependencies(node, skip_sources=True), + } + + # pylint: disable=too-many-branches + def handle_list_action(self, node, action): + """TODO write this comment""" + results = [ + self.action_to_ninja_build(node, action=act) + for act in action.list + if act is not None + ] + results = [ + result for result in results if result is not None and result["outputs"] + ] + if not results: + return None + + # No need to process the results if we only got a single result + if len(results) == 1: + return results[0] + + all_outputs = list({output for build in results for output in build["outputs"]}) + dependencies = list({dep for build in results for dep in build["implicit"]}) + + if results[0]["rule"] == "CMD" or results[0]["rule"] == "GENERATED_CMD": + cmdline = "" + for cmd in results: + + # Occasionally a command line will expand to a + # whitespace only string (i.e. ' '). Which is not a + # valid command but does not trigger the empty command + # condition if not cmdstr. So here we strip preceding + # and proceeding whitespace to make strings like the + # above become empty strings and so will be skipped. + cmdstr = cmd["variables"]["cmd"].strip() + if not cmdstr: + continue + + # Skip duplicate commands + if cmdstr in cmdline: + continue + + if cmdline: + cmdline += " && " + + cmdline += cmdstr + + # Remove all preceding and proceeding whitespace + cmdline = cmdline.strip() + + # Make sure we didn't generate an empty cmdline + if cmdline: + ninja_build = { + "outputs": all_outputs, + "rule": get_rule(node, "GENERATED_CMD"), + "variables": { + "cmd": cmdline, + "env": get_command_env(node.env if node.env else self.env), + }, + "implicit": dependencies, + } + + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["pool"] + + return ninja_build + + elif results[0]["rule"] == "phony": + return { + "outputs": all_outputs, + "rule": "phony", + "implicit": dependencies, + } + + elif results[0]["rule"] == "INSTALL": + return { + "outputs": all_outputs, + "rule": get_rule(node, "INSTALL"), + "inputs": get_inputs(node), + "implicit": dependencies, + } + + raise Exception("Unhandled list action with rule: " + results[0]["rule"]) diff --git a/SCons/Tool/ninjaCommon/Overrides.py b/SCons/Tool/ninjaCommon/Overrides.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/SCons/Tool/ninjaCommon/Rules.py b/SCons/Tool/ninjaCommon/Rules.py index 99b11e4c8e..c1c238e1d7 100644 --- a/SCons/Tool/ninjaCommon/Rules.py +++ b/SCons/Tool/ninjaCommon/Rules.py @@ -1,3 +1,26 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + from .Util import get_outputs, get_rule, get_inputs, get_dependencies @@ -55,4 +78,4 @@ def _lib_symlink_action_function(_env, node): "inputs": inputs, "rule": get_rule(node, "SYMLINK"), "implicit": get_dependencies(node), - } \ No newline at end of file + } diff --git a/SCons/Tool/ninjaCommon/Util.py b/SCons/Tool/ninjaCommon/Util.py index 5f14a96060..bb20d80a3f 100644 --- a/SCons/Tool/ninjaCommon/Util.py +++ b/SCons/Tool/ninjaCommon/Util.py @@ -1,5 +1,34 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +import os +from os.path import join as joinpath + import SCons +from SCons.Action import get_default_ENV, _string_from_cmd_list from SCons.Script import AddOption +from SCons.Tool.ninjaCommon import __NINJA_RULE_MAPPING +from SCons.Util import is_List, flatten_sequence + def ninja_add_command_line_options(): """ @@ -170,4 +199,259 @@ def get_rule(node, rule): if invalid_ninja_nodes(node, tlist): return "TEMPLATE" else: - return rule \ No newline at end of file + return rule + + +def generate_depfile(env, node, dependencies): + """ + Ninja tool function for writing a depfile. The depfile should include + the node path followed by all the dependent files in a makefile format. + + dependencies arg can be a list or a subst generator which returns a list. + """ + + depfile = os.path.join(get_path(env['NINJA_BUILDDIR']), str(node) + '.depfile') + + # subst_list will take in either a raw list or a subst callable which generates + # a list, and return a list of CmdStringHolders which can be converted into raw strings. + # If a raw list was passed in, then scons_list will make a list of lists from the original + # values and even subst items in the list if they are substitutable. Flatten will flatten + # the list in that case, to ensure for either input we have a list of CmdStringHolders. + deps_list = env.Flatten(env.subst_list(dependencies)) + + # Now that we have the deps in a list as CmdStringHolders, we can convert them into raw strings + # and make sure to escape the strings to handle spaces in paths. We also will sort the result + # keep the order of the list consistent. + escaped_depends = sorted([dep.escape(env.get("ESCAPE", lambda x: x)) for dep in deps_list]) + depfile_contents = str(node) + ": " + ' '.join(escaped_depends) + + need_rewrite = False + try: + with open(depfile, 'r') as f: + need_rewrite = (f.read() != depfile_contents) + except FileNotFoundError: + need_rewrite = True + + if need_rewrite: + os.makedirs(os.path.dirname(depfile) or '.', exist_ok=True) + with open(depfile, 'w') as f: + f.write(depfile_contents) + + +def ninja_noop(*_args, **_kwargs): + """ + A general purpose no-op function. + + There are many things that happen in SCons that we don't need and + also don't return anything. We use this to disable those functions + instead of creating multiple definitions of the same thing. + """ + return None + + +def get_command(env, node, action): # pylint: disable=too-many-branches + """Get the command to execute for node.""" + if node.env: + sub_env = node.env + else: + sub_env = env + executor = node.get_executor() + tlist, slist = get_targets_sources(node) + + # Generate a real CommandAction + if isinstance(action, SCons.Action.CommandGeneratorAction): + # pylint: disable=protected-access + action = action._generate(tlist, slist, sub_env, 1, executor=executor) + + variables = {} + + comstr = get_comstr(sub_env, action, tlist, slist) + if not comstr: + return None + + provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) + rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) + + # Get the dependencies for all targets + implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) + + # Now add in the other dependencies related to the command, + # e.g. the compiler binary. The ninja rule can be user provided so + # we must do some validation to resolve the dependency path for ninja. + for provider_dep in provider_deps: + + provider_dep = sub_env.subst(provider_dep) + if not provider_dep: + continue + + # If the tool is a node, then SCons will resolve the path later, if its not + # a node then we assume it generated from build and make sure it is existing. + if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep): + implicit.append(provider_dep) + continue + + # in some case the tool could be in the local directory and be suppled without the ext + # such as in windows, so append the executable suffix and check. + prog_suffix = sub_env.get('PROGSUFFIX', '') + provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix + if os.path.exists(provider_dep_ext): + implicit.append(provider_dep_ext) + continue + + # Many commands will assume the binary is in the path, so + # we accept this as a possible input from a given command. + + provider_dep_abspath = sub_env.WhereIs(provider_dep) or sub_env.WhereIs(provider_dep, path=os.environ["PATH"]) + if provider_dep_abspath: + implicit.append(provider_dep_abspath) + continue + + # Possibly these could be ignore and the build would still work, however it may not always + # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided + # ninja rule. + raise Exception("Could not resolve path for %s dependency on node '%s'" % (provider_dep, node)) + + ninja_build = { + "order_only": get_order_only(node), + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "implicit": implicit, + "rule": get_rule(node, rule), + "variables": variables, + } + + # Don't use sub_env here because we require that NINJA_POOL be set + # on a per-builder call basis to prevent accidental strange + # behavior like env['NINJA_POOL'] = 'console' and sub_env can be + # the global Environment object if node.env is None. + # Example: + # + # Allowed: + # + # env.Command("ls", NINJA_POOL="ls_pool") + # + # Not allowed and ignored: + # + # env["NINJA_POOL"] = "ls_pool" + # env.Command("ls") + # + # TODO: Why not alloe env['NINJA_POOL'] ? (bdbaddog) + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["NINJA_POOL"] + + return ninja_build + + +def get_command_env(env): + """ + Return a string that sets the environment for any environment variables that + differ between the OS environment and the SCons command ENV. + + It will be compatible with the default shell of the operating system. + """ + try: + return env["NINJA_ENV_VAR_CACHE"] + except KeyError: + pass + + # Scan the ENV looking for any keys which do not exist in + # os.environ or differ from it. We assume if it's a new or + # differing key from the process environment then it's + # important to pass down to commands in the Ninja file. + ENV = get_default_ENV(env) + scons_specified_env = { + key: value + for key, value in ENV.items() + # TODO: Remove this filter, unless there's a good reason to keep. SCons's behavior shouldn't depend on shell's. + if key not in os.environ or os.environ.get(key, None) != value + } + + windows = env["PLATFORM"] == "win32" + command_env = "" + for key, value in scons_specified_env.items(): + # Ensure that the ENV values are all strings: + if is_List(value): + # If the value is a list, then we assume it is a + # path list, because that's a pretty common list-like + # value to stick in an environment variable: + value = flatten_sequence(value) + value = joinpath(map(str, value)) + else: + # If it isn't a string or a list, then we just coerce + # it to a string, which is the proper way to handle + # Dir and File instances and will produce something + # reasonable for just about everything else: + value = str(value) + + if windows: + command_env += "set '{}={}' && ".format(key, value) + else: + # We address here *only* the specific case that a user might have + # an environment variable which somehow gets included and has + # spaces in the value. These are escapes that Ninja handles. This + # doesn't make builds on paths with spaces (Ninja and SCons issues) + # nor expanding response file paths with spaces (Ninja issue) work. + value = value.replace(r' ', r'$ ') + command_env += "export {}='{}';".format(key, value) + + env["NINJA_ENV_VAR_CACHE"] = command_env + return command_env + + +def get_comstr(env, action, targets, sources): + """Get the un-substituted string for action.""" + # Despite being having "list" in it's name this member is not + # actually a list. It's the pre-subst'd string of the command. We + # use it to determine if the command we're about to generate needs + # to use a custom Ninja rule. By default this redirects CC, CXX, + # AR, SHLINK, and LINK commands to their respective rules but the + # user can inject custom Ninja rules and tie them to commands by + # using their pre-subst'd string. + if hasattr(action, "process"): + return action.cmd_list + + return action.genstring(targets, sources, env) + + +def get_generic_shell_command(env, node, action, targets, sources, executor=None): + return ( + "CMD", + { + # TODO: Why is executor passed in and then ignored below? (bdbaddog) + "cmd": generate_command(env, node, action, targets, sources, executor=None), + "env": get_command_env(env), + }, + # Since this function is a rule mapping provider, it must return a list of dependencies, + # and usually this would be the path to a tool, such as a compiler, used for this rule. + # However this function is to generic to be able to reliably extract such deps + # from the command, so we return a placeholder empty list. It should be noted that + # generally this function will not be used solely and is more like a template to generate + # the basics for a custom provider which may have more specific options for a provider + # function for a custom NinjaRuleMapping. + [] + ) + + +def generate_command(env, node, action, targets, sources, executor=None): + # Actions like CommandAction have a method called process that is + # used by SCons to generate the cmd_line they need to run. So + # check if it's a thing like CommandAction and call it if we can. + if hasattr(action, "process"): + cmd_list, _, _ = action.process(targets, sources, env, executor=executor) + cmd = _string_from_cmd_list(cmd_list[0]) + else: + # Anything else works with genstring, this is most commonly hit by + # ListActions which essentially call process on all of their + # commands and concatenate it for us. + genstring = action.genstring(targets, sources, env) + if executor is not None: + cmd = env.subst(genstring, executor=executor) + else: + cmd = env.subst(genstring, targets, sources) + + cmd = cmd.replace("\n", " && ").strip() + if cmd.endswith("&&"): + cmd = cmd[0:-2].strip() + + # Escape dollars as necessary + return cmd.replace("$", "$$") \ No newline at end of file diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index 0af6c21b3e..a2745c5827 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -25,7 +25,6 @@ """Generate build.ninja files from SCons aliases.""" import importlib -import io import os import shlex import shutil @@ -33,803 +32,22 @@ import sys import textwrap from glob import glob -from os.path import join as joinpath -from os.path import splitext import SCons -from SCons.Action import _string_from_cmd_list, get_default_ENV -# from SCons.Node import SConscriptNodes +import SCons.Tool.ninjaCommon.Globals +from SCons.Action import _string_from_cmd_list from SCons.Script import GetOption -from SCons.Script import COMMAND_LINE_TARGETS -from SCons.Util import is_List, flatten_sequence +from SCons.Util import is_List +import SCons.Tool.ninjaCommon.Globals + +from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS, __NINJA_RULE_MAPPING +from .NinjaState import NinjaState from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function from .Util import ninja_add_command_line_options, alias_to_ninja_build, \ - get_order_only, get_dependencies, get_inputs, get_outputs, \ - get_targets_sources, get_path, get_rule + get_targets_sources, get_path, ninja_noop, get_command, get_command_env, get_comstr, get_generic_shell_command, \ + generate_command NINJA_STATE = None -NINJA_RULES = "__NINJA_CUSTOM_RULES" -NINJA_POOLS = "__NINJA_CUSTOM_POOLS" -NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" -NINJA_BUILD = "NINJA_BUILD" -NINJA_WHEREIS_MEMO = {} -NINJA_STAT_MEMO = {} - -__NINJA_RULE_MAPPING = {} - -# These are the types that get_command can do something with -COMMAND_TYPES = ( - SCons.Action.CommandAction, - SCons.Action.CommandGeneratorAction, -) - -ninja_builder_initialized = False - - -def generate_depfile(env, node, dependencies): - """ - Ninja tool function for writing a depfile. The depfile should include - the node path followed by all the dependent files in a makefile format. - - dependencies arg can be a list or a subst generator which returns a list. - """ - - depfile = os.path.join(get_path(env['NINJA_BUILDDIR']), str(node) + '.depfile') - - # subst_list will take in either a raw list or a subst callable which generates - # a list, and return a list of CmdStringHolders which can be converted into raw strings. - # If a raw list was passed in, then scons_list will make a list of lists from the original - # values and even subst items in the list if they are substitutable. Flatten will flatten - # the list in that case, to ensure for either input we have a list of CmdStringHolders. - deps_list = env.Flatten(env.subst_list(dependencies)) - - # Now that we have the deps in a list as CmdStringHolders, we can convert them into raw strings - # and make sure to escape the strings to handle spaces in paths. We also will sort the result - # keep the order of the list consistent. - escaped_depends = sorted([dep.escape(env.get("ESCAPE", lambda x: x)) for dep in deps_list]) - depfile_contents = str(node) + ": " + ' '.join(escaped_depends) - - need_rewrite = False - try: - with open(depfile, 'r') as f: - need_rewrite = (f.read() != depfile_contents) - except FileNotFoundError: - need_rewrite = True - - if need_rewrite: - os.makedirs(os.path.dirname(depfile) or '.', exist_ok=True) - with open(depfile, 'w') as f: - f.write(depfile_contents) - - - - - -class SConsToNinjaTranslator: - """Translates SCons Actions into Ninja build objects.""" - - def __init__(self, env): - self.env = env - self.func_handlers = { - # Skip conftest builders - "_createSource": ninja_noop, - # SCons has a custom FunctionAction that just makes sure the - # target isn't static. We let the commands that ninja runs do - # this check for us. - "SharedFlagChecker": ninja_noop, - # The install builder is implemented as a function action. - # TODO: use command action #3573 - "installFunc": _install_action_function, - "MkdirFunc": _mkdir_action_function, - "LibSymlinksActionFunction": _lib_symlink_action_function, - "Copy" : _copy_action_function - } - - self.loaded_custom = False - - # pylint: disable=too-many-return-statements - def action_to_ninja_build(self, node, action=None): - """Generate build arguments dictionary for node.""" - if not self.loaded_custom: - self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) - self.loaded_custom = True - - if node.builder is None: - return None - - if action is None: - action = node.builder.action - - if node.env and node.env.get("NINJA_SKIP"): - return None - - build = {} - env = node.env if node.env else self.env - - # Ideally this should never happen, and we do try to filter - # Ninja builders out of being sources of ninja builders but I - # can't fix every DAG problem so we just skip ninja_builders - # if we find one - global NINJA_STATE - if NINJA_STATE.ninja_file == str(node): - build = None - elif isinstance(action, SCons.Action.FunctionAction): - build = self.handle_func_action(node, action) - elif isinstance(action, SCons.Action.LazyAction): - # pylint: disable=protected-access - action = action._generate_cache(env) - build = self.action_to_ninja_build(node, action=action) - elif isinstance(action, SCons.Action.ListAction): - build = self.handle_list_action(node, action) - elif isinstance(action, COMMAND_TYPES): - build = get_command(env, node, action) - else: - raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) - - if build is not None: - build["order_only"] = get_order_only(node) - - #TODO: WPD Is this testing the filename to verify it's a configure context generated file? - if not node.is_conftest(): - node_callback = node.check_attributes("ninja_build_callback") - if callable(node_callback): - node_callback(env, node, build) - - return build - - def handle_func_action(self, node, action): - """Determine how to handle the function action.""" - name = action.function_name() - # This is the name given by the Subst/Textfile builders. So return the - # node to indicate that SCons is required. We skip sources here because - # dependencies don't really matter when we're going to shove these to - # the bottom of ninja's DAG anyway and Textfile builders can have text - # content as their source which doesn't work as an implicit dep in - # ninja. - if name == 'ninja_builder': - return None - - handler = self.func_handlers.get(name, None) - if handler is not None: - return handler(node.env if node.env else self.env, node) - elif name == "ActionCaller": - action_to_call = str(action).split('(')[0].strip() - handler = self.func_handlers.get(action_to_call, None) - if handler is not None: - return handler(node.env if node.env else self.env, node) - - SCons.Warnings.SConsWarning( - "Found unhandled function action {}, " - " generating scons command to build\n" - "Note: this is less efficient than Ninja," - " you can write your own ninja build generator for" - " this function using NinjaRegisterFunctionHandler".format(name) - ) - - return { - "rule": "TEMPLATE", - "order_only": get_order_only(node), - "outputs": get_outputs(node), - "inputs": get_inputs(node), - "implicit": get_dependencies(node, skip_sources=True), - } - - # pylint: disable=too-many-branches - def handle_list_action(self, node, action): - """TODO write this comment""" - results = [ - self.action_to_ninja_build(node, action=act) - for act in action.list - if act is not None - ] - results = [ - result for result in results if result is not None and result["outputs"] - ] - if not results: - return None - - # No need to process the results if we only got a single result - if len(results) == 1: - return results[0] - - all_outputs = list({output for build in results for output in build["outputs"]}) - dependencies = list({dep for build in results for dep in build["implicit"]}) - - if results[0]["rule"] == "CMD" or results[0]["rule"] == "GENERATED_CMD": - cmdline = "" - for cmd in results: - - # Occasionally a command line will expand to a - # whitespace only string (i.e. ' '). Which is not a - # valid command but does not trigger the empty command - # condition if not cmdstr. So here we strip preceding - # and proceeding whitespace to make strings like the - # above become empty strings and so will be skipped. - cmdstr = cmd["variables"]["cmd"].strip() - if not cmdstr: - continue - - # Skip duplicate commands - if cmdstr in cmdline: - continue - - if cmdline: - cmdline += " && " - - cmdline += cmdstr - - # Remove all preceding and proceeding whitespace - cmdline = cmdline.strip() - - # Make sure we didn't generate an empty cmdline - if cmdline: - ninja_build = { - "outputs": all_outputs, - "rule": get_rule(node, "GENERATED_CMD"), - "variables": { - "cmd": cmdline, - "env": get_command_env(node.env if node.env else self.env), - }, - "implicit": dependencies, - } - - if node.env and node.env.get("NINJA_POOL", None) is not None: - ninja_build["pool"] = node.env["pool"] - - return ninja_build - - elif results[0]["rule"] == "phony": - return { - "outputs": all_outputs, - "rule": "phony", - "implicit": dependencies, - } - - elif results[0]["rule"] == "INSTALL": - return { - "outputs": all_outputs, - "rule": get_rule(node, "INSTALL"), - "inputs": get_inputs(node), - "implicit": dependencies, - } - - raise Exception("Unhandled list action with rule: " + results[0]["rule"]) - - -# pylint: disable=too-many-instance-attributes -class NinjaState: - """Maintains state of Ninja build system as it's translated from SCons.""" - - def __init__(self, env, ninja_file, writer_class): - self.env = env - self.ninja_file = ninja_file - self.ninja_bin_path = '' - self.writer_class = writer_class - self.__generated = False - self.translator = SConsToNinjaTranslator(env) - self.generated_suffixes = env.get("NINJA_GENERATED_SOURCE_SUFFIXES", []) - - # List of generated builds that will be written at a later stage - self.builds = dict() - - # List of targets for which we have generated a build. This - # allows us to take multiple Alias nodes as sources and to not - # fail to build if they have overlapping targets. - self.built = set() - - # SCons sets this variable to a function which knows how to do - # shell quoting on whatever platform it's run on. Here we use it - # to make the SCONS_INVOCATION variable properly quoted for things - # like CCFLAGS - escape = env.get("ESCAPE", lambda x: x) - - # if SCons was invoked from python, we expect the first arg to be the scons.py - # script, otherwise scons was invoked from the scons script - python_bin = '' - if os.path.basename(sys.argv[0]) == 'scons.py': - python_bin = escape(sys.executable) - self.variables = { - "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", - "SCONS_INVOCATION": '{} {} --disable-ninja __NINJA_NO=1 $out'.format( - python_bin, - " ".join( - [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] - ), - ), - "SCONS_INVOCATION_W_TARGETS": "{} {} --disable-ninja".format( - python_bin, " ".join([escape(arg) for arg in sys.argv]) - ), - # This must be set to a global default per: - # https://ninja-build.org/manual.html#_deps - # English Visual Studio will have the default below, - # otherwise the user can define the variable in the first environment - # that initialized ninja tool - "msvc_deps_prefix": env.get("NINJA_MSVC_DEPS_PREFIX", "Note: including file:") - } - - self.rules = { - "CMD": { - "command": "cmd /c $env$cmd $in $out" if sys.platform == "win32" else "$env$cmd $in $out", - "description": "Building $out", - "pool": "local_pool", - }, - "GENERATED_CMD": { - "command": "cmd /c $env$cmd" if sys.platform == "win32" else "$env$cmd", - "description": "Building $out", - "pool": "local_pool", - }, - # We add the deps processing variables to this below. We - # don't pipe these through cmd.exe on Windows because we - # use this to generate a compile_commands.json database - # which can't use the shell command as it's compile - # command. - "CC": { - "command": "$env$CC @$out.rsp", - "description": "Compiling $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - }, - "CXX": { - "command": "$env$CXX @$out.rsp", - "description": "Compiling $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - }, - "LINK": { - "command": "$env$LINK @$out.rsp", - "description": "Linking $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - "pool": "local_pool", - }, - # Ninja does not automatically delete the archive before - # invoking ar. The ar utility will append to an existing archive, which - # can cause duplicate symbols if the symbols moved between object files. - # Native SCons will perform this operation so we need to force ninja - # to do the same. See related for more info: - # https://jira.mongodb.org/browse/SERVER-49457 - "AR": { - "command": "{}$env$AR @$out.rsp".format( - '' if sys.platform == "win32" else "rm -f $out && " - ), - "description": "Archiving $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - "pool": "local_pool", - }, - "SYMLINK": { - "command": ( - "cmd /c mklink $out $in" - if sys.platform == "win32" - else "ln -s $in $out" - ), - "description": "Symlink $in -> $out", - }, - "INSTALL": { - "command": "$COPY $in $out", - "description": "Install $out", - "pool": "install_pool", - # On Windows cmd.exe /c copy does not always correctly - # update the timestamp on the output file. This leads - # to a stuck constant timestamp in the Ninja database - # and needless rebuilds. - # - # Adding restat here ensures that Ninja always checks - # the copy updated the timestamp and that Ninja has - # the correct information. - "restat": 1, - }, - "TEMPLATE": { - "command": "$SCONS_INVOCATION $out", - "description": "Rendering $SCONS_INVOCATION $out", - "pool": "scons_pool", - "restat": 1, - }, - "SCONS": { - "command": "$SCONS_INVOCATION $out", - "description": "$SCONS_INVOCATION $out", - "pool": "scons_pool", - # restat - # if present, causes Ninja to re-stat the command's outputs - # after execution of the command. Each output whose - # modification time the command did not change will be - # treated as though it had never needed to be built. This - # may cause the output's reverse dependencies to be removed - # from the list of pending build actions. - # - # We use restat any time we execute SCons because - # SCons calls in Ninja typically create multiple - # targets. But since SCons is doing it's own up to - # date-ness checks it may only update say one of - # them. Restat will find out which of the multiple - # build targets did actually change then only rebuild - # those targets which depend specifically on that - # output. - "restat": 1, - }, - "REGENERATE": { - "command": "$SCONS_INVOCATION_W_TARGETS", - "description": "Regenerating $out", - "generator": 1, - "depfile": os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile'), - # Console pool restricts to 1 job running at a time, - # it additionally has some special handling about - # passing stdin, stdout, etc to process in this pool - # that we need for SCons to behave correctly when - # regenerating Ninja - "pool": "console", - # Again we restat in case Ninja thought the - # build.ninja should be regenerated but SCons knew - # better. - "restat": 1, - }, - } - - if env['PLATFORM'] == 'darwin' and env['AR'] == 'ar': - self.rules["AR"] = { - "command": "rm -f $out && $env$AR $rspc", - "description": "Archiving $out", - "pool": "local_pool", - } - - self.pools = { - "local_pool": self.env.GetOption("num_jobs"), - "install_pool": self.env.GetOption("num_jobs") / 2, - "scons_pool": 1, - } - - for rule in ["CC", "CXX"]: - if env["PLATFORM"] == "win32": - self.rules[rule]["deps"] = "msvc" - else: - self.rules[rule]["deps"] = "gcc" - self.rules[rule]["depfile"] = "$out.d" - - def add_build(self, node): - if not node.has_builder(): - return False - - if isinstance(node, SCons.Node.Alias.Alias): - build = alias_to_ninja_build(node) - else: - build = self.translator.action_to_ninja_build(node) - - # Some things are unbuild-able or need not be built in Ninja - if build is None: - return False - - node_string = str(node) - if node_string in self.builds: - raise Exception("Node {} added to ninja build state more than once".format(node_string)) - self.builds[node_string] = build - self.built.update(build["outputs"]) - return True - - # TODO: rely on SCons to tell us what is generated source - # or some form of user scanner maybe (Github Issue #3624) - def is_generated_source(self, output): - """Check if output ends with a known generated suffix.""" - _, suffix = splitext(output) - return suffix in self.generated_suffixes - - def has_generated_sources(self, output): - """ - Determine if output indicates this is a generated header file. - """ - for generated in output: - if self.is_generated_source(generated): - return True - return False - - # pylint: disable=too-many-branches,too-many-locals - def generate(self): - """ - Generate the build.ninja. - - This should only be called once for the lifetime of this object. - """ - if self.__generated: - return - - self.rules.update(self.env.get(NINJA_RULES, {})) - self.pools.update(self.env.get(NINJA_POOLS, {})) - - content = io.StringIO() - ninja = self.writer_class(content, width=100) - - ninja.comment("Generated by scons. DO NOT EDIT.") - - ninja.variable("builddir", get_path(self.env['NINJA_BUILDDIR'])) - - for pool_name, size in self.pools.items(): - ninja.pool(pool_name, size) - - for var, val in self.variables.items(): - ninja.variable(var, val) - - for rule, kwargs in self.rules.items(): - ninja.rule(rule, **kwargs) - - generated_source_files = sorted({ - output - # First find builds which have header files in their outputs. - for build in self.builds.values() - if self.has_generated_sources(build["outputs"]) - for output in build["outputs"] - # Collect only the header files from the builds with them - # in their output. We do this because is_generated_source - # returns True if it finds a header in any of the outputs, - # here we need to filter so we only have the headers and - # not the other outputs. - if self.is_generated_source(output) - }) - - if generated_source_files: - ninja.build( - outputs="_generated_sources", - rule="phony", - implicit=generated_source_files - ) - - template_builders = [] - - for build in [self.builds[key] for key in sorted(self.builds.keys())]: - if build["rule"] == "TEMPLATE": - template_builders.append(build) - continue - - if "implicit" in build: - build["implicit"].sort() - - # Don't make generated sources depend on each other. We - # have to check that none of the outputs are generated - # sources and none of the direct implicit dependencies are - # generated sources or else we will create a dependency - # cycle. - if ( - generated_source_files - and not build["rule"] == "INSTALL" - and set(build["outputs"]).isdisjoint(generated_source_files) - and set(build.get("implicit", [])).isdisjoint(generated_source_files) - ): - - # Make all non-generated source targets depend on - # _generated_sources. We use order_only for generated - # sources so that we don't rebuild the world if one - # generated source was rebuilt. We just need to make - # sure that all of these sources are generated before - # other builds. - order_only = build.get("order_only", []) - order_only.append("_generated_sources") - build["order_only"] = order_only - if "order_only" in build: - build["order_only"].sort() - - # When using a depfile Ninja can only have a single output - # but SCons will usually have emitted an output for every - # thing a command will create because it's caching is much - # more complex than Ninja's. This includes things like DWO - # files. Here we make sure that Ninja only ever sees one - # target when using a depfile. It will still have a command - # that will create all of the outputs but most targets don't - # depend directly on DWO files and so this assumption is safe - # to make. - rule = self.rules.get(build["rule"]) - - # Some rules like 'phony' and other builtins we don't have - # listed in self.rules so verify that we got a result - # before trying to check if it has a deps key. - # - # Anything using deps or rspfile in Ninja can only have a single - # output, but we may have a build which actually produces - # multiple outputs which other targets can depend on. Here we - # slice up the outputs so we have a single output which we will - # use for the "real" builder and multiple phony targets that - # match the file names of the remaining outputs. This way any - # build can depend on any output from any build. - # - # We assume that the first listed output is the 'key' - # output and is stably presented to us by SCons. For - # instance if -gsplit-dwarf is in play and we are - # producing foo.o and foo.dwo, we expect that outputs[0] - # from SCons will be the foo.o file and not the dwo - # file. If instead we just sorted the whole outputs array, - # we would find that the dwo file becomes the - # first_output, and this breaks, for instance, header - # dependency scanning. - if rule is not None and (rule.get("deps") or rule.get("rspfile")): - first_output, remaining_outputs = ( - build["outputs"][0], - build["outputs"][1:], - ) - - if remaining_outputs: - ninja.build( - outputs=sorted(remaining_outputs), rule="phony", implicit=first_output, - ) - - build["outputs"] = first_output - - # Optionally a rule can specify a depfile, and SCons can generate implicit - # dependencies into the depfile. This allows for dependencies to come and go - # without invalidating the ninja file. The depfile was created in ninja specifically - # for dealing with header files appearing and disappearing across rebuilds, but it can - # be repurposed for anything, as long as you have a way to regenerate the depfile. - # More specific info can be found here: https://ninja-build.org/manual.html#_depfile - if rule is not None and rule.get('depfile') and build.get('deps_files'): - path = build['outputs'] if SCons.Util.is_List(build['outputs']) else [build['outputs']] - generate_depfile(self.env, path[0], build.pop('deps_files', [])) - - if "inputs" in build: - build["inputs"].sort() - - ninja.build(**build) - - template_builds = dict() - for template_builder in template_builders: - - # Special handling for outputs and implicit since we need to - # aggregate not replace for each builder. - for agg_key in ["outputs", "implicit", "inputs"]: - new_val = template_builds.get(agg_key, []) - - # Use pop so the key is removed and so the update - # below will not overwrite our aggregated values. - cur_val = template_builder.pop(agg_key, []) - if is_List(cur_val): - new_val += cur_val - else: - new_val.append(cur_val) - template_builds[agg_key] = new_val - - # Collect all other keys - template_builds.update(template_builder) - - if template_builds.get("outputs", []): - ninja.build(**template_builds) - - # We have to glob the SCons files here to teach the ninja file - # how to regenerate itself. We'll never see ourselves in the - # DAG walk so we can't rely on action_to_ninja_build to - # generate this rule even though SCons should know we're - # dependent on SCons files. - # - # The REGENERATE rule uses depfile, so we need to generate the depfile - # in case any of the SConscripts have changed. The depfile needs to be - # path with in the build and the passed ninja file is an abspath, so - # we will use SCons to give us the path within the build. Normally - # generate_depfile should not be called like this, but instead be called - # through the use of custom rules, and filtered out in the normal - # list of build generation about. However, because the generate rule - # is hardcoded here, we need to do this generate_depfile call manually. - ninja_file_path = self.env.File(self.ninja_file).path - generate_depfile( - self.env, - ninja_file_path, - self.env['NINJA_REGENERATE_DEPS'] - ) - - ninja.build( - ninja_file_path, - rule="REGENERATE", - implicit=[__file__], - ) - - # If we ever change the name/s of the rules that include - # compile commands (i.e. something like CC) we will need to - # update this build to reflect that complete list. - ninja.build( - "compile_commands.json", - rule="CMD", - pool="console", - implicit=[str(self.ninja_file)], - variables={ - "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( - # NINJA_COMPDB_EXPAND - should only be true for ninja - # This was added to ninja's compdb tool in version 1.9.0 (merged April 2018) - # https://github.com/ninja-build/ninja/pull/1223 - # TODO: add check in generate to check version and enable this by default if it's available. - self.ninja_bin_path, str(self.ninja_file), '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' - ) - }, - ) - - ninja.build( - "compiledb", rule="phony", implicit=["compile_commands.json"], - ) - - # Look in SCons's list of DEFAULT_TARGETS, find the ones that - # we generated a ninja build rule for. - scons_default_targets = [ - get_path(tgt) - for tgt in SCons.Script.DEFAULT_TARGETS - if get_path(tgt) in self.built - ] - - # If we found an overlap between SCons's list of default - # targets and the targets we created ninja builds for then use - # those as ninja's default as well. - if scons_default_targets: - ninja.default(" ".join(scons_default_targets)) - - with open(str(self.ninja_file), "w") as build_ninja: - build_ninja.write(content.getvalue()) - - self.__generated = True - - - - - -def get_comstr(env, action, targets, sources): - """Get the un-substituted string for action.""" - # Despite being having "list" in it's name this member is not - # actually a list. It's the pre-subst'd string of the command. We - # use it to determine if the command we're about to generate needs - # to use a custom Ninja rule. By default this redirects CC, CXX, - # AR, SHLINK, and LINK commands to their respective rules but the - # user can inject custom Ninja rules and tie them to commands by - # using their pre-subst'd string. - if hasattr(action, "process"): - return action.cmd_list - - return action.genstring(targets, sources, env) - - -def get_command_env(env): - """ - Return a string that sets the environment for any environment variables that - differ between the OS environment and the SCons command ENV. - - It will be compatible with the default shell of the operating system. - """ - try: - return env["NINJA_ENV_VAR_CACHE"] - except KeyError: - pass - - # Scan the ENV looking for any keys which do not exist in - # os.environ or differ from it. We assume if it's a new or - # differing key from the process environment then it's - # important to pass down to commands in the Ninja file. - ENV = get_default_ENV(env) - scons_specified_env = { - key: value - for key, value in ENV.items() - # TODO: Remove this filter, unless there's a good reason to keep. SCons's behavior shouldn't depend on shell's. - if key not in os.environ or os.environ.get(key, None) != value - } - - windows = env["PLATFORM"] == "win32" - command_env = "" - for key, value in scons_specified_env.items(): - # Ensure that the ENV values are all strings: - if is_List(value): - # If the value is a list, then we assume it is a - # path list, because that's a pretty common list-like - # value to stick in an environment variable: - value = flatten_sequence(value) - value = joinpath(map(str, value)) - else: - # If it isn't a string or a list, then we just coerce - # it to a string, which is the proper way to handle - # Dir and File instances and will produce something - # reasonable for just about everything else: - value = str(value) - - if windows: - command_env += "set '{}={}' && ".format(key, value) - else: - # We address here *only* the specific case that a user might have - # an environment variable which somehow gets included and has - # spaces in the value. These are escapes that Ninja handles. This - # doesn't make builds on paths with spaces (Ninja and SCons issues) - # nor expanding response file paths with spaces (Ninja issue) work. - value = value.replace(r' ', r'$ ') - command_env += "export {}='{}';".format(key, value) - - env["NINJA_ENV_VAR_CACHE"] = command_env - return command_env def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom_env={}): @@ -891,143 +109,6 @@ def get_response_file_command(env, node, action, targets, sources, executor=None return get_response_file_command -def generate_command(env, node, action, targets, sources, executor=None): - # Actions like CommandAction have a method called process that is - # used by SCons to generate the cmd_line they need to run. So - # check if it's a thing like CommandAction and call it if we can. - if hasattr(action, "process"): - cmd_list, _, _ = action.process(targets, sources, env, executor=executor) - cmd = _string_from_cmd_list(cmd_list[0]) - else: - # Anything else works with genstring, this is most commonly hit by - # ListActions which essentially call process on all of their - # commands and concatenate it for us. - genstring = action.genstring(targets, sources, env) - if executor is not None: - cmd = env.subst(genstring, executor=executor) - else: - cmd = env.subst(genstring, targets, sources) - - cmd = cmd.replace("\n", " && ").strip() - if cmd.endswith("&&"): - cmd = cmd[0:-2].strip() - - # Escape dollars as necessary - return cmd.replace("$", "$$") - - -def get_generic_shell_command(env, node, action, targets, sources, executor=None): - return ( - "CMD", - { - # TODO: Why is executor passed in and then ignored below? (bdbaddog) - "cmd": generate_command(env, node, action, targets, sources, executor=None), - "env": get_command_env(env), - }, - # Since this function is a rule mapping provider, it must return a list of dependencies, - # and usually this would be the path to a tool, such as a compiler, used for this rule. - # However this function is to generic to be able to reliably extract such deps - # from the command, so we return a placeholder empty list. It should be noted that - # generally this function will not be used solely and is more like a template to generate - # the basics for a custom provider which may have more specific options for a provider - # function for a custom NinjaRuleMapping. - [] - ) - - -def get_command(env, node, action): # pylint: disable=too-many-branches - """Get the command to execute for node.""" - if node.env: - sub_env = node.env - else: - sub_env = env - executor = node.get_executor() - tlist, slist = get_targets_sources(node) - - # Generate a real CommandAction - if isinstance(action, SCons.Action.CommandGeneratorAction): - # pylint: disable=protected-access - action = action._generate(tlist, slist, sub_env, 1, executor=executor) - - variables = {} - - comstr = get_comstr(sub_env, action, tlist, slist) - if not comstr: - return None - - provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) - rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) - - # Get the dependencies for all targets - implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) - - # Now add in the other dependencies related to the command, - # e.g. the compiler binary. The ninja rule can be user provided so - # we must do some validation to resolve the dependency path for ninja. - for provider_dep in provider_deps: - - provider_dep = sub_env.subst(provider_dep) - if not provider_dep: - continue - - # If the tool is a node, then SCons will resolve the path later, if its not - # a node then we assume it generated from build and make sure it is existing. - if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep): - implicit.append(provider_dep) - continue - - # in some case the tool could be in the local directory and be suppled without the ext - # such as in windows, so append the executable suffix and check. - prog_suffix = sub_env.get('PROGSUFFIX', '') - provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix - if os.path.exists(provider_dep_ext): - implicit.append(provider_dep_ext) - continue - - # Many commands will assume the binary is in the path, so - # we accept this as a possible input from a given command. - - provider_dep_abspath = sub_env.WhereIs(provider_dep) or sub_env.WhereIs(provider_dep, path=os.environ["PATH"]) - if provider_dep_abspath: - implicit.append(provider_dep_abspath) - continue - - # Possibly these could be ignore and the build would still work, however it may not always - # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided - # ninja rule. - raise Exception("Could not resolve path for %s dependency on node '%s'" % (provider_dep, node)) - - ninja_build = { - "order_only": get_order_only(node), - "outputs": get_outputs(node), - "inputs": get_inputs(node), - "implicit": implicit, - "rule": get_rule(node, rule), - "variables": variables, - } - - # Don't use sub_env here because we require that NINJA_POOL be set - # on a per-builder call basis to prevent accidental strange - # behavior like env['NINJA_POOL'] = 'console' and sub_env can be - # the global Environment object if node.env is None. - # Example: - # - # Allowed: - # - # env.Command("ls", NINJA_POOL="ls_pool") - # - # Not allowed and ignored: - # - # env["NINJA_POOL"] = "ls_pool" - # env.Command("ls") - # - # TODO: Why not alloe env['NINJA_POOL'] ? (bdbaddog) - if node.env and node.env.get("NINJA_POOL", None) is not None: - ninja_build["pool"] = node.env["NINJA_POOL"] - - return ninja_build - - def ninja_builder(env, target, source): """Generate a build.ninja for source.""" if not isinstance(source, list): @@ -1109,8 +190,7 @@ def register_custom_handler(env, name, handler): def register_custom_rule_mapping(env, pre_subst_string, rule): """Register a function to call for a given rule.""" - global __NINJA_RULE_MAPPING - __NINJA_RULE_MAPPING[pre_subst_string] = rule + SCons.Tool.ninjaCommon.Globals.__NINJA_RULE_MAPPING[pre_subst_string] = rule def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): @@ -1190,6 +270,7 @@ def CheckNinjaCompdbExpand(env, context): context.Result(result) return result + def ninja_stat(_self, path): """ Eternally memoized stat call. @@ -1200,42 +281,29 @@ def ninja_stat(_self, path): change. For these reasons we patch SCons.Node.FS.LocalFS.stat to use our eternal memoized dictionary. """ - global NINJA_STAT_MEMO try: - return NINJA_STAT_MEMO[path] + return SCons.Tool.ninjaCommon.Globals.NINJA_STAT_MEMO[path] except KeyError: try: result = os.stat(path) except os.error: result = None - NINJA_STAT_MEMO[path] = result + SCons.Tool.ninjaCommon.Globals.NINJA_STAT_MEMO[path] = result return result -def ninja_noop(*_args, **_kwargs): - """ - A general purpose no-op function. - - There are many things that happen in SCons that we don't need and - also don't return anything. We use this to disable those functions - instead of creating multiple definitions of the same thing. - """ - return None - - def ninja_whereis(thing, *_args, **_kwargs): """Replace env.WhereIs with a much faster version""" - global NINJA_WHEREIS_MEMO # Optimize for success, this gets called significantly more often # when the value is already memoized than when it's not. try: - return NINJA_WHEREIS_MEMO[thing] + return SCons.Tool.ninjaCommon.Globals.NINJA_WHEREIS_MEMO[thing] except KeyError: # We do not honor any env['ENV'] or env[*] variables in the - # generated ninja ile. Ninja passes your raw shell environment + # generated ninja file. Ninja passes your raw shell environment # down to it's subprocess so the only sane option is to do the # same during generation. At some point, if and when we try to # upstream this, I'm sure a sticking point will be respecting @@ -1244,7 +312,7 @@ def ninja_whereis(thing, *_args, **_kwargs): # with shell quoting is nigh impossible. So I've decided to # cross that bridge when it's absolutely required. path = shutil.which(thing) - NINJA_WHEREIS_MEMO[thing] = path + SCons.Tool.ninjaCommon.Globals.NINJA_WHEREIS_MEMO[thing] = path return path @@ -1329,9 +397,10 @@ def ninja_emitter(target, source, env): def generate(env): """Generate the NINJA builders.""" - global ninja_builder_initialized - if not ninja_builder_initialized: - ninja_builder_initialized = True + global NINJA_STATE + + if not SCons.Tool.ninjaCommon.Globals.ninja_builder_initialized: + SCons.Tool.ninjaCommon.Globals.ninja_builder_initialized = True ninja_add_command_line_options() @@ -1343,8 +412,6 @@ def generate(env): env["DISABLE_AUTO_NINJA"] = GetOption('disable_execute_ninja') - global NINJA_STATE - env["NINJA_FILE_NAME"] = env.get("NINJA_FILE_NAME", "build.ninja") # Add the Ninja builder. @@ -1353,7 +420,6 @@ def generate(env): emitter=ninja_emitter) env.Append(BUILDERS={"Ninja": ninja_builder_obj}) - env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") env['NINJA_BUILDDIR'] = env.get("NINJA_BUILDDIR", env.Dir(".ninja").path) @@ -1561,20 +627,7 @@ def robust_rule_mapping(var, rule, tool): if NINJA_STATE is None: NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) - NINJA_STATE.ninja_bin_path = env.get('NINJA') - if not NINJA_STATE.ninja_bin_path: - # default to using ninja installed with python module - ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' - NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - ninja_bin)) - if not os.path.exists(NINJA_STATE.ninja_bin_path): - # couldn't find it, just give the bin name and hope - # its in the path later - NINJA_STATE.ninja_bin_path = ninja_bin + # TODO: this is hacking into scons, preferable if there were a less intrusive way # We will subvert the normal builder execute to make sure all the ninja file is dependent @@ -1607,7 +660,6 @@ def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): # file once we have the upstream support for referencing SConscripts as File # nodes. def ninja_execute(self): - global NINJA_STATE target = self.targets[0] if target.check_attributes('ninja_file') is None or not target.is_conftest: From 70b783ca59bfea4276e3bcb2b3becee20ca35eaf Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 10 Apr 2021 16:59:22 -0700 Subject: [PATCH 064/163] Address sider issues --- SCons/Tool/ninjaCommon/Util.py | 2 +- SCons/Tool/ninjaCommon/__init__.py | 12 ++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/SCons/Tool/ninjaCommon/Util.py b/SCons/Tool/ninjaCommon/Util.py index bb20d80a3f..cc30552b50 100644 --- a/SCons/Tool/ninjaCommon/Util.py +++ b/SCons/Tool/ninjaCommon/Util.py @@ -26,7 +26,7 @@ import SCons from SCons.Action import get_default_ENV, _string_from_cmd_list from SCons.Script import AddOption -from SCons.Tool.ninjaCommon import __NINJA_RULE_MAPPING +from SCons.Tool.ninjaCommon.Globals import __NINJA_RULE_MAPPING from SCons.Util import is_List, flatten_sequence diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index a2745c5827..4c5b08eb2c 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -35,16 +35,12 @@ import SCons import SCons.Tool.ninjaCommon.Globals -from SCons.Action import _string_from_cmd_list from SCons.Script import GetOption -from SCons.Util import is_List -import SCons.Tool.ninjaCommon.Globals -from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS, __NINJA_RULE_MAPPING +from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS from .NinjaState import NinjaState -from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function -from .Util import ninja_add_command_line_options, alias_to_ninja_build, \ - get_targets_sources, get_path, ninja_noop, get_command, get_command_env, get_comstr, get_generic_shell_command, \ +from .Util import ninja_add_command_line_options, \ + get_path, ninja_noop, get_command, get_command_env, get_comstr, get_generic_shell_command, \ generate_command NINJA_STATE = None @@ -405,7 +401,7 @@ def generate(env): ninja_add_command_line_options() try: - import ninja + import ninja # noqa: F401 except ImportError: SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return From 791e7939e34c3b8333a1221002ff894784d4e2e3 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 10 Apr 2021 17:09:03 -0700 Subject: [PATCH 065/163] Fix ninja.xml syntax error --- SCons/Tool/ninjaCommon/ninja.xml | 2 + doc/generated/builders.gen | 65 ++++- doc/generated/builders.mod | 4 + doc/generated/functions.gen | 360 ++++++++++++++++-------- doc/generated/tools.gen | 11 +- doc/generated/tools.mod | 2 + doc/generated/variables.gen | 464 ++++++++++++++++++++----------- doc/generated/variables.mod | 38 ++- 8 files changed, 650 insertions(+), 296 deletions(-) diff --git a/SCons/Tool/ninjaCommon/ninja.xml b/SCons/Tool/ninjaCommon/ninja.xml index 8f7de362bf..c28d697953 100644 --- a/SCons/Tool/ninjaCommon/ninja.xml +++ b/SCons/Tool/ninjaCommon/ninja.xml @@ -176,9 +176,11 @@ See its __doc__ string for a discussion of the format. This propagates directly into the generated Ninja.build file. From Ninja's docs:
+ builddir A directory for some Ninja output files. ... (You can also store other build output in this directory.) +
diff --git a/doc/generated/builders.gen b/doc/generated/builders.gen index bfb1ff117c..e2b98fe2c0 100644 --- a/doc/generated/builders.gen +++ b/doc/generated/builders.gen @@ -1217,6 +1217,59 @@ env.MSVSSolution( + + Ninja() + env.Ninja() + + &b-Ninja; is a special builder which + adds a target to create a ninja build file. + The builder does not require any source files to be specified, + + + + If called with no arguments, + the builder will default to a target name of + ninja.build. + + + If called with a single positional argument, + &scons; will "deduce" the target name from that source + argument, giving it the same name, and then + ignore the source. + This is the usual way to call the builder if a + non-default target name is wanted. + + + If called with either the + target= + or source= keyword arguments, + the value of the argument is taken as the target name. + If called with both, the + target= + value is used and source= is ignored. + If called with multiple sources, + the source list will be ignored, + since there is no way to deduce what the intent was; + in this case the default target name will be used. + + + + You must load the &t-ninja; tool prior to specifying + any part of your build or some source/output + files will not show up in the compilation database. + + + To use this tool you must install pypi's ninja + package. + This can be done via + pip install ninja + + + + Available since &scons; 4.2. + + + Object() env.Object() @@ -1322,7 +1375,7 @@ env.Package( SUMMARY="balalalalal", DESCRIPTION="this should be really really long", X_RPM_GROUP="Application/fu", - SOURCE_URL="http://foo.org/foo-1.2.3.tar.gz", + SOURCE_URL="https://foo.org/foo-1.2.3.tar.gz", ) @@ -1333,7 +1386,7 @@ since it is not under the project top directory. However, since no source is specified to the &b-Package; builder, it is selected for packaging by the default sources rule. -Since packaging is done using &cv-PACKAGEROOT;, no write is +Since packaging is done using &cv-link-PACKAGEROOT;, no write is actually done to the system's /bin directory, and the target will be selected since after rebasing to underneath &cv-PACKAGEROOT; it is now under @@ -1347,13 +1400,13 @@ the top directory of the project. env.PCH() Builds a Microsoft Visual C++ precompiled header. -Calling this builder method +Calling this builder returns a list of two targets: the PCH as the first element, and the object file as the second element. Normally the object file is ignored. -This builder method is only +This builder is only provided when Microsoft Visual C++ is being used as the compiler. -The PCH builder method is generally used in -conjunction with the PCH construction variable to force object files to use +The &b-PCH; builder is generally used in +conjunction with the &cv-link-PCH; construction variable to force object files to use the precompiled header: diff --git a/doc/generated/builders.mod b/doc/generated/builders.mod index 4835118f69..3542b99110 100644 --- a/doc/generated/builders.mod +++ b/doc/generated/builders.mod @@ -37,6 +37,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. MOFiles"> MSVSProject"> MSVSSolution"> +Ninja"> Object"> Package"> PCH"> @@ -94,6 +95,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. env.MOFiles"> env.MSVSProject"> env.MSVSSolution"> +env.Ninja"> env.Object"> env.Package"> env.PCH"> @@ -157,6 +159,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. MOFiles"> MSVSProject"> MSVSSolution"> +Ninja"> Object"> Package"> PCH"> @@ -214,6 +217,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. env.MOFiles"> env.MSVSProject"> env.MSVSSolution"> +env.Ninja"> env.Object"> env.Package"> env.PCH"> diff --git a/doc/generated/functions.gen b/doc/generated/functions.gen index 61b1fedee3..c9ffc86d2e 100644 --- a/doc/generated/functions.gen +++ b/doc/generated/functions.gen @@ -425,37 +425,153 @@ Multiple targets can be passed in to a single call to env.Append(key=val, [...]) -Appends the specified keyword arguments -to the end of construction variables in the environment. -If the Environment does not have -the specified construction variable, -it is simply added to the environment. -If the values of the construction variable -and the keyword argument are the same type, -then the two values will be simply added together. -Otherwise, the construction variable -and the value of the keyword argument -are both coerced to lists, -and the lists are added together. -(See also the &Prepend; method). +Intelligently append values to &consvars; in the &consenv; +named by env. +The &consvars; and values to add to them are passed as +key=val pairs (Python keyword arguments). +&f-env-Append; is designed to allow adding values +without normally having to know the data type of an existing &consvar;. +Regular Python syntax can also be used to manipulate the &consvar;, +but for that you must know the type of the &consvar;: +for example, different Python syntax is needed to combine +a list of values with a single string value, or vice versa. +Some pre-defined &consvars; do have type expectations +based on how &SCons; will use them, +for example &cv-link-CPPDEFINES; is normally a string or a list of strings, +but can be a string, +a list of strings, +a list of tuples, +or a dictionary, while &cv-link-LIBEMITTER; +would expect a callable or list of callables, +and &cv-link-BUILDERS; would expect a mapping type. +Consult the documentation for the various &consvars; for more details. + + + +The following descriptions apply to both the append +and prepend functions, the only difference being +the insertion point of the added values. + + +If env. does not have a &consvar; +indicated by key, +val +is added to the environment under that key as-is. + + + +val can be almost any type, +and &SCons; will combine it with an existing value into an appropriate type, +but there are a few special cases to be aware of. +When two strings are combined, +the result is normally a new string, +with the caller responsible for supplying any needed separation. +The exception to this is the &consvar; &cv-link-CPPDEFINES;, +in which each item will be postprocessed by adding a prefix +and/or suffix, +so the contents are treated as a list of strings, that is, +adding a string will result in a separate string entry, +not a combined string. For &cv-CPPDEFINES; as well as +for &cv-link-LIBS;, and the various *PATH +variables, &SCons; will supply the compiler-specific +syntax (e.g. adding a -D or /D +prefix for &cv-CPPDEFINES;), so this syntax should be omitted when +adding values to these variables. +Example (gcc syntax shown in the expansion of &CPPDEFINES;): + +env = Environment(CXXFLAGS="-std=c11", CPPDEFINES="RELEASE") +print("CXXFLAGS={}, CPPDEFINES={}".format(env['CXXFLAGS'], env['CPPDEFINES'])) +# notice including a leading space in CXXFLAGS value +env.Append(CXXFLAGS=" -O", CPPDEFINES="EXTRA") +print("CXXFLAGS={}, CPPDEFINES={}".format(env['CXXFLAGS'], env['CPPDEFINES'])) +print("CPPDEFINES will expand to {}".format(env.subst("$_CPPDEFFLAGS"))) + + + +$ scons -Q +CXXFLAGS=-std=c11, CPPDEFINES=RELEASE +CXXFLAGS=-std=c11 -O, CPPDEFINES=['RELEASE', 'EXTRA'] +CPPDEFINES will expand to -DRELEASE -DEXTRA +scons: `.' is up to date. + + -Example: +Because &cv-link-CPPDEFINES; is intended to +describe C/C++ pre-processor macro definitions, +it accepts additional syntax. +Preprocessor macros can be valued, or un-valued, as in +-DBAR=1 or +-DFOO. +The macro can be be supplied as a complete string including the value, +or as a tuple (or list) of macro, value, or as a dictionary. +Example (again gcc syntax in the expanded defines): -env.Append(CCFLAGS = ' -g', FOO = ['foo.yyy']) +env = Environment(CPPDEFINES="FOO") +print("CPPDEFINES={}".format(env['CPPDEFINES'])) +env.Append(CPPDEFINES="BAR=1") +print("CPPDEFINES={}".format(env['CPPDEFINES'])) +env.Append(CPPDEFINES=("OTHER", 2)) +print("CPPDEFINES={}".format(env['CPPDEFINES'])) +env.Append(CPPDEFINES={"EXTRA": "arg"}) +print("CPPDEFINES={}".format(env['CPPDEFINES'])) +print("CPPDEFINES will expand to {}".format(env.subst("$_CPPDEFFLAGS"))) + + +$ scons -Q +CPPDEFINES=FOO +CPPDEFINES=['FOO', 'BAR=1'] +CPPDEFINES=['FOO', 'BAR=1', ('OTHER', 2)] +CPPDEFINES=['FOO', 'BAR=1', ('OTHER', 2), {'EXTRA': 'arg'}] +CPPDEFINES will expand to -DFOO -DBAR=1 -DOTHER=2 -DEXTRA=arg +scons: `.' is up to date. + + + +Adding a string val +to a dictonary &consvar; will enter +val as the key in the dict, +and None as its value. +Using a tuple type to supply a key + value only works +for the special case of &cv-link-CPPDEFINES; +described above. + + + +Although most combinations of types work without +needing to know the details, some combinations +do not make sense and a Python exception will be raised. + + + +When using &f-env-Append; to modify &consvars; +which are path specifications (normally, +those names which end in PATH), +it is recommended to add the values as a list of strings, +even if there is only a single string to add. +The same goes for adding library names to &cv-LIBS;. + + + +env.Append(CPPPATH=["#/include"]) + + + +See also &f-link-env-AppendUnique;, +&f-link-env-Prepend; and &f-link-env-PrependUnique;. + + - env.AppendENVPath(name, newpath, [envname, sep, delete_existing]) + env.AppendENVPath(name, newpath, [envname, sep, delete_existing=False]) -This appends new path elements to the given path in the -specified external environment -(ENV -by default). +Append new path elements to the given path in the +specified external environment (&cv-link-ENV; by default). This will only add any particular path once (leaving the last one it encounters and ignoring the rest, to preserve path order), @@ -472,7 +588,7 @@ string, in which case a list will be returned instead of a string. If delete_existing -is 0, then adding a path that already exists +is False, then adding a path that already exists will not move it to the end; it will stay where it is in the list. @@ -485,29 +601,28 @@ print('before:', env['ENV']['INCLUDE']) include_path = '/foo/bar:/foo' env.AppendENVPath('INCLUDE', include_path) print('after:', env['ENV']['INCLUDE']) + -yields: +Yields: + before: /foo:/biz after: /biz:/foo/bar:/foo - + - env.AppendUnique(key=val, [...], delete_existing=0) + env.AppendUnique(key=val, [...], delete_existing=False) -Appends the specified keyword arguments -to the end of construction variables in the environment. -If the Environment does not have -the specified construction variable, -it is simply added to the environment. -If the construction variable being appended to is a list, -then any value(s) that already exist in the -construction variable will -not -be added again to the list. -However, if delete_existing is 1, -existing matching values are removed first, so -existing values in the arg list move to the end of the list. +Append values to &consvars; in the current &consenv;, +maintaining uniqueness. +Works like &f-link-env-Append; (see for details), +except that values already present in the &consvar; +will not be added again. +If delete_existing +is True, +the existing matching value is first removed, +and the requested value is added, +having the effect of moving such values to the end. @@ -515,8 +630,14 @@ Example: -env.AppendUnique(CCFLAGS = '-g', FOO = ['foo.yyy']) +env.AppendUnique(CCFLAGS='-g', FOO=['foo.yyy']) + + +See also &f-link-env-Append;, +&f-link-env-Prepend; +and &f-link-env-PrependUnique;. + @@ -540,7 +661,7 @@ including the argument, at the time it is called using the construction variables in the -env +env construction environment through which &f-env-Builder; was called. The @@ -737,7 +858,7 @@ Example: env2 = env.Clone() -env3 = env.Clone(CCFLAGS = '-g') +env3 = env.Clone(CCFLAGS='-g') @@ -746,8 +867,10 @@ the &f-link-Environment; constructor: -def MyTool(env): env['FOO'] = 'bar' -env4 = env.Clone(tools = ['msvc', MyTool]) +def MyTool(env): + env['FOO'] = 'bar' + +env4 = env.Clone(tools=['msvc', MyTool]) @@ -949,24 +1072,24 @@ timestamp, such as can happen when restoring files from backup archives. -"MD5" +"content" Specifies that a target shall be considered out of date and rebuilt if the dependency's content has changed since the last time the target was built, -as determined be performing an MD5 checksum +as determined be performing an checksum on the dependency's contents and comparing it to the checksum recorded the last time the target was built. -content +MD5 can be used as a synonym for -MD5. +content, but it is deprecated. -"MD5-timestamp" +"content-timestamp" Specifies that a target shall be considered out of date and rebuilt @@ -979,7 +1102,7 @@ assumed to be up-to-date and rebuilt. This provides behavior very similar to the -MD5 +content behavior of always checksumming file contents, with an optimization of not checking the contents of files whose timestamps haven't changed. @@ -992,6 +1115,9 @@ that runs a build, updates a file, and runs the build again, all within a single second. +MD5-timestamp +can be used as a synonym for +content-timestamp, but it is deprecated. @@ -1006,7 +1132,7 @@ Examples: # Use exact timestamp matches by default. Decider('timestamp-match') -# Use MD5 content signatures for any targets built +# Use hash content signatures for any targets built # with the attached construction environment. env.Decider('content') @@ -1262,9 +1388,9 @@ Find an executable from one or more choices: Returns the first value from progs that was found, or None. Executable is searched by checking the paths specified -by env['ENV']['PATH']. +by env['ENV']['PATH']. On Windows systems, additionally applies the filename suffixes found in -env['ENV']['PATHEXT'] +env['ENV']['PATHEXT'] but will not include any such extension in the return value. &f-env-Detect; is a wrapper around &f-link-env-WhereIs;. @@ -1384,7 +1510,7 @@ While this SConstruct: -env=Environment() +env = Environment() print(env.Dump()) @@ -1677,16 +1803,16 @@ Example: -Install( '/bin', [ 'executable_a', 'executable_b' ] ) +Install('/bin', ['executable_a', 'executable_b']) # will return the file node list -# [ '/bin/executable_a', '/bin/executable_b' ] +# ['/bin/executable_a', '/bin/executable_b'] FindInstalledFiles() -Install( '/lib', [ 'some_library' ] ) +Install('/lib', ['some_library']) # will return the file node list -# [ '/bin/executable_a', '/bin/executable_b', '/lib/some_library' ] +# ['/bin/executable_a', '/bin/executable_b', '/lib/some_library'] FindInstalledFiles() @@ -1774,15 +1900,15 @@ Example: -Program( 'src/main_a.c' ) -Program( 'src/main_b.c' ) -Program( 'main_c.c' ) +Program('src/main_a.c') +Program('src/main_b.c') +Program('main_c.c') # returns ['main_c.c', 'src/main_a.c', 'SConstruct', 'src/main_b.c'] FindSourceFiles() # returns ['src/main_b.c', 'src/main_a.c' ] -FindSourceFiles( 'src' ) +FindSourceFiles('src') @@ -2445,8 +2571,8 @@ Examples: env.Ignore('foo', 'foo.c') env.Ignore('bar', ['bar1.h', 'bar2.h']) -env.Ignore('.','foobar.obj') -env.Ignore('bar','bar/foobar.obj') +env.Ignore('.', 'foobar.obj') +env.Ignore('bar', 'bar/foobar.obj') @@ -2864,7 +2990,7 @@ Example: -env = Environment(platform = Platform('win32')) +env = Environment(platform=Platform('win32')) @@ -2916,19 +3042,10 @@ Multiple targets can be passed in to a single call to env.Prepend(key=val, [...]) -Appends the specified keyword arguments -to the beginning of construction variables in the environment. -If the Environment does not have -the specified construction variable, -it is simply added to the environment. -If the values of the construction variable -and the keyword argument are the same type, -then the two values will be simply added together. -Otherwise, the construction variable -and the value of the keyword argument -are both coerced to lists, -and the lists are added together. -(See also the Append method, above.) +Prepend values to &consvars; in the current &consenv;, +Works like &f-link-env-Append; (see for details), +except that values are added to the front, +rather than the end, of any existing value of the &consvar; @@ -2936,25 +3053,29 @@ Example: -env.Prepend(CCFLAGS = '-g ', FOO = ['foo.yyy']) +env.Prepend(CCFLAGS='-g ', FOO=['foo.yyy']) + + +See also &f-link-env-Append;, +&f-link-env-AppendUnique; +and &f-link-env-PrependUnique;. + env.PrependENVPath(name, newpath, [envname, sep, delete_existing]) -This appends new path elements to the given path in the -specified external environment -(&cv-ENV; -by default). +Prepend new path elements to the given path in the +specified external environment (&cv-link-ENV; by default). This will only add any particular path once (leaving the first one it encounters and ignoring the rest, to preserve path order), and to help assure this, will normalize all paths (using -os.path.normpath +os.path.normpath and -os.path.normcase). +os.path.normcase). This can also handle the case where the given old path variable is a list instead of a string, in which case a list will be returned instead of a string. @@ -2963,7 +3084,8 @@ string, in which case a list will be returned instead of a string. If delete_existing -is 0, then adding a path that already exists +is False, +then adding a path that already exists will not move it to the beginning; it will stay where it is in the list. @@ -2979,32 +3101,29 @@ env.PrependENVPath('INCLUDE', include_path) print('after:', env['ENV']['INCLUDE']) - -The above example will print: - +Yields: - + before: /biz:/foo after: /foo/bar:/foo:/biz - + - env.PrependUnique(key=val, delete_existing=0, [...]) + env.PrependUnique(key=val, delete_existing=False, [...]) -Appends the specified keyword arguments -to the beginning of construction variables in the environment. -If the Environment does not have -the specified construction variable, -it is simply added to the environment. -If the construction variable being appended to is a list, -then any value(s) that already exist in the -construction variable will -not -be added again to the list. -However, if delete_existing is 1, -existing matching values are removed first, so -existing values in the arg list move to the front of the list. +Prepend values to &consvars; in the current &consenv;, +maintaining uniqueness. +Works like &f-link-env-Append; (see for details), +except that values are added to the front, +rather than the end, of any existing value of the the &consvar;, +and values already present in the &consvar; +will not be added again. +If delete_existing +is True, +the existing matching value is first removed, +and the requested value is inserted, +having the effect of moving such values to the front. @@ -3012,8 +3131,14 @@ Example: -env.PrependUnique(CCFLAGS = '-g', FOO = ['foo.yyy']) +env.PrependUnique(CCFLAGS='-g', FOO=['foo.yyy']) + + +See also &f-link-env-Append;, +&f-link-env-AppendUnique; +and &f-link-env-Prepend;. + @@ -3200,7 +3325,7 @@ Example: -env.Replace(CCFLAGS = '-g', FOO = 'foo.xxx') +env.Replace(CCFLAGS='-g', FOO='foo.xxx') @@ -3738,9 +3863,9 @@ The following statements are equivalent: -env.SetDefault(FOO = 'foo') - -if 'FOO' not in env: env['FOO'] = 'foo' +env.SetDefault(FOO='foo') +if 'FOO' not in env: + env['FOO'] = 'foo' @@ -4070,12 +4195,13 @@ Example: print(env.subst("The C compiler is: $CC")) def compile(target, source, env): - sourceDir = env.subst("${SOURCE.srcdir}", - target=target, - source=source) + sourceDir = env.subst( + "${SOURCE.srcdir}", + target=target, + source=source + ) -source_nodes = env.subst('$EXPAND_TO_NODELIST', - conv=lambda x: x) +source_nodes = env.subst('$EXPAND_TO_NODELIST', conv=lambda x: x) @@ -4368,11 +4494,11 @@ searches the paths in the path keyword argument, or if None (the default) the paths listed in the &consenv; -(env['ENV']['PATH']). +(env['ENV']['PATH']). The external environment's path list (os.environ['PATH']) is used as a fallback if the key -env['ENV']['PATH'] +env['ENV']['PATH'] does not exist. @@ -4381,11 +4507,11 @@ programs with any of the file extensions listed in the pathext keyword argument, or if None (the default) the pathname extensions listed in the &consenv; -(env['ENV']['PATHEXT']). +(env['ENV']['PATHEXT']). The external environment's pathname extensions list (os.environ['PATHEXT']) is used as a fallback if the key -env['ENV']['PATHEXT'] +env['ENV']['PATHEXT'] does not exist. diff --git a/doc/generated/tools.gen b/doc/generated/tools.gen index 625e7d7e43..44118c211a 100644 --- a/doc/generated/tools.gen +++ b/doc/generated/tools.gen @@ -744,7 +744,7 @@ library archiver. Sets construction variables for the Microsoft linker. -Sets: &cv-link-LDMODULE;, &cv-link-LDMODULECOM;, &cv-link-LDMODULEFLAGS;, &cv-link-LDMODULEPREFIX;, &cv-link-LDMODULESUFFIX;, &cv-link-LIBDIRPREFIX;, &cv-link-LIBDIRSUFFIX;, &cv-link-LIBLINKPREFIX;, &cv-link-LIBLINKSUFFIX;, &cv-link-LINK;, &cv-link-LINKCOM;, &cv-link-LINKFLAGS;, &cv-link-REGSVR;, &cv-link-REGSVRCOM;, &cv-link-REGSVRFLAGS;, &cv-link-SHLINK;, &cv-link-SHLINKCOM;, &cv-link-SHLINKFLAGS;, &cv-link-WIN32DEFPREFIX;, &cv-link-WIN32DEFSUFFIX;, &cv-link-WIN32EXPPREFIX;, &cv-link-WIN32EXPSUFFIX;, &cv-link-WINDOWSDEFPREFIX;, &cv-link-WINDOWSDEFSUFFIX;, &cv-link-WINDOWSEXPPREFIX;, &cv-link-WINDOWSEXPSUFFIX;, &cv-link-WINDOWSPROGMANIFESTPREFIX;, &cv-link-WINDOWSPROGMANIFESTSUFFIX;, &cv-link-WINDOWSSHLIBMANIFESTPREFIX;, &cv-link-WINDOWSSHLIBMANIFESTSUFFIX;, &cv-link-WINDOWS_INSERT_DEF;.Uses: &cv-link-LDMODULECOMSTR;, &cv-link-LINKCOMSTR;, &cv-link-REGSVRCOMSTR;, &cv-link-SHLINKCOMSTR;. +Sets: &cv-link-LDMODULE;, &cv-link-LDMODULECOM;, &cv-link-LDMODULEFLAGS;, &cv-link-LDMODULEPREFIX;, &cv-link-LDMODULESUFFIX;, &cv-link-LIBDIRPREFIX;, &cv-link-LIBDIRSUFFIX;, &cv-link-LIBLINKPREFIX;, &cv-link-LIBLINKSUFFIX;, &cv-link-LINK;, &cv-link-LINKCOM;, &cv-link-LINKFLAGS;, &cv-link-REGSVR;, &cv-link-REGSVRCOM;, &cv-link-REGSVRFLAGS;, &cv-link-SHLINK;, &cv-link-SHLINKCOM;, &cv-link-SHLINKFLAGS;, &cv-link-WINDOWSDEFPREFIX;, &cv-link-WINDOWSDEFSUFFIX;, &cv-link-WINDOWSEXPPREFIX;, &cv-link-WINDOWSEXPSUFFIX;, &cv-link-WINDOWSPROGMANIFESTPREFIX;, &cv-link-WINDOWSPROGMANIFESTSUFFIX;, &cv-link-WINDOWSSHLIBMANIFESTPREFIX;, &cv-link-WINDOWSSHLIBMANIFESTSUFFIX;, &cv-link-WINDOWS_INSERT_DEF;.Uses: &cv-link-LDMODULECOMSTR;, &cv-link-LINKCOMSTR;, &cv-link-REGSVRCOMSTR;, &cv-link-SHLINKCOMSTR;. mssdk @@ -796,10 +796,17 @@ Sets construction variables for the Sets: &cv-link-AS;, &cv-link-ASCOM;, &cv-link-ASFLAGS;, &cv-link-ASPPCOM;, &cv-link-ASPPFLAGS;.Uses: &cv-link-ASCOMSTR;, &cv-link-ASPPCOMSTR;. + + ninja + + Sets up &b-link-Ninja; builder which generates a ninja build file, and then optionally runs ninja. + + Sets: &cv-link-DISABLE_AUTO_NINJA;, &cv-link-NINJA_ALIAS_NAME;, &cv-link-NINJA_BUILDDIR;, &cv-link-NINJA_COMPDB_EXPAND;, &cv-link-NINJA_ENV_VAR_CACHE;, &cv-link-NINJA_FILE_NAME;, &cv-link-NINJA_GENERATED_SOURCE_SUFFIXES;, &cv-link-NINJA_MSVC_DEPS_PREFIX;, &cv-link-NINJA_POOL;, &cv-link-NINJA_REGENERATE_DEPS;, &cv-link-NINJA_SYNTAX;, &cv-link-_NINJA_REGENERATE_DEPS_FUNC;, &cv-link-__NINJA_NO;.Uses: &cv-link-AR;, &cv-link-ARCOM;, &cv-link-ARFLAGS;, &cv-link-CC;, &cv-link-CCCOM;, &cv-link-CXX;, &cv-link-CXXCOM;, &cv-link-ESCAPE;, &cv-link-LINK;, &cv-link-LINKCOM;, &cv-link-PLATFORM;, &cv-link-RANLIB;, &cv-link-RANLIBCOM;, &cv-link-SHCXXCOM;, &cv-link-SHLINK;, &cv-link-SHLINKCOM;. + packaging -Sets construction variables for the &b-Package; Builder. +Sets construction variables for the &b-link-Package; Builder. If this tool is enabled, the command-line option is also enabled. diff --git a/doc/generated/tools.mod b/doc/generated/tools.mod index 78aa9ef982..35eea5e2b4 100644 --- a/doc/generated/tools.mod +++ b/doc/generated/tools.mod @@ -79,6 +79,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. mwcc"> mwld"> nasm"> +ninja"> packaging"> pdf"> pdflatex"> @@ -184,6 +185,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. mwcc"> mwld"> nasm"> +ninja"> packaging"> pdf"> pdflatex"> diff --git a/doc/generated/variables.gen b/doc/generated/variables.gen index 84ef3b1bfa..5b1edbe56a 100644 --- a/doc/generated/variables.gen +++ b/doc/generated/variables.gen @@ -22,6 +22,16 @@ if &cv-link-LDMODULEVERSION; is set. Othervise it evaluates to an empty string. + + + __NINJA_NO + + + Internal flag. Used to tell SCons whether or not to try to import pypi's ninja python package. + This is set to True when being called by Ninja? + + + __SHLIBVERSIONFLAGS @@ -152,6 +162,7 @@ and the BuildArch: field in the RPM .spec file, as well as forming part of the name of a generated RPM package file. +See the &b-link-Package; builder. @@ -527,6 +538,7 @@ the .wxs for MSI). If set, the function will be called after the SCons template for the file has been written. +See the &b-link-Package; builder. @@ -565,6 +577,7 @@ This is included as the section of the RPM .spec file. +See the &b-link-Package; builder. @@ -662,14 +675,13 @@ file. _CPPDEFFLAGS -An automatically-generated construction variable +An automatically-generated &consvar; containing the C preprocessor command-line options to define values. The value of &cv-_CPPDEFFLAGS; is created by respectively prepending and appending -&cv-CPPDEFPREFIX; and &cv-CPPDEFSUFFIX; -to the beginning and end -of each definition in &cv-CPPDEFINES;. +&cv-link-CPPDEFPREFIX; and &cv-link-CPPDEFSUFFIX; +to each definition in &cv-link-CPPDEFINES;. @@ -678,10 +690,10 @@ of each definition in &cv-CPPDEFINES;. CPPDEFINES -A platform independent specification of C preprocessor definitions. +A platform independent specification of C preprocessor macro definitions. The definitions will be added to command lines through the automatically-generated -&cv-_CPPDEFFLAGS; construction variable (see above), +&cv-link-_CPPDEFFLAGS; &consvar; (see above), which is constructed according to the type of value of &cv-CPPDEFINES;: @@ -689,10 +701,9 @@ the type of value of &cv-CPPDEFINES;: If &cv-CPPDEFINES; is a string, the values of the -&cv-CPPDEFPREFIX; and &cv-CPPDEFSUFFIX; -construction variables -will be respectively prepended and appended to the beginning and end -of each definition in &cv-CPPDEFINES;. +&cv-link-CPPDEFPREFIX; and &cv-link-CPPDEFSUFFIX; &consvars; +will be respectively prepended and appended to +each definition in &cv-CPPDEFINES;. @@ -704,10 +715,9 @@ env = Environment(CPPDEFINES='xyz') If &cv-CPPDEFINES; is a list, the values of the -&cv-CPPDEFPREFIX; and &cv-CPPDEFSUFFIX; -construction variables -will be respectively prepended and appended to the beginning and end -of each element in the list. +&cv-link-CPPDEFPREFIX; and &cv-link-CPPDEFSUFFIX; &consvars; +will be respectively prepended and appended to +each element in the list. If any element is a list or tuple, then the first item is the name being defined and the second item is its value: @@ -722,10 +732,9 @@ env = Environment(CPPDEFINES=[('B', 2), 'A']) If &cv-CPPDEFINES; is a dictionary, the values of the -&cv-CPPDEFPREFIX; and &cv-CPPDEFSUFFIX; -construction variables -will be respectively prepended and appended to the beginning and end -of each item from the dictionary. +&cv-link-CPPDEFPREFIX; and &cv-link-CPPDEFSUFFIX; &consvars; +will be respectively prepended and appended to +each item from the dictionary. The key of each dictionary item is a name being defined to the dictionary item's corresponding value; @@ -751,11 +760,11 @@ env = Environment(CPPDEFINES={'B':2, 'A':None}) CPPDEFPREFIX -The prefix used to specify preprocessor definitions +The prefix used to specify preprocessor macro definitions on the C compiler command line. -This will be prepended to the beginning of each definition -in the &cv-CPPDEFINES; construction variable -when the &cv-_CPPDEFFLAGS; variable is automatically generated. +This will be prepended to each definition +in the &cv-link-CPPDEFINES; &consvar; +when the &cv-link-_CPPDEFFLAGS; variable is automatically generated. @@ -764,11 +773,11 @@ when the &cv-_CPPDEFFLAGS; variable is automatically generated. CPPDEFSUFFIX -The suffix used to specify preprocessor definitions +The suffix used to specify preprocessor macro definitions on the C compiler command line. -This will be appended to the end of each definition -in the &cv-CPPDEFINES; construction variable -when the &cv-_CPPDEFFLAGS; variable is automatically generated. +This will be appended to each definition +in the &cv-link-CPPDEFINES; &consvar; +when the &cv-link-_CPPDEFFLAGS; variable is automatically generated. @@ -808,13 +817,13 @@ for the variable that expands to those options. _CPPINCFLAGS -An automatically-generated construction variable +An automatically-generated &consvar; containing the C preprocessor command-line options for specifying directories to be searched for include files. The value of &cv-_CPPINCFLAGS; is created -by respectively prepending and appending &cv-INCPREFIX; and &cv-INCSUFFIX; -to the beginning and end -of each directory in &cv-CPPPATH;. +by respectively prepending and appending +&cv-link-INCPREFIX; and &cv-link-INCSUFFIX; +to each directory in &cv-link-CPPPATH;. @@ -825,13 +834,24 @@ of each directory in &cv-CPPPATH;. The list of directories that the C preprocessor will search for include directories. The C/C++ implicit dependency scanner will search these -directories for include files. Don't explicitly put include directory -arguments in CCFLAGS or CXXFLAGS because the result will be non-portable -and the directories will not be searched by the dependency scanner. Note: -directory names in CPPPATH will be looked-up relative to the SConscript -directory when they are used in a command. To force -&scons; -to look-up a directory relative to the root of the source tree use #: +directories for include files. +In general it's not advised to put include directory directives +directly into &cv-link-CCFLAGS; or &cv-link-CXXFLAGS; +as the result will be non-portable +and the directories will not be searched by the dependency scanner. +&cv-CPPPATH; should be a list of path strings, +or a single string, not a pathname list joined by +Python's os.sep. + + + +Note: +directory names in &cv-link-CPPPATH; +will be looked-up relative to the directory of the SConscript file +when they are used in a command. +To force &scons; +to look-up a directory relative to the root of the source tree use +the # prefix: @@ -840,7 +860,7 @@ env = Environment(CPPPATH='#/include') The directory look-up can also be forced using the -&Dir;() +&f-link-Dir; function: @@ -852,17 +872,14 @@ env = Environment(CPPPATH=include) The directory list will be added to command lines through the automatically-generated -&cv-_CPPINCFLAGS; -construction variable, +&cv-link-_CPPINCFLAGS; &consvar;, which is constructed by -respectively prepending and appending the value of the -&cv-INCPREFIX; and &cv-INCSUFFIX; -construction variables -to the beginning and end -of each directory in &cv-CPPPATH;. +respectively prepending and appending the values of the +&cv-link-INCPREFIX; and &cv-link-INCSUFFIX; &consvars; +to each directory in &cv-link-CPPPATH;. Any command lines you define that need -the CPPPATH directory list should -include &cv-_CPPINCFLAGS;: +the &cv-CPPPATH; directory list should +include &cv-link-_CPPINCFLAGS;: @@ -1052,6 +1069,7 @@ A long description of the project being packaged. This is included in the relevant section of the file that controls the packaging build. +See the &b-link-Package; builder. @@ -1066,6 +1084,7 @@ This is used to populate a section of an RPM .spec file. +See the &b-link-Package; builder. @@ -1142,6 +1161,17 @@ into a list of Dir instances relative to the target being built. + + + DISABLE_AUTO_NINJA + + + Boolean (True|False). Default: False + When True, SCons will not run ninja automatically after creating the ninja.build file. + + + + DLIB @@ -3268,9 +3298,9 @@ env = Environment(IMPLICIT_COMMAND_DEPENDENCIES=False) The prefix used to specify an include directory on the C compiler command line. -This will be prepended to the beginning of each directory -in the &cv-CPPPATH; and &cv-FORTRANPATH; construction variables -when the &cv-_CPPINCFLAGS; and &cv-_FORTRANINCFLAGS; +This will be prepended to each directory +in the &cv-link-CPPPATH; and &cv-link-FORTRANPATH; &consvars; +when the &cv-link-_CPPINCFLAGS; and &cv-link-_FORTRANINCFLAGS; variables are automatically generated. @@ -3282,9 +3312,9 @@ variables are automatically generated. The suffix used to specify an include directory on the C compiler command line. -This will be appended to the end of each directory -in the &cv-CPPPATH; and &cv-FORTRANPATH; construction variables -when the &cv-_CPPINCFLAGS; and &cv-_FORTRANINCFLAGS; +This will be appended to each directory +in the &cv-link-CPPPATH; and &cv-link-FORTRANPATH; &consvars; +when the &cv-link-_CPPINCFLAGS; and &cv-link-_FORTRANINCFLAGS; variables are automatically generated. @@ -3924,9 +3954,9 @@ An automatically-generated construction variable containing the linker command-line options for specifying directories to be searched for library. The value of &cv-_LIBDIRFLAGS; is created -by respectively prepending and appending &cv-LIBDIRPREFIX; and &cv-LIBDIRSUFFIX; -to the beginning and end -of each directory in &cv-LIBPATH;. +by respectively prepending and appending &cv-link-LIBDIRPREFIX; +and &cv-link-LIBDIRSUFFIX; +to each directory in &cv-link-LIBPATH;. @@ -3936,9 +3966,9 @@ of each directory in &cv-LIBPATH;. The prefix used to specify a library directory on the linker command line. -This will be prepended to the beginning of each directory -in the &cv-LIBPATH; construction variable -when the &cv-_LIBDIRFLAGS; variable is automatically generated. +This will be prepended to each directory +in the &cv-link-LIBPATH; construction variable +when the &cv-link-_LIBDIRFLAGS; variable is automatically generated. @@ -3948,9 +3978,9 @@ when the &cv-_LIBDIRFLAGS; variable is automatically generated. The suffix used to specify a library directory on the linker command line. -This will be appended to the end of each directory -in the &cv-LIBPATH; construction variable -when the &cv-_LIBDIRFLAGS; variable is automatically generated. +This will be appended to each directory +in the &cv-link-LIBPATH; construction variable +when the &cv-link-_LIBDIRFLAGS; variable is automatically generated. @@ -3974,10 +4004,10 @@ general information on specifying emitters. An automatically-generated construction variable containing the linker command-line options for specifying libraries to be linked with the resulting target. -The value of &cv-_LIBFLAGS; is created -by respectively prepending and appending &cv-LIBLINKPREFIX; and &cv-LIBLINKSUFFIX; -to the beginning and end -of each filename in &cv-LIBS;. +The value of &cv-link-_LIBFLAGS; is created +by respectively prepending and appending &cv-link-LIBLINKPREFIX; +and &cv-link-LIBLINKSUFFIX; +to each filename in &cv-link-LIBS;. @@ -3987,9 +4017,9 @@ of each filename in &cv-LIBS;. The prefix used to specify a library to link on the linker command line. -This will be prepended to the beginning of each library -in the &cv-LIBS; construction variable -when the &cv-_LIBFLAGS; variable is automatically generated. +This will be prepended to each library +in the &cv-link-LIBS; construction variable +when the &cv-link-_LIBFLAGS; variable is automatically generated. @@ -3999,9 +4029,9 @@ when the &cv-_LIBFLAGS; variable is automatically generated. The suffix used to specify a library to link on the linker command line. -This will be appended to the end of each library -in the &cv-LIBS; construction variable -when the &cv-_LIBFLAGS; variable is automatically generated. +This will be appended to each library +in the &cv-link-LIBS; construction variable +when the &cv-link-_LIBFLAGS; variable is automatically generated. @@ -4010,16 +4040,33 @@ when the &cv-_LIBFLAGS; variable is automatically generated. LIBPATH -The list of directories that will be searched for libraries. +The list of directories that will be searched for libraries +specified by the &cv-link-LIBS; &consvar;. +&cv-LIBPATH; should be a list of path strings, +or a single string, not a pathname list joined by +Python's os.sep. + + +Do not put library search directives directly +into &cv-LINKFLAGS; or &cv-SHLINKFLAGS; +as the result will be non-portable. + + + + +Note: +directory names in &cv-LIBPATH; will be looked-up relative to the +directory of the SConscript file +when they are used in a command. +To force &scons; +to look-up a directory relative to the root of the source tree use +the # prefix: @@ -4028,8 +4075,7 @@ env = Environment(LIBPATH='#/libs') The directory look-up can also be forced using the -&Dir;() -function: +&f-link-Dir; function: @@ -4040,16 +4086,15 @@ env = Environment(LIBPATH=libs) The directory list will be added to command lines through the automatically-generated -&cv-_LIBDIRFLAGS; +&cv-link-_LIBDIRFLAGS; construction variable, which is constructed by respectively prepending and appending the values of the -&cv-LIBDIRPREFIX; and &cv-LIBDIRSUFFIX; +&cv-link-LIBDIRPREFIX; and &cv-link-LIBDIRSUFFIX; construction variables -to the beginning and end -of each directory in &cv-LIBPATH;. +to each directory in &cv-LIBPATH;. Any command lines you define that need -the LIBPATH directory list should +the &cv-LIBPATH; directory list should include &cv-_LIBDIRFLAGS;: @@ -4081,7 +4126,7 @@ A list of all legal prefixes for library file names. When searching for library dependencies, SCons will look for files with these prefixes, the base library name, -and suffixes in the &cv-LIBSUFFIXES; list. +and suffixes from the &cv-link-LIBSUFFIXES; list. @@ -4091,24 +4136,32 @@ and suffixes in the &cv-LIBSUFFIXES; list. A list of one or more libraries -that will be linked with -any executable programs -created by this environment. +that will be added to the link line +for linking with any executable program, shared library, or loadable module +created by the &consenv; or override. +String-valued library names should include +only the library base names, +without prefixes such as lib +or suffixes such as .so or .dll. The library list will be added to command lines through the automatically-generated -&cv-_LIBFLAGS; -construction variable, +&cv-_LIBFLAGS; &consvar; which is constructed by respectively prepending and appending the values of the -&cv-LIBLINKPREFIX; and &cv-LIBLINKSUFFIX; -construction variables -to the beginning and end -of each filename in &cv-LIBS;. +&cv-LIBLINKPREFIX; and &cv-LIBLINKSUFFIX; &consvars; +to each library name in &cv-LIBS;. +Library name strings should not include a +path component, instead the compiler will be +directed to look for libraries in the paths +specified by &cv-link-LIBPATH;. + + + Any command lines you define that need -the LIBS library list should +the &cv-LIBS; library list should include &cv-_LIBFLAGS;: @@ -4118,12 +4171,12 @@ env = Environment(LINKCOM="my_linker $_LIBDIRFLAGS $_LIBFLAGS -o $TARGET $SOURCE If you add a -File +File object to the &cv-LIBS; list, the name of that file will be added to &cv-_LIBFLAGS;, -and thus the link line, as is, without +and thus to the link line, as-is, without &cv-LIBLINKPREFIX; or &cv-LIBLINKSUFFIX;. @@ -4161,7 +4214,7 @@ to reflect the names of the libraries they create. A list of all legal suffixes for library file names. When searching for library dependencies, -SCons will look for files with prefixes, in the &cv-LIBPREFIXES; list, +SCons will look for files with prefixes from the &cv-link-LIBPREFIXES; list, the base library name, and these suffixes. @@ -4174,9 +4227,12 @@ and these suffixes. The abbreviated name, preferably the SPDX code, of the license under which this project is released (GPL-3.0, LGPL-2.1, BSD-2-Clause etc.). -See http://www.opensource.org/licenses/alphabetical +See + +http://www.opensource.org/licenses/alphabetical for a list of license names and SPDX codes. +See the &b-link-Package; builder. @@ -5097,8 +5153,141 @@ on this system. Specfies the name of the project to package. +See the &b-link-Package; builder. + + + NINJA_ALIAS_NAME + + + Name of the Alias() which is will cause SCons to create the ninja.build file, and + then (optionally) run ninja. + + + + + + NINJA_BUILDDIR + + + This propagates directly into the generated Ninja.build file. + From Ninja's docs: +
+ + builddir + A directory for some Ninja output files. ... (You can also store other build output in this + directory.) + +
+ + + + + + NINJA_COMPDB_EXPAND + + + Boolean value (True|False) to instruct ninja to expand the command line arguments normally put into + response files. + This prevents lines in the compilation database like gcc @rsp_file and instead yields + gcc -c -o myfile.o myfile.c -Ia -DXYZ + + + Ninja's compdb tool added the -x flag in Ninja V1.9.0 + + + + + + NINJA_ENV_VAR_CACHE + + + A string that sets the environment for any environment variables that + differ between the OS environment and the SCons command ENV. + + It will be compatible with the default shell of the operating system. + + If not explicitly specified, SCons will generate this dynamically from the Environment()'s 'ENV' + env['ENV'] + where those values differ from the existing shell.. + + + + + + NINJA_FILE_NAME + + + The filename for the generated Ninja build file defaults to ninja.build + + + + + + NINJA_GENERATED_SOURCE_SUFFIXES + + + The list of source file suffixes which are generated by SCons build steps. + All source files which match these suffixes will be added to the _generated_sources alias in the output + ninja.build file. + Then all other source files will be made to depend on this in the Ninja.build file, forcing the + generated sources to be built first. + + + + + + NINJA_MSVC_DEPS_PREFIX + + + This propagates directly into the generated Ninja.build file. + From Ninja's docs + defines the string which should be stripped from msvc’s /showIncludes output + + + + + + NINJA_POOL + + + Set the ninja_pool for this or all targets in scope for this env var. + + + + + + NINJA_REGENERATE_DEPS + + + A generator function used to create a ninja depsfile which includes all the files which would require + SCons to be invoked if they change. + Or a list of said files. + + + + + + _NINJA_REGENERATE_DEPS_FUNC + + + Internal value used to specify the function to call with argument env to generate the list of files + which if changed would require the ninja file to be regenerated. + + + + + + NINJA_SYNTAX + + + Theres also NINJA_SYNTAX which is the path to a custom ninja_syntax.py file which is used in generation. + The tool currently assumes you have ninja installed through pip, and grabs the syntax file from that + installation if none specified. + + + no_import_lib @@ -5139,8 +5328,9 @@ The suffix used for (static) object file names. Specifies the directory where all files in resulting archive will be -placed if applicable. The default value is "$NAME-$VERSION". +placed if applicable. The default value is &cv-NAME;-&cv-VERSION;. +See the &b-link-Package; builder. @@ -5157,6 +5347,7 @@ for the builder for the currently supported types. &cv-PACKAGETYPE; may be overridden with the command line option. +See the &b-link-Package; builder. @@ -5169,6 +5360,7 @@ This is currently only used by the rpm packager and should reflect changes in the packaging, not the underlying project code itself. +See the &b-link-Package; builder. @@ -5274,15 +5466,6 @@ You can generate PDB files with the switch by overriding the default &cv-link-CCPDBFLAGS; variable; see the entry for that variable for specific examples. - - - - - PDFCOM - - -A deprecated synonym for &cv-link-DVIPDFCOM;. - @@ -6046,7 +6229,7 @@ If this is not set, then &cv-link-RCCOM; (the command line) is displayed. RCFLAGS -The flags passed to the resource compiler by the RES builder. +The flags passed to the resource compiler by the &b-link-RES; builder. @@ -7276,6 +7459,7 @@ This is used to fill in the Source: field in the controlling information for Ipkg and RPM packages. +See the &b-link-Package; builder. @@ -7393,6 +7577,7 @@ and as the Description: field in MSI packages. +See the &b-link-Package; builder. @@ -7931,6 +8116,7 @@ and the Manufacturer: field in the controlling information for MSI packages. +See the &b-link-Package; builder. @@ -7940,6 +8126,7 @@ field in the controlling information for MSI packages. The version of the project, specified as a string. +See the &b-link-Package; builder. @@ -7990,51 +8177,6 @@ and also before &f-link-env-Tool; is called to ininitialize any of those tools: - - - - - WIN32_INSERT_DEF - - -A deprecated synonym for &cv-link-WINDOWS_INSERT_DEF;. - - - - - - WIN32DEFPREFIX - - -A deprecated synonym for &cv-link-WINDOWSDEFPREFIX;. - - - - - - WIN32DEFSUFFIX - - -A deprecated synonym for &cv-link-WINDOWSDEFSUFFIX;. - - - - - - WIN32EXPPREFIX - - -A deprecated synonym for &cv-link-WINDOWSEXPSUFFIX;. - - - - - - WIN32EXPSUFFIX - - -A deprecated synonym for &cv-link-WINDOWSEXPSUFFIX;. - @@ -8164,6 +8306,7 @@ This is used to fill in the Depends: field in the controlling information for Ipkg packages. +See the &b-link-Package; builder. @@ -8175,7 +8318,7 @@ This is used to fill in the Description: field in the controlling information for Ipkg packages. The default value is -$SUMMARY\n$DESCRIPTION +&cv-SUMMARY;\n&cv-DESCRIPTION; @@ -8221,6 +8364,7 @@ This is used to fill in the Language: attribute in the controlling information for MSI packages. +See the &b-link-Package; builder. @@ -8232,6 +8376,7 @@ The text of the software license in RTF format. Carriage return characters will be replaced with the RTF equivalent \\par. +See the &b-link-Package; builder. @@ -8253,6 +8398,7 @@ This is used to fill in the field in the RPM .spec file. +See the &b-link-Package; builder. @@ -8315,7 +8461,7 @@ field in the RPM This value is used as the default attributes for the files in the RPM package. The default value is -(-,root,root). +(-,root,root). diff --git a/doc/generated/variables.mod b/doc/generated/variables.mod index 10b62e77f1..a5fe961cd9 100644 --- a/doc/generated/variables.mod +++ b/doc/generated/variables.mod @@ -9,6 +9,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. --> $__LDMODULEVERSIONFLAGS"> +$__NINJA_NO"> $__SHLIBVERSIONFLAGS"> $APPLELINK_COMPATIBILITY_VERSION"> $_APPLELINK_COMPATIBILITY_VERSION"> @@ -82,6 +83,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $DINCSUFFIX"> $Dir"> $Dirs"> +$DISABLE_AUTO_NINJA"> $DLIB"> $DLIBCOM"> $DLIBDIRPREFIX"> @@ -348,6 +350,17 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $MWCW_VERSION"> $MWCW_VERSIONS"> $NAME"> +$NINJA_ALIAS_NAME"> +$NINJA_BUILDDIR"> +$NINJA_COMPDB_EXPAND"> +$NINJA_ENV_VAR_CACHE"> +$NINJA_FILE_NAME"> +$NINJA_GENERATED_SOURCE_SUFFIXES"> +$NINJA_MSVC_DEPS_PREFIX"> +$NINJA_POOL"> +$NINJA_REGENERATE_DEPS"> +$_NINJA_REGENERATE_DEPS_FUNC"> +$NINJA_SYNTAX"> $no_import_lib"> $OBJPREFIX"> $OBJSUFFIX"> @@ -360,7 +373,6 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $PCHPDBFLAGS"> $PCHSTOP"> $PDB"> -$PDFCOM"> $PDFLATEX"> $PDFLATEXCOM"> $PDFLATEXCOMSTR"> @@ -566,11 +578,6 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $VENDOR"> $VERSION"> $VSWHERE"> -$WIN32_INSERT_DEF"> -$WIN32DEFPREFIX"> -$WIN32DEFSUFFIX"> -$WIN32EXPPREFIX"> -$WIN32EXPSUFFIX"> $WINDOWS_EMBED_MANIFEST"> $WINDOWS_INSERT_DEF"> $WINDOWS_INSERT_MANIFEST"> @@ -653,6 +660,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. --> $__LDMODULEVERSIONFLAGS"> +$__NINJA_NO"> $__SHLIBVERSIONFLAGS"> $APPLELINK_COMPATIBILITY_VERSION"> $_APPLELINK_COMPATIBILITY_VERSION"> @@ -726,6 +734,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $DINCSUFFIX"> $Dir"> $Dirs"> +$DISABLE_AUTO_NINJA"> $DLIB"> $DLIBCOM"> $DLIBDIRPREFIX"> @@ -992,6 +1001,17 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $MWCW_VERSION"> $MWCW_VERSIONS"> $NAME"> +$NINJA_ALIAS_NAME"> +$NINJA_BUILDDIR"> +$NINJA_COMPDB_EXPAND"> +$NINJA_ENV_VAR_CACHE"> +$NINJA_FILE_NAME"> +$NINJA_GENERATED_SOURCE_SUFFIXES"> +$NINJA_MSVC_DEPS_PREFIX"> +$NINJA_POOL"> +$NINJA_REGENERATE_DEPS"> +$_NINJA_REGENERATE_DEPS_FUNC"> +$NINJA_SYNTAX"> $no_import_lib"> $OBJPREFIX"> $OBJSUFFIX"> @@ -1004,7 +1024,6 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $PCHPDBFLAGS"> $PCHSTOP"> $PDB"> -$PDFCOM"> $PDFLATEX"> $PDFLATEXCOM"> $PDFLATEXCOMSTR"> @@ -1210,11 +1229,6 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $VENDOR"> $VERSION"> $VSWHERE"> -$WIN32_INSERT_DEF"> -$WIN32DEFPREFIX"> -$WIN32DEFSUFFIX"> -$WIN32EXPPREFIX"> -$WIN32EXPSUFFIX"> $WINDOWS_EMBED_MANIFEST"> $WINDOWS_INSERT_DEF"> $WINDOWS_INSERT_MANIFEST"> From 1cb4dc8cf06af0d9032ee290a7a757c2ca37db71 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 12 Apr 2021 17:57:47 -0700 Subject: [PATCH 066/163] No need to have ninja.py which just references ninjaCommon, rename package to ninja and get rid of ninja.py --- SCons/Tool/ninja.py | 26 ------------------- SCons/Tool/{ninjaCommon => ninja}/Globals.py | 0 .../Tool/{ninjaCommon => ninja}/NinjaState.py | 5 ++-- .../Tool/{ninjaCommon => ninja}/Overrides.py | 0 SCons/Tool/{ninjaCommon => ninja}/Rules.py | 0 SCons/Tool/{ninjaCommon => ninja}/Util.py | 2 +- SCons/Tool/{ninjaCommon => ninja}/__init__.py | 16 ++++++------ SCons/Tool/{ninjaCommon => ninja}/ninja.xml | 0 8 files changed, 12 insertions(+), 37 deletions(-) delete mode 100644 SCons/Tool/ninja.py rename SCons/Tool/{ninjaCommon => ninja}/Globals.py (100%) rename SCons/Tool/{ninjaCommon => ninja}/NinjaState.py (99%) rename SCons/Tool/{ninjaCommon => ninja}/Overrides.py (100%) rename SCons/Tool/{ninjaCommon => ninja}/Rules.py (100%) rename SCons/Tool/{ninjaCommon => ninja}/Util.py (99%) rename SCons/Tool/{ninjaCommon => ninja}/__init__.py (98%) rename SCons/Tool/{ninjaCommon => ninja}/ninja.xml (100%) diff --git a/SCons/Tool/ninja.py b/SCons/Tool/ninja.py deleted file mode 100644 index 88b7d48f09..0000000000 --- a/SCons/Tool/ninja.py +++ /dev/null @@ -1,26 +0,0 @@ -# MIT License -# -# Copyright The SCons Foundation -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - - -from .ninjaCommon import generate, exists # noqa: F401 diff --git a/SCons/Tool/ninjaCommon/Globals.py b/SCons/Tool/ninja/Globals.py similarity index 100% rename from SCons/Tool/ninjaCommon/Globals.py rename to SCons/Tool/ninja/Globals.py diff --git a/SCons/Tool/ninjaCommon/NinjaState.py b/SCons/Tool/ninja/NinjaState.py similarity index 99% rename from SCons/Tool/ninjaCommon/NinjaState.py rename to SCons/Tool/ninja/NinjaState.py index 4e41934aa3..a7c3584582 100644 --- a/SCons/Tool/ninjaCommon/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -5,6 +5,7 @@ # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including +# "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to @@ -30,7 +31,7 @@ import SCons from SCons.Script import COMMAND_LINE_TARGETS from SCons.Util import is_List -import SCons.Tool.ninjaCommon.Globals +import SCons.Tool.ninja.Globals from .Globals import COMMAND_TYPES, NINJA_RULES, NINJA_POOLS, \ NINJA_CUSTOM_HANDLERS from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function @@ -562,7 +563,7 @@ def action_to_ninja_build(self, node, action=None): # Ninja builders out of being sources of ninja builders but I # can't fix every DAG problem so we just skip ninja_builders # if we find one - if SCons.Tool.ninjaCommon.NINJA_STATE.ninja_file == str(node): + if SCons.Tool.ninja.NINJA_STATE.ninja_file == str(node): build = None elif isinstance(action, SCons.Action.FunctionAction): build = self.handle_func_action(node, action) diff --git a/SCons/Tool/ninjaCommon/Overrides.py b/SCons/Tool/ninja/Overrides.py similarity index 100% rename from SCons/Tool/ninjaCommon/Overrides.py rename to SCons/Tool/ninja/Overrides.py diff --git a/SCons/Tool/ninjaCommon/Rules.py b/SCons/Tool/ninja/Rules.py similarity index 100% rename from SCons/Tool/ninjaCommon/Rules.py rename to SCons/Tool/ninja/Rules.py diff --git a/SCons/Tool/ninjaCommon/Util.py b/SCons/Tool/ninja/Util.py similarity index 99% rename from SCons/Tool/ninjaCommon/Util.py rename to SCons/Tool/ninja/Util.py index cc30552b50..80d1b16d75 100644 --- a/SCons/Tool/ninjaCommon/Util.py +++ b/SCons/Tool/ninja/Util.py @@ -26,7 +26,7 @@ import SCons from SCons.Action import get_default_ENV, _string_from_cmd_list from SCons.Script import AddOption -from SCons.Tool.ninjaCommon.Globals import __NINJA_RULE_MAPPING +from SCons.Tool.ninja.Globals import __NINJA_RULE_MAPPING from SCons.Util import is_List, flatten_sequence diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninja/__init__.py similarity index 98% rename from SCons/Tool/ninjaCommon/__init__.py rename to SCons/Tool/ninja/__init__.py index 4c5b08eb2c..9cc8b2e5f7 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -34,7 +34,7 @@ from glob import glob import SCons -import SCons.Tool.ninjaCommon.Globals +import SCons.Tool.ninja.Globals from SCons.Script import GetOption from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS @@ -186,7 +186,7 @@ def register_custom_handler(env, name, handler): def register_custom_rule_mapping(env, pre_subst_string, rule): """Register a function to call for a given rule.""" - SCons.Tool.ninjaCommon.Globals.__NINJA_RULE_MAPPING[pre_subst_string] = rule + SCons.Tool.ninja.Globals.__NINJA_RULE_MAPPING[pre_subst_string] = rule def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): @@ -279,14 +279,14 @@ def ninja_stat(_self, path): """ try: - return SCons.Tool.ninjaCommon.Globals.NINJA_STAT_MEMO[path] + return SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] except KeyError: try: result = os.stat(path) except os.error: result = None - SCons.Tool.ninjaCommon.Globals.NINJA_STAT_MEMO[path] = result + SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] = result return result @@ -296,7 +296,7 @@ def ninja_whereis(thing, *_args, **_kwargs): # Optimize for success, this gets called significantly more often # when the value is already memoized than when it's not. try: - return SCons.Tool.ninjaCommon.Globals.NINJA_WHEREIS_MEMO[thing] + return SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] except KeyError: # We do not honor any env['ENV'] or env[*] variables in the # generated ninja file. Ninja passes your raw shell environment @@ -308,7 +308,7 @@ def ninja_whereis(thing, *_args, **_kwargs): # with shell quoting is nigh impossible. So I've decided to # cross that bridge when it's absolutely required. path = shutil.which(thing) - SCons.Tool.ninjaCommon.Globals.NINJA_WHEREIS_MEMO[thing] = path + SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] = path return path @@ -395,8 +395,8 @@ def generate(env): """Generate the NINJA builders.""" global NINJA_STATE - if not SCons.Tool.ninjaCommon.Globals.ninja_builder_initialized: - SCons.Tool.ninjaCommon.Globals.ninja_builder_initialized = True + if not SCons.Tool.ninja.Globals.ninja_builder_initialized: + SCons.Tool.ninja.Globals.ninja_builder_initialized = True ninja_add_command_line_options() diff --git a/SCons/Tool/ninjaCommon/ninja.xml b/SCons/Tool/ninja/ninja.xml similarity index 100% rename from SCons/Tool/ninjaCommon/ninja.xml rename to SCons/Tool/ninja/ninja.xml From 8fa2fe1cf7367970611180d8146803bb333d4c7b Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 12 Apr 2021 21:21:39 -0700 Subject: [PATCH 067/163] Updated from MongoDB commit: https://github.com/mongodb/mongo/commit/59965cf5430b8132074771c0d9278a3a4b1e6730. Better handling of config test output --- SCons/Tool/ninja/__init__.py | 9 ++++++++- test/ninja/iterative_speedup.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 9cc8b2e5f7..ee059ced1d 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -340,6 +340,13 @@ def ninja_hack_linkcom(env): ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' +def ninja_print_conf_log(s, target, source, env): + """Command line print only for conftest to generate a correct conf log.""" + if target and target[0].is_conftest(): + action = SCons.Action._ActionAction() + action.print_cmd_line(s, target, source, env) + + class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" @@ -600,7 +607,7 @@ def robust_rule_mapping(var, rule, tool): SCons.Executor.Executor._get_unchanged_targets = SCons.Executor.Executor._get_targets # Replace false action messages with nothing. - env["PRINT_CMD_LINE_FUNC"] = ninja_noop + env["PRINT_CMD_LINE_FUNC"] = ninja_print_conf_log # This reduces unnecessary subst_list calls to add the compiler to # the implicit dependencies of targets. Since we encode full paths diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index df010f4d5b..9823550b59 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -225,7 +225,7 @@ def mod_source_orig(test_num): full_build_print = True for ninja_time, scons_time in zip(ninja_times, scons_times): if ninja_time > scons_time: - test.fail_test() + test.fail_test(message="Ninja was slower than SCons: SCons: {:.3f}s Ninja: {:.3f}s".format(scons_time, ninja_time)) if full_build_print: full_build_print = False print("Clean build {} files - SCons: {:.3f}s Ninja: {:.3f}s".format(num_source, scons_time, ninja_time)) From d9a3a4781c45a7d5551440cca508debc6166822c Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 13 Apr 2021 14:28:01 -0700 Subject: [PATCH 068/163] Add --experimental=ninja --- SCons/Script/SConsOptions.py | 4 ++-- test/option/option--experimental.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index 0233703800..c8c9cf87b8 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -39,7 +39,7 @@ diskcheck_all = SCons.Node.FS.diskcheck_types() -experimental_features = {'warp_speed', 'transporter'} +experimental_features = {'warp_speed', 'transporter', 'ninja'} def diskcheck_convert(value): @@ -734,7 +734,7 @@ def experimental_callback(option, opt, value, parser): op.add_option('--experimental', dest='experimental', action='callback', - default={}, # empty set + default=set(), # empty set type='str', # choices=experimental_options+experimental_features, callback =experimental_callback, diff --git a/test/option/option--experimental.py b/test/option/option--experimental.py index 0f0e9ec677..2e06dc3777 100644 --- a/test/option/option--experimental.py +++ b/test/option/option--experimental.py @@ -35,12 +35,13 @@ tests = [ ('.', []), - ('--experimental=all', ['transporter', 'warp_speed']), + ('--experimental=ninja', ['ninja']), + ('--experimental=all', ['ninja', 'transporter', 'warp_speed']), ('--experimental=none', []), ] for args, exper in tests: - read_string = """All Features=transporter,warp_speed + read_string = """All Features=ninja,transporter,warp_speed Experimental=%s """ % (exper) test.run(arguments=args, @@ -49,7 +50,7 @@ test.run(arguments='--experimental=warp_drive', stderr="""usage: scons [OPTION] [TARGET] ... -SCons Error: option --experimental: invalid choice: 'warp_drive' (choose from 'all','none','transporter','warp_speed') +SCons Error: option --experimental: invalid choice: 'warp_drive' (choose from 'all','none','ninja','transporter','warp_speed') """, status=2) From 0bc7e2169f79643f7306509fe422b9aa5947fae9 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 13 Apr 2021 14:28:37 -0700 Subject: [PATCH 069/163] Gate ninja tool being initialized by --experimental=ninja --- SCons/Tool/ninja/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index ee059ced1d..37e2cf8933 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -361,6 +361,9 @@ def _print_cmd_str(*_args, **_kwargs): def exists(env): """Enable if called.""" + if 'ninja' not in GetOption('experimental'): + return False + # This variable disables the tool when storing the SCons command in the # generated ninja file to ensure that the ninja tool is not loaded when # SCons should do actual work as a subprocess of a ninja build. The ninja @@ -402,6 +405,9 @@ def generate(env): """Generate the NINJA builders.""" global NINJA_STATE + if not 'ninja' in GetOption('experimental'): + return + if not SCons.Tool.ninja.Globals.ninja_builder_initialized: SCons.Tool.ninja.Globals.ninja_builder_initialized = True From 6437d84ecd2ba3d5a17981ec9130e17142a4f235 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 13 Apr 2021 14:28:53 -0700 Subject: [PATCH 070/163] update tests to use SetOption('experimental','ninja') --- test/ninja/build_libraries.py | 2 ++ test/ninja/copy_function_command.py | 2 ++ test/ninja/generate_source.py | 3 +++ test/ninja/iterative_speedup.py | 3 +++ test/ninja/multi_env.py | 3 +++ .../ninja/ninja_test_sconscripts/sconstruct_generate_and_build | 1 + .../ninja_test_sconscripts/sconstruct_generate_and_build_cxx | 1 + test/ninja/shell_command.py | 3 +++ 8 files changed, 18 insertions(+) diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 198dcced99..eb8eb74590 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -66,6 +66,8 @@ lib_suffix = '.dylib' test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) env = Environment() env.Tool('ninja') env['NINJA'] = "%(ninja_bin)s" diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index 37af62f52e..7e999b33e5 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -47,6 +47,8 @@ test.dir_fixture('ninja-fixture') test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) env = Environment() env.Tool('ninja') env.Command('foo2.c', ['foo.c'], Copy('$TARGET','$SOURCE')) diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index 18b53f249b..8300176c2e 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -49,6 +49,9 @@ shell = '' if IS_WINDOWS else './' test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + env = Environment() env.Tool('ninja') prog = env.Program(target = 'generate_source', source = 'generate_source.c') diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index 9823550b59..05e372cf56 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -166,6 +166,9 @@ def mod_source_orig(test_num): """ % locals()) test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + env = Environment() env.Tool('ninja') sources = ['main.c'] + env.Glob('source*.c') diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index d14588876b..e5da6cf885 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -49,6 +49,9 @@ test.dir_fixture('ninja-fixture') test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + env = Environment() env.Tool('ninja') env.Program(target = 'foo', source = 'foo.c') diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build index 242eb76265..81a4366755 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build @@ -1,3 +1,4 @@ +SetOption('experimental','ninja') DefaultEnvironment(tools=[]) env = Environment() diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx index 51ca6c356d..f7137dfb6e 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx @@ -1,3 +1,4 @@ +SetOption('experimental','ninja') DefaultEnvironment(tools=[]) env = Environment() env.Tool('ninja') diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index f0450d9cef..a6926c778d 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -49,6 +49,9 @@ shell = '' if IS_WINDOWS else './' test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + env = Environment() env.Tool('ninja') prog = env.Program(target = 'foo', source = 'foo.c') From 908f5eb33656b4e764accf3bc001dbd4397588a7 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 13 Apr 2021 14:32:49 -0700 Subject: [PATCH 071/163] fix sider complaint --- SCons/Tool/ninja/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 37e2cf8933..62c16cd3f7 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -405,7 +405,7 @@ def generate(env): """Generate the NINJA builders.""" global NINJA_STATE - if not 'ninja' in GetOption('experimental'): + if 'ninja' not in GetOption('experimental'): return if not SCons.Tool.ninja.Globals.ninja_builder_initialized: From 3df3a4f38fc6d8158f49393458a13a1fee5afcd9 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 13 Apr 2021 16:33:22 -0700 Subject: [PATCH 072/163] switch to using SCons.Node.SConscriptNodes to get list of sconscripts vs using Glob()'s --- SCons/Tool/ninja/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 62c16cd3f7..b239c379a4 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -443,10 +443,14 @@ def generate(env): SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] - # TODO: API for getting the SConscripts programmatically - # exists upstream: https://github.com/SCons/scons/issues/3625 def ninja_generate_deps(env): - return sorted([env.File("#SConstruct").path] + glob("**/SConscript", recursive=True)) + """Return a list of SConscripts + TODO: Should we also include files loaded from site_scons/*** + or even all loaded modules? https://stackoverflow.com/questions/4858100/how-to-list-imported-modules + TODO: Do we want this to be Nodes? + """ + return sorted([str(s) for s in SCons.Node.SConscriptNodes]) + env['_NINJA_REGENERATE_DEPS_FUNC'] = ninja_generate_deps env['NINJA_REGENERATE_DEPS'] = env.get('NINJA_REGENERATE_DEPS', '${_NINJA_REGENERATE_DEPS_FUNC(__env__)}') From 9e5a7664f0513fae6c4db18221d5b19f72a60f1b Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 18 May 2021 15:49:21 -0700 Subject: [PATCH 073/163] Quiet sider complaint --- SCons/Tool/ninja/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index b239c379a4..fd9c13c7e3 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -414,7 +414,7 @@ def generate(env): ninja_add_command_line_options() try: - import ninja # noqa: F401 + import ninja # noqa: F401 except ImportError: SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return From 08ea935321ab8858e15573e44c270da563e12ea0 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 24 May 2021 16:01:47 -0700 Subject: [PATCH 074/163] Continue refactor. Simplify __init__, and move logic to reasonably named files --- SCons/Tool/ninja/Methods.py | 269 ++++++++++++++++++++++++ SCons/Tool/ninja/NinjaState.py | 4 +- SCons/Tool/ninja/Overrides.py | 96 +++++++++ SCons/Tool/ninja/Rules.py | 2 +- SCons/Tool/ninja/{Util.py => Utils.py} | 191 +++++++---------- SCons/Tool/ninja/__init__.py | 276 +------------------------ 6 files changed, 454 insertions(+), 384 deletions(-) create mode 100644 SCons/Tool/ninja/Methods.py rename SCons/Tool/ninja/{Util.py => Utils.py} (72%) diff --git a/SCons/Tool/ninja/Methods.py b/SCons/Tool/ninja/Methods.py new file mode 100644 index 0000000000..3612236ed7 --- /dev/null +++ b/SCons/Tool/ninja/Methods.py @@ -0,0 +1,269 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os +import shlex +import textwrap + +import SCons +from SCons.Tool.ninja import NINJA_CUSTOM_HANDLERS, NINJA_RULES, NINJA_POOLS +from SCons.Tool.ninja.Globals import __NINJA_RULE_MAPPING +from SCons.Tool.ninja.Utils import get_targets_sources, get_dependencies, get_order_only, get_outputs, get_inputs, \ + get_rule, get_path, generate_command, get_command_env, get_comstr + + +def register_custom_handler(env, name, handler): + """Register a custom handler for SCons function actions.""" + env[NINJA_CUSTOM_HANDLERS][name] = handler + + +def register_custom_rule_mapping(env, pre_subst_string, rule): + """Register a function to call for a given rule.""" + SCons.Tool.ninja.Globals.__NINJA_RULE_MAPPING[pre_subst_string] = rule + + +def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): + """Allows specification of Ninja rules from inside SCons files.""" + rule_obj = { + "command": command, + "description": description if description else "{} $out".format(rule), + } + + if use_depfile: + rule_obj["depfile"] = os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile') + + if deps is not None: + rule_obj["deps"] = deps + + if pool is not None: + rule_obj["pool"] = pool + + if use_response_file: + rule_obj["rspfile"] = "$out.rsp" + rule_obj["rspfile_content"] = response_file_content + + env[NINJA_RULES][rule] = rule_obj + + +def register_custom_pool(env, pool, size): + """Allows the creation of custom Ninja pools""" + env[NINJA_POOLS][pool] = size + + +def set_build_node_callback(env, node, callback): + if not node.is_conftest(): + node.attributes.ninja_build_callback = callback + + +def get_generic_shell_command(env, node, action, targets, sources, executor=None): + return ( + "CMD", + { + # TODO: Why is executor passed in and then ignored below? (bdbaddog) + "cmd": generate_command(env, node, action, targets, sources, executor=None), + "env": get_command_env(env), + }, + # Since this function is a rule mapping provider, it must return a list of dependencies, + # and usually this would be the path to a tool, such as a compiler, used for this rule. + # However this function is to generic to be able to reliably extract such deps + # from the command, so we return a placeholder empty list. It should be noted that + # generally this function will not be used solely and is more like a template to generate + # the basics for a custom provider which may have more specific options for a provider + # function for a custom NinjaRuleMapping. + [] + ) + + +def CheckNinjaCompdbExpand(env, context): + """ Configure check testing if ninja's compdb can expand response files""" + + # TODO: When would this be false? + context.Message('Checking if ninja compdb can expand response files... ') + ret, output = context.TryAction( + action='ninja -f $SOURCE -t compdb -x CMD_RSP > $TARGET', + extension='.ninja', + text=textwrap.dedent(""" + rule CMD_RSP + command = $cmd @$out.rsp > fake_output.txt + description = Building $out + rspfile = $out.rsp + rspfile_content = $rspc + build fake_output.txt: CMD_RSP fake_input.txt + cmd = echo + pool = console + rspc = "test" + """)) + result = '@fake_output.txt.rsp' not in output + context.Result(result) + return result + + +def get_command(env, node, action): # pylint: disable=too-many-branches + """Get the command to execute for node.""" + if node.env: + sub_env = node.env + else: + sub_env = env + executor = node.get_executor() + tlist, slist = get_targets_sources(node) + + # Generate a real CommandAction + if isinstance(action, SCons.Action.CommandGeneratorAction): + # pylint: disable=protected-access + action = action._generate(tlist, slist, sub_env, 1, executor=executor) + + variables = {} + + comstr = get_comstr(sub_env, action, tlist, slist) + if not comstr: + return None + + provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) + rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) + + # Get the dependencies for all targets + implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) + + # Now add in the other dependencies related to the command, + # e.g. the compiler binary. The ninja rule can be user provided so + # we must do some validation to resolve the dependency path for ninja. + for provider_dep in provider_deps: + + provider_dep = sub_env.subst(provider_dep) + if not provider_dep: + continue + + # If the tool is a node, then SCons will resolve the path later, if its not + # a node then we assume it generated from build and make sure it is existing. + if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep): + implicit.append(provider_dep) + continue + + # in some case the tool could be in the local directory and be suppled without the ext + # such as in windows, so append the executable suffix and check. + prog_suffix = sub_env.get('PROGSUFFIX', '') + provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix + if os.path.exists(provider_dep_ext): + implicit.append(provider_dep_ext) + continue + + # Many commands will assume the binary is in the path, so + # we accept this as a possible input from a given command. + + provider_dep_abspath = sub_env.WhereIs(provider_dep) or sub_env.WhereIs(provider_dep, path=os.environ["PATH"]) + if provider_dep_abspath: + implicit.append(provider_dep_abspath) + continue + + # Possibly these could be ignore and the build would still work, however it may not always + # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided + # ninja rule. + raise Exception("Could not resolve path for %s dependency on node '%s'" % (provider_dep, node)) + + ninja_build = { + "order_only": get_order_only(node), + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "implicit": implicit, + "rule": get_rule(node, rule), + "variables": variables, + } + + # Don't use sub_env here because we require that NINJA_POOL be set + # on a per-builder call basis to prevent accidental strange + # behavior like env['NINJA_POOL'] = 'console' and sub_env can be + # the global Environment object if node.env is None. + # Example: + # + # Allowed: + # + # env.Command("ls", NINJA_POOL="ls_pool") + # + # Not allowed and ignored: + # + # env["NINJA_POOL"] = "ls_pool" + # env.Command("ls") + # + # TODO: Why not alloe env['NINJA_POOL'] ? (bdbaddog) + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["NINJA_POOL"] + + return ninja_build + + +def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom_env={}): + """Generate a response file command provider for rule name.""" + + # If win32 using the environment with a response file command will cause + # ninja to fail to create the response file. Additionally since these rules + # generally are not piping through cmd.exe /c any environment variables will + # make CreateProcess fail to start. + # + # On POSIX we can still set environment variables even for compile + # commands so we do so. + use_command_env = not env["PLATFORM"] == "win32" + if "$" in tool: + tool_is_dynamic = True + + def get_response_file_command(env, node, action, targets, sources, executor=None): + if hasattr(action, "process"): + cmd_list, _, _ = action.process(targets, sources, env, executor=executor) + cmd_list = [str(c).replace("$", "$$") for c in cmd_list[0]] + else: + command = generate_command( + env, node, action, targets, sources, executor=executor + ) + cmd_list = shlex.split(command) + + if tool_is_dynamic: + tool_command = env.subst( + tool, target=targets, source=sources, executor=executor + ) + else: + tool_command = tool + + try: + # Add 1 so we always keep the actual tool inside of cmd + tool_idx = cmd_list.index(tool_command) + 1 + except ValueError: + raise Exception( + "Could not find tool {} in {} generated from {}".format( + tool, cmd_list, get_comstr(env, action, targets, sources) + ) + ) + + cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] + rsp_content = ['"' + rsp_content_item + '"' for rsp_content_item in rsp_content] + rsp_content = " ".join(rsp_content) + + variables = {"rspc": rsp_content, rule: cmd} + if use_command_env: + variables["env"] = get_command_env(env) + + for key, value in custom_env.items(): + variables["env"] += env.subst( + "export %s=%s;" % (key, value), target=targets, source=sources, executor=executor + ) + " " + return rule, variables, [tool_command] + + return get_response_file_command \ No newline at end of file diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index a7c3584582..f906b33fb7 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -35,9 +35,9 @@ from .Globals import COMMAND_TYPES, NINJA_RULES, NINJA_POOLS, \ NINJA_CUSTOM_HANDLERS from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function -from .Util import get_path, alias_to_ninja_build, generate_depfile, ninja_noop, get_command, get_order_only, \ +from .Utils import get_path, alias_to_ninja_build, generate_depfile, ninja_noop, get_order_only, \ get_outputs, get_inputs, get_dependencies, get_rule, get_command_env - +from . import get_command # pylint: disable=too-many-instance-attributes diff --git a/SCons/Tool/ninja/Overrides.py b/SCons/Tool/ninja/Overrides.py index e69de29bb2..80516a26ea 100644 --- a/SCons/Tool/ninja/Overrides.py +++ b/SCons/Tool/ninja/Overrides.py @@ -0,0 +1,96 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +This module is to hold logic which overrides default SCons behavoirs to enable +ninja file generation +""" +import SCons + + +def ninja_hack_linkcom(env): + # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks + # without relying on the SCons hacks that SCons uses by default. + if env["PLATFORM"] == "win32": + from SCons.Tool.mslink import compositeLinkAction + + if env.get("LINKCOM", None) == compositeLinkAction: + env[ + "LINKCOM" + ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' + env[ + "SHLINKCOM" + ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' + + +def ninja_hack_arcom(env): + """ + Force ARCOM so use 's' flag on ar instead of separately running ranlib + """ + if env["PLATFORM"] != "win32" and env.get("RANLIBCOM"): + # There is no way to translate the ranlib list action into + # Ninja so add the s flag and disable ranlib. + # + # This is equivalent to Meson. + # https://github.com/mesonbuild/meson/blob/master/mesonbuild/linkers.py#L143 + old_arflags = str(env["ARFLAGS"]) + if "s" not in old_arflags: + old_arflags += "s" + + env["ARFLAGS"] = SCons.Util.CLVar([old_arflags]) + + # Disable running ranlib, since we added 's' above + env["RANLIBCOM"] = "" + + +class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): + """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" + + def __call__(self, target, source, env, for_signature): + return self.cmd + + def _print_cmd_str(*_args, **_kwargs): + """Disable this method""" + pass + + +def ninja_always_serial(self, num, taskmaster): + """Replacement for SCons.Job.Jobs constructor which always uses the Serial Job class.""" + # We still set self.num_jobs to num even though it's a lie. The + # only consumer of this attribute is the Parallel Job class AND + # the Main.py function which instantiates a Jobs class. It checks + # if Jobs.num_jobs is equal to options.num_jobs, so if the user + # provides -j12 but we set self.num_jobs = 1 they get an incorrect + # warning about this version of Python not supporting parallel + # builds. So here we lie so the Main.py will not give a false + # warning to users. + self.num_jobs = num + self.job = SCons.Job.Serial(taskmaster) + + +# pylint: disable=too-few-public-methods +class AlwaysExecAction(SCons.Action.FunctionAction): + """Override FunctionAction.__call__ to always execute.""" + + def __call__(self, *args, **kwargs): + kwargs["execute"] = 1 + return super().__call__(*args, **kwargs) \ No newline at end of file diff --git a/SCons/Tool/ninja/Rules.py b/SCons/Tool/ninja/Rules.py index c1c238e1d7..a2f6bc5456 100644 --- a/SCons/Tool/ninja/Rules.py +++ b/SCons/Tool/ninja/Rules.py @@ -21,7 +21,7 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from .Util import get_outputs, get_rule, get_inputs, get_dependencies +from .Utils import get_outputs, get_rule, get_inputs, get_dependencies def _install_action_function(_env, node): diff --git a/SCons/Tool/ninja/Util.py b/SCons/Tool/ninja/Utils.py similarity index 72% rename from SCons/Tool/ninja/Util.py rename to SCons/Tool/ninja/Utils.py index 80d1b16d75..18d54dcdf2 100644 --- a/SCons/Tool/ninja/Util.py +++ b/SCons/Tool/ninja/Utils.py @@ -21,12 +21,12 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import os +import shutil from os.path import join as joinpath import SCons from SCons.Action import get_default_ENV, _string_from_cmd_list from SCons.Script import AddOption -from SCons.Tool.ninja.Globals import __NINJA_RULE_MAPPING from SCons.Util import is_List, flatten_sequence @@ -249,99 +249,6 @@ def ninja_noop(*_args, **_kwargs): return None -def get_command(env, node, action): # pylint: disable=too-many-branches - """Get the command to execute for node.""" - if node.env: - sub_env = node.env - else: - sub_env = env - executor = node.get_executor() - tlist, slist = get_targets_sources(node) - - # Generate a real CommandAction - if isinstance(action, SCons.Action.CommandGeneratorAction): - # pylint: disable=protected-access - action = action._generate(tlist, slist, sub_env, 1, executor=executor) - - variables = {} - - comstr = get_comstr(sub_env, action, tlist, slist) - if not comstr: - return None - - provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) - rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) - - # Get the dependencies for all targets - implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) - - # Now add in the other dependencies related to the command, - # e.g. the compiler binary. The ninja rule can be user provided so - # we must do some validation to resolve the dependency path for ninja. - for provider_dep in provider_deps: - - provider_dep = sub_env.subst(provider_dep) - if not provider_dep: - continue - - # If the tool is a node, then SCons will resolve the path later, if its not - # a node then we assume it generated from build and make sure it is existing. - if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep): - implicit.append(provider_dep) - continue - - # in some case the tool could be in the local directory and be suppled without the ext - # such as in windows, so append the executable suffix and check. - prog_suffix = sub_env.get('PROGSUFFIX', '') - provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix - if os.path.exists(provider_dep_ext): - implicit.append(provider_dep_ext) - continue - - # Many commands will assume the binary is in the path, so - # we accept this as a possible input from a given command. - - provider_dep_abspath = sub_env.WhereIs(provider_dep) or sub_env.WhereIs(provider_dep, path=os.environ["PATH"]) - if provider_dep_abspath: - implicit.append(provider_dep_abspath) - continue - - # Possibly these could be ignore and the build would still work, however it may not always - # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided - # ninja rule. - raise Exception("Could not resolve path for %s dependency on node '%s'" % (provider_dep, node)) - - ninja_build = { - "order_only": get_order_only(node), - "outputs": get_outputs(node), - "inputs": get_inputs(node), - "implicit": implicit, - "rule": get_rule(node, rule), - "variables": variables, - } - - # Don't use sub_env here because we require that NINJA_POOL be set - # on a per-builder call basis to prevent accidental strange - # behavior like env['NINJA_POOL'] = 'console' and sub_env can be - # the global Environment object if node.env is None. - # Example: - # - # Allowed: - # - # env.Command("ls", NINJA_POOL="ls_pool") - # - # Not allowed and ignored: - # - # env["NINJA_POOL"] = "ls_pool" - # env.Command("ls") - # - # TODO: Why not alloe env['NINJA_POOL'] ? (bdbaddog) - if node.env and node.env.get("NINJA_POOL", None) is not None: - ninja_build["pool"] = node.env["NINJA_POOL"] - - return ninja_build - - def get_command_env(env): """ Return a string that sets the environment for any environment variables that @@ -413,25 +320,6 @@ def get_comstr(env, action, targets, sources): return action.genstring(targets, sources, env) -def get_generic_shell_command(env, node, action, targets, sources, executor=None): - return ( - "CMD", - { - # TODO: Why is executor passed in and then ignored below? (bdbaddog) - "cmd": generate_command(env, node, action, targets, sources, executor=None), - "env": get_command_env(env), - }, - # Since this function is a rule mapping provider, it must return a list of dependencies, - # and usually this would be the path to a tool, such as a compiler, used for this rule. - # However this function is to generic to be able to reliably extract such deps - # from the command, so we return a placeholder empty list. It should be noted that - # generally this function will not be used solely and is more like a template to generate - # the basics for a custom provider which may have more specific options for a provider - # function for a custom NinjaRuleMapping. - [] - ) - - def generate_command(env, node, action, targets, sources, executor=None): # Actions like CommandAction have a method called process that is # used by SCons to generate the cmd_line they need to run. So @@ -454,4 +342,79 @@ def generate_command(env, node, action, targets, sources, executor=None): cmd = cmd[0:-2].strip() # Escape dollars as necessary - return cmd.replace("$", "$$") \ No newline at end of file + return cmd.replace("$", "$$") + + +def ninja_csig(original): + """Return a dummy csig""" + + def wrapper(self): + if isinstance(self, SCons.Node.Node) and self.is_sconscript(): + return original(self) + return "dummy_ninja_csig" + + return wrapper + + +def ninja_contents(original): + """Return a dummy content without doing IO""" + + def wrapper(self): + if isinstance(self, SCons.Node.Node) and self.is_sconscript(): + return original(self) + return bytes("dummy_ninja_contents", encoding="utf-8") + + return wrapper + + +def ninja_stat(_self, path): + """ + Eternally memoized stat call. + + SCons is very aggressive about clearing out cached values. For our + purposes everything should only ever call stat once since we're + running in a no_exec build the file system state should not + change. For these reasons we patch SCons.Node.FS.LocalFS.stat to + use our eternal memoized dictionary. + """ + + try: + return SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] + except KeyError: + try: + result = os.stat(path) + except os.error: + result = None + + SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] = result + return result + + +def ninja_whereis(thing, *_args, **_kwargs): + """Replace env.WhereIs with a much faster version""" + + # Optimize for success, this gets called significantly more often + # when the value is already memoized than when it's not. + try: + return SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] + except KeyError: + # TODO: Fix this to respect env['ENV']['PATH']... WPD + # We do not honor any env['ENV'] or env[*] variables in the + # generated ninja file. Ninja passes your raw shell environment + # down to it's subprocess so the only sane option is to do the + # same during generation. At some point, if and when we try to + # upstream this, I'm sure a sticking point will be respecting + # env['ENV'] variables and such but it's actually quite + # complicated. I have a naive version but making it always work + # with shell quoting is nigh impossible. So I've decided to + # cross that bridge when it's absolutely required. + path = shutil.which(thing) + SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] = path + return path + + +def ninja_print_conf_log(s, target, source, env): + """Command line print only for conftest to generate a correct conf log.""" + if target and target[0].is_conftest(): + action = SCons.Action._ActionAction() + action.print_cmd_line(s, target, source, env) \ No newline at end of file diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index fd9c13c7e3..ee852a704a 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -26,85 +26,25 @@ import importlib import os -import shlex -import shutil import subprocess import sys -import textwrap -from glob import glob import SCons import SCons.Tool.ninja.Globals from SCons.Script import GetOption from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS +from .Methods import register_custom_handler, register_custom_rule_mapping, register_custom_rule, register_custom_pool, \ + set_build_node_callback, get_generic_shell_command, CheckNinjaCompdbExpand, get_command, \ + gen_get_response_file_command from .NinjaState import NinjaState -from .Util import ninja_add_command_line_options, \ - get_path, ninja_noop, get_command, get_command_env, get_comstr, get_generic_shell_command, \ - generate_command +from .Overrides import ninja_hack_linkcom, ninja_hack_arcom, NinjaNoResponseFiles, ninja_always_serial, AlwaysExecAction +from .Utils import ninja_add_command_line_options, \ + get_path, ninja_noop, ninja_print_conf_log, get_command_env, get_comstr, generate_command, ninja_csig, ninja_contents, ninja_stat, ninja_whereis, ninja_csig, ninja_contents NINJA_STATE = None -def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom_env={}): - """Generate a response file command provider for rule name.""" - - # If win32 using the environment with a response file command will cause - # ninja to fail to create the response file. Additionally since these rules - # generally are not piping through cmd.exe /c any environment variables will - # make CreateProcess fail to start. - # - # On POSIX we can still set environment variables even for compile - # commands so we do so. - use_command_env = not env["PLATFORM"] == "win32" - if "$" in tool: - tool_is_dynamic = True - - def get_response_file_command(env, node, action, targets, sources, executor=None): - if hasattr(action, "process"): - cmd_list, _, _ = action.process(targets, sources, env, executor=executor) - cmd_list = [str(c).replace("$", "$$") for c in cmd_list[0]] - else: - command = generate_command( - env, node, action, targets, sources, executor=executor - ) - cmd_list = shlex.split(command) - - if tool_is_dynamic: - tool_command = env.subst( - tool, target=targets, source=sources, executor=executor - ) - else: - tool_command = tool - - try: - # Add 1 so we always keep the actual tool inside of cmd - tool_idx = cmd_list.index(tool_command) + 1 - except ValueError: - raise Exception( - "Could not find tool {} in {} generated from {}".format( - tool, cmd_list, get_comstr(env, action, targets, sources) - ) - ) - - cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] - rsp_content = ['"' + rsp_content_item + '"' for rsp_content_item in rsp_content] - rsp_content = " ".join(rsp_content) - - variables = {"rspc": rsp_content} - variables[rule] = cmd - if use_command_env: - variables["env"] = get_command_env(env) - - for key, value in custom_env.items(): - variables["env"] += env.subst( - "export %s=%s;" % (key, value), target=targets, source=sources, executor=executor - ) + " " - return rule, variables, [tool_command] - - return get_response_file_command - - def ninja_builder(env, target, source): """Generate a build.ninja for source.""" if not isinstance(source, list): @@ -120,6 +60,7 @@ def ninja_builder(env, target, source): NINJA_STATE.generate() if env["PLATFORM"] == "win32": + # TODO: Is this necessary as you set env variable in the ninja build file per target? # this is not great, its doesn't consider specific # node environments, which means on linux the build could # behave differently, because on linux you can set the environment @@ -170,193 +111,6 @@ def execute_ninja(): # prone to failure with such a simple check erase_previous = output.startswith('[') -# pylint: disable=too-few-public-methods -class AlwaysExecAction(SCons.Action.FunctionAction): - """Override FunctionAction.__call__ to always execute.""" - - def __call__(self, *args, **kwargs): - kwargs["execute"] = 1 - return super().__call__(*args, **kwargs) - - -def register_custom_handler(env, name, handler): - """Register a custom handler for SCons function actions.""" - env[NINJA_CUSTOM_HANDLERS][name] = handler - - -def register_custom_rule_mapping(env, pre_subst_string, rule): - """Register a function to call for a given rule.""" - SCons.Tool.ninja.Globals.__NINJA_RULE_MAPPING[pre_subst_string] = rule - - -def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): - """Allows specification of Ninja rules from inside SCons files.""" - rule_obj = { - "command": command, - "description": description if description else "{} $out".format(rule), - } - - if use_depfile: - rule_obj["depfile"] = os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile') - - if deps is not None: - rule_obj["deps"] = deps - - if pool is not None: - rule_obj["pool"] = pool - - if use_response_file: - rule_obj["rspfile"] = "$out.rsp" - rule_obj["rspfile_content"] = response_file_content - - env[NINJA_RULES][rule] = rule_obj - - -def register_custom_pool(env, pool, size): - """Allows the creation of custom Ninja pools""" - env[NINJA_POOLS][pool] = size - - -def set_build_node_callback(env, node, callback): - if not node.is_conftest(): - node.attributes.ninja_build_callback = callback - - -def ninja_csig(original): - """Return a dummy csig""" - - def wrapper(self): - if isinstance(self, SCons.Node.Node) and self.is_sconscript(): - return original(self) - return "dummy_ninja_csig" - - return wrapper - - -def ninja_contents(original): - """Return a dummy content without doing IO""" - - def wrapper(self): - if isinstance(self, SCons.Node.Node) and self.is_sconscript(): - return original(self) - return bytes("dummy_ninja_contents", encoding="utf-8") - - return wrapper - - -def CheckNinjaCompdbExpand(env, context): - """ Configure check testing if ninja's compdb can expand response files""" - - context.Message('Checking if ninja compdb can expand response files... ') - ret, output = context.TryAction( - action='ninja -f $SOURCE -t compdb -x CMD_RSP > $TARGET', - extension='.ninja', - text=textwrap.dedent(""" - rule CMD_RSP - command = $cmd @$out.rsp > fake_output.txt - description = Building $out - rspfile = $out.rsp - rspfile_content = $rspc - build fake_output.txt: CMD_RSP fake_input.txt - cmd = echo - pool = console - rspc = "test" - """)) - result = '@fake_output.txt.rsp' not in output - context.Result(result) - return result - - -def ninja_stat(_self, path): - """ - Eternally memoized stat call. - - SCons is very aggressive about clearing out cached values. For our - purposes everything should only ever call stat once since we're - running in a no_exec build the file system state should not - change. For these reasons we patch SCons.Node.FS.LocalFS.stat to - use our eternal memoized dictionary. - """ - - try: - return SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] - except KeyError: - try: - result = os.stat(path) - except os.error: - result = None - - SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] = result - return result - - -def ninja_whereis(thing, *_args, **_kwargs): - """Replace env.WhereIs with a much faster version""" - - # Optimize for success, this gets called significantly more often - # when the value is already memoized than when it's not. - try: - return SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] - except KeyError: - # We do not honor any env['ENV'] or env[*] variables in the - # generated ninja file. Ninja passes your raw shell environment - # down to it's subprocess so the only sane option is to do the - # same during generation. At some point, if and when we try to - # upstream this, I'm sure a sticking point will be respecting - # env['ENV'] variables and such but it's actually quite - # complicated. I have a naive version but making it always work - # with shell quoting is nigh impossible. So I've decided to - # cross that bridge when it's absolutely required. - path = shutil.which(thing) - SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] = path - return path - - -def ninja_always_serial(self, num, taskmaster): - """Replacement for SCons.Job.Jobs constructor which always uses the Serial Job class.""" - # We still set self.num_jobs to num even though it's a lie. The - # only consumer of this attribute is the Parallel Job class AND - # the Main.py function which instantiates a Jobs class. It checks - # if Jobs.num_jobs is equal to options.num_jobs, so if the user - # provides -j12 but we set self.num_jobs = 1 they get an incorrect - # warning about this version of Python not supporting parallel - # builds. So here we lie so the Main.py will not give a false - # warning to users. - self.num_jobs = num - self.job = SCons.Job.Serial(taskmaster) - -def ninja_hack_linkcom(env): - # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks - # without relying on the SCons hacks that SCons uses by default. - if env["PLATFORM"] == "win32": - from SCons.Tool.mslink import compositeLinkAction - - if env.get("LINKCOM", None) == compositeLinkAction: - env[ - "LINKCOM" - ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' - env[ - "SHLINKCOM" - ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' - - -def ninja_print_conf_log(s, target, source, env): - """Command line print only for conftest to generate a correct conf log.""" - if target and target[0].is_conftest(): - action = SCons.Action._ActionAction() - action.print_cmd_line(s, target, source, env) - - -class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): - """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" - - def __call__(self, target, source, env, for_signature): - return self.cmd - - def _print_cmd_str(*_args, **_kwargs): - """Disable this method""" - pass - def exists(env): """Enable if called.""" @@ -529,20 +283,8 @@ def robust_rule_mapping(var, rule, tool): # TODO: switch to using SCons to help determine this (Github Issue #3624) env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] - if env["PLATFORM"] != "win32" and env.get("RANLIBCOM"): - # There is no way to translate the ranlib list action into - # Ninja so add the s flag and disable ranlib. - # - # This is equivalent to Meson. - # https://github.com/mesonbuild/meson/blob/master/mesonbuild/linkers.py#L143 - old_arflags = str(env["ARFLAGS"]) - if "s" not in old_arflags: - old_arflags += "s" - - env["ARFLAGS"] = SCons.Util.CLVar([old_arflags]) - - # Disable running ranlib, since we added 's' above - env["RANLIBCOM"] = "" + # Force ARCOM so use 's' flag on ar instead of separately running ranlib + ninja_hack_arcom(env) if GetOption('disable_ninja'): return env From bd30cb0867419f8c8caa40a802be7f2c2d3e191e Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 24 May 2021 16:11:14 -0700 Subject: [PATCH 075/163] Fix sider complaints --- SCons/Tool/ninja/Overrides.py | 2 +- SCons/Tool/ninja/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/ninja/Overrides.py b/SCons/Tool/ninja/Overrides.py index 80516a26ea..c6f12074b7 100644 --- a/SCons/Tool/ninja/Overrides.py +++ b/SCons/Tool/ninja/Overrides.py @@ -21,7 +21,7 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -This module is to hold logic which overrides default SCons behavoirs to enable +This module is to hold logic which overrides default SCons behaviors to enable ninja file generation """ import SCons diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index ee852a704a..8207b15860 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -40,7 +40,7 @@ from .NinjaState import NinjaState from .Overrides import ninja_hack_linkcom, ninja_hack_arcom, NinjaNoResponseFiles, ninja_always_serial, AlwaysExecAction from .Utils import ninja_add_command_line_options, \ - get_path, ninja_noop, ninja_print_conf_log, get_command_env, get_comstr, generate_command, ninja_csig, ninja_contents, ninja_stat, ninja_whereis, ninja_csig, ninja_contents + ninja_noop, ninja_print_conf_log, ninja_csig, ninja_contents, ninja_stat, ninja_whereis NINJA_STATE = None From ceba1ffedee631797f91b811dfb3bd197c92efce Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 30 May 2021 14:42:27 -0700 Subject: [PATCH 076/163] Address mwichmann's comments on PR. Mostly doc and a few import related code changes --- SCons/Tool/ninja/NinjaState.py | 6 +- SCons/Tool/ninja/ninja.xml | 5 +- SCons/__init__.py | 6 +- doc/generated/builders.gen | 4 +- .../examples/caching_ex-random_1.xml | 2 +- .../examples/troubleshoot_Dump_1.xml | 4 +- .../examples/troubleshoot_Dump_2.xml | 4 +- .../examples/troubleshoot_explain1_3.xml | 2 +- .../examples/troubleshoot_stacktrace_2.xml | 4 +- doc/generated/functions.gen | 816 ++++++++++-------- doc/generated/tools.gen | 2 +- doc/user/external.xml | 521 +++++------ 12 files changed, 745 insertions(+), 631 deletions(-) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index f906b33fb7..14704eb10a 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -31,13 +31,13 @@ import SCons from SCons.Script import COMMAND_LINE_TARGETS from SCons.Util import is_List -import SCons.Tool.ninja.Globals +from SCons.Errors import InternalError from .Globals import COMMAND_TYPES, NINJA_RULES, NINJA_POOLS, \ NINJA_CUSTOM_HANDLERS from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function from .Utils import get_path, alias_to_ninja_build, generate_depfile, ninja_noop, get_order_only, \ get_outputs, get_inputs, get_dependencies, get_rule, get_command_env -from . import get_command +from .Methods import get_command # pylint: disable=too-many-instance-attributes @@ -259,7 +259,7 @@ def add_build(self, node): node_string = str(node) if node_string in self.builds: - raise Exception("Node {} added to ninja build state more than once".format(node_string)) + raise InternalError("Node {} added to ninja build state more than once".format(node_string)) self.builds[node_string] = build self.built.update(build["outputs"]) return True diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index c28d697953..2ea9fd67ac 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -74,6 +74,7 @@ See its __doc__ string for a discussion of the format. CCCOM CXXCOM SHCXXCOM + SHCCCOM CC CXX @@ -130,9 +131,7 @@ See its __doc__ string for a discussion of the format. - You must load the &t-ninja; tool prior to specifying - any part of your build or some source/output - files will not show up in the compilation database. + You must specify scons --experimental=ninja to enable this tool. To use this tool you must install pypi's ninja diff --git a/SCons/__init__.py b/SCons/__init__.py index bc0f428f53..cd521d1f1c 100644 --- a/SCons/__init__.py +++ b/SCons/__init__.py @@ -1,9 +1,9 @@ __version__="4.1.1a" __copyright__="Copyright (c) 2001 - 2021 The SCons Foundation" __developer__="bdbaddog" -__date__="2021-04-18 22:22:37" +__date__="2021-05-30 21:33:41" __buildsys__="ProDog2020" -__revision__="1850bdd13c7668aac493b9f5b896578813e60ca9" -__build__="1850bdd13c7668aac493b9f5b896578813e60ca9" +__revision__="b4c7a0966c36d37d0c2ea125dd98bff0061e28f2" +__build__="b4c7a0966c36d37d0c2ea125dd98bff0061e28f2" # make sure compatibility is always in place import SCons.compat # noqa \ No newline at end of file diff --git a/doc/generated/builders.gen b/doc/generated/builders.gen index d3490f0ca0..ba90e6cfbd 100644 --- a/doc/generated/builders.gen +++ b/doc/generated/builders.gen @@ -1258,9 +1258,7 @@ env.MSVSSolution( - You must load the &t-ninja; tool prior to specifying - any part of your build or some source/output - files will not show up in the compilation database. + You must specify scons --experimental=ninja to enable this tool. To use this tool you must install pypi's ninja diff --git a/doc/generated/examples/caching_ex-random_1.xml b/doc/generated/examples/caching_ex-random_1.xml index a5593d302d..1570d7df30 100644 --- a/doc/generated/examples/caching_ex-random_1.xml +++ b/doc/generated/examples/caching_ex-random_1.xml @@ -1,8 +1,8 @@ % scons -Q +cc -o f4.o -c f4.c cc -o f2.o -c f2.c cc -o f1.o -c f1.c cc -o f3.o -c f3.c -cc -o f4.o -c f4.c cc -o f5.o -c f5.c cc -o prog f1.o f2.o f3.o f4.o f5.o diff --git a/doc/generated/examples/troubleshoot_Dump_1.xml b/doc/generated/examples/troubleshoot_Dump_1.xml index 8f4e624d94..2b4178bd8d 100644 --- a/doc/generated/examples/troubleshoot_Dump_1.xml +++ b/doc/generated/examples/troubleshoot_Dump_1.xml @@ -62,8 +62,8 @@ scons: Reading SConscript files ... 'TEMPFILEARGJOIN': ' ', 'TEMPFILEPREFIX': '@', 'TOOLS': ['install', 'install'], - '_CPPDEFFLAGS': '${_defines(CPPDEFPREFIX, CPPDEFINES, CPPDEFSUFFIX, ' - '__env__)}', + '_CPPDEFFLAGS': '${_defines(CPPDEFPREFIX, CPPDEFINES, CPPDEFSUFFIX, __env__, ' + 'TARGET, SOURCE)}', '_CPPINCFLAGS': '$( ${_concat(INCPREFIX, CPPPATH, INCSUFFIX, __env__, RDirs, ' 'TARGET, SOURCE)} $)', '_LIBDIRFLAGS': '$( ${_concat(LIBDIRPREFIX, LIBPATH, LIBDIRSUFFIX, __env__, ' diff --git a/doc/generated/examples/troubleshoot_Dump_2.xml b/doc/generated/examples/troubleshoot_Dump_2.xml index f2fc5e6f6f..c2fced13c1 100644 --- a/doc/generated/examples/troubleshoot_Dump_2.xml +++ b/doc/generated/examples/troubleshoot_Dump_2.xml @@ -107,8 +107,8 @@ scons: Reading SConscript files ... 'TOOLS': ['msvc', 'install', 'install'], 'VSWHERE': None, '_CCCOMCOM': '$CPPFLAGS $_CPPDEFFLAGS $_CPPINCFLAGS $CCPCHFLAGS $CCPDBFLAGS', - '_CPPDEFFLAGS': '${_defines(CPPDEFPREFIX, CPPDEFINES, CPPDEFSUFFIX, ' - '__env__)}', + '_CPPDEFFLAGS': '${_defines(CPPDEFPREFIX, CPPDEFINES, CPPDEFSUFFIX, __env__, ' + 'TARGET, SOURCE)}', '_CPPINCFLAGS': '$( ${_concat(INCPREFIX, CPPPATH, INCSUFFIX, __env__, RDirs, ' 'TARGET, SOURCE)} $)', '_LIBDIRFLAGS': '$( ${_concat(LIBDIRPREFIX, LIBPATH, LIBDIRSUFFIX, __env__, ' diff --git a/doc/generated/examples/troubleshoot_explain1_3.xml b/doc/generated/examples/troubleshoot_explain1_3.xml index 6b90e1cabe..14a105e048 100644 --- a/doc/generated/examples/troubleshoot_explain1_3.xml +++ b/doc/generated/examples/troubleshoot_explain1_3.xml @@ -2,5 +2,5 @@ cp file.in file.oout scons: warning: Cannot find target file.out after building -File "/Users/bdbaddog/devel/scons/git/as_scons/scripts/scons.py", line 96, in <module> +File "/Users/bdbaddog/devel/scons/git/scons-bugfixes-4/scripts/scons.py", line 96, in <module> diff --git a/doc/generated/examples/troubleshoot_stacktrace_2.xml b/doc/generated/examples/troubleshoot_stacktrace_2.xml index 8e8809c461..75dd6fba03 100644 --- a/doc/generated/examples/troubleshoot_stacktrace_2.xml +++ b/doc/generated/examples/troubleshoot_stacktrace_2.xml @@ -3,10 +3,10 @@ scons: *** [prog.o] Source `prog.c' not found, needed by target `prog.o'. scons: internal stack trace: File "SCons/Job.py", line 195, in start task.prepare() - File "SCons/Script/Main.py", line 179, in prepare + File "SCons/Script/Main.py", line 178, in prepare return SCons.Taskmaster.OutOfDateTask.prepare(self) File "SCons/Taskmaster.py", line 186, in prepare executor.prepare() - File "SCons/Executor.py", line 424, in prepare + File "SCons/Executor.py", line 418, in prepare raise SCons.Errors.StopError(msg % (s, self.batches[0].targets[0])) diff --git a/doc/generated/functions.gen b/doc/generated/functions.gen index d1e730a4e3..fe2f516b5d 100644 --- a/doc/generated/functions.gen +++ b/doc/generated/functions.gen @@ -13,8 +13,8 @@ - Action(action, [cmd/str/fun, [var, ...]] [option=value, ...]) - env.Action(action, [cmd/str/fun, [var, ...]] [option=value, ...]) + Action(action, [output, [var, ...]] [key=value, ...]) + env.Action(action, [output, [var, ...]] [key=value, ...]) A factory function to create an Action object for the specified @@ -507,6 +507,58 @@ Preprocessor macros can be valued, or un-valued, as in The macro can be be supplied as a complete string including the value, or as a tuple (or list) of macro, value, or as a dictionary. Example (again gcc syntax in the expanded defines): + + + +env = Environment(CPPDEFINES="FOO") +print("CPPDEFINES={}".format(env['CPPDEFINES'])) +env.Append(CPPDEFINES="BAR=1") +print("CPPDEFINES={}".format(env['CPPDEFINES'])) +env.Append(CPPDEFINES=("OTHER", 2)) +print("CPPDEFINES={}".format(env['CPPDEFINES'])) +env.Append(CPPDEFINES={"EXTRA": "arg"}) +print("CPPDEFINES={}".format(env['CPPDEFINES'])) +print("CPPDEFINES will expand to {}".format(env.subst("$_CPPDEFFLAGS"))) + + + +$ scons -Q +CPPDEFINES=FOO +CPPDEFINES=['FOO', 'BAR=1'] +CPPDEFINES=['FOO', 'BAR=1', ('OTHER', 2)] +CPPDEFINES=['FOO', 'BAR=1', ('OTHER', 2), {'EXTRA': 'arg'}] +CPPDEFINES will expand to -DFOO -DBAR=1 -DOTHER=2 -DEXTRA=arg +scons: `.' is up to date. + + + +Adding a string val +to a dictonary &consvar; will enter +val as the key in the dict, +and None as its value. +Using a tuple type to supply a key + value only works +for the special case of &cv-link-CPPDEFINES; +described above. + + + +Although most combinations of types work without +needing to know the details, some combinations +do not make sense and a Python exception will be raised. + + + +When using &f-env-Append; to modify &consvars; +which are path specifications (normally, +those names which end in PATH), +it is recommended to add the values as a list of strings, +even if there is only a single string to add. +The same goes for adding library names to &cv-LIBS;. + + + +env.Append(CPPPATH=["#/include"]) + See also &f-link-env-AppendUnique;, @@ -2072,261 +2124,242 @@ file is found. env.GetOption(name) This function provides a way to query the value of -SCons options set on scons command line -(or set using the -&f-link-SetOption; -function). -The options supported are: - - - - - -cache_debug - - -which corresponds to ; - - - - -cache_disable - - -which corresponds to ; - - - - -cache_force - - -which corresponds to ; - - - - -cache_show - - -which corresponds to ; - - - - -clean - - -which corresponds to , -and ; - - - - -config - - -which corresponds to ; - - - - -directory - - -which corresponds to and ; - - - - -diskcheck - - -which corresponds to ; - - - - -duplicate - - -which corresponds to ; - - - - -file - - -which corresponds to , , and ; - - - - -help - - -which corresponds to and ; - - - - -ignore_errors - - -which corresponds to ; - - - - -implicit_cache - - -which corresponds to ; - - - - -implicit_deps_changed - - -which corresponds to ; - - - - -implicit_deps_unchanged - - -which corresponds to ; - - - - -interactive - - -which corresponds to and ; +options which can be set via the command line or using the +&f-link-SetOption; function. - - - -keep_going - - -which corresponds to and ; - - - - -max_drift - - -which corresponds to ; - - - - -no_exec - - -which corresponds to , -, , - and ; - - - - -no_site_dir - - -which corresponds to ; - - - - -num_jobs - - -which corresponds to and ; - - - - -profile_file - - -which corresponds to ; - - - - -question - - -which corresponds to and ; - - - - -random - - -which corresponds to ; - - - - -repository - -which corresponds to , and ; - - - - -silent - - -which corresponds to , and ; - - - - -site_dir - - -which corresponds to ; - - - - -stack_size - - -which corresponds to ; - - - - -taskmastertrace_file - - -which corresponds to ; and - - - - -warn - - -which corresponds to and . - - - - +name can be an entry from the following table, +which shows the corresponding command line arguments +that could affect the value. +name can be also be the destination +variable name from a project-specific option added using the +&f-link-AddOption; function, as long as the addition +happens prior to the &f-GetOption; call in the SConscript files. + + + + + Query name + Command-line options + Notes + + + + + + cache_debug + + + + cache_disable + + , + + + + + cache_force + + , + + + + + cache_readonly + + + + cache_show + + + + clean + + , + , + + + + + climb_up + + + + + + + + + + config + + + + debug + + + + directory + , + + + diskcheck + + + + duplicate + + + + enable_virtualenv + + + + experimental + + since 4.2 + + + file + + , + , + , + + + + + hash_format + + since 4.2 + + + help + , + + + ignore_errors + , + + + ignore_virtualenv + + + + implicit_cache + + + + implicit_deps_changed + + + + implicit_deps_unchanged + + + + include_dir + , + + + install_sandbox + + Available only if the &t-link-install; tool has been called + + + keep_going + , + + + max_drift + + + + md5_chunksize + + , + + + since 4.2 + + + no_exec + + , + , + , + , + + + + + no_progress + + + + num_jobs + , + + + package_type + + Available only if the &t-link-packaging; tool has been called + + + profile_file + + + + question + , + + + random + + + + repository + + , + , + + + + + silent + + , + , + + + + + site_dir + , + + + stack_size + + + + taskmastertrace_file + + + + tree_printers + + + + warn + , + + + + + + See the documentation for the corresponding command line option for information about each specific @@ -3827,15 +3860,33 @@ if 'FOO' not in env: Sets &scons; option variable name to value. These options are all also settable via -&scons; command-line options but the variable name -may differ from the command-line option name (see table). +command-line options but the variable name +may differ from the command-line option name - +see the table for correspondences. A value set via command-line option will take precedence over one set with &f-SetOption;, which allows setting a project default in the scripts and temporarily overriding it via command line. +&f-SetOption; calls can also be placed in the +site_init.py file. + + + +See the documentation in the manpage for the +corresponding command line option for information about each specific option. +The value parameter is mandatory, +for option values which are boolean in nature +(that is, the command line option does not take an argument) +use a value +which evaluates to true (e.g. True, +1) or false (e.g. False, +0). + + + Options which affect the reading and processing of SConscript files -are not settable this way, since those files must -be read in order to find the &f-SetOption; call. +are not settable using &f-SetOption; since those files must +be read in order to find the &f-SetOption; call in the first place. @@ -3843,126 +3894,161 @@ The settable variables with their associated command-line options are: - + -VariableCommand-line options + + Settable name + Command-line options + Notes + + - -clean - -, , - - -diskcheck - - - - - -duplicate - - - - - - - - experimental - - - - - - - - -help - -, - - -implicit_cache - - - - - -max_drift - - - - -md5_chunksize - - - - -no_exec - -, , -, , - - - -no_progress - - - - -num_jobs - -, - - -random - - - - -silent - -. - - -stack_size - - - - -warn - -. - + + clean + + , + , + + + + + + diskcheck + + + + + duplicate + + + + + experimental + + since 4.2 + + + + hash_chunksize + + + Actually sets md5_chunksize. + since 4.2 + + + + + hash_format + + since 4.2 + + + + help + , + + + + implicit_cache + + + + + implicit_deps_changed + + + Also sets implicit_cache. + (settable since 4.2) + + + + + implicit_deps_unchanged + + + Also sets implicit_cache. + (settable since 4.2) + + + + + max_drift + + + + + md5_chunksize + + + + + no_exec + + , + , + , + , + + + + + + no_progress + + See + + If no_progress is set via &f-SetOption; + in an SConscript file + (but not if set in a site_init.py file) + there will still be an initial status message about + reading SConscript files since &SCons; has + to start reading them before it can see the + &f-SetOption;. + + + + + + + num_jobs + , + + + + random + + + + + silent + + , + , + + + + + + stack_size + + + + + warn + + + - -See the documentation in the manpage for the -corresponding command line option for information about each specific option. -Option values which are boolean in nature (that is, they are -either on or off) should be set to a true value (True, -1) or a false value (False, -0). - - - - -If no_progress is set via &f-SetOption; -there will still be initial progress output as &SCons; has -to start reading SConscript files before it can see the -&f-SetOption; in an SConscript file: -scons: Reading SConscript files ... - - - Example: -SetOption('max_drift', True) +SetOption('max_drift', 0) diff --git a/doc/generated/tools.gen b/doc/generated/tools.gen index bdb353ce67..01b69f593e 100644 --- a/doc/generated/tools.gen +++ b/doc/generated/tools.gen @@ -816,7 +816,7 @@ Sets construction variables for the Sets up &b-link-Ninja; builder which generates a ninja build file, and then optionally runs ninja. - Sets: &cv-link-DISABLE_AUTO_NINJA;, &cv-link-NINJA_ALIAS_NAME;, &cv-link-NINJA_BUILDDIR;, &cv-link-NINJA_COMPDB_EXPAND;, &cv-link-NINJA_ENV_VAR_CACHE;, &cv-link-NINJA_FILE_NAME;, &cv-link-NINJA_GENERATED_SOURCE_SUFFIXES;, &cv-link-NINJA_MSVC_DEPS_PREFIX;, &cv-link-NINJA_POOL;, &cv-link-NINJA_REGENERATE_DEPS;, &cv-link-NINJA_SYNTAX;, &cv-link-_NINJA_REGENERATE_DEPS_FUNC;, &cv-link-__NINJA_NO;.Uses: &cv-link-AR;, &cv-link-ARCOM;, &cv-link-ARFLAGS;, &cv-link-CC;, &cv-link-CCCOM;, &cv-link-CXX;, &cv-link-CXXCOM;, &cv-link-ESCAPE;, &cv-link-LINK;, &cv-link-LINKCOM;, &cv-link-PLATFORM;, &cv-link-RANLIB;, &cv-link-RANLIBCOM;, &cv-link-SHCXXCOM;, &cv-link-SHLINK;, &cv-link-SHLINKCOM;. + Sets: &cv-link-DISABLE_AUTO_NINJA;, &cv-link-NINJA_ALIAS_NAME;, &cv-link-NINJA_BUILDDIR;, &cv-link-NINJA_COMPDB_EXPAND;, &cv-link-NINJA_ENV_VAR_CACHE;, &cv-link-NINJA_FILE_NAME;, &cv-link-NINJA_GENERATED_SOURCE_SUFFIXES;, &cv-link-NINJA_MSVC_DEPS_PREFIX;, &cv-link-NINJA_POOL;, &cv-link-NINJA_REGENERATE_DEPS;, &cv-link-NINJA_SYNTAX;, &cv-link-_NINJA_REGENERATE_DEPS_FUNC;, &cv-link-__NINJA_NO;.Uses: &cv-link-AR;, &cv-link-ARCOM;, &cv-link-ARFLAGS;, &cv-link-CC;, &cv-link-CCCOM;, &cv-link-CXX;, &cv-link-CXXCOM;, &cv-link-ESCAPE;, &cv-link-LINK;, &cv-link-LINKCOM;, &cv-link-PLATFORM;, &cv-link-RANLIB;, &cv-link-RANLIBCOM;, &cv-link-SHCCCOM;, &cv-link-SHCXXCOM;, &cv-link-SHLINK;, &cv-link-SHLINKCOM;. packaging diff --git a/doc/user/external.xml b/doc/user/external.xml index f580d3ff46..78e0a7d488 100644 --- a/doc/user/external.xml +++ b/doc/user/external.xml @@ -1,290 +1,305 @@ - %scons; + + %scons; - - %builders-mod; - - %functions-mod; - - %tools-mod; - - %variables-mod; + + %builders-mod; + + %functions-mod; + + %tools-mod; + + %variables-mod; -]> + ]> -Using SCons with other build tools + Using SCons with other build tools - + --> - + - Sometimes a project needs to interact with other projects - in various ways. For example, many open source projects - make use of components from other open source projects, - and want to use those in their released form, not recode - their builds into &SCons;. As another example, sometimes - the flexibility and power of &SCons; is useful for managing the - overall project, but developers might like faster incremental - builds when making small changes by using a different tool. + Sometimes a project needs to interact with other projects + in various ways. For example, many open source projects + make use of components from other open source projects, + and want to use those in their released form, not recode + their builds into &SCons;. As another example, sometimes + the flexibility and power of &SCons; is useful for managing the + overall project, but developers might like faster incremental + builds when making small changes by using a different tool. - + - + - This chapter shows some techniques for interacting with other - projects and tools effectively from within &SCons;. + This chapter shows some techniques for interacting with other + projects and tools effectively from within &SCons;. - + -
- Creating a Compilation Database +
+ Creating a Compilation Database - + - Tooling to perform analysis and modification - of source code often needs to know not only the source code - itself, but also how it will be compiled, as the compilation line - affects the behavior of macros, includes, etc. &SCons; has a - record of this information once it has run, in the form of - Actions associated with the sources, and can emit this information - so tools can use it. + Tooling to perform analysis and modification + of source code often needs to know not only the source code + itself, but also how it will be compiled, as the compilation line + affects the behavior of macros, includes, etc. &SCons; has a + record of this information once it has run, in the form of + Actions associated with the sources, and can emit this information + so tools can use it. - + - + - The Clang project has defined a JSON Compilation Database. - This database is in common use as input into Clang tools - and many IDEs and editors as well. - See - - JSON Compilation Database Format Specification - - for complete information. &SCons; can emit a - compilation database in this format - by enabling the &t-link-compilation_db; tool - and calling the &b-link-CompilationDatabase; builder - (available since &scons; 4.0). + The Clang project has defined a JSON Compilation Database. + This database is in common use as input into Clang tools + and many IDEs and editors as well. + See + + JSON Compilation Database Format Specification + + for complete information. &SCons; can emit a + compilation database in this format + by enabling the &t-link-compilation_db; tool + and calling the &b-link-CompilationDatabase; builder + (available since &scons; 4.0). - + - + - The compilation database can be populated with - source and output files either with paths relative - to the top of the build, or using absolute paths. - This is controlled by - COMPILATIONDB_USE_ABSPATH=(True|False) - which defaults to False. - The entries in this file can be filtered by using - - COMPILATIONDB_PATH_FILTER='pattern' - where the filter pattern is a string following the Python - - fnmatch - - syntax. - This filtering can be used for outputting different - build variants to different compilation database files. + The compilation database can be populated with + source and output files either with paths relative + to the top of the build, or using absolute paths. + This is controlled by + COMPILATIONDB_USE_ABSPATH=(True|False) + which defaults to False. + The entries in this file can be filtered by using + + COMPILATIONDB_PATH_FILTER='pattern' + where the filter pattern is a string following the Python + + fnmatch + + syntax. + This filtering can be used for outputting different + build variants to different compilation database files. - + - + - The following example illustrates generating a compilation - database containing absolute paths: + The following example illustrates generating a compilation + database containing absolute paths: - + - - -env = Environment(COMPILATIONDB_USE_ABSPATH=True) -env.Tool('compilation_db') -env.CompilationDatabase() -env.Program('hello.c') - - -int main( int argc, char* argv[] ) -{ - return 0; -} - - - - - scons -Q - - - compile_commands.json contains: - - -[ - { - "command": "gcc -o hello.o -c hello.c", - "directory": "/home/user/sandbox", - "file": "/home/user/sandbox/hello.c", - "output": "/home/user/sandbox/hello.o" - } -] - + + + env = Environment(COMPILATIONDB_USE_ABSPATH=True) + env.Tool('compilation_db') + env.CompilationDatabase() + env.Program('hello.c') + + + int main( int argc, char* argv[] ) + { + return 0; + } + + + + + scons -Q + - + + compile_commands.json + contains: + - Notice that the generated database contains only an entry for - the hello.c/hello.o pairing, - and nothing for the generation of the final executable - hello - the transformation of - hello.o to hello - does not have any information that affects interpretation - of the source code, - so it is not interesting to the compilation database. + + [ + { + "command": "gcc -o hello.o -c hello.c", + "directory": "/home/user/sandbox", + "file": "/home/user/sandbox/hello.c", + "output": "/home/user/sandbox/hello.o" + } + ] + - + - + Notice that the generated database contains only an entry for + the hello.c/hello.o pairing, + and nothing for the generation of the final executable + hello + - the transformation of + hello.o + to + hello + does not have any information that affects interpretation + of the source code, + so it is not interesting to the compilation database. - Although it can be a little surprising at first glance, - a compilation database target is, like any other target, - subject to &scons; target selection rules. - This means if you set a default target (that does not - include the compilation database), or use command-line - targets, it might not be selected for building. - This can actually be an advantage, since you don't - necessarily want to regenerate the compilation database - every build. - The following example - shows selecting relative paths (the default) - for output and source, - and also giving a non-default name to the database. - In order to be able to generate the database separately from building, - an alias is set referring to the database, - which can then be used as a target - here we are only - building the compilation database target, not the code. + - + - - -env = Environment() -env.Tool('compilation_db') -cdb = env.CompilationDatabase('compile_database.json') -Alias('cdb', cdb) -env.Program('test_main.c') - - -#include "test_main.h" -int main( int argc, char* argv[] ) -{ - return 0; -} - - -/* dummy include file */ - - - - - scons -Q cdb - - - compile_database.json contains: - - -[ - { - "command": "gcc -o test_main.o -c test_main.c", - "directory": "/home/user/sandbox", - "file": "test_main.c", - "output": "test_main.o" - } -] - + Although it can be a little surprising at first glance, + a compilation database target is, like any other target, + subject to &scons; target selection rules. + This means if you set a default target (that does not + include the compilation database), or use command-line + targets, it might not be selected for building. + This can actually be an advantage, since you don't + necessarily want to regenerate the compilation database + every build. + The following example + shows selecting relative paths (the default) + for output and source, + and also giving a non-default name to the database. + In order to be able to generate the database separately from building, + an alias is set referring to the database, + which can then be used as a target - here we are only + building the compilation database target, not the code. - + - The following (incomplete) example shows using filtering - to separate build variants. - In the case of using variants, - you want different compilation databases for each, - since the build parameters differ, so the code analysis - needs to see the correct build lines for the 32-bit build - and 64-bit build hinted at here. - For simplicity of presentation, - the example omits the setup details of the variant directories: + + + env = Environment() + env.Tool('compilation_db') + cdb = env.CompilationDatabase('compile_database.json') + Alias('cdb', cdb) + env.Program('test_main.c') + + + #include "test_main.h" + int main( int argc, char* argv[] ) + { + return 0; + } + + + /* dummy include file */ + + + + + scons -Q cdb + - + + compile_database.json + contains: + + + + [ + { + "command": "gcc -o test_main.o -c test_main.c", + "directory": "/home/user/sandbox", + "file": "test_main.c", + "output": "test_main.o" + } + ] + + + + + The following (incomplete) example shows using filtering + to separate build variants. + In the case of using variants, + you want different compilation databases for each, + since the build parameters differ, so the code analysis + needs to see the correct build lines for the 32-bit build + and 64-bit build hinted at here. + For simplicity of presentation, + the example omits the setup details of the variant directories: + + + + + env = Environment() + env.Tool('compilation_db') + + env1 = env.Clone() + env1['COMPILATIONDB_PATH_FILTER'] = 'build/linux32/*' + env1.CompilationDatabase('compile_commands-linux32.json') - -env = Environment() -env.Tool('compilation_db') - -env1 = env.Clone() -env1['COMPILATIONDB_PATH_FILTER'] = 'build/linux32/*' -env1.CompilationDatabase('compile_commands-linux32.json') - -env2 = env.Clone() -env2['COMPILATIONDB_PATH_FILTER'] = 'build/linux64/*' -env2.CompilationDatabase('compile_commands-linux64.json') - - - compile_commands-linux32.json contains: - - -[ - { - "command": "gcc -m32 -o build/linux32/test_main.o -c test_main.c", - "directory": "/home/user/sandbox", - "file": "test_main.c", - "output": "build/linux32/test_main.o" - } -] - - - compile_commands-linux64.json contains: - - -[ - { - "command": "gcc -m64 -o build/linux64/test_main.o -c test_main.c", - "directory": "/home/user/sandbox", - "file": "test_main.c", - "output": "build/linux64/test_main.o" - } -] - - -
+ env2 = env.Clone() + env2['COMPILATIONDB_PATH_FILTER'] = 'build/linux64/*' + env2.CompilationDatabase('compile_commands-linux64.json') + + + + compile_commands-linux32.json + contains: + + + + [ + { + "command": "gcc -m32 -o build/linux32/test_main.o -c test_main.c", + "directory": "/home/user/sandbox", + "file": "test_main.c", + "output": "build/linux32/test_main.o" + } + ] + + + + compile_commands-linux64.json + contains: + + + + [ + { + "command": "gcc -m64 -o build/linux64/test_main.o -c test_main.c", + "directory": "/home/user/sandbox", + "file": "test_main.c", + "output": "build/linux64/test_main.o" + } + ] + + +
Ninja Build Generator @@ -293,16 +308,32 @@ env2.CompilationDatabase('compile_commands-linux64.json') This is an experimental new feature. - Using the + This tool will enabled creating a ninja build file from your SCons based build system. It can then invoke + ninja to run your build. For most builds ninja will be significantly faster, but you may have to give up + some accuracy. You are NOT advised to use this for production builds. It can however significantly speed up + your build/debug/compile iterations. + + + It's not expected that the ninja builder will work for all builds at this point. It's still under active + development. If you find that your build doesn't work with ninja please bring this to the users mailing list + or devel channel on our discord server. + + + Specifically if your build has many (or even any) python function actions you may find that the ninja build + will be slower as it will run ninja, which will then run SCons for each target created by a python action. + To alleviate some of these, especially those python based actions built into SCons there is special logic to + implement those actions via shell commands in the ninja build file. - Ninja Build System + + Ninja Build System + - - Ninja File Format Specification - + + Ninja File Format Specification +
From 0a80633d2d92892df4fb2c6d5860cf7f4749bfe5 Mon Sep 17 00:00:00 2001 From: Mathew Robinson Date: Fri, 24 Jan 2020 17:25:31 -0500 Subject: [PATCH 077/163] [WIP] write a tool to generate build.ninja files from SCons --- SCons/Script/SConscript.py | 3 + SCons/Script/__init__.py | 1 + src/engine/SCons/Tool/ninja.py | 1396 ++++++++++++++++++++++++++++++++ 3 files changed, 1400 insertions(+) create mode 100644 src/engine/SCons/Tool/ninja.py diff --git a/SCons/Script/SConscript.py b/SCons/Script/SConscript.py index 596fca0463..ded0fcfef9 100644 --- a/SCons/Script/SConscript.py +++ b/SCons/Script/SConscript.py @@ -203,9 +203,11 @@ def _SConscript(fs, *files, **kw): if f.rexists(): actual = f.rfile() _file_ = open(actual.get_abspath(), "rb") + SCons.Script.LOADED_SCONSCRIPTS.append(actual.get_abspath()) elif f.srcnode().rexists(): actual = f.srcnode().rfile() _file_ = open(actual.get_abspath(), "rb") + SCons.Script.LOADED_SCONSCRIPTS.append(actual.get_abspath()) elif f.has_src_builder(): # The SConscript file apparently exists in a source # code management system. Build it, but then clear @@ -214,6 +216,7 @@ def _SConscript(fs, *files, **kw): f.build() f.built() f.builder_set(None) + SCons.Script.LOADED_SCONSCRIPTS.append(f.get_abspath()) if f.exists(): _file_ = open(f.get_abspath(), "rb") if _file_: diff --git a/SCons/Script/__init__.py b/SCons/Script/__init__.py index 5f58d9972d..e409f06ef4 100644 --- a/SCons/Script/__init__.py +++ b/SCons/Script/__init__.py @@ -187,6 +187,7 @@ def _clear(self): BUILD_TARGETS = TargetList() COMMAND_LINE_TARGETS = [] DEFAULT_TARGETS = [] +LOADED_SCONSCRIPTS = [] # BUILD_TARGETS can be modified in the SConscript files. If so, we # want to treat the modified BUILD_TARGETS list as if they specified diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py new file mode 100644 index 0000000000..b34759e401 --- /dev/null +++ b/src/engine/SCons/Tool/ninja.py @@ -0,0 +1,1396 @@ +# Copyright 2019 MongoDB Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Generate build.ninja files from SCons aliases.""" + +import sys +import os +import importlib +import io +import shutil + +from threading import Lock +from os.path import join as joinpath +from os.path import splitext + +import SCons +from SCons.Action import _string_from_cmd_list, get_default_ENV +from SCons.Util import is_String, is_List +from SCons.Script import COMMAND_LINE_TARGETS, LOADED_SCONSCRIPTS + +NINJA_SYNTAX = "NINJA_SYNTAX" +NINJA_RULES = "__NINJA_CUSTOM_RULES" +NINJA_POOLS = "__NINJA_CUSTOM_POOLS" +NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" +NINJA_BUILD = "NINJA_BUILD" +NINJA_WHEREIS_MEMO = {} +NINJA_STAT_MEMO = {} +MEMO_LOCK = Lock() + +__NINJA_RULE_MAPPING = {} + +# These are the types that get_command can do something with +COMMAND_TYPES = ( + SCons.Action.CommandAction, + SCons.Action.CommandGeneratorAction, +) + + +def _install_action_function(_env, node): + """Install files using the install or copy commands""" + return { + "outputs": get_outputs(node), + "rule": "INSTALL", + "pool": "install_pool", + "inputs": [get_path(src_file(s)) for s in node.sources], + "implicit": get_dependencies(node), + } + + +def _lib_symlink_action_function(_env, node): + """Create shared object symlinks if any need to be created""" + symlinks = getattr(getattr(node, "attributes", None), "shliblinks", None) + + if not symlinks or symlinks is None: + return None + + outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] + inputs = [link.get_path() for link, _ in symlinks] + + return { + "outputs": outputs, + "inputs": inputs, + "rule": "SYMLINK", + "implicit": get_dependencies(node), + } + + +def is_valid_dependent_node(node): + """ + Return True if node is not an alias or is an alias that has children + + This prevents us from making phony targets that depend on other + phony targets that will never have an associated ninja build + target. + + We also have to specify that it's an alias when doing the builder + check because some nodes (like src files) won't have builders but + are valid implicit dependencies. + """ + return not isinstance(node, SCons.Node.Alias.Alias) or node.children() + + +def alias_to_ninja_build(node): + """Convert an Alias node into a Ninja phony target""" + return { + "outputs": get_outputs(node), + "rule": "phony", + "implicit": [ + get_path(n) for n in node.children() if is_valid_dependent_node(n) + ], + } + + +def get_dependencies(node): + """Return a list of dependencies for node.""" + return [get_path(src_file(child)) for child in node.children()] + + +def get_inputs(node): + """Collect the Ninja inputs for node.""" + executor = node.get_executor() + if executor is not None: + inputs = executor.get_all_sources() + else: + inputs = node.sources + + inputs = [get_path(src_file(o)) for o in inputs] + return inputs + + +def get_outputs(node): + """Collect the Ninja outputs for node.""" + executor = node.get_executor() + if executor is not None: + outputs = executor.get_all_targets() + else: + if hasattr(node, "target_peers"): + outputs = node.target_peers + else: + outputs = [node] + + outputs = [get_path(o) for o in outputs] + return outputs + + +class SConsToNinjaTranslator: + """Translates SCons Actions into Ninja build objects.""" + + def __init__(self, env): + self.env = env + self.func_handlers = { + # Skip conftest builders + "_createSource": ninja_noop, + # SCons has a custom FunctionAction that just makes sure the + # target isn't static. We let the commands that ninja runs do + # this check for us. + "SharedFlagChecker": ninja_noop, + # The install builder is implemented as a function action. + "installFunc": _install_action_function, + "LibSymlinksActionFunction": _lib_symlink_action_function, + } + + self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) + + # pylint: disable=too-many-return-statements + def action_to_ninja_build(self, node, action=None): + """Generate build arguments dictionary for node.""" + # Use False since None is a valid value for this Attribute + build = getattr(node.attributes, NINJA_BUILD, False) + if build is not False: + return build + + if node.builder is None: + return None + + if action is None: + action = node.builder.action + + # Ideally this should never happen, and we do try to filter + # Ninja builders out of being sources of ninja builders but I + # can't fix every DAG problem so we just skip ninja_builders + # if we find one + if node.builder == self.env["BUILDERS"]["Ninja"]: + return None + + if isinstance(action, SCons.Action.FunctionAction): + return self.handle_func_action(node, action) + + if isinstance(action, SCons.Action.LazyAction): + # pylint: disable=protected-access + action = action._generate_cache(node.env if node.env else self.env) + return self.action_to_ninja_build(node, action=action) + + if isinstance(action, SCons.Action.ListAction): + return self.handle_list_action(node, action) + + if isinstance(action, COMMAND_TYPES): + return get_command(node.env if node.env else self.env, node, action) + + # Return the node to indicate that SCons is required + return { + "rule": "SCONS", + "outputs": get_outputs(node), + "implicit": get_dependencies(node), + } + + def handle_func_action(self, node, action): + """Determine how to handle the function action.""" + name = action.function_name() + # This is the name given by the Subst/Textfile builders. So return the + # node to indicate that SCons is required + if name == "_action": + return { + "rule": "TEMPLATE", + "outputs": get_outputs(node), + "implicit": get_dependencies(node), + } + + handler = self.func_handlers.get(name, None) + if handler is not None: + return handler(node.env if node.env else self.env, node) + + print( + "Found unhandled function action {}, " + " generating scons command to build\n" + "Note: this is less efficient than Ninja," + " you can write your own ninja build generator for" + " this function using NinjaRegisterFunctionHandler".format(name) + ) + + return { + "rule": "SCONS", + "outputs": get_outputs(node), + "implicit": get_dependencies(node), + } + + # pylint: disable=too-many-branches + def handle_list_action(self, node, action): + """ + Attempt to translate list actions to Ninja. + + List actions are tricky to move to ninja. First we translate + each individual action in the action list to a Ninja + build. Then we process the resulting ninja builds to see if + they are all the same ninja rule. If they are not all the same + rule we cannot make them a single resulting ninja build, so + instead we make them a single SCons invocation to build all of + the targets. + + If they are all the same rule and the rule is CMD we attempt + to combine the cmdlines together using ' && ' which we then + combine into a single ninja build. + + If they are all phony targets we simple combine the outputs + and dependencies. + + If they are all INSTALL rules we combine all of the sources + and outputs. + + If they are all SCONS rules we do the same as if they are not + the same rule and make a build that will use SCons to generate + them. + + If they're all one rule and None of the above rules we throw an Exception. + """ + + results = [ + self.action_to_ninja_build(node, action=act) + for act in action.list + if act is not None + ] + results = [result for result in results if result is not None] + if not results: + return None + + # No need to process the results if we only got a single result + if len(results) == 1: + return results[0] + + all_outputs = list({output for build in results for output in build["outputs"]}) + # If we have no outputs we're done + if not all_outputs: + return None + + # Used to verify if all rules are the same + all_one_rule = len( + [ + r + for r in results + if isinstance(r, dict) and r["rule"] == results[0]["rule"] + ] + ) == len(results) + dependencies = get_dependencies(node) + + if not all_one_rule: + # If they aren't all the same rule use scons to generate these + # outputs. At this time nothing hits this case. + return { + "outputs": all_outputs, + "rule": "SCONS", + "implicit": dependencies, + } + + if results[0]["rule"] == "CMD": + cmdline = "" + for cmd in results: + + # Occasionally a command line will expand to a + # whitespace only string (i.e. ' '). Which is not a + # valid command but does not trigger the empty command + # condition if not cmdstr. So here we strip preceding + # and proceeding whitespace to make strings like the + # above become empty strings and so will be skipped. + cmdstr = cmd["variables"]["cmd"].strip() + if not cmdstr: + continue + + # Skip duplicate commands + if cmdstr in cmdline: + continue + + if cmdline: + cmdline += " && " + + cmdline += cmdstr + + # Remove all preceding and proceeding whitespace + cmdline = cmdline.strip() + + # Make sure we didn't generate an empty cmdline + if cmdline: + ninja_build = { + "outputs": all_outputs, + "rule": "CMD", + "variables": {"cmd": cmdline}, + "implicit": dependencies, + } + + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["pool"] + + return ninja_build + + elif results[0]["rule"] == "phony": + return { + "outputs": all_outputs, + "rule": "phony", + "implicit": dependencies, + } + + elif results[0]["rule"] == "INSTALL": + return { + "outputs": all_outputs, + "rule": "INSTALL", + "pool": "install_pool", + "inputs": [get_path(src_file(s)) for s in node.sources], + "implicit": dependencies, + } + + elif results[0]["rule"] == "SCONS": + return { + "outputs": all_outputs, + "rule": "SCONS", + "inputs": dependencies, + } + + raise Exception("Unhandled list action with rule: " + results[0]["rule"]) + + +# pylint: disable=too-many-instance-attributes +class NinjaState: + """Maintains state of Ninja build system as it's translated from SCons.""" + + def __init__(self, env, writer_class): + self.env = env + self.writer_class = writer_class + self.__generated = False + self.translator = SConsToNinjaTranslator(env) + self.generated_suffixes = env.get("NINJA_GENERATED_SOURCE_SUFFIXES", []) + + # List of generated builds that will be written at a later stage + self.builds = list() + + # List of targets for which we have generated a build. This + # allows us to take multiple Alias nodes as sources and to not + # fail to build if they have overlapping targets. + self.built = set() + + # SCons sets this variable to a function which knows how to do + # shell quoting on whatever platform it's run on. Here we use it + # to make the SCONS_INVOCATION variable properly quoted for things + # like CCFLAGS + escape = env.get("ESCAPE", lambda x: x) + + self.variables = { + "COPY": "cmd.exe /c copy" if sys.platform == "win32" else "cp", + "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format( + sys.executable, + " ".join( + [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] + ), + ), + "SCONS_INVOCATION_W_TARGETS": "{} {}".format( + sys.executable, " ".join([escape(arg) for arg in sys.argv]) + ), + # This must be set to a global default per: + # https://ninja-build.org/manual.html + # + # (The deps section) + "msvc_deps_prefix": "Note: including file:", + } + + self.rules = { + "CMD": { + "command": "cmd /c $cmd" if sys.platform == "win32" else "$cmd", + "description": "Building $out", + }, + # We add the deps processing variables to this below. We + # don't pipe this through cmd.exe on Windows because we + # use this to generate a compile_commands.json database + # which can't use the shell command as it's compile + # command. This does mean that we assume anything using + # CMD_W_DEPS is a straight up compile which is true today. + "CMD_W_DEPS": {"command": "$cmd", "description": "Building $out"}, + "SYMLINK": { + "command": ( + "cmd /c mklink $out $in" + if sys.platform == "win32" + else "ln -s $in $out" + ), + "description": "Symlink $in -> $out", + }, + "INSTALL": { + "command": "$COPY $in $out", + "description": "Install $out", + # On Windows cmd.exe /c copy does not always correctly + # update the timestamp on the output file. This leads + # to a stuck constant timestamp in the Ninja database + # and needless rebuilds. + # + # Adding restat here ensures that Ninja always checks + # the copy updated the timestamp and that Ninja has + # the correct information. + "restat": 1, + }, + "TEMPLATE": { + "command": "$SCONS_INVOCATION $out", + "description": "Rendering $out", + "pool": "scons_pool", + "restat": 1, + }, + "SCONS": { + "command": "$SCONS_INVOCATION $out", + "description": "SCons $out", + "pool": "scons_pool", + # restat + # if present, causes Ninja to re-stat the command's outputs + # after execution of the command. Each output whose + # modification time the command did not change will be + # treated as though it had never needed to be built. This + # may cause the output's reverse dependencies to be removed + # from the list of pending build actions. + # + # We use restat any time we execute SCons because + # SCons calls in Ninja typically create multiple + # targets. But since SCons is doing it's own up to + # date-ness checks it may only update say one of + # them. Restat will find out which of the multiple + # build targets did actually change then only rebuild + # those targets which depend specifically on that + # output. + "restat": 1, + }, + "REGENERATE": { + "command": "$SCONS_INVOCATION_W_TARGETS", + "description": "Regenerating $out", + "generator": 1, + # Console pool restricts to 1 job running at a time, + # it additionally has some special handling about + # passing stdin, stdout, etc to process in this pool + # that we need for SCons to behave correctly when + # regenerating Ninja + "pool": "console", + # Again we restat in case Ninja thought the + # build.ninja should be regenerated but SCons knew + # better. + "restat": 1, + }, + } + + self.pools = { + "install_pool": self.env.GetOption("num_jobs") / 2, + "scons_pool": 1, + } + + if env["PLATFORM"] == "win32": + self.rules["CMD_W_DEPS"]["deps"] = "msvc" + else: + self.rules["CMD_W_DEPS"]["deps"] = "gcc" + self.rules["CMD_W_DEPS"]["depfile"] = "$out.d" + + self.rules.update(env.get(NINJA_RULES, {})) + self.pools.update(env.get(NINJA_POOLS, {})) + + def generate_builds(self, node): + """Generate a ninja build rule for node and it's children.""" + # Filter out nodes with no builder. They are likely source files + # and so no work needs to be done, it will be used in the + # generation for some real target. + # + # Note that all nodes have a builder attribute but it is sometimes + # set to None. So we cannot use a simpler hasattr check here. + if getattr(node, "builder", None) is None: + return + + stack = [[node]] + while stack: + frame = stack.pop() + for child in frame: + outputs = set(get_outputs(child)) + # Check if all the outputs are in self.built, if they + # are we've already seen this node and it's children. + if not outputs.isdisjoint(self.built): + continue + + self.built = self.built.union(outputs) + stack.append(child.children()) + + if isinstance(child, SCons.Node.Alias.Alias): + build = alias_to_ninja_build(child) + elif node.builder is not None: + # Use False since None is a valid value for this attribute + build = getattr(child.attributes, NINJA_BUILD, False) + if build is False: + build = self.translator.action_to_ninja_build(child) + setattr(child.attributes, NINJA_BUILD, build) + else: + build = None + + # Some things are unbuild-able or need not be built in Ninja + if build is None or build == 0: + continue + + self.builds.append(build) + + def is_generated_source(self, output): + """Check if output ends with a known generated suffix.""" + _, suffix = splitext(output) + return suffix in self.generated_suffixes + + def has_generated_sources(self, output): + """ + Determine if output indicates this is a generated header file. + """ + for generated in output: + if self.is_generated_source(generated): + return True + return False + + # pylint: disable=too-many-branches,too-many-locals + def generate(self, ninja_file, fallback_default_target=None): + """ + Generate the build.ninja. + + This should only be called once for the lifetime of this object. + """ + if self.__generated: + return + + content = io.StringIO() + ninja = self.writer_class(content, width=100) + + ninja.comment("Generated by scons. DO NOT EDIT.") + + for pool_name, size in self.pools.items(): + ninja.pool(pool_name, size) + + for var, val in self.variables.items(): + ninja.variable(var, val) + + for rule, kwargs in self.rules.items(): + ninja.rule(rule, **kwargs) + + generated_source_files = { + output + # First find builds which have header files in their outputs. + for build in self.builds + if self.has_generated_sources(build["outputs"]) + for output in build["outputs"] + # Collect only the header files from the builds with them + # in their output. We do this because is_generated_source + # returns True if it finds a header in any of the outputs, + # here we need to filter so we only have the headers and + # not the other outputs. + if self.is_generated_source(output) + } + + if generated_source_files: + ninja.build( + outputs="_generated_sources", + rule="phony", + implicit=list(generated_source_files), + ) + + template_builders = [] + + for build in self.builds: + if build["rule"] == "TEMPLATE": + template_builders.append(build) + continue + + implicit = build.get("implicit", []) + implicit.append(ninja_file) + build["implicit"] = implicit + + # Don't make generated sources depend on each other. We + # have to check that none of the outputs are generated + # sources and none of the direct implicit dependencies are + # generated sources or else we will create a dependency + # cycle. + if ( + generated_source_files + and not build["rule"] == "INSTALL" + and set(build["outputs"]).isdisjoint(generated_source_files) + and set(implicit).isdisjoint(generated_source_files) + ): + + # Make all non-generated source targets depend on + # _generated_sources. We use order_only for generated + # sources so that we don't rebuild the world if one + # generated source was rebuilt. We just need to make + # sure that all of these sources are generated before + # other builds. + build["order_only"] = "_generated_sources" + + # When using a depfile Ninja can only have a single output + # but SCons will usually have emitted an output for every + # thing a command will create because it's caching is much + # more complex than Ninja's. This includes things like DWO + # files. Here we make sure that Ninja only ever sees one + # target when using a depfile. It will still have a command + # that will create all of the outputs but most targets don't + # depend direclty on DWO files and so this assumption is safe + # to make. + rule = self.rules.get(build["rule"]) + + # Some rules like 'phony' and other builtins we don't have + # listed in self.rules so verify that we got a result + # before trying to check if it has a deps key. + if rule is not None and rule.get("deps"): + + # Anything using deps in Ninja can only have a single + # output, but we may have a build which actually + # produces multiple outputs which other targets can + # depend on. Here we slice up the outputs so we have a + # single output which we will use for the "real" + # builder and multiple phony targets that match the + # file names of the remaining outputs. This way any + # build can depend on any output from any build. + first_output, remaining_outputs = build["outputs"][0], build["outputs"][1:] + if remaining_outputs: + ninja.build( + outputs=remaining_outputs, + rule="phony", + implicit=first_output, + ) + + build["outputs"] = first_output + + ninja.build(**build) + + template_builds = dict() + for template_builder in template_builders: + + # Special handling for outputs and implicit since we need to + # aggregate not replace for each builder. + for agg_key in ["outputs", "implicit"]: + new_val = template_builds.get(agg_key, []) + + # Use pop so the key is removed and so the update + # below will not overwrite our aggregated values. + cur_val = template_builder.pop(agg_key, []) + if isinstance(cur_val, list): + new_val += cur_val + else: + new_val.append(cur_val) + template_builds[agg_key] = new_val + + # Collect all other keys + template_builds.update(template_builder) + + if template_builds.get("outputs", []): + ninja.build(**template_builds) + + # Here to teach the ninja file how to regenerate itself. We'll + # never see ourselves in the DAG walk so we can't rely on + # action_to_ninja_build to generate this rule + ninja.build( + ninja_file, + rule="REGENERATE", + implicit=[ + self.env.File("#SConstruct").get_abspath(), + os.path.abspath(__file__), + ] + + LOADED_SCONSCRIPTS, + ) + + ninja.build( + "scons-invocation", + rule="CMD", + pool="console", + variables={"cmd": "echo $SCONS_INVOCATION_W_TARGETS"}, + ) + + # Note the use of CMD_W_DEPS below. CMD_W_DEPS are always + # compile commands in this generator. If we ever change the + # name/s of the rules that include compile commands + # (i.e. something like CC/CXX) we will need to update this + # build to reflect that complete list. + ninja.build( + "compile_commands.json", + rule="CMD", + pool="console", + variables={ + "cmd": "ninja -f {} -t compdb CMD_W_DEPS > compile_commands.json".format( + ninja_file + ) + }, + ) + + ninja.build( + "compiledb", rule="phony", implicit=["compile_commands.json"], + ) + + # Look in SCons's list of DEFAULT_TARGETS, find the ones that + # we generated a ninja build rule for. + scons_default_targets = [ + get_path(tgt) + for tgt in SCons.Script.DEFAULT_TARGETS + if get_path(tgt) in self.built + ] + + # If we found an overlap between SCons's list of default + # targets and the targets we created ninja builds for then use + # those as ninja's default as well. + if scons_default_targets: + ninja.default(" ".join(scons_default_targets)) + + # If not then set the default to the fallback_default_target we were given. + # Otherwise we won't create a default ninja target. + elif fallback_default_target is not None: + ninja.default(fallback_default_target) + + with open(ninja_file, "w") as build_ninja: + build_ninja.write(content.getvalue()) + + self.__generated = True + + +def get_path(node): + """ + Return a fake path if necessary. + + As an example Aliases use this as their target name in Ninja. + """ + if hasattr(node, "get_path"): + return node.get_path() + return str(node) + + +def rfile(node): + """ + Return the repository file for node if it has one. Otherwise return node + """ + if hasattr(node, "rfile"): + return node.rfile() + return node + + +def src_file(node): + """Returns the src code file if it exists.""" + if hasattr(node, "srcnode"): + src = node.srcnode() + if src.stat() is not None: + return src + return get_path(node) + + +# TODO: Make the Rules smarter. Instead of just using a "cmd" rule +# everywhere we should be smarter about generating CC, CXX, LINK, +# etc. rules +def get_command(env, node, action): # pylint: disable=too-many-branches + """Get the command to execute for node.""" + if node.env: + sub_env = node.env + else: + sub_env = env + + executor = node.get_executor() + if executor is not None: + tlist = executor.get_all_targets() + slist = executor.get_all_sources() + else: + if hasattr(node, "target_peers"): + tlist = node.target_peers + else: + tlist = [node] + slist = node.sources + + # Retrieve the repository file for all sources + slist = [rfile(s) for s in slist] + + # Get the dependencies for all targets + implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) + + # Generate a real CommandAction + if isinstance(action, SCons.Action.CommandGeneratorAction): + # pylint: disable=protected-access + action = action._generate(tlist, slist, sub_env, 1, executor=executor) + + rule = "CMD" + + # Actions like CommandAction have a method called process that is + # used by SCons to generate the cmd_line they need to run. So + # check if it's a thing like CommandAction and call it if we can. + if hasattr(action, "process"): + cmd_list, _, _ = action.process(tlist, slist, sub_env, executor=executor) + + # Despite being having "list" in it's name this member is not + # actually a list. It's the pre-subst'd string of the command. We + # use it to determine if the command we generated needs to use a + # custom Ninja rule. By default this redirects CC/CXX commands to + # CMD_W_DEPS but the user can inject custom Ninja rules and tie + # them to commands by using their pre-subst'd string. + rule = __NINJA_RULE_MAPPING.get(action.cmd_list, "CMD") + + cmd = _string_from_cmd_list(cmd_list[0]) + else: + # Anything else works with genstring, this is most commonly hit by + # ListActions which essentially call process on all of their + # commands and concatenate it for us. + genstring = action.genstring(tlist, slist, sub_env) + + # Detect if we have a custom rule for this + # "ListActionCommandAction" type thing. + rule = __NINJA_RULE_MAPPING.get(genstring, "CMD") + + if executor is not None: + cmd = sub_env.subst(genstring, executor=executor) + else: + cmd = sub_env.subst(genstring, target=tlist, source=slist) + + # Since we're only enabling Ninja for developer builds right + # now we skip all Manifest related work on Windows as it's not + # necessary. We shouldn't have gotten here but on Windows + # SCons has a ListAction which shows as a + # CommandGeneratorAction for linking. That ListAction ends + # with a FunctionAction (embedManifestExeCheck, + # embedManifestDllCheck) that simply say "does + # target[0].manifest exist?" if so execute the real command + # action underlying me, otherwise do nothing. + # + # Eventually we'll want to find a way to translate this to + # Ninja but for now, and partially because the existing Ninja + # generator does so, we just disable it all together. + cmd = cmd.replace("\n", " && ").strip() + if env["PLATFORM"] == "win32" and ( + "embedManifestExeCheck" in cmd or "embedManifestDllCheck" in cmd + ): + cmd = " && ".join(cmd.split(" && ")[0:-1]) + + if cmd.endswith("&&"): + cmd = cmd[0:-2].strip() + + outputs = get_outputs(node) + command_env = "" + windows = env["PLATFORM"] == "win32" + + # If win32 and rule == CMD_W_DEPS then we don't want to calculate + # an environment for this command. It's a compile command and + # compiledb doesn't support shell syntax on Windows. We need the + # shell syntax to use environment variables on Windows so we just + # skip this platform / rule combination to keep the compiledb + # working. + # + # On POSIX we can still set environment variables even for compile + # commands so we do so. + if not (windows and rule == "CMD_W_DEPS"): + + # Scan the ENV looking for any keys which do not exist in + # os.environ or differ from it. We assume if it's a new or + # differing key from the process environment then it's + # important to pass down to commands in the Ninja file. + ENV = get_default_ENV(sub_env) + scons_specified_env = { + key: value + for key, value in ENV.items() + if key not in os.environ or os.environ.get(key, None) != value + } + + for key, value in scons_specified_env.items(): + # Ensure that the ENV values are all strings: + if is_List(value): + # If the value is a list, then we assume it is a + # path list, because that's a pretty common list-like + # value to stick in an environment variable: + value = flatten_sequence(value) + value = joinpath(map(str, value)) + else: + # If it isn't a string or a list, then we just coerce + # it to a string, which is the proper way to handle + # Dir and File instances and will produce something + # reasonable for just about everything else: + value = str(value) + + if windows: + command_env += "set '{}={}' && ".format(key, value) + else: + command_env += "{}={} ".format(key, value) + + variables = {"cmd": command_env + cmd} + extra_vars = getattr(node.attributes, "NINJA_EXTRA_VARS", {}) + if extra_vars: + variables.update(extra_vars) + + ninja_build = { + "outputs": outputs, + "inputs": get_inputs(node), + "implicit": implicit, + "rule": rule, + "variables": variables, + } + + # Don't use sub_env here because we require that NINJA_POOL be set + # on a per-builder call basis to prevent accidental strange + # behavior like env['NINJA_POOL'] = 'console' and sub_env can be + # the global Environment object if node.env is None. + # Example: + # + # Allowed: + # + # env.Command("ls", NINJA_POOL="ls_pool") + # + # Not allowed and ignored: + # + # env["NINJA_POOL"] = "ls_pool" + # env.Command("ls") + # + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["NINJA_POOL"] + + return ninja_build + + +def ninja_builder(env, target, source): + """Generate a build.ninja for source.""" + if not isinstance(source, list): + source = [source] + if not isinstance(target, list): + target = [target] + + # We have no COMSTR equivalent so print that we're generating + # here. + print("Generating:", str(target[0])) + + # The environment variable NINJA_SYNTAX points to the + # ninja_syntax.py module from the ninja sources found here: + # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py + # + # This should be vendored into the build sources and it's location + # set in NINJA_SYNTAX. This code block loads the location from + # that variable, gets the absolute path to the vendored file, gets + # it's parent directory then uses importlib to import the module + # dynamically. + ninja_syntax_file = env[NINJA_SYNTAX] + if isinstance(ninja_syntax_file, str): + ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() + ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) + sys.path.append(ninja_syntax_mod_dir) + ninja_syntax_mod_name = os.path.basename(ninja_syntax_file) + ninja_syntax = importlib.import_module(ninja_syntax_mod_name.replace(".py", "")) + + suffix = env.get("NINJA_SUFFIX", "") + if suffix and not suffix[0] == ".": + suffix = "." + suffix + + generated_build_ninja = target[0].get_abspath() + suffix + ninja_state = NinjaState(env, ninja_syntax.Writer) + + for src in source: + ninja_state.generate_builds(src) + + ninja_state.generate(generated_build_ninja, str(source[0])) + + return 0 + + +# pylint: disable=too-few-public-methods +class AlwaysExecAction(SCons.Action.FunctionAction): + """Override FunctionAction.__call__ to always execute.""" + + def __call__(self, *args, **kwargs): + kwargs["execute"] = 1 + return super().__call__(*args, **kwargs) + + +def ninja_print(_cmd, target, _source, env): + """Tag targets with the commands to build them.""" + if target: + for tgt in target: + if ( + tgt.has_builder() + # Use 'is False' because not would still trigger on + # None's which we don't want to regenerate + and getattr(tgt.attributes, NINJA_BUILD, False) is False + and isinstance(tgt.builder.action, COMMAND_TYPES) + ): + ninja_action = get_command(env, tgt, tgt.builder.action) + setattr(tgt.attributes, NINJA_BUILD, ninja_action) + # Preload the attributes dependencies while we're still running + # multithreaded + get_dependencies(tgt) + return 0 + + +def register_custom_handler(env, name, handler): + """Register a custom handler for SCons function actions.""" + env[NINJA_CUSTOM_HANDLERS][name] = handler + + +def register_custom_rule_mapping(env, pre_subst_string, rule): + """Register a custom handler for SCons function actions.""" + global __NINJA_RULE_MAPPING + __NINJA_RULE_MAPPING[pre_subst_string] = rule + + +def register_custom_rule(env, rule, command, description="", deps=None): + """Allows specification of Ninja rules from inside SCons files.""" + rule_obj = { + "command": command, + "description": description if description else "{} $out".format(rule), + } + + if deps is not None: + rule_obj["deps"] = deps + + env[NINJA_RULES][rule] = rule_obj + + +def register_custom_pool(env, pool, size): + """Allows the creation of custom Ninja pools""" + env[NINJA_POOLS][pool] = size + + +def ninja_csig(original): + """Return a dummy csig""" + + def wrapper(self): + name = str(self) + if "SConscript" in name or "SConstruct" in name: + return original(self) + return "dummy_ninja_csig" + + return wrapper + + +def ninja_contents(original): + """Return a dummy content without doing IO""" + + def wrapper(self): + name = str(self) + if "SConscript" in name or "SConstruct" in name: + return original(self) + return bytes("dummy_ninja_contents", encoding="utf-8") + + return wrapper + + +def ninja_stat(_self, path): + """ + Eternally memoized stat call. + + SCons is very aggressive about clearing out cached values. For our + purposes everything should only ever call stat once since we're + running in a no_exec build the file system state should not + change. For these reasons we patch SCons.Node.FS.LocalFS.stat to + use our eternal memoized dictionary. + + Since this is happening during the Node walk it's being run while + threaded, we have to protect adding to the memoized dictionary + with a threading.Lock otherwise many targets miss the memoization + due to racing. + """ + global NINJA_STAT_MEMO + + try: + return NINJA_STAT_MEMO[path] + except KeyError: + try: + result = os.stat(path) + except os.error: + result = None + + with MEMO_LOCK: + NINJA_STAT_MEMO[path] = result + + return result + + +def ninja_noop(*_args, **_kwargs): + """ + A general purpose no-op function. + + There are many things that happen in SCons that we don't need and + also don't return anything. We use this to disable those functions + instead of creating multiple definitions of the same thing. + """ + return None + + +def ninja_whereis(thing, *_args, **_kwargs): + """Replace env.WhereIs with a much faster version""" + global NINJA_WHEREIS_MEMO + + # Optimize for success, this gets called significantly more often + # when the value is already memoized than when it's not. + try: + return NINJA_WHEREIS_MEMO[thing] + except KeyError: + # We do not honor any env['ENV'] or env[*] variables in the + # generated ninja ile. Ninja passes your raw shell environment + # down to it's subprocess so the only sane option is to do the + # same during generation. At some point, if and when we try to + # upstream this, I'm sure a sticking point will be respecting + # env['ENV'] variables and such but it's actually quite + # complicated. I have a naive version but making it always work + # with shell quoting is nigh impossible. So I've decided to + # cross that bridge when it's absolutely required. + path = shutil.which(thing) + NINJA_WHEREIS_MEMO[thing] = path + return path + + +def ninja_always_serial(self, num, taskmaster): + """Replacement for SCons.Job.Jobs constructor which always uses the Serial Job class.""" + # We still set self.num_jobs to num even though it's a lie. The + # only consumer of this attribute is the Parallel Job class AND + # the Main.py function which instantiates a Jobs class. It checks + # if Jobs.num_jobs is equal to options.num_jobs, so if the user + # provides -j12 but we set self.num_jobs = 1 they get an incorrect + # warning about this version of Python not supporting parallel + # builds. So here we lie so the Main.py will not give a false + # warning to users. + self.num_jobs = num + self.job = SCons.Job.Serial(taskmaster) + + +class NinjaEternalTempFile(SCons.Platform.TempFileMunge): + """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" + + def __call__(self, target, source, env, for_signature): + if for_signature: + return self.cmd + + node = target[0] if SCons.Util.is_List(target) else target + if node is not None: + cmdlist = getattr(node.attributes, "tempfile_cmdlist", None) + if cmdlist is not None: + return cmdlist + + cmd = super().__call__(target, source, env, for_signature) + + # If TempFileMunge.__call__ returns a string it means that no + # response file was needed. No processing required so just + # return the command. + if isinstance(cmd, str): + return cmd + + # Strip the removal commands from the command list. + # + # SCons' TempFileMunge class has some very strange + # behavior where it, as part of the command line, tries to + # delete the response file after executing the link + # command. We want to keep those response files since + # Ninja will keep using them over and over. The + # TempFileMunge class creates a cmdlist to do this, a + # common SCons convention for executing commands see: + # https://github.com/SCons/scons/blob/master/src/engine/SCons/Action.py#L949 + # + # This deletion behavior is not configurable. So we wanted + # to remove the deletion command from the command list by + # simply slicing it out here. Unfortunately for some + # strange reason TempFileMunge doesn't make the "rm" + # command it's own list element. It appends it to the + # tempfile argument to cmd[0] (which is CC/CXX) and then + # adds the tempfile again as it's own element. + # + # So we just kind of skip that middle element. Since the + # tempfile is in the command list on it's own at the end we + # can cut it out entirely. This is what I would call + # "likely to break" in future SCons updates. Hopefully it + # breaks because they start doing the right thing and not + # weirdly splitting these arguments up. For reference a + # command list that we get back from the OG TempFileMunge + # looks like this: + # + # [ + # 'g++', + # '@/mats/tempfiles/random_string.lnk\nrm', + # '/mats/tempfiles/random_string.lnk', + # ] + # + # Note the weird newline and rm command in the middle + # element and the lack of TEMPFILEPREFIX on the last + # element. + prefix = env.subst("$TEMPFILEPREFIX") + if not prefix: + prefix = "@" + + new_cmdlist = [cmd[0], prefix + cmd[-1]] + setattr(node.attributes, "tempfile_cmdlist", new_cmdlist) + return new_cmdlist + + def _print_cmd_str(*_args, **_kwargs): + """Disable this method""" + pass + + +def exists(env): + """Enable if called.""" + + # This variable disables the tool when storing the SCons command in the + # generated ninja file to ensure that the ninja tool is not loaded when + # SCons should do actual work as a subprocess of a ninja build. The ninja + # tool is very invasive into the internals of SCons and so should never be + # enabled when SCons needs to build a target. + if env.get("__NINJA_NO", "0") == "1": + return False + + return True + + +def generate(env): + """Generate the NINJA builders.""" + env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") + + # Add the Ninja builder. + always_exec_ninja_action = AlwaysExecAction(ninja_builder, {}) + ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action) + env.Append(BUILDERS={"Ninja": ninja_builder_obj}) + + # This adds the required flags such that the generated compile + # commands will create depfiles as appropriate in the Ninja file. + if env["PLATFORM"] == "win32": + env.Append(CCFLAGS=["/showIncludes"]) + else: + env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) + + # Provides a way for users to handle custom FunctionActions they + # want to translate to Ninja. + env[NINJA_CUSTOM_HANDLERS] = {} + env.AddMethod(register_custom_handler, "NinjaRegisterFunctionHandler") + + # Provides a mechanism for inject custom Ninja rules which can + # then be mapped using NinjaRuleMapping. + env[NINJA_RULES] = {} + env.AddMethod(register_custom_rule, "NinjaRule") + + # Provides a mechanism for inject custom Ninja pools which can + # be used by providing the NINJA_POOL="name" as an + # OverrideEnvironment variable in a builder call. + env[NINJA_POOLS] = {} + env.AddMethod(register_custom_pool, "NinjaPool") + + # Add the ability to register custom NinjaRuleMappings for Command + # builders. We don't store this dictionary in the env to prevent + # accidental deletion of the CC/XXCOM mappings. You can still + # overwrite them if you really want to but you have to explicit + # about it this way. The reason is that if they were accidentally + # deleted you would get a very subtly incorrect Ninja file and + # might not catch it. + env.AddMethod(register_custom_rule_mapping, "NinjaRuleMapping") + env.NinjaRuleMapping("${CCCOM}", "CMD_W_DEPS") + env.NinjaRuleMapping("${CXXCOM}", "CMD_W_DEPS") + + # Make SCons node walk faster by preventing unnecessary work + env.Decider("timestamp-match") + + # Used to determine if a build generates a source file. Ninja + # requires that all generated sources are added as order_only + # dependencies to any builds that *might* use them. + env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] + + # This is the point of no return, anything after this comment + # makes changes to SCons that are irreversible and incompatible + # with a normal SCons build. We return early if __NINJA_NO=1 has + # been given on the command line (i.e. by us in the generated + # ninja file) here to prevent these modifications from happening + # when we want SCons to do work. Everything before this was + # necessary to setup the builder and other functions so that the + # tool can be unconditionally used in the users's SCons files. + + if not exists(env): + return + + # Set a known variable that other tools can query so they can + # behave correctly during ninja generation. + env["GENERATING_NINJA"] = True + + # These methods are no-op'd because they do not work during ninja + # generation, expected to do no work, or simply fail. All of which + # are slow in SCons. So we overwrite them with no logic. + SCons.Node.FS.File.make_ready = ninja_noop + SCons.Node.FS.File.prepare = ninja_noop + SCons.Node.FS.File.push_to_cache = ninja_noop + SCons.Executor.Executor.prepare = ninja_noop + SCons.Node.FS.File.built = ninja_noop + + # We make lstat a no-op because it is only used for SONAME + # symlinks which we're not producing. + SCons.Node.FS.LocalFS.lstat = ninja_noop + + # This is a slow method that isn't memoized. We make it a noop + # since during our generation we will never use the results of + # this or change the results. + SCons.Node.FS.is_up_to_date = ninja_noop + + # We overwrite stat and WhereIs with eternally memoized + # implementations. See the docstring of ninja_stat and + # ninja_whereis for detailed explanations. + SCons.Node.FS.LocalFS.stat = ninja_stat + SCons.Util.WhereIs = ninja_whereis + + # Monkey patch get_csig and get_contents for some classes. It + # slows down the build significantly and we don't need contents or + # content signatures calculated when generating a ninja file since + # we're not doing any SCons caching or building. + SCons.Executor.Executor.get_contents = ninja_contents( + SCons.Executor.Executor.get_contents + ) + SCons.Node.Alias.Alias.get_contents = ninja_contents( + SCons.Node.Alias.Alias.get_contents + ) + SCons.Node.FS.File.get_contents = ninja_contents(SCons.Node.FS.File.get_contents) + SCons.Node.FS.File.get_csig = ninja_csig(SCons.Node.FS.File.get_csig) + SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) + SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig) + + # Replace false Compiling* messages with a more accurate output + # + # We also use this to tag all Nodes with Builders using + # CommandActions with the final command that was used to compile + # it for passing to Ninja. If we don't inject this behavior at + # this stage in the build too much state is lost to generate the + # command at the actual ninja_builder execution time for most + # commands. + # + # We do attempt command generation again in ninja_builder if it + # hasn't been tagged and it seems to work for anything that + # doesn't represent as a non-FunctionAction during the print_func + # call. + env["PRINT_CMD_LINE_FUNC"] = ninja_print + + # This reduces unnecessary subst_list calls to add the compiler to + # the implicit dependencies of targets. Since we encode full paths + # in our generated commands we do not need these slow subst calls + # as executing the command will fail if the file is not found + # where we expect it. + env["IMPLICIT_COMMAND_DEPENDENCIES"] = False + + # Set build to no_exec, our sublcass of FunctionAction will force + # an execution for ninja_builder so this simply effects all other + # Builders. + env.SetOption("no_exec", True) + + # This makes SCons more aggressively cache MD5 signatures in the + # SConsign file. + env.SetOption("max_drift", 1) + + # The Serial job class is SIGNIFICANTLY (almost twice as) faster + # than the Parallel job class for generating Ninja files. So we + # monkey the Jobs constructor to only use the Serial Job class. + SCons.Job.Jobs.__init__ = ninja_always_serial + + # We will eventually need to overwrite TempFileMunge to make it + # handle persistent tempfiles or get an upstreamed change to add + # some configurability to it's behavior in regards to tempfiles. + # + # Set all three environment variables that Python's + # tempfile.mkstemp looks at as it behaves differently on different + # platforms and versions of Python. + os.environ["TMPDIR"] = env.Dir("$BUILD_DIR/response_files").get_abspath() + os.environ["TEMP"] = os.environ["TMPDIR"] + os.environ["TMP"] = os.environ["TMPDIR"] + if not os.path.isdir(os.environ["TMPDIR"]): + env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) + + env["TEMPFILE"] = NinjaEternalTempFile + + # Force the SConsign to be written, we benefit from SCons caching of + # implicit dependencies and conftests. Unfortunately, we have to do this + # using an atexit handler because SCons will not write the file when in a + # no_exec build. + import atexit + + atexit.register(SCons.SConsign.write) From 4b1efb90f9dfaaa1a4533e2a99bf40295133da20 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 5 May 2020 23:44:01 -0400 Subject: [PATCH 078/163] updated to ninja-next, added some small fixes, and added simple test --- src/engine/SCons/Tool/ninja.py | 1071 +++++++++++++++++------------- test/ninja/CC.py | 66 ++ test/ninja/ninja-fixture/bar.c | 10 + test/ninja/ninja-fixture/foo.c | 10 + test/ninja/ninja-fixture/test1.c | 3 + test/ninja/ninja-fixture/test2.C | 3 + 6 files changed, 691 insertions(+), 472 deletions(-) create mode 100644 test/ninja/CC.py create mode 100644 test/ninja/ninja-fixture/bar.c create mode 100644 test/ninja/ninja-fixture/foo.c create mode 100644 test/ninja/ninja-fixture/test1.c create mode 100644 test/ninja/ninja-fixture/test2.C diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index b34759e401..d1cbafa495 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -1,16 +1,25 @@ -# Copyright 2019 MongoDB Inc. +# Copyright 2020 MongoDB Inc. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: # -# http://www.apache.org/licenses/LICENSE-2.0 +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + """Generate build.ninja files from SCons aliases.""" import sys @@ -18,16 +27,19 @@ import importlib import io import shutil +import shlex +import subprocess -from threading import Lock +from glob import glob from os.path import join as joinpath from os.path import splitext import SCons from SCons.Action import _string_from_cmd_list, get_default_ENV -from SCons.Util import is_String, is_List -from SCons.Script import COMMAND_LINE_TARGETS, LOADED_SCONSCRIPTS +from SCons.Util import is_List, flatten_sequence +from SCons.Script import COMMAND_LINE_TARGETS +NINJA_STATE = None NINJA_SYNTAX = "NINJA_SYNTAX" NINJA_RULES = "__NINJA_CUSTOM_RULES" NINJA_POOLS = "__NINJA_CUSTOM_POOLS" @@ -35,7 +47,6 @@ NINJA_BUILD = "NINJA_BUILD" NINJA_WHEREIS_MEMO = {} NINJA_STAT_MEMO = {} -MEMO_LOCK = Lock() __NINJA_RULE_MAPPING = {} @@ -51,11 +62,43 @@ def _install_action_function(_env, node): return { "outputs": get_outputs(node), "rule": "INSTALL", - "pool": "install_pool", "inputs": [get_path(src_file(s)) for s in node.sources], "implicit": get_dependencies(node), } +def _mkdir_action_function(env, node): + return { + "outputs": get_outputs(node), + "rule": "CMD", + # implicit explicitly omitted, we translate these so they can be + # used by anything that depends on these but commonly this is + # hit with a node that will depend on all of the fake + # srcnode's that SCons will never give us a rule for leading + # to an invalid ninja file. + "variables": { + # On Windows mkdir "-p" is always on + "cmd": "{mkdir} $out".format( + mkdir="mkdir" if env["PLATFORM"] == "win32" else "mkdir -p", + ), + }, + } + +def _copy_action_function(env, node): + return { + "outputs": get_outputs(node), + "inputs": [get_path(src_file(s)) for s in node.sources], + "rule": "CMD", + # implicit explicitly omitted, we translate these so they can be + # used by anything that depends on these but commonly this is + # hit with a node that will depend on all of the fake + # srcnode's that SCons will never give us a rule for leading + # to an invalid ninja file. + "variables": { + # On Windows mkdir "-p" is always on + "cmd": "$COPY $in $out", + }, + } + def _lib_symlink_action_function(_env, node): """Create shared object symlinks if any need to be created""" @@ -87,7 +130,13 @@ def is_valid_dependent_node(node): check because some nodes (like src files) won't have builders but are valid implicit dependencies. """ - return not isinstance(node, SCons.Node.Alias.Alias) or node.children() + if isinstance(node, SCons.Node.Alias.Alias): + return node.children() + + if not node.env: + return True + + return not node.env.get("NINJA_SKIP") def alias_to_ninja_build(node): @@ -96,13 +145,26 @@ def alias_to_ninja_build(node): "outputs": get_outputs(node), "rule": "phony", "implicit": [ - get_path(n) for n in node.children() if is_valid_dependent_node(n) + get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) ], } -def get_dependencies(node): +def get_order_only(node): + """Return a list of order only dependencies for node.""" + if node.prerequisites is None: + return [] + return [get_path(src_file(prereq)) for prereq in node.prerequisites] + + +def get_dependencies(node, skip_sources=False): """Return a list of dependencies for node.""" + if skip_sources: + return [ + get_path(src_file(child)) + for child in node.children() + if child not in node.sources + ] return [get_path(src_file(child)) for child in node.children()] @@ -130,6 +192,7 @@ def get_outputs(node): outputs = [node] outputs = [get_path(o) for o in outputs] + return outputs @@ -147,18 +210,19 @@ def __init__(self, env): "SharedFlagChecker": ninja_noop, # The install builder is implemented as a function action. "installFunc": _install_action_function, + "MkdirFunc": _mkdir_action_function, "LibSymlinksActionFunction": _lib_symlink_action_function, + "Copy" : _copy_action_function } - self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) + self.loaded_custom = False # pylint: disable=too-many-return-statements def action_to_ninja_build(self, node, action=None): """Generate build arguments dictionary for node.""" - # Use False since None is a valid value for this Attribute - build = getattr(node.attributes, NINJA_BUILD, False) - if build is not False: - return build + if not self.loaded_custom: + self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) + self.loaded_custom = True if node.builder is None: return None @@ -166,51 +230,60 @@ def action_to_ninja_build(self, node, action=None): if action is None: action = node.builder.action + if node.env and node.env.get("NINJA_SKIP"): + return None + + build = {} + # Ideally this should never happen, and we do try to filter # Ninja builders out of being sources of ninja builders but I # can't fix every DAG problem so we just skip ninja_builders # if we find one if node.builder == self.env["BUILDERS"]["Ninja"]: - return None - - if isinstance(action, SCons.Action.FunctionAction): - return self.handle_func_action(node, action) - - if isinstance(action, SCons.Action.LazyAction): + build = None + elif isinstance(action, SCons.Action.FunctionAction): + build = self.handle_func_action(node, action) + elif isinstance(action, SCons.Action.LazyAction): # pylint: disable=protected-access action = action._generate_cache(node.env if node.env else self.env) - return self.action_to_ninja_build(node, action=action) - - if isinstance(action, SCons.Action.ListAction): - return self.handle_list_action(node, action) + build = self.action_to_ninja_build(node, action=action) + elif isinstance(action, SCons.Action.ListAction): + build = self.handle_list_action(node, action) + elif isinstance(action, COMMAND_TYPES): + build = get_command(node.env if node.env else self.env, node, action) + else: + raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) - if isinstance(action, COMMAND_TYPES): - return get_command(node.env if node.env else self.env, node, action) + if build is not None: + build["order_only"] = get_order_only(node) - # Return the node to indicate that SCons is required - return { - "rule": "SCONS", - "outputs": get_outputs(node), - "implicit": get_dependencies(node), - } + return build def handle_func_action(self, node, action): """Determine how to handle the function action.""" name = action.function_name() # This is the name given by the Subst/Textfile builders. So return the - # node to indicate that SCons is required + # node to indicate that SCons is required. We skip sources here because + # dependencies don't really matter when we're going to shove these to + # the bottom of ninja's DAG anyway and Textfile builders can have text + # content as their source which doesn't work as an implicit dep in + # ninja. if name == "_action": return { "rule": "TEMPLATE", "outputs": get_outputs(node), - "implicit": get_dependencies(node), + "implicit": get_dependencies(node, skip_sources=True), } - handler = self.func_handlers.get(name, None) if handler is not None: return handler(node.env if node.env else self.env, node) + elif name == "ActionCaller": + action_to_call = str(action).split('(')[0].strip() + handler = self.func_handlers.get(action_to_call, None) + if handler is not None: + return handler(node.env if node.env else self.env, node) - print( + raise Exception( "Found unhandled function action {}, " " generating scons command to build\n" "Note: this is less efficient than Ninja," @@ -218,48 +291,17 @@ def handle_func_action(self, node, action): " this function using NinjaRegisterFunctionHandler".format(name) ) - return { - "rule": "SCONS", - "outputs": get_outputs(node), - "implicit": get_dependencies(node), - } - # pylint: disable=too-many-branches def handle_list_action(self, node, action): - """ - Attempt to translate list actions to Ninja. - - List actions are tricky to move to ninja. First we translate - each individual action in the action list to a Ninja - build. Then we process the resulting ninja builds to see if - they are all the same ninja rule. If they are not all the same - rule we cannot make them a single resulting ninja build, so - instead we make them a single SCons invocation to build all of - the targets. - - If they are all the same rule and the rule is CMD we attempt - to combine the cmdlines together using ' && ' which we then - combine into a single ninja build. - - If they are all phony targets we simple combine the outputs - and dependencies. - - If they are all INSTALL rules we combine all of the sources - and outputs. - - If they are all SCONS rules we do the same as if they are not - the same rule and make a build that will use SCons to generate - them. - - If they're all one rule and None of the above rules we throw an Exception. - """ - + """TODO write this comment""" results = [ self.action_to_ninja_build(node, action=act) for act in action.list if act is not None ] - results = [result for result in results if result is not None] + results = [ + result for result in results if result is not None and result["outputs"] + ] if not results: return None @@ -268,28 +310,7 @@ def handle_list_action(self, node, action): return results[0] all_outputs = list({output for build in results for output in build["outputs"]}) - # If we have no outputs we're done - if not all_outputs: - return None - - # Used to verify if all rules are the same - all_one_rule = len( - [ - r - for r in results - if isinstance(r, dict) and r["rule"] == results[0]["rule"] - ] - ) == len(results) - dependencies = get_dependencies(node) - - if not all_one_rule: - # If they aren't all the same rule use scons to generate these - # outputs. At this time nothing hits this case. - return { - "outputs": all_outputs, - "rule": "SCONS", - "implicit": dependencies, - } + dependencies = list({dep for build in results for dep in build["implicit"]}) if results[0]["rule"] == "CMD": cmdline = "" @@ -322,7 +343,10 @@ def handle_list_action(self, node, action): ninja_build = { "outputs": all_outputs, "rule": "CMD", - "variables": {"cmd": cmdline}, + "variables": { + "cmd": cmdline, + "env": get_command_env(node.env if node.env else self.env), + }, "implicit": dependencies, } @@ -342,18 +366,10 @@ def handle_list_action(self, node, action): return { "outputs": all_outputs, "rule": "INSTALL", - "pool": "install_pool", "inputs": [get_path(src_file(s)) for s in node.sources], "implicit": dependencies, } - elif results[0]["rule"] == "SCONS": - return { - "outputs": all_outputs, - "rule": "SCONS", - "inputs": dependencies, - } - raise Exception("Unhandled list action with rule: " + results[0]["rule"]) @@ -369,7 +385,7 @@ def __init__(self, env, writer_class): self.generated_suffixes = env.get("NINJA_GENERATED_SOURCE_SUFFIXES", []) # List of generated builds that will be written at a later stage - self.builds = list() + self.builds = dict() # List of targets for which we have generated a build. This # allows us to take multiple Alias nodes as sources and to not @@ -383,7 +399,7 @@ def __init__(self, env, writer_class): escape = env.get("ESCAPE", lambda x: x) self.variables = { - "COPY": "cmd.exe /c copy" if sys.platform == "win32" else "cp", + "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format( sys.executable, " ".join( @@ -402,16 +418,46 @@ def __init__(self, env, writer_class): self.rules = { "CMD": { - "command": "cmd /c $cmd" if sys.platform == "win32" else "$cmd", + "command": "cmd /c $env$cmd $in $out" if sys.platform == "win32" else "$env$cmd $in $out", + "description": "Building $out", + "pool": "local_pool", + }, + "GENERATED_CMD": { + "command": "cmd /c $env$cmd" if sys.platform == "win32" else "$env$cmd", "description": "Building $out", + "pool": "local_pool", }, # We add the deps processing variables to this below. We - # don't pipe this through cmd.exe on Windows because we + # don't pipe these through cmd.exe on Windows because we # use this to generate a compile_commands.json database # which can't use the shell command as it's compile - # command. This does mean that we assume anything using - # CMD_W_DEPS is a straight up compile which is true today. - "CMD_W_DEPS": {"command": "$cmd", "description": "Building $out"}, + # command. + "CC": { + "command": "$env$CC @$out.rsp", + "description": "Compiling $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + }, + "CXX": { + "command": "$env$CXX @$out.rsp", + "description": "Compiling $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + }, + "LINK": { + "command": "$env$LINK @$out.rsp", + "description": "Linking $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + "pool": "local_pool", + }, + "AR": { + "command": "$env$AR @$out.rsp", + "description": "Archiving $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + "pool": "local_pool", + }, "SYMLINK": { "command": ( "cmd /c mklink $out $in" @@ -423,6 +469,7 @@ def __init__(self, env, writer_class): "INSTALL": { "command": "$COPY $in $out", "description": "Install $out", + "pool": "install_pool", # On Windows cmd.exe /c copy does not always correctly # update the timestamp on the output file. This leads # to a stuck constant timestamp in the Ninja database @@ -479,65 +526,43 @@ def __init__(self, env, writer_class): } self.pools = { + "local_pool": self.env.GetOption("num_jobs"), "install_pool": self.env.GetOption("num_jobs") / 2, "scons_pool": 1, } - if env["PLATFORM"] == "win32": - self.rules["CMD_W_DEPS"]["deps"] = "msvc" - else: - self.rules["CMD_W_DEPS"]["deps"] = "gcc" - self.rules["CMD_W_DEPS"]["depfile"] = "$out.d" - - self.rules.update(env.get(NINJA_RULES, {})) - self.pools.update(env.get(NINJA_POOLS, {})) - - def generate_builds(self, node): - """Generate a ninja build rule for node and it's children.""" - # Filter out nodes with no builder. They are likely source files - # and so no work needs to be done, it will be used in the - # generation for some real target. - # - # Note that all nodes have a builder attribute but it is sometimes - # set to None. So we cannot use a simpler hasattr check here. - if getattr(node, "builder", None) is None: - return + for rule in ["CC", "CXX"]: + if env["PLATFORM"] == "win32": + self.rules[rule]["deps"] = "msvc" + else: + self.rules[rule]["deps"] = "gcc" + self.rules[rule]["depfile"] = "$out.d" - stack = [[node]] - while stack: - frame = stack.pop() - for child in frame: - outputs = set(get_outputs(child)) - # Check if all the outputs are in self.built, if they - # are we've already seen this node and it's children. - if not outputs.isdisjoint(self.built): - continue + def add_build(self, node): + if not node.has_builder(): + return False - self.built = self.built.union(outputs) - stack.append(child.children()) - - if isinstance(child, SCons.Node.Alias.Alias): - build = alias_to_ninja_build(child) - elif node.builder is not None: - # Use False since None is a valid value for this attribute - build = getattr(child.attributes, NINJA_BUILD, False) - if build is False: - build = self.translator.action_to_ninja_build(child) - setattr(child.attributes, NINJA_BUILD, build) - else: - build = None + if isinstance(node, SCons.Node.Alias.Alias): + build = alias_to_ninja_build(node) + else: + build = self.translator.action_to_ninja_build(node) - # Some things are unbuild-able or need not be built in Ninja - if build is None or build == 0: - continue + # Some things are unbuild-able or need not be built in Ninja + if build is None: + return False - self.builds.append(build) + node_string = str(node) + if node_string in self.builds: + raise Exception("Node {} added to ninja build state more than once".format(node_string)) + self.builds[node_string] = build + self.built.update(build["outputs"]) + return True def is_generated_source(self, output): """Check if output ends with a known generated suffix.""" _, suffix = splitext(output) return suffix in self.generated_suffixes - + def has_generated_sources(self, output): """ Determine if output indicates this is a generated header file. @@ -548,7 +573,7 @@ def has_generated_sources(self, output): return False # pylint: disable=too-many-branches,too-many-locals - def generate(self, ninja_file, fallback_default_target=None): + def generate(self, ninja_file): """ Generate the build.ninja. @@ -557,6 +582,9 @@ def generate(self, ninja_file, fallback_default_target=None): if self.__generated: return + self.rules.update(self.env.get(NINJA_RULES, {})) + self.pools.update(self.env.get(NINJA_POOLS, {})) + content = io.StringIO() ninja = self.writer_class(content, width=100) @@ -571,10 +599,10 @@ def generate(self, ninja_file, fallback_default_target=None): for rule, kwargs in self.rules.items(): ninja.rule(rule, **kwargs) - generated_source_files = { + generated_source_files = sorted({ output # First find builds which have header files in their outputs. - for build in self.builds + for build in self.builds.values() if self.has_generated_sources(build["outputs"]) for output in build["outputs"] # Collect only the header files from the builds with them @@ -583,25 +611,24 @@ def generate(self, ninja_file, fallback_default_target=None): # here we need to filter so we only have the headers and # not the other outputs. if self.is_generated_source(output) - } + }) if generated_source_files: ninja.build( outputs="_generated_sources", rule="phony", - implicit=list(generated_source_files), + implicit=generated_source_files ) template_builders = [] - for build in self.builds: + for build in [self.builds[key] for key in sorted(self.builds.keys())]: if build["rule"] == "TEMPLATE": template_builders.append(build) continue - implicit = build.get("implicit", []) - implicit.append(ninja_file) - build["implicit"] = implicit + if "implicit" in build: + build["implicit"].sort() # Don't make generated sources depend on each other. We # have to check that none of the outputs are generated @@ -612,7 +639,7 @@ def generate(self, ninja_file, fallback_default_target=None): generated_source_files and not build["rule"] == "INSTALL" and set(build["outputs"]).isdisjoint(generated_source_files) - and set(implicit).isdisjoint(generated_source_files) + and set(build.get("implicit", [])).isdisjoint(generated_source_files) ): # Make all non-generated source targets depend on @@ -621,7 +648,11 @@ def generate(self, ninja_file, fallback_default_target=None): # generated source was rebuilt. We just need to make # sure that all of these sources are generated before # other builds. - build["order_only"] = "_generated_sources" + order_only = build.get("order_only", []) + order_only.append("_generated_sources") + build["order_only"] = order_only + if "order_only" in build: + build["order_only"].sort() # When using a depfile Ninja can only have a single output # but SCons will usually have emitted an output for every @@ -637,26 +668,31 @@ def generate(self, ninja_file, fallback_default_target=None): # Some rules like 'phony' and other builtins we don't have # listed in self.rules so verify that we got a result # before trying to check if it has a deps key. - if rule is not None and rule.get("deps"): - - # Anything using deps in Ninja can only have a single - # output, but we may have a build which actually - # produces multiple outputs which other targets can - # depend on. Here we slice up the outputs so we have a - # single output which we will use for the "real" - # builder and multiple phony targets that match the - # file names of the remaining outputs. This way any - # build can depend on any output from any build. - first_output, remaining_outputs = build["outputs"][0], build["outputs"][1:] + # + # Anything using deps or rspfile in Ninja can only have a single + # output, but we may have a build which actually produces + # multiple outputs which other targets can depend on. Here we + # slice up the outputs so we have a single output which we will + # use for the "real" builder and multiple phony targets that + # match the file names of the remaining outputs. This way any + # build can depend on any output from any build. + build["outputs"].sort() + if rule is not None and (rule.get("deps") or rule.get("rspfile")): + first_output, remaining_outputs = ( + build["outputs"][0], + build["outputs"][1:], + ) + if remaining_outputs: ninja.build( - outputs=remaining_outputs, - rule="phony", - implicit=first_output, + outputs=remaining_outputs, rule="phony", implicit=first_output, ) build["outputs"] = first_output + if "inputs" in build: + build["inputs"].sort() + ninja.build(**build) template_builds = dict() @@ -682,37 +718,37 @@ def generate(self, ninja_file, fallback_default_target=None): if template_builds.get("outputs", []): ninja.build(**template_builds) - # Here to teach the ninja file how to regenerate itself. We'll - # never see ourselves in the DAG walk so we can't rely on - # action_to_ninja_build to generate this rule + # We have to glob the SCons files here to teach the ninja file + # how to regenerate itself. We'll never see ourselves in the + # DAG walk so we can't rely on action_to_ninja_build to + # generate this rule even though SCons should know we're + # dependent on SCons files. + # + # TODO: We're working on getting an API into SCons that will + # allow us to query the actual SConscripts used. Right now + # this glob method has deficiencies like skipping + # jstests/SConscript and being specific to the MongoDB + # repository layout. ninja.build( - ninja_file, + self.env.File(ninja_file).path, rule="REGENERATE", implicit=[ - self.env.File("#SConstruct").get_abspath(), - os.path.abspath(__file__), + self.env.File("#SConstruct").path, + __file__, ] - + LOADED_SCONSCRIPTS, + + sorted(glob("src/**/SConscript", recursive=True)), ) - ninja.build( - "scons-invocation", - rule="CMD", - pool="console", - variables={"cmd": "echo $SCONS_INVOCATION_W_TARGETS"}, - ) - - # Note the use of CMD_W_DEPS below. CMD_W_DEPS are always - # compile commands in this generator. If we ever change the - # name/s of the rules that include compile commands - # (i.e. something like CC/CXX) we will need to update this - # build to reflect that complete list. + # If we ever change the name/s of the rules that include + # compile commands (i.e. something like CC) we will need to + # update this build to reflect that complete list. ninja.build( "compile_commands.json", rule="CMD", pool="console", + implicit=[ninja_file], variables={ - "cmd": "ninja -f {} -t compdb CMD_W_DEPS > compile_commands.json".format( + "cmd": "ninja -f {} -t compdb CC CXX > compile_commands.json".format( ninja_file ) }, @@ -736,11 +772,6 @@ def generate(self, ninja_file, fallback_default_target=None): if scons_default_targets: ninja.default(" ".join(scons_default_targets)) - # If not then set the default to the fallback_default_target we were given. - # Otherwise we won't create a default ninja target. - elif fallback_default_target is not None: - ninja.default(fallback_default_target) - with open(ninja_file, "w") as build_ninja: build_ninja.write(content.getvalue()) @@ -776,9 +807,158 @@ def src_file(node): return get_path(node) -# TODO: Make the Rules smarter. Instead of just using a "cmd" rule -# everywhere we should be smarter about generating CC, CXX, LINK, -# etc. rules +def get_comstr(env, action, targets, sources): + """Get the un-substituted string for action.""" + # Despite being having "list" in it's name this member is not + # actually a list. It's the pre-subst'd string of the command. We + # use it to determine if the command we're about to generate needs + # to use a custom Ninja rule. By default this redirects CC, CXX, + # AR, SHLINK, and LINK commands to their respective rules but the + # user can inject custom Ninja rules and tie them to commands by + # using their pre-subst'd string. + if hasattr(action, "process"): + return action.cmd_list + + return action.genstring(targets, sources, env) + + +def get_command_env(env): + """ + Return a string that sets the enrivonment for any environment variables that + differ between the OS environment and the SCons command ENV. + + It will be compatible with the default shell of the operating system. + """ + try: + return env["NINJA_ENV_VAR_CACHE"] + except KeyError: + pass + + # Scan the ENV looking for any keys which do not exist in + # os.environ or differ from it. We assume if it's a new or + # differing key from the process environment then it's + # important to pass down to commands in the Ninja file. + ENV = get_default_ENV(env) + scons_specified_env = { + key: value + for key, value in ENV.items() + if key not in os.environ or os.environ.get(key, None) != value + } + + windows = env["PLATFORM"] == "win32" + command_env = "" + for key, value in scons_specified_env.items(): + # Ensure that the ENV values are all strings: + if is_List(value): + # If the value is a list, then we assume it is a + # path list, because that's a pretty common list-like + # value to stick in an environment variable: + value = flatten_sequence(value) + value = joinpath(map(str, value)) + else: + # If it isn't a string or a list, then we just coerce + # it to a string, which is the proper way to handle + # Dir and File instances and will produce something + # reasonable for just about everything else: + value = str(value) + + if windows: + command_env += "set '{}={}' && ".format(key, value) + else: + command_env += "{}={} ".format(key, value) + + env["NINJA_ENV_VAR_CACHE"] = command_env + return command_env + + +def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False): + """Generate a response file command provider for rule name.""" + + # If win32 using the environment with a response file command will cause + # ninja to fail to create the response file. Additionally since these rules + # generally are not piping through cmd.exe /c any environment variables will + # make CreateProcess fail to start. + # + # On POSIX we can still set environment variables even for compile + # commands so we do so. + use_command_env = not env["PLATFORM"] == "win32" + if "$" in tool: + tool_is_dynamic = True + + def get_response_file_command(env, node, action, targets, sources, executor=None): + if hasattr(action, "process"): + cmd_list, _, _ = action.process(targets, sources, env, executor=executor) + cmd_list = [str(c).replace("$", "$$") for c in cmd_list[0]] + else: + command = generate_command( + env, node, action, targets, sources, executor=executor + ) + cmd_list = shlex.split(command) + + if tool_is_dynamic: + tool_command = env.subst( + tool, target=targets, source=sources, executor=executor + ) + else: + tool_command = tool + + try: + # Add 1 so we always keep the actual tool inside of cmd + tool_idx = cmd_list.index(tool_command) + 1 + except ValueError: + raise Exception( + "Could not find tool {} in {} generated from {}".format( + tool_command, cmd_list, get_comstr(env, action, targets, sources) + ) + ) + + cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] + rsp_content = " ".join(rsp_content) + + variables = {"rspc": rsp_content} + variables[rule] = cmd + if use_command_env: + variables["env"] = get_command_env(env) + return rule, variables + + return get_response_file_command + + +def generate_command(env, node, action, targets, sources, executor=None): + # Actions like CommandAction have a method called process that is + # used by SCons to generate the cmd_line they need to run. So + # check if it's a thing like CommandAction and call it if we can. + if hasattr(action, "process"): + cmd_list, _, _ = action.process(targets, sources, env, executor=executor) + cmd = _string_from_cmd_list(cmd_list[0]) + else: + # Anything else works with genstring, this is most commonly hit by + # ListActions which essentially call process on all of their + # commands and concatenate it for us. + genstring = action.genstring(targets, sources, env) + if executor is not None: + cmd = env.subst(genstring, executor=executor) + else: + cmd = env.subst(genstring, targets, sources) + + cmd = cmd.replace("\n", " && ").strip() + if cmd.endswith("&&"): + cmd = cmd[0:-2].strip() + + # Escape dollars as necessary + return cmd.replace("$", "$$") + + +def get_shell_command(env, node, action, targets, sources, executor=None): + return ( + "GENERATED_CMD", + { + "cmd": generate_command(env, node, action, targets, sources, executor=None), + "env": get_command_env(env), + }, + ) + + def get_command(env, node, action): # pylint: disable=too-many-branches """Get the command to execute for node.""" if node.env: @@ -800,121 +980,26 @@ def get_command(env, node, action): # pylint: disable=too-many-branches # Retrieve the repository file for all sources slist = [rfile(s) for s in slist] - # Get the dependencies for all targets - implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) - # Generate a real CommandAction if isinstance(action, SCons.Action.CommandGeneratorAction): # pylint: disable=protected-access action = action._generate(tlist, slist, sub_env, 1, executor=executor) - rule = "CMD" - - # Actions like CommandAction have a method called process that is - # used by SCons to generate the cmd_line they need to run. So - # check if it's a thing like CommandAction and call it if we can. - if hasattr(action, "process"): - cmd_list, _, _ = action.process(tlist, slist, sub_env, executor=executor) - - # Despite being having "list" in it's name this member is not - # actually a list. It's the pre-subst'd string of the command. We - # use it to determine if the command we generated needs to use a - # custom Ninja rule. By default this redirects CC/CXX commands to - # CMD_W_DEPS but the user can inject custom Ninja rules and tie - # them to commands by using their pre-subst'd string. - rule = __NINJA_RULE_MAPPING.get(action.cmd_list, "CMD") - - cmd = _string_from_cmd_list(cmd_list[0]) - else: - # Anything else works with genstring, this is most commonly hit by - # ListActions which essentially call process on all of their - # commands and concatenate it for us. - genstring = action.genstring(tlist, slist, sub_env) + variables = {} - # Detect if we have a custom rule for this - # "ListActionCommandAction" type thing. - rule = __NINJA_RULE_MAPPING.get(genstring, "CMD") - - if executor is not None: - cmd = sub_env.subst(genstring, executor=executor) - else: - cmd = sub_env.subst(genstring, target=tlist, source=slist) - - # Since we're only enabling Ninja for developer builds right - # now we skip all Manifest related work on Windows as it's not - # necessary. We shouldn't have gotten here but on Windows - # SCons has a ListAction which shows as a - # CommandGeneratorAction for linking. That ListAction ends - # with a FunctionAction (embedManifestExeCheck, - # embedManifestDllCheck) that simply say "does - # target[0].manifest exist?" if so execute the real command - # action underlying me, otherwise do nothing. - # - # Eventually we'll want to find a way to translate this to - # Ninja but for now, and partially because the existing Ninja - # generator does so, we just disable it all together. - cmd = cmd.replace("\n", " && ").strip() - if env["PLATFORM"] == "win32" and ( - "embedManifestExeCheck" in cmd or "embedManifestDllCheck" in cmd - ): - cmd = " && ".join(cmd.split(" && ")[0:-1]) - - if cmd.endswith("&&"): - cmd = cmd[0:-2].strip() - - outputs = get_outputs(node) - command_env = "" - windows = env["PLATFORM"] == "win32" - - # If win32 and rule == CMD_W_DEPS then we don't want to calculate - # an environment for this command. It's a compile command and - # compiledb doesn't support shell syntax on Windows. We need the - # shell syntax to use environment variables on Windows so we just - # skip this platform / rule combination to keep the compiledb - # working. - # - # On POSIX we can still set environment variables even for compile - # commands so we do so. - if not (windows and rule == "CMD_W_DEPS"): - - # Scan the ENV looking for any keys which do not exist in - # os.environ or differ from it. We assume if it's a new or - # differing key from the process environment then it's - # important to pass down to commands in the Ninja file. - ENV = get_default_ENV(sub_env) - scons_specified_env = { - key: value - for key, value in ENV.items() - if key not in os.environ or os.environ.get(key, None) != value - } + comstr = get_comstr(sub_env, action, tlist, slist) + if not comstr: + return None - for key, value in scons_specified_env.items(): - # Ensure that the ENV values are all strings: - if is_List(value): - # If the value is a list, then we assume it is a - # path list, because that's a pretty common list-like - # value to stick in an environment variable: - value = flatten_sequence(value) - value = joinpath(map(str, value)) - else: - # If it isn't a string or a list, then we just coerce - # it to a string, which is the proper way to handle - # Dir and File instances and will produce something - # reasonable for just about everything else: - value = str(value) - - if windows: - command_env += "set '{}={}' && ".format(key, value) - else: - command_env += "{}={} ".format(key, value) + provider = __NINJA_RULE_MAPPING.get(comstr, get_shell_command) + rule, variables = provider(sub_env, node, action, tlist, slist, executor=executor) - variables = {"cmd": command_env + cmd} - extra_vars = getattr(node.attributes, "NINJA_EXTRA_VARS", {}) - if extra_vars: - variables.update(extra_vars) + # Get the dependencies for all targets + implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) ninja_build = { - "outputs": outputs, + "order_only": get_order_only(node), + "outputs": get_outputs(node), "inputs": get_inputs(node), "implicit": implicit, "rule": rule, @@ -953,37 +1038,30 @@ def ninja_builder(env, target, source): # here. print("Generating:", str(target[0])) - # The environment variable NINJA_SYNTAX points to the - # ninja_syntax.py module from the ninja sources found here: - # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py - # - # This should be vendored into the build sources and it's location - # set in NINJA_SYNTAX. This code block loads the location from - # that variable, gets the absolute path to the vendored file, gets - # it's parent directory then uses importlib to import the module - # dynamically. - ninja_syntax_file = env[NINJA_SYNTAX] - if isinstance(ninja_syntax_file, str): - ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() - ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) - sys.path.append(ninja_syntax_mod_dir) - ninja_syntax_mod_name = os.path.basename(ninja_syntax_file) - ninja_syntax = importlib.import_module(ninja_syntax_mod_name.replace(".py", "")) - - suffix = env.get("NINJA_SUFFIX", "") - if suffix and not suffix[0] == ".": - suffix = "." + suffix + generated_build_ninja = target[0].get_abspath() + NINJA_STATE.generate(generated_build_ninja) + if env.get("DISABLE_AUTO_NINJA") != True: + print("Executing:", str(target[0])) - generated_build_ninja = target[0].get_abspath() + suffix - ninja_state = NinjaState(env, ninja_syntax.Writer) - - for src in source: - ninja_state.generate_builds(src) - - ninja_state.generate(generated_build_ninja, str(source[0])) - - return 0 + def execute_ninja(): + proc = subprocess.Popen( ['ninja', '-f', generated_build_ninja], + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + universal_newlines=True + ) + for stdout_line in iter(proc.stdout.readline, ""): + yield stdout_line + proc.stdout.close() + return_code = proc.wait() + if return_code: + raise subprocess.CalledProcessError(return_code, 'ninja') + + for output in execute_ninja(): + output = output.strip() + sys.stdout.write('\x1b[2K') # erase previous line + sys.stdout.write(output + "\r") + sys.stdout.flush() # pylint: disable=too-few-public-methods class AlwaysExecAction(SCons.Action.FunctionAction): @@ -994,25 +1072,6 @@ def __call__(self, *args, **kwargs): return super().__call__(*args, **kwargs) -def ninja_print(_cmd, target, _source, env): - """Tag targets with the commands to build them.""" - if target: - for tgt in target: - if ( - tgt.has_builder() - # Use 'is False' because not would still trigger on - # None's which we don't want to regenerate - and getattr(tgt.attributes, NINJA_BUILD, False) is False - and isinstance(tgt.builder.action, COMMAND_TYPES) - ): - ninja_action = get_command(env, tgt, tgt.builder.action) - setattr(tgt.attributes, NINJA_BUILD, ninja_action) - # Preload the attributes dependencies while we're still running - # multithreaded - get_dependencies(tgt) - return 0 - - def register_custom_handler(env, name, handler): """Register a custom handler for SCons function actions.""" env[NINJA_CUSTOM_HANDLERS][name] = handler @@ -1024,7 +1083,7 @@ def register_custom_rule_mapping(env, pre_subst_string, rule): __NINJA_RULE_MAPPING[pre_subst_string] = rule -def register_custom_rule(env, rule, command, description="", deps=None): +def register_custom_rule(env, rule, command, description="", deps=None, pool=None): """Allows specification of Ninja rules from inside SCons files.""" rule_obj = { "command": command, @@ -1034,6 +1093,9 @@ def register_custom_rule(env, rule, command, description="", deps=None): if deps is not None: rule_obj["deps"] = deps + if pool is not None: + rule_obj["pool"] = pool + env[NINJA_RULES][rule] = rule_obj @@ -1075,11 +1137,6 @@ def ninja_stat(_self, path): running in a no_exec build the file system state should not change. For these reasons we patch SCons.Node.FS.LocalFS.stat to use our eternal memoized dictionary. - - Since this is happening during the Node walk it's being run while - threaded, we have to protect adding to the memoized dictionary - with a threading.Lock otherwise many targets miss the memoization - due to racing. """ global NINJA_STAT_MEMO @@ -1091,9 +1148,7 @@ def ninja_stat(_self, path): except os.error: result = None - with MEMO_LOCK: - NINJA_STAT_MEMO[path] = result - + NINJA_STAT_MEMO[path] = result return result @@ -1145,71 +1200,11 @@ def ninja_always_serial(self, num, taskmaster): self.job = SCons.Job.Serial(taskmaster) -class NinjaEternalTempFile(SCons.Platform.TempFileMunge): +class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" def __call__(self, target, source, env, for_signature): - if for_signature: - return self.cmd - - node = target[0] if SCons.Util.is_List(target) else target - if node is not None: - cmdlist = getattr(node.attributes, "tempfile_cmdlist", None) - if cmdlist is not None: - return cmdlist - - cmd = super().__call__(target, source, env, for_signature) - - # If TempFileMunge.__call__ returns a string it means that no - # response file was needed. No processing required so just - # return the command. - if isinstance(cmd, str): - return cmd - - # Strip the removal commands from the command list. - # - # SCons' TempFileMunge class has some very strange - # behavior where it, as part of the command line, tries to - # delete the response file after executing the link - # command. We want to keep those response files since - # Ninja will keep using them over and over. The - # TempFileMunge class creates a cmdlist to do this, a - # common SCons convention for executing commands see: - # https://github.com/SCons/scons/blob/master/src/engine/SCons/Action.py#L949 - # - # This deletion behavior is not configurable. So we wanted - # to remove the deletion command from the command list by - # simply slicing it out here. Unfortunately for some - # strange reason TempFileMunge doesn't make the "rm" - # command it's own list element. It appends it to the - # tempfile argument to cmd[0] (which is CC/CXX) and then - # adds the tempfile again as it's own element. - # - # So we just kind of skip that middle element. Since the - # tempfile is in the command list on it's own at the end we - # can cut it out entirely. This is what I would call - # "likely to break" in future SCons updates. Hopefully it - # breaks because they start doing the right thing and not - # weirdly splitting these arguments up. For reference a - # command list that we get back from the OG TempFileMunge - # looks like this: - # - # [ - # 'g++', - # '@/mats/tempfiles/random_string.lnk\nrm', - # '/mats/tempfiles/random_string.lnk', - # ] - # - # Note the weird newline and rm command in the middle - # element and the lack of TEMPFILEPREFIX on the last - # element. - prefix = env.subst("$TEMPFILEPREFIX") - if not prefix: - prefix = "@" - - new_cmdlist = [cmd[0], prefix + cmd[-1]] - setattr(node.attributes, "tempfile_cmdlist", new_cmdlist) - return new_cmdlist + return self.cmd def _print_cmd_str(*_args, **_kwargs): """Disable this method""" @@ -1229,9 +1224,22 @@ def exists(env): return True +added = None def generate(env): """Generate the NINJA builders.""" + from SCons.Script import AddOption, GetOption + global added + if not added: + added = 1 + AddOption('--disable-auto-ninja', + dest='disable_auto_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + env["DISABLE_AUTO_NINJA"] = GetOption('disable_auto_ninja') + env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") # Add the Ninja builder. @@ -1239,6 +1247,15 @@ def generate(env): ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action) env.Append(BUILDERS={"Ninja": ninja_builder_obj}) + env["NINJA_PREFIX"] = env.get("NINJA_PREFIX", "build") + env["NINJA_SUFFIX"] = env.get("NINJA_SUFFIX", "ninja") + env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") + + ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") + ninja_file = env.Ninja(target=ninja_file_name, source=[]) + env.AlwaysBuild(ninja_file) + env.Alias("$NINJA_ALIAS_NAME", ninja_file) + # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. if env["PLATFORM"] == "win32": @@ -1246,6 +1263,11 @@ def generate(env): else: env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) + # Provide a way for custom rule authors to easily access command + # generation. + env.AddMethod(get_shell_command, "NinjaGetShellCommand") + env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider") + # Provides a way for users to handle custom FunctionActions they # want to translate to Ninja. env[NINJA_CUSTOM_HANDLERS] = {} @@ -1270,8 +1292,40 @@ def generate(env): # deleted you would get a very subtly incorrect Ninja file and # might not catch it. env.AddMethod(register_custom_rule_mapping, "NinjaRuleMapping") - env.NinjaRuleMapping("${CCCOM}", "CMD_W_DEPS") - env.NinjaRuleMapping("${CXXCOM}", "CMD_W_DEPS") + + # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks + # without relying on the SCons hacks that SCons uses by default. + if env["PLATFORM"] == "win32": + from SCons.Tool.mslink import compositeLinkAction + + if env["LINKCOM"] == compositeLinkAction: + env[ + "LINKCOM" + ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' + env[ + "SHLINKCOM" + ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' + + # Normally in SCons actions for the Program and *Library builders + # will return "${*COM}" as their pre-subst'd command line. However + # if a user in a SConscript overwrites those values via key access + # like env["LINKCOM"] = "$( $ICERUN $)" + env["LINKCOM"] then + # those actions no longer return the "bracketted" string and + # instead return something that looks more expanded. So to + # continue working even if a user has done this we map both the + # "bracketted" and semi-expanded versions. + def robust_rule_mapping(var, rule, tool): + provider = gen_get_response_file_command(env, rule, tool) + env.NinjaRuleMapping("${" + var + "}", provider) + env.NinjaRuleMapping(env[var], provider) + + robust_rule_mapping("CCCOM", "CC", "$CC") + robust_rule_mapping("SHCCCOM", "CC", "$CC") + robust_rule_mapping("CXXCOM", "CXX", "$CXX") + robust_rule_mapping("SHCXXCOM", "CXX", "$CXX") + robust_rule_mapping("LINKCOM", "LINK", "$LINK") + robust_rule_mapping("SHLINKCOM", "LINK", "$SHLINK") + robust_rule_mapping("ARCOM", "AR", "$AR") # Make SCons node walk faster by preventing unnecessary work env.Decider("timestamp-match") @@ -1281,6 +1335,21 @@ def generate(env): # dependencies to any builds that *might* use them. env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] + if env["PLATFORM"] != "win32" and env.get("RANLIBCOM"): + # There is no way to translate the ranlib list action into + # Ninja so add the s flag and disable ranlib. + # + # This is equivalent to Meson. + # https://github.com/mesonbuild/meson/blob/master/mesonbuild/linkers.py#L143 + old_arflags = str(env["ARFLAGS"]) + if "s" not in old_arflags: + old_arflags += "s" + + env["ARFLAGS"] = SCons.Util.CLVar([old_arflags]) + + # Disable running ranlib, since we added 's' above + env["RANLIBCOM"] = "" + # This is the point of no return, anything after this comment # makes changes to SCons that are irreversible and incompatible # with a normal SCons build. We return early if __NINJA_NO=1 has @@ -1304,7 +1373,9 @@ def generate(env): SCons.Node.FS.File.prepare = ninja_noop SCons.Node.FS.File.push_to_cache = ninja_noop SCons.Executor.Executor.prepare = ninja_noop + SCons.Taskmaster.Task.prepare = ninja_noop SCons.Node.FS.File.built = ninja_noop + SCons.Node.Node.visited = ninja_noop # We make lstat a no-op because it is only used for SONAME # symlinks which we're not producing. @@ -1336,20 +1407,8 @@ def generate(env): SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig) - # Replace false Compiling* messages with a more accurate output - # - # We also use this to tag all Nodes with Builders using - # CommandActions with the final command that was used to compile - # it for passing to Ninja. If we don't inject this behavior at - # this stage in the build too much state is lost to generate the - # command at the actual ninja_builder execution time for most - # commands. - # - # We do attempt command generation again in ninja_builder if it - # hasn't been tagged and it seems to work for anything that - # doesn't represent as a non-FunctionAction during the print_func - # call. - env["PRINT_CMD_LINE_FUNC"] = ninja_print + # Replace false action messages with nothing. + env["PRINT_CMD_LINE_FUNC"] = ninja_noop # This reduces unnecessary subst_list calls to add the compiler to # the implicit dependencies of targets. Since we encode full paths @@ -1358,11 +1417,6 @@ def generate(env): # where we expect it. env["IMPLICIT_COMMAND_DEPENDENCIES"] = False - # Set build to no_exec, our sublcass of FunctionAction will force - # an execution for ninja_builder so this simply effects all other - # Builders. - env.SetOption("no_exec", True) - # This makes SCons more aggressively cache MD5 signatures in the # SConsign file. env.SetOption("max_drift", 1) @@ -1372,6 +1426,84 @@ def generate(env): # monkey the Jobs constructor to only use the Serial Job class. SCons.Job.Jobs.__init__ = ninja_always_serial + # The environment variable NINJA_SYNTAX points to the + # ninja_syntax.py module from the ninja sources found here: + # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py + # + # This should be vendored into the build sources and it's location + # set in NINJA_SYNTAX. This code block loads the location from + # that variable, gets the absolute path to the vendored file, gets + # it's parent directory then uses importlib to import the module + # dynamically. + ninja_syntax_file = env[NINJA_SYNTAX] + + if os.path.exists(ninja_syntax_file): + if isinstance(ninja_syntax_file, str): + ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() + ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) + sys.path.append(ninja_syntax_mod_dir) + ninja_syntax_mod_name = os.path.basename(ninja_syntax_file).replace(".py", "") + ninja_syntax = importlib.import_module(ninja_syntax_mod_name) + else: + ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') + + global NINJA_STATE + NINJA_STATE = NinjaState(env, ninja_syntax.Writer) + + # Here we will force every builder to use an emitter which makes the ninja + # file depend on it's target. This forces the ninja file to the bottom of + # the DAG which is required so that we walk every target, and therefore add + # it to the global NINJA_STATE, before we try to write the ninja file. + def ninja_file_depends_on_all(target, source, env): + if not any("conftest" in str(t) for t in target): + env.Depends(ninja_file, target) + return target, source + + # The "Alias Builder" isn't in the BUILDERS map so we have to + # modify it directly. + SCons.Environment.AliasBuilder.emitter = ninja_file_depends_on_all + + for _, builder in env["BUILDERS"].items(): + try: + emitter = builder.emitter + if emitter is not None: + builder.emitter = SCons.Builder.ListEmitter( + [emitter, ninja_file_depends_on_all] + ) + else: + builder.emitter = ninja_file_depends_on_all + # Users can inject whatever they want into the BUILDERS + # dictionary so if the thing doesn't have an emitter we'll + # just ignore it. + except AttributeError: + pass + + # Here we monkey patch the Task.execute method to not do a bunch of + # unnecessary work. If a build is a regular builder (i.e not a conftest and + # not our own Ninja builder) then we add it to the NINJA_STATE. Otherwise we + # build it like normal. This skips all of the caching work that this method + # would normally do since we aren't pulling any of these targets from the + # cache. + # + # In the future we may be able to use this to actually cache the build.ninja + # file once we have the upstream support for referencing SConscripts as File + # nodes. + def ninja_execute(self): + global NINJA_STATE + + target = self.targets[0] + target_name = str(target) + if target_name != ninja_file_name and "conftest" not in target_name: + NINJA_STATE.add_build(target) + else: + target.build() + + SCons.Taskmaster.Task.execute = ninja_execute + + # Make needs_execute always return true instead of determining out of + # date-ness. + SCons.Script.Main.BuildTask.needs_execute = lambda x: True + # We will eventually need to overwrite TempFileMunge to make it # handle persistent tempfiles or get an upstreamed change to add # some configurability to it's behavior in regards to tempfiles. @@ -1379,18 +1511,13 @@ def generate(env): # Set all three environment variables that Python's # tempfile.mkstemp looks at as it behaves differently on different # platforms and versions of Python. - os.environ["TMPDIR"] = env.Dir("$BUILD_DIR/response_files").get_abspath() + build_dir = env.subst("$BUILD_DIR") + if build_dir == "": + build_dir = "." + os.environ["TMPDIR"] = env.Dir("{}/.response_files".format(build_dir)).get_abspath() os.environ["TEMP"] = os.environ["TMPDIR"] os.environ["TMP"] = os.environ["TMPDIR"] if not os.path.isdir(os.environ["TMPDIR"]): env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) - env["TEMPFILE"] = NinjaEternalTempFile - - # Force the SConsign to be written, we benefit from SCons caching of - # implicit dependencies and conftests. Unfortunately, we have to do this - # using an atexit handler because SCons will not write the file when in a - # no_exec build. - import atexit - - atexit.register(SCons.SConsign.write) + env["TEMPFILE"] = NinjaNoResponseFiles \ No newline at end of file diff --git a/test/ninja/CC.py b/test/ninja/CC.py new file mode 100644 index 0000000000..fe18721b9d --- /dev/null +++ b/test/ninja/CC.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import sys +import TestSCons + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') +env.Program(target = 'foo', source = 'foo.c') +""" % locals()) + +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) + +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed foo.o', + 'Removed foo', + 'Removed build.ninja']) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/ninja-fixture/bar.c b/test/ninja/ninja-fixture/bar.c new file mode 100644 index 0000000000..de1e6e50b6 --- /dev/null +++ b/test/ninja/ninja-fixture/bar.c @@ -0,0 +1,10 @@ +#include +#include + +int +main(int argc, char *argv[]) +{ + argv[argc++] = "--"; + printf("foo.c\n"); + exit (0); +} diff --git a/test/ninja/ninja-fixture/foo.c b/test/ninja/ninja-fixture/foo.c new file mode 100644 index 0000000000..de1e6e50b6 --- /dev/null +++ b/test/ninja/ninja-fixture/foo.c @@ -0,0 +1,10 @@ +#include +#include + +int +main(int argc, char *argv[]) +{ + argv[argc++] = "--"; + printf("foo.c\n"); + exit (0); +} diff --git a/test/ninja/ninja-fixture/test1.c b/test/ninja/ninja-fixture/test1.c new file mode 100644 index 0000000000..7535b0aa56 --- /dev/null +++ b/test/ninja/ninja-fixture/test1.c @@ -0,0 +1,3 @@ +This is a .c file. +/*cc*/ +/*link*/ diff --git a/test/ninja/ninja-fixture/test2.C b/test/ninja/ninja-fixture/test2.C new file mode 100644 index 0000000000..a1ee9e32b9 --- /dev/null +++ b/test/ninja/ninja-fixture/test2.C @@ -0,0 +1,3 @@ +This is a .C file. +/*cc*/ +/*link*/ From a242c6c61078558753f85439fea9275388df3de5 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 6 May 2020 13:40:11 -0400 Subject: [PATCH 079/163] added some more test and update ninja tool to handle commands --- src/engine/SCons/Tool/ninja.py | 15 +++- test/ninja/copy_function_command.py | 81 ++++++++++++++++++++ test/ninja/{CC.py => generate_and_build.py} | 15 +++- test/ninja/shell_command.py | 83 +++++++++++++++++++++ 4 files changed, 191 insertions(+), 3 deletions(-) create mode 100644 test/ninja/copy_function_command.py rename test/ninja/{CC.py => generate_and_build.py} (85%) create mode 100644 test/ninja/shell_command.py diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index d1cbafa495..e0129d8bcd 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -692,7 +692,7 @@ def generate(self, ninja_file): if "inputs" in build: build["inputs"].sort() - + ninja.build(**build) template_builds = dict() @@ -990,7 +990,7 @@ def get_command(env, node, action): # pylint: disable=too-many-branches comstr = get_comstr(sub_env, action, tlist, slist) if not comstr: return None - + provider = __NINJA_RULE_MAPPING.get(comstr, get_shell_command) rule, variables = provider(sub_env, node, action, tlist, slist, executor=executor) @@ -1478,6 +1478,17 @@ def ninja_file_depends_on_all(target, source, env): except AttributeError: pass + # We will subvert the normal Command to make sure all targets generated + # from commands will be linked to the ninja file + SconsCommand = SCons.Environment.Environment.Command + + def NinjaCommand(self, target, source, action, **kw): + targets = SconsCommand(env, target, source, action, **kw) + env.Depends(ninja_file, targets) + return targets + + SCons.Environment.Environment.Command = NinjaCommand + # Here we monkey patch the Task.execute method to not do a bunch of # unnecessary work. If a build is a regular builder (i.e not a conftest and # not our own Ninja builder) then we add it to the NINJA_STATE. Otherwise we diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py new file mode 100644 index 0000000000..f86f717e6f --- /dev/null +++ b/test/ninja/copy_function_command.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import sys +import TestSCons + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') +env.Command('foo2.c', ['foo.c'], Copy('$TARGET','$SOURCE')) +env.Program(target = 'foo', source = 'foo2.c') +""") + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed foo2.o', + 'Removed foo2.c', + 'Removed foo', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) + +# run ninja independently +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/CC.py b/test/ninja/generate_and_build.py similarity index 85% rename from test/ninja/CC.py rename to test/ninja/generate_and_build.py index fe18721b9d..fd111a29be 100644 --- a/test/ninja/CC.py +++ b/test/ninja/generate_and_build.py @@ -35,28 +35,41 @@ test.dir_fixture('ninja-fixture') +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + test.write('SConstruct', """ env = Environment() env.Tool('ninja') env.Program(target = 'foo', source = 'foo.c') -""" % locals()) +""") +# generate simple build test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +# clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo.o', 'Removed foo', 'Removed build.ninja']) + +# only generate the ninja file test.run(arguments='--disable-auto-ninja', stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_not_contain_any_line(test.stdout(), ['Executing: build.ninja']) +# run ninja independently +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) + test.pass_test() # Local Variables: diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py new file mode 100644 index 0000000000..fd0e35f8de --- /dev/null +++ b/test/ninja/shell_command.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import sys +import TestSCons + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') +env.Program(target = 'foo', source = 'foo.c') +env.Command('foo.out', ['foo'], './foo > foo.out') +""") + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_match('foo.out', 'foo.c' + os.linesep) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed foo.o', + 'Removed foo', + 'Removed foo.out', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) + +# run ninja independently +test.run(program = ninja, stdout=None) +test.must_match('foo.out', 'foo.c' + os.linesep) + + + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: From ef730840ae465134cab0d88057f404c181e4d3b0 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 6 May 2020 17:23:53 -0400 Subject: [PATCH 080/163] update to pass import.py test and support multiple environments --- src/engine/SCons/Tool/ninja.py | 41 ++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index e0129d8bcd..63ba243c61 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -274,6 +274,9 @@ def handle_func_action(self, node, action): "outputs": get_outputs(node), "implicit": get_dependencies(node, skip_sources=True), } + if name == 'ninja_builder': + return None + handler = self.func_handlers.get(name, None) if handler is not None: return handler(node.env if node.env else self.env, node) @@ -377,8 +380,9 @@ def handle_list_action(self, node, action): class NinjaState: """Maintains state of Ninja build system as it's translated from SCons.""" - def __init__(self, env, writer_class): + def __init__(self, env, ninja_file, writer_class): self.env = env + self.ninja_file = ninja_file self.writer_class = writer_class self.__generated = False self.translator = SConsToNinjaTranslator(env) @@ -573,7 +577,7 @@ def has_generated_sources(self, output): return False # pylint: disable=too-many-branches,too-many-locals - def generate(self, ninja_file): + def generate(self): """ Generate the build.ninja. @@ -730,7 +734,7 @@ def generate(self, ninja_file): # jstests/SConscript and being specific to the MongoDB # repository layout. ninja.build( - self.env.File(ninja_file).path, + self.ninja_file.path, rule="REGENERATE", implicit=[ self.env.File("#SConstruct").path, @@ -746,10 +750,10 @@ def generate(self, ninja_file): "compile_commands.json", rule="CMD", pool="console", - implicit=[ninja_file], + implicit=[str(self.ninja_file)], variables={ "cmd": "ninja -f {} -t compdb CC CXX > compile_commands.json".format( - ninja_file + str(self.ninja_file) ) }, ) @@ -772,7 +776,7 @@ def generate(self, ninja_file): if scons_default_targets: ninja.default(" ".join(scons_default_targets)) - with open(ninja_file, "w") as build_ninja: + with open(str(self.ninja_file), "w") as build_ninja: build_ninja.write(content.getvalue()) self.__generated = True @@ -1039,7 +1043,7 @@ def ninja_builder(env, target, source): print("Generating:", str(target[0])) generated_build_ninja = target[0].get_abspath() - NINJA_STATE.generate(generated_build_ninja) + NINJA_STATE.generate() if env.get("DISABLE_AUTO_NINJA") != True: print("Executing:", str(target[0])) @@ -1240,6 +1244,7 @@ def generate(env): help='Disable ninja automatically building after scons') env["DISABLE_AUTO_NINJA"] = GetOption('disable_auto_ninja') + global NINJA_STATE env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") # Add the Ninja builder. @@ -1252,10 +1257,18 @@ def generate(env): env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") - ninja_file = env.Ninja(target=ninja_file_name, source=[]) - env.AlwaysBuild(ninja_file) - env.Alias("$NINJA_ALIAS_NAME", ninja_file) - + # here we allow multiple environments to construct rules and builds + # into the same ninja file + if NINJA_STATE is None: + ninja_file = env.Ninja(target=ninja_file_name, source=[]) + env.AlwaysBuild(ninja_file) + env.Alias("$NINJA_ALIAS_NAME", ninja_file) + else: + if str(NINJA_STATE.ninja_file) != ninja_file_name: + raise Exception("Generating multiple ninja files not supported.") + else: + ninja_file = [NINJA_STATE.ninja_file] + # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. if env["PLATFORM"] == "win32": @@ -1317,7 +1330,7 @@ def generate(env): def robust_rule_mapping(var, rule, tool): provider = gen_get_response_file_command(env, rule, tool) env.NinjaRuleMapping("${" + var + "}", provider) - env.NinjaRuleMapping(env[var], provider) + env.NinjaRuleMapping(env.get(var, None), provider) robust_rule_mapping("CCCOM", "CC", "$CC") robust_rule_mapping("SHCCCOM", "CC", "$CC") @@ -1447,8 +1460,8 @@ def robust_rule_mapping(var, rule, tool): else: ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') - global NINJA_STATE - NINJA_STATE = NinjaState(env, ninja_syntax.Writer) + if NINJA_STATE is None: + NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) # Here we will force every builder to use an emitter which makes the ninja # file depend on it's target. This forces the ninja file to the bottom of From bdf6f50c774db5e16a9fe5c0f054adc16a55468d Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 7 May 2020 00:25:15 -0400 Subject: [PATCH 081/163] added more test, including ninja speed test --- test/ninja/build_libraries.py | 92 +++++++++++ test/ninja/generate_source.py | 102 ++++++++++++ test/ninja/iterative_speedup.py | 226 +++++++++++++++++++++++++++ test/ninja/multi_env.py | 89 +++++++++++ test/ninja/ninja-fixture/bar.c | 2 +- test/ninja/ninja-fixture/test1.c | 13 +- test/ninja/ninja-fixture/test2.C | 3 - test/ninja/ninja-fixture/test_impl.c | 8 + 8 files changed, 528 insertions(+), 7 deletions(-) create mode 100644 test/ninja/build_libraries.py create mode 100644 test/ninja/generate_source.py create mode 100644 test/ninja/iterative_speedup.py create mode 100644 test/ninja/multi_env.py delete mode 100644 test/ninja/ninja-fixture/test2.C create mode 100644 test/ninja/ninja-fixture/test_impl.c diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py new file mode 100644 index 0000000000..662c584fe8 --- /dev/null +++ b/test/ninja/build_libraries.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import sys +import TestSCons + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') + +shared_lib = env.SharedLibrary(target = 'test_impl', source = 'test_impl.c') +env.Program(target = 'test', source = 'test1.c', LIBS=[shared_lib], LIBPATH=['.'], RPATH='.') + +static_lib = env.StaticLibrary(target = 'test_impl_static', source = 'test_impl.c') +static_obj = env.Object(target = 'test_static.o', source = 'test1.c') +env.Program(target = 'test_static', source = static_obj, LIBS=[static_lib], LIBPATH=['.']) +""") +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('test'), stdout="library_function" + os.linesep) +test.run(program = test.workpath('test_static'), stdout="library_function" + os.linesep) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed test_impl.os', + 'Removed libtest_impl.so', + 'Removed test1.o', + 'Removed test', + 'Removed test_impl.o', + 'Removed libtest_impl_static.a', + 'Removed test_static.o', + 'Removed test_static', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) + +# run ninja independently +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('test'), stdout="library_function" + os.linesep) +test.run(program = test.workpath('test_static'), stdout="library_function" + os.linesep) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py new file mode 100644 index 0000000000..8ae0a80c39 --- /dev/null +++ b/test/ninja/generate_source.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import sys +import TestSCons + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') +env.Program(target = 'generate_source', source = 'generate_source.c') +env.Command('generated_source.c', ['generate_source'], './generate_source') +env.Program(target = 'generated_source', source = 'generated_source.c') +""") + +test.write('generate_source.c', """ +#include + +int main(int argc, char *argv[]) { + FILE *fp; + + fp = fopen("generated_source.c", "w"); + fprintf(fp, "#include \\n"); + fprintf(fp, "#include \\n"); + fprintf(fp, "\\n"); + fprintf(fp, "int\\n"); + fprintf(fp, "main(int argc, char *argv[])\\n"); + fprintf(fp, "{\\n"); + fprintf(fp, " printf(\\"generated_source.c\\\\n\\");\\n"); + fprintf(fp, " exit (0);\\n"); + fprintf(fp, "}\\n"); + fclose(fp); +} +""") + +# generate simple build +test.run(stdout=None) +test.run(program = test.workpath('generated_source'), stdout="generated_source.c" + os.linesep) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed generate_source.o', + 'Removed generate_source', + 'Removed generated_source.c', + 'Removed generated_source.o', + 'Removed generated_source', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) + +# run ninja independently +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('generated_source'), stdout="generated_source.c" + os.linesep) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py new file mode 100644 index 0000000000..018ba7beda --- /dev/null +++ b/test/ninja/iterative_speedup.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import sys +import time +import random +import TestSCons + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + +test.write('source_0.c', """ +#include +#include +#include "source_0.h" + +int +print_function0() +{ + printf("main print\\n"); +} +""") + +test.write('source_0.h', """ +#include +#include + +int +print_function0(); +""") + +def get_num_cpus(): + """ + Function to get the number of CPUs the system has. + """ + # Linux, Unix and MacOS: + if hasattr(os, "sysconf"): + if 'SC_NPROCESSORS_ONLN' in os.sysconf_names: + # Linux & Unix: + ncpus = os.sysconf("SC_NPROCESSORS_ONLN") + if isinstance(ncpus, int) and ncpus > 0: + return ncpus + # OSX: + return int(os.popen("sysctl -n hw.ncpu")[1].read()) + # Windows: + if 'NUMBER_OF_PROCESSORS' in os.environ: + ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]) + if ncpus > 0: + return ncpus + # Default + return 1 + +def generate_source(parent_source, current_source): + test.write('source_{}.c'.format(current_source), """ + #include + #include + #include "source_%(parent_source)s.h" + #include "source_%(current_source)s.h" + + int + print_function%(current_source)s() + { + print_function%(parent_source)s(); + } + """ % locals()) + + test.write('source_{}.h'.format(current_source), """ + #include + #include + + int + print_function%(current_source)s(); + """ % locals()) + +def mod_source_return(test_num): + parent_source = test_num-1 + test.write('source_{}.c'.format(test_num), """ + #include + #include + #include "source_%(parent_source)s.h" + #include "source_%(test_num)s.h" + + int + print_function%(test_num)s() + { + int test = 5 + 5; + print_function%(parent_source)s(); + return test; + } + """ % locals()) + +def mod_source_orig(test_num): + parent_source = test_num-1 + test.write('source_{}.c'.format(test_num), """ + #include + #include + #include "source_%(parent_source)s.h" + #include "source_%(test_num)s.h" + + int + print_function%(test_num)s() + { + print_function%(parent_source)s(); + } + """ % locals()) + +num_source = 200 +for i in range(1, num_source+1): + generate_source(i-1, i) + +test.write('main.c', """ +#include +#include +#include "source_%(num_source)s.h" +int +main() +{ + print_function%(num_source)s(); +} +""" % locals()) + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') +sources = ['main.c'] + env.Glob('source*.c') +env.Program(target = 'print_bin', source = sources) +""") + +test.write('SConstruct_no_ninja', """ +env = Environment() +sources = ['main.c'] + env.Glob('source*.c') +env.Program(target = 'print_bin', source = sources) +""") + +tests_mods = [] +ninja_times = [] +scons_times = [] +for _ in range(10): + tests_mods += [random.randrange(1, num_source, 1)] +jobs = '-j' + str(get_num_cpus()) + +start = time.perf_counter() +test.run(arguments='--disable-auto-ninja', stdout=None) +test.run(program = ninja, arguments=[jobs], stdout=None) +stop = time.perf_counter() +ninja_times += [stop - start] +test.run(program = test.workpath('print_bin'), stdout="main print" + os.linesep) + +for test_mod in tests_mods: + mod_source_return(test_mod) + start = time.perf_counter() + test.run(program = ninja, arguments=[jobs], stdout=None) + stop = time.perf_counter() + ninja_times += [stop - start] + +for test_mod in tests_mods: + mod_source_orig(test_mod) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed build.ninja']) + +start = time.perf_counter() +test.run(arguments = ["-f", "SConstruct_no_ninja", jobs], stdout=None) +stop = time.perf_counter() +scons_times += [stop - start] +test.run(program = test.workpath('print_bin'), stdout="main print" + os.linesep) + +for test_mod in tests_mods: + mod_source_return(test_mod) + start = time.perf_counter() + test.run(arguments = ["-f", "SConstruct_no_ninja", jobs], stdout=None) + stop = time.perf_counter() + scons_times += [stop - start] + +full_build_print = True +for ninja, scons in zip(ninja_times, scons_times): + if ninja > scons: + test.fail_test() + if full_build_print: + full_build_print = False + print("Clean build {} files - SCons: {:.3f}s Ninja: {:.3f}s".format(num_source, scons, ninja)) + else: + print("Single File Rebuild - SCons: {:.3f}s Ninja: {:.3f}s".format(scons, ninja)) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py new file mode 100644 index 0000000000..3612d7b84a --- /dev/null +++ b/test/ninja/multi_env.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import sys +import TestSCons + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') +env.Program(target = 'foo', source = 'foo.c') + +env2 = Environment() +env2.Tool('ninja') +env2.Program(target = 'bar', source = 'bar.c') +""") + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +test.run(program = test.workpath('bar'), stdout="bar.c" + os.linesep) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed foo.o', + 'Removed foo', + 'Removed bar.o', + 'Removed bar', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) + +# run ninja independently +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +test.run(program = test.workpath('bar'), stdout="bar.c" + os.linesep) + + + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/ninja-fixture/bar.c b/test/ninja/ninja-fixture/bar.c index de1e6e50b6..3767857b00 100644 --- a/test/ninja/ninja-fixture/bar.c +++ b/test/ninja/ninja-fixture/bar.c @@ -5,6 +5,6 @@ int main(int argc, char *argv[]) { argv[argc++] = "--"; - printf("foo.c\n"); + printf("bar.c\n"); exit (0); } diff --git a/test/ninja/ninja-fixture/test1.c b/test/ninja/ninja-fixture/test1.c index 7535b0aa56..c53f54ac85 100644 --- a/test/ninja/ninja-fixture/test1.c +++ b/test/ninja/ninja-fixture/test1.c @@ -1,3 +1,10 @@ -This is a .c file. -/*cc*/ -/*link*/ +#include +#include + +extern int library_function(void); + +int +main(int argc, char *argv[]) +{ + library_function(); +} diff --git a/test/ninja/ninja-fixture/test2.C b/test/ninja/ninja-fixture/test2.C deleted file mode 100644 index a1ee9e32b9..0000000000 --- a/test/ninja/ninja-fixture/test2.C +++ /dev/null @@ -1,3 +0,0 @@ -This is a .C file. -/*cc*/ -/*link*/ diff --git a/test/ninja/ninja-fixture/test_impl.c b/test/ninja/ninja-fixture/test_impl.c new file mode 100644 index 0000000000..ae5effc965 --- /dev/null +++ b/test/ninja/ninja-fixture/test_impl.c @@ -0,0 +1,8 @@ +#include +#include + +int +library_function(void) +{ + printf("library_function\n"); +} From dd17dd4f8281057ae7dda5ad3d130389229c00b2 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 7 May 2020 00:31:28 -0400 Subject: [PATCH 082/163] fixed sider issues --- src/engine/SCons/Tool/ninja.py | 4 ++-- test/ninja/build_libraries.py | 1 - test/ninja/copy_function_command.py | 1 - test/ninja/generate_and_build.py | 1 - test/ninja/generate_source.py | 1 - test/ninja/iterative_speedup.py | 1 - test/ninja/multi_env.py | 1 - test/ninja/shell_command.py | 1 - 8 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 63ba243c61..bf399e2592 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -665,7 +665,7 @@ def generate(self): # files. Here we make sure that Ninja only ever sees one # target when using a depfile. It will still have a command # that will create all of the outputs but most targets don't - # depend direclty on DWO files and so this assumption is safe + # depend directly on DWO files and so this assumption is safe # to make. rule = self.rules.get(build["rule"]) @@ -1044,7 +1044,7 @@ def ninja_builder(env, target, source): generated_build_ninja = target[0].get_abspath() NINJA_STATE.generate() - if env.get("DISABLE_AUTO_NINJA") != True: + if not env.get("DISABLE_AUTO_NINJA"): print("Executing:", str(target[0])) def execute_ninja(): diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 662c584fe8..7e0ec2365a 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index f86f717e6f..a1e72b7181 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index fd111a29be..82aab5e53b 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index 8ae0a80c39..d1bfe34a6b 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index 018ba7beda..cf999d8568 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import time import random import TestSCons diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 3612d7b84a..5360fd215c 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index fd0e35f8de..b5c8323f43 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ From d2a46f8583f9376ecb8e77e31e4d04a8c391a1c9 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 8 May 2020 12:44:27 -0500 Subject: [PATCH 083/163] update tests to work on windows, added some environment support for windows and msvc --- src/engine/SCons/Tool/ninja.py | 47 +++++++++++++++++++++------- test/ninja/build_libraries.py | 41 +++++++++++++----------- test/ninja/copy_function_command.py | 10 +++--- test/ninja/generate_and_build.py | 9 ++++-- test/ninja/generate_source.py | 22 +++++++------ test/ninja/iterative_speedup.py | 19 ++++++----- test/ninja/multi_env.py | 18 +++++------ test/ninja/ninja-fixture/bar.c | 2 +- test/ninja/ninja-fixture/foo.c | 2 +- test/ninja/ninja-fixture/test1.c | 13 +++++++- test/ninja/ninja-fixture/test_impl.c | 15 +++++++-- test/ninja/shell_command.py | 20 ++++++------ 12 files changed, 142 insertions(+), 76 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index bf399e2592..10e27f3423 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -77,8 +77,8 @@ def _mkdir_action_function(env, node): # to an invalid ninja file. "variables": { # On Windows mkdir "-p" is always on - "cmd": "{mkdir} $out".format( - mkdir="mkdir" if env["PLATFORM"] == "win32" else "mkdir -p", + "cmd": "{mkdir}".format( + mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", ), }, } @@ -383,6 +383,7 @@ class NinjaState: def __init__(self, env, ninja_file, writer_class): self.env = env self.ninja_file = ninja_file + self.ninja_bin_path = '' self.writer_class = writer_class self.__generated = False self.translator = SConsToNinjaTranslator(env) @@ -752,7 +753,7 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "ninja -f {} -t compdb CC CXX > compile_commands.json".format( + "cmd": "{}/ninja -f {} -t compdb CC CXX > compile_commands.json".format(self.ninja_bin_path, str(self.ninja_file) ) }, @@ -1044,15 +1045,32 @@ def ninja_builder(env, target, source): generated_build_ninja = target[0].get_abspath() NINJA_STATE.generate() + + if env["PLATFORM"] == "win32": + # this is not great, it executes everytime + # and its doesn't consider specific node environments + # also a bit quirky to use, but usually MSVC is not + # setup system wide for command line use so this is needed + # on the standard MSVC setup, this is only needed if + # running ninja directly from a command line that hasn't + # had the environment setup (vcvarsall.bat) + # todo: hook this into a command so that it only regnerates + # the .bat if the env['ENV'] changes + with open('ninja_env.bat', 'w') as f: + for key in env['ENV']: + f.write('set {}={}\n'.format(key, env['ENV'][key])) + if not env.get("DISABLE_AUTO_NINJA"): print("Executing:", str(target[0])) def execute_ninja(): - proc = subprocess.Popen( ['ninja', '-f', generated_build_ninja], - stderr=subprocess.STDOUT, + env.AppendENVPath('PATH', NINJA_STATE.ninja_bin_path) + proc = subprocess.Popen(['ninja', '-f', generated_build_ninja], + stderr=sys.stderr, stdout=subprocess.PIPE, - universal_newlines=True + universal_newlines=True, + env=env['ENV'] ) for stdout_line in iter(proc.stdout.readline, ""): yield stdout_line @@ -1060,12 +1078,17 @@ def execute_ninja(): return_code = proc.wait() if return_code: raise subprocess.CalledProcessError(return_code, 'ninja') - + erase_previous = False for output in execute_ninja(): output = output.strip() - sys.stdout.write('\x1b[2K') # erase previous line - sys.stdout.write(output + "\r") + if erase_previous: + sys.stdout.write('\x1b[2K') # erase previous line + sys.stdout.write("\r") + else: + sys.stdout.write(os.linesep) + sys.stdout.write(output) sys.stdout.flush() + erase_previous = output.startswith('[') # pylint: disable=too-few-public-methods class AlwaysExecAction(SCons.Action.FunctionAction): @@ -1311,7 +1334,7 @@ def generate(env): if env["PLATFORM"] == "win32": from SCons.Tool.mslink import compositeLinkAction - if env["LINKCOM"] == compositeLinkAction: + if env.get("LINKCOM", None) == compositeLinkAction: env[ "LINKCOM" ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' @@ -1459,10 +1482,10 @@ def robust_rule_mapping(var, rule, tool): ninja_syntax = importlib.import_module(ninja_syntax_mod_name) else: ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') - + if NINJA_STATE is None: NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) - + NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join(ninja_syntax.__file__, os.pardir, 'data', 'bin')) # Here we will force every builder to use an emitter which makes the ninja # file depend on it's target. This forces the ninja file to the bottom of # the DAG which is required so that we walk every target, and therefore add diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 7e0ec2365a..5e491cc90b 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -39,35 +40,38 @@ if not ninja: test.skip_test("Could not find ninja in environment") +lib_suffix = '.lib' if IS_WINDOWS else '.so' +staticlib_suffix = '.lib' if IS_WINDOWS else '.a' +lib_prefix = '' if IS_WINDOWS else 'lib' + +win32 = ", 'WIN32'" if IS_WINDOWS else '' + test.write('SConstruct', """ env = Environment() env.Tool('ninja') -shared_lib = env.SharedLibrary(target = 'test_impl', source = 'test_impl.c') -env.Program(target = 'test', source = 'test1.c', LIBS=[shared_lib], LIBPATH=['.'], RPATH='.') +shared_lib = env.SharedLibrary(target = 'test_impl', source = 'test_impl.c', CPPDEFINES=['LIBRARY_BUILD'%(win32)s]) +env.Program(target = 'test', source = 'test1.c', LIBS=['test_impl'], LIBPATH=['.'], RPATH='.') -static_lib = env.StaticLibrary(target = 'test_impl_static', source = 'test_impl.c') -static_obj = env.Object(target = 'test_static.o', source = 'test1.c') +static_obj = env.Object(target = 'test_impl_static', source = 'test_impl.c') +static_lib = env.StaticLibrary(target = 'test_impl_static', source = static_obj) +static_obj = env.Object(target = 'test_static', source = 'test1.c') env.Program(target = 'test_static', source = static_obj, LIBS=[static_lib], LIBPATH=['.']) -""") +""" % locals()) # generate simple build test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('test'), stdout="library_function" + os.linesep) -test.run(program = test.workpath('test_static'), stdout="library_function" + os.linesep) +test.run(program = test.workpath('test'), stdout="library_function") +test.run(program = test.workpath('test_static'), stdout="library_function") # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ - 'Removed test_impl.os', - 'Removed libtest_impl.so', - 'Removed test1.o', - 'Removed test', - 'Removed test_impl.o', - 'Removed libtest_impl_static.a', - 'Removed test_static.o', - 'Removed test_static', + ('Removed %stest_impl' % lib_prefix) + lib_suffix, + 'Removed test' + _exe, + ('Removed %stest_impl_static' % lib_prefix) + staticlib_suffix, + 'Removed test_static' + _exe, 'Removed build.ninja']) # only generate the ninja file @@ -78,9 +82,10 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('test'), stdout="library_function" + os.linesep) -test.run(program = test.workpath('test_static'), stdout="library_function" + os.linesep) +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('test'), stdout="library_function") +test.run(program = test.workpath('test_static'), stdout="library_function") test.pass_test() diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index a1e72b7181..06991d375c 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -50,14 +51,14 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +test.run(program = test.workpath('foo'), stdout="foo.c") # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo2.o', 'Removed foo2.c', - 'Removed foo', + 'Removed foo' + _exe, 'Removed build.ninja']) # only generate the ninja file @@ -68,8 +69,9 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('foo'), stdout="foo.c") test.pass_test() diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index 82aab5e53b..904e46a3cd 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -45,11 +46,12 @@ env.Program(target = 'foo', source = 'foo.c') """) + # generate simple build test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +test.run(program = test.workpath('foo' + _exe), stdout="foo.c") # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -66,8 +68,9 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('foo' + _exe), stdout="foo.c") test.pass_test() diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index d1bfe34a6b..298d227fea 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -39,13 +40,15 @@ if not ninja: test.skip_test("Could not find ninja in environment") +shell = '' if IS_WINDOWS else './' + test.write('SConstruct', """ env = Environment() env.Tool('ninja') -env.Program(target = 'generate_source', source = 'generate_source.c') -env.Command('generated_source.c', ['generate_source'], './generate_source') +prog = env.Program(target = 'generate_source', source = 'generate_source.c') +env.Command('generated_source.c', prog, '%(shell)sgenerate_source%(_exe)s') env.Program(target = 'generated_source', source = 'generated_source.c') -""") +""" % locals()) test.write('generate_source.c', """ #include @@ -60,7 +63,7 @@ fprintf(fp, "int\\n"); fprintf(fp, "main(int argc, char *argv[])\\n"); fprintf(fp, "{\\n"); - fprintf(fp, " printf(\\"generated_source.c\\\\n\\");\\n"); + fprintf(fp, " printf(\\"generated_source.c\\");\\n"); fprintf(fp, " exit (0);\\n"); fprintf(fp, "}\\n"); fclose(fp); @@ -69,16 +72,16 @@ # generate simple build test.run(stdout=None) -test.run(program = test.workpath('generated_source'), stdout="generated_source.c" + os.linesep) +test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed generate_source.o', - 'Removed generate_source', + 'Removed generate_source' + _exe, 'Removed generated_source.c', 'Removed generated_source.o', - 'Removed generated_source', + 'Removed generated_source' + _exe, 'Removed build.ninja']) # only generate the ninja file @@ -89,8 +92,9 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('generated_source'), stdout="generated_source.c" + os.linesep) +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") test.pass_test() diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index cf999d8568..6675bf3350 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -28,6 +28,7 @@ import time import random import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -49,7 +50,8 @@ int print_function0() { - printf("main print\\n"); + printf("main print"); + return 0; } """) @@ -92,7 +94,7 @@ def generate_source(parent_source, current_source): int print_function%(current_source)s() { - print_function%(parent_source)s(); + return print_function%(parent_source)s(); } """ % locals()) @@ -132,7 +134,7 @@ def mod_source_orig(test_num): int print_function%(test_num)s() { - print_function%(parent_source)s(); + return print_function%(parent_source)s(); } """ % locals()) @@ -148,6 +150,7 @@ def mod_source_orig(test_num): main() { print_function%(num_source)s(); + exit(0); } """ % locals()) @@ -171,17 +174,19 @@ def mod_source_orig(test_num): tests_mods += [random.randrange(1, num_source, 1)] jobs = '-j' + str(get_num_cpus()) +ninja_program = [test.workpath('ninja_env.bat'), '&', ninja, jobs] if IS_WINDOWS else [ninja, jobs] + start = time.perf_counter() test.run(arguments='--disable-auto-ninja', stdout=None) -test.run(program = ninja, arguments=[jobs], stdout=None) +test.run(program = ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] -test.run(program = test.workpath('print_bin'), stdout="main print" + os.linesep) +test.run(program = test.workpath('print_bin'), stdout="main print") for test_mod in tests_mods: mod_source_return(test_mod) start = time.perf_counter() - test.run(program = ninja, arguments=[jobs], stdout=None) + test.run(program = ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] @@ -197,7 +202,7 @@ def mod_source_orig(test_num): test.run(arguments = ["-f", "SConstruct_no_ninja", jobs], stdout=None) stop = time.perf_counter() scons_times += [stop - start] -test.run(program = test.workpath('print_bin'), stdout="main print" + os.linesep) +test.run(program = test.workpath('print_bin'), stdout="main print") for test_mod in tests_mods: mod_source_return(test_mod) diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 5360fd215c..087d392b52 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -53,16 +54,16 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) -test.run(program = test.workpath('bar'), stdout="bar.c" + os.linesep) +test.run(program = test.workpath('foo' + _exe), stdout="foo.c") +test.run(program = test.workpath('bar' + _exe), stdout="bar.c") # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo.o', - 'Removed foo', + 'Removed foo' + _exe, 'Removed bar.o', - 'Removed bar', + 'Removed bar' + _exe, 'Removed build.ninja']) # only generate the ninja file @@ -73,11 +74,10 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) -test.run(program = test.workpath('bar'), stdout="bar.c" + os.linesep) - - +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('foo' + _exe), stdout="foo.c") +test.run(program = test.workpath('bar' + _exe), stdout="bar.c") test.pass_test() diff --git a/test/ninja/ninja-fixture/bar.c b/test/ninja/ninja-fixture/bar.c index 3767857b00..15b2ecc46a 100644 --- a/test/ninja/ninja-fixture/bar.c +++ b/test/ninja/ninja-fixture/bar.c @@ -5,6 +5,6 @@ int main(int argc, char *argv[]) { argv[argc++] = "--"; - printf("bar.c\n"); + printf("bar.c"); exit (0); } diff --git a/test/ninja/ninja-fixture/foo.c b/test/ninja/ninja-fixture/foo.c index de1e6e50b6..ba35c687a2 100644 --- a/test/ninja/ninja-fixture/foo.c +++ b/test/ninja/ninja-fixture/foo.c @@ -5,6 +5,6 @@ int main(int argc, char *argv[]) { argv[argc++] = "--"; - printf("foo.c\n"); + printf("foo.c"); exit (0); } diff --git a/test/ninja/ninja-fixture/test1.c b/test/ninja/ninja-fixture/test1.c index c53f54ac85..678461f508 100644 --- a/test/ninja/ninja-fixture/test1.c +++ b/test/ninja/ninja-fixture/test1.c @@ -1,10 +1,21 @@ #include #include -extern int library_function(void); +#ifdef WIN32 +#ifdef LIBRARY_BUILD +#define DLLEXPORT __declspec(dllexport) +#else +#define DLLEXPORT __declspec(dllimport) +#endif +#else +#define DLLEXPORT +#endif + +DLLEXPORT extern int library_function(void); int main(int argc, char *argv[]) { library_function(); + exit(0); } diff --git a/test/ninja/ninja-fixture/test_impl.c b/test/ninja/ninja-fixture/test_impl.c index ae5effc965..89c26ede6f 100644 --- a/test/ninja/ninja-fixture/test_impl.c +++ b/test/ninja/ninja-fixture/test_impl.c @@ -1,8 +1,19 @@ #include #include -int +#ifdef WIN32 +#ifdef LIBRARY_BUILD +#define DLLEXPORT __declspec(dllexport) +#else +#define DLLEXPORT __declspec(dllimport) +#endif +#else +#define DLLEXPORT +#endif + + +DLLEXPORT int library_function(void) { - printf("library_function\n"); + printf("library_function"); } diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index b5c8323f43..5d491645c6 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -39,24 +40,26 @@ if not ninja: test.skip_test("Could not find ninja in environment") +shell = '' if IS_WINDOWS else './' + test.write('SConstruct', """ env = Environment() env.Tool('ninja') -env.Program(target = 'foo', source = 'foo.c') -env.Command('foo.out', ['foo'], './foo > foo.out') -""") +prog = env.Program(target = 'foo', source = 'foo.c') +env.Command('foo.out', prog, '%(shell)sfoo%(_exe)s > foo.out') +""" % locals()) # generate simple build test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.must_match('foo.out', 'foo.c' + os.linesep) +test.must_match('foo.out', 'foo.c') # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo.o', - 'Removed foo', + 'Removed foo%(_exe)s' % locals(), 'Removed foo.out', 'Removed build.ninja']) @@ -68,10 +71,9 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.must_match('foo.out', 'foo.c' + os.linesep) - - +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.must_match('foo.out', 'foo.c') test.pass_test() From 8450dc18977c5da3e7294265b06357c44faffad2 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 14 May 2020 02:52:16 -0400 Subject: [PATCH 084/163] used different method for pushing ninja file to bottom of DAG, use import ninja to get ninja_syntax and ninja bin, and added some more basic testing. --- src/engine/SCons/Tool/ninja.py | 196 +++++++++++++-------------- test/ninja/build_libraries.py | 34 +++-- test/ninja/copy_function_command.py | 33 +++-- test/ninja/generate_and_build.py | 33 +++-- test/ninja/generate_and_build_cxx.py | 106 +++++++++++++++ test/ninja/generate_source.py | 31 +++-- test/ninja/iterative_speedup.py | 22 ++- test/ninja/multi_env.py | 35 +++-- test/ninja/ninja-fixture/test2.cpp | 16 +++ test/ninja/ninja-fixture/test2.hpp | 9 ++ test/ninja/shell_command.py | 33 +++-- 11 files changed, 364 insertions(+), 184 deletions(-) create mode 100644 test/ninja/generate_and_build_cxx.py create mode 100644 test/ninja/ninja-fixture/test2.cpp create mode 100644 test/ninja/ninja-fixture/test2.hpp diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 10e27f3423..bc505b8937 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -88,13 +88,7 @@ def _copy_action_function(env, node): "outputs": get_outputs(node), "inputs": [get_path(src_file(s)) for s in node.sources], "rule": "CMD", - # implicit explicitly omitted, we translate these so they can be - # used by anything that depends on these but commonly this is - # hit with a node that will depend on all of the fake - # srcnode's that SCons will never give us a rule for leading - # to an invalid ninja file. "variables": { - # On Windows mkdir "-p" is always on "cmd": "$COPY $in $out", }, } @@ -753,7 +747,7 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "{}/ninja -f {} -t compdb CC CXX > compile_commands.json".format(self.ninja_bin_path, + "cmd": "{} -f {} -t compdb CC CXX > compile_commands.json".format(self.ninja_bin_path, str(self.ninja_file) ) }, @@ -1047,30 +1041,30 @@ def ninja_builder(env, target, source): NINJA_STATE.generate() if env["PLATFORM"] == "win32": - # this is not great, it executes everytime - # and its doesn't consider specific node environments - # also a bit quirky to use, but usually MSVC is not - # setup system wide for command line use so this is needed - # on the standard MSVC setup, this is only needed if + # this is not great, its doesn't consider specific + # node environments, which means on linux the build could + # behave differently, becuase on linux you can set the environment + # per command in the ninja file. This is only needed if # running ninja directly from a command line that hasn't # had the environment setup (vcvarsall.bat) - # todo: hook this into a command so that it only regnerates - # the .bat if the env['ENV'] changes - with open('ninja_env.bat', 'w') as f: + with open('run_ninja_env.bat', 'w') as f: for key in env['ENV']: f.write('set {}={}\n'.format(key, env['ENV'][key])) + f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) if not env.get("DISABLE_AUTO_NINJA"): - print("Executing:", str(target[0])) + cmd = [NINJA_STATE.ninja_bin_path, '-f', generated_build_ninja] + print("Executing:", str(' '.join(cmd))) + # execute the ninja build at the end of SCons, trying to + # reproduce the output like a ninja build would def execute_ninja(): - env.AppendENVPath('PATH', NINJA_STATE.ninja_bin_path) - proc = subprocess.Popen(['ninja', '-f', generated_build_ninja], + proc = subprocess.Popen(cmd, stderr=sys.stderr, stdout=subprocess.PIPE, universal_newlines=True, - env=env['ENV'] + env=env['ENV'] # ninja build items won't consider node env on win32 ) for stdout_line in iter(proc.stdout.readline, ""): yield stdout_line @@ -1078,6 +1072,7 @@ def execute_ninja(): return_code = proc.wait() if return_code: raise subprocess.CalledProcessError(return_code, 'ninja') + erase_previous = False for output in execute_ninja(): output = output.strip() @@ -1088,6 +1083,9 @@ def execute_ninja(): sys.stdout.write(os.linesep) sys.stdout.write(output) sys.stdout.flush() + # this will only erase ninjas [#/#] lines + # leaving warnings and other output, seems a bit + # prone to failure with such a simple check erase_previous = output.startswith('[') # pylint: disable=too-few-public-methods @@ -1226,6 +1224,20 @@ def ninja_always_serial(self, num, taskmaster): self.num_jobs = num self.job = SCons.Job.Serial(taskmaster) +def ninja_hack_linkcom(env): + # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks + # without relying on the SCons hacks that SCons uses by default. + if env["PLATFORM"] == "win32": + from SCons.Tool.mslink import compositeLinkAction + + if env.get("LINKCOM", None) == compositeLinkAction: + env[ + "LINKCOM" + ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' + env[ + "SHLINKCOM" + ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' + class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" @@ -1259,13 +1271,31 @@ def generate(env): global added if not added: added = 1 - AddOption('--disable-auto-ninja', - dest='disable_auto_ninja', - metavar='BOOL', - action="store_true", - default=False, - help='Disable ninja automatically building after scons') - env["DISABLE_AUTO_NINJA"] = GetOption('disable_auto_ninja') + + AddOption('--disable-execute-ninja', + dest='disable_execute_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + + AddOption('--disable-ninja', + dest='disable_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + + if GetOption('disable_ninja'): + return env + + try: + import ninja + except ImportError: + SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") + return + + env["DISABLE_AUTO_NINJA"] = GetOption('disable_execute_ninja') global NINJA_STATE env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") @@ -1288,16 +1318,15 @@ def generate(env): env.Alias("$NINJA_ALIAS_NAME", ninja_file) else: if str(NINJA_STATE.ninja_file) != ninja_file_name: - raise Exception("Generating multiple ninja files not supported.") - else: - ninja_file = [NINJA_STATE.ninja_file] + SCons.Warnings.Warning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") + ninja_file = [NINJA_STATE.ninja_file] # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. if env["PLATFORM"] == "win32": env.Append(CCFLAGS=["/showIncludes"]) else: - env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) + env.Append(CCFLAGS=["-MD", "-MF", "${TARGET}.d"]) # Provide a way for custom rule authors to easily access command # generation. @@ -1329,18 +1358,8 @@ def generate(env): # might not catch it. env.AddMethod(register_custom_rule_mapping, "NinjaRuleMapping") - # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks - # without relying on the SCons hacks that SCons uses by default. - if env["PLATFORM"] == "win32": - from SCons.Tool.mslink import compositeLinkAction - - if env.get("LINKCOM", None) == compositeLinkAction: - env[ - "LINKCOM" - ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' - env[ - "SHLINKCOM" - ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' + # on windows we need to change the link action + ninja_hack_linkcom(env) # Normally in SCons actions for the Program and *Library builders # will return "${*COM}" as their pre-subst'd command line. However @@ -1386,6 +1405,8 @@ def robust_rule_mapping(var, rule, tool): # Disable running ranlib, since we added 's' above env["RANLIBCOM"] = "" + SCons.Warnings.Warning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") + # This is the point of no return, anything after this comment # makes changes to SCons that are irreversible and incompatible # with a normal SCons build. We return early if __NINJA_NO=1 has @@ -1462,68 +1483,43 @@ def robust_rule_mapping(var, rule, tool): # monkey the Jobs constructor to only use the Serial Job class. SCons.Job.Jobs.__init__ = ninja_always_serial - # The environment variable NINJA_SYNTAX points to the - # ninja_syntax.py module from the ninja sources found here: - # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py - # - # This should be vendored into the build sources and it's location - # set in NINJA_SYNTAX. This code block loads the location from - # that variable, gets the absolute path to the vendored file, gets - # it's parent directory then uses importlib to import the module - # dynamically. - ninja_syntax_file = env[NINJA_SYNTAX] - - if os.path.exists(ninja_syntax_file): - if isinstance(ninja_syntax_file, str): - ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() - ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) - sys.path.append(ninja_syntax_mod_dir) - ninja_syntax_mod_name = os.path.basename(ninja_syntax_file).replace(".py", "") - ninja_syntax = importlib.import_module(ninja_syntax_mod_name) - else: - ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') + ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') if NINJA_STATE is None: NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) - NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join(ninja_syntax.__file__, os.pardir, 'data', 'bin')) - # Here we will force every builder to use an emitter which makes the ninja - # file depend on it's target. This forces the ninja file to the bottom of - # the DAG which is required so that we walk every target, and therefore add - # it to the global NINJA_STATE, before we try to write the ninja file. - def ninja_file_depends_on_all(target, source, env): - if not any("conftest" in str(t) for t in target): - env.Depends(ninja_file, target) - return target, source - - # The "Alias Builder" isn't in the BUILDERS map so we have to - # modify it directly. - SCons.Environment.AliasBuilder.emitter = ninja_file_depends_on_all - - for _, builder in env["BUILDERS"].items(): - try: - emitter = builder.emitter - if emitter is not None: - builder.emitter = SCons.Builder.ListEmitter( - [emitter, ninja_file_depends_on_all] - ) - else: - builder.emitter = ninja_file_depends_on_all - # Users can inject whatever they want into the BUILDERS - # dictionary so if the thing doesn't have an emitter we'll - # just ignore it. - except AttributeError: - pass - - # We will subvert the normal Command to make sure all targets generated - # from commands will be linked to the ninja file - SconsCommand = SCons.Environment.Environment.Command - - def NinjaCommand(self, target, source, action, **kw): - targets = SconsCommand(env, target, source, action, **kw) - env.Depends(ninja_file, targets) + NINJA_STATE.ninja_bin_path = env.get('NINJA_BIN') + if not NINJA_STATE.ninja_bin_path: + # default to using ninja installed with python module + ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' + NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( + ninja_syntax.__file__, + os.pardir, + 'data', + 'bin', + ninja_bin)) + if not os.path.exists(NINJA_STATE.ninja_bin_path): + # couldn't find it, just give the bin name and hope + # its in the path later + NINJA_STATE.ninja_bin_path = ninja_bin + + # TODO: this is hacking into scons, preferable if there were a less intrusive way + # We will subvert the normal builder execute to make sure all the ninja file is dependent + # on all targets generated from any builders + SCons_Builder_BuilderBase__execute = SCons.Builder.BuilderBase._execute + def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): + # this ensures all environments in which a builder executes from will + # not create list actions for linking on windows + ninja_hack_linkcom(env) + targets = SCons_Builder_BuilderBase__execute(self, env, target, source, overwarn=overwarn, executor_kw=executor_kw) + + if not SCons.Util.is_List(target): + target = [target] + + for target in targets: + if str(target) != ninja_file_name and "conftest" not in str(target): + env.Depends(ninja_file, targets) return targets - - SCons.Environment.Environment.Command = NinjaCommand + SCons.Builder.BuilderBase._execute = NinjaBuilderExecute # Here we monkey patch the Task.execute method to not do a bunch of # unnecessary work. If a build is a regular builder (i.e not a conftest and diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 5e491cc90b..347d63902c 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - lib_suffix = '.lib' if IS_WINDOWS else '.so' staticlib_suffix = '.lib' if IS_WINDOWS else '.a' lib_prefix = '' if IS_WINDOWS else 'lib' @@ -60,8 +68,9 @@ """ % locals()) # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('test'), stdout="library_function") test.run(program = test.workpath('test_static'), stdout="library_function") @@ -75,14 +84,13 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('test')) +test.must_not_exist(test.workpath('test_static')) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('test'), stdout="library_function") test.run(program = test.workpath('test_static'), stdout="library_function") diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index 06991d375c..0beb8de11c 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - test.write('SConstruct', """ env = Environment() env.Tool('ninja') @@ -49,8 +57,9 @@ # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('foo'), stdout="foo.c") # clean build and ninja files @@ -62,14 +71,12 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo')) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('foo'), stdout="foo.c") diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index 904e46a3cd..73c71f1b40 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - test.write('SConstruct', """ env = Environment() env.Tool('ninja') @@ -49,8 +57,9 @@ # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('foo' + _exe), stdout="foo.c") # clean build and ninja files @@ -61,14 +70,12 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo' + _exe)) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('foo' + _exe), stdout="foo.c") diff --git a/test/ninja/generate_and_build_cxx.py b/test/ninja/generate_and_build_cxx.py new file mode 100644 index 0000000000..ac0f55444f --- /dev/null +++ b/test/ninja/generate_and_build_cxx.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import importlib +import TestSCons +from TestCmd import IS_WINDOWS + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') +env.Program(target = 'test2', source = 'test2.cpp') +""") + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) +test.run(program = test.workpath('test2' + _exe), stdout="print_function") + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed test2.o', + 'Removed test2', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('test2' + _exe)) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program = program, stdout=None) +test.run(program = test.workpath('test2' + _exe), stdout="print_function") + +test.write('test2.hpp', """ +#include +#include + +class Foo +{ +public: + int print_function(); + int print_function2(){ + std::cout << "2"; + return 0; + }; +}; +""") + +# generate simple build +test.run(program = program, stdout=None) +test.run(program = test.workpath('test2' + _exe), stdout="print_function2") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index 298d227fea..d9b9c4ed59 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - shell = '' if IS_WINDOWS else './' test.write('SConstruct', """ @@ -72,6 +80,9 @@ # generate simple build test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") # clean build and ninja files @@ -85,14 +96,12 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('generated_source' + _exe)) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index 6675bf3350..b03e9cbcb0 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -27,21 +27,29 @@ import os import time import random +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - test.write('source_0.c', """ #include #include @@ -174,10 +182,10 @@ def mod_source_orig(test_num): tests_mods += [random.randrange(1, num_source, 1)] jobs = '-j' + str(get_num_cpus()) -ninja_program = [test.workpath('ninja_env.bat'), '&', ninja, jobs] if IS_WINDOWS else [ninja, jobs] +ninja_program = [test.workpath('run_ninja_env.bat'), jobs] if IS_WINDOWS else [ninja_bin, jobs] start = time.perf_counter() -test.run(arguments='--disable-auto-ninja', stdout=None) +test.run(arguments='--disable-execute-ninja', stdout=None) test.run(program = ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 087d392b52..c9b21b7d70 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -25,35 +25,43 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - test.write('SConstruct', """ env = Environment() env.Tool('ninja') env.Program(target = 'foo', source = 'foo.c') env2 = Environment() -env2.Tool('ninja') env2.Program(target = 'bar', source = 'bar.c') """) # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('foo' + _exe), stdout="foo.c") test.run(program = test.workpath('bar' + _exe), stdout="bar.c") @@ -67,14 +75,13 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo' + _exe)) +test.must_not_exist(test.workpath('bar' + _exe)) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('foo' + _exe), stdout="foo.c") test.run(program = test.workpath('bar' + _exe), stdout="bar.c") diff --git a/test/ninja/ninja-fixture/test2.cpp b/test/ninja/ninja-fixture/test2.cpp new file mode 100644 index 0000000000..69b54c97f7 --- /dev/null +++ b/test/ninja/ninja-fixture/test2.cpp @@ -0,0 +1,16 @@ +#include "test2.hpp" + +int +main(int argc, char *argv[]) +{ + Foo* test = new Foo(); + test->print_function(); + test->print_function2(); + return 0; +} + +int Foo::print_function() +{ + std::cout << "print_function"; + return 0; +} \ No newline at end of file diff --git a/test/ninja/ninja-fixture/test2.hpp b/test/ninja/ninja-fixture/test2.hpp new file mode 100644 index 0000000000..f0583fc547 --- /dev/null +++ b/test/ninja/ninja-fixture/test2.hpp @@ -0,0 +1,9 @@ +#include +#include + +class Foo +{ +public: + int print_function(); + int print_function2(){return 0;}; +}; diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index 5d491645c6..b35a52b6c0 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - shell = '' if IS_WINDOWS else './' test.write('SConstruct', """ @@ -51,8 +59,9 @@ # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.must_match('foo.out', 'foo.c') # clean build and ninja files @@ -64,14 +73,12 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo.out')) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.must_match('foo.out', 'foo.c') From c342e53632090c5bcbd91eeb9b0bcb355884e685 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 14 May 2020 12:28:17 -0400 Subject: [PATCH 085/163] is link should use the base nodes lstat instead of local fs stat builder is not garunteed to be in the environment, so check if the node is the ninja_file fix sider issues --- src/engine/SCons/Tool/ninja.py | 9 +++++---- test/ninja/build_libraries.py | 7 +++---- test/ninja/copy_function_command.py | 7 +++---- test/ninja/generate_and_build.py | 7 +++---- test/ninja/generate_and_build_cxx.py | 7 +++---- test/ninja/generate_source.py | 7 +++---- test/ninja/iterative_speedup.py | 15 +++++++-------- test/ninja/multi_env.py | 5 +++-- test/ninja/shell_command.py | 7 +++---- 9 files changed, 33 insertions(+), 38 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index bc505b8937..a68655ef1d 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -233,9 +233,10 @@ def action_to_ninja_build(self, node, action=None): # Ninja builders out of being sources of ninja builders but I # can't fix every DAG problem so we just skip ninja_builders # if we find one - if node.builder == self.env["BUILDERS"]["Ninja"]: + global NINJA_STATE + if NINJA_STATE.ninja_file == str(node): build = None - elif isinstance(action, SCons.Action.FunctionAction): + if isinstance(action, SCons.Action.FunctionAction): build = self.handle_func_action(node, action) elif isinstance(action, SCons.Action.LazyAction): # pylint: disable=protected-access @@ -1043,7 +1044,7 @@ def ninja_builder(env, target, source): if env["PLATFORM"] == "win32": # this is not great, its doesn't consider specific # node environments, which means on linux the build could - # behave differently, becuase on linux you can set the environment + # behave differently, because on linux you can set the environment # per command in the ninja file. This is only needed if # running ninja directly from a command line that hasn't # had the environment setup (vcvarsall.bat) @@ -1492,7 +1493,7 @@ def robust_rule_mapping(var, rule, tool): # default to using ninja installed with python module ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( - ninja_syntax.__file__, + ninja.__file__, os.pardir, 'data', 'bin', diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 347d63902c..40404152fa 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') lib_suffix = '.lib' if IS_WINDOWS else '.so' diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index 0beb8de11c..8e7acff7b7 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') test.write('SConstruct', """ diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index 73c71f1b40..faf395a1ef 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') test.write('SConstruct', """ diff --git a/test/ninja/generate_and_build_cxx.py b/test/ninja/generate_and_build_cxx.py index ac0f55444f..663282bd92 100644 --- a/test/ninja/generate_and_build_cxx.py +++ b/test/ninja/generate_and_build_cxx.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') test.write('SConstruct', """ diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index d9b9c4ed59..76c79bb7da 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') shell = '' if IS_WINDOWS else './' diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index b03e9cbcb0..ff50f502a3 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -27,10 +27,11 @@ import os import time import random -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -40,14 +41,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') test.write('source_0.c', """ @@ -220,14 +219,14 @@ def mod_source_orig(test_num): scons_times += [stop - start] full_build_print = True -for ninja, scons in zip(ninja_times, scons_times): - if ninja > scons: +for ninja_time, scons_time in zip(ninja_times, scons_times): + if ninja_time > scons_time: test.fail_test() if full_build_print: full_build_print = False - print("Clean build {} files - SCons: {:.3f}s Ninja: {:.3f}s".format(num_source, scons, ninja)) + print("Clean build {} files - SCons: {:.3f}s Ninja: {:.3f}s".format(num_source, scons_time, ninja_time)) else: - print("Single File Rebuild - SCons: {:.3f}s Ninja: {:.3f}s".format(scons, ninja)) + print("Single File Rebuild - SCons: {:.3f}s Ninja: {:.3f}s".format(scons_time, ninja_time)) test.pass_test() diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index c9b21b7d70..18ca3cbc69 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,7 +39,7 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index b35a52b6c0..5d7f97e215 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') shell = '' if IS_WINDOWS else './' From ed07dc9b383393b2d257e05c44acbad633514975 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 21 May 2020 16:23:13 -0400 Subject: [PATCH 086/163] removed NINJA_SYNTAX completely --- src/engine/SCons/Tool/ninja.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index a68655ef1d..5515a35413 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -40,7 +40,6 @@ from SCons.Script import COMMAND_LINE_TARGETS NINJA_STATE = None -NINJA_SYNTAX = "NINJA_SYNTAX" NINJA_RULES = "__NINJA_CUSTOM_RULES" NINJA_POOLS = "__NINJA_CUSTOM_POOLS" NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" @@ -1299,7 +1298,6 @@ def generate(env): env["DISABLE_AUTO_NINJA"] = GetOption('disable_execute_ninja') global NINJA_STATE - env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") # Add the Ninja builder. always_exec_ninja_action = AlwaysExecAction(ninja_builder, {}) From ef4413d6ac14c760b25438c6cb0ab6cbb0b2f548 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 22 May 2020 00:49:03 -0400 Subject: [PATCH 087/163] removed old sconscript changes --- SCons/Script/SConscript.py | 3 --- SCons/Script/__init__.py | 1 - src/engine/SCons/Tool/ninja.py | 2 +- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/SCons/Script/SConscript.py b/SCons/Script/SConscript.py index ded0fcfef9..596fca0463 100644 --- a/SCons/Script/SConscript.py +++ b/SCons/Script/SConscript.py @@ -203,11 +203,9 @@ def _SConscript(fs, *files, **kw): if f.rexists(): actual = f.rfile() _file_ = open(actual.get_abspath(), "rb") - SCons.Script.LOADED_SCONSCRIPTS.append(actual.get_abspath()) elif f.srcnode().rexists(): actual = f.srcnode().rfile() _file_ = open(actual.get_abspath(), "rb") - SCons.Script.LOADED_SCONSCRIPTS.append(actual.get_abspath()) elif f.has_src_builder(): # The SConscript file apparently exists in a source # code management system. Build it, but then clear @@ -216,7 +214,6 @@ def _SConscript(fs, *files, **kw): f.build() f.built() f.builder_set(None) - SCons.Script.LOADED_SCONSCRIPTS.append(f.get_abspath()) if f.exists(): _file_ = open(f.get_abspath(), "rb") if _file_: diff --git a/SCons/Script/__init__.py b/SCons/Script/__init__.py index e409f06ef4..5f58d9972d 100644 --- a/SCons/Script/__init__.py +++ b/SCons/Script/__init__.py @@ -187,7 +187,6 @@ def _clear(self): BUILD_TARGETS = TargetList() COMMAND_LINE_TARGETS = [] DEFAULT_TARGETS = [] -LOADED_SCONSCRIPTS = [] # BUILD_TARGETS can be modified in the SConscript files. If so, we # want to treat the modified BUILD_TARGETS list as if they specified diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 5515a35413..205bfa79a9 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -727,7 +727,7 @@ def generate(self): # allow us to query the actual SConscripts used. Right now # this glob method has deficiencies like skipping # jstests/SConscript and being specific to the MongoDB - # repository layout. + # repository layout. (github issue #3625) ninja.build( self.ninja_file.path, rule="REGENERATE", From e6c20b33d16638103919943fe504777ea1bbedbe Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 3 Jun 2020 10:47:54 -0400 Subject: [PATCH 088/163] merge commit a7541c60e5904e7deafdedf5bb040cc8924ac7d3 from https://github.com/mongodb/mongo --- src/engine/SCons/Tool/ninja.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 205bfa79a9..689f2ee262 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -1463,6 +1463,16 @@ def robust_rule_mapping(var, rule, tool): SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig) + # Ignore CHANGED_SOURCES and CHANGED_TARGETS. We don't want those + # to have effect in a generation pass because the generator + # shouldn't generate differently depending on the current local + # state. Without this, when generating on Windows, if you already + # had a foo.obj, you would omit foo.cpp from the response file. Do the same for UNCHANGED. + SCons.Executor.Executor._get_changed_sources = SCons.Executor.Executor._get_sources + SCons.Executor.Executor._get_changed_targets = SCons.Executor.Executor._get_targets + SCons.Executor.Executor._get_unchanged_sources = SCons.Executor.Executor._get_sources + SCons.Executor.Executor._get_unchanged_targets = SCons.Executor.Executor._get_targets + # Replace false action messages with nothing. env["PRINT_CMD_LINE_FUNC"] = ninja_noop From 05e9accc7c19b1c87eb44752ad5054d2c1277f2c Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 3 Jun 2020 10:49:15 -0400 Subject: [PATCH 089/163] merge commit 18cbf0d581162b2d15d66577b1fe08fe22006699 from https://github.com/mongodb/mongo --- src/engine/SCons/Tool/ninja.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 689f2ee262..3a1034fea2 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -675,7 +675,16 @@ def generate(self): # use for the "real" builder and multiple phony targets that # match the file names of the remaining outputs. This way any # build can depend on any output from any build. - build["outputs"].sort() + # + # We assume that the first listed output is the 'key' + # output and is stably presented to us by SCons. For + # instance if -gsplit-dwarf is in play and we are + # producing foo.o and foo.dwo, we expect that outputs[0] + # from SCons will be the foo.o file and not the dwo + # file. If instead we just sorted the whole outputs array, + # we would find that the dwo file becomes the + # first_output, and this breaks, for instance, header + # dependency scanning. if rule is not None and (rule.get("deps") or rule.get("rspfile")): first_output, remaining_outputs = ( build["outputs"][0], @@ -684,7 +693,7 @@ def generate(self): if remaining_outputs: ninja.build( - outputs=remaining_outputs, rule="phony", implicit=first_output, + outputs=sorted(remaining_outputs), rule="phony", implicit=first_output, ) build["outputs"] = first_output From 4b7ddc9af1a349ac05ff6eb8df66558898a2f5cf Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 8 Jun 2020 21:43:59 -0400 Subject: [PATCH 090/163] update to build godot reinvoke scons for unhandled actions Ignore Python.Values (need fix) escape rsp content check is_sconscript fix sider issues --- src/engine/SCons/Tool/ninja.py | 98 +++++++++++++++++++--------------- 1 file changed, 56 insertions(+), 42 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 3a1034fea2..3fdadf9d77 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -30,7 +30,6 @@ import shlex import subprocess -from glob import glob from os.path import join as joinpath from os.path import splitext @@ -38,6 +37,7 @@ from SCons.Action import _string_from_cmd_list, get_default_ENV from SCons.Util import is_List, flatten_sequence from SCons.Script import COMMAND_LINE_TARGETS +from SCons.Node import SConscriptNodes NINJA_STATE = None NINJA_RULES = "__NINJA_CUSTOM_RULES" @@ -58,10 +58,11 @@ def _install_action_function(_env, node): """Install files using the install or copy commands""" + #TODO: handle Python.Value nodes return { "outputs": get_outputs(node), "rule": "INSTALL", - "inputs": [get_path(src_file(s)) for s in node.sources], + "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], "implicit": get_dependencies(node), } @@ -83,9 +84,10 @@ def _mkdir_action_function(env, node): } def _copy_action_function(env, node): + #TODO: handle Python.Value nodes return { "outputs": get_outputs(node), - "inputs": [get_path(src_file(s)) for s in node.sources], + "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], "rule": "CMD", "variables": { "cmd": "$COPY $in $out", @@ -134,11 +136,12 @@ def is_valid_dependent_node(node): def alias_to_ninja_build(node): """Convert an Alias node into a Ninja phony target""" + # TODO: handle Python.Values return { "outputs": get_outputs(node), "rule": "phony", "implicit": [ - get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) + get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) and not isinstance(n, SCons.Node.Python.Value) ], } @@ -147,18 +150,20 @@ def get_order_only(node): """Return a list of order only dependencies for node.""" if node.prerequisites is None: return [] - return [get_path(src_file(prereq)) for prereq in node.prerequisites] + #TODO: handle Python.Value nodes + return [get_path(src_file(prereq)) for prereq in node.prerequisites if not isinstance(prereq, SCons.Node.Python.Value)] def get_dependencies(node, skip_sources=False): """Return a list of dependencies for node.""" + #TODO: handle Python.Value nodes if skip_sources: return [ get_path(src_file(child)) for child in node.children() - if child not in node.sources + if child not in node.sources and not isinstance(child, SCons.Node.Python.Value) ] - return [get_path(src_file(child)) for child in node.children()] + return [get_path(src_file(child)) for child in node.children() if not isinstance(child, SCons.Node.Python.Value)] def get_inputs(node): @@ -168,8 +173,8 @@ def get_inputs(node): inputs = executor.get_all_sources() else: inputs = node.sources - - inputs = [get_path(src_file(o)) for o in inputs] + #TODO: handle Python.Value nodes + inputs = [get_path(src_file(o)) for o in inputs if not isinstance(o, SCons.Node.Python.Value)] return inputs @@ -202,6 +207,7 @@ def __init__(self, env): # this check for us. "SharedFlagChecker": ninja_noop, # The install builder is implemented as a function action. + # TODO: use command action #3573 "installFunc": _install_action_function, "MkdirFunc": _mkdir_action_function, "LibSymlinksActionFunction": _lib_symlink_action_function, @@ -262,12 +268,6 @@ def handle_func_action(self, node, action): # the bottom of ninja's DAG anyway and Textfile builders can have text # content as their source which doesn't work as an implicit dep in # ninja. - if name == "_action": - return { - "rule": "TEMPLATE", - "outputs": get_outputs(node), - "implicit": get_dependencies(node, skip_sources=True), - } if name == 'ninja_builder': return None @@ -280,7 +280,7 @@ def handle_func_action(self, node, action): if handler is not None: return handler(node.env if node.env else self.env, node) - raise Exception( + SCons.Warnings.Warning( "Found unhandled function action {}, " " generating scons command to build\n" "Note: this is less efficient than Ninja," @@ -288,6 +288,12 @@ def handle_func_action(self, node, action): " this function using NinjaRegisterFunctionHandler".format(name) ) + return { + "rule": "TEMPLATE", + "outputs": get_outputs(node), + "implicit": get_dependencies(node, skip_sources=True), + } + # pylint: disable=too-many-branches def handle_list_action(self, node, action): """TODO write this comment""" @@ -309,7 +315,7 @@ def handle_list_action(self, node, action): all_outputs = list({output for build in results for output in build["outputs"]}) dependencies = list({dep for build in results for dep in build["implicit"]}) - if results[0]["rule"] == "CMD": + if results[0]["rule"] == "CMD" or results[0]["rule"] == "GENERATED_CMD": cmdline = "" for cmd in results: @@ -339,7 +345,7 @@ def handle_list_action(self, node, action): if cmdline: ninja_build = { "outputs": all_outputs, - "rule": "CMD", + "rule": "GENERATED_CMD", "variables": { "cmd": cmdline, "env": get_command_env(node.env if node.env else self.env), @@ -360,10 +366,11 @@ def handle_list_action(self, node, action): } elif results[0]["rule"] == "INSTALL": + #TODO: handle Python.Value nodes return { "outputs": all_outputs, "rule": "INSTALL", - "inputs": [get_path(src_file(s)) for s in node.sources], + "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], "implicit": dependencies, } @@ -397,22 +404,28 @@ def __init__(self, env, ninja_file, writer_class): # like CCFLAGS escape = env.get("ESCAPE", lambda x: x) + # if SCons was invoked from python, we expect the first arg to be the scons.py + # script, otherwise scons was invoked from the scons script + python_bin = '' + if os.path.basename(sys.argv[0]) == 'scons.py': + python_bin = escape(sys.executable) self.variables = { "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", - "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format( - sys.executable, + "SCONS_INVOCATION": '{} {} --disable-ninja __NINJA_NO=1 $out'.format( + python_bin, " ".join( [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] ), ), - "SCONS_INVOCATION_W_TARGETS": "{} {}".format( - sys.executable, " ".join([escape(arg) for arg in sys.argv]) + "SCONS_INVOCATION_W_TARGETS": "{} {} --disable-ninja".format( + python_bin, " ".join([escape(arg) for arg in sys.argv]) ), # This must be set to a global default per: - # https://ninja-build.org/manual.html - # - # (The deps section) - "msvc_deps_prefix": "Note: including file:", + # https://ninja-build.org/manual.html#_deps + # English Visual Studio will have the default below, + # otherwise the user can define the variable in the first environment + # that initialized ninja tool + "msvc_deps_prefix": env.get("NINJA_MSVC_DEPS_PREFIX", "Note: including file:") } self.rules = { @@ -481,13 +494,13 @@ def __init__(self, env, ninja_file, writer_class): }, "TEMPLATE": { "command": "$SCONS_INVOCATION $out", - "description": "Rendering $out", + "description": "Rendering $SCONS_INVOCATION $out", "pool": "scons_pool", "restat": 1, }, "SCONS": { "command": "$SCONS_INVOCATION $out", - "description": "SCons $out", + "description": "$SCONS_INVOCATION $out", "pool": "scons_pool", # restat # if present, causes Ninja to re-stat the command's outputs @@ -557,6 +570,8 @@ def add_build(self, node): self.built.update(build["outputs"]) return True + # TODO: rely on SCons to tell us what is generated source + # or some form of user scanner maybe (Github Issue #3624) def is_generated_source(self, output): """Check if output ends with a known generated suffix.""" _, suffix = splitext(output) @@ -740,11 +755,7 @@ def generate(self): ninja.build( self.ninja_file.path, rule="REGENERATE", - implicit=[ - self.env.File("#SConstruct").path, - __file__, - ] - + sorted(glob("src/**/SConscript", recursive=True)), + implicit=[__file__] + [str(node) for node in SConscriptNodes], ) # If we ever change the name/s of the rules that include @@ -921,6 +932,7 @@ def get_response_file_command(env, node, action, targets, sources, executor=None ) cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] + rsp_content = ['"' + rsp_content_item + '"' for rsp_content_item in rsp_content] rsp_content = " ".join(rsp_content) variables = {"rspc": rsp_content} @@ -1060,20 +1072,23 @@ def ninja_builder(env, target, source): for key in env['ENV']: f.write('set {}={}\n'.format(key, env['ENV'][key])) f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) - - if not env.get("DISABLE_AUTO_NINJA"): + cmd = ['run_ninja_env.bat'] + + else: cmd = [NINJA_STATE.ninja_bin_path, '-f', generated_build_ninja] + + if not env.get("DISABLE_AUTO_NINJA"): print("Executing:", str(' '.join(cmd))) # execute the ninja build at the end of SCons, trying to # reproduce the output like a ninja build would def execute_ninja(): - + proc = subprocess.Popen(cmd, stderr=sys.stderr, stdout=subprocess.PIPE, universal_newlines=True, - env=env['ENV'] # ninja build items won't consider node env on win32 + env=os.environ if env["PLATFORM"] == "win32" else env['ENV'] ) for stdout_line in iter(proc.stdout.readline, ""): yield stdout_line @@ -1142,8 +1157,7 @@ def ninja_csig(original): """Return a dummy csig""" def wrapper(self): - name = str(self) - if "SConscript" in name or "SConstruct" in name: + if isinstance(self, SCons.Node.Node) and self.is_sconscript(): return original(self) return "dummy_ninja_csig" @@ -1154,8 +1168,7 @@ def ninja_contents(original): """Return a dummy content without doing IO""" def wrapper(self): - name = str(self) - if "SConscript" in name or "SConstruct" in name: + if isinstance(self, SCons.Node.Node) and self.is_sconscript(): return original(self) return bytes("dummy_ninja_contents", encoding="utf-8") @@ -1396,6 +1409,7 @@ def robust_rule_mapping(var, rule, tool): # Used to determine if a build generates a source file. Ninja # requires that all generated sources are added as order_only # dependencies to any builds that *might* use them. + # TODO: switch to using SCons to help determine this (Github Issue #3624) env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] if env["PLATFORM"] != "win32" and env.get("RANLIBCOM"): From c6764c5c0412cb875229e96d97a2ee94ac1c5263 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 22 Jun 2020 15:21:34 -0400 Subject: [PATCH 091/163] revert ninja install requirement expand response file in ninja comdb output --- src/engine/SCons/Tool/ninja.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 3fdadf9d77..ad36758788 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -767,7 +767,7 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "{} -f {} -t compdb CC CXX > compile_commands.json".format(self.ninja_bin_path, + "cmd": "{} -f {} -t compdb -x CC CXX > compile_commands.json".format(self.ninja_bin_path, str(self.ninja_file) ) }, From df31cad0ca8a178a019464c1513f61d558b2afa5 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 25 Jun 2020 23:05:28 -0400 Subject: [PATCH 092/163] handle files which are not file or alias by reinvoking scons --- src/engine/SCons/Tool/ninja.py | 228 +++++++++++++++++++-------------- 1 file changed, 134 insertions(+), 94 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index ad36758788..50a9518aae 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -56,63 +56,6 @@ ) -def _install_action_function(_env, node): - """Install files using the install or copy commands""" - #TODO: handle Python.Value nodes - return { - "outputs": get_outputs(node), - "rule": "INSTALL", - "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], - "implicit": get_dependencies(node), - } - -def _mkdir_action_function(env, node): - return { - "outputs": get_outputs(node), - "rule": "CMD", - # implicit explicitly omitted, we translate these so they can be - # used by anything that depends on these but commonly this is - # hit with a node that will depend on all of the fake - # srcnode's that SCons will never give us a rule for leading - # to an invalid ninja file. - "variables": { - # On Windows mkdir "-p" is always on - "cmd": "{mkdir}".format( - mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", - ), - }, - } - -def _copy_action_function(env, node): - #TODO: handle Python.Value nodes - return { - "outputs": get_outputs(node), - "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], - "rule": "CMD", - "variables": { - "cmd": "$COPY $in $out", - }, - } - - -def _lib_symlink_action_function(_env, node): - """Create shared object symlinks if any need to be created""" - symlinks = getattr(getattr(node, "attributes", None), "shliblinks", None) - - if not symlinks or symlinks is None: - return None - - outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] - inputs = [link.get_path() for link, _ in symlinks] - - return { - "outputs": outputs, - "inputs": inputs, - "rule": "SYMLINK", - "implicit": get_dependencies(node), - } - - def is_valid_dependent_node(node): """ Return True if node is not an alias or is an alias that has children @@ -136,46 +79,65 @@ def is_valid_dependent_node(node): def alias_to_ninja_build(node): """Convert an Alias node into a Ninja phony target""" - # TODO: handle Python.Values return { "outputs": get_outputs(node), "rule": "phony", "implicit": [ - get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) and not isinstance(n, SCons.Node.Python.Value) + get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) ], } +def check_invalid_ninja_node(node): + return not isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)) + + +def filter_ninja_nodes(node_list): + ninja_nodes = [] + for node in node_list: + if isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)): + ninja_nodes.append(node) + else: + continue + return ninja_nodes + +def get_input_nodes(node): + if node.get_executor() is not None: + inputs = node.get_executor().get_all_sources() + else: + inputs = node.sources + return inputs + + +def invalid_ninja_nodes(node, targets): + result = False + for node_list in [node.prerequisites, get_input_nodes(node), node.children(), targets]: + if node_list: + result = result or any([check_invalid_ninja_node(node) for node in node_list]) + return result + + def get_order_only(node): """Return a list of order only dependencies for node.""" if node.prerequisites is None: return [] - #TODO: handle Python.Value nodes - return [get_path(src_file(prereq)) for prereq in node.prerequisites if not isinstance(prereq, SCons.Node.Python.Value)] + return [get_path(src_file(prereq)) for prereq in filter_ninja_nodes(node.prerequisites)] def get_dependencies(node, skip_sources=False): """Return a list of dependencies for node.""" - #TODO: handle Python.Value nodes if skip_sources: return [ get_path(src_file(child)) - for child in node.children() - if child not in node.sources and not isinstance(child, SCons.Node.Python.Value) + for child in filter_ninja_nodes(node.children()) + if child not in node.sources ] - return [get_path(src_file(child)) for child in node.children() if not isinstance(child, SCons.Node.Python.Value)] + return [get_path(src_file(child)) for child in filter_ninja_nodes(node.children())] def get_inputs(node): """Collect the Ninja inputs for node.""" - executor = node.get_executor() - if executor is not None: - inputs = executor.get_all_sources() - else: - inputs = node.sources - #TODO: handle Python.Value nodes - inputs = [get_path(src_file(o)) for o in inputs if not isinstance(o, SCons.Node.Python.Value)] - return inputs + return [get_path(src_file(o)) for o in filter_ninja_nodes(get_input_nodes(node))] def get_outputs(node): @@ -189,11 +151,93 @@ def get_outputs(node): else: outputs = [node] - outputs = [get_path(o) for o in outputs] + outputs = [get_path(o) for o in filter_ninja_nodes(outputs)] return outputs +def get_targets_sources(node): + executor = node.get_executor() + if executor is not None: + tlist = executor.get_all_targets() + slist = executor.get_all_sources() + else: + if hasattr(node, "target_peers"): + tlist = node.target_peers + else: + tlist = [node] + slist = node.sources + + # Retrieve the repository file for all sources + slist = [rfile(s) for s in slist] + return tlist, slist + + +def get_rule(node, rule): + tlist, slist = get_targets_sources(node) + if invalid_ninja_nodes(node, tlist): + return "TEMPLATE" + else: + return rule + + +def _install_action_function(_env, node): + """Install files using the install or copy commands""" + return { + "outputs": get_outputs(node), + "rule": get_rule(node, "INSTALL"), + "inputs": get_inputs(node), + "implicit": get_dependencies(node), + } + + +def _mkdir_action_function(env, node): + return { + "outputs": get_outputs(node), + "rule": get_rule(node, "CMD"), + # implicit explicitly omitted, we translate these so they can be + # used by anything that depends on these but commonly this is + # hit with a node that will depend on all of the fake + # srcnode's that SCons will never give us a rule for leading + # to an invalid ninja file. + "variables": { + # On Windows mkdir "-p" is always on + "cmd": "{mkdir}".format( + mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", + ), + }, + } + + +def _copy_action_function(env, node): + return { + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "rule": get_rule(node, "CMD"), + "variables": { + "cmd": "$COPY $in $out", + }, + } + + +def _lib_symlink_action_function(_env, node): + """Create shared object symlinks if any need to be created""" + symlinks = getattr(getattr(node, "attributes", None), "shliblinks", None) + + if not symlinks or symlinks is None: + return None + + outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] + inputs = [link.get_path() for link, _ in symlinks] + + return { + "outputs": outputs, + "inputs": inputs, + "rule": get_rule(node, "SYMLINK"), + "implicit": get_dependencies(node), + } + + class SConsToNinjaTranslator: """Translates SCons Actions into Ninja build objects.""" @@ -290,7 +334,9 @@ def handle_func_action(self, node, action): return { "rule": "TEMPLATE", + "order_only": get_order_only(node), "outputs": get_outputs(node), + "inputs": get_inputs(node), "implicit": get_dependencies(node, skip_sources=True), } @@ -345,7 +391,7 @@ def handle_list_action(self, node, action): if cmdline: ninja_build = { "outputs": all_outputs, - "rule": "GENERATED_CMD", + "rule": get_rule(node, "GENERATED_CMD"), "variables": { "cmd": cmdline, "env": get_command_env(node.env if node.env else self.env), @@ -366,11 +412,10 @@ def handle_list_action(self, node, action): } elif results[0]["rule"] == "INSTALL": - #TODO: handle Python.Value nodes return { "outputs": all_outputs, - "rule": "INSTALL", - "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], + "rule": get_rule(node, "INSTALL"), + "inputs": get_inputs(node), "implicit": dependencies, } @@ -985,20 +1030,8 @@ def get_command(env, node, action): # pylint: disable=too-many-branches sub_env = node.env else: sub_env = env - executor = node.get_executor() - if executor is not None: - tlist = executor.get_all_targets() - slist = executor.get_all_sources() - else: - if hasattr(node, "target_peers"): - tlist = node.target_peers - else: - tlist = [node] - slist = node.sources - - # Retrieve the repository file for all sources - slist = [rfile(s) for s in slist] + tlist, slist = get_targets_sources(node) # Generate a real CommandAction if isinstance(action, SCons.Action.CommandGeneratorAction): @@ -1022,10 +1055,11 @@ def get_command(env, node, action): # pylint: disable=too-many-branches "outputs": get_outputs(node), "inputs": get_inputs(node), "implicit": implicit, - "rule": rule, + "rule": get_rule(node, rule), "variables": variables, } + # Don't use sub_env here because we require that NINJA_POOL be set # on a per-builder call basis to prevent accidental strange # behavior like env['NINJA_POOL'] = 'console' and sub_env can be @@ -1283,6 +1317,11 @@ def exists(env): if env.get("__NINJA_NO", "0") == "1": return False + try: + import ninja + except ImportError: + SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") + return False return True added = None @@ -1308,8 +1347,6 @@ def generate(env): default=False, help='Disable ninja automatically building after scons') - if GetOption('disable_ninja'): - return env try: import ninja @@ -1427,6 +1464,9 @@ def robust_rule_mapping(var, rule, tool): # Disable running ranlib, since we added 's' above env["RANLIBCOM"] = "" + if GetOption('disable_ninja'): + return env + SCons.Warnings.Warning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") # This is the point of no return, anything after this comment From 5eb4b761b98b7c7c89a368229cd73f714e2f61dc Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 20 Jul 2020 17:31:56 +0000 Subject: [PATCH 093/163] updated with some changes from latest mongodb version: 21075112a999e252a22e9c9bd64e403cec892df3 5fe923a0aa312044062df044eb4eaa47951f70ec c7348f391124e681d9c62aceb0e13e0d07fca8bc --- src/engine/SCons/Tool/ninja.py | 40 +++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 50a9518aae..bf7c45add0 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -29,6 +29,7 @@ import shutil import shlex import subprocess +import textwrap from os.path import join as joinpath from os.path import splitext @@ -768,13 +769,13 @@ def generate(self): # Special handling for outputs and implicit since we need to # aggregate not replace for each builder. - for agg_key in ["outputs", "implicit"]: + for agg_key in ["outputs", "implicit", 'inputs']: new_val = template_builds.get(agg_key, []) # Use pop so the key is removed and so the update # below will not overwrite our aggregated values. cur_val = template_builder.pop(agg_key, []) - if isinstance(cur_val, list): + if is_List(cur_val): new_val += cur_val else: new_val.append(cur_val) @@ -812,8 +813,8 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "{} -f {} -t compdb -x CC CXX > compile_commands.json".format(self.ninja_bin_path, - str(self.ninja_file) + "cmd": "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format( + ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' ) }, ) @@ -929,7 +930,13 @@ def get_command_env(env): if windows: command_env += "set '{}={}' && ".format(key, value) else: - command_env += "{}={} ".format(key, value) + # We address here *only* the specific case that a user might have + # an environment variable which somehow gets included and has + # spaces in the value. These are escapes that Ninja handles. This + # doesn't make builds on paths with spaces (Ninja and SCons issues) + # nor expanding response file paths with spaces (Ninja issue) work. + value = value.replace(r' ', r'$ ') + command_env += "{}='{}' ".format(key, value) env["NINJA_ENV_VAR_CACHE"] = command_env return command_env @@ -1208,6 +1215,27 @@ def wrapper(self): return wrapper +def CheckNinjaCompdbExpand(env, context): + """ Configure check testing if ninja's compdb can expand response files""" + + context.Message('Checking if ninja compdb can expand response files... ') + ret, output = context.TryAction( + action='ninja -f $SOURCE -t compdb -x CMD_RSP > $TARGET', + extension='.ninja', + text=textwrap.dedent(""" + rule CMD_RSP + command = $cmd @$out.rsp > fake_output.txt + description = Building $out + rspfile = $out.rsp + rspfile_content = $rspc + build fake_output.txt: CMD_RSP fake_input.txt + cmd = echo + pool = console + rspc = "test" + """)) + result = '@fake_output.txt.rsp' not in output + context.Result(result) + return result def ninja_stat(_self, path): """ @@ -1386,6 +1414,8 @@ def generate(env): else: env.Append(CCFLAGS=["-MD", "-MF", "${TARGET}.d"]) + env.AddMethod(CheckNinjaCompdbExpand, "CheckNinjaCompdbExpand") + # Provide a way for custom rule authors to easily access command # generation. env.AddMethod(get_shell_command, "NinjaGetShellCommand") From 53758dc83078043cc33f4da349b2eea20af3a00b Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 20 Jul 2020 17:43:48 +0000 Subject: [PATCH 094/163] fixed sider issues --- src/engine/SCons/Tool/ninja.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index bf7c45add0..153a8943b5 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -813,8 +813,8 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format( - ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' + "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( + self.ninja_bin_path, str(self.ninja_file), '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' ) }, ) @@ -1347,10 +1347,11 @@ def exists(env): try: import ninja + return ninja.__file__ except ImportError: SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") return False - return True + added = None From 492b34ce675d52729c18a17672d6e7ef7b3b4309 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 20 Jul 2020 18:25:39 +0000 Subject: [PATCH 095/163] updated warning to the latest API --- src/engine/SCons/Tool/ninja.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 153a8943b5..502d365014 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -325,7 +325,7 @@ def handle_func_action(self, node, action): if handler is not None: return handler(node.env if node.env else self.env, node) - SCons.Warnings.Warning( + SCons.Warnings.SConsWarning( "Found unhandled function action {}, " " generating scons command to build\n" "Note: this is less efficient than Ninja," @@ -1349,7 +1349,7 @@ def exists(env): import ninja return ninja.__file__ except ImportError: - SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") + SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return False @@ -1380,7 +1380,7 @@ def generate(env): try: import ninja except ImportError: - SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") + SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return env["DISABLE_AUTO_NINJA"] = GetOption('disable_execute_ninja') @@ -1405,7 +1405,7 @@ def generate(env): env.Alias("$NINJA_ALIAS_NAME", ninja_file) else: if str(NINJA_STATE.ninja_file) != ninja_file_name: - SCons.Warnings.Warning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") + SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] # This adds the required flags such that the generated compile @@ -1498,7 +1498,7 @@ def robust_rule_mapping(var, rule, tool): if GetOption('disable_ninja'): return env - SCons.Warnings.Warning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") + SCons.Warnings.SConsWarning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") # This is the point of no return, anything after this comment # makes changes to SCons that are irreversible and incompatible From 5f758433d6f1facb6bcff79ac2b430a20550131a Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 31 Dec 2020 15:58:31 +0000 Subject: [PATCH 096/163] Sync with mongo ninja file --- src/engine/SCons/Tool/ninja.py | 230 ++++++++++++++++++++++++++------- 1 file changed, 186 insertions(+), 44 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 502d365014..ea01d590ee 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -31,6 +31,7 @@ import subprocess import textwrap +from glob import glob from os.path import join as joinpath from os.path import splitext @@ -156,6 +157,41 @@ def get_outputs(node): return outputs +def generate_depfile(env, node, dependencies): + """ + Ninja tool function for writing a depfile. The depfile should include + the node path followed by all the dependent files in a makefile format. + + dependencies arg can be a list or a subst generator which returns a list. + """ + + depfile = os.path.join(get_path(env['NINJA_BUILDDIR']), str(node) + '.depfile') + + # subst_list will take in either a raw list or a subst callable which generates + # a list, and return a list of CmdStringHolders which can be converted into raw strings. + # If a raw list was passed in, then scons_list will make a list of lists from the original + # values and even subst items in the list if they are substitutable. Flatten will flatten + # the list in that case, to ensure for either input we have a list of CmdStringHolders. + deps_list = env.Flatten(env.subst_list(dependencies)) + + # Now that we have the deps in a list as CmdStringHolders, we can convert them into raw strings + # and make sure to escape the strings to handle spaces in paths. We also will sort the result + # keep the order of the list consistent. + escaped_depends = sorted([dep.escape(env.get("ESCAPE", lambda x: x)) for dep in deps_list]) + depfile_contents = str(node) + ": " + ' '.join(escaped_depends) + + need_rewrite = False + try: + with open(depfile, 'r') as f: + need_rewrite = (f.read() != depfile_contents) + except FileNotFoundError: + need_rewrite = True + + if need_rewrite: + os.makedirs(os.path.dirname(depfile) or '.', exist_ok=True) + with open(depfile, 'w') as f: + f.write(depfile_contents) + def get_targets_sources(node): executor = node.get_executor() @@ -278,6 +314,7 @@ def action_to_ninja_build(self, node, action=None): return None build = {} + env = node.env if node.env else self.env # Ideally this should never happen, and we do try to filter # Ninja builders out of being sources of ninja builders but I @@ -286,22 +323,27 @@ def action_to_ninja_build(self, node, action=None): global NINJA_STATE if NINJA_STATE.ninja_file == str(node): build = None - if isinstance(action, SCons.Action.FunctionAction): + elif isinstance(action, SCons.Action.FunctionAction): build = self.handle_func_action(node, action) elif isinstance(action, SCons.Action.LazyAction): # pylint: disable=protected-access - action = action._generate_cache(node.env if node.env else self.env) + action = action._generate_cache(env) build = self.action_to_ninja_build(node, action=action) elif isinstance(action, SCons.Action.ListAction): build = self.handle_list_action(node, action) elif isinstance(action, COMMAND_TYPES): - build = get_command(node.env if node.env else self.env, node, action) + build = get_command(env, node, action) else: raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) if build is not None: build["order_only"] = get_order_only(node) + if 'conftest' not in str(node): + node_callback = getattr(node.attributes, "ninja_build_callback", None) + if callable(node_callback): + node_callback(env, node, build) + return build def handle_func_action(self, node, action): @@ -509,8 +551,16 @@ def __init__(self, env, ninja_file, writer_class): "rspfile_content": "$rspc", "pool": "local_pool", }, + # Ninja does not automatically delete the archive before + # invoking ar. The ar utility will append to an existing archive, which + # can cause duplicate symbols if the symbols moved between object files. + # Native SCons will perform this operation so we need to force ninja + # to do the same. See related for more info: + # https://jira.mongodb.org/browse/SERVER-49457 "AR": { - "command": "$env$AR @$out.rsp", + "command": "{}$env$AR @$out.rsp".format( + '' if sys.platform == "win32" else "rm -f $out && " + ), "description": "Archiving $out", "rspfile": "$out.rsp", "rspfile_content": "$rspc", @@ -540,7 +590,7 @@ def __init__(self, env, ninja_file, writer_class): }, "TEMPLATE": { "command": "$SCONS_INVOCATION $out", - "description": "Rendering $SCONS_INVOCATION $out", + "description": "Rendering $SCONS_INVOCATION $out", "pool": "scons_pool", "restat": 1, }, @@ -570,6 +620,7 @@ def __init__(self, env, ninja_file, writer_class): "command": "$SCONS_INVOCATION_W_TARGETS", "description": "Regenerating $out", "generator": 1, + "depfile": os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile'), # Console pool restricts to 1 job running at a time, # it additionally has some special handling about # passing stdin, stdout, etc to process in this pool @@ -650,6 +701,8 @@ def generate(self): ninja.comment("Generated by scons. DO NOT EDIT.") + ninja.variable("builddir", get_path(self.env['NINJA_BUILDDIR'])) + for pool_name, size in self.pools.items(): ninja.pool(pool_name, size) @@ -759,9 +812,19 @@ def generate(self): build["outputs"] = first_output + # Optionally a rule can specify a depfile, and SCons can generate implicit + # dependencies into the depfile. This allows for dependencies to come and go + # without invalidating the ninja file. The depfile was created in ninja specifically + # for dealing with header files appearing and disappearing across rebuilds, but it can + # be repurposed for anything, as long as you have a way to regenerate the depfile. + # More specific info can be found here: https://ninja-build.org/manual.html#_depfile + if rule is not None and rule.get('depfile') and build.get('deps_files'): + path = build['outputs'] if SCons.Util.is_List(build['outputs']) else [build['outputs']] + generate_depfile(self.env, path[0], build.pop('deps_files', [])) + if "inputs" in build: build["inputs"].sort() - + ninja.build(**build) template_builds = dict() @@ -769,7 +832,7 @@ def generate(self): # Special handling for outputs and implicit since we need to # aggregate not replace for each builder. - for agg_key in ["outputs", "implicit", 'inputs']: + for agg_key in ["outputs", "implicit", "inputs"]: new_val = template_builds.get(agg_key, []) # Use pop so the key is removed and so the update @@ -793,15 +856,25 @@ def generate(self): # generate this rule even though SCons should know we're # dependent on SCons files. # - # TODO: We're working on getting an API into SCons that will - # allow us to query the actual SConscripts used. Right now - # this glob method has deficiencies like skipping - # jstests/SConscript and being specific to the MongoDB - # repository layout. (github issue #3625) + # The REGENERATE rule uses depfile, so we need to generate the depfile + # in case any of the SConscripts have changed. The depfile needs to be + # path with in the build and the passed ninja file is an abspath, so + # we will use SCons to give us the path within the build. Normally + # generate_depfile should not be called like this, but instead be called + # through the use of custom rules, and filtered out in the normal + # list of build generation about. However, because the generate rule + # is hardcoded here, we need to do this generate_depfile call manually. + ninja_file_path = self.env.File(self.ninja_file).path + generate_depfile( + self.env, + ninja_file_path, + self.env['NINJA_REGENERATE_DEPS'] + ) + ninja.build( - self.ninja_file.path, + ninja_file_path, rule="REGENERATE", - implicit=[__file__] + [str(node) for node in SConscriptNodes], + implicit=[__file__], ) # If we ever change the name/s of the rules that include @@ -936,13 +1009,13 @@ def get_command_env(env): # doesn't make builds on paths with spaces (Ninja and SCons issues) # nor expanding response file paths with spaces (Ninja issue) work. value = value.replace(r' ', r'$ ') - command_env += "{}='{}' ".format(key, value) + command_env += "export {}='{}';".format(key, value) env["NINJA_ENV_VAR_CACHE"] = command_env return command_env -def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False): +def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom_env={}): """Generate a response file command provider for rule name.""" # If win32 using the environment with a response file command will cause @@ -979,7 +1052,7 @@ def get_response_file_command(env, node, action, targets, sources, executor=None except ValueError: raise Exception( "Could not find tool {} in {} generated from {}".format( - tool_command, cmd_list, get_comstr(env, action, targets, sources) + tool, cmd_list, get_comstr(env, action, targets, sources) ) ) @@ -991,7 +1064,12 @@ def get_response_file_command(env, node, action, targets, sources, executor=None variables[rule] = cmd if use_command_env: variables["env"] = get_command_env(env) - return rule, variables + + for key, value in custom_env.items(): + variables["env"] += env.subst( + f"export {key}={value};", target=targets, source=sources, executor=executor + ) + " " + return rule, variables, [tool_command] return get_response_file_command @@ -1021,13 +1099,21 @@ def generate_command(env, node, action, targets, sources, executor=None): return cmd.replace("$", "$$") -def get_shell_command(env, node, action, targets, sources, executor=None): +def get_generic_shell_command(env, node, action, targets, sources, executor=None): return ( - "GENERATED_CMD", + "CMD", { "cmd": generate_command(env, node, action, targets, sources, executor=None), "env": get_command_env(env), }, + # Since this function is a rule mapping provider, it must return a list of dependencies, + # and usually this would be the path to a tool, such as a compiler, used for this rule. + # However this function is to generic to be able to reliably extract such deps + # from the command, so we return a placeholder empty list. It should be noted that + # generally this function will not be used soley and is more like a template to generate + # the basics for a custom provider which may have more specific options for a provier + # function for a custom NinjaRuleMapping. + [] ) @@ -1050,13 +1136,49 @@ def get_command(env, node, action): # pylint: disable=too-many-branches comstr = get_comstr(sub_env, action, tlist, slist) if not comstr: return None - - provider = __NINJA_RULE_MAPPING.get(comstr, get_shell_command) - rule, variables = provider(sub_env, node, action, tlist, slist, executor=executor) + + provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) + rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) # Get the dependencies for all targets implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) + # Now add in the other dependencies related to the command, + # e.g. the compiler binary. The ninja rule can be user provided so + # we must do some validation to resolve the dependency path for ninja. + for provider_dep in provider_deps: + + provider_dep = sub_env.subst(provider_dep) + if not provider_dep: + continue + + # If the tool is a node, then SCons will resolve the path later, if its not + # a node then we assume it generated from build and make sure it is existing. + if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep): + implicit.append(provider_dep) + continue + + # in some case the tool could be in the local directory and be suppled without the ext + # such as in windows, so append the executable suffix and check. + prog_suffix = sub_env.get('PROGSUFFIX', '') + provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix + if os.path.exists(provider_dep_ext): + implicit.append(provider_dep_ext) + continue + + # Many commands will assume the binary is in the path, so + # we accept this as a possible input from a given command. + + provider_dep_abspath = sub_env.WhereIs(provider_dep) or sub_env.WhereIs(provider_dep, path=os.environ["PATH"]) + if provider_dep_abspath: + implicit.append(provider_dep_abspath) + continue + + # Possibly these could be ignore and the build would still work, however it may not always + # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided + # ninja rule. + raise Exception(f"Could not resolve path for {provider_dep} dependency on node '{node}'") + ninja_build = { "order_only": get_order_only(node), "outputs": get_outputs(node), @@ -1066,7 +1188,6 @@ def get_command(env, node, action): # pylint: disable=too-many-branches "variables": variables, } - # Don't use sub_env here because we require that NINJA_POOL be set # on a per-builder call basis to prevent accidental strange # behavior like env['NINJA_POOL'] = 'console' and sub_env can be @@ -1103,28 +1224,28 @@ def ninja_builder(env, target, source): NINJA_STATE.generate() if env["PLATFORM"] == "win32": - # this is not great, its doesn't consider specific + # this is not great, its doesn't consider specific # node environments, which means on linux the build could # behave differently, because on linux you can set the environment - # per command in the ninja file. This is only needed if + # per command in the ninja file. This is only needed if # running ninja directly from a command line that hasn't # had the environment setup (vcvarsall.bat) with open('run_ninja_env.bat', 'w') as f: for key in env['ENV']: f.write('set {}={}\n'.format(key, env['ENV'][key])) - f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) - cmd = ['run_ninja_env.bat'] + f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) + cmd = ['run_ninja_env.bat'] else: cmd = [NINJA_STATE.ninja_bin_path, '-f', generated_build_ninja] - + if not env.get("DISABLE_AUTO_NINJA"): print("Executing:", str(' '.join(cmd))) # execute the ninja build at the end of SCons, trying to # reproduce the output like a ninja build would def execute_ninja(): - + proc = subprocess.Popen(cmd, stderr=sys.stderr, stdout=subprocess.PIPE, @@ -1137,7 +1258,7 @@ def execute_ninja(): return_code = proc.wait() if return_code: raise subprocess.CalledProcessError(return_code, 'ninja') - + erase_previous = False for output in execute_ninja(): output = output.strip() @@ -1168,24 +1289,31 @@ def register_custom_handler(env, name, handler): def register_custom_rule_mapping(env, pre_subst_string, rule): - """Register a custom handler for SCons function actions.""" + """Register a function to call for a given rule.""" global __NINJA_RULE_MAPPING __NINJA_RULE_MAPPING[pre_subst_string] = rule -def register_custom_rule(env, rule, command, description="", deps=None, pool=None): +def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): """Allows specification of Ninja rules from inside SCons files.""" rule_obj = { "command": command, "description": description if description else "{} $out".format(rule), } + if use_depfile: + rule_obj["depfile"] = os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile') + if deps is not None: rule_obj["deps"] = deps if pool is not None: rule_obj["pool"] = pool + if use_response_file: + rule_obj["rspfile"] = "$out.rsp" + rule_obj["rspfile_content"] = response_file_content + env[NINJA_RULES][rule] = rule_obj @@ -1193,6 +1321,9 @@ def register_custom_pool(env, pool, size): """Allows the creation of custom Ninja pools""" env[NINJA_POOLS][pool] = size +def set_build_node_callback(env, node, callback): + if 'conftest' not in str(node): + setattr(node.attributes, "ninja_build_callback", callback) def ninja_csig(original): """Return a dummy csig""" @@ -1395,7 +1526,7 @@ def generate(env): env["NINJA_PREFIX"] = env.get("NINJA_PREFIX", "build") env["NINJA_SUFFIX"] = env.get("NINJA_SUFFIX", "ninja") env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") - + env['NINJA_BUILDDIR'] = env.get("NINJA_BUILDDIR", env.Dir(".ninja").path) ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") # here we allow multiple environments to construct rules and builds # into the same ninja file @@ -1407,20 +1538,31 @@ def generate(env): if str(NINJA_STATE.ninja_file) != ninja_file_name: SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] - + + + # TODO: API for getting the SConscripts programmatically + # exists upstream: https://github.com/SCons/scons/issues/3625 + def ninja_generate_deps(env): + return sorted([env.File("#SConstruct").path] + glob("**/SConscript", recursive=True)) + env['_NINJA_REGENERATE_DEPS_FUNC'] = ninja_generate_deps + + env['NINJA_REGENERATE_DEPS'] = env.get('NINJA_REGENERATE_DEPS', '${_NINJA_REGENERATE_DEPS_FUNC(__env__)}') + # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. if env["PLATFORM"] == "win32": env.Append(CCFLAGS=["/showIncludes"]) else: - env.Append(CCFLAGS=["-MD", "-MF", "${TARGET}.d"]) + env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) env.AddMethod(CheckNinjaCompdbExpand, "CheckNinjaCompdbExpand") # Provide a way for custom rule authors to easily access command # generation. - env.AddMethod(get_shell_command, "NinjaGetShellCommand") + env.AddMethod(get_generic_shell_command, "NinjaGetGenericShellCommand") + env.AddMethod(get_command, "NinjaGetCommand") env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider") + env.AddMethod(set_build_node_callback, "NinjaSetBuildNodeCallback") # Provides a way for users to handle custom FunctionActions they # want to translate to Ninja. @@ -1587,7 +1729,7 @@ def robust_rule_mapping(var, rule, tool): SCons.Job.Jobs.__init__ = ninja_always_serial ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') - + if NINJA_STATE is None: NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) NINJA_STATE.ninja_bin_path = env.get('NINJA_BIN') @@ -1595,10 +1737,10 @@ def robust_rule_mapping(var, rule, tool): # default to using ninja installed with python module ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', + ninja.__file__, + os.pardir, + 'data', + 'bin', ninja_bin)) if not os.path.exists(NINJA_STATE.ninja_bin_path): # couldn't find it, just give the bin name and hope @@ -1607,7 +1749,7 @@ def robust_rule_mapping(var, rule, tool): # TODO: this is hacking into scons, preferable if there were a less intrusive way # We will subvert the normal builder execute to make sure all the ninja file is dependent - # on all targets generated from any builders + # on all targets generated from any builders SCons_Builder_BuilderBase__execute = SCons.Builder.BuilderBase._execute def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): # this ensures all environments in which a builder executes from will @@ -1666,4 +1808,4 @@ def ninja_execute(self): if not os.path.isdir(os.environ["TMPDIR"]): env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) - env["TEMPFILE"] = NinjaNoResponseFiles \ No newline at end of file + env["TEMPFILE"] = NinjaNoResponseFiles From b9d9518ad89882ea17d2db51c25a0be5040e8ea1 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 31 Dec 2020 16:06:53 +0000 Subject: [PATCH 097/163] Update ninja to new scons layout --- {src/engine/SCons => SCons}/Tool/ninja.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {src/engine/SCons => SCons}/Tool/ninja.py (100%) diff --git a/src/engine/SCons/Tool/ninja.py b/SCons/Tool/ninja.py similarity index 100% rename from src/engine/SCons/Tool/ninja.py rename to SCons/Tool/ninja.py From 366e83d8f28f1b9e5efb5346affb320178701103 Mon Sep 17 00:00:00 2001 From: Mathew Robinson Date: Fri, 24 Jan 2020 17:25:31 -0500 Subject: [PATCH 098/163] [WIP] write a tool to generate build.ninja files from SCons --- SCons/Script/SConscript.py | 3 + SCons/Script/__init__.py | 1 + src/engine/SCons/Tool/ninja.py | 1396 ++++++++++++++++++++++++++++++++ 3 files changed, 1400 insertions(+) create mode 100644 src/engine/SCons/Tool/ninja.py diff --git a/SCons/Script/SConscript.py b/SCons/Script/SConscript.py index 596fca0463..ded0fcfef9 100644 --- a/SCons/Script/SConscript.py +++ b/SCons/Script/SConscript.py @@ -203,9 +203,11 @@ def _SConscript(fs, *files, **kw): if f.rexists(): actual = f.rfile() _file_ = open(actual.get_abspath(), "rb") + SCons.Script.LOADED_SCONSCRIPTS.append(actual.get_abspath()) elif f.srcnode().rexists(): actual = f.srcnode().rfile() _file_ = open(actual.get_abspath(), "rb") + SCons.Script.LOADED_SCONSCRIPTS.append(actual.get_abspath()) elif f.has_src_builder(): # The SConscript file apparently exists in a source # code management system. Build it, but then clear @@ -214,6 +216,7 @@ def _SConscript(fs, *files, **kw): f.build() f.built() f.builder_set(None) + SCons.Script.LOADED_SCONSCRIPTS.append(f.get_abspath()) if f.exists(): _file_ = open(f.get_abspath(), "rb") if _file_: diff --git a/SCons/Script/__init__.py b/SCons/Script/__init__.py index 5f58d9972d..e409f06ef4 100644 --- a/SCons/Script/__init__.py +++ b/SCons/Script/__init__.py @@ -187,6 +187,7 @@ def _clear(self): BUILD_TARGETS = TargetList() COMMAND_LINE_TARGETS = [] DEFAULT_TARGETS = [] +LOADED_SCONSCRIPTS = [] # BUILD_TARGETS can be modified in the SConscript files. If so, we # want to treat the modified BUILD_TARGETS list as if they specified diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py new file mode 100644 index 0000000000..b34759e401 --- /dev/null +++ b/src/engine/SCons/Tool/ninja.py @@ -0,0 +1,1396 @@ +# Copyright 2019 MongoDB Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Generate build.ninja files from SCons aliases.""" + +import sys +import os +import importlib +import io +import shutil + +from threading import Lock +from os.path import join as joinpath +from os.path import splitext + +import SCons +from SCons.Action import _string_from_cmd_list, get_default_ENV +from SCons.Util import is_String, is_List +from SCons.Script import COMMAND_LINE_TARGETS, LOADED_SCONSCRIPTS + +NINJA_SYNTAX = "NINJA_SYNTAX" +NINJA_RULES = "__NINJA_CUSTOM_RULES" +NINJA_POOLS = "__NINJA_CUSTOM_POOLS" +NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" +NINJA_BUILD = "NINJA_BUILD" +NINJA_WHEREIS_MEMO = {} +NINJA_STAT_MEMO = {} +MEMO_LOCK = Lock() + +__NINJA_RULE_MAPPING = {} + +# These are the types that get_command can do something with +COMMAND_TYPES = ( + SCons.Action.CommandAction, + SCons.Action.CommandGeneratorAction, +) + + +def _install_action_function(_env, node): + """Install files using the install or copy commands""" + return { + "outputs": get_outputs(node), + "rule": "INSTALL", + "pool": "install_pool", + "inputs": [get_path(src_file(s)) for s in node.sources], + "implicit": get_dependencies(node), + } + + +def _lib_symlink_action_function(_env, node): + """Create shared object symlinks if any need to be created""" + symlinks = getattr(getattr(node, "attributes", None), "shliblinks", None) + + if not symlinks or symlinks is None: + return None + + outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] + inputs = [link.get_path() for link, _ in symlinks] + + return { + "outputs": outputs, + "inputs": inputs, + "rule": "SYMLINK", + "implicit": get_dependencies(node), + } + + +def is_valid_dependent_node(node): + """ + Return True if node is not an alias or is an alias that has children + + This prevents us from making phony targets that depend on other + phony targets that will never have an associated ninja build + target. + + We also have to specify that it's an alias when doing the builder + check because some nodes (like src files) won't have builders but + are valid implicit dependencies. + """ + return not isinstance(node, SCons.Node.Alias.Alias) or node.children() + + +def alias_to_ninja_build(node): + """Convert an Alias node into a Ninja phony target""" + return { + "outputs": get_outputs(node), + "rule": "phony", + "implicit": [ + get_path(n) for n in node.children() if is_valid_dependent_node(n) + ], + } + + +def get_dependencies(node): + """Return a list of dependencies for node.""" + return [get_path(src_file(child)) for child in node.children()] + + +def get_inputs(node): + """Collect the Ninja inputs for node.""" + executor = node.get_executor() + if executor is not None: + inputs = executor.get_all_sources() + else: + inputs = node.sources + + inputs = [get_path(src_file(o)) for o in inputs] + return inputs + + +def get_outputs(node): + """Collect the Ninja outputs for node.""" + executor = node.get_executor() + if executor is not None: + outputs = executor.get_all_targets() + else: + if hasattr(node, "target_peers"): + outputs = node.target_peers + else: + outputs = [node] + + outputs = [get_path(o) for o in outputs] + return outputs + + +class SConsToNinjaTranslator: + """Translates SCons Actions into Ninja build objects.""" + + def __init__(self, env): + self.env = env + self.func_handlers = { + # Skip conftest builders + "_createSource": ninja_noop, + # SCons has a custom FunctionAction that just makes sure the + # target isn't static. We let the commands that ninja runs do + # this check for us. + "SharedFlagChecker": ninja_noop, + # The install builder is implemented as a function action. + "installFunc": _install_action_function, + "LibSymlinksActionFunction": _lib_symlink_action_function, + } + + self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) + + # pylint: disable=too-many-return-statements + def action_to_ninja_build(self, node, action=None): + """Generate build arguments dictionary for node.""" + # Use False since None is a valid value for this Attribute + build = getattr(node.attributes, NINJA_BUILD, False) + if build is not False: + return build + + if node.builder is None: + return None + + if action is None: + action = node.builder.action + + # Ideally this should never happen, and we do try to filter + # Ninja builders out of being sources of ninja builders but I + # can't fix every DAG problem so we just skip ninja_builders + # if we find one + if node.builder == self.env["BUILDERS"]["Ninja"]: + return None + + if isinstance(action, SCons.Action.FunctionAction): + return self.handle_func_action(node, action) + + if isinstance(action, SCons.Action.LazyAction): + # pylint: disable=protected-access + action = action._generate_cache(node.env if node.env else self.env) + return self.action_to_ninja_build(node, action=action) + + if isinstance(action, SCons.Action.ListAction): + return self.handle_list_action(node, action) + + if isinstance(action, COMMAND_TYPES): + return get_command(node.env if node.env else self.env, node, action) + + # Return the node to indicate that SCons is required + return { + "rule": "SCONS", + "outputs": get_outputs(node), + "implicit": get_dependencies(node), + } + + def handle_func_action(self, node, action): + """Determine how to handle the function action.""" + name = action.function_name() + # This is the name given by the Subst/Textfile builders. So return the + # node to indicate that SCons is required + if name == "_action": + return { + "rule": "TEMPLATE", + "outputs": get_outputs(node), + "implicit": get_dependencies(node), + } + + handler = self.func_handlers.get(name, None) + if handler is not None: + return handler(node.env if node.env else self.env, node) + + print( + "Found unhandled function action {}, " + " generating scons command to build\n" + "Note: this is less efficient than Ninja," + " you can write your own ninja build generator for" + " this function using NinjaRegisterFunctionHandler".format(name) + ) + + return { + "rule": "SCONS", + "outputs": get_outputs(node), + "implicit": get_dependencies(node), + } + + # pylint: disable=too-many-branches + def handle_list_action(self, node, action): + """ + Attempt to translate list actions to Ninja. + + List actions are tricky to move to ninja. First we translate + each individual action in the action list to a Ninja + build. Then we process the resulting ninja builds to see if + they are all the same ninja rule. If they are not all the same + rule we cannot make them a single resulting ninja build, so + instead we make them a single SCons invocation to build all of + the targets. + + If they are all the same rule and the rule is CMD we attempt + to combine the cmdlines together using ' && ' which we then + combine into a single ninja build. + + If they are all phony targets we simple combine the outputs + and dependencies. + + If they are all INSTALL rules we combine all of the sources + and outputs. + + If they are all SCONS rules we do the same as if they are not + the same rule and make a build that will use SCons to generate + them. + + If they're all one rule and None of the above rules we throw an Exception. + """ + + results = [ + self.action_to_ninja_build(node, action=act) + for act in action.list + if act is not None + ] + results = [result for result in results if result is not None] + if not results: + return None + + # No need to process the results if we only got a single result + if len(results) == 1: + return results[0] + + all_outputs = list({output for build in results for output in build["outputs"]}) + # If we have no outputs we're done + if not all_outputs: + return None + + # Used to verify if all rules are the same + all_one_rule = len( + [ + r + for r in results + if isinstance(r, dict) and r["rule"] == results[0]["rule"] + ] + ) == len(results) + dependencies = get_dependencies(node) + + if not all_one_rule: + # If they aren't all the same rule use scons to generate these + # outputs. At this time nothing hits this case. + return { + "outputs": all_outputs, + "rule": "SCONS", + "implicit": dependencies, + } + + if results[0]["rule"] == "CMD": + cmdline = "" + for cmd in results: + + # Occasionally a command line will expand to a + # whitespace only string (i.e. ' '). Which is not a + # valid command but does not trigger the empty command + # condition if not cmdstr. So here we strip preceding + # and proceeding whitespace to make strings like the + # above become empty strings and so will be skipped. + cmdstr = cmd["variables"]["cmd"].strip() + if not cmdstr: + continue + + # Skip duplicate commands + if cmdstr in cmdline: + continue + + if cmdline: + cmdline += " && " + + cmdline += cmdstr + + # Remove all preceding and proceeding whitespace + cmdline = cmdline.strip() + + # Make sure we didn't generate an empty cmdline + if cmdline: + ninja_build = { + "outputs": all_outputs, + "rule": "CMD", + "variables": {"cmd": cmdline}, + "implicit": dependencies, + } + + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["pool"] + + return ninja_build + + elif results[0]["rule"] == "phony": + return { + "outputs": all_outputs, + "rule": "phony", + "implicit": dependencies, + } + + elif results[0]["rule"] == "INSTALL": + return { + "outputs": all_outputs, + "rule": "INSTALL", + "pool": "install_pool", + "inputs": [get_path(src_file(s)) for s in node.sources], + "implicit": dependencies, + } + + elif results[0]["rule"] == "SCONS": + return { + "outputs": all_outputs, + "rule": "SCONS", + "inputs": dependencies, + } + + raise Exception("Unhandled list action with rule: " + results[0]["rule"]) + + +# pylint: disable=too-many-instance-attributes +class NinjaState: + """Maintains state of Ninja build system as it's translated from SCons.""" + + def __init__(self, env, writer_class): + self.env = env + self.writer_class = writer_class + self.__generated = False + self.translator = SConsToNinjaTranslator(env) + self.generated_suffixes = env.get("NINJA_GENERATED_SOURCE_SUFFIXES", []) + + # List of generated builds that will be written at a later stage + self.builds = list() + + # List of targets for which we have generated a build. This + # allows us to take multiple Alias nodes as sources and to not + # fail to build if they have overlapping targets. + self.built = set() + + # SCons sets this variable to a function which knows how to do + # shell quoting on whatever platform it's run on. Here we use it + # to make the SCONS_INVOCATION variable properly quoted for things + # like CCFLAGS + escape = env.get("ESCAPE", lambda x: x) + + self.variables = { + "COPY": "cmd.exe /c copy" if sys.platform == "win32" else "cp", + "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format( + sys.executable, + " ".join( + [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] + ), + ), + "SCONS_INVOCATION_W_TARGETS": "{} {}".format( + sys.executable, " ".join([escape(arg) for arg in sys.argv]) + ), + # This must be set to a global default per: + # https://ninja-build.org/manual.html + # + # (The deps section) + "msvc_deps_prefix": "Note: including file:", + } + + self.rules = { + "CMD": { + "command": "cmd /c $cmd" if sys.platform == "win32" else "$cmd", + "description": "Building $out", + }, + # We add the deps processing variables to this below. We + # don't pipe this through cmd.exe on Windows because we + # use this to generate a compile_commands.json database + # which can't use the shell command as it's compile + # command. This does mean that we assume anything using + # CMD_W_DEPS is a straight up compile which is true today. + "CMD_W_DEPS": {"command": "$cmd", "description": "Building $out"}, + "SYMLINK": { + "command": ( + "cmd /c mklink $out $in" + if sys.platform == "win32" + else "ln -s $in $out" + ), + "description": "Symlink $in -> $out", + }, + "INSTALL": { + "command": "$COPY $in $out", + "description": "Install $out", + # On Windows cmd.exe /c copy does not always correctly + # update the timestamp on the output file. This leads + # to a stuck constant timestamp in the Ninja database + # and needless rebuilds. + # + # Adding restat here ensures that Ninja always checks + # the copy updated the timestamp and that Ninja has + # the correct information. + "restat": 1, + }, + "TEMPLATE": { + "command": "$SCONS_INVOCATION $out", + "description": "Rendering $out", + "pool": "scons_pool", + "restat": 1, + }, + "SCONS": { + "command": "$SCONS_INVOCATION $out", + "description": "SCons $out", + "pool": "scons_pool", + # restat + # if present, causes Ninja to re-stat the command's outputs + # after execution of the command. Each output whose + # modification time the command did not change will be + # treated as though it had never needed to be built. This + # may cause the output's reverse dependencies to be removed + # from the list of pending build actions. + # + # We use restat any time we execute SCons because + # SCons calls in Ninja typically create multiple + # targets. But since SCons is doing it's own up to + # date-ness checks it may only update say one of + # them. Restat will find out which of the multiple + # build targets did actually change then only rebuild + # those targets which depend specifically on that + # output. + "restat": 1, + }, + "REGENERATE": { + "command": "$SCONS_INVOCATION_W_TARGETS", + "description": "Regenerating $out", + "generator": 1, + # Console pool restricts to 1 job running at a time, + # it additionally has some special handling about + # passing stdin, stdout, etc to process in this pool + # that we need for SCons to behave correctly when + # regenerating Ninja + "pool": "console", + # Again we restat in case Ninja thought the + # build.ninja should be regenerated but SCons knew + # better. + "restat": 1, + }, + } + + self.pools = { + "install_pool": self.env.GetOption("num_jobs") / 2, + "scons_pool": 1, + } + + if env["PLATFORM"] == "win32": + self.rules["CMD_W_DEPS"]["deps"] = "msvc" + else: + self.rules["CMD_W_DEPS"]["deps"] = "gcc" + self.rules["CMD_W_DEPS"]["depfile"] = "$out.d" + + self.rules.update(env.get(NINJA_RULES, {})) + self.pools.update(env.get(NINJA_POOLS, {})) + + def generate_builds(self, node): + """Generate a ninja build rule for node and it's children.""" + # Filter out nodes with no builder. They are likely source files + # and so no work needs to be done, it will be used in the + # generation for some real target. + # + # Note that all nodes have a builder attribute but it is sometimes + # set to None. So we cannot use a simpler hasattr check here. + if getattr(node, "builder", None) is None: + return + + stack = [[node]] + while stack: + frame = stack.pop() + for child in frame: + outputs = set(get_outputs(child)) + # Check if all the outputs are in self.built, if they + # are we've already seen this node and it's children. + if not outputs.isdisjoint(self.built): + continue + + self.built = self.built.union(outputs) + stack.append(child.children()) + + if isinstance(child, SCons.Node.Alias.Alias): + build = alias_to_ninja_build(child) + elif node.builder is not None: + # Use False since None is a valid value for this attribute + build = getattr(child.attributes, NINJA_BUILD, False) + if build is False: + build = self.translator.action_to_ninja_build(child) + setattr(child.attributes, NINJA_BUILD, build) + else: + build = None + + # Some things are unbuild-able or need not be built in Ninja + if build is None or build == 0: + continue + + self.builds.append(build) + + def is_generated_source(self, output): + """Check if output ends with a known generated suffix.""" + _, suffix = splitext(output) + return suffix in self.generated_suffixes + + def has_generated_sources(self, output): + """ + Determine if output indicates this is a generated header file. + """ + for generated in output: + if self.is_generated_source(generated): + return True + return False + + # pylint: disable=too-many-branches,too-many-locals + def generate(self, ninja_file, fallback_default_target=None): + """ + Generate the build.ninja. + + This should only be called once for the lifetime of this object. + """ + if self.__generated: + return + + content = io.StringIO() + ninja = self.writer_class(content, width=100) + + ninja.comment("Generated by scons. DO NOT EDIT.") + + for pool_name, size in self.pools.items(): + ninja.pool(pool_name, size) + + for var, val in self.variables.items(): + ninja.variable(var, val) + + for rule, kwargs in self.rules.items(): + ninja.rule(rule, **kwargs) + + generated_source_files = { + output + # First find builds which have header files in their outputs. + for build in self.builds + if self.has_generated_sources(build["outputs"]) + for output in build["outputs"] + # Collect only the header files from the builds with them + # in their output. We do this because is_generated_source + # returns True if it finds a header in any of the outputs, + # here we need to filter so we only have the headers and + # not the other outputs. + if self.is_generated_source(output) + } + + if generated_source_files: + ninja.build( + outputs="_generated_sources", + rule="phony", + implicit=list(generated_source_files), + ) + + template_builders = [] + + for build in self.builds: + if build["rule"] == "TEMPLATE": + template_builders.append(build) + continue + + implicit = build.get("implicit", []) + implicit.append(ninja_file) + build["implicit"] = implicit + + # Don't make generated sources depend on each other. We + # have to check that none of the outputs are generated + # sources and none of the direct implicit dependencies are + # generated sources or else we will create a dependency + # cycle. + if ( + generated_source_files + and not build["rule"] == "INSTALL" + and set(build["outputs"]).isdisjoint(generated_source_files) + and set(implicit).isdisjoint(generated_source_files) + ): + + # Make all non-generated source targets depend on + # _generated_sources. We use order_only for generated + # sources so that we don't rebuild the world if one + # generated source was rebuilt. We just need to make + # sure that all of these sources are generated before + # other builds. + build["order_only"] = "_generated_sources" + + # When using a depfile Ninja can only have a single output + # but SCons will usually have emitted an output for every + # thing a command will create because it's caching is much + # more complex than Ninja's. This includes things like DWO + # files. Here we make sure that Ninja only ever sees one + # target when using a depfile. It will still have a command + # that will create all of the outputs but most targets don't + # depend direclty on DWO files and so this assumption is safe + # to make. + rule = self.rules.get(build["rule"]) + + # Some rules like 'phony' and other builtins we don't have + # listed in self.rules so verify that we got a result + # before trying to check if it has a deps key. + if rule is not None and rule.get("deps"): + + # Anything using deps in Ninja can only have a single + # output, but we may have a build which actually + # produces multiple outputs which other targets can + # depend on. Here we slice up the outputs so we have a + # single output which we will use for the "real" + # builder and multiple phony targets that match the + # file names of the remaining outputs. This way any + # build can depend on any output from any build. + first_output, remaining_outputs = build["outputs"][0], build["outputs"][1:] + if remaining_outputs: + ninja.build( + outputs=remaining_outputs, + rule="phony", + implicit=first_output, + ) + + build["outputs"] = first_output + + ninja.build(**build) + + template_builds = dict() + for template_builder in template_builders: + + # Special handling for outputs and implicit since we need to + # aggregate not replace for each builder. + for agg_key in ["outputs", "implicit"]: + new_val = template_builds.get(agg_key, []) + + # Use pop so the key is removed and so the update + # below will not overwrite our aggregated values. + cur_val = template_builder.pop(agg_key, []) + if isinstance(cur_val, list): + new_val += cur_val + else: + new_val.append(cur_val) + template_builds[agg_key] = new_val + + # Collect all other keys + template_builds.update(template_builder) + + if template_builds.get("outputs", []): + ninja.build(**template_builds) + + # Here to teach the ninja file how to regenerate itself. We'll + # never see ourselves in the DAG walk so we can't rely on + # action_to_ninja_build to generate this rule + ninja.build( + ninja_file, + rule="REGENERATE", + implicit=[ + self.env.File("#SConstruct").get_abspath(), + os.path.abspath(__file__), + ] + + LOADED_SCONSCRIPTS, + ) + + ninja.build( + "scons-invocation", + rule="CMD", + pool="console", + variables={"cmd": "echo $SCONS_INVOCATION_W_TARGETS"}, + ) + + # Note the use of CMD_W_DEPS below. CMD_W_DEPS are always + # compile commands in this generator. If we ever change the + # name/s of the rules that include compile commands + # (i.e. something like CC/CXX) we will need to update this + # build to reflect that complete list. + ninja.build( + "compile_commands.json", + rule="CMD", + pool="console", + variables={ + "cmd": "ninja -f {} -t compdb CMD_W_DEPS > compile_commands.json".format( + ninja_file + ) + }, + ) + + ninja.build( + "compiledb", rule="phony", implicit=["compile_commands.json"], + ) + + # Look in SCons's list of DEFAULT_TARGETS, find the ones that + # we generated a ninja build rule for. + scons_default_targets = [ + get_path(tgt) + for tgt in SCons.Script.DEFAULT_TARGETS + if get_path(tgt) in self.built + ] + + # If we found an overlap between SCons's list of default + # targets and the targets we created ninja builds for then use + # those as ninja's default as well. + if scons_default_targets: + ninja.default(" ".join(scons_default_targets)) + + # If not then set the default to the fallback_default_target we were given. + # Otherwise we won't create a default ninja target. + elif fallback_default_target is not None: + ninja.default(fallback_default_target) + + with open(ninja_file, "w") as build_ninja: + build_ninja.write(content.getvalue()) + + self.__generated = True + + +def get_path(node): + """ + Return a fake path if necessary. + + As an example Aliases use this as their target name in Ninja. + """ + if hasattr(node, "get_path"): + return node.get_path() + return str(node) + + +def rfile(node): + """ + Return the repository file for node if it has one. Otherwise return node + """ + if hasattr(node, "rfile"): + return node.rfile() + return node + + +def src_file(node): + """Returns the src code file if it exists.""" + if hasattr(node, "srcnode"): + src = node.srcnode() + if src.stat() is not None: + return src + return get_path(node) + + +# TODO: Make the Rules smarter. Instead of just using a "cmd" rule +# everywhere we should be smarter about generating CC, CXX, LINK, +# etc. rules +def get_command(env, node, action): # pylint: disable=too-many-branches + """Get the command to execute for node.""" + if node.env: + sub_env = node.env + else: + sub_env = env + + executor = node.get_executor() + if executor is not None: + tlist = executor.get_all_targets() + slist = executor.get_all_sources() + else: + if hasattr(node, "target_peers"): + tlist = node.target_peers + else: + tlist = [node] + slist = node.sources + + # Retrieve the repository file for all sources + slist = [rfile(s) for s in slist] + + # Get the dependencies for all targets + implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) + + # Generate a real CommandAction + if isinstance(action, SCons.Action.CommandGeneratorAction): + # pylint: disable=protected-access + action = action._generate(tlist, slist, sub_env, 1, executor=executor) + + rule = "CMD" + + # Actions like CommandAction have a method called process that is + # used by SCons to generate the cmd_line they need to run. So + # check if it's a thing like CommandAction and call it if we can. + if hasattr(action, "process"): + cmd_list, _, _ = action.process(tlist, slist, sub_env, executor=executor) + + # Despite being having "list" in it's name this member is not + # actually a list. It's the pre-subst'd string of the command. We + # use it to determine if the command we generated needs to use a + # custom Ninja rule. By default this redirects CC/CXX commands to + # CMD_W_DEPS but the user can inject custom Ninja rules and tie + # them to commands by using their pre-subst'd string. + rule = __NINJA_RULE_MAPPING.get(action.cmd_list, "CMD") + + cmd = _string_from_cmd_list(cmd_list[0]) + else: + # Anything else works with genstring, this is most commonly hit by + # ListActions which essentially call process on all of their + # commands and concatenate it for us. + genstring = action.genstring(tlist, slist, sub_env) + + # Detect if we have a custom rule for this + # "ListActionCommandAction" type thing. + rule = __NINJA_RULE_MAPPING.get(genstring, "CMD") + + if executor is not None: + cmd = sub_env.subst(genstring, executor=executor) + else: + cmd = sub_env.subst(genstring, target=tlist, source=slist) + + # Since we're only enabling Ninja for developer builds right + # now we skip all Manifest related work on Windows as it's not + # necessary. We shouldn't have gotten here but on Windows + # SCons has a ListAction which shows as a + # CommandGeneratorAction for linking. That ListAction ends + # with a FunctionAction (embedManifestExeCheck, + # embedManifestDllCheck) that simply say "does + # target[0].manifest exist?" if so execute the real command + # action underlying me, otherwise do nothing. + # + # Eventually we'll want to find a way to translate this to + # Ninja but for now, and partially because the existing Ninja + # generator does so, we just disable it all together. + cmd = cmd.replace("\n", " && ").strip() + if env["PLATFORM"] == "win32" and ( + "embedManifestExeCheck" in cmd or "embedManifestDllCheck" in cmd + ): + cmd = " && ".join(cmd.split(" && ")[0:-1]) + + if cmd.endswith("&&"): + cmd = cmd[0:-2].strip() + + outputs = get_outputs(node) + command_env = "" + windows = env["PLATFORM"] == "win32" + + # If win32 and rule == CMD_W_DEPS then we don't want to calculate + # an environment for this command. It's a compile command and + # compiledb doesn't support shell syntax on Windows. We need the + # shell syntax to use environment variables on Windows so we just + # skip this platform / rule combination to keep the compiledb + # working. + # + # On POSIX we can still set environment variables even for compile + # commands so we do so. + if not (windows and rule == "CMD_W_DEPS"): + + # Scan the ENV looking for any keys which do not exist in + # os.environ or differ from it. We assume if it's a new or + # differing key from the process environment then it's + # important to pass down to commands in the Ninja file. + ENV = get_default_ENV(sub_env) + scons_specified_env = { + key: value + for key, value in ENV.items() + if key not in os.environ or os.environ.get(key, None) != value + } + + for key, value in scons_specified_env.items(): + # Ensure that the ENV values are all strings: + if is_List(value): + # If the value is a list, then we assume it is a + # path list, because that's a pretty common list-like + # value to stick in an environment variable: + value = flatten_sequence(value) + value = joinpath(map(str, value)) + else: + # If it isn't a string or a list, then we just coerce + # it to a string, which is the proper way to handle + # Dir and File instances and will produce something + # reasonable for just about everything else: + value = str(value) + + if windows: + command_env += "set '{}={}' && ".format(key, value) + else: + command_env += "{}={} ".format(key, value) + + variables = {"cmd": command_env + cmd} + extra_vars = getattr(node.attributes, "NINJA_EXTRA_VARS", {}) + if extra_vars: + variables.update(extra_vars) + + ninja_build = { + "outputs": outputs, + "inputs": get_inputs(node), + "implicit": implicit, + "rule": rule, + "variables": variables, + } + + # Don't use sub_env here because we require that NINJA_POOL be set + # on a per-builder call basis to prevent accidental strange + # behavior like env['NINJA_POOL'] = 'console' and sub_env can be + # the global Environment object if node.env is None. + # Example: + # + # Allowed: + # + # env.Command("ls", NINJA_POOL="ls_pool") + # + # Not allowed and ignored: + # + # env["NINJA_POOL"] = "ls_pool" + # env.Command("ls") + # + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["NINJA_POOL"] + + return ninja_build + + +def ninja_builder(env, target, source): + """Generate a build.ninja for source.""" + if not isinstance(source, list): + source = [source] + if not isinstance(target, list): + target = [target] + + # We have no COMSTR equivalent so print that we're generating + # here. + print("Generating:", str(target[0])) + + # The environment variable NINJA_SYNTAX points to the + # ninja_syntax.py module from the ninja sources found here: + # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py + # + # This should be vendored into the build sources and it's location + # set in NINJA_SYNTAX. This code block loads the location from + # that variable, gets the absolute path to the vendored file, gets + # it's parent directory then uses importlib to import the module + # dynamically. + ninja_syntax_file = env[NINJA_SYNTAX] + if isinstance(ninja_syntax_file, str): + ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() + ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) + sys.path.append(ninja_syntax_mod_dir) + ninja_syntax_mod_name = os.path.basename(ninja_syntax_file) + ninja_syntax = importlib.import_module(ninja_syntax_mod_name.replace(".py", "")) + + suffix = env.get("NINJA_SUFFIX", "") + if suffix and not suffix[0] == ".": + suffix = "." + suffix + + generated_build_ninja = target[0].get_abspath() + suffix + ninja_state = NinjaState(env, ninja_syntax.Writer) + + for src in source: + ninja_state.generate_builds(src) + + ninja_state.generate(generated_build_ninja, str(source[0])) + + return 0 + + +# pylint: disable=too-few-public-methods +class AlwaysExecAction(SCons.Action.FunctionAction): + """Override FunctionAction.__call__ to always execute.""" + + def __call__(self, *args, **kwargs): + kwargs["execute"] = 1 + return super().__call__(*args, **kwargs) + + +def ninja_print(_cmd, target, _source, env): + """Tag targets with the commands to build them.""" + if target: + for tgt in target: + if ( + tgt.has_builder() + # Use 'is False' because not would still trigger on + # None's which we don't want to regenerate + and getattr(tgt.attributes, NINJA_BUILD, False) is False + and isinstance(tgt.builder.action, COMMAND_TYPES) + ): + ninja_action = get_command(env, tgt, tgt.builder.action) + setattr(tgt.attributes, NINJA_BUILD, ninja_action) + # Preload the attributes dependencies while we're still running + # multithreaded + get_dependencies(tgt) + return 0 + + +def register_custom_handler(env, name, handler): + """Register a custom handler for SCons function actions.""" + env[NINJA_CUSTOM_HANDLERS][name] = handler + + +def register_custom_rule_mapping(env, pre_subst_string, rule): + """Register a custom handler for SCons function actions.""" + global __NINJA_RULE_MAPPING + __NINJA_RULE_MAPPING[pre_subst_string] = rule + + +def register_custom_rule(env, rule, command, description="", deps=None): + """Allows specification of Ninja rules from inside SCons files.""" + rule_obj = { + "command": command, + "description": description if description else "{} $out".format(rule), + } + + if deps is not None: + rule_obj["deps"] = deps + + env[NINJA_RULES][rule] = rule_obj + + +def register_custom_pool(env, pool, size): + """Allows the creation of custom Ninja pools""" + env[NINJA_POOLS][pool] = size + + +def ninja_csig(original): + """Return a dummy csig""" + + def wrapper(self): + name = str(self) + if "SConscript" in name or "SConstruct" in name: + return original(self) + return "dummy_ninja_csig" + + return wrapper + + +def ninja_contents(original): + """Return a dummy content without doing IO""" + + def wrapper(self): + name = str(self) + if "SConscript" in name or "SConstruct" in name: + return original(self) + return bytes("dummy_ninja_contents", encoding="utf-8") + + return wrapper + + +def ninja_stat(_self, path): + """ + Eternally memoized stat call. + + SCons is very aggressive about clearing out cached values. For our + purposes everything should only ever call stat once since we're + running in a no_exec build the file system state should not + change. For these reasons we patch SCons.Node.FS.LocalFS.stat to + use our eternal memoized dictionary. + + Since this is happening during the Node walk it's being run while + threaded, we have to protect adding to the memoized dictionary + with a threading.Lock otherwise many targets miss the memoization + due to racing. + """ + global NINJA_STAT_MEMO + + try: + return NINJA_STAT_MEMO[path] + except KeyError: + try: + result = os.stat(path) + except os.error: + result = None + + with MEMO_LOCK: + NINJA_STAT_MEMO[path] = result + + return result + + +def ninja_noop(*_args, **_kwargs): + """ + A general purpose no-op function. + + There are many things that happen in SCons that we don't need and + also don't return anything. We use this to disable those functions + instead of creating multiple definitions of the same thing. + """ + return None + + +def ninja_whereis(thing, *_args, **_kwargs): + """Replace env.WhereIs with a much faster version""" + global NINJA_WHEREIS_MEMO + + # Optimize for success, this gets called significantly more often + # when the value is already memoized than when it's not. + try: + return NINJA_WHEREIS_MEMO[thing] + except KeyError: + # We do not honor any env['ENV'] or env[*] variables in the + # generated ninja ile. Ninja passes your raw shell environment + # down to it's subprocess so the only sane option is to do the + # same during generation. At some point, if and when we try to + # upstream this, I'm sure a sticking point will be respecting + # env['ENV'] variables and such but it's actually quite + # complicated. I have a naive version but making it always work + # with shell quoting is nigh impossible. So I've decided to + # cross that bridge when it's absolutely required. + path = shutil.which(thing) + NINJA_WHEREIS_MEMO[thing] = path + return path + + +def ninja_always_serial(self, num, taskmaster): + """Replacement for SCons.Job.Jobs constructor which always uses the Serial Job class.""" + # We still set self.num_jobs to num even though it's a lie. The + # only consumer of this attribute is the Parallel Job class AND + # the Main.py function which instantiates a Jobs class. It checks + # if Jobs.num_jobs is equal to options.num_jobs, so if the user + # provides -j12 but we set self.num_jobs = 1 they get an incorrect + # warning about this version of Python not supporting parallel + # builds. So here we lie so the Main.py will not give a false + # warning to users. + self.num_jobs = num + self.job = SCons.Job.Serial(taskmaster) + + +class NinjaEternalTempFile(SCons.Platform.TempFileMunge): + """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" + + def __call__(self, target, source, env, for_signature): + if for_signature: + return self.cmd + + node = target[0] if SCons.Util.is_List(target) else target + if node is not None: + cmdlist = getattr(node.attributes, "tempfile_cmdlist", None) + if cmdlist is not None: + return cmdlist + + cmd = super().__call__(target, source, env, for_signature) + + # If TempFileMunge.__call__ returns a string it means that no + # response file was needed. No processing required so just + # return the command. + if isinstance(cmd, str): + return cmd + + # Strip the removal commands from the command list. + # + # SCons' TempFileMunge class has some very strange + # behavior where it, as part of the command line, tries to + # delete the response file after executing the link + # command. We want to keep those response files since + # Ninja will keep using them over and over. The + # TempFileMunge class creates a cmdlist to do this, a + # common SCons convention for executing commands see: + # https://github.com/SCons/scons/blob/master/src/engine/SCons/Action.py#L949 + # + # This deletion behavior is not configurable. So we wanted + # to remove the deletion command from the command list by + # simply slicing it out here. Unfortunately for some + # strange reason TempFileMunge doesn't make the "rm" + # command it's own list element. It appends it to the + # tempfile argument to cmd[0] (which is CC/CXX) and then + # adds the tempfile again as it's own element. + # + # So we just kind of skip that middle element. Since the + # tempfile is in the command list on it's own at the end we + # can cut it out entirely. This is what I would call + # "likely to break" in future SCons updates. Hopefully it + # breaks because they start doing the right thing and not + # weirdly splitting these arguments up. For reference a + # command list that we get back from the OG TempFileMunge + # looks like this: + # + # [ + # 'g++', + # '@/mats/tempfiles/random_string.lnk\nrm', + # '/mats/tempfiles/random_string.lnk', + # ] + # + # Note the weird newline and rm command in the middle + # element and the lack of TEMPFILEPREFIX on the last + # element. + prefix = env.subst("$TEMPFILEPREFIX") + if not prefix: + prefix = "@" + + new_cmdlist = [cmd[0], prefix + cmd[-1]] + setattr(node.attributes, "tempfile_cmdlist", new_cmdlist) + return new_cmdlist + + def _print_cmd_str(*_args, **_kwargs): + """Disable this method""" + pass + + +def exists(env): + """Enable if called.""" + + # This variable disables the tool when storing the SCons command in the + # generated ninja file to ensure that the ninja tool is not loaded when + # SCons should do actual work as a subprocess of a ninja build. The ninja + # tool is very invasive into the internals of SCons and so should never be + # enabled when SCons needs to build a target. + if env.get("__NINJA_NO", "0") == "1": + return False + + return True + + +def generate(env): + """Generate the NINJA builders.""" + env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") + + # Add the Ninja builder. + always_exec_ninja_action = AlwaysExecAction(ninja_builder, {}) + ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action) + env.Append(BUILDERS={"Ninja": ninja_builder_obj}) + + # This adds the required flags such that the generated compile + # commands will create depfiles as appropriate in the Ninja file. + if env["PLATFORM"] == "win32": + env.Append(CCFLAGS=["/showIncludes"]) + else: + env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) + + # Provides a way for users to handle custom FunctionActions they + # want to translate to Ninja. + env[NINJA_CUSTOM_HANDLERS] = {} + env.AddMethod(register_custom_handler, "NinjaRegisterFunctionHandler") + + # Provides a mechanism for inject custom Ninja rules which can + # then be mapped using NinjaRuleMapping. + env[NINJA_RULES] = {} + env.AddMethod(register_custom_rule, "NinjaRule") + + # Provides a mechanism for inject custom Ninja pools which can + # be used by providing the NINJA_POOL="name" as an + # OverrideEnvironment variable in a builder call. + env[NINJA_POOLS] = {} + env.AddMethod(register_custom_pool, "NinjaPool") + + # Add the ability to register custom NinjaRuleMappings for Command + # builders. We don't store this dictionary in the env to prevent + # accidental deletion of the CC/XXCOM mappings. You can still + # overwrite them if you really want to but you have to explicit + # about it this way. The reason is that if they were accidentally + # deleted you would get a very subtly incorrect Ninja file and + # might not catch it. + env.AddMethod(register_custom_rule_mapping, "NinjaRuleMapping") + env.NinjaRuleMapping("${CCCOM}", "CMD_W_DEPS") + env.NinjaRuleMapping("${CXXCOM}", "CMD_W_DEPS") + + # Make SCons node walk faster by preventing unnecessary work + env.Decider("timestamp-match") + + # Used to determine if a build generates a source file. Ninja + # requires that all generated sources are added as order_only + # dependencies to any builds that *might* use them. + env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] + + # This is the point of no return, anything after this comment + # makes changes to SCons that are irreversible and incompatible + # with a normal SCons build. We return early if __NINJA_NO=1 has + # been given on the command line (i.e. by us in the generated + # ninja file) here to prevent these modifications from happening + # when we want SCons to do work. Everything before this was + # necessary to setup the builder and other functions so that the + # tool can be unconditionally used in the users's SCons files. + + if not exists(env): + return + + # Set a known variable that other tools can query so they can + # behave correctly during ninja generation. + env["GENERATING_NINJA"] = True + + # These methods are no-op'd because they do not work during ninja + # generation, expected to do no work, or simply fail. All of which + # are slow in SCons. So we overwrite them with no logic. + SCons.Node.FS.File.make_ready = ninja_noop + SCons.Node.FS.File.prepare = ninja_noop + SCons.Node.FS.File.push_to_cache = ninja_noop + SCons.Executor.Executor.prepare = ninja_noop + SCons.Node.FS.File.built = ninja_noop + + # We make lstat a no-op because it is only used for SONAME + # symlinks which we're not producing. + SCons.Node.FS.LocalFS.lstat = ninja_noop + + # This is a slow method that isn't memoized. We make it a noop + # since during our generation we will never use the results of + # this or change the results. + SCons.Node.FS.is_up_to_date = ninja_noop + + # We overwrite stat and WhereIs with eternally memoized + # implementations. See the docstring of ninja_stat and + # ninja_whereis for detailed explanations. + SCons.Node.FS.LocalFS.stat = ninja_stat + SCons.Util.WhereIs = ninja_whereis + + # Monkey patch get_csig and get_contents for some classes. It + # slows down the build significantly and we don't need contents or + # content signatures calculated when generating a ninja file since + # we're not doing any SCons caching or building. + SCons.Executor.Executor.get_contents = ninja_contents( + SCons.Executor.Executor.get_contents + ) + SCons.Node.Alias.Alias.get_contents = ninja_contents( + SCons.Node.Alias.Alias.get_contents + ) + SCons.Node.FS.File.get_contents = ninja_contents(SCons.Node.FS.File.get_contents) + SCons.Node.FS.File.get_csig = ninja_csig(SCons.Node.FS.File.get_csig) + SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) + SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig) + + # Replace false Compiling* messages with a more accurate output + # + # We also use this to tag all Nodes with Builders using + # CommandActions with the final command that was used to compile + # it for passing to Ninja. If we don't inject this behavior at + # this stage in the build too much state is lost to generate the + # command at the actual ninja_builder execution time for most + # commands. + # + # We do attempt command generation again in ninja_builder if it + # hasn't been tagged and it seems to work for anything that + # doesn't represent as a non-FunctionAction during the print_func + # call. + env["PRINT_CMD_LINE_FUNC"] = ninja_print + + # This reduces unnecessary subst_list calls to add the compiler to + # the implicit dependencies of targets. Since we encode full paths + # in our generated commands we do not need these slow subst calls + # as executing the command will fail if the file is not found + # where we expect it. + env["IMPLICIT_COMMAND_DEPENDENCIES"] = False + + # Set build to no_exec, our sublcass of FunctionAction will force + # an execution for ninja_builder so this simply effects all other + # Builders. + env.SetOption("no_exec", True) + + # This makes SCons more aggressively cache MD5 signatures in the + # SConsign file. + env.SetOption("max_drift", 1) + + # The Serial job class is SIGNIFICANTLY (almost twice as) faster + # than the Parallel job class for generating Ninja files. So we + # monkey the Jobs constructor to only use the Serial Job class. + SCons.Job.Jobs.__init__ = ninja_always_serial + + # We will eventually need to overwrite TempFileMunge to make it + # handle persistent tempfiles or get an upstreamed change to add + # some configurability to it's behavior in regards to tempfiles. + # + # Set all three environment variables that Python's + # tempfile.mkstemp looks at as it behaves differently on different + # platforms and versions of Python. + os.environ["TMPDIR"] = env.Dir("$BUILD_DIR/response_files").get_abspath() + os.environ["TEMP"] = os.environ["TMPDIR"] + os.environ["TMP"] = os.environ["TMPDIR"] + if not os.path.isdir(os.environ["TMPDIR"]): + env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) + + env["TEMPFILE"] = NinjaEternalTempFile + + # Force the SConsign to be written, we benefit from SCons caching of + # implicit dependencies and conftests. Unfortunately, we have to do this + # using an atexit handler because SCons will not write the file when in a + # no_exec build. + import atexit + + atexit.register(SCons.SConsign.write) From d9c27b862028b22ded6cc68ddccbfce5a6ba25dc Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 5 May 2020 23:44:01 -0400 Subject: [PATCH 099/163] updated to ninja-next, added some small fixes, and added simple test --- src/engine/SCons/Tool/ninja.py | 1071 +++++++++++++++++------------- test/ninja/CC.py | 66 ++ test/ninja/ninja-fixture/test2.C | 3 + 3 files changed, 668 insertions(+), 472 deletions(-) create mode 100644 test/ninja/CC.py create mode 100644 test/ninja/ninja-fixture/test2.C diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index b34759e401..d1cbafa495 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -1,16 +1,25 @@ -# Copyright 2019 MongoDB Inc. +# Copyright 2020 MongoDB Inc. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: # -# http://www.apache.org/licenses/LICENSE-2.0 +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + """Generate build.ninja files from SCons aliases.""" import sys @@ -18,16 +27,19 @@ import importlib import io import shutil +import shlex +import subprocess -from threading import Lock +from glob import glob from os.path import join as joinpath from os.path import splitext import SCons from SCons.Action import _string_from_cmd_list, get_default_ENV -from SCons.Util import is_String, is_List -from SCons.Script import COMMAND_LINE_TARGETS, LOADED_SCONSCRIPTS +from SCons.Util import is_List, flatten_sequence +from SCons.Script import COMMAND_LINE_TARGETS +NINJA_STATE = None NINJA_SYNTAX = "NINJA_SYNTAX" NINJA_RULES = "__NINJA_CUSTOM_RULES" NINJA_POOLS = "__NINJA_CUSTOM_POOLS" @@ -35,7 +47,6 @@ NINJA_BUILD = "NINJA_BUILD" NINJA_WHEREIS_MEMO = {} NINJA_STAT_MEMO = {} -MEMO_LOCK = Lock() __NINJA_RULE_MAPPING = {} @@ -51,11 +62,43 @@ def _install_action_function(_env, node): return { "outputs": get_outputs(node), "rule": "INSTALL", - "pool": "install_pool", "inputs": [get_path(src_file(s)) for s in node.sources], "implicit": get_dependencies(node), } +def _mkdir_action_function(env, node): + return { + "outputs": get_outputs(node), + "rule": "CMD", + # implicit explicitly omitted, we translate these so they can be + # used by anything that depends on these but commonly this is + # hit with a node that will depend on all of the fake + # srcnode's that SCons will never give us a rule for leading + # to an invalid ninja file. + "variables": { + # On Windows mkdir "-p" is always on + "cmd": "{mkdir} $out".format( + mkdir="mkdir" if env["PLATFORM"] == "win32" else "mkdir -p", + ), + }, + } + +def _copy_action_function(env, node): + return { + "outputs": get_outputs(node), + "inputs": [get_path(src_file(s)) for s in node.sources], + "rule": "CMD", + # implicit explicitly omitted, we translate these so they can be + # used by anything that depends on these but commonly this is + # hit with a node that will depend on all of the fake + # srcnode's that SCons will never give us a rule for leading + # to an invalid ninja file. + "variables": { + # On Windows mkdir "-p" is always on + "cmd": "$COPY $in $out", + }, + } + def _lib_symlink_action_function(_env, node): """Create shared object symlinks if any need to be created""" @@ -87,7 +130,13 @@ def is_valid_dependent_node(node): check because some nodes (like src files) won't have builders but are valid implicit dependencies. """ - return not isinstance(node, SCons.Node.Alias.Alias) or node.children() + if isinstance(node, SCons.Node.Alias.Alias): + return node.children() + + if not node.env: + return True + + return not node.env.get("NINJA_SKIP") def alias_to_ninja_build(node): @@ -96,13 +145,26 @@ def alias_to_ninja_build(node): "outputs": get_outputs(node), "rule": "phony", "implicit": [ - get_path(n) for n in node.children() if is_valid_dependent_node(n) + get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) ], } -def get_dependencies(node): +def get_order_only(node): + """Return a list of order only dependencies for node.""" + if node.prerequisites is None: + return [] + return [get_path(src_file(prereq)) for prereq in node.prerequisites] + + +def get_dependencies(node, skip_sources=False): """Return a list of dependencies for node.""" + if skip_sources: + return [ + get_path(src_file(child)) + for child in node.children() + if child not in node.sources + ] return [get_path(src_file(child)) for child in node.children()] @@ -130,6 +192,7 @@ def get_outputs(node): outputs = [node] outputs = [get_path(o) for o in outputs] + return outputs @@ -147,18 +210,19 @@ def __init__(self, env): "SharedFlagChecker": ninja_noop, # The install builder is implemented as a function action. "installFunc": _install_action_function, + "MkdirFunc": _mkdir_action_function, "LibSymlinksActionFunction": _lib_symlink_action_function, + "Copy" : _copy_action_function } - self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) + self.loaded_custom = False # pylint: disable=too-many-return-statements def action_to_ninja_build(self, node, action=None): """Generate build arguments dictionary for node.""" - # Use False since None is a valid value for this Attribute - build = getattr(node.attributes, NINJA_BUILD, False) - if build is not False: - return build + if not self.loaded_custom: + self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) + self.loaded_custom = True if node.builder is None: return None @@ -166,51 +230,60 @@ def action_to_ninja_build(self, node, action=None): if action is None: action = node.builder.action + if node.env and node.env.get("NINJA_SKIP"): + return None + + build = {} + # Ideally this should never happen, and we do try to filter # Ninja builders out of being sources of ninja builders but I # can't fix every DAG problem so we just skip ninja_builders # if we find one if node.builder == self.env["BUILDERS"]["Ninja"]: - return None - - if isinstance(action, SCons.Action.FunctionAction): - return self.handle_func_action(node, action) - - if isinstance(action, SCons.Action.LazyAction): + build = None + elif isinstance(action, SCons.Action.FunctionAction): + build = self.handle_func_action(node, action) + elif isinstance(action, SCons.Action.LazyAction): # pylint: disable=protected-access action = action._generate_cache(node.env if node.env else self.env) - return self.action_to_ninja_build(node, action=action) - - if isinstance(action, SCons.Action.ListAction): - return self.handle_list_action(node, action) + build = self.action_to_ninja_build(node, action=action) + elif isinstance(action, SCons.Action.ListAction): + build = self.handle_list_action(node, action) + elif isinstance(action, COMMAND_TYPES): + build = get_command(node.env if node.env else self.env, node, action) + else: + raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) - if isinstance(action, COMMAND_TYPES): - return get_command(node.env if node.env else self.env, node, action) + if build is not None: + build["order_only"] = get_order_only(node) - # Return the node to indicate that SCons is required - return { - "rule": "SCONS", - "outputs": get_outputs(node), - "implicit": get_dependencies(node), - } + return build def handle_func_action(self, node, action): """Determine how to handle the function action.""" name = action.function_name() # This is the name given by the Subst/Textfile builders. So return the - # node to indicate that SCons is required + # node to indicate that SCons is required. We skip sources here because + # dependencies don't really matter when we're going to shove these to + # the bottom of ninja's DAG anyway and Textfile builders can have text + # content as their source which doesn't work as an implicit dep in + # ninja. if name == "_action": return { "rule": "TEMPLATE", "outputs": get_outputs(node), - "implicit": get_dependencies(node), + "implicit": get_dependencies(node, skip_sources=True), } - handler = self.func_handlers.get(name, None) if handler is not None: return handler(node.env if node.env else self.env, node) + elif name == "ActionCaller": + action_to_call = str(action).split('(')[0].strip() + handler = self.func_handlers.get(action_to_call, None) + if handler is not None: + return handler(node.env if node.env else self.env, node) - print( + raise Exception( "Found unhandled function action {}, " " generating scons command to build\n" "Note: this is less efficient than Ninja," @@ -218,48 +291,17 @@ def handle_func_action(self, node, action): " this function using NinjaRegisterFunctionHandler".format(name) ) - return { - "rule": "SCONS", - "outputs": get_outputs(node), - "implicit": get_dependencies(node), - } - # pylint: disable=too-many-branches def handle_list_action(self, node, action): - """ - Attempt to translate list actions to Ninja. - - List actions are tricky to move to ninja. First we translate - each individual action in the action list to a Ninja - build. Then we process the resulting ninja builds to see if - they are all the same ninja rule. If they are not all the same - rule we cannot make them a single resulting ninja build, so - instead we make them a single SCons invocation to build all of - the targets. - - If they are all the same rule and the rule is CMD we attempt - to combine the cmdlines together using ' && ' which we then - combine into a single ninja build. - - If they are all phony targets we simple combine the outputs - and dependencies. - - If they are all INSTALL rules we combine all of the sources - and outputs. - - If they are all SCONS rules we do the same as if they are not - the same rule and make a build that will use SCons to generate - them. - - If they're all one rule and None of the above rules we throw an Exception. - """ - + """TODO write this comment""" results = [ self.action_to_ninja_build(node, action=act) for act in action.list if act is not None ] - results = [result for result in results if result is not None] + results = [ + result for result in results if result is not None and result["outputs"] + ] if not results: return None @@ -268,28 +310,7 @@ def handle_list_action(self, node, action): return results[0] all_outputs = list({output for build in results for output in build["outputs"]}) - # If we have no outputs we're done - if not all_outputs: - return None - - # Used to verify if all rules are the same - all_one_rule = len( - [ - r - for r in results - if isinstance(r, dict) and r["rule"] == results[0]["rule"] - ] - ) == len(results) - dependencies = get_dependencies(node) - - if not all_one_rule: - # If they aren't all the same rule use scons to generate these - # outputs. At this time nothing hits this case. - return { - "outputs": all_outputs, - "rule": "SCONS", - "implicit": dependencies, - } + dependencies = list({dep for build in results for dep in build["implicit"]}) if results[0]["rule"] == "CMD": cmdline = "" @@ -322,7 +343,10 @@ def handle_list_action(self, node, action): ninja_build = { "outputs": all_outputs, "rule": "CMD", - "variables": {"cmd": cmdline}, + "variables": { + "cmd": cmdline, + "env": get_command_env(node.env if node.env else self.env), + }, "implicit": dependencies, } @@ -342,18 +366,10 @@ def handle_list_action(self, node, action): return { "outputs": all_outputs, "rule": "INSTALL", - "pool": "install_pool", "inputs": [get_path(src_file(s)) for s in node.sources], "implicit": dependencies, } - elif results[0]["rule"] == "SCONS": - return { - "outputs": all_outputs, - "rule": "SCONS", - "inputs": dependencies, - } - raise Exception("Unhandled list action with rule: " + results[0]["rule"]) @@ -369,7 +385,7 @@ def __init__(self, env, writer_class): self.generated_suffixes = env.get("NINJA_GENERATED_SOURCE_SUFFIXES", []) # List of generated builds that will be written at a later stage - self.builds = list() + self.builds = dict() # List of targets for which we have generated a build. This # allows us to take multiple Alias nodes as sources and to not @@ -383,7 +399,7 @@ def __init__(self, env, writer_class): escape = env.get("ESCAPE", lambda x: x) self.variables = { - "COPY": "cmd.exe /c copy" if sys.platform == "win32" else "cp", + "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format( sys.executable, " ".join( @@ -402,16 +418,46 @@ def __init__(self, env, writer_class): self.rules = { "CMD": { - "command": "cmd /c $cmd" if sys.platform == "win32" else "$cmd", + "command": "cmd /c $env$cmd $in $out" if sys.platform == "win32" else "$env$cmd $in $out", + "description": "Building $out", + "pool": "local_pool", + }, + "GENERATED_CMD": { + "command": "cmd /c $env$cmd" if sys.platform == "win32" else "$env$cmd", "description": "Building $out", + "pool": "local_pool", }, # We add the deps processing variables to this below. We - # don't pipe this through cmd.exe on Windows because we + # don't pipe these through cmd.exe on Windows because we # use this to generate a compile_commands.json database # which can't use the shell command as it's compile - # command. This does mean that we assume anything using - # CMD_W_DEPS is a straight up compile which is true today. - "CMD_W_DEPS": {"command": "$cmd", "description": "Building $out"}, + # command. + "CC": { + "command": "$env$CC @$out.rsp", + "description": "Compiling $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + }, + "CXX": { + "command": "$env$CXX @$out.rsp", + "description": "Compiling $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + }, + "LINK": { + "command": "$env$LINK @$out.rsp", + "description": "Linking $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + "pool": "local_pool", + }, + "AR": { + "command": "$env$AR @$out.rsp", + "description": "Archiving $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + "pool": "local_pool", + }, "SYMLINK": { "command": ( "cmd /c mklink $out $in" @@ -423,6 +469,7 @@ def __init__(self, env, writer_class): "INSTALL": { "command": "$COPY $in $out", "description": "Install $out", + "pool": "install_pool", # On Windows cmd.exe /c copy does not always correctly # update the timestamp on the output file. This leads # to a stuck constant timestamp in the Ninja database @@ -479,65 +526,43 @@ def __init__(self, env, writer_class): } self.pools = { + "local_pool": self.env.GetOption("num_jobs"), "install_pool": self.env.GetOption("num_jobs") / 2, "scons_pool": 1, } - if env["PLATFORM"] == "win32": - self.rules["CMD_W_DEPS"]["deps"] = "msvc" - else: - self.rules["CMD_W_DEPS"]["deps"] = "gcc" - self.rules["CMD_W_DEPS"]["depfile"] = "$out.d" - - self.rules.update(env.get(NINJA_RULES, {})) - self.pools.update(env.get(NINJA_POOLS, {})) - - def generate_builds(self, node): - """Generate a ninja build rule for node and it's children.""" - # Filter out nodes with no builder. They are likely source files - # and so no work needs to be done, it will be used in the - # generation for some real target. - # - # Note that all nodes have a builder attribute but it is sometimes - # set to None. So we cannot use a simpler hasattr check here. - if getattr(node, "builder", None) is None: - return + for rule in ["CC", "CXX"]: + if env["PLATFORM"] == "win32": + self.rules[rule]["deps"] = "msvc" + else: + self.rules[rule]["deps"] = "gcc" + self.rules[rule]["depfile"] = "$out.d" - stack = [[node]] - while stack: - frame = stack.pop() - for child in frame: - outputs = set(get_outputs(child)) - # Check if all the outputs are in self.built, if they - # are we've already seen this node and it's children. - if not outputs.isdisjoint(self.built): - continue + def add_build(self, node): + if not node.has_builder(): + return False - self.built = self.built.union(outputs) - stack.append(child.children()) - - if isinstance(child, SCons.Node.Alias.Alias): - build = alias_to_ninja_build(child) - elif node.builder is not None: - # Use False since None is a valid value for this attribute - build = getattr(child.attributes, NINJA_BUILD, False) - if build is False: - build = self.translator.action_to_ninja_build(child) - setattr(child.attributes, NINJA_BUILD, build) - else: - build = None + if isinstance(node, SCons.Node.Alias.Alias): + build = alias_to_ninja_build(node) + else: + build = self.translator.action_to_ninja_build(node) - # Some things are unbuild-able or need not be built in Ninja - if build is None or build == 0: - continue + # Some things are unbuild-able or need not be built in Ninja + if build is None: + return False - self.builds.append(build) + node_string = str(node) + if node_string in self.builds: + raise Exception("Node {} added to ninja build state more than once".format(node_string)) + self.builds[node_string] = build + self.built.update(build["outputs"]) + return True def is_generated_source(self, output): """Check if output ends with a known generated suffix.""" _, suffix = splitext(output) return suffix in self.generated_suffixes - + def has_generated_sources(self, output): """ Determine if output indicates this is a generated header file. @@ -548,7 +573,7 @@ def has_generated_sources(self, output): return False # pylint: disable=too-many-branches,too-many-locals - def generate(self, ninja_file, fallback_default_target=None): + def generate(self, ninja_file): """ Generate the build.ninja. @@ -557,6 +582,9 @@ def generate(self, ninja_file, fallback_default_target=None): if self.__generated: return + self.rules.update(self.env.get(NINJA_RULES, {})) + self.pools.update(self.env.get(NINJA_POOLS, {})) + content = io.StringIO() ninja = self.writer_class(content, width=100) @@ -571,10 +599,10 @@ def generate(self, ninja_file, fallback_default_target=None): for rule, kwargs in self.rules.items(): ninja.rule(rule, **kwargs) - generated_source_files = { + generated_source_files = sorted({ output # First find builds which have header files in their outputs. - for build in self.builds + for build in self.builds.values() if self.has_generated_sources(build["outputs"]) for output in build["outputs"] # Collect only the header files from the builds with them @@ -583,25 +611,24 @@ def generate(self, ninja_file, fallback_default_target=None): # here we need to filter so we only have the headers and # not the other outputs. if self.is_generated_source(output) - } + }) if generated_source_files: ninja.build( outputs="_generated_sources", rule="phony", - implicit=list(generated_source_files), + implicit=generated_source_files ) template_builders = [] - for build in self.builds: + for build in [self.builds[key] for key in sorted(self.builds.keys())]: if build["rule"] == "TEMPLATE": template_builders.append(build) continue - implicit = build.get("implicit", []) - implicit.append(ninja_file) - build["implicit"] = implicit + if "implicit" in build: + build["implicit"].sort() # Don't make generated sources depend on each other. We # have to check that none of the outputs are generated @@ -612,7 +639,7 @@ def generate(self, ninja_file, fallback_default_target=None): generated_source_files and not build["rule"] == "INSTALL" and set(build["outputs"]).isdisjoint(generated_source_files) - and set(implicit).isdisjoint(generated_source_files) + and set(build.get("implicit", [])).isdisjoint(generated_source_files) ): # Make all non-generated source targets depend on @@ -621,7 +648,11 @@ def generate(self, ninja_file, fallback_default_target=None): # generated source was rebuilt. We just need to make # sure that all of these sources are generated before # other builds. - build["order_only"] = "_generated_sources" + order_only = build.get("order_only", []) + order_only.append("_generated_sources") + build["order_only"] = order_only + if "order_only" in build: + build["order_only"].sort() # When using a depfile Ninja can only have a single output # but SCons will usually have emitted an output for every @@ -637,26 +668,31 @@ def generate(self, ninja_file, fallback_default_target=None): # Some rules like 'phony' and other builtins we don't have # listed in self.rules so verify that we got a result # before trying to check if it has a deps key. - if rule is not None and rule.get("deps"): - - # Anything using deps in Ninja can only have a single - # output, but we may have a build which actually - # produces multiple outputs which other targets can - # depend on. Here we slice up the outputs so we have a - # single output which we will use for the "real" - # builder and multiple phony targets that match the - # file names of the remaining outputs. This way any - # build can depend on any output from any build. - first_output, remaining_outputs = build["outputs"][0], build["outputs"][1:] + # + # Anything using deps or rspfile in Ninja can only have a single + # output, but we may have a build which actually produces + # multiple outputs which other targets can depend on. Here we + # slice up the outputs so we have a single output which we will + # use for the "real" builder and multiple phony targets that + # match the file names of the remaining outputs. This way any + # build can depend on any output from any build. + build["outputs"].sort() + if rule is not None and (rule.get("deps") or rule.get("rspfile")): + first_output, remaining_outputs = ( + build["outputs"][0], + build["outputs"][1:], + ) + if remaining_outputs: ninja.build( - outputs=remaining_outputs, - rule="phony", - implicit=first_output, + outputs=remaining_outputs, rule="phony", implicit=first_output, ) build["outputs"] = first_output + if "inputs" in build: + build["inputs"].sort() + ninja.build(**build) template_builds = dict() @@ -682,37 +718,37 @@ def generate(self, ninja_file, fallback_default_target=None): if template_builds.get("outputs", []): ninja.build(**template_builds) - # Here to teach the ninja file how to regenerate itself. We'll - # never see ourselves in the DAG walk so we can't rely on - # action_to_ninja_build to generate this rule + # We have to glob the SCons files here to teach the ninja file + # how to regenerate itself. We'll never see ourselves in the + # DAG walk so we can't rely on action_to_ninja_build to + # generate this rule even though SCons should know we're + # dependent on SCons files. + # + # TODO: We're working on getting an API into SCons that will + # allow us to query the actual SConscripts used. Right now + # this glob method has deficiencies like skipping + # jstests/SConscript and being specific to the MongoDB + # repository layout. ninja.build( - ninja_file, + self.env.File(ninja_file).path, rule="REGENERATE", implicit=[ - self.env.File("#SConstruct").get_abspath(), - os.path.abspath(__file__), + self.env.File("#SConstruct").path, + __file__, ] - + LOADED_SCONSCRIPTS, + + sorted(glob("src/**/SConscript", recursive=True)), ) - ninja.build( - "scons-invocation", - rule="CMD", - pool="console", - variables={"cmd": "echo $SCONS_INVOCATION_W_TARGETS"}, - ) - - # Note the use of CMD_W_DEPS below. CMD_W_DEPS are always - # compile commands in this generator. If we ever change the - # name/s of the rules that include compile commands - # (i.e. something like CC/CXX) we will need to update this - # build to reflect that complete list. + # If we ever change the name/s of the rules that include + # compile commands (i.e. something like CC) we will need to + # update this build to reflect that complete list. ninja.build( "compile_commands.json", rule="CMD", pool="console", + implicit=[ninja_file], variables={ - "cmd": "ninja -f {} -t compdb CMD_W_DEPS > compile_commands.json".format( + "cmd": "ninja -f {} -t compdb CC CXX > compile_commands.json".format( ninja_file ) }, @@ -736,11 +772,6 @@ def generate(self, ninja_file, fallback_default_target=None): if scons_default_targets: ninja.default(" ".join(scons_default_targets)) - # If not then set the default to the fallback_default_target we were given. - # Otherwise we won't create a default ninja target. - elif fallback_default_target is not None: - ninja.default(fallback_default_target) - with open(ninja_file, "w") as build_ninja: build_ninja.write(content.getvalue()) @@ -776,9 +807,158 @@ def src_file(node): return get_path(node) -# TODO: Make the Rules smarter. Instead of just using a "cmd" rule -# everywhere we should be smarter about generating CC, CXX, LINK, -# etc. rules +def get_comstr(env, action, targets, sources): + """Get the un-substituted string for action.""" + # Despite being having "list" in it's name this member is not + # actually a list. It's the pre-subst'd string of the command. We + # use it to determine if the command we're about to generate needs + # to use a custom Ninja rule. By default this redirects CC, CXX, + # AR, SHLINK, and LINK commands to their respective rules but the + # user can inject custom Ninja rules and tie them to commands by + # using their pre-subst'd string. + if hasattr(action, "process"): + return action.cmd_list + + return action.genstring(targets, sources, env) + + +def get_command_env(env): + """ + Return a string that sets the enrivonment for any environment variables that + differ between the OS environment and the SCons command ENV. + + It will be compatible with the default shell of the operating system. + """ + try: + return env["NINJA_ENV_VAR_CACHE"] + except KeyError: + pass + + # Scan the ENV looking for any keys which do not exist in + # os.environ or differ from it. We assume if it's a new or + # differing key from the process environment then it's + # important to pass down to commands in the Ninja file. + ENV = get_default_ENV(env) + scons_specified_env = { + key: value + for key, value in ENV.items() + if key not in os.environ or os.environ.get(key, None) != value + } + + windows = env["PLATFORM"] == "win32" + command_env = "" + for key, value in scons_specified_env.items(): + # Ensure that the ENV values are all strings: + if is_List(value): + # If the value is a list, then we assume it is a + # path list, because that's a pretty common list-like + # value to stick in an environment variable: + value = flatten_sequence(value) + value = joinpath(map(str, value)) + else: + # If it isn't a string or a list, then we just coerce + # it to a string, which is the proper way to handle + # Dir and File instances and will produce something + # reasonable for just about everything else: + value = str(value) + + if windows: + command_env += "set '{}={}' && ".format(key, value) + else: + command_env += "{}={} ".format(key, value) + + env["NINJA_ENV_VAR_CACHE"] = command_env + return command_env + + +def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False): + """Generate a response file command provider for rule name.""" + + # If win32 using the environment with a response file command will cause + # ninja to fail to create the response file. Additionally since these rules + # generally are not piping through cmd.exe /c any environment variables will + # make CreateProcess fail to start. + # + # On POSIX we can still set environment variables even for compile + # commands so we do so. + use_command_env = not env["PLATFORM"] == "win32" + if "$" in tool: + tool_is_dynamic = True + + def get_response_file_command(env, node, action, targets, sources, executor=None): + if hasattr(action, "process"): + cmd_list, _, _ = action.process(targets, sources, env, executor=executor) + cmd_list = [str(c).replace("$", "$$") for c in cmd_list[0]] + else: + command = generate_command( + env, node, action, targets, sources, executor=executor + ) + cmd_list = shlex.split(command) + + if tool_is_dynamic: + tool_command = env.subst( + tool, target=targets, source=sources, executor=executor + ) + else: + tool_command = tool + + try: + # Add 1 so we always keep the actual tool inside of cmd + tool_idx = cmd_list.index(tool_command) + 1 + except ValueError: + raise Exception( + "Could not find tool {} in {} generated from {}".format( + tool_command, cmd_list, get_comstr(env, action, targets, sources) + ) + ) + + cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] + rsp_content = " ".join(rsp_content) + + variables = {"rspc": rsp_content} + variables[rule] = cmd + if use_command_env: + variables["env"] = get_command_env(env) + return rule, variables + + return get_response_file_command + + +def generate_command(env, node, action, targets, sources, executor=None): + # Actions like CommandAction have a method called process that is + # used by SCons to generate the cmd_line they need to run. So + # check if it's a thing like CommandAction and call it if we can. + if hasattr(action, "process"): + cmd_list, _, _ = action.process(targets, sources, env, executor=executor) + cmd = _string_from_cmd_list(cmd_list[0]) + else: + # Anything else works with genstring, this is most commonly hit by + # ListActions which essentially call process on all of their + # commands and concatenate it for us. + genstring = action.genstring(targets, sources, env) + if executor is not None: + cmd = env.subst(genstring, executor=executor) + else: + cmd = env.subst(genstring, targets, sources) + + cmd = cmd.replace("\n", " && ").strip() + if cmd.endswith("&&"): + cmd = cmd[0:-2].strip() + + # Escape dollars as necessary + return cmd.replace("$", "$$") + + +def get_shell_command(env, node, action, targets, sources, executor=None): + return ( + "GENERATED_CMD", + { + "cmd": generate_command(env, node, action, targets, sources, executor=None), + "env": get_command_env(env), + }, + ) + + def get_command(env, node, action): # pylint: disable=too-many-branches """Get the command to execute for node.""" if node.env: @@ -800,121 +980,26 @@ def get_command(env, node, action): # pylint: disable=too-many-branches # Retrieve the repository file for all sources slist = [rfile(s) for s in slist] - # Get the dependencies for all targets - implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) - # Generate a real CommandAction if isinstance(action, SCons.Action.CommandGeneratorAction): # pylint: disable=protected-access action = action._generate(tlist, slist, sub_env, 1, executor=executor) - rule = "CMD" - - # Actions like CommandAction have a method called process that is - # used by SCons to generate the cmd_line they need to run. So - # check if it's a thing like CommandAction and call it if we can. - if hasattr(action, "process"): - cmd_list, _, _ = action.process(tlist, slist, sub_env, executor=executor) - - # Despite being having "list" in it's name this member is not - # actually a list. It's the pre-subst'd string of the command. We - # use it to determine if the command we generated needs to use a - # custom Ninja rule. By default this redirects CC/CXX commands to - # CMD_W_DEPS but the user can inject custom Ninja rules and tie - # them to commands by using their pre-subst'd string. - rule = __NINJA_RULE_MAPPING.get(action.cmd_list, "CMD") - - cmd = _string_from_cmd_list(cmd_list[0]) - else: - # Anything else works with genstring, this is most commonly hit by - # ListActions which essentially call process on all of their - # commands and concatenate it for us. - genstring = action.genstring(tlist, slist, sub_env) + variables = {} - # Detect if we have a custom rule for this - # "ListActionCommandAction" type thing. - rule = __NINJA_RULE_MAPPING.get(genstring, "CMD") - - if executor is not None: - cmd = sub_env.subst(genstring, executor=executor) - else: - cmd = sub_env.subst(genstring, target=tlist, source=slist) - - # Since we're only enabling Ninja for developer builds right - # now we skip all Manifest related work on Windows as it's not - # necessary. We shouldn't have gotten here but on Windows - # SCons has a ListAction which shows as a - # CommandGeneratorAction for linking. That ListAction ends - # with a FunctionAction (embedManifestExeCheck, - # embedManifestDllCheck) that simply say "does - # target[0].manifest exist?" if so execute the real command - # action underlying me, otherwise do nothing. - # - # Eventually we'll want to find a way to translate this to - # Ninja but for now, and partially because the existing Ninja - # generator does so, we just disable it all together. - cmd = cmd.replace("\n", " && ").strip() - if env["PLATFORM"] == "win32" and ( - "embedManifestExeCheck" in cmd or "embedManifestDllCheck" in cmd - ): - cmd = " && ".join(cmd.split(" && ")[0:-1]) - - if cmd.endswith("&&"): - cmd = cmd[0:-2].strip() - - outputs = get_outputs(node) - command_env = "" - windows = env["PLATFORM"] == "win32" - - # If win32 and rule == CMD_W_DEPS then we don't want to calculate - # an environment for this command. It's a compile command and - # compiledb doesn't support shell syntax on Windows. We need the - # shell syntax to use environment variables on Windows so we just - # skip this platform / rule combination to keep the compiledb - # working. - # - # On POSIX we can still set environment variables even for compile - # commands so we do so. - if not (windows and rule == "CMD_W_DEPS"): - - # Scan the ENV looking for any keys which do not exist in - # os.environ or differ from it. We assume if it's a new or - # differing key from the process environment then it's - # important to pass down to commands in the Ninja file. - ENV = get_default_ENV(sub_env) - scons_specified_env = { - key: value - for key, value in ENV.items() - if key not in os.environ or os.environ.get(key, None) != value - } + comstr = get_comstr(sub_env, action, tlist, slist) + if not comstr: + return None - for key, value in scons_specified_env.items(): - # Ensure that the ENV values are all strings: - if is_List(value): - # If the value is a list, then we assume it is a - # path list, because that's a pretty common list-like - # value to stick in an environment variable: - value = flatten_sequence(value) - value = joinpath(map(str, value)) - else: - # If it isn't a string or a list, then we just coerce - # it to a string, which is the proper way to handle - # Dir and File instances and will produce something - # reasonable for just about everything else: - value = str(value) - - if windows: - command_env += "set '{}={}' && ".format(key, value) - else: - command_env += "{}={} ".format(key, value) + provider = __NINJA_RULE_MAPPING.get(comstr, get_shell_command) + rule, variables = provider(sub_env, node, action, tlist, slist, executor=executor) - variables = {"cmd": command_env + cmd} - extra_vars = getattr(node.attributes, "NINJA_EXTRA_VARS", {}) - if extra_vars: - variables.update(extra_vars) + # Get the dependencies for all targets + implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) ninja_build = { - "outputs": outputs, + "order_only": get_order_only(node), + "outputs": get_outputs(node), "inputs": get_inputs(node), "implicit": implicit, "rule": rule, @@ -953,37 +1038,30 @@ def ninja_builder(env, target, source): # here. print("Generating:", str(target[0])) - # The environment variable NINJA_SYNTAX points to the - # ninja_syntax.py module from the ninja sources found here: - # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py - # - # This should be vendored into the build sources and it's location - # set in NINJA_SYNTAX. This code block loads the location from - # that variable, gets the absolute path to the vendored file, gets - # it's parent directory then uses importlib to import the module - # dynamically. - ninja_syntax_file = env[NINJA_SYNTAX] - if isinstance(ninja_syntax_file, str): - ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() - ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) - sys.path.append(ninja_syntax_mod_dir) - ninja_syntax_mod_name = os.path.basename(ninja_syntax_file) - ninja_syntax = importlib.import_module(ninja_syntax_mod_name.replace(".py", "")) - - suffix = env.get("NINJA_SUFFIX", "") - if suffix and not suffix[0] == ".": - suffix = "." + suffix + generated_build_ninja = target[0].get_abspath() + NINJA_STATE.generate(generated_build_ninja) + if env.get("DISABLE_AUTO_NINJA") != True: + print("Executing:", str(target[0])) - generated_build_ninja = target[0].get_abspath() + suffix - ninja_state = NinjaState(env, ninja_syntax.Writer) - - for src in source: - ninja_state.generate_builds(src) - - ninja_state.generate(generated_build_ninja, str(source[0])) - - return 0 + def execute_ninja(): + proc = subprocess.Popen( ['ninja', '-f', generated_build_ninja], + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + universal_newlines=True + ) + for stdout_line in iter(proc.stdout.readline, ""): + yield stdout_line + proc.stdout.close() + return_code = proc.wait() + if return_code: + raise subprocess.CalledProcessError(return_code, 'ninja') + + for output in execute_ninja(): + output = output.strip() + sys.stdout.write('\x1b[2K') # erase previous line + sys.stdout.write(output + "\r") + sys.stdout.flush() # pylint: disable=too-few-public-methods class AlwaysExecAction(SCons.Action.FunctionAction): @@ -994,25 +1072,6 @@ def __call__(self, *args, **kwargs): return super().__call__(*args, **kwargs) -def ninja_print(_cmd, target, _source, env): - """Tag targets with the commands to build them.""" - if target: - for tgt in target: - if ( - tgt.has_builder() - # Use 'is False' because not would still trigger on - # None's which we don't want to regenerate - and getattr(tgt.attributes, NINJA_BUILD, False) is False - and isinstance(tgt.builder.action, COMMAND_TYPES) - ): - ninja_action = get_command(env, tgt, tgt.builder.action) - setattr(tgt.attributes, NINJA_BUILD, ninja_action) - # Preload the attributes dependencies while we're still running - # multithreaded - get_dependencies(tgt) - return 0 - - def register_custom_handler(env, name, handler): """Register a custom handler for SCons function actions.""" env[NINJA_CUSTOM_HANDLERS][name] = handler @@ -1024,7 +1083,7 @@ def register_custom_rule_mapping(env, pre_subst_string, rule): __NINJA_RULE_MAPPING[pre_subst_string] = rule -def register_custom_rule(env, rule, command, description="", deps=None): +def register_custom_rule(env, rule, command, description="", deps=None, pool=None): """Allows specification of Ninja rules from inside SCons files.""" rule_obj = { "command": command, @@ -1034,6 +1093,9 @@ def register_custom_rule(env, rule, command, description="", deps=None): if deps is not None: rule_obj["deps"] = deps + if pool is not None: + rule_obj["pool"] = pool + env[NINJA_RULES][rule] = rule_obj @@ -1075,11 +1137,6 @@ def ninja_stat(_self, path): running in a no_exec build the file system state should not change. For these reasons we patch SCons.Node.FS.LocalFS.stat to use our eternal memoized dictionary. - - Since this is happening during the Node walk it's being run while - threaded, we have to protect adding to the memoized dictionary - with a threading.Lock otherwise many targets miss the memoization - due to racing. """ global NINJA_STAT_MEMO @@ -1091,9 +1148,7 @@ def ninja_stat(_self, path): except os.error: result = None - with MEMO_LOCK: - NINJA_STAT_MEMO[path] = result - + NINJA_STAT_MEMO[path] = result return result @@ -1145,71 +1200,11 @@ def ninja_always_serial(self, num, taskmaster): self.job = SCons.Job.Serial(taskmaster) -class NinjaEternalTempFile(SCons.Platform.TempFileMunge): +class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" def __call__(self, target, source, env, for_signature): - if for_signature: - return self.cmd - - node = target[0] if SCons.Util.is_List(target) else target - if node is not None: - cmdlist = getattr(node.attributes, "tempfile_cmdlist", None) - if cmdlist is not None: - return cmdlist - - cmd = super().__call__(target, source, env, for_signature) - - # If TempFileMunge.__call__ returns a string it means that no - # response file was needed. No processing required so just - # return the command. - if isinstance(cmd, str): - return cmd - - # Strip the removal commands from the command list. - # - # SCons' TempFileMunge class has some very strange - # behavior where it, as part of the command line, tries to - # delete the response file after executing the link - # command. We want to keep those response files since - # Ninja will keep using them over and over. The - # TempFileMunge class creates a cmdlist to do this, a - # common SCons convention for executing commands see: - # https://github.com/SCons/scons/blob/master/src/engine/SCons/Action.py#L949 - # - # This deletion behavior is not configurable. So we wanted - # to remove the deletion command from the command list by - # simply slicing it out here. Unfortunately for some - # strange reason TempFileMunge doesn't make the "rm" - # command it's own list element. It appends it to the - # tempfile argument to cmd[0] (which is CC/CXX) and then - # adds the tempfile again as it's own element. - # - # So we just kind of skip that middle element. Since the - # tempfile is in the command list on it's own at the end we - # can cut it out entirely. This is what I would call - # "likely to break" in future SCons updates. Hopefully it - # breaks because they start doing the right thing and not - # weirdly splitting these arguments up. For reference a - # command list that we get back from the OG TempFileMunge - # looks like this: - # - # [ - # 'g++', - # '@/mats/tempfiles/random_string.lnk\nrm', - # '/mats/tempfiles/random_string.lnk', - # ] - # - # Note the weird newline and rm command in the middle - # element and the lack of TEMPFILEPREFIX on the last - # element. - prefix = env.subst("$TEMPFILEPREFIX") - if not prefix: - prefix = "@" - - new_cmdlist = [cmd[0], prefix + cmd[-1]] - setattr(node.attributes, "tempfile_cmdlist", new_cmdlist) - return new_cmdlist + return self.cmd def _print_cmd_str(*_args, **_kwargs): """Disable this method""" @@ -1229,9 +1224,22 @@ def exists(env): return True +added = None def generate(env): """Generate the NINJA builders.""" + from SCons.Script import AddOption, GetOption + global added + if not added: + added = 1 + AddOption('--disable-auto-ninja', + dest='disable_auto_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + env["DISABLE_AUTO_NINJA"] = GetOption('disable_auto_ninja') + env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") # Add the Ninja builder. @@ -1239,6 +1247,15 @@ def generate(env): ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action) env.Append(BUILDERS={"Ninja": ninja_builder_obj}) + env["NINJA_PREFIX"] = env.get("NINJA_PREFIX", "build") + env["NINJA_SUFFIX"] = env.get("NINJA_SUFFIX", "ninja") + env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") + + ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") + ninja_file = env.Ninja(target=ninja_file_name, source=[]) + env.AlwaysBuild(ninja_file) + env.Alias("$NINJA_ALIAS_NAME", ninja_file) + # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. if env["PLATFORM"] == "win32": @@ -1246,6 +1263,11 @@ def generate(env): else: env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) + # Provide a way for custom rule authors to easily access command + # generation. + env.AddMethod(get_shell_command, "NinjaGetShellCommand") + env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider") + # Provides a way for users to handle custom FunctionActions they # want to translate to Ninja. env[NINJA_CUSTOM_HANDLERS] = {} @@ -1270,8 +1292,40 @@ def generate(env): # deleted you would get a very subtly incorrect Ninja file and # might not catch it. env.AddMethod(register_custom_rule_mapping, "NinjaRuleMapping") - env.NinjaRuleMapping("${CCCOM}", "CMD_W_DEPS") - env.NinjaRuleMapping("${CXXCOM}", "CMD_W_DEPS") + + # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks + # without relying on the SCons hacks that SCons uses by default. + if env["PLATFORM"] == "win32": + from SCons.Tool.mslink import compositeLinkAction + + if env["LINKCOM"] == compositeLinkAction: + env[ + "LINKCOM" + ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' + env[ + "SHLINKCOM" + ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' + + # Normally in SCons actions for the Program and *Library builders + # will return "${*COM}" as their pre-subst'd command line. However + # if a user in a SConscript overwrites those values via key access + # like env["LINKCOM"] = "$( $ICERUN $)" + env["LINKCOM"] then + # those actions no longer return the "bracketted" string and + # instead return something that looks more expanded. So to + # continue working even if a user has done this we map both the + # "bracketted" and semi-expanded versions. + def robust_rule_mapping(var, rule, tool): + provider = gen_get_response_file_command(env, rule, tool) + env.NinjaRuleMapping("${" + var + "}", provider) + env.NinjaRuleMapping(env[var], provider) + + robust_rule_mapping("CCCOM", "CC", "$CC") + robust_rule_mapping("SHCCCOM", "CC", "$CC") + robust_rule_mapping("CXXCOM", "CXX", "$CXX") + robust_rule_mapping("SHCXXCOM", "CXX", "$CXX") + robust_rule_mapping("LINKCOM", "LINK", "$LINK") + robust_rule_mapping("SHLINKCOM", "LINK", "$SHLINK") + robust_rule_mapping("ARCOM", "AR", "$AR") # Make SCons node walk faster by preventing unnecessary work env.Decider("timestamp-match") @@ -1281,6 +1335,21 @@ def generate(env): # dependencies to any builds that *might* use them. env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] + if env["PLATFORM"] != "win32" and env.get("RANLIBCOM"): + # There is no way to translate the ranlib list action into + # Ninja so add the s flag and disable ranlib. + # + # This is equivalent to Meson. + # https://github.com/mesonbuild/meson/blob/master/mesonbuild/linkers.py#L143 + old_arflags = str(env["ARFLAGS"]) + if "s" not in old_arflags: + old_arflags += "s" + + env["ARFLAGS"] = SCons.Util.CLVar([old_arflags]) + + # Disable running ranlib, since we added 's' above + env["RANLIBCOM"] = "" + # This is the point of no return, anything after this comment # makes changes to SCons that are irreversible and incompatible # with a normal SCons build. We return early if __NINJA_NO=1 has @@ -1304,7 +1373,9 @@ def generate(env): SCons.Node.FS.File.prepare = ninja_noop SCons.Node.FS.File.push_to_cache = ninja_noop SCons.Executor.Executor.prepare = ninja_noop + SCons.Taskmaster.Task.prepare = ninja_noop SCons.Node.FS.File.built = ninja_noop + SCons.Node.Node.visited = ninja_noop # We make lstat a no-op because it is only used for SONAME # symlinks which we're not producing. @@ -1336,20 +1407,8 @@ def generate(env): SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig) - # Replace false Compiling* messages with a more accurate output - # - # We also use this to tag all Nodes with Builders using - # CommandActions with the final command that was used to compile - # it for passing to Ninja. If we don't inject this behavior at - # this stage in the build too much state is lost to generate the - # command at the actual ninja_builder execution time for most - # commands. - # - # We do attempt command generation again in ninja_builder if it - # hasn't been tagged and it seems to work for anything that - # doesn't represent as a non-FunctionAction during the print_func - # call. - env["PRINT_CMD_LINE_FUNC"] = ninja_print + # Replace false action messages with nothing. + env["PRINT_CMD_LINE_FUNC"] = ninja_noop # This reduces unnecessary subst_list calls to add the compiler to # the implicit dependencies of targets. Since we encode full paths @@ -1358,11 +1417,6 @@ def generate(env): # where we expect it. env["IMPLICIT_COMMAND_DEPENDENCIES"] = False - # Set build to no_exec, our sublcass of FunctionAction will force - # an execution for ninja_builder so this simply effects all other - # Builders. - env.SetOption("no_exec", True) - # This makes SCons more aggressively cache MD5 signatures in the # SConsign file. env.SetOption("max_drift", 1) @@ -1372,6 +1426,84 @@ def generate(env): # monkey the Jobs constructor to only use the Serial Job class. SCons.Job.Jobs.__init__ = ninja_always_serial + # The environment variable NINJA_SYNTAX points to the + # ninja_syntax.py module from the ninja sources found here: + # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py + # + # This should be vendored into the build sources and it's location + # set in NINJA_SYNTAX. This code block loads the location from + # that variable, gets the absolute path to the vendored file, gets + # it's parent directory then uses importlib to import the module + # dynamically. + ninja_syntax_file = env[NINJA_SYNTAX] + + if os.path.exists(ninja_syntax_file): + if isinstance(ninja_syntax_file, str): + ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() + ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) + sys.path.append(ninja_syntax_mod_dir) + ninja_syntax_mod_name = os.path.basename(ninja_syntax_file).replace(".py", "") + ninja_syntax = importlib.import_module(ninja_syntax_mod_name) + else: + ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') + + global NINJA_STATE + NINJA_STATE = NinjaState(env, ninja_syntax.Writer) + + # Here we will force every builder to use an emitter which makes the ninja + # file depend on it's target. This forces the ninja file to the bottom of + # the DAG which is required so that we walk every target, and therefore add + # it to the global NINJA_STATE, before we try to write the ninja file. + def ninja_file_depends_on_all(target, source, env): + if not any("conftest" in str(t) for t in target): + env.Depends(ninja_file, target) + return target, source + + # The "Alias Builder" isn't in the BUILDERS map so we have to + # modify it directly. + SCons.Environment.AliasBuilder.emitter = ninja_file_depends_on_all + + for _, builder in env["BUILDERS"].items(): + try: + emitter = builder.emitter + if emitter is not None: + builder.emitter = SCons.Builder.ListEmitter( + [emitter, ninja_file_depends_on_all] + ) + else: + builder.emitter = ninja_file_depends_on_all + # Users can inject whatever they want into the BUILDERS + # dictionary so if the thing doesn't have an emitter we'll + # just ignore it. + except AttributeError: + pass + + # Here we monkey patch the Task.execute method to not do a bunch of + # unnecessary work. If a build is a regular builder (i.e not a conftest and + # not our own Ninja builder) then we add it to the NINJA_STATE. Otherwise we + # build it like normal. This skips all of the caching work that this method + # would normally do since we aren't pulling any of these targets from the + # cache. + # + # In the future we may be able to use this to actually cache the build.ninja + # file once we have the upstream support for referencing SConscripts as File + # nodes. + def ninja_execute(self): + global NINJA_STATE + + target = self.targets[0] + target_name = str(target) + if target_name != ninja_file_name and "conftest" not in target_name: + NINJA_STATE.add_build(target) + else: + target.build() + + SCons.Taskmaster.Task.execute = ninja_execute + + # Make needs_execute always return true instead of determining out of + # date-ness. + SCons.Script.Main.BuildTask.needs_execute = lambda x: True + # We will eventually need to overwrite TempFileMunge to make it # handle persistent tempfiles or get an upstreamed change to add # some configurability to it's behavior in regards to tempfiles. @@ -1379,18 +1511,13 @@ def generate(env): # Set all three environment variables that Python's # tempfile.mkstemp looks at as it behaves differently on different # platforms and versions of Python. - os.environ["TMPDIR"] = env.Dir("$BUILD_DIR/response_files").get_abspath() + build_dir = env.subst("$BUILD_DIR") + if build_dir == "": + build_dir = "." + os.environ["TMPDIR"] = env.Dir("{}/.response_files".format(build_dir)).get_abspath() os.environ["TEMP"] = os.environ["TMPDIR"] os.environ["TMP"] = os.environ["TMPDIR"] if not os.path.isdir(os.environ["TMPDIR"]): env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) - env["TEMPFILE"] = NinjaEternalTempFile - - # Force the SConsign to be written, we benefit from SCons caching of - # implicit dependencies and conftests. Unfortunately, we have to do this - # using an atexit handler because SCons will not write the file when in a - # no_exec build. - import atexit - - atexit.register(SCons.SConsign.write) + env["TEMPFILE"] = NinjaNoResponseFiles \ No newline at end of file diff --git a/test/ninja/CC.py b/test/ninja/CC.py new file mode 100644 index 0000000000..fe18721b9d --- /dev/null +++ b/test/ninja/CC.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import os +import sys +import TestSCons + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +test = TestSCons.TestSCons() + +test.dir_fixture('ninja-fixture') + +test.write('SConstruct', """ +env = Environment() +env.Tool('ninja') +env.Program(target = 'foo', source = 'foo.c') +""" % locals()) + +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) + +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed foo.o', + 'Removed foo', + 'Removed build.ninja']) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/ninja-fixture/test2.C b/test/ninja/ninja-fixture/test2.C new file mode 100644 index 0000000000..a1ee9e32b9 --- /dev/null +++ b/test/ninja/ninja-fixture/test2.C @@ -0,0 +1,3 @@ +This is a .C file. +/*cc*/ +/*link*/ From c715e94831dfd8249379eb2f648522a9a7cbdc56 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 6 May 2020 13:40:11 -0400 Subject: [PATCH 100/163] added some more test and update ninja tool to handle commands --- src/engine/SCons/Tool/ninja.py | 15 ++++++- test/ninja/CC.py | 66 ----------------------------- test/ninja/copy_function_command.py | 43 ++++++++----------- test/ninja/generate_and_build.py | 42 ++++++++---------- test/ninja/shell_command.py | 51 ++++++++++------------ 5 files changed, 70 insertions(+), 147 deletions(-) delete mode 100644 test/ninja/CC.py diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index d1cbafa495..e0129d8bcd 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -692,7 +692,7 @@ def generate(self, ninja_file): if "inputs" in build: build["inputs"].sort() - + ninja.build(**build) template_builds = dict() @@ -990,7 +990,7 @@ def get_command(env, node, action): # pylint: disable=too-many-branches comstr = get_comstr(sub_env, action, tlist, slist) if not comstr: return None - + provider = __NINJA_RULE_MAPPING.get(comstr, get_shell_command) rule, variables = provider(sub_env, node, action, tlist, slist, executor=executor) @@ -1478,6 +1478,17 @@ def ninja_file_depends_on_all(target, source, env): except AttributeError: pass + # We will subvert the normal Command to make sure all targets generated + # from commands will be linked to the ninja file + SconsCommand = SCons.Environment.Environment.Command + + def NinjaCommand(self, target, source, action, **kw): + targets = SconsCommand(env, target, source, action, **kw) + env.Depends(ninja_file, targets) + return targets + + SCons.Environment.Environment.Command = NinjaCommand + # Here we monkey patch the Task.execute method to not do a bunch of # unnecessary work. If a build is a regular builder (i.e not a conftest and # not our own Ninja builder) then we add it to the NINJA_STATE. Otherwise we diff --git a/test/ninja/CC.py b/test/ninja/CC.py deleted file mode 100644 index fe18721b9d..0000000000 --- a/test/ninja/CC.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -import os -import sys -import TestSCons - -_python_ = TestSCons._python_ -_exe = TestSCons._exe - -test = TestSCons.TestSCons() - -test.dir_fixture('ninja-fixture') - -test.write('SConstruct', """ -env = Environment() -env.Tool('ninja') -env.Program(target = 'foo', source = 'foo.c') -""" % locals()) - -test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) - -test.run(arguments='-c', stdout=None) -test.must_contain_all_lines(test.stdout(), [ - 'Removed foo.o', - 'Removed foo', - 'Removed build.ninja']) -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) - -test.pass_test() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index 8e7acff7b7..f86f717e6f 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -25,28 +25,21 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import sys import TestSCons -from TestCmd import IS_WINDOWS - -test = TestSCons.TestSCons() - -try: - import ninja -except ImportError: - test.skip_test("Could not find module in python") _python_ = TestSCons._python_ _exe = TestSCons._exe -ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - 'ninja' + _exe)) +test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + test.write('SConstruct', """ env = Environment() env.Tool('ninja') @@ -56,28 +49,28 @@ # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('foo'), stdout="foo.c") +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo2.o', 'Removed foo2.c', - 'Removed foo' + _exe, + 'Removed foo', 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-execute-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_not_exist(test.workpath('foo')) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) # run ninja independently -program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('foo'), stdout="foo.c") +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) test.pass_test() diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index faf395a1ef..fd111a29be 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -25,41 +25,32 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import sys import TestSCons -from TestCmd import IS_WINDOWS - -test = TestSCons.TestSCons() - -try: - import ninja -except ImportError: - test.skip_test("Could not find module in python") _python_ = TestSCons._python_ _exe = TestSCons._exe -ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - 'ninja' + _exe)) +test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + test.write('SConstruct', """ env = Environment() env.Tool('ninja') env.Program(target = 'foo', source = 'foo.c') """) - # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('foo' + _exe), stdout="foo.c") +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -69,14 +60,15 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-execute-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_not_exist(test.workpath('foo' + _exe)) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) # run ninja independently -program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('foo' + _exe), stdout="foo.c") +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) test.pass_test() diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index 5d7f97e215..fd0e35f8de 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -25,61 +25,54 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import sys import TestSCons -from TestCmd import IS_WINDOWS - -test = TestSCons.TestSCons() - -try: - import ninja -except ImportError: - test.skip_test("Could not find module in python") _python_ = TestSCons._python_ _exe = TestSCons._exe -ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - 'ninja' + _exe)) +test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -shell = '' if IS_WINDOWS else './' +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") test.write('SConstruct', """ env = Environment() env.Tool('ninja') -prog = env.Program(target = 'foo', source = 'foo.c') -env.Command('foo.out', prog, '%(shell)sfoo%(_exe)s > foo.out') -""" % locals()) +env.Program(target = 'foo', source = 'foo.c') +env.Command('foo.out', ['foo'], './foo > foo.out') +""") # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.must_match('foo.out', 'foo.c') +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_match('foo.out', 'foo.c' + os.linesep) # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo.o', - 'Removed foo%(_exe)s' % locals(), + 'Removed foo', 'Removed foo.out', 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-execute-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_not_exist(test.workpath('foo.out')) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) # run ninja independently -program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.must_match('foo.out', 'foo.c') +test.run(program = ninja, stdout=None) +test.must_match('foo.out', 'foo.c' + os.linesep) + + test.pass_test() From b7bf996569174458d20edde3baa33243923f5a3b Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 6 May 2020 17:23:53 -0400 Subject: [PATCH 101/163] update to pass import.py test and support multiple environments --- src/engine/SCons/Tool/ninja.py | 41 ++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index e0129d8bcd..63ba243c61 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -274,6 +274,9 @@ def handle_func_action(self, node, action): "outputs": get_outputs(node), "implicit": get_dependencies(node, skip_sources=True), } + if name == 'ninja_builder': + return None + handler = self.func_handlers.get(name, None) if handler is not None: return handler(node.env if node.env else self.env, node) @@ -377,8 +380,9 @@ def handle_list_action(self, node, action): class NinjaState: """Maintains state of Ninja build system as it's translated from SCons.""" - def __init__(self, env, writer_class): + def __init__(self, env, ninja_file, writer_class): self.env = env + self.ninja_file = ninja_file self.writer_class = writer_class self.__generated = False self.translator = SConsToNinjaTranslator(env) @@ -573,7 +577,7 @@ def has_generated_sources(self, output): return False # pylint: disable=too-many-branches,too-many-locals - def generate(self, ninja_file): + def generate(self): """ Generate the build.ninja. @@ -730,7 +734,7 @@ def generate(self, ninja_file): # jstests/SConscript and being specific to the MongoDB # repository layout. ninja.build( - self.env.File(ninja_file).path, + self.ninja_file.path, rule="REGENERATE", implicit=[ self.env.File("#SConstruct").path, @@ -746,10 +750,10 @@ def generate(self, ninja_file): "compile_commands.json", rule="CMD", pool="console", - implicit=[ninja_file], + implicit=[str(self.ninja_file)], variables={ "cmd": "ninja -f {} -t compdb CC CXX > compile_commands.json".format( - ninja_file + str(self.ninja_file) ) }, ) @@ -772,7 +776,7 @@ def generate(self, ninja_file): if scons_default_targets: ninja.default(" ".join(scons_default_targets)) - with open(ninja_file, "w") as build_ninja: + with open(str(self.ninja_file), "w") as build_ninja: build_ninja.write(content.getvalue()) self.__generated = True @@ -1039,7 +1043,7 @@ def ninja_builder(env, target, source): print("Generating:", str(target[0])) generated_build_ninja = target[0].get_abspath() - NINJA_STATE.generate(generated_build_ninja) + NINJA_STATE.generate() if env.get("DISABLE_AUTO_NINJA") != True: print("Executing:", str(target[0])) @@ -1240,6 +1244,7 @@ def generate(env): help='Disable ninja automatically building after scons') env["DISABLE_AUTO_NINJA"] = GetOption('disable_auto_ninja') + global NINJA_STATE env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") # Add the Ninja builder. @@ -1252,10 +1257,18 @@ def generate(env): env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") - ninja_file = env.Ninja(target=ninja_file_name, source=[]) - env.AlwaysBuild(ninja_file) - env.Alias("$NINJA_ALIAS_NAME", ninja_file) - + # here we allow multiple environments to construct rules and builds + # into the same ninja file + if NINJA_STATE is None: + ninja_file = env.Ninja(target=ninja_file_name, source=[]) + env.AlwaysBuild(ninja_file) + env.Alias("$NINJA_ALIAS_NAME", ninja_file) + else: + if str(NINJA_STATE.ninja_file) != ninja_file_name: + raise Exception("Generating multiple ninja files not supported.") + else: + ninja_file = [NINJA_STATE.ninja_file] + # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. if env["PLATFORM"] == "win32": @@ -1317,7 +1330,7 @@ def generate(env): def robust_rule_mapping(var, rule, tool): provider = gen_get_response_file_command(env, rule, tool) env.NinjaRuleMapping("${" + var + "}", provider) - env.NinjaRuleMapping(env[var], provider) + env.NinjaRuleMapping(env.get(var, None), provider) robust_rule_mapping("CCCOM", "CC", "$CC") robust_rule_mapping("SHCCCOM", "CC", "$CC") @@ -1447,8 +1460,8 @@ def robust_rule_mapping(var, rule, tool): else: ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') - global NINJA_STATE - NINJA_STATE = NinjaState(env, ninja_syntax.Writer) + if NINJA_STATE is None: + NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) # Here we will force every builder to use an emitter which makes the ninja # file depend on it's target. This forces the ninja file to the bottom of From 13aafe523675040db20e2a4936951ffe47d2b9f0 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 6 May 2020 17:59:10 -0400 Subject: [PATCH 102/163] update CI to install ninja pypi package --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 520b90bb74..4093f19278 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ sphinx<3.5.0 sphinx_rtd_theme lxml==4.6.3 rst2pdf +ninja \ No newline at end of file From aa38ad58b742eda215c4ac9743f13a7644717f9f Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 7 May 2020 00:25:15 -0400 Subject: [PATCH 103/163] added more test, including ninja speed test --- test/ninja/build_libraries.py | 71 ++++++++++++---------------- test/ninja/generate_source.py | 51 ++++++++------------ test/ninja/iterative_speedup.py | 49 ++++++++----------- test/ninja/multi_env.py | 53 +++++++++------------ test/ninja/ninja-fixture/bar.c | 2 +- test/ninja/ninja-fixture/test1.c | 13 +---- test/ninja/ninja-fixture/test2.C | 3 -- test/ninja/ninja-fixture/test_impl.c | 15 +----- 8 files changed, 96 insertions(+), 161 deletions(-) delete mode 100644 test/ninja/ninja-fixture/test2.C diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 40404152fa..662c584fe8 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -25,74 +25,63 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import sys import TestSCons -from TestCmd import IS_WINDOWS - -test = TestSCons.TestSCons() - -try: - import ninja -except ImportError: - test.skip_test("Could not find module in python") _python_ = TestSCons._python_ _exe = TestSCons._exe -ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - 'ninja' + _exe)) +test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -lib_suffix = '.lib' if IS_WINDOWS else '.so' -staticlib_suffix = '.lib' if IS_WINDOWS else '.a' -lib_prefix = '' if IS_WINDOWS else 'lib' +ninja = test.where_is('ninja', os.environ['PATH']) -win32 = ", 'WIN32'" if IS_WINDOWS else '' +if not ninja: + test.skip_test("Could not find ninja in environment") test.write('SConstruct', """ env = Environment() env.Tool('ninja') -shared_lib = env.SharedLibrary(target = 'test_impl', source = 'test_impl.c', CPPDEFINES=['LIBRARY_BUILD'%(win32)s]) -env.Program(target = 'test', source = 'test1.c', LIBS=['test_impl'], LIBPATH=['.'], RPATH='.') +shared_lib = env.SharedLibrary(target = 'test_impl', source = 'test_impl.c') +env.Program(target = 'test', source = 'test1.c', LIBS=[shared_lib], LIBPATH=['.'], RPATH='.') -static_obj = env.Object(target = 'test_impl_static', source = 'test_impl.c') -static_lib = env.StaticLibrary(target = 'test_impl_static', source = static_obj) -static_obj = env.Object(target = 'test_static', source = 'test1.c') +static_lib = env.StaticLibrary(target = 'test_impl_static', source = 'test_impl.c') +static_obj = env.Object(target = 'test_static.o', source = 'test1.c') env.Program(target = 'test_static', source = static_obj, LIBS=[static_lib], LIBPATH=['.']) -""" % locals()) +""") # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('test'), stdout="library_function") -test.run(program = test.workpath('test_static'), stdout="library_function") +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('test'), stdout="library_function" + os.linesep) +test.run(program = test.workpath('test_static'), stdout="library_function" + os.linesep) # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ - ('Removed %stest_impl' % lib_prefix) + lib_suffix, - 'Removed test' + _exe, - ('Removed %stest_impl_static' % lib_prefix) + staticlib_suffix, - 'Removed test_static' + _exe, + 'Removed test_impl.os', + 'Removed libtest_impl.so', + 'Removed test1.o', + 'Removed test', + 'Removed test_impl.o', + 'Removed libtest_impl_static.a', + 'Removed test_static.o', + 'Removed test_static', 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-execute-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_not_exist(test.workpath('test')) -test.must_not_exist(test.workpath('test_static')) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) # run ninja independently -program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('test'), stdout="library_function") -test.run(program = test.workpath('test_static'), stdout="library_function") +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('test'), stdout="library_function" + os.linesep) +test.run(program = test.workpath('test_static'), stdout="library_function" + os.linesep) test.pass_test() diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index 76c79bb7da..8ae0a80c39 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -25,37 +25,28 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import sys import TestSCons -from TestCmd import IS_WINDOWS - -test = TestSCons.TestSCons() - -try: - import ninja -except ImportError: - test.skip_test("Could not find module in python") _python_ = TestSCons._python_ _exe = TestSCons._exe -ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - 'ninja' + _exe)) +test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -shell = '' if IS_WINDOWS else './' +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") test.write('SConstruct', """ env = Environment() env.Tool('ninja') -prog = env.Program(target = 'generate_source', source = 'generate_source.c') -env.Command('generated_source.c', prog, '%(shell)sgenerate_source%(_exe)s') +env.Program(target = 'generate_source', source = 'generate_source.c') +env.Command('generated_source.c', ['generate_source'], './generate_source') env.Program(target = 'generated_source', source = 'generated_source.c') -""" % locals()) +""") test.write('generate_source.c', """ #include @@ -70,7 +61,7 @@ fprintf(fp, "int\\n"); fprintf(fp, "main(int argc, char *argv[])\\n"); fprintf(fp, "{\\n"); - fprintf(fp, " printf(\\"generated_source.c\\");\\n"); + fprintf(fp, " printf(\\"generated_source.c\\\\n\\");\\n"); fprintf(fp, " exit (0);\\n"); fprintf(fp, "}\\n"); fclose(fp); @@ -79,30 +70,28 @@ # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") +test.run(program = test.workpath('generated_source'), stdout="generated_source.c" + os.linesep) # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed generate_source.o', - 'Removed generate_source' + _exe, + 'Removed generate_source', 'Removed generated_source.c', 'Removed generated_source.o', - 'Removed generated_source' + _exe, + 'Removed generated_source', 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-execute-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_not_exist(test.workpath('generated_source' + _exe)) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) # run ninja independently -program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('generated_source'), stdout="generated_source.c" + os.linesep) test.pass_test() diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index ff50f502a3..018ba7beda 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -25,30 +25,23 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import sys import time import random import TestSCons -from TestCmd import IS_WINDOWS - -test = TestSCons.TestSCons() - -try: - import ninja -except ImportError: - test.skip_test("Could not find module in python") _python_ = TestSCons._python_ _exe = TestSCons._exe -ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - 'ninja' + _exe)) +test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + test.write('source_0.c', """ #include #include @@ -57,8 +50,7 @@ int print_function0() { - printf("main print"); - return 0; + printf("main print\\n"); } """) @@ -101,7 +93,7 @@ def generate_source(parent_source, current_source): int print_function%(current_source)s() { - return print_function%(parent_source)s(); + print_function%(parent_source)s(); } """ % locals()) @@ -141,7 +133,7 @@ def mod_source_orig(test_num): int print_function%(test_num)s() { - return print_function%(parent_source)s(); + print_function%(parent_source)s(); } """ % locals()) @@ -157,7 +149,6 @@ def mod_source_orig(test_num): main() { print_function%(num_source)s(); - exit(0); } """ % locals()) @@ -181,19 +172,17 @@ def mod_source_orig(test_num): tests_mods += [random.randrange(1, num_source, 1)] jobs = '-j' + str(get_num_cpus()) -ninja_program = [test.workpath('run_ninja_env.bat'), jobs] if IS_WINDOWS else [ninja_bin, jobs] - start = time.perf_counter() -test.run(arguments='--disable-execute-ninja', stdout=None) -test.run(program = ninja_program, stdout=None) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.run(program = ninja, arguments=[jobs], stdout=None) stop = time.perf_counter() ninja_times += [stop - start] -test.run(program = test.workpath('print_bin'), stdout="main print") +test.run(program = test.workpath('print_bin'), stdout="main print" + os.linesep) for test_mod in tests_mods: mod_source_return(test_mod) start = time.perf_counter() - test.run(program = ninja_program, stdout=None) + test.run(program = ninja, arguments=[jobs], stdout=None) stop = time.perf_counter() ninja_times += [stop - start] @@ -209,7 +198,7 @@ def mod_source_orig(test_num): test.run(arguments = ["-f", "SConstruct_no_ninja", jobs], stdout=None) stop = time.perf_counter() scons_times += [stop - start] -test.run(program = test.workpath('print_bin'), stdout="main print") +test.run(program = test.workpath('print_bin'), stdout="main print" + os.linesep) for test_mod in tests_mods: mod_source_return(test_mod) @@ -219,14 +208,14 @@ def mod_source_orig(test_num): scons_times += [stop - start] full_build_print = True -for ninja_time, scons_time in zip(ninja_times, scons_times): - if ninja_time > scons_time: +for ninja, scons in zip(ninja_times, scons_times): + if ninja > scons: test.fail_test() if full_build_print: full_build_print = False - print("Clean build {} files - SCons: {:.3f}s Ninja: {:.3f}s".format(num_source, scons_time, ninja_time)) + print("Clean build {} files - SCons: {:.3f}s Ninja: {:.3f}s".format(num_source, scons, ninja)) else: - print("Single File Rebuild - SCons: {:.3f}s Ninja: {:.3f}s".format(scons_time, ninja_time)) + print("Single File Rebuild - SCons: {:.3f}s Ninja: {:.3f}s".format(scons, ninja)) test.pass_test() diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 18ca3cbc69..3612d7b84a 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -25,67 +25,60 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import sys import TestSCons -from TestCmd import IS_WINDOWS - -test = TestSCons.TestSCons() - -try: - import ninja -except ImportError: - test.skip_test("Could not find module in python") _python_ = TestSCons._python_ _exe = TestSCons._exe -ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - 'ninja' + _exe)) - test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') +ninja = test.where_is('ninja', os.environ['PATH']) + +if not ninja: + test.skip_test("Could not find ninja in environment") + test.write('SConstruct', """ env = Environment() env.Tool('ninja') env.Program(target = 'foo', source = 'foo.c') env2 = Environment() +env2.Tool('ninja') env2.Program(target = 'bar', source = 'bar.c') """) # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('foo' + _exe), stdout="foo.c") -test.run(program = test.workpath('bar' + _exe), stdout="bar.c") +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja', 'Executing: build.ninja']) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +test.run(program = test.workpath('bar'), stdout="bar.c" + os.linesep) # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo.o', - 'Removed foo' + _exe, + 'Removed foo', 'Removed bar.o', - 'Removed bar' + _exe, + 'Removed bar', 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-execute-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_not_exist(test.workpath('foo' + _exe)) -test.must_not_exist(test.workpath('bar' + _exe)) +test.run(arguments='--disable-auto-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), + ['Generating: build.ninja']) +test.must_not_contain_any_line(test.stdout(), + ['Executing: build.ninja']) # run ninja independently -program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('foo' + _exe), stdout="foo.c") -test.run(program = test.workpath('bar' + _exe), stdout="bar.c") +test.run(program = ninja, stdout=None) +test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +test.run(program = test.workpath('bar'), stdout="bar.c" + os.linesep) + + test.pass_test() diff --git a/test/ninja/ninja-fixture/bar.c b/test/ninja/ninja-fixture/bar.c index 15b2ecc46a..3767857b00 100644 --- a/test/ninja/ninja-fixture/bar.c +++ b/test/ninja/ninja-fixture/bar.c @@ -5,6 +5,6 @@ int main(int argc, char *argv[]) { argv[argc++] = "--"; - printf("bar.c"); + printf("bar.c\n"); exit (0); } diff --git a/test/ninja/ninja-fixture/test1.c b/test/ninja/ninja-fixture/test1.c index 678461f508..c53f54ac85 100644 --- a/test/ninja/ninja-fixture/test1.c +++ b/test/ninja/ninja-fixture/test1.c @@ -1,21 +1,10 @@ #include #include -#ifdef WIN32 -#ifdef LIBRARY_BUILD -#define DLLEXPORT __declspec(dllexport) -#else -#define DLLEXPORT __declspec(dllimport) -#endif -#else -#define DLLEXPORT -#endif - -DLLEXPORT extern int library_function(void); +extern int library_function(void); int main(int argc, char *argv[]) { library_function(); - exit(0); } diff --git a/test/ninja/ninja-fixture/test2.C b/test/ninja/ninja-fixture/test2.C deleted file mode 100644 index a1ee9e32b9..0000000000 --- a/test/ninja/ninja-fixture/test2.C +++ /dev/null @@ -1,3 +0,0 @@ -This is a .C file. -/*cc*/ -/*link*/ diff --git a/test/ninja/ninja-fixture/test_impl.c b/test/ninja/ninja-fixture/test_impl.c index 89c26ede6f..ae5effc965 100644 --- a/test/ninja/ninja-fixture/test_impl.c +++ b/test/ninja/ninja-fixture/test_impl.c @@ -1,19 +1,8 @@ #include #include -#ifdef WIN32 -#ifdef LIBRARY_BUILD -#define DLLEXPORT __declspec(dllexport) -#else -#define DLLEXPORT __declspec(dllimport) -#endif -#else -#define DLLEXPORT -#endif - - -DLLEXPORT int +int library_function(void) { - printf("library_function"); + printf("library_function\n"); } From f901137d77c90fa411677ded3b053ed65f9fa80d Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 7 May 2020 00:31:28 -0400 Subject: [PATCH 104/163] fixed sider issues --- src/engine/SCons/Tool/ninja.py | 4 ++-- test/ninja/build_libraries.py | 1 - test/ninja/copy_function_command.py | 1 - test/ninja/generate_and_build.py | 1 - test/ninja/generate_source.py | 1 - test/ninja/iterative_speedup.py | 1 - test/ninja/multi_env.py | 1 - test/ninja/shell_command.py | 1 - 8 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 63ba243c61..bf399e2592 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -665,7 +665,7 @@ def generate(self): # files. Here we make sure that Ninja only ever sees one # target when using a depfile. It will still have a command # that will create all of the outputs but most targets don't - # depend direclty on DWO files and so this assumption is safe + # depend directly on DWO files and so this assumption is safe # to make. rule = self.rules.get(build["rule"]) @@ -1044,7 +1044,7 @@ def ninja_builder(env, target, source): generated_build_ninja = target[0].get_abspath() NINJA_STATE.generate() - if env.get("DISABLE_AUTO_NINJA") != True: + if not env.get("DISABLE_AUTO_NINJA"): print("Executing:", str(target[0])) def execute_ninja(): diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 662c584fe8..7e0ec2365a 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index f86f717e6f..a1e72b7181 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index fd111a29be..82aab5e53b 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index 8ae0a80c39..d1bfe34a6b 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index 018ba7beda..cf999d8568 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import time import random import TestSCons diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 3612d7b84a..5360fd215c 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index fd0e35f8de..b5c8323f43 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -25,7 +25,6 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import sys import TestSCons _python_ = TestSCons._python_ From 1c77822f983a29dc7e879e61d4bf0c6db44871d1 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 8 May 2020 12:44:27 -0500 Subject: [PATCH 105/163] update tests to work on windows, added some environment support for windows and msvc --- src/engine/SCons/Tool/ninja.py | 47 +++++++++++++++++++++------- test/ninja/build_libraries.py | 41 +++++++++++++----------- test/ninja/copy_function_command.py | 10 +++--- test/ninja/generate_and_build.py | 9 ++++-- test/ninja/generate_source.py | 22 +++++++------ test/ninja/iterative_speedup.py | 19 ++++++----- test/ninja/multi_env.py | 18 +++++------ test/ninja/ninja-fixture/bar.c | 2 +- test/ninja/ninja-fixture/test1.c | 13 +++++++- test/ninja/ninja-fixture/test_impl.c | 15 +++++++-- test/ninja/shell_command.py | 20 ++++++------ 11 files changed, 141 insertions(+), 75 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index bf399e2592..10e27f3423 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -77,8 +77,8 @@ def _mkdir_action_function(env, node): # to an invalid ninja file. "variables": { # On Windows mkdir "-p" is always on - "cmd": "{mkdir} $out".format( - mkdir="mkdir" if env["PLATFORM"] == "win32" else "mkdir -p", + "cmd": "{mkdir}".format( + mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", ), }, } @@ -383,6 +383,7 @@ class NinjaState: def __init__(self, env, ninja_file, writer_class): self.env = env self.ninja_file = ninja_file + self.ninja_bin_path = '' self.writer_class = writer_class self.__generated = False self.translator = SConsToNinjaTranslator(env) @@ -752,7 +753,7 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "ninja -f {} -t compdb CC CXX > compile_commands.json".format( + "cmd": "{}/ninja -f {} -t compdb CC CXX > compile_commands.json".format(self.ninja_bin_path, str(self.ninja_file) ) }, @@ -1044,15 +1045,32 @@ def ninja_builder(env, target, source): generated_build_ninja = target[0].get_abspath() NINJA_STATE.generate() + + if env["PLATFORM"] == "win32": + # this is not great, it executes everytime + # and its doesn't consider specific node environments + # also a bit quirky to use, but usually MSVC is not + # setup system wide for command line use so this is needed + # on the standard MSVC setup, this is only needed if + # running ninja directly from a command line that hasn't + # had the environment setup (vcvarsall.bat) + # todo: hook this into a command so that it only regnerates + # the .bat if the env['ENV'] changes + with open('ninja_env.bat', 'w') as f: + for key in env['ENV']: + f.write('set {}={}\n'.format(key, env['ENV'][key])) + if not env.get("DISABLE_AUTO_NINJA"): print("Executing:", str(target[0])) def execute_ninja(): - proc = subprocess.Popen( ['ninja', '-f', generated_build_ninja], - stderr=subprocess.STDOUT, + env.AppendENVPath('PATH', NINJA_STATE.ninja_bin_path) + proc = subprocess.Popen(['ninja', '-f', generated_build_ninja], + stderr=sys.stderr, stdout=subprocess.PIPE, - universal_newlines=True + universal_newlines=True, + env=env['ENV'] ) for stdout_line in iter(proc.stdout.readline, ""): yield stdout_line @@ -1060,12 +1078,17 @@ def execute_ninja(): return_code = proc.wait() if return_code: raise subprocess.CalledProcessError(return_code, 'ninja') - + erase_previous = False for output in execute_ninja(): output = output.strip() - sys.stdout.write('\x1b[2K') # erase previous line - sys.stdout.write(output + "\r") + if erase_previous: + sys.stdout.write('\x1b[2K') # erase previous line + sys.stdout.write("\r") + else: + sys.stdout.write(os.linesep) + sys.stdout.write(output) sys.stdout.flush() + erase_previous = output.startswith('[') # pylint: disable=too-few-public-methods class AlwaysExecAction(SCons.Action.FunctionAction): @@ -1311,7 +1334,7 @@ def generate(env): if env["PLATFORM"] == "win32": from SCons.Tool.mslink import compositeLinkAction - if env["LINKCOM"] == compositeLinkAction: + if env.get("LINKCOM", None) == compositeLinkAction: env[ "LINKCOM" ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' @@ -1459,10 +1482,10 @@ def robust_rule_mapping(var, rule, tool): ninja_syntax = importlib.import_module(ninja_syntax_mod_name) else: ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') - + if NINJA_STATE is None: NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) - + NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join(ninja_syntax.__file__, os.pardir, 'data', 'bin')) # Here we will force every builder to use an emitter which makes the ninja # file depend on it's target. This forces the ninja file to the bottom of # the DAG which is required so that we walk every target, and therefore add diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 7e0ec2365a..5e491cc90b 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -39,35 +40,38 @@ if not ninja: test.skip_test("Could not find ninja in environment") +lib_suffix = '.lib' if IS_WINDOWS else '.so' +staticlib_suffix = '.lib' if IS_WINDOWS else '.a' +lib_prefix = '' if IS_WINDOWS else 'lib' + +win32 = ", 'WIN32'" if IS_WINDOWS else '' + test.write('SConstruct', """ env = Environment() env.Tool('ninja') -shared_lib = env.SharedLibrary(target = 'test_impl', source = 'test_impl.c') -env.Program(target = 'test', source = 'test1.c', LIBS=[shared_lib], LIBPATH=['.'], RPATH='.') +shared_lib = env.SharedLibrary(target = 'test_impl', source = 'test_impl.c', CPPDEFINES=['LIBRARY_BUILD'%(win32)s]) +env.Program(target = 'test', source = 'test1.c', LIBS=['test_impl'], LIBPATH=['.'], RPATH='.') -static_lib = env.StaticLibrary(target = 'test_impl_static', source = 'test_impl.c') -static_obj = env.Object(target = 'test_static.o', source = 'test1.c') +static_obj = env.Object(target = 'test_impl_static', source = 'test_impl.c') +static_lib = env.StaticLibrary(target = 'test_impl_static', source = static_obj) +static_obj = env.Object(target = 'test_static', source = 'test1.c') env.Program(target = 'test_static', source = static_obj, LIBS=[static_lib], LIBPATH=['.']) -""") +""" % locals()) # generate simple build test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('test'), stdout="library_function" + os.linesep) -test.run(program = test.workpath('test_static'), stdout="library_function" + os.linesep) +test.run(program = test.workpath('test'), stdout="library_function") +test.run(program = test.workpath('test_static'), stdout="library_function") # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ - 'Removed test_impl.os', - 'Removed libtest_impl.so', - 'Removed test1.o', - 'Removed test', - 'Removed test_impl.o', - 'Removed libtest_impl_static.a', - 'Removed test_static.o', - 'Removed test_static', + ('Removed %stest_impl' % lib_prefix) + lib_suffix, + 'Removed test' + _exe, + ('Removed %stest_impl_static' % lib_prefix) + staticlib_suffix, + 'Removed test_static' + _exe, 'Removed build.ninja']) # only generate the ninja file @@ -78,9 +82,10 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('test'), stdout="library_function" + os.linesep) -test.run(program = test.workpath('test_static'), stdout="library_function" + os.linesep) +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('test'), stdout="library_function") +test.run(program = test.workpath('test_static'), stdout="library_function") test.pass_test() diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index a1e72b7181..06991d375c 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -50,14 +51,14 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +test.run(program = test.workpath('foo'), stdout="foo.c") # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo2.o', 'Removed foo2.c', - 'Removed foo', + 'Removed foo' + _exe, 'Removed build.ninja']) # only generate the ninja file @@ -68,8 +69,9 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('foo'), stdout="foo.c") test.pass_test() diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index 82aab5e53b..904e46a3cd 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -45,11 +46,12 @@ env.Program(target = 'foo', source = 'foo.c') """) + # generate simple build test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +test.run(program = test.workpath('foo' + _exe), stdout="foo.c") # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -66,8 +68,9 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('foo' + _exe), stdout="foo.c") test.pass_test() diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index d1bfe34a6b..298d227fea 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -39,13 +40,15 @@ if not ninja: test.skip_test("Could not find ninja in environment") +shell = '' if IS_WINDOWS else './' + test.write('SConstruct', """ env = Environment() env.Tool('ninja') -env.Program(target = 'generate_source', source = 'generate_source.c') -env.Command('generated_source.c', ['generate_source'], './generate_source') +prog = env.Program(target = 'generate_source', source = 'generate_source.c') +env.Command('generated_source.c', prog, '%(shell)sgenerate_source%(_exe)s') env.Program(target = 'generated_source', source = 'generated_source.c') -""") +""" % locals()) test.write('generate_source.c', """ #include @@ -60,7 +63,7 @@ fprintf(fp, "int\\n"); fprintf(fp, "main(int argc, char *argv[])\\n"); fprintf(fp, "{\\n"); - fprintf(fp, " printf(\\"generated_source.c\\\\n\\");\\n"); + fprintf(fp, " printf(\\"generated_source.c\\");\\n"); fprintf(fp, " exit (0);\\n"); fprintf(fp, "}\\n"); fclose(fp); @@ -69,16 +72,16 @@ # generate simple build test.run(stdout=None) -test.run(program = test.workpath('generated_source'), stdout="generated_source.c" + os.linesep) +test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed generate_source.o', - 'Removed generate_source', + 'Removed generate_source' + _exe, 'Removed generated_source.c', 'Removed generated_source.o', - 'Removed generated_source', + 'Removed generated_source' + _exe, 'Removed build.ninja']) # only generate the ninja file @@ -89,8 +92,9 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('generated_source'), stdout="generated_source.c" + os.linesep) +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") test.pass_test() diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index cf999d8568..6675bf3350 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -28,6 +28,7 @@ import time import random import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -49,7 +50,8 @@ int print_function0() { - printf("main print\\n"); + printf("main print"); + return 0; } """) @@ -92,7 +94,7 @@ def generate_source(parent_source, current_source): int print_function%(current_source)s() { - print_function%(parent_source)s(); + return print_function%(parent_source)s(); } """ % locals()) @@ -132,7 +134,7 @@ def mod_source_orig(test_num): int print_function%(test_num)s() { - print_function%(parent_source)s(); + return print_function%(parent_source)s(); } """ % locals()) @@ -148,6 +150,7 @@ def mod_source_orig(test_num): main() { print_function%(num_source)s(); + exit(0); } """ % locals()) @@ -171,17 +174,19 @@ def mod_source_orig(test_num): tests_mods += [random.randrange(1, num_source, 1)] jobs = '-j' + str(get_num_cpus()) +ninja_program = [test.workpath('ninja_env.bat'), '&', ninja, jobs] if IS_WINDOWS else [ninja, jobs] + start = time.perf_counter() test.run(arguments='--disable-auto-ninja', stdout=None) -test.run(program = ninja, arguments=[jobs], stdout=None) +test.run(program = ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] -test.run(program = test.workpath('print_bin'), stdout="main print" + os.linesep) +test.run(program = test.workpath('print_bin'), stdout="main print") for test_mod in tests_mods: mod_source_return(test_mod) start = time.perf_counter() - test.run(program = ninja, arguments=[jobs], stdout=None) + test.run(program = ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] @@ -197,7 +202,7 @@ def mod_source_orig(test_num): test.run(arguments = ["-f", "SConstruct_no_ninja", jobs], stdout=None) stop = time.perf_counter() scons_times += [stop - start] -test.run(program = test.workpath('print_bin'), stdout="main print" + os.linesep) +test.run(program = test.workpath('print_bin'), stdout="main print") for test_mod in tests_mods: mod_source_return(test_mod) diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 5360fd215c..087d392b52 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -53,16 +54,16 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) -test.run(program = test.workpath('bar'), stdout="bar.c" + os.linesep) +test.run(program = test.workpath('foo' + _exe), stdout="foo.c") +test.run(program = test.workpath('bar' + _exe), stdout="bar.c") # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo.o', - 'Removed foo', + 'Removed foo' + _exe, 'Removed bar.o', - 'Removed bar', + 'Removed bar' + _exe, 'Removed build.ninja']) # only generate the ninja file @@ -73,11 +74,10 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.run(program = test.workpath('foo'), stdout="foo.c" + os.linesep) -test.run(program = test.workpath('bar'), stdout="bar.c" + os.linesep) - - +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.run(program = test.workpath('foo' + _exe), stdout="foo.c") +test.run(program = test.workpath('bar' + _exe), stdout="bar.c") test.pass_test() diff --git a/test/ninja/ninja-fixture/bar.c b/test/ninja/ninja-fixture/bar.c index 3767857b00..15b2ecc46a 100644 --- a/test/ninja/ninja-fixture/bar.c +++ b/test/ninja/ninja-fixture/bar.c @@ -5,6 +5,6 @@ int main(int argc, char *argv[]) { argv[argc++] = "--"; - printf("bar.c\n"); + printf("bar.c"); exit (0); } diff --git a/test/ninja/ninja-fixture/test1.c b/test/ninja/ninja-fixture/test1.c index c53f54ac85..678461f508 100644 --- a/test/ninja/ninja-fixture/test1.c +++ b/test/ninja/ninja-fixture/test1.c @@ -1,10 +1,21 @@ #include #include -extern int library_function(void); +#ifdef WIN32 +#ifdef LIBRARY_BUILD +#define DLLEXPORT __declspec(dllexport) +#else +#define DLLEXPORT __declspec(dllimport) +#endif +#else +#define DLLEXPORT +#endif + +DLLEXPORT extern int library_function(void); int main(int argc, char *argv[]) { library_function(); + exit(0); } diff --git a/test/ninja/ninja-fixture/test_impl.c b/test/ninja/ninja-fixture/test_impl.c index ae5effc965..89c26ede6f 100644 --- a/test/ninja/ninja-fixture/test_impl.c +++ b/test/ninja/ninja-fixture/test_impl.c @@ -1,8 +1,19 @@ #include #include -int +#ifdef WIN32 +#ifdef LIBRARY_BUILD +#define DLLEXPORT __declspec(dllexport) +#else +#define DLLEXPORT __declspec(dllimport) +#endif +#else +#define DLLEXPORT +#endif + + +DLLEXPORT int library_function(void) { - printf("library_function\n"); + printf("library_function"); } diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index b5c8323f43..5d491645c6 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -26,6 +26,7 @@ import os import TestSCons +from TestCmd import IS_WINDOWS _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -39,24 +40,26 @@ if not ninja: test.skip_test("Could not find ninja in environment") +shell = '' if IS_WINDOWS else './' + test.write('SConstruct', """ env = Environment() env.Tool('ninja') -env.Program(target = 'foo', source = 'foo.c') -env.Command('foo.out', ['foo'], './foo > foo.out') -""") +prog = env.Program(target = 'foo', source = 'foo.c') +env.Command('foo.out', prog, '%(shell)sfoo%(_exe)s > foo.out') +""" % locals()) # generate simple build test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'Executing: build.ninja']) -test.must_match('foo.out', 'foo.c' + os.linesep) +test.must_match('foo.out', 'foo.c') # clean build and ninja files test.run(arguments='-c', stdout=None) test.must_contain_all_lines(test.stdout(), [ 'Removed foo.o', - 'Removed foo', + 'Removed foo%(_exe)s' % locals(), 'Removed foo.out', 'Removed build.ninja']) @@ -68,10 +71,9 @@ ['Executing: build.ninja']) # run ninja independently -test.run(program = ninja, stdout=None) -test.must_match('foo.out', 'foo.c' + os.linesep) - - +program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +test.run(program = program, stdout=None) +test.must_match('foo.out', 'foo.c') test.pass_test() From c2fda149ad794c4072c50a07d65c5b04c3fe1317 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 14 May 2020 02:52:16 -0400 Subject: [PATCH 106/163] used different method for pushing ninja file to bottom of DAG, use import ninja to get ninja_syntax and ninja bin, and added some more basic testing. --- src/engine/SCons/Tool/ninja.py | 196 +++++++++++++-------------- test/ninja/build_libraries.py | 34 +++-- test/ninja/copy_function_command.py | 33 +++-- test/ninja/generate_and_build.py | 33 +++-- test/ninja/generate_and_build_cxx.py | 7 +- test/ninja/generate_source.py | 31 +++-- test/ninja/iterative_speedup.py | 22 ++- test/ninja/multi_env.py | 35 +++-- test/ninja/shell_command.py | 33 +++-- 9 files changed, 237 insertions(+), 187 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 10e27f3423..bc505b8937 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -88,13 +88,7 @@ def _copy_action_function(env, node): "outputs": get_outputs(node), "inputs": [get_path(src_file(s)) for s in node.sources], "rule": "CMD", - # implicit explicitly omitted, we translate these so they can be - # used by anything that depends on these but commonly this is - # hit with a node that will depend on all of the fake - # srcnode's that SCons will never give us a rule for leading - # to an invalid ninja file. "variables": { - # On Windows mkdir "-p" is always on "cmd": "$COPY $in $out", }, } @@ -753,7 +747,7 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "{}/ninja -f {} -t compdb CC CXX > compile_commands.json".format(self.ninja_bin_path, + "cmd": "{} -f {} -t compdb CC CXX > compile_commands.json".format(self.ninja_bin_path, str(self.ninja_file) ) }, @@ -1047,30 +1041,30 @@ def ninja_builder(env, target, source): NINJA_STATE.generate() if env["PLATFORM"] == "win32": - # this is not great, it executes everytime - # and its doesn't consider specific node environments - # also a bit quirky to use, but usually MSVC is not - # setup system wide for command line use so this is needed - # on the standard MSVC setup, this is only needed if + # this is not great, its doesn't consider specific + # node environments, which means on linux the build could + # behave differently, becuase on linux you can set the environment + # per command in the ninja file. This is only needed if # running ninja directly from a command line that hasn't # had the environment setup (vcvarsall.bat) - # todo: hook this into a command so that it only regnerates - # the .bat if the env['ENV'] changes - with open('ninja_env.bat', 'w') as f: + with open('run_ninja_env.bat', 'w') as f: for key in env['ENV']: f.write('set {}={}\n'.format(key, env['ENV'][key])) + f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) if not env.get("DISABLE_AUTO_NINJA"): - print("Executing:", str(target[0])) + cmd = [NINJA_STATE.ninja_bin_path, '-f', generated_build_ninja] + print("Executing:", str(' '.join(cmd))) + # execute the ninja build at the end of SCons, trying to + # reproduce the output like a ninja build would def execute_ninja(): - env.AppendENVPath('PATH', NINJA_STATE.ninja_bin_path) - proc = subprocess.Popen(['ninja', '-f', generated_build_ninja], + proc = subprocess.Popen(cmd, stderr=sys.stderr, stdout=subprocess.PIPE, universal_newlines=True, - env=env['ENV'] + env=env['ENV'] # ninja build items won't consider node env on win32 ) for stdout_line in iter(proc.stdout.readline, ""): yield stdout_line @@ -1078,6 +1072,7 @@ def execute_ninja(): return_code = proc.wait() if return_code: raise subprocess.CalledProcessError(return_code, 'ninja') + erase_previous = False for output in execute_ninja(): output = output.strip() @@ -1088,6 +1083,9 @@ def execute_ninja(): sys.stdout.write(os.linesep) sys.stdout.write(output) sys.stdout.flush() + # this will only erase ninjas [#/#] lines + # leaving warnings and other output, seems a bit + # prone to failure with such a simple check erase_previous = output.startswith('[') # pylint: disable=too-few-public-methods @@ -1226,6 +1224,20 @@ def ninja_always_serial(self, num, taskmaster): self.num_jobs = num self.job = SCons.Job.Serial(taskmaster) +def ninja_hack_linkcom(env): + # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks + # without relying on the SCons hacks that SCons uses by default. + if env["PLATFORM"] == "win32": + from SCons.Tool.mslink import compositeLinkAction + + if env.get("LINKCOM", None) == compositeLinkAction: + env[ + "LINKCOM" + ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' + env[ + "SHLINKCOM" + ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' + class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" @@ -1259,13 +1271,31 @@ def generate(env): global added if not added: added = 1 - AddOption('--disable-auto-ninja', - dest='disable_auto_ninja', - metavar='BOOL', - action="store_true", - default=False, - help='Disable ninja automatically building after scons') - env["DISABLE_AUTO_NINJA"] = GetOption('disable_auto_ninja') + + AddOption('--disable-execute-ninja', + dest='disable_execute_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + + AddOption('--disable-ninja', + dest='disable_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + + if GetOption('disable_ninja'): + return env + + try: + import ninja + except ImportError: + SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") + return + + env["DISABLE_AUTO_NINJA"] = GetOption('disable_execute_ninja') global NINJA_STATE env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") @@ -1288,16 +1318,15 @@ def generate(env): env.Alias("$NINJA_ALIAS_NAME", ninja_file) else: if str(NINJA_STATE.ninja_file) != ninja_file_name: - raise Exception("Generating multiple ninja files not supported.") - else: - ninja_file = [NINJA_STATE.ninja_file] + SCons.Warnings.Warning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") + ninja_file = [NINJA_STATE.ninja_file] # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. if env["PLATFORM"] == "win32": env.Append(CCFLAGS=["/showIncludes"]) else: - env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) + env.Append(CCFLAGS=["-MD", "-MF", "${TARGET}.d"]) # Provide a way for custom rule authors to easily access command # generation. @@ -1329,18 +1358,8 @@ def generate(env): # might not catch it. env.AddMethod(register_custom_rule_mapping, "NinjaRuleMapping") - # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks - # without relying on the SCons hacks that SCons uses by default. - if env["PLATFORM"] == "win32": - from SCons.Tool.mslink import compositeLinkAction - - if env.get("LINKCOM", None) == compositeLinkAction: - env[ - "LINKCOM" - ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' - env[ - "SHLINKCOM" - ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' + # on windows we need to change the link action + ninja_hack_linkcom(env) # Normally in SCons actions for the Program and *Library builders # will return "${*COM}" as their pre-subst'd command line. However @@ -1386,6 +1405,8 @@ def robust_rule_mapping(var, rule, tool): # Disable running ranlib, since we added 's' above env["RANLIBCOM"] = "" + SCons.Warnings.Warning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") + # This is the point of no return, anything after this comment # makes changes to SCons that are irreversible and incompatible # with a normal SCons build. We return early if __NINJA_NO=1 has @@ -1462,68 +1483,43 @@ def robust_rule_mapping(var, rule, tool): # monkey the Jobs constructor to only use the Serial Job class. SCons.Job.Jobs.__init__ = ninja_always_serial - # The environment variable NINJA_SYNTAX points to the - # ninja_syntax.py module from the ninja sources found here: - # https://github.com/ninja-build/ninja/blob/master/misc/ninja_syntax.py - # - # This should be vendored into the build sources and it's location - # set in NINJA_SYNTAX. This code block loads the location from - # that variable, gets the absolute path to the vendored file, gets - # it's parent directory then uses importlib to import the module - # dynamically. - ninja_syntax_file = env[NINJA_SYNTAX] - - if os.path.exists(ninja_syntax_file): - if isinstance(ninja_syntax_file, str): - ninja_syntax_file = env.File(ninja_syntax_file).get_abspath() - ninja_syntax_mod_dir = os.path.dirname(ninja_syntax_file) - sys.path.append(ninja_syntax_mod_dir) - ninja_syntax_mod_name = os.path.basename(ninja_syntax_file).replace(".py", "") - ninja_syntax = importlib.import_module(ninja_syntax_mod_name) - else: - ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') + ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') if NINJA_STATE is None: NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) - NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join(ninja_syntax.__file__, os.pardir, 'data', 'bin')) - # Here we will force every builder to use an emitter which makes the ninja - # file depend on it's target. This forces the ninja file to the bottom of - # the DAG which is required so that we walk every target, and therefore add - # it to the global NINJA_STATE, before we try to write the ninja file. - def ninja_file_depends_on_all(target, source, env): - if not any("conftest" in str(t) for t in target): - env.Depends(ninja_file, target) - return target, source - - # The "Alias Builder" isn't in the BUILDERS map so we have to - # modify it directly. - SCons.Environment.AliasBuilder.emitter = ninja_file_depends_on_all - - for _, builder in env["BUILDERS"].items(): - try: - emitter = builder.emitter - if emitter is not None: - builder.emitter = SCons.Builder.ListEmitter( - [emitter, ninja_file_depends_on_all] - ) - else: - builder.emitter = ninja_file_depends_on_all - # Users can inject whatever they want into the BUILDERS - # dictionary so if the thing doesn't have an emitter we'll - # just ignore it. - except AttributeError: - pass - - # We will subvert the normal Command to make sure all targets generated - # from commands will be linked to the ninja file - SconsCommand = SCons.Environment.Environment.Command - - def NinjaCommand(self, target, source, action, **kw): - targets = SconsCommand(env, target, source, action, **kw) - env.Depends(ninja_file, targets) + NINJA_STATE.ninja_bin_path = env.get('NINJA_BIN') + if not NINJA_STATE.ninja_bin_path: + # default to using ninja installed with python module + ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' + NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( + ninja_syntax.__file__, + os.pardir, + 'data', + 'bin', + ninja_bin)) + if not os.path.exists(NINJA_STATE.ninja_bin_path): + # couldn't find it, just give the bin name and hope + # its in the path later + NINJA_STATE.ninja_bin_path = ninja_bin + + # TODO: this is hacking into scons, preferable if there were a less intrusive way + # We will subvert the normal builder execute to make sure all the ninja file is dependent + # on all targets generated from any builders + SCons_Builder_BuilderBase__execute = SCons.Builder.BuilderBase._execute + def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): + # this ensures all environments in which a builder executes from will + # not create list actions for linking on windows + ninja_hack_linkcom(env) + targets = SCons_Builder_BuilderBase__execute(self, env, target, source, overwarn=overwarn, executor_kw=executor_kw) + + if not SCons.Util.is_List(target): + target = [target] + + for target in targets: + if str(target) != ninja_file_name and "conftest" not in str(target): + env.Depends(ninja_file, targets) return targets - - SCons.Environment.Environment.Command = NinjaCommand + SCons.Builder.BuilderBase._execute = NinjaBuilderExecute # Here we monkey patch the Task.execute method to not do a bunch of # unnecessary work. If a build is a regular builder (i.e not a conftest and diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 5e491cc90b..347d63902c 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - lib_suffix = '.lib' if IS_WINDOWS else '.so' staticlib_suffix = '.lib' if IS_WINDOWS else '.a' lib_prefix = '' if IS_WINDOWS else 'lib' @@ -60,8 +68,9 @@ """ % locals()) # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('test'), stdout="library_function") test.run(program = test.workpath('test_static'), stdout="library_function") @@ -75,14 +84,13 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('test')) +test.must_not_exist(test.workpath('test_static')) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('test'), stdout="library_function") test.run(program = test.workpath('test_static'), stdout="library_function") diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index 06991d375c..0beb8de11c 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - test.write('SConstruct', """ env = Environment() env.Tool('ninja') @@ -49,8 +57,9 @@ # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('foo'), stdout="foo.c") # clean build and ninja files @@ -62,14 +71,12 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo')) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('foo'), stdout="foo.c") diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index 904e46a3cd..73c71f1b40 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - test.write('SConstruct', """ env = Environment() env.Tool('ninja') @@ -49,8 +57,9 @@ # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('foo' + _exe), stdout="foo.c") # clean build and ninja files @@ -61,14 +70,12 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo' + _exe)) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('foo' + _exe), stdout="foo.c") diff --git a/test/ninja/generate_and_build_cxx.py b/test/ninja/generate_and_build_cxx.py index 663282bd92..ac0f55444f 100644 --- a/test/ninja/generate_and_build_cxx.py +++ b/test/ninja/generate_and_build_cxx.py @@ -25,11 +25,10 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS -test = TestSCons.TestSCons() - try: import ninja except ImportError: @@ -39,12 +38,14 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, + importlib.import_module(".ninja_syntax", package='ninja').__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) +test = TestSCons.TestSCons() + test.dir_fixture('ninja-fixture') test.write('SConstruct', """ diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index 298d227fea..d9b9c4ed59 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - shell = '' if IS_WINDOWS else './' test.write('SConstruct', """ @@ -72,6 +80,9 @@ # generate simple build test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") # clean build and ninja files @@ -85,14 +96,12 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('generated_source' + _exe)) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index 6675bf3350..b03e9cbcb0 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -27,21 +27,29 @@ import os import time import random +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - test.write('source_0.c', """ #include #include @@ -174,10 +182,10 @@ def mod_source_orig(test_num): tests_mods += [random.randrange(1, num_source, 1)] jobs = '-j' + str(get_num_cpus()) -ninja_program = [test.workpath('ninja_env.bat'), '&', ninja, jobs] if IS_WINDOWS else [ninja, jobs] +ninja_program = [test.workpath('run_ninja_env.bat'), jobs] if IS_WINDOWS else [ninja_bin, jobs] start = time.perf_counter() -test.run(arguments='--disable-auto-ninja', stdout=None) +test.run(arguments='--disable-execute-ninja', stdout=None) test.run(program = ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 087d392b52..c9b21b7d70 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -25,35 +25,43 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - test.write('SConstruct', """ env = Environment() env.Tool('ninja') env.Program(target = 'foo', source = 'foo.c') env2 = Environment() -env2.Tool('ninja') env2.Program(target = 'bar', source = 'bar.c') """) # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.run(program = test.workpath('foo' + _exe), stdout="foo.c") test.run(program = test.workpath('bar' + _exe), stdout="bar.c") @@ -67,14 +75,13 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo' + _exe)) +test.must_not_exist(test.workpath('bar' + _exe)) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.run(program = test.workpath('foo' + _exe), stdout="foo.c") test.run(program = test.workpath('bar' + _exe), stdout="bar.c") diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index 5d491645c6..b35a52b6c0 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -25,21 +25,29 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os +import importlib import TestSCons from TestCmd import IS_WINDOWS +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + _python_ = TestSCons._python_ _exe = TestSCons._exe +ninja_bin = os.path.abspath(os.path.join( + importlib.import_module(".ninja_syntax", package='ninja').__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + test = TestSCons.TestSCons() test.dir_fixture('ninja-fixture') -ninja = test.where_is('ninja', os.environ['PATH']) - -if not ninja: - test.skip_test("Could not find ninja in environment") - shell = '' if IS_WINDOWS else './' test.write('SConstruct', """ @@ -51,8 +59,9 @@ # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja', 'Executing: build.ninja']) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) test.must_match('foo.out', 'foo.c') # clean build and ninja files @@ -64,14 +73,12 @@ 'Removed build.ninja']) # only generate the ninja file -test.run(arguments='--disable-auto-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), - ['Generating: build.ninja']) -test.must_not_contain_any_line(test.stdout(), - ['Executing: build.ninja']) +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo.out')) # run ninja independently -program = ['ninja_env.bat', '&', ninja] if IS_WINDOWS else ninja +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin test.run(program = program, stdout=None) test.must_match('foo.out', 'foo.c') From cb94dc138f9e28de62c851152c76df410d639072 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 14 May 2020 12:28:17 -0400 Subject: [PATCH 107/163] is link should use the base nodes lstat instead of local fs stat builder is not garunteed to be in the environment, so check if the node is the ninja_file fix sider issues --- src/engine/SCons/Tool/ninja.py | 9 +++++---- test/ninja/build_libraries.py | 7 +++---- test/ninja/copy_function_command.py | 7 +++---- test/ninja/generate_and_build.py | 7 +++---- test/ninja/generate_and_build_cxx.py | 7 +++---- test/ninja/generate_source.py | 7 +++---- test/ninja/iterative_speedup.py | 15 +++++++-------- test/ninja/multi_env.py | 5 +++-- test/ninja/shell_command.py | 7 +++---- 9 files changed, 33 insertions(+), 38 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index bc505b8937..a68655ef1d 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -233,9 +233,10 @@ def action_to_ninja_build(self, node, action=None): # Ninja builders out of being sources of ninja builders but I # can't fix every DAG problem so we just skip ninja_builders # if we find one - if node.builder == self.env["BUILDERS"]["Ninja"]: + global NINJA_STATE + if NINJA_STATE.ninja_file == str(node): build = None - elif isinstance(action, SCons.Action.FunctionAction): + if isinstance(action, SCons.Action.FunctionAction): build = self.handle_func_action(node, action) elif isinstance(action, SCons.Action.LazyAction): # pylint: disable=protected-access @@ -1043,7 +1044,7 @@ def ninja_builder(env, target, source): if env["PLATFORM"] == "win32": # this is not great, its doesn't consider specific # node environments, which means on linux the build could - # behave differently, becuase on linux you can set the environment + # behave differently, because on linux you can set the environment # per command in the ninja file. This is only needed if # running ninja directly from a command line that hasn't # had the environment setup (vcvarsall.bat) @@ -1492,7 +1493,7 @@ def robust_rule_mapping(var, rule, tool): # default to using ninja installed with python module ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( - ninja_syntax.__file__, + ninja.__file__, os.pardir, 'data', 'bin', diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 347d63902c..40404152fa 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') lib_suffix = '.lib' if IS_WINDOWS else '.so' diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index 0beb8de11c..8e7acff7b7 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') test.write('SConstruct', """ diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index 73c71f1b40..faf395a1ef 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') test.write('SConstruct', """ diff --git a/test/ninja/generate_and_build_cxx.py b/test/ninja/generate_and_build_cxx.py index ac0f55444f..663282bd92 100644 --- a/test/ninja/generate_and_build_cxx.py +++ b/test/ninja/generate_and_build_cxx.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') test.write('SConstruct', """ diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index d9b9c4ed59..76c79bb7da 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') shell = '' if IS_WINDOWS else './' diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index b03e9cbcb0..ff50f502a3 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -27,10 +27,11 @@ import os import time import random -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -40,14 +41,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') test.write('source_0.c', """ @@ -220,14 +219,14 @@ def mod_source_orig(test_num): scons_times += [stop - start] full_build_print = True -for ninja, scons in zip(ninja_times, scons_times): - if ninja > scons: +for ninja_time, scons_time in zip(ninja_times, scons_times): + if ninja_time > scons_time: test.fail_test() if full_build_print: full_build_print = False - print("Clean build {} files - SCons: {:.3f}s Ninja: {:.3f}s".format(num_source, scons, ninja)) + print("Clean build {} files - SCons: {:.3f}s Ninja: {:.3f}s".format(num_source, scons_time, ninja_time)) else: - print("Single File Rebuild - SCons: {:.3f}s Ninja: {:.3f}s".format(scons, ninja)) + print("Single File Rebuild - SCons: {:.3f}s Ninja: {:.3f}s".format(scons_time, ninja_time)) test.pass_test() diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index c9b21b7d70..18ca3cbc69 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,7 +39,7 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index b35a52b6c0..5d7f97e215 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -25,10 +25,11 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os -import importlib import TestSCons from TestCmd import IS_WINDOWS +test = TestSCons.TestSCons() + try: import ninja except ImportError: @@ -38,14 +39,12 @@ _exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( - importlib.import_module(".ninja_syntax", package='ninja').__file__, + ninja.__file__, os.pardir, 'data', 'bin', 'ninja' + _exe)) -test = TestSCons.TestSCons() - test.dir_fixture('ninja-fixture') shell = '' if IS_WINDOWS else './' From ac0bc43ed1fe945f7e5fc5ce0c0ea3ce74b5ef05 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 21 May 2020 16:23:13 -0400 Subject: [PATCH 108/163] removed NINJA_SYNTAX completely --- src/engine/SCons/Tool/ninja.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index a68655ef1d..5515a35413 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -40,7 +40,6 @@ from SCons.Script import COMMAND_LINE_TARGETS NINJA_STATE = None -NINJA_SYNTAX = "NINJA_SYNTAX" NINJA_RULES = "__NINJA_CUSTOM_RULES" NINJA_POOLS = "__NINJA_CUSTOM_POOLS" NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" @@ -1299,7 +1298,6 @@ def generate(env): env["DISABLE_AUTO_NINJA"] = GetOption('disable_execute_ninja') global NINJA_STATE - env[NINJA_SYNTAX] = env.get(NINJA_SYNTAX, "ninja_syntax.py") # Add the Ninja builder. always_exec_ninja_action = AlwaysExecAction(ninja_builder, {}) From 53fdf4f35ce2b7cbd630b6f16b9b151cfb9fec5e Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 22 May 2020 00:49:03 -0400 Subject: [PATCH 109/163] removed old sconscript changes --- SCons/Script/SConscript.py | 3 --- SCons/Script/__init__.py | 1 - src/engine/SCons/Tool/ninja.py | 2 +- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/SCons/Script/SConscript.py b/SCons/Script/SConscript.py index ded0fcfef9..596fca0463 100644 --- a/SCons/Script/SConscript.py +++ b/SCons/Script/SConscript.py @@ -203,11 +203,9 @@ def _SConscript(fs, *files, **kw): if f.rexists(): actual = f.rfile() _file_ = open(actual.get_abspath(), "rb") - SCons.Script.LOADED_SCONSCRIPTS.append(actual.get_abspath()) elif f.srcnode().rexists(): actual = f.srcnode().rfile() _file_ = open(actual.get_abspath(), "rb") - SCons.Script.LOADED_SCONSCRIPTS.append(actual.get_abspath()) elif f.has_src_builder(): # The SConscript file apparently exists in a source # code management system. Build it, but then clear @@ -216,7 +214,6 @@ def _SConscript(fs, *files, **kw): f.build() f.built() f.builder_set(None) - SCons.Script.LOADED_SCONSCRIPTS.append(f.get_abspath()) if f.exists(): _file_ = open(f.get_abspath(), "rb") if _file_: diff --git a/SCons/Script/__init__.py b/SCons/Script/__init__.py index e409f06ef4..5f58d9972d 100644 --- a/SCons/Script/__init__.py +++ b/SCons/Script/__init__.py @@ -187,7 +187,6 @@ def _clear(self): BUILD_TARGETS = TargetList() COMMAND_LINE_TARGETS = [] DEFAULT_TARGETS = [] -LOADED_SCONSCRIPTS = [] # BUILD_TARGETS can be modified in the SConscript files. If so, we # want to treat the modified BUILD_TARGETS list as if they specified diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 5515a35413..205bfa79a9 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -727,7 +727,7 @@ def generate(self): # allow us to query the actual SConscripts used. Right now # this glob method has deficiencies like skipping # jstests/SConscript and being specific to the MongoDB - # repository layout. + # repository layout. (github issue #3625) ninja.build( self.ninja_file.path, rule="REGENERATE", From 60c6d882b551eb84c331f3dba2758bbc3cd3572d Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 3 Jun 2020 10:47:54 -0400 Subject: [PATCH 110/163] merge commit a7541c60e5904e7deafdedf5bb040cc8924ac7d3 from https://github.com/mongodb/mongo --- src/engine/SCons/Tool/ninja.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 205bfa79a9..689f2ee262 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -1463,6 +1463,16 @@ def robust_rule_mapping(var, rule, tool): SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig) + # Ignore CHANGED_SOURCES and CHANGED_TARGETS. We don't want those + # to have effect in a generation pass because the generator + # shouldn't generate differently depending on the current local + # state. Without this, when generating on Windows, if you already + # had a foo.obj, you would omit foo.cpp from the response file. Do the same for UNCHANGED. + SCons.Executor.Executor._get_changed_sources = SCons.Executor.Executor._get_sources + SCons.Executor.Executor._get_changed_targets = SCons.Executor.Executor._get_targets + SCons.Executor.Executor._get_unchanged_sources = SCons.Executor.Executor._get_sources + SCons.Executor.Executor._get_unchanged_targets = SCons.Executor.Executor._get_targets + # Replace false action messages with nothing. env["PRINT_CMD_LINE_FUNC"] = ninja_noop From d1092d6ce0b8d92a30f93bcd5fa7a8f6d403a2d9 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 3 Jun 2020 10:49:15 -0400 Subject: [PATCH 111/163] merge commit 18cbf0d581162b2d15d66577b1fe08fe22006699 from https://github.com/mongodb/mongo --- src/engine/SCons/Tool/ninja.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 689f2ee262..3a1034fea2 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -675,7 +675,16 @@ def generate(self): # use for the "real" builder and multiple phony targets that # match the file names of the remaining outputs. This way any # build can depend on any output from any build. - build["outputs"].sort() + # + # We assume that the first listed output is the 'key' + # output and is stably presented to us by SCons. For + # instance if -gsplit-dwarf is in play and we are + # producing foo.o and foo.dwo, we expect that outputs[0] + # from SCons will be the foo.o file and not the dwo + # file. If instead we just sorted the whole outputs array, + # we would find that the dwo file becomes the + # first_output, and this breaks, for instance, header + # dependency scanning. if rule is not None and (rule.get("deps") or rule.get("rspfile")): first_output, remaining_outputs = ( build["outputs"][0], @@ -684,7 +693,7 @@ def generate(self): if remaining_outputs: ninja.build( - outputs=remaining_outputs, rule="phony", implicit=first_output, + outputs=sorted(remaining_outputs), rule="phony", implicit=first_output, ) build["outputs"] = first_output From d63a2d66df15b0447241426e1df758bcab257879 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 8 Jun 2020 21:43:59 -0400 Subject: [PATCH 112/163] update to build godot reinvoke scons for unhandled actions Ignore Python.Values (need fix) escape rsp content check is_sconscript fix sider issues --- src/engine/SCons/Tool/ninja.py | 98 +++++++++++++++++++--------------- 1 file changed, 56 insertions(+), 42 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 3a1034fea2..3fdadf9d77 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -30,7 +30,6 @@ import shlex import subprocess -from glob import glob from os.path import join as joinpath from os.path import splitext @@ -38,6 +37,7 @@ from SCons.Action import _string_from_cmd_list, get_default_ENV from SCons.Util import is_List, flatten_sequence from SCons.Script import COMMAND_LINE_TARGETS +from SCons.Node import SConscriptNodes NINJA_STATE = None NINJA_RULES = "__NINJA_CUSTOM_RULES" @@ -58,10 +58,11 @@ def _install_action_function(_env, node): """Install files using the install or copy commands""" + #TODO: handle Python.Value nodes return { "outputs": get_outputs(node), "rule": "INSTALL", - "inputs": [get_path(src_file(s)) for s in node.sources], + "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], "implicit": get_dependencies(node), } @@ -83,9 +84,10 @@ def _mkdir_action_function(env, node): } def _copy_action_function(env, node): + #TODO: handle Python.Value nodes return { "outputs": get_outputs(node), - "inputs": [get_path(src_file(s)) for s in node.sources], + "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], "rule": "CMD", "variables": { "cmd": "$COPY $in $out", @@ -134,11 +136,12 @@ def is_valid_dependent_node(node): def alias_to_ninja_build(node): """Convert an Alias node into a Ninja phony target""" + # TODO: handle Python.Values return { "outputs": get_outputs(node), "rule": "phony", "implicit": [ - get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) + get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) and not isinstance(n, SCons.Node.Python.Value) ], } @@ -147,18 +150,20 @@ def get_order_only(node): """Return a list of order only dependencies for node.""" if node.prerequisites is None: return [] - return [get_path(src_file(prereq)) for prereq in node.prerequisites] + #TODO: handle Python.Value nodes + return [get_path(src_file(prereq)) for prereq in node.prerequisites if not isinstance(prereq, SCons.Node.Python.Value)] def get_dependencies(node, skip_sources=False): """Return a list of dependencies for node.""" + #TODO: handle Python.Value nodes if skip_sources: return [ get_path(src_file(child)) for child in node.children() - if child not in node.sources + if child not in node.sources and not isinstance(child, SCons.Node.Python.Value) ] - return [get_path(src_file(child)) for child in node.children()] + return [get_path(src_file(child)) for child in node.children() if not isinstance(child, SCons.Node.Python.Value)] def get_inputs(node): @@ -168,8 +173,8 @@ def get_inputs(node): inputs = executor.get_all_sources() else: inputs = node.sources - - inputs = [get_path(src_file(o)) for o in inputs] + #TODO: handle Python.Value nodes + inputs = [get_path(src_file(o)) for o in inputs if not isinstance(o, SCons.Node.Python.Value)] return inputs @@ -202,6 +207,7 @@ def __init__(self, env): # this check for us. "SharedFlagChecker": ninja_noop, # The install builder is implemented as a function action. + # TODO: use command action #3573 "installFunc": _install_action_function, "MkdirFunc": _mkdir_action_function, "LibSymlinksActionFunction": _lib_symlink_action_function, @@ -262,12 +268,6 @@ def handle_func_action(self, node, action): # the bottom of ninja's DAG anyway and Textfile builders can have text # content as their source which doesn't work as an implicit dep in # ninja. - if name == "_action": - return { - "rule": "TEMPLATE", - "outputs": get_outputs(node), - "implicit": get_dependencies(node, skip_sources=True), - } if name == 'ninja_builder': return None @@ -280,7 +280,7 @@ def handle_func_action(self, node, action): if handler is not None: return handler(node.env if node.env else self.env, node) - raise Exception( + SCons.Warnings.Warning( "Found unhandled function action {}, " " generating scons command to build\n" "Note: this is less efficient than Ninja," @@ -288,6 +288,12 @@ def handle_func_action(self, node, action): " this function using NinjaRegisterFunctionHandler".format(name) ) + return { + "rule": "TEMPLATE", + "outputs": get_outputs(node), + "implicit": get_dependencies(node, skip_sources=True), + } + # pylint: disable=too-many-branches def handle_list_action(self, node, action): """TODO write this comment""" @@ -309,7 +315,7 @@ def handle_list_action(self, node, action): all_outputs = list({output for build in results for output in build["outputs"]}) dependencies = list({dep for build in results for dep in build["implicit"]}) - if results[0]["rule"] == "CMD": + if results[0]["rule"] == "CMD" or results[0]["rule"] == "GENERATED_CMD": cmdline = "" for cmd in results: @@ -339,7 +345,7 @@ def handle_list_action(self, node, action): if cmdline: ninja_build = { "outputs": all_outputs, - "rule": "CMD", + "rule": "GENERATED_CMD", "variables": { "cmd": cmdline, "env": get_command_env(node.env if node.env else self.env), @@ -360,10 +366,11 @@ def handle_list_action(self, node, action): } elif results[0]["rule"] == "INSTALL": + #TODO: handle Python.Value nodes return { "outputs": all_outputs, "rule": "INSTALL", - "inputs": [get_path(src_file(s)) for s in node.sources], + "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], "implicit": dependencies, } @@ -397,22 +404,28 @@ def __init__(self, env, ninja_file, writer_class): # like CCFLAGS escape = env.get("ESCAPE", lambda x: x) + # if SCons was invoked from python, we expect the first arg to be the scons.py + # script, otherwise scons was invoked from the scons script + python_bin = '' + if os.path.basename(sys.argv[0]) == 'scons.py': + python_bin = escape(sys.executable) self.variables = { "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", - "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format( - sys.executable, + "SCONS_INVOCATION": '{} {} --disable-ninja __NINJA_NO=1 $out'.format( + python_bin, " ".join( [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] ), ), - "SCONS_INVOCATION_W_TARGETS": "{} {}".format( - sys.executable, " ".join([escape(arg) for arg in sys.argv]) + "SCONS_INVOCATION_W_TARGETS": "{} {} --disable-ninja".format( + python_bin, " ".join([escape(arg) for arg in sys.argv]) ), # This must be set to a global default per: - # https://ninja-build.org/manual.html - # - # (The deps section) - "msvc_deps_prefix": "Note: including file:", + # https://ninja-build.org/manual.html#_deps + # English Visual Studio will have the default below, + # otherwise the user can define the variable in the first environment + # that initialized ninja tool + "msvc_deps_prefix": env.get("NINJA_MSVC_DEPS_PREFIX", "Note: including file:") } self.rules = { @@ -481,13 +494,13 @@ def __init__(self, env, ninja_file, writer_class): }, "TEMPLATE": { "command": "$SCONS_INVOCATION $out", - "description": "Rendering $out", + "description": "Rendering $SCONS_INVOCATION $out", "pool": "scons_pool", "restat": 1, }, "SCONS": { "command": "$SCONS_INVOCATION $out", - "description": "SCons $out", + "description": "$SCONS_INVOCATION $out", "pool": "scons_pool", # restat # if present, causes Ninja to re-stat the command's outputs @@ -557,6 +570,8 @@ def add_build(self, node): self.built.update(build["outputs"]) return True + # TODO: rely on SCons to tell us what is generated source + # or some form of user scanner maybe (Github Issue #3624) def is_generated_source(self, output): """Check if output ends with a known generated suffix.""" _, suffix = splitext(output) @@ -740,11 +755,7 @@ def generate(self): ninja.build( self.ninja_file.path, rule="REGENERATE", - implicit=[ - self.env.File("#SConstruct").path, - __file__, - ] - + sorted(glob("src/**/SConscript", recursive=True)), + implicit=[__file__] + [str(node) for node in SConscriptNodes], ) # If we ever change the name/s of the rules that include @@ -921,6 +932,7 @@ def get_response_file_command(env, node, action, targets, sources, executor=None ) cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] + rsp_content = ['"' + rsp_content_item + '"' for rsp_content_item in rsp_content] rsp_content = " ".join(rsp_content) variables = {"rspc": rsp_content} @@ -1060,20 +1072,23 @@ def ninja_builder(env, target, source): for key in env['ENV']: f.write('set {}={}\n'.format(key, env['ENV'][key])) f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) - - if not env.get("DISABLE_AUTO_NINJA"): + cmd = ['run_ninja_env.bat'] + + else: cmd = [NINJA_STATE.ninja_bin_path, '-f', generated_build_ninja] + + if not env.get("DISABLE_AUTO_NINJA"): print("Executing:", str(' '.join(cmd))) # execute the ninja build at the end of SCons, trying to # reproduce the output like a ninja build would def execute_ninja(): - + proc = subprocess.Popen(cmd, stderr=sys.stderr, stdout=subprocess.PIPE, universal_newlines=True, - env=env['ENV'] # ninja build items won't consider node env on win32 + env=os.environ if env["PLATFORM"] == "win32" else env['ENV'] ) for stdout_line in iter(proc.stdout.readline, ""): yield stdout_line @@ -1142,8 +1157,7 @@ def ninja_csig(original): """Return a dummy csig""" def wrapper(self): - name = str(self) - if "SConscript" in name or "SConstruct" in name: + if isinstance(self, SCons.Node.Node) and self.is_sconscript(): return original(self) return "dummy_ninja_csig" @@ -1154,8 +1168,7 @@ def ninja_contents(original): """Return a dummy content without doing IO""" def wrapper(self): - name = str(self) - if "SConscript" in name or "SConstruct" in name: + if isinstance(self, SCons.Node.Node) and self.is_sconscript(): return original(self) return bytes("dummy_ninja_contents", encoding="utf-8") @@ -1396,6 +1409,7 @@ def robust_rule_mapping(var, rule, tool): # Used to determine if a build generates a source file. Ninja # requires that all generated sources are added as order_only # dependencies to any builds that *might* use them. + # TODO: switch to using SCons to help determine this (Github Issue #3624) env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] if env["PLATFORM"] != "win32" and env.get("RANLIBCOM"): From e7fc941d25f3bc5a70efab20755035a076ef93af Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 22 Jun 2020 15:21:34 -0400 Subject: [PATCH 113/163] revert ninja install requirement expand response file in ninja comdb output --- src/engine/SCons/Tool/ninja.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 3fdadf9d77..ad36758788 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -767,7 +767,7 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "{} -f {} -t compdb CC CXX > compile_commands.json".format(self.ninja_bin_path, + "cmd": "{} -f {} -t compdb -x CC CXX > compile_commands.json".format(self.ninja_bin_path, str(self.ninja_file) ) }, From ea80011d05ce5eebec447a77d02980de57887133 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 25 Jun 2020 23:05:28 -0400 Subject: [PATCH 114/163] handle files which are not file or alias by reinvoking scons --- src/engine/SCons/Tool/ninja.py | 228 +++++++++++++++++++-------------- 1 file changed, 134 insertions(+), 94 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index ad36758788..50a9518aae 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -56,63 +56,6 @@ ) -def _install_action_function(_env, node): - """Install files using the install or copy commands""" - #TODO: handle Python.Value nodes - return { - "outputs": get_outputs(node), - "rule": "INSTALL", - "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], - "implicit": get_dependencies(node), - } - -def _mkdir_action_function(env, node): - return { - "outputs": get_outputs(node), - "rule": "CMD", - # implicit explicitly omitted, we translate these so they can be - # used by anything that depends on these but commonly this is - # hit with a node that will depend on all of the fake - # srcnode's that SCons will never give us a rule for leading - # to an invalid ninja file. - "variables": { - # On Windows mkdir "-p" is always on - "cmd": "{mkdir}".format( - mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", - ), - }, - } - -def _copy_action_function(env, node): - #TODO: handle Python.Value nodes - return { - "outputs": get_outputs(node), - "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], - "rule": "CMD", - "variables": { - "cmd": "$COPY $in $out", - }, - } - - -def _lib_symlink_action_function(_env, node): - """Create shared object symlinks if any need to be created""" - symlinks = getattr(getattr(node, "attributes", None), "shliblinks", None) - - if not symlinks or symlinks is None: - return None - - outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] - inputs = [link.get_path() for link, _ in symlinks] - - return { - "outputs": outputs, - "inputs": inputs, - "rule": "SYMLINK", - "implicit": get_dependencies(node), - } - - def is_valid_dependent_node(node): """ Return True if node is not an alias or is an alias that has children @@ -136,46 +79,65 @@ def is_valid_dependent_node(node): def alias_to_ninja_build(node): """Convert an Alias node into a Ninja phony target""" - # TODO: handle Python.Values return { "outputs": get_outputs(node), "rule": "phony", "implicit": [ - get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) and not isinstance(n, SCons.Node.Python.Value) + get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) ], } +def check_invalid_ninja_node(node): + return not isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)) + + +def filter_ninja_nodes(node_list): + ninja_nodes = [] + for node in node_list: + if isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)): + ninja_nodes.append(node) + else: + continue + return ninja_nodes + +def get_input_nodes(node): + if node.get_executor() is not None: + inputs = node.get_executor().get_all_sources() + else: + inputs = node.sources + return inputs + + +def invalid_ninja_nodes(node, targets): + result = False + for node_list in [node.prerequisites, get_input_nodes(node), node.children(), targets]: + if node_list: + result = result or any([check_invalid_ninja_node(node) for node in node_list]) + return result + + def get_order_only(node): """Return a list of order only dependencies for node.""" if node.prerequisites is None: return [] - #TODO: handle Python.Value nodes - return [get_path(src_file(prereq)) for prereq in node.prerequisites if not isinstance(prereq, SCons.Node.Python.Value)] + return [get_path(src_file(prereq)) for prereq in filter_ninja_nodes(node.prerequisites)] def get_dependencies(node, skip_sources=False): """Return a list of dependencies for node.""" - #TODO: handle Python.Value nodes if skip_sources: return [ get_path(src_file(child)) - for child in node.children() - if child not in node.sources and not isinstance(child, SCons.Node.Python.Value) + for child in filter_ninja_nodes(node.children()) + if child not in node.sources ] - return [get_path(src_file(child)) for child in node.children() if not isinstance(child, SCons.Node.Python.Value)] + return [get_path(src_file(child)) for child in filter_ninja_nodes(node.children())] def get_inputs(node): """Collect the Ninja inputs for node.""" - executor = node.get_executor() - if executor is not None: - inputs = executor.get_all_sources() - else: - inputs = node.sources - #TODO: handle Python.Value nodes - inputs = [get_path(src_file(o)) for o in inputs if not isinstance(o, SCons.Node.Python.Value)] - return inputs + return [get_path(src_file(o)) for o in filter_ninja_nodes(get_input_nodes(node))] def get_outputs(node): @@ -189,11 +151,93 @@ def get_outputs(node): else: outputs = [node] - outputs = [get_path(o) for o in outputs] + outputs = [get_path(o) for o in filter_ninja_nodes(outputs)] return outputs +def get_targets_sources(node): + executor = node.get_executor() + if executor is not None: + tlist = executor.get_all_targets() + slist = executor.get_all_sources() + else: + if hasattr(node, "target_peers"): + tlist = node.target_peers + else: + tlist = [node] + slist = node.sources + + # Retrieve the repository file for all sources + slist = [rfile(s) for s in slist] + return tlist, slist + + +def get_rule(node, rule): + tlist, slist = get_targets_sources(node) + if invalid_ninja_nodes(node, tlist): + return "TEMPLATE" + else: + return rule + + +def _install_action_function(_env, node): + """Install files using the install or copy commands""" + return { + "outputs": get_outputs(node), + "rule": get_rule(node, "INSTALL"), + "inputs": get_inputs(node), + "implicit": get_dependencies(node), + } + + +def _mkdir_action_function(env, node): + return { + "outputs": get_outputs(node), + "rule": get_rule(node, "CMD"), + # implicit explicitly omitted, we translate these so they can be + # used by anything that depends on these but commonly this is + # hit with a node that will depend on all of the fake + # srcnode's that SCons will never give us a rule for leading + # to an invalid ninja file. + "variables": { + # On Windows mkdir "-p" is always on + "cmd": "{mkdir}".format( + mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", + ), + }, + } + + +def _copy_action_function(env, node): + return { + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "rule": get_rule(node, "CMD"), + "variables": { + "cmd": "$COPY $in $out", + }, + } + + +def _lib_symlink_action_function(_env, node): + """Create shared object symlinks if any need to be created""" + symlinks = getattr(getattr(node, "attributes", None), "shliblinks", None) + + if not symlinks or symlinks is None: + return None + + outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] + inputs = [link.get_path() for link, _ in symlinks] + + return { + "outputs": outputs, + "inputs": inputs, + "rule": get_rule(node, "SYMLINK"), + "implicit": get_dependencies(node), + } + + class SConsToNinjaTranslator: """Translates SCons Actions into Ninja build objects.""" @@ -290,7 +334,9 @@ def handle_func_action(self, node, action): return { "rule": "TEMPLATE", + "order_only": get_order_only(node), "outputs": get_outputs(node), + "inputs": get_inputs(node), "implicit": get_dependencies(node, skip_sources=True), } @@ -345,7 +391,7 @@ def handle_list_action(self, node, action): if cmdline: ninja_build = { "outputs": all_outputs, - "rule": "GENERATED_CMD", + "rule": get_rule(node, "GENERATED_CMD"), "variables": { "cmd": cmdline, "env": get_command_env(node.env if node.env else self.env), @@ -366,11 +412,10 @@ def handle_list_action(self, node, action): } elif results[0]["rule"] == "INSTALL": - #TODO: handle Python.Value nodes return { "outputs": all_outputs, - "rule": "INSTALL", - "inputs": [get_path(src_file(s)) for s in node.sources if not isinstance(s, SCons.Node.Python.Value)], + "rule": get_rule(node, "INSTALL"), + "inputs": get_inputs(node), "implicit": dependencies, } @@ -985,20 +1030,8 @@ def get_command(env, node, action): # pylint: disable=too-many-branches sub_env = node.env else: sub_env = env - executor = node.get_executor() - if executor is not None: - tlist = executor.get_all_targets() - slist = executor.get_all_sources() - else: - if hasattr(node, "target_peers"): - tlist = node.target_peers - else: - tlist = [node] - slist = node.sources - - # Retrieve the repository file for all sources - slist = [rfile(s) for s in slist] + tlist, slist = get_targets_sources(node) # Generate a real CommandAction if isinstance(action, SCons.Action.CommandGeneratorAction): @@ -1022,10 +1055,11 @@ def get_command(env, node, action): # pylint: disable=too-many-branches "outputs": get_outputs(node), "inputs": get_inputs(node), "implicit": implicit, - "rule": rule, + "rule": get_rule(node, rule), "variables": variables, } + # Don't use sub_env here because we require that NINJA_POOL be set # on a per-builder call basis to prevent accidental strange # behavior like env['NINJA_POOL'] = 'console' and sub_env can be @@ -1283,6 +1317,11 @@ def exists(env): if env.get("__NINJA_NO", "0") == "1": return False + try: + import ninja + except ImportError: + SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") + return False return True added = None @@ -1308,8 +1347,6 @@ def generate(env): default=False, help='Disable ninja automatically building after scons') - if GetOption('disable_ninja'): - return env try: import ninja @@ -1427,6 +1464,9 @@ def robust_rule_mapping(var, rule, tool): # Disable running ranlib, since we added 's' above env["RANLIBCOM"] = "" + if GetOption('disable_ninja'): + return env + SCons.Warnings.Warning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") # This is the point of no return, anything after this comment From e9d0fd6fdf1658993ec346b5d49b45bed0ebe667 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 20 Jul 2020 17:31:56 +0000 Subject: [PATCH 115/163] updated with some changes from latest mongodb version: 21075112a999e252a22e9c9bd64e403cec892df3 5fe923a0aa312044062df044eb4eaa47951f70ec c7348f391124e681d9c62aceb0e13e0d07fca8bc --- src/engine/SCons/Tool/ninja.py | 40 +++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 50a9518aae..bf7c45add0 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -29,6 +29,7 @@ import shutil import shlex import subprocess +import textwrap from os.path import join as joinpath from os.path import splitext @@ -768,13 +769,13 @@ def generate(self): # Special handling for outputs and implicit since we need to # aggregate not replace for each builder. - for agg_key in ["outputs", "implicit"]: + for agg_key in ["outputs", "implicit", 'inputs']: new_val = template_builds.get(agg_key, []) # Use pop so the key is removed and so the update # below will not overwrite our aggregated values. cur_val = template_builder.pop(agg_key, []) - if isinstance(cur_val, list): + if is_List(cur_val): new_val += cur_val else: new_val.append(cur_val) @@ -812,8 +813,8 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "{} -f {} -t compdb -x CC CXX > compile_commands.json".format(self.ninja_bin_path, - str(self.ninja_file) + "cmd": "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format( + ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' ) }, ) @@ -929,7 +930,13 @@ def get_command_env(env): if windows: command_env += "set '{}={}' && ".format(key, value) else: - command_env += "{}={} ".format(key, value) + # We address here *only* the specific case that a user might have + # an environment variable which somehow gets included and has + # spaces in the value. These are escapes that Ninja handles. This + # doesn't make builds on paths with spaces (Ninja and SCons issues) + # nor expanding response file paths with spaces (Ninja issue) work. + value = value.replace(r' ', r'$ ') + command_env += "{}='{}' ".format(key, value) env["NINJA_ENV_VAR_CACHE"] = command_env return command_env @@ -1208,6 +1215,27 @@ def wrapper(self): return wrapper +def CheckNinjaCompdbExpand(env, context): + """ Configure check testing if ninja's compdb can expand response files""" + + context.Message('Checking if ninja compdb can expand response files... ') + ret, output = context.TryAction( + action='ninja -f $SOURCE -t compdb -x CMD_RSP > $TARGET', + extension='.ninja', + text=textwrap.dedent(""" + rule CMD_RSP + command = $cmd @$out.rsp > fake_output.txt + description = Building $out + rspfile = $out.rsp + rspfile_content = $rspc + build fake_output.txt: CMD_RSP fake_input.txt + cmd = echo + pool = console + rspc = "test" + """)) + result = '@fake_output.txt.rsp' not in output + context.Result(result) + return result def ninja_stat(_self, path): """ @@ -1386,6 +1414,8 @@ def generate(env): else: env.Append(CCFLAGS=["-MD", "-MF", "${TARGET}.d"]) + env.AddMethod(CheckNinjaCompdbExpand, "CheckNinjaCompdbExpand") + # Provide a way for custom rule authors to easily access command # generation. env.AddMethod(get_shell_command, "NinjaGetShellCommand") From 830641d7dffbb61e85fa61f4cfca686ab0a295aa Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 20 Jul 2020 17:43:48 +0000 Subject: [PATCH 116/163] fixed sider issues --- src/engine/SCons/Tool/ninja.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index bf7c45add0..153a8943b5 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -813,8 +813,8 @@ def generate(self): pool="console", implicit=[str(self.ninja_file)], variables={ - "cmd": "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format( - ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' + "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( + self.ninja_bin_path, str(self.ninja_file), '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' ) }, ) @@ -1347,10 +1347,11 @@ def exists(env): try: import ninja + return ninja.__file__ except ImportError: SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") return False - return True + added = None From 941468745e201263fb993aae21b30d63958d9049 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 20 Jul 2020 18:25:39 +0000 Subject: [PATCH 117/163] updated warning to the latest API --- src/engine/SCons/Tool/ninja.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 153a8943b5..502d365014 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -325,7 +325,7 @@ def handle_func_action(self, node, action): if handler is not None: return handler(node.env if node.env else self.env, node) - SCons.Warnings.Warning( + SCons.Warnings.SConsWarning( "Found unhandled function action {}, " " generating scons command to build\n" "Note: this is less efficient than Ninja," @@ -1349,7 +1349,7 @@ def exists(env): import ninja return ninja.__file__ except ImportError: - SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") + SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return False @@ -1380,7 +1380,7 @@ def generate(env): try: import ninja except ImportError: - SCons.Warnings.Warning("Failed to import ninja, attempt normal SCons build.") + SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return env["DISABLE_AUTO_NINJA"] = GetOption('disable_execute_ninja') @@ -1405,7 +1405,7 @@ def generate(env): env.Alias("$NINJA_ALIAS_NAME", ninja_file) else: if str(NINJA_STATE.ninja_file) != ninja_file_name: - SCons.Warnings.Warning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") + SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] # This adds the required flags such that the generated compile @@ -1498,7 +1498,7 @@ def robust_rule_mapping(var, rule, tool): if GetOption('disable_ninja'): return env - SCons.Warnings.Warning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") + SCons.Warnings.SConsWarning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") # This is the point of no return, anything after this comment # makes changes to SCons that are irreversible and incompatible From 00c20a8859016fc087a8f4030a46a423b3bebc8f Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 31 Dec 2020 15:58:31 +0000 Subject: [PATCH 118/163] Sync with mongo ninja file --- src/engine/SCons/Tool/ninja.py | 230 ++++++++++++++++++++++++++------- 1 file changed, 186 insertions(+), 44 deletions(-) diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py index 502d365014..ea01d590ee 100644 --- a/src/engine/SCons/Tool/ninja.py +++ b/src/engine/SCons/Tool/ninja.py @@ -31,6 +31,7 @@ import subprocess import textwrap +from glob import glob from os.path import join as joinpath from os.path import splitext @@ -156,6 +157,41 @@ def get_outputs(node): return outputs +def generate_depfile(env, node, dependencies): + """ + Ninja tool function for writing a depfile. The depfile should include + the node path followed by all the dependent files in a makefile format. + + dependencies arg can be a list or a subst generator which returns a list. + """ + + depfile = os.path.join(get_path(env['NINJA_BUILDDIR']), str(node) + '.depfile') + + # subst_list will take in either a raw list or a subst callable which generates + # a list, and return a list of CmdStringHolders which can be converted into raw strings. + # If a raw list was passed in, then scons_list will make a list of lists from the original + # values and even subst items in the list if they are substitutable. Flatten will flatten + # the list in that case, to ensure for either input we have a list of CmdStringHolders. + deps_list = env.Flatten(env.subst_list(dependencies)) + + # Now that we have the deps in a list as CmdStringHolders, we can convert them into raw strings + # and make sure to escape the strings to handle spaces in paths. We also will sort the result + # keep the order of the list consistent. + escaped_depends = sorted([dep.escape(env.get("ESCAPE", lambda x: x)) for dep in deps_list]) + depfile_contents = str(node) + ": " + ' '.join(escaped_depends) + + need_rewrite = False + try: + with open(depfile, 'r') as f: + need_rewrite = (f.read() != depfile_contents) + except FileNotFoundError: + need_rewrite = True + + if need_rewrite: + os.makedirs(os.path.dirname(depfile) or '.', exist_ok=True) + with open(depfile, 'w') as f: + f.write(depfile_contents) + def get_targets_sources(node): executor = node.get_executor() @@ -278,6 +314,7 @@ def action_to_ninja_build(self, node, action=None): return None build = {} + env = node.env if node.env else self.env # Ideally this should never happen, and we do try to filter # Ninja builders out of being sources of ninja builders but I @@ -286,22 +323,27 @@ def action_to_ninja_build(self, node, action=None): global NINJA_STATE if NINJA_STATE.ninja_file == str(node): build = None - if isinstance(action, SCons.Action.FunctionAction): + elif isinstance(action, SCons.Action.FunctionAction): build = self.handle_func_action(node, action) elif isinstance(action, SCons.Action.LazyAction): # pylint: disable=protected-access - action = action._generate_cache(node.env if node.env else self.env) + action = action._generate_cache(env) build = self.action_to_ninja_build(node, action=action) elif isinstance(action, SCons.Action.ListAction): build = self.handle_list_action(node, action) elif isinstance(action, COMMAND_TYPES): - build = get_command(node.env if node.env else self.env, node, action) + build = get_command(env, node, action) else: raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) if build is not None: build["order_only"] = get_order_only(node) + if 'conftest' not in str(node): + node_callback = getattr(node.attributes, "ninja_build_callback", None) + if callable(node_callback): + node_callback(env, node, build) + return build def handle_func_action(self, node, action): @@ -509,8 +551,16 @@ def __init__(self, env, ninja_file, writer_class): "rspfile_content": "$rspc", "pool": "local_pool", }, + # Ninja does not automatically delete the archive before + # invoking ar. The ar utility will append to an existing archive, which + # can cause duplicate symbols if the symbols moved between object files. + # Native SCons will perform this operation so we need to force ninja + # to do the same. See related for more info: + # https://jira.mongodb.org/browse/SERVER-49457 "AR": { - "command": "$env$AR @$out.rsp", + "command": "{}$env$AR @$out.rsp".format( + '' if sys.platform == "win32" else "rm -f $out && " + ), "description": "Archiving $out", "rspfile": "$out.rsp", "rspfile_content": "$rspc", @@ -540,7 +590,7 @@ def __init__(self, env, ninja_file, writer_class): }, "TEMPLATE": { "command": "$SCONS_INVOCATION $out", - "description": "Rendering $SCONS_INVOCATION $out", + "description": "Rendering $SCONS_INVOCATION $out", "pool": "scons_pool", "restat": 1, }, @@ -570,6 +620,7 @@ def __init__(self, env, ninja_file, writer_class): "command": "$SCONS_INVOCATION_W_TARGETS", "description": "Regenerating $out", "generator": 1, + "depfile": os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile'), # Console pool restricts to 1 job running at a time, # it additionally has some special handling about # passing stdin, stdout, etc to process in this pool @@ -650,6 +701,8 @@ def generate(self): ninja.comment("Generated by scons. DO NOT EDIT.") + ninja.variable("builddir", get_path(self.env['NINJA_BUILDDIR'])) + for pool_name, size in self.pools.items(): ninja.pool(pool_name, size) @@ -759,9 +812,19 @@ def generate(self): build["outputs"] = first_output + # Optionally a rule can specify a depfile, and SCons can generate implicit + # dependencies into the depfile. This allows for dependencies to come and go + # without invalidating the ninja file. The depfile was created in ninja specifically + # for dealing with header files appearing and disappearing across rebuilds, but it can + # be repurposed for anything, as long as you have a way to regenerate the depfile. + # More specific info can be found here: https://ninja-build.org/manual.html#_depfile + if rule is not None and rule.get('depfile') and build.get('deps_files'): + path = build['outputs'] if SCons.Util.is_List(build['outputs']) else [build['outputs']] + generate_depfile(self.env, path[0], build.pop('deps_files', [])) + if "inputs" in build: build["inputs"].sort() - + ninja.build(**build) template_builds = dict() @@ -769,7 +832,7 @@ def generate(self): # Special handling for outputs and implicit since we need to # aggregate not replace for each builder. - for agg_key in ["outputs", "implicit", 'inputs']: + for agg_key in ["outputs", "implicit", "inputs"]: new_val = template_builds.get(agg_key, []) # Use pop so the key is removed and so the update @@ -793,15 +856,25 @@ def generate(self): # generate this rule even though SCons should know we're # dependent on SCons files. # - # TODO: We're working on getting an API into SCons that will - # allow us to query the actual SConscripts used. Right now - # this glob method has deficiencies like skipping - # jstests/SConscript and being specific to the MongoDB - # repository layout. (github issue #3625) + # The REGENERATE rule uses depfile, so we need to generate the depfile + # in case any of the SConscripts have changed. The depfile needs to be + # path with in the build and the passed ninja file is an abspath, so + # we will use SCons to give us the path within the build. Normally + # generate_depfile should not be called like this, but instead be called + # through the use of custom rules, and filtered out in the normal + # list of build generation about. However, because the generate rule + # is hardcoded here, we need to do this generate_depfile call manually. + ninja_file_path = self.env.File(self.ninja_file).path + generate_depfile( + self.env, + ninja_file_path, + self.env['NINJA_REGENERATE_DEPS'] + ) + ninja.build( - self.ninja_file.path, + ninja_file_path, rule="REGENERATE", - implicit=[__file__] + [str(node) for node in SConscriptNodes], + implicit=[__file__], ) # If we ever change the name/s of the rules that include @@ -936,13 +1009,13 @@ def get_command_env(env): # doesn't make builds on paths with spaces (Ninja and SCons issues) # nor expanding response file paths with spaces (Ninja issue) work. value = value.replace(r' ', r'$ ') - command_env += "{}='{}' ".format(key, value) + command_env += "export {}='{}';".format(key, value) env["NINJA_ENV_VAR_CACHE"] = command_env return command_env -def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False): +def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom_env={}): """Generate a response file command provider for rule name.""" # If win32 using the environment with a response file command will cause @@ -979,7 +1052,7 @@ def get_response_file_command(env, node, action, targets, sources, executor=None except ValueError: raise Exception( "Could not find tool {} in {} generated from {}".format( - tool_command, cmd_list, get_comstr(env, action, targets, sources) + tool, cmd_list, get_comstr(env, action, targets, sources) ) ) @@ -991,7 +1064,12 @@ def get_response_file_command(env, node, action, targets, sources, executor=None variables[rule] = cmd if use_command_env: variables["env"] = get_command_env(env) - return rule, variables + + for key, value in custom_env.items(): + variables["env"] += env.subst( + f"export {key}={value};", target=targets, source=sources, executor=executor + ) + " " + return rule, variables, [tool_command] return get_response_file_command @@ -1021,13 +1099,21 @@ def generate_command(env, node, action, targets, sources, executor=None): return cmd.replace("$", "$$") -def get_shell_command(env, node, action, targets, sources, executor=None): +def get_generic_shell_command(env, node, action, targets, sources, executor=None): return ( - "GENERATED_CMD", + "CMD", { "cmd": generate_command(env, node, action, targets, sources, executor=None), "env": get_command_env(env), }, + # Since this function is a rule mapping provider, it must return a list of dependencies, + # and usually this would be the path to a tool, such as a compiler, used for this rule. + # However this function is to generic to be able to reliably extract such deps + # from the command, so we return a placeholder empty list. It should be noted that + # generally this function will not be used soley and is more like a template to generate + # the basics for a custom provider which may have more specific options for a provier + # function for a custom NinjaRuleMapping. + [] ) @@ -1050,13 +1136,49 @@ def get_command(env, node, action): # pylint: disable=too-many-branches comstr = get_comstr(sub_env, action, tlist, slist) if not comstr: return None - - provider = __NINJA_RULE_MAPPING.get(comstr, get_shell_command) - rule, variables = provider(sub_env, node, action, tlist, slist, executor=executor) + + provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) + rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) # Get the dependencies for all targets implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) + # Now add in the other dependencies related to the command, + # e.g. the compiler binary. The ninja rule can be user provided so + # we must do some validation to resolve the dependency path for ninja. + for provider_dep in provider_deps: + + provider_dep = sub_env.subst(provider_dep) + if not provider_dep: + continue + + # If the tool is a node, then SCons will resolve the path later, if its not + # a node then we assume it generated from build and make sure it is existing. + if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep): + implicit.append(provider_dep) + continue + + # in some case the tool could be in the local directory and be suppled without the ext + # such as in windows, so append the executable suffix and check. + prog_suffix = sub_env.get('PROGSUFFIX', '') + provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix + if os.path.exists(provider_dep_ext): + implicit.append(provider_dep_ext) + continue + + # Many commands will assume the binary is in the path, so + # we accept this as a possible input from a given command. + + provider_dep_abspath = sub_env.WhereIs(provider_dep) or sub_env.WhereIs(provider_dep, path=os.environ["PATH"]) + if provider_dep_abspath: + implicit.append(provider_dep_abspath) + continue + + # Possibly these could be ignore and the build would still work, however it may not always + # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided + # ninja rule. + raise Exception(f"Could not resolve path for {provider_dep} dependency on node '{node}'") + ninja_build = { "order_only": get_order_only(node), "outputs": get_outputs(node), @@ -1066,7 +1188,6 @@ def get_command(env, node, action): # pylint: disable=too-many-branches "variables": variables, } - # Don't use sub_env here because we require that NINJA_POOL be set # on a per-builder call basis to prevent accidental strange # behavior like env['NINJA_POOL'] = 'console' and sub_env can be @@ -1103,28 +1224,28 @@ def ninja_builder(env, target, source): NINJA_STATE.generate() if env["PLATFORM"] == "win32": - # this is not great, its doesn't consider specific + # this is not great, its doesn't consider specific # node environments, which means on linux the build could # behave differently, because on linux you can set the environment - # per command in the ninja file. This is only needed if + # per command in the ninja file. This is only needed if # running ninja directly from a command line that hasn't # had the environment setup (vcvarsall.bat) with open('run_ninja_env.bat', 'w') as f: for key in env['ENV']: f.write('set {}={}\n'.format(key, env['ENV'][key])) - f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) - cmd = ['run_ninja_env.bat'] + f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) + cmd = ['run_ninja_env.bat'] else: cmd = [NINJA_STATE.ninja_bin_path, '-f', generated_build_ninja] - + if not env.get("DISABLE_AUTO_NINJA"): print("Executing:", str(' '.join(cmd))) # execute the ninja build at the end of SCons, trying to # reproduce the output like a ninja build would def execute_ninja(): - + proc = subprocess.Popen(cmd, stderr=sys.stderr, stdout=subprocess.PIPE, @@ -1137,7 +1258,7 @@ def execute_ninja(): return_code = proc.wait() if return_code: raise subprocess.CalledProcessError(return_code, 'ninja') - + erase_previous = False for output in execute_ninja(): output = output.strip() @@ -1168,24 +1289,31 @@ def register_custom_handler(env, name, handler): def register_custom_rule_mapping(env, pre_subst_string, rule): - """Register a custom handler for SCons function actions.""" + """Register a function to call for a given rule.""" global __NINJA_RULE_MAPPING __NINJA_RULE_MAPPING[pre_subst_string] = rule -def register_custom_rule(env, rule, command, description="", deps=None, pool=None): +def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): """Allows specification of Ninja rules from inside SCons files.""" rule_obj = { "command": command, "description": description if description else "{} $out".format(rule), } + if use_depfile: + rule_obj["depfile"] = os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile') + if deps is not None: rule_obj["deps"] = deps if pool is not None: rule_obj["pool"] = pool + if use_response_file: + rule_obj["rspfile"] = "$out.rsp" + rule_obj["rspfile_content"] = response_file_content + env[NINJA_RULES][rule] = rule_obj @@ -1193,6 +1321,9 @@ def register_custom_pool(env, pool, size): """Allows the creation of custom Ninja pools""" env[NINJA_POOLS][pool] = size +def set_build_node_callback(env, node, callback): + if 'conftest' not in str(node): + setattr(node.attributes, "ninja_build_callback", callback) def ninja_csig(original): """Return a dummy csig""" @@ -1395,7 +1526,7 @@ def generate(env): env["NINJA_PREFIX"] = env.get("NINJA_PREFIX", "build") env["NINJA_SUFFIX"] = env.get("NINJA_SUFFIX", "ninja") env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") - + env['NINJA_BUILDDIR'] = env.get("NINJA_BUILDDIR", env.Dir(".ninja").path) ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") # here we allow multiple environments to construct rules and builds # into the same ninja file @@ -1407,20 +1538,31 @@ def generate(env): if str(NINJA_STATE.ninja_file) != ninja_file_name: SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] - + + + # TODO: API for getting the SConscripts programmatically + # exists upstream: https://github.com/SCons/scons/issues/3625 + def ninja_generate_deps(env): + return sorted([env.File("#SConstruct").path] + glob("**/SConscript", recursive=True)) + env['_NINJA_REGENERATE_DEPS_FUNC'] = ninja_generate_deps + + env['NINJA_REGENERATE_DEPS'] = env.get('NINJA_REGENERATE_DEPS', '${_NINJA_REGENERATE_DEPS_FUNC(__env__)}') + # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. if env["PLATFORM"] == "win32": env.Append(CCFLAGS=["/showIncludes"]) else: - env.Append(CCFLAGS=["-MD", "-MF", "${TARGET}.d"]) + env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) env.AddMethod(CheckNinjaCompdbExpand, "CheckNinjaCompdbExpand") # Provide a way for custom rule authors to easily access command # generation. - env.AddMethod(get_shell_command, "NinjaGetShellCommand") + env.AddMethod(get_generic_shell_command, "NinjaGetGenericShellCommand") + env.AddMethod(get_command, "NinjaGetCommand") env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider") + env.AddMethod(set_build_node_callback, "NinjaSetBuildNodeCallback") # Provides a way for users to handle custom FunctionActions they # want to translate to Ninja. @@ -1587,7 +1729,7 @@ def robust_rule_mapping(var, rule, tool): SCons.Job.Jobs.__init__ = ninja_always_serial ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') - + if NINJA_STATE is None: NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) NINJA_STATE.ninja_bin_path = env.get('NINJA_BIN') @@ -1595,10 +1737,10 @@ def robust_rule_mapping(var, rule, tool): # default to using ninja installed with python module ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', + ninja.__file__, + os.pardir, + 'data', + 'bin', ninja_bin)) if not os.path.exists(NINJA_STATE.ninja_bin_path): # couldn't find it, just give the bin name and hope @@ -1607,7 +1749,7 @@ def robust_rule_mapping(var, rule, tool): # TODO: this is hacking into scons, preferable if there were a less intrusive way # We will subvert the normal builder execute to make sure all the ninja file is dependent - # on all targets generated from any builders + # on all targets generated from any builders SCons_Builder_BuilderBase__execute = SCons.Builder.BuilderBase._execute def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): # this ensures all environments in which a builder executes from will @@ -1666,4 +1808,4 @@ def ninja_execute(self): if not os.path.isdir(os.environ["TMPDIR"]): env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) - env["TEMPFILE"] = NinjaNoResponseFiles \ No newline at end of file + env["TEMPFILE"] = NinjaNoResponseFiles From 5206ffccc76760f544e668e18878f3733367be5c Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 31 Dec 2020 16:06:53 +0000 Subject: [PATCH 119/163] Update ninja to new scons layout --- src/engine/SCons/Tool/ninja.py | 1811 -------------------------------- 1 file changed, 1811 deletions(-) delete mode 100644 src/engine/SCons/Tool/ninja.py diff --git a/src/engine/SCons/Tool/ninja.py b/src/engine/SCons/Tool/ninja.py deleted file mode 100644 index ea01d590ee..0000000000 --- a/src/engine/SCons/Tool/ninja.py +++ /dev/null @@ -1,1811 +0,0 @@ -# Copyright 2020 MongoDB Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -"""Generate build.ninja files from SCons aliases.""" - -import sys -import os -import importlib -import io -import shutil -import shlex -import subprocess -import textwrap - -from glob import glob -from os.path import join as joinpath -from os.path import splitext - -import SCons -from SCons.Action import _string_from_cmd_list, get_default_ENV -from SCons.Util import is_List, flatten_sequence -from SCons.Script import COMMAND_LINE_TARGETS -from SCons.Node import SConscriptNodes - -NINJA_STATE = None -NINJA_RULES = "__NINJA_CUSTOM_RULES" -NINJA_POOLS = "__NINJA_CUSTOM_POOLS" -NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" -NINJA_BUILD = "NINJA_BUILD" -NINJA_WHEREIS_MEMO = {} -NINJA_STAT_MEMO = {} - -__NINJA_RULE_MAPPING = {} - -# These are the types that get_command can do something with -COMMAND_TYPES = ( - SCons.Action.CommandAction, - SCons.Action.CommandGeneratorAction, -) - - -def is_valid_dependent_node(node): - """ - Return True if node is not an alias or is an alias that has children - - This prevents us from making phony targets that depend on other - phony targets that will never have an associated ninja build - target. - - We also have to specify that it's an alias when doing the builder - check because some nodes (like src files) won't have builders but - are valid implicit dependencies. - """ - if isinstance(node, SCons.Node.Alias.Alias): - return node.children() - - if not node.env: - return True - - return not node.env.get("NINJA_SKIP") - - -def alias_to_ninja_build(node): - """Convert an Alias node into a Ninja phony target""" - return { - "outputs": get_outputs(node), - "rule": "phony", - "implicit": [ - get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) - ], - } - - -def check_invalid_ninja_node(node): - return not isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)) - - -def filter_ninja_nodes(node_list): - ninja_nodes = [] - for node in node_list: - if isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)): - ninja_nodes.append(node) - else: - continue - return ninja_nodes - -def get_input_nodes(node): - if node.get_executor() is not None: - inputs = node.get_executor().get_all_sources() - else: - inputs = node.sources - return inputs - - -def invalid_ninja_nodes(node, targets): - result = False - for node_list in [node.prerequisites, get_input_nodes(node), node.children(), targets]: - if node_list: - result = result or any([check_invalid_ninja_node(node) for node in node_list]) - return result - - -def get_order_only(node): - """Return a list of order only dependencies for node.""" - if node.prerequisites is None: - return [] - return [get_path(src_file(prereq)) for prereq in filter_ninja_nodes(node.prerequisites)] - - -def get_dependencies(node, skip_sources=False): - """Return a list of dependencies for node.""" - if skip_sources: - return [ - get_path(src_file(child)) - for child in filter_ninja_nodes(node.children()) - if child not in node.sources - ] - return [get_path(src_file(child)) for child in filter_ninja_nodes(node.children())] - - -def get_inputs(node): - """Collect the Ninja inputs for node.""" - return [get_path(src_file(o)) for o in filter_ninja_nodes(get_input_nodes(node))] - - -def get_outputs(node): - """Collect the Ninja outputs for node.""" - executor = node.get_executor() - if executor is not None: - outputs = executor.get_all_targets() - else: - if hasattr(node, "target_peers"): - outputs = node.target_peers - else: - outputs = [node] - - outputs = [get_path(o) for o in filter_ninja_nodes(outputs)] - - return outputs - -def generate_depfile(env, node, dependencies): - """ - Ninja tool function for writing a depfile. The depfile should include - the node path followed by all the dependent files in a makefile format. - - dependencies arg can be a list or a subst generator which returns a list. - """ - - depfile = os.path.join(get_path(env['NINJA_BUILDDIR']), str(node) + '.depfile') - - # subst_list will take in either a raw list or a subst callable which generates - # a list, and return a list of CmdStringHolders which can be converted into raw strings. - # If a raw list was passed in, then scons_list will make a list of lists from the original - # values and even subst items in the list if they are substitutable. Flatten will flatten - # the list in that case, to ensure for either input we have a list of CmdStringHolders. - deps_list = env.Flatten(env.subst_list(dependencies)) - - # Now that we have the deps in a list as CmdStringHolders, we can convert them into raw strings - # and make sure to escape the strings to handle spaces in paths. We also will sort the result - # keep the order of the list consistent. - escaped_depends = sorted([dep.escape(env.get("ESCAPE", lambda x: x)) for dep in deps_list]) - depfile_contents = str(node) + ": " + ' '.join(escaped_depends) - - need_rewrite = False - try: - with open(depfile, 'r') as f: - need_rewrite = (f.read() != depfile_contents) - except FileNotFoundError: - need_rewrite = True - - if need_rewrite: - os.makedirs(os.path.dirname(depfile) or '.', exist_ok=True) - with open(depfile, 'w') as f: - f.write(depfile_contents) - - -def get_targets_sources(node): - executor = node.get_executor() - if executor is not None: - tlist = executor.get_all_targets() - slist = executor.get_all_sources() - else: - if hasattr(node, "target_peers"): - tlist = node.target_peers - else: - tlist = [node] - slist = node.sources - - # Retrieve the repository file for all sources - slist = [rfile(s) for s in slist] - return tlist, slist - - -def get_rule(node, rule): - tlist, slist = get_targets_sources(node) - if invalid_ninja_nodes(node, tlist): - return "TEMPLATE" - else: - return rule - - -def _install_action_function(_env, node): - """Install files using the install or copy commands""" - return { - "outputs": get_outputs(node), - "rule": get_rule(node, "INSTALL"), - "inputs": get_inputs(node), - "implicit": get_dependencies(node), - } - - -def _mkdir_action_function(env, node): - return { - "outputs": get_outputs(node), - "rule": get_rule(node, "CMD"), - # implicit explicitly omitted, we translate these so they can be - # used by anything that depends on these but commonly this is - # hit with a node that will depend on all of the fake - # srcnode's that SCons will never give us a rule for leading - # to an invalid ninja file. - "variables": { - # On Windows mkdir "-p" is always on - "cmd": "{mkdir}".format( - mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", - ), - }, - } - - -def _copy_action_function(env, node): - return { - "outputs": get_outputs(node), - "inputs": get_inputs(node), - "rule": get_rule(node, "CMD"), - "variables": { - "cmd": "$COPY $in $out", - }, - } - - -def _lib_symlink_action_function(_env, node): - """Create shared object symlinks if any need to be created""" - symlinks = getattr(getattr(node, "attributes", None), "shliblinks", None) - - if not symlinks or symlinks is None: - return None - - outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] - inputs = [link.get_path() for link, _ in symlinks] - - return { - "outputs": outputs, - "inputs": inputs, - "rule": get_rule(node, "SYMLINK"), - "implicit": get_dependencies(node), - } - - -class SConsToNinjaTranslator: - """Translates SCons Actions into Ninja build objects.""" - - def __init__(self, env): - self.env = env - self.func_handlers = { - # Skip conftest builders - "_createSource": ninja_noop, - # SCons has a custom FunctionAction that just makes sure the - # target isn't static. We let the commands that ninja runs do - # this check for us. - "SharedFlagChecker": ninja_noop, - # The install builder is implemented as a function action. - # TODO: use command action #3573 - "installFunc": _install_action_function, - "MkdirFunc": _mkdir_action_function, - "LibSymlinksActionFunction": _lib_symlink_action_function, - "Copy" : _copy_action_function - } - - self.loaded_custom = False - - # pylint: disable=too-many-return-statements - def action_to_ninja_build(self, node, action=None): - """Generate build arguments dictionary for node.""" - if not self.loaded_custom: - self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) - self.loaded_custom = True - - if node.builder is None: - return None - - if action is None: - action = node.builder.action - - if node.env and node.env.get("NINJA_SKIP"): - return None - - build = {} - env = node.env if node.env else self.env - - # Ideally this should never happen, and we do try to filter - # Ninja builders out of being sources of ninja builders but I - # can't fix every DAG problem so we just skip ninja_builders - # if we find one - global NINJA_STATE - if NINJA_STATE.ninja_file == str(node): - build = None - elif isinstance(action, SCons.Action.FunctionAction): - build = self.handle_func_action(node, action) - elif isinstance(action, SCons.Action.LazyAction): - # pylint: disable=protected-access - action = action._generate_cache(env) - build = self.action_to_ninja_build(node, action=action) - elif isinstance(action, SCons.Action.ListAction): - build = self.handle_list_action(node, action) - elif isinstance(action, COMMAND_TYPES): - build = get_command(env, node, action) - else: - raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) - - if build is not None: - build["order_only"] = get_order_only(node) - - if 'conftest' not in str(node): - node_callback = getattr(node.attributes, "ninja_build_callback", None) - if callable(node_callback): - node_callback(env, node, build) - - return build - - def handle_func_action(self, node, action): - """Determine how to handle the function action.""" - name = action.function_name() - # This is the name given by the Subst/Textfile builders. So return the - # node to indicate that SCons is required. We skip sources here because - # dependencies don't really matter when we're going to shove these to - # the bottom of ninja's DAG anyway and Textfile builders can have text - # content as their source which doesn't work as an implicit dep in - # ninja. - if name == 'ninja_builder': - return None - - handler = self.func_handlers.get(name, None) - if handler is not None: - return handler(node.env if node.env else self.env, node) - elif name == "ActionCaller": - action_to_call = str(action).split('(')[0].strip() - handler = self.func_handlers.get(action_to_call, None) - if handler is not None: - return handler(node.env if node.env else self.env, node) - - SCons.Warnings.SConsWarning( - "Found unhandled function action {}, " - " generating scons command to build\n" - "Note: this is less efficient than Ninja," - " you can write your own ninja build generator for" - " this function using NinjaRegisterFunctionHandler".format(name) - ) - - return { - "rule": "TEMPLATE", - "order_only": get_order_only(node), - "outputs": get_outputs(node), - "inputs": get_inputs(node), - "implicit": get_dependencies(node, skip_sources=True), - } - - # pylint: disable=too-many-branches - def handle_list_action(self, node, action): - """TODO write this comment""" - results = [ - self.action_to_ninja_build(node, action=act) - for act in action.list - if act is not None - ] - results = [ - result for result in results if result is not None and result["outputs"] - ] - if not results: - return None - - # No need to process the results if we only got a single result - if len(results) == 1: - return results[0] - - all_outputs = list({output for build in results for output in build["outputs"]}) - dependencies = list({dep for build in results for dep in build["implicit"]}) - - if results[0]["rule"] == "CMD" or results[0]["rule"] == "GENERATED_CMD": - cmdline = "" - for cmd in results: - - # Occasionally a command line will expand to a - # whitespace only string (i.e. ' '). Which is not a - # valid command but does not trigger the empty command - # condition if not cmdstr. So here we strip preceding - # and proceeding whitespace to make strings like the - # above become empty strings and so will be skipped. - cmdstr = cmd["variables"]["cmd"].strip() - if not cmdstr: - continue - - # Skip duplicate commands - if cmdstr in cmdline: - continue - - if cmdline: - cmdline += " && " - - cmdline += cmdstr - - # Remove all preceding and proceeding whitespace - cmdline = cmdline.strip() - - # Make sure we didn't generate an empty cmdline - if cmdline: - ninja_build = { - "outputs": all_outputs, - "rule": get_rule(node, "GENERATED_CMD"), - "variables": { - "cmd": cmdline, - "env": get_command_env(node.env if node.env else self.env), - }, - "implicit": dependencies, - } - - if node.env and node.env.get("NINJA_POOL", None) is not None: - ninja_build["pool"] = node.env["pool"] - - return ninja_build - - elif results[0]["rule"] == "phony": - return { - "outputs": all_outputs, - "rule": "phony", - "implicit": dependencies, - } - - elif results[0]["rule"] == "INSTALL": - return { - "outputs": all_outputs, - "rule": get_rule(node, "INSTALL"), - "inputs": get_inputs(node), - "implicit": dependencies, - } - - raise Exception("Unhandled list action with rule: " + results[0]["rule"]) - - -# pylint: disable=too-many-instance-attributes -class NinjaState: - """Maintains state of Ninja build system as it's translated from SCons.""" - - def __init__(self, env, ninja_file, writer_class): - self.env = env - self.ninja_file = ninja_file - self.ninja_bin_path = '' - self.writer_class = writer_class - self.__generated = False - self.translator = SConsToNinjaTranslator(env) - self.generated_suffixes = env.get("NINJA_GENERATED_SOURCE_SUFFIXES", []) - - # List of generated builds that will be written at a later stage - self.builds = dict() - - # List of targets for which we have generated a build. This - # allows us to take multiple Alias nodes as sources and to not - # fail to build if they have overlapping targets. - self.built = set() - - # SCons sets this variable to a function which knows how to do - # shell quoting on whatever platform it's run on. Here we use it - # to make the SCONS_INVOCATION variable properly quoted for things - # like CCFLAGS - escape = env.get("ESCAPE", lambda x: x) - - # if SCons was invoked from python, we expect the first arg to be the scons.py - # script, otherwise scons was invoked from the scons script - python_bin = '' - if os.path.basename(sys.argv[0]) == 'scons.py': - python_bin = escape(sys.executable) - self.variables = { - "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", - "SCONS_INVOCATION": '{} {} --disable-ninja __NINJA_NO=1 $out'.format( - python_bin, - " ".join( - [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] - ), - ), - "SCONS_INVOCATION_W_TARGETS": "{} {} --disable-ninja".format( - python_bin, " ".join([escape(arg) for arg in sys.argv]) - ), - # This must be set to a global default per: - # https://ninja-build.org/manual.html#_deps - # English Visual Studio will have the default below, - # otherwise the user can define the variable in the first environment - # that initialized ninja tool - "msvc_deps_prefix": env.get("NINJA_MSVC_DEPS_PREFIX", "Note: including file:") - } - - self.rules = { - "CMD": { - "command": "cmd /c $env$cmd $in $out" if sys.platform == "win32" else "$env$cmd $in $out", - "description": "Building $out", - "pool": "local_pool", - }, - "GENERATED_CMD": { - "command": "cmd /c $env$cmd" if sys.platform == "win32" else "$env$cmd", - "description": "Building $out", - "pool": "local_pool", - }, - # We add the deps processing variables to this below. We - # don't pipe these through cmd.exe on Windows because we - # use this to generate a compile_commands.json database - # which can't use the shell command as it's compile - # command. - "CC": { - "command": "$env$CC @$out.rsp", - "description": "Compiling $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - }, - "CXX": { - "command": "$env$CXX @$out.rsp", - "description": "Compiling $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - }, - "LINK": { - "command": "$env$LINK @$out.rsp", - "description": "Linking $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - "pool": "local_pool", - }, - # Ninja does not automatically delete the archive before - # invoking ar. The ar utility will append to an existing archive, which - # can cause duplicate symbols if the symbols moved between object files. - # Native SCons will perform this operation so we need to force ninja - # to do the same. See related for more info: - # https://jira.mongodb.org/browse/SERVER-49457 - "AR": { - "command": "{}$env$AR @$out.rsp".format( - '' if sys.platform == "win32" else "rm -f $out && " - ), - "description": "Archiving $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - "pool": "local_pool", - }, - "SYMLINK": { - "command": ( - "cmd /c mklink $out $in" - if sys.platform == "win32" - else "ln -s $in $out" - ), - "description": "Symlink $in -> $out", - }, - "INSTALL": { - "command": "$COPY $in $out", - "description": "Install $out", - "pool": "install_pool", - # On Windows cmd.exe /c copy does not always correctly - # update the timestamp on the output file. This leads - # to a stuck constant timestamp in the Ninja database - # and needless rebuilds. - # - # Adding restat here ensures that Ninja always checks - # the copy updated the timestamp and that Ninja has - # the correct information. - "restat": 1, - }, - "TEMPLATE": { - "command": "$SCONS_INVOCATION $out", - "description": "Rendering $SCONS_INVOCATION $out", - "pool": "scons_pool", - "restat": 1, - }, - "SCONS": { - "command": "$SCONS_INVOCATION $out", - "description": "$SCONS_INVOCATION $out", - "pool": "scons_pool", - # restat - # if present, causes Ninja to re-stat the command's outputs - # after execution of the command. Each output whose - # modification time the command did not change will be - # treated as though it had never needed to be built. This - # may cause the output's reverse dependencies to be removed - # from the list of pending build actions. - # - # We use restat any time we execute SCons because - # SCons calls in Ninja typically create multiple - # targets. But since SCons is doing it's own up to - # date-ness checks it may only update say one of - # them. Restat will find out which of the multiple - # build targets did actually change then only rebuild - # those targets which depend specifically on that - # output. - "restat": 1, - }, - "REGENERATE": { - "command": "$SCONS_INVOCATION_W_TARGETS", - "description": "Regenerating $out", - "generator": 1, - "depfile": os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile'), - # Console pool restricts to 1 job running at a time, - # it additionally has some special handling about - # passing stdin, stdout, etc to process in this pool - # that we need for SCons to behave correctly when - # regenerating Ninja - "pool": "console", - # Again we restat in case Ninja thought the - # build.ninja should be regenerated but SCons knew - # better. - "restat": 1, - }, - } - - self.pools = { - "local_pool": self.env.GetOption("num_jobs"), - "install_pool": self.env.GetOption("num_jobs") / 2, - "scons_pool": 1, - } - - for rule in ["CC", "CXX"]: - if env["PLATFORM"] == "win32": - self.rules[rule]["deps"] = "msvc" - else: - self.rules[rule]["deps"] = "gcc" - self.rules[rule]["depfile"] = "$out.d" - - def add_build(self, node): - if not node.has_builder(): - return False - - if isinstance(node, SCons.Node.Alias.Alias): - build = alias_to_ninja_build(node) - else: - build = self.translator.action_to_ninja_build(node) - - # Some things are unbuild-able or need not be built in Ninja - if build is None: - return False - - node_string = str(node) - if node_string in self.builds: - raise Exception("Node {} added to ninja build state more than once".format(node_string)) - self.builds[node_string] = build - self.built.update(build["outputs"]) - return True - - # TODO: rely on SCons to tell us what is generated source - # or some form of user scanner maybe (Github Issue #3624) - def is_generated_source(self, output): - """Check if output ends with a known generated suffix.""" - _, suffix = splitext(output) - return suffix in self.generated_suffixes - - def has_generated_sources(self, output): - """ - Determine if output indicates this is a generated header file. - """ - for generated in output: - if self.is_generated_source(generated): - return True - return False - - # pylint: disable=too-many-branches,too-many-locals - def generate(self): - """ - Generate the build.ninja. - - This should only be called once for the lifetime of this object. - """ - if self.__generated: - return - - self.rules.update(self.env.get(NINJA_RULES, {})) - self.pools.update(self.env.get(NINJA_POOLS, {})) - - content = io.StringIO() - ninja = self.writer_class(content, width=100) - - ninja.comment("Generated by scons. DO NOT EDIT.") - - ninja.variable("builddir", get_path(self.env['NINJA_BUILDDIR'])) - - for pool_name, size in self.pools.items(): - ninja.pool(pool_name, size) - - for var, val in self.variables.items(): - ninja.variable(var, val) - - for rule, kwargs in self.rules.items(): - ninja.rule(rule, **kwargs) - - generated_source_files = sorted({ - output - # First find builds which have header files in their outputs. - for build in self.builds.values() - if self.has_generated_sources(build["outputs"]) - for output in build["outputs"] - # Collect only the header files from the builds with them - # in their output. We do this because is_generated_source - # returns True if it finds a header in any of the outputs, - # here we need to filter so we only have the headers and - # not the other outputs. - if self.is_generated_source(output) - }) - - if generated_source_files: - ninja.build( - outputs="_generated_sources", - rule="phony", - implicit=generated_source_files - ) - - template_builders = [] - - for build in [self.builds[key] for key in sorted(self.builds.keys())]: - if build["rule"] == "TEMPLATE": - template_builders.append(build) - continue - - if "implicit" in build: - build["implicit"].sort() - - # Don't make generated sources depend on each other. We - # have to check that none of the outputs are generated - # sources and none of the direct implicit dependencies are - # generated sources or else we will create a dependency - # cycle. - if ( - generated_source_files - and not build["rule"] == "INSTALL" - and set(build["outputs"]).isdisjoint(generated_source_files) - and set(build.get("implicit", [])).isdisjoint(generated_source_files) - ): - - # Make all non-generated source targets depend on - # _generated_sources. We use order_only for generated - # sources so that we don't rebuild the world if one - # generated source was rebuilt. We just need to make - # sure that all of these sources are generated before - # other builds. - order_only = build.get("order_only", []) - order_only.append("_generated_sources") - build["order_only"] = order_only - if "order_only" in build: - build["order_only"].sort() - - # When using a depfile Ninja can only have a single output - # but SCons will usually have emitted an output for every - # thing a command will create because it's caching is much - # more complex than Ninja's. This includes things like DWO - # files. Here we make sure that Ninja only ever sees one - # target when using a depfile. It will still have a command - # that will create all of the outputs but most targets don't - # depend directly on DWO files and so this assumption is safe - # to make. - rule = self.rules.get(build["rule"]) - - # Some rules like 'phony' and other builtins we don't have - # listed in self.rules so verify that we got a result - # before trying to check if it has a deps key. - # - # Anything using deps or rspfile in Ninja can only have a single - # output, but we may have a build which actually produces - # multiple outputs which other targets can depend on. Here we - # slice up the outputs so we have a single output which we will - # use for the "real" builder and multiple phony targets that - # match the file names of the remaining outputs. This way any - # build can depend on any output from any build. - # - # We assume that the first listed output is the 'key' - # output and is stably presented to us by SCons. For - # instance if -gsplit-dwarf is in play and we are - # producing foo.o and foo.dwo, we expect that outputs[0] - # from SCons will be the foo.o file and not the dwo - # file. If instead we just sorted the whole outputs array, - # we would find that the dwo file becomes the - # first_output, and this breaks, for instance, header - # dependency scanning. - if rule is not None and (rule.get("deps") or rule.get("rspfile")): - first_output, remaining_outputs = ( - build["outputs"][0], - build["outputs"][1:], - ) - - if remaining_outputs: - ninja.build( - outputs=sorted(remaining_outputs), rule="phony", implicit=first_output, - ) - - build["outputs"] = first_output - - # Optionally a rule can specify a depfile, and SCons can generate implicit - # dependencies into the depfile. This allows for dependencies to come and go - # without invalidating the ninja file. The depfile was created in ninja specifically - # for dealing with header files appearing and disappearing across rebuilds, but it can - # be repurposed for anything, as long as you have a way to regenerate the depfile. - # More specific info can be found here: https://ninja-build.org/manual.html#_depfile - if rule is not None and rule.get('depfile') and build.get('deps_files'): - path = build['outputs'] if SCons.Util.is_List(build['outputs']) else [build['outputs']] - generate_depfile(self.env, path[0], build.pop('deps_files', [])) - - if "inputs" in build: - build["inputs"].sort() - - ninja.build(**build) - - template_builds = dict() - for template_builder in template_builders: - - # Special handling for outputs and implicit since we need to - # aggregate not replace for each builder. - for agg_key in ["outputs", "implicit", "inputs"]: - new_val = template_builds.get(agg_key, []) - - # Use pop so the key is removed and so the update - # below will not overwrite our aggregated values. - cur_val = template_builder.pop(agg_key, []) - if is_List(cur_val): - new_val += cur_val - else: - new_val.append(cur_val) - template_builds[agg_key] = new_val - - # Collect all other keys - template_builds.update(template_builder) - - if template_builds.get("outputs", []): - ninja.build(**template_builds) - - # We have to glob the SCons files here to teach the ninja file - # how to regenerate itself. We'll never see ourselves in the - # DAG walk so we can't rely on action_to_ninja_build to - # generate this rule even though SCons should know we're - # dependent on SCons files. - # - # The REGENERATE rule uses depfile, so we need to generate the depfile - # in case any of the SConscripts have changed. The depfile needs to be - # path with in the build and the passed ninja file is an abspath, so - # we will use SCons to give us the path within the build. Normally - # generate_depfile should not be called like this, but instead be called - # through the use of custom rules, and filtered out in the normal - # list of build generation about. However, because the generate rule - # is hardcoded here, we need to do this generate_depfile call manually. - ninja_file_path = self.env.File(self.ninja_file).path - generate_depfile( - self.env, - ninja_file_path, - self.env['NINJA_REGENERATE_DEPS'] - ) - - ninja.build( - ninja_file_path, - rule="REGENERATE", - implicit=[__file__], - ) - - # If we ever change the name/s of the rules that include - # compile commands (i.e. something like CC) we will need to - # update this build to reflect that complete list. - ninja.build( - "compile_commands.json", - rule="CMD", - pool="console", - implicit=[str(self.ninja_file)], - variables={ - "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( - self.ninja_bin_path, str(self.ninja_file), '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' - ) - }, - ) - - ninja.build( - "compiledb", rule="phony", implicit=["compile_commands.json"], - ) - - # Look in SCons's list of DEFAULT_TARGETS, find the ones that - # we generated a ninja build rule for. - scons_default_targets = [ - get_path(tgt) - for tgt in SCons.Script.DEFAULT_TARGETS - if get_path(tgt) in self.built - ] - - # If we found an overlap between SCons's list of default - # targets and the targets we created ninja builds for then use - # those as ninja's default as well. - if scons_default_targets: - ninja.default(" ".join(scons_default_targets)) - - with open(str(self.ninja_file), "w") as build_ninja: - build_ninja.write(content.getvalue()) - - self.__generated = True - - -def get_path(node): - """ - Return a fake path if necessary. - - As an example Aliases use this as their target name in Ninja. - """ - if hasattr(node, "get_path"): - return node.get_path() - return str(node) - - -def rfile(node): - """ - Return the repository file for node if it has one. Otherwise return node - """ - if hasattr(node, "rfile"): - return node.rfile() - return node - - -def src_file(node): - """Returns the src code file if it exists.""" - if hasattr(node, "srcnode"): - src = node.srcnode() - if src.stat() is not None: - return src - return get_path(node) - - -def get_comstr(env, action, targets, sources): - """Get the un-substituted string for action.""" - # Despite being having "list" in it's name this member is not - # actually a list. It's the pre-subst'd string of the command. We - # use it to determine if the command we're about to generate needs - # to use a custom Ninja rule. By default this redirects CC, CXX, - # AR, SHLINK, and LINK commands to their respective rules but the - # user can inject custom Ninja rules and tie them to commands by - # using their pre-subst'd string. - if hasattr(action, "process"): - return action.cmd_list - - return action.genstring(targets, sources, env) - - -def get_command_env(env): - """ - Return a string that sets the enrivonment for any environment variables that - differ between the OS environment and the SCons command ENV. - - It will be compatible with the default shell of the operating system. - """ - try: - return env["NINJA_ENV_VAR_CACHE"] - except KeyError: - pass - - # Scan the ENV looking for any keys which do not exist in - # os.environ or differ from it. We assume if it's a new or - # differing key from the process environment then it's - # important to pass down to commands in the Ninja file. - ENV = get_default_ENV(env) - scons_specified_env = { - key: value - for key, value in ENV.items() - if key not in os.environ or os.environ.get(key, None) != value - } - - windows = env["PLATFORM"] == "win32" - command_env = "" - for key, value in scons_specified_env.items(): - # Ensure that the ENV values are all strings: - if is_List(value): - # If the value is a list, then we assume it is a - # path list, because that's a pretty common list-like - # value to stick in an environment variable: - value = flatten_sequence(value) - value = joinpath(map(str, value)) - else: - # If it isn't a string or a list, then we just coerce - # it to a string, which is the proper way to handle - # Dir and File instances and will produce something - # reasonable for just about everything else: - value = str(value) - - if windows: - command_env += "set '{}={}' && ".format(key, value) - else: - # We address here *only* the specific case that a user might have - # an environment variable which somehow gets included and has - # spaces in the value. These are escapes that Ninja handles. This - # doesn't make builds on paths with spaces (Ninja and SCons issues) - # nor expanding response file paths with spaces (Ninja issue) work. - value = value.replace(r' ', r'$ ') - command_env += "export {}='{}';".format(key, value) - - env["NINJA_ENV_VAR_CACHE"] = command_env - return command_env - - -def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom_env={}): - """Generate a response file command provider for rule name.""" - - # If win32 using the environment with a response file command will cause - # ninja to fail to create the response file. Additionally since these rules - # generally are not piping through cmd.exe /c any environment variables will - # make CreateProcess fail to start. - # - # On POSIX we can still set environment variables even for compile - # commands so we do so. - use_command_env = not env["PLATFORM"] == "win32" - if "$" in tool: - tool_is_dynamic = True - - def get_response_file_command(env, node, action, targets, sources, executor=None): - if hasattr(action, "process"): - cmd_list, _, _ = action.process(targets, sources, env, executor=executor) - cmd_list = [str(c).replace("$", "$$") for c in cmd_list[0]] - else: - command = generate_command( - env, node, action, targets, sources, executor=executor - ) - cmd_list = shlex.split(command) - - if tool_is_dynamic: - tool_command = env.subst( - tool, target=targets, source=sources, executor=executor - ) - else: - tool_command = tool - - try: - # Add 1 so we always keep the actual tool inside of cmd - tool_idx = cmd_list.index(tool_command) + 1 - except ValueError: - raise Exception( - "Could not find tool {} in {} generated from {}".format( - tool, cmd_list, get_comstr(env, action, targets, sources) - ) - ) - - cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] - rsp_content = ['"' + rsp_content_item + '"' for rsp_content_item in rsp_content] - rsp_content = " ".join(rsp_content) - - variables = {"rspc": rsp_content} - variables[rule] = cmd - if use_command_env: - variables["env"] = get_command_env(env) - - for key, value in custom_env.items(): - variables["env"] += env.subst( - f"export {key}={value};", target=targets, source=sources, executor=executor - ) + " " - return rule, variables, [tool_command] - - return get_response_file_command - - -def generate_command(env, node, action, targets, sources, executor=None): - # Actions like CommandAction have a method called process that is - # used by SCons to generate the cmd_line they need to run. So - # check if it's a thing like CommandAction and call it if we can. - if hasattr(action, "process"): - cmd_list, _, _ = action.process(targets, sources, env, executor=executor) - cmd = _string_from_cmd_list(cmd_list[0]) - else: - # Anything else works with genstring, this is most commonly hit by - # ListActions which essentially call process on all of their - # commands and concatenate it for us. - genstring = action.genstring(targets, sources, env) - if executor is not None: - cmd = env.subst(genstring, executor=executor) - else: - cmd = env.subst(genstring, targets, sources) - - cmd = cmd.replace("\n", " && ").strip() - if cmd.endswith("&&"): - cmd = cmd[0:-2].strip() - - # Escape dollars as necessary - return cmd.replace("$", "$$") - - -def get_generic_shell_command(env, node, action, targets, sources, executor=None): - return ( - "CMD", - { - "cmd": generate_command(env, node, action, targets, sources, executor=None), - "env": get_command_env(env), - }, - # Since this function is a rule mapping provider, it must return a list of dependencies, - # and usually this would be the path to a tool, such as a compiler, used for this rule. - # However this function is to generic to be able to reliably extract such deps - # from the command, so we return a placeholder empty list. It should be noted that - # generally this function will not be used soley and is more like a template to generate - # the basics for a custom provider which may have more specific options for a provier - # function for a custom NinjaRuleMapping. - [] - ) - - -def get_command(env, node, action): # pylint: disable=too-many-branches - """Get the command to execute for node.""" - if node.env: - sub_env = node.env - else: - sub_env = env - executor = node.get_executor() - tlist, slist = get_targets_sources(node) - - # Generate a real CommandAction - if isinstance(action, SCons.Action.CommandGeneratorAction): - # pylint: disable=protected-access - action = action._generate(tlist, slist, sub_env, 1, executor=executor) - - variables = {} - - comstr = get_comstr(sub_env, action, tlist, slist) - if not comstr: - return None - - provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) - rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) - - # Get the dependencies for all targets - implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) - - # Now add in the other dependencies related to the command, - # e.g. the compiler binary. The ninja rule can be user provided so - # we must do some validation to resolve the dependency path for ninja. - for provider_dep in provider_deps: - - provider_dep = sub_env.subst(provider_dep) - if not provider_dep: - continue - - # If the tool is a node, then SCons will resolve the path later, if its not - # a node then we assume it generated from build and make sure it is existing. - if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep): - implicit.append(provider_dep) - continue - - # in some case the tool could be in the local directory and be suppled without the ext - # such as in windows, so append the executable suffix and check. - prog_suffix = sub_env.get('PROGSUFFIX', '') - provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix - if os.path.exists(provider_dep_ext): - implicit.append(provider_dep_ext) - continue - - # Many commands will assume the binary is in the path, so - # we accept this as a possible input from a given command. - - provider_dep_abspath = sub_env.WhereIs(provider_dep) or sub_env.WhereIs(provider_dep, path=os.environ["PATH"]) - if provider_dep_abspath: - implicit.append(provider_dep_abspath) - continue - - # Possibly these could be ignore and the build would still work, however it may not always - # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided - # ninja rule. - raise Exception(f"Could not resolve path for {provider_dep} dependency on node '{node}'") - - ninja_build = { - "order_only": get_order_only(node), - "outputs": get_outputs(node), - "inputs": get_inputs(node), - "implicit": implicit, - "rule": get_rule(node, rule), - "variables": variables, - } - - # Don't use sub_env here because we require that NINJA_POOL be set - # on a per-builder call basis to prevent accidental strange - # behavior like env['NINJA_POOL'] = 'console' and sub_env can be - # the global Environment object if node.env is None. - # Example: - # - # Allowed: - # - # env.Command("ls", NINJA_POOL="ls_pool") - # - # Not allowed and ignored: - # - # env["NINJA_POOL"] = "ls_pool" - # env.Command("ls") - # - if node.env and node.env.get("NINJA_POOL", None) is not None: - ninja_build["pool"] = node.env["NINJA_POOL"] - - return ninja_build - - -def ninja_builder(env, target, source): - """Generate a build.ninja for source.""" - if not isinstance(source, list): - source = [source] - if not isinstance(target, list): - target = [target] - - # We have no COMSTR equivalent so print that we're generating - # here. - print("Generating:", str(target[0])) - - generated_build_ninja = target[0].get_abspath() - NINJA_STATE.generate() - - if env["PLATFORM"] == "win32": - # this is not great, its doesn't consider specific - # node environments, which means on linux the build could - # behave differently, because on linux you can set the environment - # per command in the ninja file. This is only needed if - # running ninja directly from a command line that hasn't - # had the environment setup (vcvarsall.bat) - with open('run_ninja_env.bat', 'w') as f: - for key in env['ENV']: - f.write('set {}={}\n'.format(key, env['ENV'][key])) - f.write('{} -f {} %*\n'.format(NINJA_STATE.ninja_bin_path, generated_build_ninja)) - cmd = ['run_ninja_env.bat'] - - else: - cmd = [NINJA_STATE.ninja_bin_path, '-f', generated_build_ninja] - - if not env.get("DISABLE_AUTO_NINJA"): - print("Executing:", str(' '.join(cmd))) - - # execute the ninja build at the end of SCons, trying to - # reproduce the output like a ninja build would - def execute_ninja(): - - proc = subprocess.Popen(cmd, - stderr=sys.stderr, - stdout=subprocess.PIPE, - universal_newlines=True, - env=os.environ if env["PLATFORM"] == "win32" else env['ENV'] - ) - for stdout_line in iter(proc.stdout.readline, ""): - yield stdout_line - proc.stdout.close() - return_code = proc.wait() - if return_code: - raise subprocess.CalledProcessError(return_code, 'ninja') - - erase_previous = False - for output in execute_ninja(): - output = output.strip() - if erase_previous: - sys.stdout.write('\x1b[2K') # erase previous line - sys.stdout.write("\r") - else: - sys.stdout.write(os.linesep) - sys.stdout.write(output) - sys.stdout.flush() - # this will only erase ninjas [#/#] lines - # leaving warnings and other output, seems a bit - # prone to failure with such a simple check - erase_previous = output.startswith('[') - -# pylint: disable=too-few-public-methods -class AlwaysExecAction(SCons.Action.FunctionAction): - """Override FunctionAction.__call__ to always execute.""" - - def __call__(self, *args, **kwargs): - kwargs["execute"] = 1 - return super().__call__(*args, **kwargs) - - -def register_custom_handler(env, name, handler): - """Register a custom handler for SCons function actions.""" - env[NINJA_CUSTOM_HANDLERS][name] = handler - - -def register_custom_rule_mapping(env, pre_subst_string, rule): - """Register a function to call for a given rule.""" - global __NINJA_RULE_MAPPING - __NINJA_RULE_MAPPING[pre_subst_string] = rule - - -def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): - """Allows specification of Ninja rules from inside SCons files.""" - rule_obj = { - "command": command, - "description": description if description else "{} $out".format(rule), - } - - if use_depfile: - rule_obj["depfile"] = os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile') - - if deps is not None: - rule_obj["deps"] = deps - - if pool is not None: - rule_obj["pool"] = pool - - if use_response_file: - rule_obj["rspfile"] = "$out.rsp" - rule_obj["rspfile_content"] = response_file_content - - env[NINJA_RULES][rule] = rule_obj - - -def register_custom_pool(env, pool, size): - """Allows the creation of custom Ninja pools""" - env[NINJA_POOLS][pool] = size - -def set_build_node_callback(env, node, callback): - if 'conftest' not in str(node): - setattr(node.attributes, "ninja_build_callback", callback) - -def ninja_csig(original): - """Return a dummy csig""" - - def wrapper(self): - if isinstance(self, SCons.Node.Node) and self.is_sconscript(): - return original(self) - return "dummy_ninja_csig" - - return wrapper - - -def ninja_contents(original): - """Return a dummy content without doing IO""" - - def wrapper(self): - if isinstance(self, SCons.Node.Node) and self.is_sconscript(): - return original(self) - return bytes("dummy_ninja_contents", encoding="utf-8") - - return wrapper - -def CheckNinjaCompdbExpand(env, context): - """ Configure check testing if ninja's compdb can expand response files""" - - context.Message('Checking if ninja compdb can expand response files... ') - ret, output = context.TryAction( - action='ninja -f $SOURCE -t compdb -x CMD_RSP > $TARGET', - extension='.ninja', - text=textwrap.dedent(""" - rule CMD_RSP - command = $cmd @$out.rsp > fake_output.txt - description = Building $out - rspfile = $out.rsp - rspfile_content = $rspc - build fake_output.txt: CMD_RSP fake_input.txt - cmd = echo - pool = console - rspc = "test" - """)) - result = '@fake_output.txt.rsp' not in output - context.Result(result) - return result - -def ninja_stat(_self, path): - """ - Eternally memoized stat call. - - SCons is very aggressive about clearing out cached values. For our - purposes everything should only ever call stat once since we're - running in a no_exec build the file system state should not - change. For these reasons we patch SCons.Node.FS.LocalFS.stat to - use our eternal memoized dictionary. - """ - global NINJA_STAT_MEMO - - try: - return NINJA_STAT_MEMO[path] - except KeyError: - try: - result = os.stat(path) - except os.error: - result = None - - NINJA_STAT_MEMO[path] = result - return result - - -def ninja_noop(*_args, **_kwargs): - """ - A general purpose no-op function. - - There are many things that happen in SCons that we don't need and - also don't return anything. We use this to disable those functions - instead of creating multiple definitions of the same thing. - """ - return None - - -def ninja_whereis(thing, *_args, **_kwargs): - """Replace env.WhereIs with a much faster version""" - global NINJA_WHEREIS_MEMO - - # Optimize for success, this gets called significantly more often - # when the value is already memoized than when it's not. - try: - return NINJA_WHEREIS_MEMO[thing] - except KeyError: - # We do not honor any env['ENV'] or env[*] variables in the - # generated ninja ile. Ninja passes your raw shell environment - # down to it's subprocess so the only sane option is to do the - # same during generation. At some point, if and when we try to - # upstream this, I'm sure a sticking point will be respecting - # env['ENV'] variables and such but it's actually quite - # complicated. I have a naive version but making it always work - # with shell quoting is nigh impossible. So I've decided to - # cross that bridge when it's absolutely required. - path = shutil.which(thing) - NINJA_WHEREIS_MEMO[thing] = path - return path - - -def ninja_always_serial(self, num, taskmaster): - """Replacement for SCons.Job.Jobs constructor which always uses the Serial Job class.""" - # We still set self.num_jobs to num even though it's a lie. The - # only consumer of this attribute is the Parallel Job class AND - # the Main.py function which instantiates a Jobs class. It checks - # if Jobs.num_jobs is equal to options.num_jobs, so if the user - # provides -j12 but we set self.num_jobs = 1 they get an incorrect - # warning about this version of Python not supporting parallel - # builds. So here we lie so the Main.py will not give a false - # warning to users. - self.num_jobs = num - self.job = SCons.Job.Serial(taskmaster) - -def ninja_hack_linkcom(env): - # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks - # without relying on the SCons hacks that SCons uses by default. - if env["PLATFORM"] == "win32": - from SCons.Tool.mslink import compositeLinkAction - - if env.get("LINKCOM", None) == compositeLinkAction: - env[ - "LINKCOM" - ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' - env[ - "SHLINKCOM" - ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' - - -class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): - """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" - - def __call__(self, target, source, env, for_signature): - return self.cmd - - def _print_cmd_str(*_args, **_kwargs): - """Disable this method""" - pass - - -def exists(env): - """Enable if called.""" - - # This variable disables the tool when storing the SCons command in the - # generated ninja file to ensure that the ninja tool is not loaded when - # SCons should do actual work as a subprocess of a ninja build. The ninja - # tool is very invasive into the internals of SCons and so should never be - # enabled when SCons needs to build a target. - if env.get("__NINJA_NO", "0") == "1": - return False - - try: - import ninja - return ninja.__file__ - except ImportError: - SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") - return False - - -added = None - -def generate(env): - """Generate the NINJA builders.""" - from SCons.Script import AddOption, GetOption - global added - if not added: - added = 1 - - AddOption('--disable-execute-ninja', - dest='disable_execute_ninja', - metavar='BOOL', - action="store_true", - default=False, - help='Disable ninja automatically building after scons') - - AddOption('--disable-ninja', - dest='disable_ninja', - metavar='BOOL', - action="store_true", - default=False, - help='Disable ninja automatically building after scons') - - - try: - import ninja - except ImportError: - SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") - return - - env["DISABLE_AUTO_NINJA"] = GetOption('disable_execute_ninja') - - global NINJA_STATE - - # Add the Ninja builder. - always_exec_ninja_action = AlwaysExecAction(ninja_builder, {}) - ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action) - env.Append(BUILDERS={"Ninja": ninja_builder_obj}) - - env["NINJA_PREFIX"] = env.get("NINJA_PREFIX", "build") - env["NINJA_SUFFIX"] = env.get("NINJA_SUFFIX", "ninja") - env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") - env['NINJA_BUILDDIR'] = env.get("NINJA_BUILDDIR", env.Dir(".ninja").path) - ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") - # here we allow multiple environments to construct rules and builds - # into the same ninja file - if NINJA_STATE is None: - ninja_file = env.Ninja(target=ninja_file_name, source=[]) - env.AlwaysBuild(ninja_file) - env.Alias("$NINJA_ALIAS_NAME", ninja_file) - else: - if str(NINJA_STATE.ninja_file) != ninja_file_name: - SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") - ninja_file = [NINJA_STATE.ninja_file] - - - # TODO: API for getting the SConscripts programmatically - # exists upstream: https://github.com/SCons/scons/issues/3625 - def ninja_generate_deps(env): - return sorted([env.File("#SConstruct").path] + glob("**/SConscript", recursive=True)) - env['_NINJA_REGENERATE_DEPS_FUNC'] = ninja_generate_deps - - env['NINJA_REGENERATE_DEPS'] = env.get('NINJA_REGENERATE_DEPS', '${_NINJA_REGENERATE_DEPS_FUNC(__env__)}') - - # This adds the required flags such that the generated compile - # commands will create depfiles as appropriate in the Ninja file. - if env["PLATFORM"] == "win32": - env.Append(CCFLAGS=["/showIncludes"]) - else: - env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) - - env.AddMethod(CheckNinjaCompdbExpand, "CheckNinjaCompdbExpand") - - # Provide a way for custom rule authors to easily access command - # generation. - env.AddMethod(get_generic_shell_command, "NinjaGetGenericShellCommand") - env.AddMethod(get_command, "NinjaGetCommand") - env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider") - env.AddMethod(set_build_node_callback, "NinjaSetBuildNodeCallback") - - # Provides a way for users to handle custom FunctionActions they - # want to translate to Ninja. - env[NINJA_CUSTOM_HANDLERS] = {} - env.AddMethod(register_custom_handler, "NinjaRegisterFunctionHandler") - - # Provides a mechanism for inject custom Ninja rules which can - # then be mapped using NinjaRuleMapping. - env[NINJA_RULES] = {} - env.AddMethod(register_custom_rule, "NinjaRule") - - # Provides a mechanism for inject custom Ninja pools which can - # be used by providing the NINJA_POOL="name" as an - # OverrideEnvironment variable in a builder call. - env[NINJA_POOLS] = {} - env.AddMethod(register_custom_pool, "NinjaPool") - - # Add the ability to register custom NinjaRuleMappings for Command - # builders. We don't store this dictionary in the env to prevent - # accidental deletion of the CC/XXCOM mappings. You can still - # overwrite them if you really want to but you have to explicit - # about it this way. The reason is that if they were accidentally - # deleted you would get a very subtly incorrect Ninja file and - # might not catch it. - env.AddMethod(register_custom_rule_mapping, "NinjaRuleMapping") - - # on windows we need to change the link action - ninja_hack_linkcom(env) - - # Normally in SCons actions for the Program and *Library builders - # will return "${*COM}" as their pre-subst'd command line. However - # if a user in a SConscript overwrites those values via key access - # like env["LINKCOM"] = "$( $ICERUN $)" + env["LINKCOM"] then - # those actions no longer return the "bracketted" string and - # instead return something that looks more expanded. So to - # continue working even if a user has done this we map both the - # "bracketted" and semi-expanded versions. - def robust_rule_mapping(var, rule, tool): - provider = gen_get_response_file_command(env, rule, tool) - env.NinjaRuleMapping("${" + var + "}", provider) - env.NinjaRuleMapping(env.get(var, None), provider) - - robust_rule_mapping("CCCOM", "CC", "$CC") - robust_rule_mapping("SHCCCOM", "CC", "$CC") - robust_rule_mapping("CXXCOM", "CXX", "$CXX") - robust_rule_mapping("SHCXXCOM", "CXX", "$CXX") - robust_rule_mapping("LINKCOM", "LINK", "$LINK") - robust_rule_mapping("SHLINKCOM", "LINK", "$SHLINK") - robust_rule_mapping("ARCOM", "AR", "$AR") - - # Make SCons node walk faster by preventing unnecessary work - env.Decider("timestamp-match") - - # Used to determine if a build generates a source file. Ninja - # requires that all generated sources are added as order_only - # dependencies to any builds that *might* use them. - # TODO: switch to using SCons to help determine this (Github Issue #3624) - env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] - - if env["PLATFORM"] != "win32" and env.get("RANLIBCOM"): - # There is no way to translate the ranlib list action into - # Ninja so add the s flag and disable ranlib. - # - # This is equivalent to Meson. - # https://github.com/mesonbuild/meson/blob/master/mesonbuild/linkers.py#L143 - old_arflags = str(env["ARFLAGS"]) - if "s" not in old_arflags: - old_arflags += "s" - - env["ARFLAGS"] = SCons.Util.CLVar([old_arflags]) - - # Disable running ranlib, since we added 's' above - env["RANLIBCOM"] = "" - - if GetOption('disable_ninja'): - return env - - SCons.Warnings.SConsWarning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") - - # This is the point of no return, anything after this comment - # makes changes to SCons that are irreversible and incompatible - # with a normal SCons build. We return early if __NINJA_NO=1 has - # been given on the command line (i.e. by us in the generated - # ninja file) here to prevent these modifications from happening - # when we want SCons to do work. Everything before this was - # necessary to setup the builder and other functions so that the - # tool can be unconditionally used in the users's SCons files. - - if not exists(env): - return - - # Set a known variable that other tools can query so they can - # behave correctly during ninja generation. - env["GENERATING_NINJA"] = True - - # These methods are no-op'd because they do not work during ninja - # generation, expected to do no work, or simply fail. All of which - # are slow in SCons. So we overwrite them with no logic. - SCons.Node.FS.File.make_ready = ninja_noop - SCons.Node.FS.File.prepare = ninja_noop - SCons.Node.FS.File.push_to_cache = ninja_noop - SCons.Executor.Executor.prepare = ninja_noop - SCons.Taskmaster.Task.prepare = ninja_noop - SCons.Node.FS.File.built = ninja_noop - SCons.Node.Node.visited = ninja_noop - - # We make lstat a no-op because it is only used for SONAME - # symlinks which we're not producing. - SCons.Node.FS.LocalFS.lstat = ninja_noop - - # This is a slow method that isn't memoized. We make it a noop - # since during our generation we will never use the results of - # this or change the results. - SCons.Node.FS.is_up_to_date = ninja_noop - - # We overwrite stat and WhereIs with eternally memoized - # implementations. See the docstring of ninja_stat and - # ninja_whereis for detailed explanations. - SCons.Node.FS.LocalFS.stat = ninja_stat - SCons.Util.WhereIs = ninja_whereis - - # Monkey patch get_csig and get_contents for some classes. It - # slows down the build significantly and we don't need contents or - # content signatures calculated when generating a ninja file since - # we're not doing any SCons caching or building. - SCons.Executor.Executor.get_contents = ninja_contents( - SCons.Executor.Executor.get_contents - ) - SCons.Node.Alias.Alias.get_contents = ninja_contents( - SCons.Node.Alias.Alias.get_contents - ) - SCons.Node.FS.File.get_contents = ninja_contents(SCons.Node.FS.File.get_contents) - SCons.Node.FS.File.get_csig = ninja_csig(SCons.Node.FS.File.get_csig) - SCons.Node.FS.Dir.get_csig = ninja_csig(SCons.Node.FS.Dir.get_csig) - SCons.Node.Alias.Alias.get_csig = ninja_csig(SCons.Node.Alias.Alias.get_csig) - - # Ignore CHANGED_SOURCES and CHANGED_TARGETS. We don't want those - # to have effect in a generation pass because the generator - # shouldn't generate differently depending on the current local - # state. Without this, when generating on Windows, if you already - # had a foo.obj, you would omit foo.cpp from the response file. Do the same for UNCHANGED. - SCons.Executor.Executor._get_changed_sources = SCons.Executor.Executor._get_sources - SCons.Executor.Executor._get_changed_targets = SCons.Executor.Executor._get_targets - SCons.Executor.Executor._get_unchanged_sources = SCons.Executor.Executor._get_sources - SCons.Executor.Executor._get_unchanged_targets = SCons.Executor.Executor._get_targets - - # Replace false action messages with nothing. - env["PRINT_CMD_LINE_FUNC"] = ninja_noop - - # This reduces unnecessary subst_list calls to add the compiler to - # the implicit dependencies of targets. Since we encode full paths - # in our generated commands we do not need these slow subst calls - # as executing the command will fail if the file is not found - # where we expect it. - env["IMPLICIT_COMMAND_DEPENDENCIES"] = False - - # This makes SCons more aggressively cache MD5 signatures in the - # SConsign file. - env.SetOption("max_drift", 1) - - # The Serial job class is SIGNIFICANTLY (almost twice as) faster - # than the Parallel job class for generating Ninja files. So we - # monkey the Jobs constructor to only use the Serial Job class. - SCons.Job.Jobs.__init__ = ninja_always_serial - - ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') - - if NINJA_STATE is None: - NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) - NINJA_STATE.ninja_bin_path = env.get('NINJA_BIN') - if not NINJA_STATE.ninja_bin_path: - # default to using ninja installed with python module - ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' - NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - ninja_bin)) - if not os.path.exists(NINJA_STATE.ninja_bin_path): - # couldn't find it, just give the bin name and hope - # its in the path later - NINJA_STATE.ninja_bin_path = ninja_bin - - # TODO: this is hacking into scons, preferable if there were a less intrusive way - # We will subvert the normal builder execute to make sure all the ninja file is dependent - # on all targets generated from any builders - SCons_Builder_BuilderBase__execute = SCons.Builder.BuilderBase._execute - def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): - # this ensures all environments in which a builder executes from will - # not create list actions for linking on windows - ninja_hack_linkcom(env) - targets = SCons_Builder_BuilderBase__execute(self, env, target, source, overwarn=overwarn, executor_kw=executor_kw) - - if not SCons.Util.is_List(target): - target = [target] - - for target in targets: - if str(target) != ninja_file_name and "conftest" not in str(target): - env.Depends(ninja_file, targets) - return targets - SCons.Builder.BuilderBase._execute = NinjaBuilderExecute - - # Here we monkey patch the Task.execute method to not do a bunch of - # unnecessary work. If a build is a regular builder (i.e not a conftest and - # not our own Ninja builder) then we add it to the NINJA_STATE. Otherwise we - # build it like normal. This skips all of the caching work that this method - # would normally do since we aren't pulling any of these targets from the - # cache. - # - # In the future we may be able to use this to actually cache the build.ninja - # file once we have the upstream support for referencing SConscripts as File - # nodes. - def ninja_execute(self): - global NINJA_STATE - - target = self.targets[0] - target_name = str(target) - if target_name != ninja_file_name and "conftest" not in target_name: - NINJA_STATE.add_build(target) - else: - target.build() - - SCons.Taskmaster.Task.execute = ninja_execute - - # Make needs_execute always return true instead of determining out of - # date-ness. - SCons.Script.Main.BuildTask.needs_execute = lambda x: True - - # We will eventually need to overwrite TempFileMunge to make it - # handle persistent tempfiles or get an upstreamed change to add - # some configurability to it's behavior in regards to tempfiles. - # - # Set all three environment variables that Python's - # tempfile.mkstemp looks at as it behaves differently on different - # platforms and versions of Python. - build_dir = env.subst("$BUILD_DIR") - if build_dir == "": - build_dir = "." - os.environ["TMPDIR"] = env.Dir("{}/.response_files".format(build_dir)).get_abspath() - os.environ["TEMP"] = os.environ["TMPDIR"] - os.environ["TMP"] = os.environ["TMPDIR"] - if not os.path.isdir(os.environ["TMPDIR"]): - env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) - - env["TEMPFILE"] = NinjaNoResponseFiles From 987c7a3d7ddc337c1fc8dd7810f657932e8a2bcb Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 27 Jan 2021 15:28:05 -0800 Subject: [PATCH 120/163] Fix int function not returning int value --- test/ninja/ninja-fixture/test_impl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/test/ninja/ninja-fixture/test_impl.c b/test/ninja/ninja-fixture/test_impl.c index 89c26ede6f..ac3fd88656 100644 --- a/test/ninja/ninja-fixture/test_impl.c +++ b/test/ninja/ninja-fixture/test_impl.c @@ -16,4 +16,5 @@ DLLEXPORT int library_function(void) { printf("library_function"); + return 0; } From 0f496329207b59a0d080a148573df0647c250fad Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 27 Jan 2021 15:29:09 -0800 Subject: [PATCH 121/163] Refactor ninja -> module, change NINJA_BIN -> NINJA --- SCons/Tool/{ninja.py => ninja/__init__.py} | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) rename SCons/Tool/{ninja.py => ninja/__init__.py} (99%) diff --git a/SCons/Tool/ninja.py b/SCons/Tool/ninja/__init__.py similarity index 99% rename from SCons/Tool/ninja.py rename to SCons/Tool/ninja/__init__.py index ea01d590ee..213b12ac8f 100644 --- a/SCons/Tool/ninja.py +++ b/SCons/Tool/ninja/__init__.py @@ -339,6 +339,7 @@ def action_to_ninja_build(self, node, action=None): if build is not None: build["order_only"] = get_order_only(node) + #TODO: WPD Is this testing the filename to verify it's a configure context generated file? if 'conftest' not in str(node): node_callback = getattr(node.attributes, "ninja_build_callback", None) if callable(node_callback): @@ -1477,8 +1478,8 @@ def exists(env): return False try: - import ninja - return ninja.__file__ + import __init__ + return __init__.__file__ except ImportError: SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return False @@ -1509,7 +1510,7 @@ def generate(env): try: - import ninja + import __init__ except ImportError: SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return @@ -1721,6 +1722,7 @@ def robust_rule_mapping(var, rule, tool): # This makes SCons more aggressively cache MD5 signatures in the # SConsign file. + # TODO: WPD shouldn't this be set to 0? env.SetOption("max_drift", 1) # The Serial job class is SIGNIFICANTLY (almost twice as) faster @@ -1732,12 +1734,12 @@ def robust_rule_mapping(var, rule, tool): if NINJA_STATE is None: NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) - NINJA_STATE.ninja_bin_path = env.get('NINJA_BIN') + NINJA_STATE.ninja_bin_path = env.get('NINJA') if not NINJA_STATE.ninja_bin_path: # default to using ninja installed with python module ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( - ninja.__file__, + __init__.__file__, os.pardir, 'data', 'bin', @@ -1801,7 +1803,7 @@ def ninja_execute(self): # platforms and versions of Python. build_dir = env.subst("$BUILD_DIR") if build_dir == "": - build_dir = "." + build_dir = ".." os.environ["TMPDIR"] = env.Dir("{}/.response_files".format(build_dir)).get_abspath() os.environ["TEMP"] = os.environ["TMPDIR"] os.environ["TMP"] = os.environ["TMPDIR"] From 400b3a370ba2d047188f82cccf6759189f548ebb Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 27 Jan 2021 15:29:31 -0800 Subject: [PATCH 122/163] refactor test a bit, explicitly use ninja found by test framework --- test/ninja/build_libraries.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 40404152fa..a1c3282475 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -33,7 +33,13 @@ try: import ninja except ImportError: - test.skip_test("Could not find module in python") + test.skip_test("Could not find ninja module in python") + + +ninja_binary = test.where_is('ninja') +if not ninja_binary: + test.skip_test("Could not find ninja. Skipping test.") + _python_ = TestSCons._python_ _exe = TestSCons._exe @@ -47,15 +53,21 @@ test.dir_fixture('ninja-fixture') -lib_suffix = '.lib' if IS_WINDOWS else '.so' -staticlib_suffix = '.lib' if IS_WINDOWS else '.a' -lib_prefix = '' if IS_WINDOWS else 'lib' - -win32 = ", 'WIN32'" if IS_WINDOWS else '' +if IS_WINDOWS: + lib_suffix = '.lib' + staticlib_suffix = '.lib' + lib_prefix = '' + win32 = ", 'WIN32'" +else: + lib_suffix = '.so' + staticlib_suffix = '.a' + lib_prefix = 'lib' + win32 = '' test.write('SConstruct', """ env = Environment() env.Tool('ninja') +env['NINJA'] = "%(ninja_bin)s" shared_lib = env.SharedLibrary(target = 'test_impl', source = 'test_impl.c', CPPDEFINES=['LIBRARY_BUILD'%(win32)s]) env.Program(target = 'test', source = 'test1.c', LIBS=['test_impl'], LIBPATH=['.'], RPATH='.') From f3b33c83b2d317c3a2dc322491575a83760696de Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 12 Feb 2021 11:27:57 -0800 Subject: [PATCH 123/163] remove f-string to retain py3.5 compatibility --- SCons/Tool/ninja/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 213b12ac8f..fe5e084dd1 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -1068,7 +1068,7 @@ def get_response_file_command(env, node, action, targets, sources, executor=None for key, value in custom_env.items(): variables["env"] += env.subst( - f"export {key}={value};", target=targets, source=sources, executor=executor + "export %s=%s;"%(key, value), target=targets, source=sources, executor=executor ) + " " return rule, variables, [tool_command] @@ -1178,7 +1178,7 @@ def get_command(env, node, action): # pylint: disable=too-many-branches # Possibly these could be ignore and the build would still work, however it may not always # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided # ninja rule. - raise Exception(f"Could not resolve path for {provider_dep} dependency on node '{node}'") + raise Exception("Could not resolve path for %s dependency on node '%s'"%(provider_dep, node)) ninja_build = { "order_only": get_order_only(node), From ef2cdf14a874f7eabf203882f87fcde41ef5d374 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 27 Jan 2021 21:21:27 -0800 Subject: [PATCH 124/163] can't name module ninja as it will conflict with the non-SCons ninja module --- SCons/Tool/ninja.py | 26 +++++++++++++++++++ SCons/Tool/{ninja => ninjaCommon}/__init__.py | 12 +++++---- 2 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 SCons/Tool/ninja.py rename SCons/Tool/{ninja => ninjaCommon}/__init__.py (99%) diff --git a/SCons/Tool/ninja.py b/SCons/Tool/ninja.py new file mode 100644 index 0000000000..e597971d2d --- /dev/null +++ b/SCons/Tool/ninja.py @@ -0,0 +1,26 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + + +from .ninjaCommon import generate, exists diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninjaCommon/__init__.py similarity index 99% rename from SCons/Tool/ninja/__init__.py rename to SCons/Tool/ninjaCommon/__init__.py index fe5e084dd1..9b330d3387 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -1,3 +1,5 @@ +# MIT License +# # Copyright 2020 MongoDB Inc. # # Permission is hereby granted, free of charge, to any person obtaining @@ -1478,8 +1480,8 @@ def exists(env): return False try: - import __init__ - return __init__.__file__ + import ninja + return ninja.__file__ except ImportError: SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return False @@ -1510,7 +1512,7 @@ def generate(env): try: - import __init__ + import ninja except ImportError: SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return @@ -1739,7 +1741,7 @@ def robust_rule_mapping(var, rule, tool): # default to using ninja installed with python module ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( - __init__.__file__, + ninja.__file__, os.pardir, 'data', 'bin', @@ -1803,7 +1805,7 @@ def ninja_execute(self): # platforms and versions of Python. build_dir = env.subst("$BUILD_DIR") if build_dir == "": - build_dir = ".." + build_dir = "." os.environ["TMPDIR"] = env.Dir("{}/.response_files".format(build_dir)).get_abspath() os.environ["TEMP"] = os.environ["TMPDIR"] os.environ["TMP"] = os.environ["TMPDIR"] From e7f455f06332071ee340ef6d56ec49439a4ad303 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 12 Feb 2021 13:59:09 -0800 Subject: [PATCH 125/163] refactor initiazation state variable to be more module specific --- SCons/Tool/ninjaCommon/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index 9b330d3387..090d3d8d1b 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -1487,14 +1487,15 @@ def exists(env): return False -added = None +ninja_builder_initialized = False + def generate(env): """Generate the NINJA builders.""" from SCons.Script import AddOption, GetOption - global added - if not added: - added = 1 + global ninja_builder_initialized + if not ninja_builder_initialized: + ninja_builder_initialized = 1 AddOption('--disable-execute-ninja', dest='disable_execute_ninja', From 37ae2b8c8a8c714b100fe2a3ec9b308e8ca04b4d Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 12 Feb 2021 14:28:53 -0800 Subject: [PATCH 126/163] Fix ninja tool rules for macos/ar for static libs to skip response files for now. Also fix build_libraries to have proper shlib suffix --- SCons/Tool/ninjaCommon/__init__.py | 7 +++++++ test/ninja/build_libraries.py | 5 ++++- testing/framework/TestCmd.py | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index 090d3d8d1b..d46d692669 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -637,6 +637,13 @@ def __init__(self, env, ninja_file, writer_class): }, } + if env['PLATFORM'] == 'darwin' and env['AR'] == 'ar': + self.rules["AR"] = { + "command": "rm -f $out && $env$AR $rspc", + "description": "Archiving $out", + "pool": "local_pool", + } + self.pools = { "local_pool": self.env.GetOption("num_jobs"), "install_pool": self.env.GetOption("num_jobs") / 2, diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index a1c3282475..b3c7b54611 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -26,7 +26,7 @@ import os import TestSCons -from TestCmd import IS_WINDOWS +from TestCmd import IS_WINDOWS, IS_MACOS test = TestSCons.TestSCons() @@ -64,6 +64,9 @@ lib_prefix = 'lib' win32 = '' +if IS_MACOS: + lib_suffix = '.dylib' + test.write('SConstruct', """ env = Environment() env.Tool('ninja') diff --git a/testing/framework/TestCmd.py b/testing/framework/TestCmd.py index 333901c6a9..903bee693a 100644 --- a/testing/framework/TestCmd.py +++ b/testing/framework/TestCmd.py @@ -314,6 +314,7 @@ IS_WINDOWS = sys.platform == 'win32' +IS_MACOS = sys.platform == 'darwin' IS_64_BIT = sys.maxsize > 2**32 IS_PYPY = hasattr(sys, 'pypy_translation_info') From a7f25ea4a2b219278df4544f9cf9e2037a0d30f1 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 16 Feb 2021 11:51:47 -0800 Subject: [PATCH 127/163] [ci skip] Initial doc structure added --- SCons/Tool/ninjaCommon/__init__.py | 2 +- SCons/Tool/ninjaCommon/ninja.xml | 203 +++++++++++++++++++++++++++++ doc/user/external.xml | 56 +++++--- 3 files changed, 240 insertions(+), 21 deletions(-) create mode 100644 SCons/Tool/ninjaCommon/ninja.xml diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index d46d692669..b1bbb68779 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -972,7 +972,7 @@ def get_comstr(env, action, targets, sources): def get_command_env(env): """ - Return a string that sets the enrivonment for any environment variables that + Return a string that sets the environment for any environment variables that differ between the OS environment and the SCons command ENV. It will be compatible with the default shell of the operating system. diff --git a/SCons/Tool/ninjaCommon/ninja.xml b/SCons/Tool/ninjaCommon/ninja.xml new file mode 100644 index 0000000000..c7f693f350 --- /dev/null +++ b/SCons/Tool/ninjaCommon/ninja.xml @@ -0,0 +1,203 @@ + + + + + %scons; + + %builders-mod; + + %functions-mod; + + %tools-mod; + + %variables-mod; + ]> + + + + + + + Generate ninja build files and run ninja to build the bulk of your targets. + + + + NINJA_GENERATED_SOURCE_SUFFIXES + NINJA_MSVC_DEPS_PREFIX + NINJA_BUILDDIR + NINJA_REGENERATE_DEPS + NINJA_COMPDB_EXPAND + NINJA_ENV_VAR_CACHE + NINJA_POOL + DISABLE_AUTO_NINJA + __NINJA_NO + + NINJA_PREFIX + NINJA_SUFFIX + NINJA_ALIAS_NAME + + _NINJA_REGENERATE_DEPS_FUNC + + + + CCCOM + CXXCOM + SHCXXCOM + CC + CXX + + LINKCOM + LINK + + SHLINKCOM + SHLINK + + ARCOM + AR + RANLIB + ARFLAGS + RANLIBCOM + PLATFORM + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + + + + The C compiler. + + + + + + + + + The C compiler. + + + + + + + + + The C compiler. + + + + + + + + The C compiler. + + + + + + diff --git a/doc/user/external.xml b/doc/user/external.xml index 9900e93688..8c466dcdb2 100644 --- a/doc/user/external.xml +++ b/doc/user/external.xml @@ -22,26 +22,28 @@ @@ -283,4 +285,18 @@ env2.CompilationDatabase('compile_commands-linux64.json') + +
+ Ninja Build Generator + + + Ninja Build System + + + + + Ninja File Format Specification + + +
From 1af172f2174a9df3f81524f74234e4cbaea089bf Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 16 Feb 2021 15:04:42 -0800 Subject: [PATCH 128/163] Incremental checkin. Fleshing out the various env vars used by ninja builders. Also annotated code with some TODO: questions --- SCons/Tool/ninjaCommon/__init__.py | 8 ++++ SCons/Tool/ninjaCommon/ninja.xml | 72 ++++++++++++++++++++++++------ doc/user/external.xml | 2 +- 3 files changed, 68 insertions(+), 14 deletions(-) diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index b1bbb68779..288e3eaaad 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -897,6 +897,10 @@ def generate(self): implicit=[str(self.ninja_file)], variables={ "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( + # NINJA_COMPDB_EXPAND - should only be true for ninja + # This was added to ninja's compdb tool in version 1.9.0 (merged April 2018) + # https://github.com/ninja-build/ninja/pull/1223 + # TODO: add check in generate to check version and enable this by default if it's available. self.ninja_bin_path, str(self.ninja_file), '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' ) }, @@ -990,6 +994,7 @@ def get_command_env(env): scons_specified_env = { key: value for key, value in ENV.items() + # TODO: Remove this filter, unless there's a good reason to keep. SCons's behavior shouldn't depend on shell's. if key not in os.environ or os.environ.get(key, None) != value } @@ -1213,6 +1218,7 @@ def get_command(env, node, action): # pylint: disable=too-many-branches # env["NINJA_POOL"] = "ls_pool" # env.Command("ls") # + # TODO: Why not alloe env['NINJA_POOL'] ? (bdbaddog) if node.env and node.env.get("NINJA_POOL", None) is not None: ninja_build["pool"] = node.env["NINJA_POOL"] @@ -1534,8 +1540,10 @@ def generate(env): ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action) env.Append(BUILDERS={"Ninja": ninja_builder_obj}) + # TODO: Can't we simplify this to just be NINJA_FILENAME ? (bdbaddog) env["NINJA_PREFIX"] = env.get("NINJA_PREFIX", "build") env["NINJA_SUFFIX"] = env.get("NINJA_SUFFIX", "ninja") + env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") env['NINJA_BUILDDIR'] = env.get("NINJA_BUILDDIR", env.Dir(".ninja").path) ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") diff --git a/SCons/Tool/ninjaCommon/ninja.xml b/SCons/Tool/ninjaCommon/ninja.xml index c7f693f350..9eb7febef3 100644 --- a/SCons/Tool/ninjaCommon/ninja.xml +++ b/SCons/Tool/ninjaCommon/ninja.xml @@ -65,6 +65,7 @@ See its __doc__ string for a discussion of the format. NINJA_PREFIX NINJA_SUFFIX NINJA_ALIAS_NAME + NINJA_SYNTAX _NINJA_REGENERATE_DEPS_FUNC @@ -88,6 +89,7 @@ See its __doc__ string for a discussion of the format. ARFLAGS RANLIBCOM PLATFORM + ESCAPE @@ -95,7 +97,11 @@ See its __doc__ string for a discussion of the format. - The C compiler. + The list of source file suffixes which are generated by SCons build steps. + All source files which match these suffixes will be added to the _generated_sources alias in the output + ninja.build file. + Then all other source files will be made to depend on this in the Ninja.build file, forcing the + generated sources to be built first. @@ -103,7 +109,9 @@ See its __doc__ string for a discussion of the format. - The C compiler. + This propagates directly into the generated Ninja.build file. + From Ninja's docs + defines the string which should be stripped from msvc’s /showIncludes output @@ -111,7 +119,13 @@ See its __doc__ string for a discussion of the format. - The C compiler. + This propagates directly into the generated Ninja.build file. + From Ninja's docs: +
+ builddir + A directory for some Ninja output files. ... (You can also store other build output in this + directory.) +
@@ -119,7 +133,9 @@ See its __doc__ string for a discussion of the format. - The C compiler. + A generator function used to create a ninja depsfile which includes all the files which would require + SCons to be invoked if they change. + Or a list of said files. @@ -127,7 +143,13 @@ See its __doc__ string for a discussion of the format. - The C compiler. + Boolean value (True|False) to instruct ninja to expand the command line arguments normally put into + response files. + This prevents lines in the compilation database like gcc @rsp_file and instead yields + gcc -c -o myfile.o myfile.c -Ia -DXYZ + + + Ninja's compdb tool added the -x flag in Ninja V1.9.0 @@ -135,7 +157,14 @@ See its __doc__ string for a discussion of the format. - The C compiler. + A string that sets the environment for any environment variables that + differ between the OS environment and the SCons command ENV. + + It will be compatible with the default shell of the operating system. + + If not explicitly specified, SCons will generate this dynamically from the Environment()'s 'ENV' + env['ENV'] + where those values differ from the existing shell.. @@ -143,7 +172,7 @@ See its __doc__ string for a discussion of the format. - The C compiler. + Set the ninja_pool for this or all targets in scope for this env var. @@ -151,7 +180,9 @@ See its __doc__ string for a discussion of the format. - The C compiler. + Boolean (True|False). Default: False + When True, SCons will not run ninja automatically after creating the ninja.build file. + @@ -159,7 +190,8 @@ See its __doc__ string for a discussion of the format. - The C compiler. + Internal flag. Used to tell SCons whether or not to try to import pypi's ninja python package. + This is set to True when being called by Ninja? @@ -168,7 +200,8 @@ See its __doc__ string for a discussion of the format. - The C compiler. + The basename for the ninja.build file (so in that case just + ninja @@ -177,7 +210,8 @@ See its __doc__ string for a discussion of the format. - The C compiler. + The suffix for the ninja.build file (so in that case just + build @@ -186,7 +220,18 @@ See its __doc__ string for a discussion of the format. - The C compiler. + Name of the Alias() which is will cause SCons to create the ninja.build file, and + then (optionally) run ninja. + + + + + + + + Theres also NINJA_SYNTAX which is the path to a custom ninja_syntax.py file which is used in generation. + The tool currently assumes you have ninja installed through pip, and grabs the syntax file from that + installation if none specified. @@ -194,7 +239,8 @@ See its __doc__ string for a discussion of the format. - The C compiler. + Internal value used to specify the function to call with argument env to generate the list of files + which if changed would require the ninja file to be regenerated. diff --git a/doc/user/external.xml b/doc/user/external.xml index 8c466dcdb2..5d24845c51 100644 --- a/doc/user/external.xml +++ b/doc/user/external.xml @@ -292,7 +292,7 @@ env2.CompilationDatabase('compile_commands-linux64.json') Ninja Build System - + Ninja File Format Specification From 1c561508013b94577c378d8b3aca55eb5439035e Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 18 Feb 2021 10:47:01 -0800 Subject: [PATCH 129/163] Fix path to SCons/Docbooks style files. Add some content. Still a WIP --- SCons/Tool/ninjaCommon/ninja.xml | 86 ++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 22 deletions(-) diff --git a/SCons/Tool/ninjaCommon/ninja.xml b/SCons/Tool/ninjaCommon/ninja.xml index 9eb7febef3..8f7de362bf 100644 --- a/SCons/Tool/ninjaCommon/ninja.xml +++ b/SCons/Tool/ninjaCommon/ninja.xml @@ -29,15 +29,15 @@ See its __doc__ string for a discussion of the format. --> + %scons; - + %builders-mod; - + %functions-mod; - + %tools-mod; - + %variables-mod; ]> @@ -48,7 +48,7 @@ See its __doc__ string for a discussion of the format. - Generate ninja build files and run ninja to build the bulk of your targets. + Sets up &b-link-Ninja; builder which generates a ninja build file, and then optionally runs ninja. @@ -62,8 +62,8 @@ See its __doc__ string for a discussion of the format. DISABLE_AUTO_NINJA __NINJA_NO - NINJA_PREFIX - NINJA_SUFFIX + NINJA_FILE_NAME + NINJA_ALIAS_NAME NINJA_SYNTAX @@ -94,6 +94,60 @@ See its __doc__ string for a discussion of the format. + + + + &b-Ninja; is a special builder which + adds a target to create a ninja build file. + The builder does not require any source files to be specified, + + + + If called with no arguments, + the builder will default to a target name of + ninja.build. + + + If called with a single positional argument, + &scons; will "deduce" the target name from that source + argument, giving it the same name, and then + ignore the source. + This is the usual way to call the builder if a + non-default target name is wanted. + + + If called with either the + target= + or source= keyword arguments, + the value of the argument is taken as the target name. + If called with both, the + target= + value is used and source= is ignored. + If called with multiple sources, + the source list will be ignored, + since there is no way to deduce what the intent was; + in this case the default target name will be used. + + + + You must load the &t-ninja; tool prior to specifying + any part of your build or some source/output + files will not show up in the compilation database. + + + To use this tool you must install pypi's ninja + package. + This can be done via + pip install ninja + + + + Available since &scons; 4.2. + + + + + @@ -197,26 +251,14 @@ See its __doc__ string for a discussion of the format. - - - - The basename for the ninja.build file (so in that case just - ninja - - - - - - + - The suffix for the ninja.build file (so in that case just - build + The filename for the generated Ninja build file defaults to ninja.build - From 87566574533e94c1d2d5af30ecfa5113b72ce88c Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 14 Mar 2021 17:24:33 -0700 Subject: [PATCH 130/163] [ci skip] fix copyright header --- test/ninja/generate_and_build_cxx.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/ninja/generate_and_build_cxx.py b/test/ninja/generate_and_build_cxx.py index 663282bd92..1d149a9822 100644 --- a/test/ninja/generate_and_build_cxx.py +++ b/test/ninja/generate_and_build_cxx.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -22,8 +22,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - import os import TestSCons from TestCmd import IS_WINDOWS From b4786e19ce31e9b6a696728c84bb8987fd621731 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 14 Mar 2021 17:58:31 -0700 Subject: [PATCH 131/163] Add API to see if a node has a given attribute stored in node.attributes --- SCons/Node/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/SCons/Node/__init__.py b/SCons/Node/__init__.py index a449082a06..619211b8ef 100644 --- a/SCons/Node/__init__.py +++ b/SCons/Node/__init__.py @@ -946,6 +946,11 @@ def is_conftest(self): return False return True + def check_attributes(self, name): + """ Simple API to check if the node.attributes for name has been set""" + return getattr(getattr(self,"attributes", None), name, None) + + def alter_targets(self): """Return a list of alternate targets for this Node. """ From 1ddb0bdeddbcb4bbc9702ee1f6c286310e3b3c87 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 15 Mar 2021 08:32:33 -0700 Subject: [PATCH 132/163] Incremental update with code reorganization. Currently broken. Some debug code enabled --- SCons/Tool/ninjaCommon/Rules.py | 58 ++++ SCons/Tool/ninjaCommon/Util.py | 173 +++++++++++ SCons/Tool/ninjaCommon/__init__.py | 289 +++--------------- test/ninja/generate_and_build.py | 4 +- test/ninja/generate_and_build_cxx.py | 13 +- .../sconstruct_generate_and_build_cxx.py | 4 + 6 files changed, 293 insertions(+), 248 deletions(-) create mode 100644 SCons/Tool/ninjaCommon/Rules.py create mode 100644 SCons/Tool/ninjaCommon/Util.py create mode 100644 test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py diff --git a/SCons/Tool/ninjaCommon/Rules.py b/SCons/Tool/ninjaCommon/Rules.py new file mode 100644 index 0000000000..99b11e4c8e --- /dev/null +++ b/SCons/Tool/ninjaCommon/Rules.py @@ -0,0 +1,58 @@ +from .Util import get_outputs, get_rule, get_inputs, get_dependencies + + +def _install_action_function(_env, node): + """Install files using the install or copy commands""" + return { + "outputs": get_outputs(node), + "rule": get_rule(node, "INSTALL"), + "inputs": get_inputs(node), + "implicit": get_dependencies(node), + } + + +def _mkdir_action_function(env, node): + return { + "outputs": get_outputs(node), + "rule": get_rule(node, "CMD"), + # implicit explicitly omitted, we translate these so they can be + # used by anything that depends on these but commonly this is + # hit with a node that will depend on all of the fake + # srcnode's that SCons will never give us a rule for leading + # to an invalid ninja file. + "variables": { + # On Windows mkdir "-p" is always on + "cmd": "{mkdir}".format( + mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", + ), + }, + } + + +def _copy_action_function(env, node): + return { + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "rule": get_rule(node, "CMD"), + "variables": { + "cmd": "$COPY $in $out", + }, + } + + +def _lib_symlink_action_function(_env, node): + """Create shared object symlinks if any need to be created""" + symlinks = node.check_attributes("shliblinks") + + if not symlinks or symlinks is None: + return None + + outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] + inputs = [link.get_path() for link, _ in symlinks] + + return { + "outputs": outputs, + "inputs": inputs, + "rule": get_rule(node, "SYMLINK"), + "implicit": get_dependencies(node), + } \ No newline at end of file diff --git a/SCons/Tool/ninjaCommon/Util.py b/SCons/Tool/ninjaCommon/Util.py new file mode 100644 index 0000000000..5f14a96060 --- /dev/null +++ b/SCons/Tool/ninjaCommon/Util.py @@ -0,0 +1,173 @@ +import SCons +from SCons.Script import AddOption + +def ninja_add_command_line_options(): + """ + Add additional command line arguments to SCons specific to the ninja tool + """ + AddOption('--disable-execute-ninja', + dest='disable_execute_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + + AddOption('--disable-ninja', + dest='disable_ninja', + metavar='BOOL', + action="store_true", + default=False, + help='Disable ninja automatically building after scons') + + +def is_valid_dependent_node(node): + """ + Return True if node is not an alias or is an alias that has children + + This prevents us from making phony targets that depend on other + phony targets that will never have an associated ninja build + target. + + We also have to specify that it's an alias when doing the builder + check because some nodes (like src files) won't have builders but + are valid implicit dependencies. + """ + if isinstance(node, SCons.Node.Alias.Alias): + return node.children() + + if not node.env: + return True + + return not node.env.get("NINJA_SKIP") + + +def alias_to_ninja_build(node): + """Convert an Alias node into a Ninja phony target""" + return { + "outputs": get_outputs(node), + "rule": "phony", + "implicit": [ + get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) + ], + } + + +def check_invalid_ninja_node(node): + return not isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)) + + +def filter_ninja_nodes(node_list): + ninja_nodes = [] + for node in node_list: + if isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)): + ninja_nodes.append(node) + else: + continue + return ninja_nodes + + +def get_input_nodes(node): + if node.get_executor() is not None: + inputs = node.get_executor().get_all_sources() + else: + inputs = node.sources + return inputs + + +def invalid_ninja_nodes(node, targets): + result = False + for node_list in [node.prerequisites, get_input_nodes(node), node.children(), targets]: + if node_list: + result = result or any([check_invalid_ninja_node(node) for node in node_list]) + return result + + +def get_order_only(node): + """Return a list of order only dependencies for node.""" + if node.prerequisites is None: + return [] + return [get_path(src_file(prereq)) for prereq in filter_ninja_nodes(node.prerequisites)] + + +def get_dependencies(node, skip_sources=False): + """Return a list of dependencies for node.""" + if skip_sources: + return [ + get_path(src_file(child)) + for child in filter_ninja_nodes(node.children()) + if child not in node.sources + ] + return [get_path(src_file(child)) for child in filter_ninja_nodes(node.children())] + + +def get_inputs(node): + """Collect the Ninja inputs for node.""" + return [get_path(src_file(o)) for o in filter_ninja_nodes(get_input_nodes(node))] + + +def get_outputs(node): + """Collect the Ninja outputs for node.""" + executor = node.get_executor() + if executor is not None: + outputs = executor.get_all_targets() + else: + if hasattr(node, "target_peers"): + outputs = node.target_peers + else: + outputs = [node] + + outputs = [get_path(o) for o in filter_ninja_nodes(outputs)] + + return outputs + + +def get_targets_sources(node): + executor = node.get_executor() + if executor is not None: + tlist = executor.get_all_targets() + slist = executor.get_all_sources() + else: + if hasattr(node, "target_peers"): + tlist = node.target_peers + else: + tlist = [node] + slist = node.sources + + # Retrieve the repository file for all sources + slist = [rfile(s) for s in slist] + return tlist, slist + +def get_path(node): + """ + Return a fake path if necessary. + + As an example Aliases use this as their target name in Ninja. + """ + if hasattr(node, "get_path"): + return node.get_path() + return str(node) + + +def rfile(node): + """ + Return the repository file for node if it has one. Otherwise return node + """ + if hasattr(node, "rfile"): + return node.rfile() + return node + + +def src_file(node): + """Returns the src code file if it exists.""" + if hasattr(node, "srcnode"): + src = node.srcnode() + if src.stat() is not None: + return src + return get_path(node) + +def get_rule(node, rule): + tlist, slist = get_targets_sources(node) + if invalid_ninja_nodes(node, tlist): + return "TEMPLATE" + else: + return rule \ No newline at end of file diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index 288e3eaaad..bb74845a90 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -24,24 +24,30 @@ """Generate build.ninja files from SCons aliases.""" -import sys -import os import importlib import io -import shutil +import os import shlex +import shutil import subprocess +import sys import textwrap - from glob import glob from os.path import join as joinpath from os.path import splitext import SCons from SCons.Action import _string_from_cmd_list, get_default_ENV -from SCons.Util import is_List, flatten_sequence -from SCons.Script import COMMAND_LINE_TARGETS from SCons.Node import SConscriptNodes +from SCons.Script import AddOption, GetOption +from SCons.Script import COMMAND_LINE_TARGETS + +from .Util import ninja_add_command_line_options, is_valid_dependent_node, alias_to_ninja_build, \ + filter_ninja_nodes, get_input_nodes, invalid_ninja_nodes, get_order_only, get_dependencies, get_inputs, get_outputs, \ + get_targets_sources, get_path +from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function + +from SCons.Util import is_List, flatten_sequence NINJA_STATE = None NINJA_RULES = "__NINJA_CUSTOM_RULES" @@ -59,105 +65,8 @@ SCons.Action.CommandGeneratorAction, ) +ninja_builder_initialized = False -def is_valid_dependent_node(node): - """ - Return True if node is not an alias or is an alias that has children - - This prevents us from making phony targets that depend on other - phony targets that will never have an associated ninja build - target. - - We also have to specify that it's an alias when doing the builder - check because some nodes (like src files) won't have builders but - are valid implicit dependencies. - """ - if isinstance(node, SCons.Node.Alias.Alias): - return node.children() - - if not node.env: - return True - - return not node.env.get("NINJA_SKIP") - - -def alias_to_ninja_build(node): - """Convert an Alias node into a Ninja phony target""" - return { - "outputs": get_outputs(node), - "rule": "phony", - "implicit": [ - get_path(src_file(n)) for n in node.children() if is_valid_dependent_node(n) - ], - } - - -def check_invalid_ninja_node(node): - return not isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)) - - -def filter_ninja_nodes(node_list): - ninja_nodes = [] - for node in node_list: - if isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)): - ninja_nodes.append(node) - else: - continue - return ninja_nodes - -def get_input_nodes(node): - if node.get_executor() is not None: - inputs = node.get_executor().get_all_sources() - else: - inputs = node.sources - return inputs - - -def invalid_ninja_nodes(node, targets): - result = False - for node_list in [node.prerequisites, get_input_nodes(node), node.children(), targets]: - if node_list: - result = result or any([check_invalid_ninja_node(node) for node in node_list]) - return result - - -def get_order_only(node): - """Return a list of order only dependencies for node.""" - if node.prerequisites is None: - return [] - return [get_path(src_file(prereq)) for prereq in filter_ninja_nodes(node.prerequisites)] - - -def get_dependencies(node, skip_sources=False): - """Return a list of dependencies for node.""" - if skip_sources: - return [ - get_path(src_file(child)) - for child in filter_ninja_nodes(node.children()) - if child not in node.sources - ] - return [get_path(src_file(child)) for child in filter_ninja_nodes(node.children())] - - -def get_inputs(node): - """Collect the Ninja inputs for node.""" - return [get_path(src_file(o)) for o in filter_ninja_nodes(get_input_nodes(node))] - - -def get_outputs(node): - """Collect the Ninja outputs for node.""" - executor = node.get_executor() - if executor is not None: - outputs = executor.get_all_targets() - else: - if hasattr(node, "target_peers"): - outputs = node.target_peers - else: - outputs = [node] - - outputs = [get_path(o) for o in filter_ninja_nodes(outputs)] - - return outputs def generate_depfile(env, node, dependencies): """ @@ -195,87 +104,8 @@ def generate_depfile(env, node, dependencies): f.write(depfile_contents) -def get_targets_sources(node): - executor = node.get_executor() - if executor is not None: - tlist = executor.get_all_targets() - slist = executor.get_all_sources() - else: - if hasattr(node, "target_peers"): - tlist = node.target_peers - else: - tlist = [node] - slist = node.sources - - # Retrieve the repository file for all sources - slist = [rfile(s) for s in slist] - return tlist, slist - - -def get_rule(node, rule): - tlist, slist = get_targets_sources(node) - if invalid_ninja_nodes(node, tlist): - return "TEMPLATE" - else: - return rule - - -def _install_action_function(_env, node): - """Install files using the install or copy commands""" - return { - "outputs": get_outputs(node), - "rule": get_rule(node, "INSTALL"), - "inputs": get_inputs(node), - "implicit": get_dependencies(node), - } - - -def _mkdir_action_function(env, node): - return { - "outputs": get_outputs(node), - "rule": get_rule(node, "CMD"), - # implicit explicitly omitted, we translate these so they can be - # used by anything that depends on these but commonly this is - # hit with a node that will depend on all of the fake - # srcnode's that SCons will never give us a rule for leading - # to an invalid ninja file. - "variables": { - # On Windows mkdir "-p" is always on - "cmd": "{mkdir}".format( - mkdir="mkdir $out & exit 0" if env["PLATFORM"] == "win32" else "mkdir -p $out", - ), - }, - } -def _copy_action_function(env, node): - return { - "outputs": get_outputs(node), - "inputs": get_inputs(node), - "rule": get_rule(node, "CMD"), - "variables": { - "cmd": "$COPY $in $out", - }, - } - - -def _lib_symlink_action_function(_env, node): - """Create shared object symlinks if any need to be created""" - symlinks = getattr(getattr(node, "attributes", None), "shliblinks", None) - - if not symlinks or symlinks is None: - return None - - outputs = [link.get_dir().rel_path(linktgt) for link, linktgt in symlinks] - inputs = [link.get_path() for link, _ in symlinks] - - return { - "outputs": outputs, - "inputs": inputs, - "rule": get_rule(node, "SYMLINK"), - "implicit": get_dependencies(node), - } - class SConsToNinjaTranslator: """Translates SCons Actions into Ninja build objects.""" @@ -342,8 +172,8 @@ def action_to_ninja_build(self, node, action=None): build["order_only"] = get_order_only(node) #TODO: WPD Is this testing the filename to verify it's a configure context generated file? - if 'conftest' not in str(node): - node_callback = getattr(node.attributes, "ninja_build_callback", None) + if not node.is_conftest(): + node_callback = node.check_attr("ninja_build_callback") if callable(node_callback): node_callback(env, node, build) @@ -930,33 +760,7 @@ def generate(self): self.__generated = True -def get_path(node): - """ - Return a fake path if necessary. - As an example Aliases use this as their target name in Ninja. - """ - if hasattr(node, "get_path"): - return node.get_path() - return str(node) - - -def rfile(node): - """ - Return the repository file for node if it has one. Otherwise return node - """ - if hasattr(node, "rfile"): - return node.rfile() - return node - - -def src_file(node): - """Returns the src code file if it exists.""" - if hasattr(node, "srcnode"): - src = node.srcnode() - if src.stat() is not None: - return src - return get_path(node) def get_comstr(env, action, targets, sources): @@ -1118,6 +922,7 @@ def get_generic_shell_command(env, node, action, targets, sources, executor=None return ( "CMD", { + # TODO: Why is executor passed in and then ignored below? (bdbaddog) "cmd": generate_command(env, node, action, targets, sources, executor=None), "env": get_command_env(env), }, @@ -1337,9 +1142,11 @@ def register_custom_pool(env, pool, size): """Allows the creation of custom Ninja pools""" env[NINJA_POOLS][pool] = size + def set_build_node_callback(env, node, callback): - if 'conftest' not in str(node): - setattr(node.attributes, "ninja_build_callback", callback) + if not node.is_conftest(): + node.attributes.ninja_build_callback = callback + def ninja_csig(original): """Return a dummy csig""" @@ -1362,6 +1169,7 @@ def wrapper(self): return wrapper + def CheckNinjaCompdbExpand(env, context): """ Configure check testing if ninja's compdb can expand response files""" @@ -1500,30 +1308,34 @@ def exists(env): return False -ninja_builder_initialized = False +def ninja_emitter(target, source, env): + """ fix up the source/targets """ + + ninja_file = env.File(env.subst("$NINJA_FILE_NAME")) + ninja_file.attributes.ninja_file = True + + # Someone called env.Ninja('my_targetname.ninja') + if not target and len(source) == 1: + target = source + + # Default target name is $NINJA_PREFIX.$NINJA.SUFFIX + if not target: + target = [ninja_file, ] + + # No source should have been passed. Drop it. + if source: + source = [] + + return target, source def generate(env): """Generate the NINJA builders.""" - from SCons.Script import AddOption, GetOption global ninja_builder_initialized if not ninja_builder_initialized: - ninja_builder_initialized = 1 - - AddOption('--disable-execute-ninja', - dest='disable_execute_ninja', - metavar='BOOL', - action="store_true", - default=False, - help='Disable ninja automatically building after scons') - - AddOption('--disable-ninja', - dest='disable_ninja', - metavar='BOOL', - action="store_true", - default=False, - help='Disable ninja automatically building after scons') + ninja_builder_initialized = True + ninja_add_command_line_options() try: import ninja @@ -1535,22 +1347,22 @@ def generate(env): global NINJA_STATE + env["NINJA_FILE_NAME"] = env.get("NINJA_FILE_NAME", "build.ninja") + # Add the Ninja builder. always_exec_ninja_action = AlwaysExecAction(ninja_builder, {}) - ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action) + ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action, + emitter=ninja_emitter) env.Append(BUILDERS={"Ninja": ninja_builder_obj}) - # TODO: Can't we simplify this to just be NINJA_FILENAME ? (bdbaddog) - env["NINJA_PREFIX"] = env.get("NINJA_PREFIX", "build") - env["NINJA_SUFFIX"] = env.get("NINJA_SUFFIX", "ninja") env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") env['NINJA_BUILDDIR'] = env.get("NINJA_BUILDDIR", env.Dir(".ninja").path) - ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") + # here we allow multiple environments to construct rules and builds # into the same ninja file if NINJA_STATE is None: - ninja_file = env.Ninja(target=ninja_file_name, source=[]) + ninja_file = env.Ninja() env.AlwaysBuild(ninja_file) env.Alias("$NINJA_ALIAS_NAME", ninja_file) else: @@ -1558,7 +1370,6 @@ def generate(env): SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] - # TODO: API for getting the SConscripts programmatically # exists upstream: https://github.com/SCons/scons/issues/3625 def ninja_generate_deps(env): @@ -1781,7 +1592,7 @@ def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): target = [target] for target in targets: - if str(target) != ninja_file_name and "conftest" not in str(target): + if target.check_attributes('ninja_file') is None and not target.is_conftest(): env.Depends(ninja_file, targets) return targets SCons.Builder.BuilderBase._execute = NinjaBuilderExecute @@ -1801,7 +1612,7 @@ def ninja_execute(self): target = self.targets[0] target_name = str(target) - if target_name != ninja_file_name and "conftest" not in target_name: + if target.check_attributes('ninja_file') is None and not target.is_conftest: NINJA_STATE.add_build(target) else: target.build() diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index faf395a1ef..147ba3a006 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -22,8 +22,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - import os import TestSCons from TestCmd import IS_WINDOWS diff --git a/test/ninja/generate_and_build_cxx.py b/test/ninja/generate_and_build_cxx.py index 1d149a9822..ee6e0bd8e3 100644 --- a/test/ninja/generate_and_build_cxx.py +++ b/test/ninja/generate_and_build_cxx.py @@ -45,14 +45,15 @@ test.dir_fixture('ninja-fixture') -test.write('SConstruct', """ -env = Environment() -env.Tool('ninja') -env.Program(target = 'test2', source = 'test2.cpp') -""") +# test.write('SConstruct', """ +# env = Environment() +# env.Tool('ninja') +# env.Program(target = 'test2', source = 'test2.cpp') +# """) +test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py','SConstruct') # generate simple build -test.run(stdout=None) +test.run() test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py new file mode 100644 index 0000000000..0b16d819c9 --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py @@ -0,0 +1,4 @@ +# DefaultEnvironment(tools=[]) +env = Environment() +env.Tool('ninja') +env.Program(target = 'test2', source = 'test2.cpp') From ccc5e26fa492d53788364c68eeb73b8d96563051 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 24 Mar 2021 17:27:36 -0700 Subject: [PATCH 133/163] add sconstest.skip to file fixture dir for ninja system tests --- test/ninja/ninja_test_sconscripts/sconstest.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 test/ninja/ninja_test_sconscripts/sconstest.skip diff --git a/test/ninja/ninja_test_sconscripts/sconstest.skip b/test/ninja/ninja_test_sconscripts/sconstest.skip new file mode 100644 index 0000000000..e69de29bb2 From 41e4859f18fecea0c5fbf59e38cb9643fa0d52fd Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 27 Mar 2021 20:35:22 -0700 Subject: [PATCH 134/163] fix broken target check in ninja_execute. It was improperly adding all nodes to NINJA_STATE.add_build(). Instead of only ones which weren't ninja files, nor conftest files --- SCons/Node/__init__.py | 2 +- SCons/Tool/ninjaCommon/__init__.py | 7 ++++--- doc/user/external.xml | 6 ++++++ test/ninja/generate_and_build.py | 18 +++++++----------- .../sconstruct_generate_and_build.py | 5 +++++ .../sconstruct_generate_and_build_cxx.py | 2 +- 6 files changed, 24 insertions(+), 16 deletions(-) create mode 100644 test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py diff --git a/SCons/Node/__init__.py b/SCons/Node/__init__.py index 619211b8ef..ec742a686b 100644 --- a/SCons/Node/__init__.py +++ b/SCons/Node/__init__.py @@ -948,7 +948,7 @@ def is_conftest(self): def check_attributes(self, name): """ Simple API to check if the node.attributes for name has been set""" - return getattr(getattr(self,"attributes", None), name, None) + return getattr(getattr(self, "attributes", None), name, None) def alter_targets(self): diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index bb74845a90..e6eb9421e1 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -44,7 +44,7 @@ from .Util import ninja_add_command_line_options, is_valid_dependent_node, alias_to_ninja_build, \ filter_ninja_nodes, get_input_nodes, invalid_ninja_nodes, get_order_only, get_dependencies, get_inputs, get_outputs, \ - get_targets_sources, get_path + get_targets_sources, get_path, get_rule from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function from SCons.Util import is_List, flatten_sequence @@ -173,7 +173,7 @@ def action_to_ninja_build(self, node, action=None): #TODO: WPD Is this testing the filename to verify it's a configure context generated file? if not node.is_conftest(): - node_callback = node.check_attr("ninja_build_callback") + node_callback = node.check_attributes("ninja_build_callback") if callable(node_callback): node_callback(env, node, build) @@ -1612,7 +1612,8 @@ def ninja_execute(self): target = self.targets[0] target_name = str(target) - if target.check_attributes('ninja_file') is None and not target.is_conftest: + # print("File:%s -> %s"%(str(target), target.check_attributes('ninja_file'))) + if target.check_attributes('ninja_file') is None or not target.is_conftest: NINJA_STATE.add_build(target) else: target.build() diff --git a/doc/user/external.xml b/doc/user/external.xml index 5d24845c51..f580d3ff46 100644 --- a/doc/user/external.xml +++ b/doc/user/external.xml @@ -289,6 +289,12 @@ env2.CompilationDatabase('compile_commands-linux64.json')
Ninja Build Generator + + This is an experimental new feature. + + + Using the + Ninja Build System diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index 147ba3a006..6972a54aa5 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -23,6 +23,7 @@ # import os + import TestSCons from TestCmd import IS_WINDOWS @@ -34,7 +35,7 @@ test.skip_test("Could not find module in python") _python_ = TestSCons._python_ -_exe = TestSCons._exe +_exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( ninja.__file__, @@ -45,19 +46,14 @@ test.dir_fixture('ninja-fixture') -test.write('SConstruct', """ -env = Environment() -env.Tool('ninja') -env.Program(target = 'foo', source = 'foo.c') -""") - +test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build.py', 'SConstruct') # generate simple build test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('foo' + _exe), stdout="foo.c") +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('foo' + _exe), stdout="foo.c") # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -73,8 +69,8 @@ # run ninja independently program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('foo' + _exe), stdout="foo.c") +test.run(program=program, stdout=None) +test.run(program=test.workpath('foo' + _exe), stdout="foo.c") test.pass_test() diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py new file mode 100644 index 0000000000..242eb76265 --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py @@ -0,0 +1,5 @@ +DefaultEnvironment(tools=[]) + +env = Environment() +env.Tool('ninja') +env.Program(target='foo', source='foo.c') diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py index 0b16d819c9..51ca6c356d 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py @@ -1,4 +1,4 @@ -# DefaultEnvironment(tools=[]) +DefaultEnvironment(tools=[]) env = Environment() env.Tool('ninja') env.Program(target = 'test2', source = 'test2.cpp') From 554a02f8998e8a5ffd3d5405e531ec334c1ba11f Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 27 Mar 2021 21:10:10 -0700 Subject: [PATCH 135/163] address sider issues. Also update copyright text to current version --- SCons/Tool/ninja.py | 2 +- SCons/Tool/ninjaCommon/__init__.py | 39 +++++++++---------- test/ninja/build_libraries.py | 3 +- test/ninja/copy_function_command.py | 4 +- test/ninja/generate_source.py | 4 +- test/ninja/iterative_speedup.py | 4 +- test/ninja/multi_env.py | 4 +- .../sconstruct_generate_and_build.py | 1 + .../sconstruct_generate_and_build_cxx.py | 2 + test/ninja/shell_command.py | 4 +- 10 files changed, 28 insertions(+), 39 deletions(-) diff --git a/SCons/Tool/ninja.py b/SCons/Tool/ninja.py index e597971d2d..36daec6ae8 100644 --- a/SCons/Tool/ninja.py +++ b/SCons/Tool/ninja.py @@ -22,5 +22,5 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # - +# noqa: F401 from .ninjaCommon import generate, exists diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index e6eb9421e1..0af6c21b3e 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -38,16 +38,14 @@ import SCons from SCons.Action import _string_from_cmd_list, get_default_ENV -from SCons.Node import SConscriptNodes -from SCons.Script import AddOption, GetOption +# from SCons.Node import SConscriptNodes +from SCons.Script import GetOption from SCons.Script import COMMAND_LINE_TARGETS - -from .Util import ninja_add_command_line_options, is_valid_dependent_node, alias_to_ninja_build, \ - filter_ninja_nodes, get_input_nodes, invalid_ninja_nodes, get_order_only, get_dependencies, get_inputs, get_outputs, \ - get_targets_sources, get_path, get_rule -from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function - from SCons.Util import is_List, flatten_sequence +from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function +from .Util import ninja_add_command_line_options, alias_to_ninja_build, \ + get_order_only, get_dependencies, get_inputs, get_outputs, \ + get_targets_sources, get_path, get_rule NINJA_STATE = None NINJA_RULES = "__NINJA_CUSTOM_RULES" @@ -886,7 +884,7 @@ def get_response_file_command(env, node, action, targets, sources, executor=None for key, value in custom_env.items(): variables["env"] += env.subst( - "export %s=%s;"%(key, value), target=targets, source=sources, executor=executor + "export %s=%s;" % (key, value), target=targets, source=sources, executor=executor ) + " " return rule, variables, [tool_command] @@ -930,8 +928,8 @@ def get_generic_shell_command(env, node, action, targets, sources, executor=None # and usually this would be the path to a tool, such as a compiler, used for this rule. # However this function is to generic to be able to reliably extract such deps # from the command, so we return a placeholder empty list. It should be noted that - # generally this function will not be used soley and is more like a template to generate - # the basics for a custom provider which may have more specific options for a provier + # generally this function will not be used solely and is more like a template to generate + # the basics for a custom provider which may have more specific options for a provider # function for a custom NinjaRuleMapping. [] ) @@ -997,7 +995,7 @@ def get_command(env, node, action): # pylint: disable=too-many-branches # Possibly these could be ignore and the build would still work, however it may not always # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided # ninja rule. - raise Exception("Could not resolve path for %s dependency on node '%s'"%(provider_dep, node)) + raise Exception("Could not resolve path for %s dependency on node '%s'" % (provider_dep, node)) ninja_build = { "order_only": get_order_only(node), @@ -1068,11 +1066,11 @@ def ninja_builder(env, target, source): def execute_ninja(): proc = subprocess.Popen(cmd, - stderr=sys.stderr, - stdout=subprocess.PIPE, - universal_newlines=True, - env=os.environ if env["PLATFORM"] == "win32" else env['ENV'] - ) + stderr=sys.stderr, + stdout=subprocess.PIPE, + universal_newlines=True, + env=os.environ if env["PLATFORM"] == "win32" else env['ENV'] + ) for stdout_line in iter(proc.stdout.readline, ""): yield stdout_line proc.stdout.close() @@ -1084,7 +1082,7 @@ def execute_ninja(): for output in execute_ninja(): output = output.strip() if erase_previous: - sys.stdout.write('\x1b[2K') # erase previous line + sys.stdout.write('\x1b[2K') # erase previous line sys.stdout.write("\r") else: sys.stdout.write(os.linesep) @@ -1366,7 +1364,7 @@ def generate(env): env.AlwaysBuild(ninja_file) env.Alias("$NINJA_ALIAS_NAME", ninja_file) else: - if str(NINJA_STATE.ninja_file) != ninja_file_name: + if str(NINJA_STATE.ninja_file) != env["NINJA_FILE_NAME"]: SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] @@ -1582,6 +1580,7 @@ def robust_rule_mapping(var, rule, tool): # We will subvert the normal builder execute to make sure all the ninja file is dependent # on all targets generated from any builders SCons_Builder_BuilderBase__execute = SCons.Builder.BuilderBase._execute + def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): # this ensures all environments in which a builder executes from will # not create list actions for linking on windows @@ -1611,8 +1610,6 @@ def ninja_execute(self): global NINJA_STATE target = self.targets[0] - target_name = str(target) - # print("File:%s -> %s"%(str(target), target.check_attributes('ninja_file'))) if target.check_attributes('ninja_file') is None or not target.is_conftest: NINJA_STATE.add_build(target) else: diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index b3c7b54611..d1e7c13dc2 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -22,7 +22,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os import TestSCons diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index 8e7acff7b7..08e8274622 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -22,8 +22,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - import os import TestSCons from TestCmd import IS_WINDOWS diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index 76c79bb7da..141b19c24d 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -22,8 +22,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - import os import TestSCons from TestCmd import IS_WINDOWS diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index ff50f502a3..5fd4bc5aa2 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -22,8 +22,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - import os import time import random diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 18ca3cbc69..53ec83b505 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -22,8 +22,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - import os import TestSCons from TestCmd import IS_WINDOWS diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py index 242eb76265..4123fd87b0 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py @@ -1,3 +1,4 @@ +# noqa: f821 DefaultEnvironment(tools=[]) env = Environment() diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py index 51ca6c356d..ab304fabe1 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py @@ -1,3 +1,5 @@ +# noqa: f821 + DefaultEnvironment(tools=[]) env = Environment() env.Tool('ninja') diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index 5d7f97e215..f477505293 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -22,8 +22,6 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - import os import TestSCons from TestCmd import IS_WINDOWS From 6da28597cc5b38df31c2e1f32934234d98de9a6c Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 27 Mar 2021 21:22:43 -0700 Subject: [PATCH 136/163] Continue fixing sider complaints. --- test/ninja/build_libraries.py | 17 +++++------ test/ninja/copy_function_command.py | 11 +++---- test/ninja/generate_and_build.py | 2 +- test/ninja/generate_and_build_cxx.py | 3 +- test/ninja/iterative_speedup.py | 30 +++++++++++-------- test/ninja/multi_env.py | 15 +++++----- ...build.py => sconstruct_generate_and_build} | 1 - ...x.py => sconstruct_generate_and_build_cxx} | 2 -- 8 files changed, 43 insertions(+), 38 deletions(-) rename test/ninja/ninja_test_sconscripts/{sconstruct_generate_and_build.py => sconstruct_generate_and_build} (89%) rename test/ninja/ninja_test_sconscripts/{sconstruct_generate_and_build_cxx.py => sconstruct_generate_and_build_cxx} (89%) diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index d1e7c13dc2..198dcced99 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -24,6 +24,7 @@ import os + import TestSCons from TestCmd import IS_WINDOWS, IS_MACOS @@ -34,14 +35,12 @@ except ImportError: test.skip_test("Could not find ninja module in python") - ninja_binary = test.where_is('ninja') if not ninja_binary: test.skip_test("Could not find ninja. Skipping test.") - _python_ = TestSCons._python_ -_exe = TestSCons._exe +_exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( ninja.__file__, @@ -83,9 +82,9 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('test'), stdout="library_function") -test.run(program = test.workpath('test_static'), stdout="library_function") +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('test'), stdout="library_function") +test.run(program=test.workpath('test_static'), stdout="library_function") # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -104,9 +103,9 @@ # run ninja independently program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('test'), stdout="library_function") -test.run(program = test.workpath('test_static'), stdout="library_function") +test.run(program=program, stdout=None) +test.run(program=test.workpath('test'), stdout="library_function") +test.run(program=test.workpath('test_static'), stdout="library_function") test.pass_test() diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index 08e8274622..37af62f52e 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -23,6 +23,7 @@ # import os + import TestSCons from TestCmd import IS_WINDOWS @@ -34,7 +35,7 @@ test.skip_test("Could not find module in python") _python_ = TestSCons._python_ -_exe = TestSCons._exe +_exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( ninja.__file__, @@ -56,8 +57,8 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('foo'), stdout="foo.c") +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('foo'), stdout="foo.c") # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -74,8 +75,8 @@ # run ninja independently program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('foo'), stdout="foo.c") +test.run(program=program, stdout=None) +test.run(program=test.workpath('foo'), stdout="foo.c") test.pass_test() diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index 6972a54aa5..91be108176 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -46,7 +46,7 @@ test.dir_fixture('ninja-fixture') -test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build.py', 'SConstruct') +test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build', 'SConstruct') # generate simple build test.run(stdout=None) diff --git a/test/ninja/generate_and_build_cxx.py b/test/ninja/generate_and_build_cxx.py index ee6e0bd8e3..481a7e51b1 100644 --- a/test/ninja/generate_and_build_cxx.py +++ b/test/ninja/generate_and_build_cxx.py @@ -23,6 +23,7 @@ # import os + import TestSCons from TestCmd import IS_WINDOWS @@ -50,7 +51,7 @@ # env.Tool('ninja') # env.Program(target = 'test2', source = 'test2.cpp') # """) -test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py','SConstruct') +test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build_cxx', 'SConstruct') # generate simple build test.run() diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index 5fd4bc5aa2..df010f4d5b 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -23,8 +23,9 @@ # import os -import time import random +import time + import TestSCons from TestCmd import IS_WINDOWS @@ -36,7 +37,7 @@ test.skip_test("Could not find module in python") _python_ = TestSCons._python_ -_exe = TestSCons._exe +_exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( ninja.__file__, @@ -68,6 +69,7 @@ print_function0(); """) + def get_num_cpus(): """ Function to get the number of CPUs the system has. @@ -89,6 +91,7 @@ def get_num_cpus(): # Default return 1 + def generate_source(parent_source, current_source): test.write('source_{}.c'.format(current_source), """ #include @@ -111,8 +114,9 @@ def generate_source(parent_source, current_source): print_function%(current_source)s(); """ % locals()) + def mod_source_return(test_num): - parent_source = test_num-1 + parent_source = test_num - 1 test.write('source_{}.c'.format(test_num), """ #include #include @@ -128,8 +132,9 @@ def mod_source_return(test_num): } """ % locals()) + def mod_source_orig(test_num): - parent_source = test_num-1 + parent_source = test_num - 1 test.write('source_{}.c'.format(test_num), """ #include #include @@ -143,9 +148,10 @@ def mod_source_orig(test_num): } """ % locals()) + num_source = 200 -for i in range(1, num_source+1): - generate_source(i-1, i) +for i in range(1, num_source + 1): + generate_source(i - 1, i) test.write('main.c', """ #include @@ -183,15 +189,15 @@ def mod_source_orig(test_num): start = time.perf_counter() test.run(arguments='--disable-execute-ninja', stdout=None) -test.run(program = ninja_program, stdout=None) +test.run(program=ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] -test.run(program = test.workpath('print_bin'), stdout="main print") +test.run(program=test.workpath('print_bin'), stdout="main print") for test_mod in tests_mods: mod_source_return(test_mod) start = time.perf_counter() - test.run(program = ninja_program, stdout=None) + test.run(program=ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] @@ -204,15 +210,15 @@ def mod_source_orig(test_num): 'Removed build.ninja']) start = time.perf_counter() -test.run(arguments = ["-f", "SConstruct_no_ninja", jobs], stdout=None) +test.run(arguments=["-f", "SConstruct_no_ninja", jobs], stdout=None) stop = time.perf_counter() scons_times += [stop - start] -test.run(program = test.workpath('print_bin'), stdout="main print") +test.run(program=test.workpath('print_bin'), stdout="main print") for test_mod in tests_mods: mod_source_return(test_mod) start = time.perf_counter() - test.run(arguments = ["-f", "SConstruct_no_ninja", jobs], stdout=None) + test.run(arguments=["-f", "SConstruct_no_ninja", jobs], stdout=None) stop = time.perf_counter() scons_times += [stop - start] diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index 53ec83b505..d14588876b 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -23,6 +23,7 @@ # import os + import TestSCons from TestCmd import IS_WINDOWS @@ -34,7 +35,7 @@ test.skip_test("Could not find module in python") _python_ = TestSCons._python_ -_exe = TestSCons._exe +_exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( ninja.__file__, @@ -60,9 +61,9 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('foo' + _exe), stdout="foo.c") -test.run(program = test.workpath('bar' + _exe), stdout="bar.c") +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('foo' + _exe), stdout="foo.c") +test.run(program=test.workpath('bar' + _exe), stdout="bar.c") # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -81,9 +82,9 @@ # run ninja independently program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('foo' + _exe), stdout="foo.c") -test.run(program = test.workpath('bar' + _exe), stdout="bar.c") +test.run(program=program, stdout=None) +test.run(program=test.workpath('foo' + _exe), stdout="foo.c") +test.run(program=test.workpath('bar' + _exe), stdout="bar.c") test.pass_test() diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build similarity index 89% rename from test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py rename to test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build index 4123fd87b0..242eb76265 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build.py +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build @@ -1,4 +1,3 @@ -# noqa: f821 DefaultEnvironment(tools=[]) env = Environment() diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx similarity index 89% rename from test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py rename to test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx index ab304fabe1..51ca6c356d 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx.py +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx @@ -1,5 +1,3 @@ -# noqa: f821 - DefaultEnvironment(tools=[]) env = Environment() env.Tool('ninja') From a368c73a472a7c4f1fa56a5256f516530ae4ba57 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 27 Mar 2021 21:27:52 -0700 Subject: [PATCH 137/163] more sider fixes --- SCons/Tool/ninja.py | 4 ++-- test/ninja/generate_and_build_cxx.py | 22 +++++++++------------- test/ninja/generate_source.py | 11 ++++++----- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/SCons/Tool/ninja.py b/SCons/Tool/ninja.py index 36daec6ae8..88b7d48f09 100644 --- a/SCons/Tool/ninja.py +++ b/SCons/Tool/ninja.py @@ -22,5 +22,5 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -# noqa: F401 -from .ninjaCommon import generate, exists + +from .ninjaCommon import generate, exists # noqa: F401 diff --git a/test/ninja/generate_and_build_cxx.py b/test/ninja/generate_and_build_cxx.py index 481a7e51b1..074a5cb9af 100644 --- a/test/ninja/generate_and_build_cxx.py +++ b/test/ninja/generate_and_build_cxx.py @@ -35,7 +35,7 @@ test.skip_test("Could not find module in python") _python_ = TestSCons._python_ -_exe = TestSCons._exe +_exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( ninja.__file__, @@ -46,19 +46,15 @@ test.dir_fixture('ninja-fixture') -# test.write('SConstruct', """ -# env = Environment() -# env.Tool('ninja') -# env.Program(target = 'test2', source = 'test2.cpp') -# """) -test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build_cxx', 'SConstruct') +test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build_cxx', + 'SConstruct') # generate simple build test.run() test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('test2' + _exe), stdout="print_function") +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('test2' + _exe), stdout="print_function") # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -74,8 +70,8 @@ # run ninja independently program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('test2' + _exe), stdout="print_function") +test.run(program=program, stdout=None) +test.run(program=test.workpath('test2' + _exe), stdout="print_function") test.write('test2.hpp', """ #include @@ -93,8 +89,8 @@ class Foo """) # generate simple build -test.run(program = program, stdout=None) -test.run(program = test.workpath('test2' + _exe), stdout="print_function2") +test.run(program=program, stdout=None) +test.run(program=test.workpath('test2' + _exe), stdout="print_function2") test.pass_test() diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index 141b19c24d..18b53f249b 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -23,6 +23,7 @@ # import os + import TestSCons from TestCmd import IS_WINDOWS @@ -34,7 +35,7 @@ test.skip_test("Could not find module in python") _python_ = TestSCons._python_ -_exe = TestSCons._exe +_exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( ninja.__file__, @@ -79,8 +80,8 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) -test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('generated_source' + _exe), stdout="generated_source.c") # clean build and ninja files test.run(arguments='-c', stdout=None) @@ -99,8 +100,8 @@ # run ninja independently program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) -test.run(program = test.workpath('generated_source' + _exe), stdout="generated_source.c") +test.run(program=program, stdout=None) +test.run(program=test.workpath('generated_source' + _exe), stdout="generated_source.c") test.pass_test() From a40215bf8bd8e4d892a68860b7cc1739b42f3392 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 27 Mar 2021 21:33:01 -0700 Subject: [PATCH 138/163] fix sider issue --- test/ninja/shell_command.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index f477505293..f0450d9cef 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -23,6 +23,7 @@ # import os + import TestSCons from TestCmd import IS_WINDOWS @@ -34,7 +35,7 @@ test.skip_test("Could not find module in python") _python_ = TestSCons._python_ -_exe = TestSCons._exe +_exe = TestSCons._exe ninja_bin = os.path.abspath(os.path.join( ninja.__file__, @@ -58,7 +59,7 @@ test.run(stdout=None) test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' %locals()) +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) test.must_match('foo.out', 'foo.c') # clean build and ninja files @@ -76,7 +77,7 @@ # run ninja independently program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin -test.run(program = program, stdout=None) +test.run(program=program, stdout=None) test.must_match('foo.out', 'foo.c') test.pass_test() From 5c871e3a747a0baa3b9793f8af374d2d9865226d Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 10 Apr 2021 16:50:43 -0700 Subject: [PATCH 139/163] Incremental checkin. Functionality restored after refactor broke it. --- SCons/Tool/ninjaCommon/Globals.py | 40 ++ SCons/Tool/ninjaCommon/NinjaState.py | 707 +++++++++++++++++++ SCons/Tool/ninjaCommon/Overrides.py | 0 SCons/Tool/ninjaCommon/Rules.py | 25 +- SCons/Tool/ninjaCommon/Util.py | 286 +++++++- SCons/Tool/ninjaCommon/__init__.py | 990 +-------------------------- 6 files changed, 1077 insertions(+), 971 deletions(-) create mode 100644 SCons/Tool/ninjaCommon/Globals.py create mode 100644 SCons/Tool/ninjaCommon/NinjaState.py create mode 100644 SCons/Tool/ninjaCommon/Overrides.py diff --git a/SCons/Tool/ninjaCommon/Globals.py b/SCons/Tool/ninjaCommon/Globals.py new file mode 100644 index 0000000000..0dc46ea840 --- /dev/null +++ b/SCons/Tool/ninjaCommon/Globals.py @@ -0,0 +1,40 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import SCons.Action + +NINJA_RULES = "__NINJA_CUSTOM_RULES" +NINJA_POOLS = "__NINJA_CUSTOM_POOLS" +NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" +NINJA_BUILD = "NINJA_BUILD" +NINJA_WHEREIS_MEMO = {} +NINJA_STAT_MEMO = {} +__NINJA_RULE_MAPPING = {} + + +# These are the types that get_command can do something with +COMMAND_TYPES = ( + SCons.Action.CommandAction, + SCons.Action.CommandGeneratorAction, +) +ninja_builder_initialized = False \ No newline at end of file diff --git a/SCons/Tool/ninjaCommon/NinjaState.py b/SCons/Tool/ninjaCommon/NinjaState.py new file mode 100644 index 0000000000..4e41934aa3 --- /dev/null +++ b/SCons/Tool/ninjaCommon/NinjaState.py @@ -0,0 +1,707 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import io +import os +import sys +from os.path import splitext +import ninja + +import SCons +from SCons.Script import COMMAND_LINE_TARGETS +from SCons.Util import is_List +import SCons.Tool.ninjaCommon.Globals +from .Globals import COMMAND_TYPES, NINJA_RULES, NINJA_POOLS, \ + NINJA_CUSTOM_HANDLERS +from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function +from .Util import get_path, alias_to_ninja_build, generate_depfile, ninja_noop, get_command, get_order_only, \ + get_outputs, get_inputs, get_dependencies, get_rule, get_command_env + + + +# pylint: disable=too-many-instance-attributes +class NinjaState: + """Maintains state of Ninja build system as it's translated from SCons.""" + + def __init__(self, env, ninja_file, writer_class): + self.env = env + self.ninja_file = ninja_file + + self.ninja_bin_path = env.get('NINJA') + if not self.ninja_bin_path: + # default to using ninja installed with python module + ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' + self.ninja_bin_path = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + ninja_bin)) + if not os.path.exists(self.ninja_bin_path): + # couldn't find it, just give the bin name and hope + # its in the path later + self.ninja_bin_path = ninja_bin + + self.writer_class = writer_class + self.__generated = False + self.translator = SConsToNinjaTranslator(env) + self.generated_suffixes = env.get("NINJA_GENERATED_SOURCE_SUFFIXES", []) + + # List of generated builds that will be written at a later stage + self.builds = dict() + + # List of targets for which we have generated a build. This + # allows us to take multiple Alias nodes as sources and to not + # fail to build if they have overlapping targets. + self.built = set() + + # SCons sets this variable to a function which knows how to do + # shell quoting on whatever platform it's run on. Here we use it + # to make the SCONS_INVOCATION variable properly quoted for things + # like CCFLAGS + escape = env.get("ESCAPE", lambda x: x) + + # if SCons was invoked from python, we expect the first arg to be the scons.py + # script, otherwise scons was invoked from the scons script + python_bin = '' + if os.path.basename(sys.argv[0]) == 'scons.py': + python_bin = escape(sys.executable) + self.variables = { + "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", + "SCONS_INVOCATION": '{} {} --disable-ninja __NINJA_NO=1 $out'.format( + python_bin, + " ".join( + [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] + ), + ), + "SCONS_INVOCATION_W_TARGETS": "{} {} --disable-ninja".format( + python_bin, " ".join([escape(arg) for arg in sys.argv]) + ), + # This must be set to a global default per: + # https://ninja-build.org/manual.html#_deps + # English Visual Studio will have the default below, + # otherwise the user can define the variable in the first environment + # that initialized ninja tool + "msvc_deps_prefix": env.get("NINJA_MSVC_DEPS_PREFIX", "Note: including file:") + } + + self.rules = { + "CMD": { + "command": "cmd /c $env$cmd $in $out" if sys.platform == "win32" else "$env$cmd $in $out", + "description": "Building $out", + "pool": "local_pool", + }, + "GENERATED_CMD": { + "command": "cmd /c $env$cmd" if sys.platform == "win32" else "$env$cmd", + "description": "Building $out", + "pool": "local_pool", + }, + # We add the deps processing variables to this below. We + # don't pipe these through cmd.exe on Windows because we + # use this to generate a compile_commands.json database + # which can't use the shell command as it's compile + # command. + "CC": { + "command": "$env$CC @$out.rsp", + "description": "Compiling $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + }, + "CXX": { + "command": "$env$CXX @$out.rsp", + "description": "Compiling $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + }, + "LINK": { + "command": "$env$LINK @$out.rsp", + "description": "Linking $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + "pool": "local_pool", + }, + # Ninja does not automatically delete the archive before + # invoking ar. The ar utility will append to an existing archive, which + # can cause duplicate symbols if the symbols moved between object files. + # Native SCons will perform this operation so we need to force ninja + # to do the same. See related for more info: + # https://jira.mongodb.org/browse/SERVER-49457 + "AR": { + "command": "{}$env$AR @$out.rsp".format( + '' if sys.platform == "win32" else "rm -f $out && " + ), + "description": "Archiving $out", + "rspfile": "$out.rsp", + "rspfile_content": "$rspc", + "pool": "local_pool", + }, + "SYMLINK": { + "command": ( + "cmd /c mklink $out $in" + if sys.platform == "win32" + else "ln -s $in $out" + ), + "description": "Symlink $in -> $out", + }, + "INSTALL": { + "command": "$COPY $in $out", + "description": "Install $out", + "pool": "install_pool", + # On Windows cmd.exe /c copy does not always correctly + # update the timestamp on the output file. This leads + # to a stuck constant timestamp in the Ninja database + # and needless rebuilds. + # + # Adding restat here ensures that Ninja always checks + # the copy updated the timestamp and that Ninja has + # the correct information. + "restat": 1, + }, + "TEMPLATE": { + "command": "$SCONS_INVOCATION $out", + "description": "Rendering $SCONS_INVOCATION $out", + "pool": "scons_pool", + "restat": 1, + }, + "SCONS": { + "command": "$SCONS_INVOCATION $out", + "description": "$SCONS_INVOCATION $out", + "pool": "scons_pool", + # restat + # if present, causes Ninja to re-stat the command's outputs + # after execution of the command. Each output whose + # modification time the command did not change will be + # treated as though it had never needed to be built. This + # may cause the output's reverse dependencies to be removed + # from the list of pending build actions. + # + # We use restat any time we execute SCons because + # SCons calls in Ninja typically create multiple + # targets. But since SCons is doing it's own up to + # date-ness checks it may only update say one of + # them. Restat will find out which of the multiple + # build targets did actually change then only rebuild + # those targets which depend specifically on that + # output. + "restat": 1, + }, + "REGENERATE": { + "command": "$SCONS_INVOCATION_W_TARGETS", + "description": "Regenerating $out", + "generator": 1, + "depfile": os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile'), + # Console pool restricts to 1 job running at a time, + # it additionally has some special handling about + # passing stdin, stdout, etc to process in this pool + # that we need for SCons to behave correctly when + # regenerating Ninja + "pool": "console", + # Again we restat in case Ninja thought the + # build.ninja should be regenerated but SCons knew + # better. + "restat": 1, + }, + } + + if env['PLATFORM'] == 'darwin' and env['AR'] == 'ar': + self.rules["AR"] = { + "command": "rm -f $out && $env$AR $rspc", + "description": "Archiving $out", + "pool": "local_pool", + } + + self.pools = { + "local_pool": self.env.GetOption("num_jobs"), + "install_pool": self.env.GetOption("num_jobs") / 2, + "scons_pool": 1, + } + + for rule in ["CC", "CXX"]: + if env["PLATFORM"] == "win32": + self.rules[rule]["deps"] = "msvc" + else: + self.rules[rule]["deps"] = "gcc" + self.rules[rule]["depfile"] = "$out.d" + + def add_build(self, node): + if not node.has_builder(): + return False + + if isinstance(node, SCons.Node.Alias.Alias): + build = alias_to_ninja_build(node) + else: + build = self.translator.action_to_ninja_build(node) + + # Some things are unbuild-able or need not be built in Ninja + if build is None: + return False + + node_string = str(node) + if node_string in self.builds: + raise Exception("Node {} added to ninja build state more than once".format(node_string)) + self.builds[node_string] = build + self.built.update(build["outputs"]) + return True + + # TODO: rely on SCons to tell us what is generated source + # or some form of user scanner maybe (Github Issue #3624) + def is_generated_source(self, output): + """Check if output ends with a known generated suffix.""" + _, suffix = splitext(output) + return suffix in self.generated_suffixes + + def has_generated_sources(self, output): + """ + Determine if output indicates this is a generated header file. + """ + for generated in output: + if self.is_generated_source(generated): + return True + return False + + # pylint: disable=too-many-branches,too-many-locals + def generate(self): + """ + Generate the build.ninja. + + This should only be called once for the lifetime of this object. + """ + if self.__generated: + return + + self.rules.update(self.env.get(NINJA_RULES, {})) + self.pools.update(self.env.get(NINJA_POOLS, {})) + + content = io.StringIO() + ninja = self.writer_class(content, width=100) + + ninja.comment("Generated by scons. DO NOT EDIT.") + + ninja.variable("builddir", get_path(self.env['NINJA_BUILDDIR'])) + + for pool_name, size in self.pools.items(): + ninja.pool(pool_name, size) + + for var, val in self.variables.items(): + ninja.variable(var, val) + + for rule, kwargs in self.rules.items(): + ninja.rule(rule, **kwargs) + + generated_source_files = sorted({ + output + # First find builds which have header files in their outputs. + for build in self.builds.values() + if self.has_generated_sources(build["outputs"]) + for output in build["outputs"] + # Collect only the header files from the builds with them + # in their output. We do this because is_generated_source + # returns True if it finds a header in any of the outputs, + # here we need to filter so we only have the headers and + # not the other outputs. + if self.is_generated_source(output) + }) + + if generated_source_files: + ninja.build( + outputs="_generated_sources", + rule="phony", + implicit=generated_source_files + ) + + template_builders = [] + + for build in [self.builds[key] for key in sorted(self.builds.keys())]: + if build["rule"] == "TEMPLATE": + template_builders.append(build) + continue + + if "implicit" in build: + build["implicit"].sort() + + # Don't make generated sources depend on each other. We + # have to check that none of the outputs are generated + # sources and none of the direct implicit dependencies are + # generated sources or else we will create a dependency + # cycle. + if ( + generated_source_files + and not build["rule"] == "INSTALL" + and set(build["outputs"]).isdisjoint(generated_source_files) + and set(build.get("implicit", [])).isdisjoint(generated_source_files) + ): + # Make all non-generated source targets depend on + # _generated_sources. We use order_only for generated + # sources so that we don't rebuild the world if one + # generated source was rebuilt. We just need to make + # sure that all of these sources are generated before + # other builds. + order_only = build.get("order_only", []) + order_only.append("_generated_sources") + build["order_only"] = order_only + if "order_only" in build: + build["order_only"].sort() + + # When using a depfile Ninja can only have a single output + # but SCons will usually have emitted an output for every + # thing a command will create because it's caching is much + # more complex than Ninja's. This includes things like DWO + # files. Here we make sure that Ninja only ever sees one + # target when using a depfile. It will still have a command + # that will create all of the outputs but most targets don't + # depend directly on DWO files and so this assumption is safe + # to make. + rule = self.rules.get(build["rule"]) + + # Some rules like 'phony' and other builtins we don't have + # listed in self.rules so verify that we got a result + # before trying to check if it has a deps key. + # + # Anything using deps or rspfile in Ninja can only have a single + # output, but we may have a build which actually produces + # multiple outputs which other targets can depend on. Here we + # slice up the outputs so we have a single output which we will + # use for the "real" builder and multiple phony targets that + # match the file names of the remaining outputs. This way any + # build can depend on any output from any build. + # + # We assume that the first listed output is the 'key' + # output and is stably presented to us by SCons. For + # instance if -gsplit-dwarf is in play and we are + # producing foo.o and foo.dwo, we expect that outputs[0] + # from SCons will be the foo.o file and not the dwo + # file. If instead we just sorted the whole outputs array, + # we would find that the dwo file becomes the + # first_output, and this breaks, for instance, header + # dependency scanning. + if rule is not None and (rule.get("deps") or rule.get("rspfile")): + first_output, remaining_outputs = ( + build["outputs"][0], + build["outputs"][1:], + ) + + if remaining_outputs: + ninja.build( + outputs=sorted(remaining_outputs), rule="phony", implicit=first_output, + ) + + build["outputs"] = first_output + + # Optionally a rule can specify a depfile, and SCons can generate implicit + # dependencies into the depfile. This allows for dependencies to come and go + # without invalidating the ninja file. The depfile was created in ninja specifically + # for dealing with header files appearing and disappearing across rebuilds, but it can + # be repurposed for anything, as long as you have a way to regenerate the depfile. + # More specific info can be found here: https://ninja-build.org/manual.html#_depfile + if rule is not None and rule.get('depfile') and build.get('deps_files'): + path = build['outputs'] if SCons.Util.is_List(build['outputs']) else [build['outputs']] + generate_depfile(self.env, path[0], build.pop('deps_files', [])) + + if "inputs" in build: + build["inputs"].sort() + + ninja.build(**build) + + template_builds = dict() + for template_builder in template_builders: + + # Special handling for outputs and implicit since we need to + # aggregate not replace for each builder. + for agg_key in ["outputs", "implicit", "inputs"]: + new_val = template_builds.get(agg_key, []) + + # Use pop so the key is removed and so the update + # below will not overwrite our aggregated values. + cur_val = template_builder.pop(agg_key, []) + if is_List(cur_val): + new_val += cur_val + else: + new_val.append(cur_val) + template_builds[agg_key] = new_val + + # Collect all other keys + template_builds.update(template_builder) + + if template_builds.get("outputs", []): + ninja.build(**template_builds) + + # We have to glob the SCons files here to teach the ninja file + # how to regenerate itself. We'll never see ourselves in the + # DAG walk so we can't rely on action_to_ninja_build to + # generate this rule even though SCons should know we're + # dependent on SCons files. + # + # The REGENERATE rule uses depfile, so we need to generate the depfile + # in case any of the SConscripts have changed. The depfile needs to be + # path with in the build and the passed ninja file is an abspath, so + # we will use SCons to give us the path within the build. Normally + # generate_depfile should not be called like this, but instead be called + # through the use of custom rules, and filtered out in the normal + # list of build generation about. However, because the generate rule + # is hardcoded here, we need to do this generate_depfile call manually. + ninja_file_path = self.env.File(self.ninja_file).path + generate_depfile( + self.env, + ninja_file_path, + self.env['NINJA_REGENERATE_DEPS'] + ) + + ninja.build( + ninja_file_path, + rule="REGENERATE", + implicit=[__file__], + ) + + # If we ever change the name/s of the rules that include + # compile commands (i.e. something like CC) we will need to + # update this build to reflect that complete list. + ninja.build( + "compile_commands.json", + rule="CMD", + pool="console", + implicit=[str(self.ninja_file)], + variables={ + "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( + # NINJA_COMPDB_EXPAND - should only be true for ninja + # This was added to ninja's compdb tool in version 1.9.0 (merged April 2018) + # https://github.com/ninja-build/ninja/pull/1223 + # TODO: add check in generate to check version and enable this by default if it's available. + self.ninja_bin_path, str(self.ninja_file), + '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' + ) + }, + ) + + ninja.build( + "compiledb", rule="phony", implicit=["compile_commands.json"], + ) + + # Look in SCons's list of DEFAULT_TARGETS, find the ones that + # we generated a ninja build rule for. + scons_default_targets = [ + get_path(tgt) + for tgt in SCons.Script.DEFAULT_TARGETS + if get_path(tgt) in self.built + ] + + # If we found an overlap between SCons's list of default + # targets and the targets we created ninja builds for then use + # those as ninja's default as well. + if scons_default_targets: + ninja.default(" ".join(scons_default_targets)) + + with open(str(self.ninja_file), "w") as build_ninja: + build_ninja.write(content.getvalue()) + + self.__generated = True + + +class SConsToNinjaTranslator: + """Translates SCons Actions into Ninja build objects.""" + + def __init__(self, env): + self.env = env + self.func_handlers = { + # Skip conftest builders + "_createSource": ninja_noop, + # SCons has a custom FunctionAction that just makes sure the + # target isn't static. We let the commands that ninja runs do + # this check for us. + "SharedFlagChecker": ninja_noop, + # The install builder is implemented as a function action. + # TODO: use command action #3573 + "installFunc": _install_action_function, + "MkdirFunc": _mkdir_action_function, + "LibSymlinksActionFunction": _lib_symlink_action_function, + "Copy": _copy_action_function + } + + self.loaded_custom = False + + # pylint: disable=too-many-return-statements + def action_to_ninja_build(self, node, action=None): + """Generate build arguments dictionary for node.""" + + if not self.loaded_custom: + self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) + self.loaded_custom = True + + if node.builder is None: + return None + + if action is None: + action = node.builder.action + + if node.env and node.env.get("NINJA_SKIP"): + return None + + build = {} + env = node.env if node.env else self.env + + # Ideally this should never happen, and we do try to filter + # Ninja builders out of being sources of ninja builders but I + # can't fix every DAG problem so we just skip ninja_builders + # if we find one + if SCons.Tool.ninjaCommon.NINJA_STATE.ninja_file == str(node): + build = None + elif isinstance(action, SCons.Action.FunctionAction): + build = self.handle_func_action(node, action) + elif isinstance(action, SCons.Action.LazyAction): + # pylint: disable=protected-access + action = action._generate_cache(env) + build = self.action_to_ninja_build(node, action=action) + elif isinstance(action, SCons.Action.ListAction): + build = self.handle_list_action(node, action) + elif isinstance(action, COMMAND_TYPES): + build = get_command(env, node, action) + else: + raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) + + if build is not None: + build["order_only"] = get_order_only(node) + + # TODO: WPD Is this testing the filename to verify it's a configure context generated file? + if not node.is_conftest(): + node_callback = node.check_attributes("ninja_build_callback") + if callable(node_callback): + node_callback(env, node, build) + + return build + + def handle_func_action(self, node, action): + """Determine how to handle the function action.""" + name = action.function_name() + # This is the name given by the Subst/Textfile builders. So return the + # node to indicate that SCons is required. We skip sources here because + # dependencies don't really matter when we're going to shove these to + # the bottom of ninja's DAG anyway and Textfile builders can have text + # content as their source which doesn't work as an implicit dep in + # ninja. + if name == 'ninja_builder': + return None + + handler = self.func_handlers.get(name, None) + if handler is not None: + return handler(node.env if node.env else self.env, node) + elif name == "ActionCaller": + action_to_call = str(action).split('(')[0].strip() + handler = self.func_handlers.get(action_to_call, None) + if handler is not None: + return handler(node.env if node.env else self.env, node) + + SCons.Warnings.SConsWarning( + "Found unhandled function action {}, " + " generating scons command to build\n" + "Note: this is less efficient than Ninja," + " you can write your own ninja build generator for" + " this function using NinjaRegisterFunctionHandler".format(name) + ) + + return { + "rule": "TEMPLATE", + "order_only": get_order_only(node), + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "implicit": get_dependencies(node, skip_sources=True), + } + + # pylint: disable=too-many-branches + def handle_list_action(self, node, action): + """TODO write this comment""" + results = [ + self.action_to_ninja_build(node, action=act) + for act in action.list + if act is not None + ] + results = [ + result for result in results if result is not None and result["outputs"] + ] + if not results: + return None + + # No need to process the results if we only got a single result + if len(results) == 1: + return results[0] + + all_outputs = list({output for build in results for output in build["outputs"]}) + dependencies = list({dep for build in results for dep in build["implicit"]}) + + if results[0]["rule"] == "CMD" or results[0]["rule"] == "GENERATED_CMD": + cmdline = "" + for cmd in results: + + # Occasionally a command line will expand to a + # whitespace only string (i.e. ' '). Which is not a + # valid command but does not trigger the empty command + # condition if not cmdstr. So here we strip preceding + # and proceeding whitespace to make strings like the + # above become empty strings and so will be skipped. + cmdstr = cmd["variables"]["cmd"].strip() + if not cmdstr: + continue + + # Skip duplicate commands + if cmdstr in cmdline: + continue + + if cmdline: + cmdline += " && " + + cmdline += cmdstr + + # Remove all preceding and proceeding whitespace + cmdline = cmdline.strip() + + # Make sure we didn't generate an empty cmdline + if cmdline: + ninja_build = { + "outputs": all_outputs, + "rule": get_rule(node, "GENERATED_CMD"), + "variables": { + "cmd": cmdline, + "env": get_command_env(node.env if node.env else self.env), + }, + "implicit": dependencies, + } + + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["pool"] + + return ninja_build + + elif results[0]["rule"] == "phony": + return { + "outputs": all_outputs, + "rule": "phony", + "implicit": dependencies, + } + + elif results[0]["rule"] == "INSTALL": + return { + "outputs": all_outputs, + "rule": get_rule(node, "INSTALL"), + "inputs": get_inputs(node), + "implicit": dependencies, + } + + raise Exception("Unhandled list action with rule: " + results[0]["rule"]) diff --git a/SCons/Tool/ninjaCommon/Overrides.py b/SCons/Tool/ninjaCommon/Overrides.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/SCons/Tool/ninjaCommon/Rules.py b/SCons/Tool/ninjaCommon/Rules.py index 99b11e4c8e..c1c238e1d7 100644 --- a/SCons/Tool/ninjaCommon/Rules.py +++ b/SCons/Tool/ninjaCommon/Rules.py @@ -1,3 +1,26 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + from .Util import get_outputs, get_rule, get_inputs, get_dependencies @@ -55,4 +78,4 @@ def _lib_symlink_action_function(_env, node): "inputs": inputs, "rule": get_rule(node, "SYMLINK"), "implicit": get_dependencies(node), - } \ No newline at end of file + } diff --git a/SCons/Tool/ninjaCommon/Util.py b/SCons/Tool/ninjaCommon/Util.py index 5f14a96060..bb20d80a3f 100644 --- a/SCons/Tool/ninjaCommon/Util.py +++ b/SCons/Tool/ninjaCommon/Util.py @@ -1,5 +1,34 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +import os +from os.path import join as joinpath + import SCons +from SCons.Action import get_default_ENV, _string_from_cmd_list from SCons.Script import AddOption +from SCons.Tool.ninjaCommon import __NINJA_RULE_MAPPING +from SCons.Util import is_List, flatten_sequence + def ninja_add_command_line_options(): """ @@ -170,4 +199,259 @@ def get_rule(node, rule): if invalid_ninja_nodes(node, tlist): return "TEMPLATE" else: - return rule \ No newline at end of file + return rule + + +def generate_depfile(env, node, dependencies): + """ + Ninja tool function for writing a depfile. The depfile should include + the node path followed by all the dependent files in a makefile format. + + dependencies arg can be a list or a subst generator which returns a list. + """ + + depfile = os.path.join(get_path(env['NINJA_BUILDDIR']), str(node) + '.depfile') + + # subst_list will take in either a raw list or a subst callable which generates + # a list, and return a list of CmdStringHolders which can be converted into raw strings. + # If a raw list was passed in, then scons_list will make a list of lists from the original + # values and even subst items in the list if they are substitutable. Flatten will flatten + # the list in that case, to ensure for either input we have a list of CmdStringHolders. + deps_list = env.Flatten(env.subst_list(dependencies)) + + # Now that we have the deps in a list as CmdStringHolders, we can convert them into raw strings + # and make sure to escape the strings to handle spaces in paths. We also will sort the result + # keep the order of the list consistent. + escaped_depends = sorted([dep.escape(env.get("ESCAPE", lambda x: x)) for dep in deps_list]) + depfile_contents = str(node) + ": " + ' '.join(escaped_depends) + + need_rewrite = False + try: + with open(depfile, 'r') as f: + need_rewrite = (f.read() != depfile_contents) + except FileNotFoundError: + need_rewrite = True + + if need_rewrite: + os.makedirs(os.path.dirname(depfile) or '.', exist_ok=True) + with open(depfile, 'w') as f: + f.write(depfile_contents) + + +def ninja_noop(*_args, **_kwargs): + """ + A general purpose no-op function. + + There are many things that happen in SCons that we don't need and + also don't return anything. We use this to disable those functions + instead of creating multiple definitions of the same thing. + """ + return None + + +def get_command(env, node, action): # pylint: disable=too-many-branches + """Get the command to execute for node.""" + if node.env: + sub_env = node.env + else: + sub_env = env + executor = node.get_executor() + tlist, slist = get_targets_sources(node) + + # Generate a real CommandAction + if isinstance(action, SCons.Action.CommandGeneratorAction): + # pylint: disable=protected-access + action = action._generate(tlist, slist, sub_env, 1, executor=executor) + + variables = {} + + comstr = get_comstr(sub_env, action, tlist, slist) + if not comstr: + return None + + provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) + rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) + + # Get the dependencies for all targets + implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) + + # Now add in the other dependencies related to the command, + # e.g. the compiler binary. The ninja rule can be user provided so + # we must do some validation to resolve the dependency path for ninja. + for provider_dep in provider_deps: + + provider_dep = sub_env.subst(provider_dep) + if not provider_dep: + continue + + # If the tool is a node, then SCons will resolve the path later, if its not + # a node then we assume it generated from build and make sure it is existing. + if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep): + implicit.append(provider_dep) + continue + + # in some case the tool could be in the local directory and be suppled without the ext + # such as in windows, so append the executable suffix and check. + prog_suffix = sub_env.get('PROGSUFFIX', '') + provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix + if os.path.exists(provider_dep_ext): + implicit.append(provider_dep_ext) + continue + + # Many commands will assume the binary is in the path, so + # we accept this as a possible input from a given command. + + provider_dep_abspath = sub_env.WhereIs(provider_dep) or sub_env.WhereIs(provider_dep, path=os.environ["PATH"]) + if provider_dep_abspath: + implicit.append(provider_dep_abspath) + continue + + # Possibly these could be ignore and the build would still work, however it may not always + # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided + # ninja rule. + raise Exception("Could not resolve path for %s dependency on node '%s'" % (provider_dep, node)) + + ninja_build = { + "order_only": get_order_only(node), + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "implicit": implicit, + "rule": get_rule(node, rule), + "variables": variables, + } + + # Don't use sub_env here because we require that NINJA_POOL be set + # on a per-builder call basis to prevent accidental strange + # behavior like env['NINJA_POOL'] = 'console' and sub_env can be + # the global Environment object if node.env is None. + # Example: + # + # Allowed: + # + # env.Command("ls", NINJA_POOL="ls_pool") + # + # Not allowed and ignored: + # + # env["NINJA_POOL"] = "ls_pool" + # env.Command("ls") + # + # TODO: Why not alloe env['NINJA_POOL'] ? (bdbaddog) + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["NINJA_POOL"] + + return ninja_build + + +def get_command_env(env): + """ + Return a string that sets the environment for any environment variables that + differ between the OS environment and the SCons command ENV. + + It will be compatible with the default shell of the operating system. + """ + try: + return env["NINJA_ENV_VAR_CACHE"] + except KeyError: + pass + + # Scan the ENV looking for any keys which do not exist in + # os.environ or differ from it. We assume if it's a new or + # differing key from the process environment then it's + # important to pass down to commands in the Ninja file. + ENV = get_default_ENV(env) + scons_specified_env = { + key: value + for key, value in ENV.items() + # TODO: Remove this filter, unless there's a good reason to keep. SCons's behavior shouldn't depend on shell's. + if key not in os.environ or os.environ.get(key, None) != value + } + + windows = env["PLATFORM"] == "win32" + command_env = "" + for key, value in scons_specified_env.items(): + # Ensure that the ENV values are all strings: + if is_List(value): + # If the value is a list, then we assume it is a + # path list, because that's a pretty common list-like + # value to stick in an environment variable: + value = flatten_sequence(value) + value = joinpath(map(str, value)) + else: + # If it isn't a string or a list, then we just coerce + # it to a string, which is the proper way to handle + # Dir and File instances and will produce something + # reasonable for just about everything else: + value = str(value) + + if windows: + command_env += "set '{}={}' && ".format(key, value) + else: + # We address here *only* the specific case that a user might have + # an environment variable which somehow gets included and has + # spaces in the value. These are escapes that Ninja handles. This + # doesn't make builds on paths with spaces (Ninja and SCons issues) + # nor expanding response file paths with spaces (Ninja issue) work. + value = value.replace(r' ', r'$ ') + command_env += "export {}='{}';".format(key, value) + + env["NINJA_ENV_VAR_CACHE"] = command_env + return command_env + + +def get_comstr(env, action, targets, sources): + """Get the un-substituted string for action.""" + # Despite being having "list" in it's name this member is not + # actually a list. It's the pre-subst'd string of the command. We + # use it to determine if the command we're about to generate needs + # to use a custom Ninja rule. By default this redirects CC, CXX, + # AR, SHLINK, and LINK commands to their respective rules but the + # user can inject custom Ninja rules and tie them to commands by + # using their pre-subst'd string. + if hasattr(action, "process"): + return action.cmd_list + + return action.genstring(targets, sources, env) + + +def get_generic_shell_command(env, node, action, targets, sources, executor=None): + return ( + "CMD", + { + # TODO: Why is executor passed in and then ignored below? (bdbaddog) + "cmd": generate_command(env, node, action, targets, sources, executor=None), + "env": get_command_env(env), + }, + # Since this function is a rule mapping provider, it must return a list of dependencies, + # and usually this would be the path to a tool, such as a compiler, used for this rule. + # However this function is to generic to be able to reliably extract such deps + # from the command, so we return a placeholder empty list. It should be noted that + # generally this function will not be used solely and is more like a template to generate + # the basics for a custom provider which may have more specific options for a provider + # function for a custom NinjaRuleMapping. + [] + ) + + +def generate_command(env, node, action, targets, sources, executor=None): + # Actions like CommandAction have a method called process that is + # used by SCons to generate the cmd_line they need to run. So + # check if it's a thing like CommandAction and call it if we can. + if hasattr(action, "process"): + cmd_list, _, _ = action.process(targets, sources, env, executor=executor) + cmd = _string_from_cmd_list(cmd_list[0]) + else: + # Anything else works with genstring, this is most commonly hit by + # ListActions which essentially call process on all of their + # commands and concatenate it for us. + genstring = action.genstring(targets, sources, env) + if executor is not None: + cmd = env.subst(genstring, executor=executor) + else: + cmd = env.subst(genstring, targets, sources) + + cmd = cmd.replace("\n", " && ").strip() + if cmd.endswith("&&"): + cmd = cmd[0:-2].strip() + + # Escape dollars as necessary + return cmd.replace("$", "$$") \ No newline at end of file diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index 0af6c21b3e..a2745c5827 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -25,7 +25,6 @@ """Generate build.ninja files from SCons aliases.""" import importlib -import io import os import shlex import shutil @@ -33,803 +32,22 @@ import sys import textwrap from glob import glob -from os.path import join as joinpath -from os.path import splitext import SCons -from SCons.Action import _string_from_cmd_list, get_default_ENV -# from SCons.Node import SConscriptNodes +import SCons.Tool.ninjaCommon.Globals +from SCons.Action import _string_from_cmd_list from SCons.Script import GetOption -from SCons.Script import COMMAND_LINE_TARGETS -from SCons.Util import is_List, flatten_sequence +from SCons.Util import is_List +import SCons.Tool.ninjaCommon.Globals + +from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS, __NINJA_RULE_MAPPING +from .NinjaState import NinjaState from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function from .Util import ninja_add_command_line_options, alias_to_ninja_build, \ - get_order_only, get_dependencies, get_inputs, get_outputs, \ - get_targets_sources, get_path, get_rule + get_targets_sources, get_path, ninja_noop, get_command, get_command_env, get_comstr, get_generic_shell_command, \ + generate_command NINJA_STATE = None -NINJA_RULES = "__NINJA_CUSTOM_RULES" -NINJA_POOLS = "__NINJA_CUSTOM_POOLS" -NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" -NINJA_BUILD = "NINJA_BUILD" -NINJA_WHEREIS_MEMO = {} -NINJA_STAT_MEMO = {} - -__NINJA_RULE_MAPPING = {} - -# These are the types that get_command can do something with -COMMAND_TYPES = ( - SCons.Action.CommandAction, - SCons.Action.CommandGeneratorAction, -) - -ninja_builder_initialized = False - - -def generate_depfile(env, node, dependencies): - """ - Ninja tool function for writing a depfile. The depfile should include - the node path followed by all the dependent files in a makefile format. - - dependencies arg can be a list or a subst generator which returns a list. - """ - - depfile = os.path.join(get_path(env['NINJA_BUILDDIR']), str(node) + '.depfile') - - # subst_list will take in either a raw list or a subst callable which generates - # a list, and return a list of CmdStringHolders which can be converted into raw strings. - # If a raw list was passed in, then scons_list will make a list of lists from the original - # values and even subst items in the list if they are substitutable. Flatten will flatten - # the list in that case, to ensure for either input we have a list of CmdStringHolders. - deps_list = env.Flatten(env.subst_list(dependencies)) - - # Now that we have the deps in a list as CmdStringHolders, we can convert them into raw strings - # and make sure to escape the strings to handle spaces in paths. We also will sort the result - # keep the order of the list consistent. - escaped_depends = sorted([dep.escape(env.get("ESCAPE", lambda x: x)) for dep in deps_list]) - depfile_contents = str(node) + ": " + ' '.join(escaped_depends) - - need_rewrite = False - try: - with open(depfile, 'r') as f: - need_rewrite = (f.read() != depfile_contents) - except FileNotFoundError: - need_rewrite = True - - if need_rewrite: - os.makedirs(os.path.dirname(depfile) or '.', exist_ok=True) - with open(depfile, 'w') as f: - f.write(depfile_contents) - - - - - -class SConsToNinjaTranslator: - """Translates SCons Actions into Ninja build objects.""" - - def __init__(self, env): - self.env = env - self.func_handlers = { - # Skip conftest builders - "_createSource": ninja_noop, - # SCons has a custom FunctionAction that just makes sure the - # target isn't static. We let the commands that ninja runs do - # this check for us. - "SharedFlagChecker": ninja_noop, - # The install builder is implemented as a function action. - # TODO: use command action #3573 - "installFunc": _install_action_function, - "MkdirFunc": _mkdir_action_function, - "LibSymlinksActionFunction": _lib_symlink_action_function, - "Copy" : _copy_action_function - } - - self.loaded_custom = False - - # pylint: disable=too-many-return-statements - def action_to_ninja_build(self, node, action=None): - """Generate build arguments dictionary for node.""" - if not self.loaded_custom: - self.func_handlers.update(self.env[NINJA_CUSTOM_HANDLERS]) - self.loaded_custom = True - - if node.builder is None: - return None - - if action is None: - action = node.builder.action - - if node.env and node.env.get("NINJA_SKIP"): - return None - - build = {} - env = node.env if node.env else self.env - - # Ideally this should never happen, and we do try to filter - # Ninja builders out of being sources of ninja builders but I - # can't fix every DAG problem so we just skip ninja_builders - # if we find one - global NINJA_STATE - if NINJA_STATE.ninja_file == str(node): - build = None - elif isinstance(action, SCons.Action.FunctionAction): - build = self.handle_func_action(node, action) - elif isinstance(action, SCons.Action.LazyAction): - # pylint: disable=protected-access - action = action._generate_cache(env) - build = self.action_to_ninja_build(node, action=action) - elif isinstance(action, SCons.Action.ListAction): - build = self.handle_list_action(node, action) - elif isinstance(action, COMMAND_TYPES): - build = get_command(env, node, action) - else: - raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) - - if build is not None: - build["order_only"] = get_order_only(node) - - #TODO: WPD Is this testing the filename to verify it's a configure context generated file? - if not node.is_conftest(): - node_callback = node.check_attributes("ninja_build_callback") - if callable(node_callback): - node_callback(env, node, build) - - return build - - def handle_func_action(self, node, action): - """Determine how to handle the function action.""" - name = action.function_name() - # This is the name given by the Subst/Textfile builders. So return the - # node to indicate that SCons is required. We skip sources here because - # dependencies don't really matter when we're going to shove these to - # the bottom of ninja's DAG anyway and Textfile builders can have text - # content as their source which doesn't work as an implicit dep in - # ninja. - if name == 'ninja_builder': - return None - - handler = self.func_handlers.get(name, None) - if handler is not None: - return handler(node.env if node.env else self.env, node) - elif name == "ActionCaller": - action_to_call = str(action).split('(')[0].strip() - handler = self.func_handlers.get(action_to_call, None) - if handler is not None: - return handler(node.env if node.env else self.env, node) - - SCons.Warnings.SConsWarning( - "Found unhandled function action {}, " - " generating scons command to build\n" - "Note: this is less efficient than Ninja," - " you can write your own ninja build generator for" - " this function using NinjaRegisterFunctionHandler".format(name) - ) - - return { - "rule": "TEMPLATE", - "order_only": get_order_only(node), - "outputs": get_outputs(node), - "inputs": get_inputs(node), - "implicit": get_dependencies(node, skip_sources=True), - } - - # pylint: disable=too-many-branches - def handle_list_action(self, node, action): - """TODO write this comment""" - results = [ - self.action_to_ninja_build(node, action=act) - for act in action.list - if act is not None - ] - results = [ - result for result in results if result is not None and result["outputs"] - ] - if not results: - return None - - # No need to process the results if we only got a single result - if len(results) == 1: - return results[0] - - all_outputs = list({output for build in results for output in build["outputs"]}) - dependencies = list({dep for build in results for dep in build["implicit"]}) - - if results[0]["rule"] == "CMD" or results[0]["rule"] == "GENERATED_CMD": - cmdline = "" - for cmd in results: - - # Occasionally a command line will expand to a - # whitespace only string (i.e. ' '). Which is not a - # valid command but does not trigger the empty command - # condition if not cmdstr. So here we strip preceding - # and proceeding whitespace to make strings like the - # above become empty strings and so will be skipped. - cmdstr = cmd["variables"]["cmd"].strip() - if not cmdstr: - continue - - # Skip duplicate commands - if cmdstr in cmdline: - continue - - if cmdline: - cmdline += " && " - - cmdline += cmdstr - - # Remove all preceding and proceeding whitespace - cmdline = cmdline.strip() - - # Make sure we didn't generate an empty cmdline - if cmdline: - ninja_build = { - "outputs": all_outputs, - "rule": get_rule(node, "GENERATED_CMD"), - "variables": { - "cmd": cmdline, - "env": get_command_env(node.env if node.env else self.env), - }, - "implicit": dependencies, - } - - if node.env and node.env.get("NINJA_POOL", None) is not None: - ninja_build["pool"] = node.env["pool"] - - return ninja_build - - elif results[0]["rule"] == "phony": - return { - "outputs": all_outputs, - "rule": "phony", - "implicit": dependencies, - } - - elif results[0]["rule"] == "INSTALL": - return { - "outputs": all_outputs, - "rule": get_rule(node, "INSTALL"), - "inputs": get_inputs(node), - "implicit": dependencies, - } - - raise Exception("Unhandled list action with rule: " + results[0]["rule"]) - - -# pylint: disable=too-many-instance-attributes -class NinjaState: - """Maintains state of Ninja build system as it's translated from SCons.""" - - def __init__(self, env, ninja_file, writer_class): - self.env = env - self.ninja_file = ninja_file - self.ninja_bin_path = '' - self.writer_class = writer_class - self.__generated = False - self.translator = SConsToNinjaTranslator(env) - self.generated_suffixes = env.get("NINJA_GENERATED_SOURCE_SUFFIXES", []) - - # List of generated builds that will be written at a later stage - self.builds = dict() - - # List of targets for which we have generated a build. This - # allows us to take multiple Alias nodes as sources and to not - # fail to build if they have overlapping targets. - self.built = set() - - # SCons sets this variable to a function which knows how to do - # shell quoting on whatever platform it's run on. Here we use it - # to make the SCONS_INVOCATION variable properly quoted for things - # like CCFLAGS - escape = env.get("ESCAPE", lambda x: x) - - # if SCons was invoked from python, we expect the first arg to be the scons.py - # script, otherwise scons was invoked from the scons script - python_bin = '' - if os.path.basename(sys.argv[0]) == 'scons.py': - python_bin = escape(sys.executable) - self.variables = { - "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", - "SCONS_INVOCATION": '{} {} --disable-ninja __NINJA_NO=1 $out'.format( - python_bin, - " ".join( - [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] - ), - ), - "SCONS_INVOCATION_W_TARGETS": "{} {} --disable-ninja".format( - python_bin, " ".join([escape(arg) for arg in sys.argv]) - ), - # This must be set to a global default per: - # https://ninja-build.org/manual.html#_deps - # English Visual Studio will have the default below, - # otherwise the user can define the variable in the first environment - # that initialized ninja tool - "msvc_deps_prefix": env.get("NINJA_MSVC_DEPS_PREFIX", "Note: including file:") - } - - self.rules = { - "CMD": { - "command": "cmd /c $env$cmd $in $out" if sys.platform == "win32" else "$env$cmd $in $out", - "description": "Building $out", - "pool": "local_pool", - }, - "GENERATED_CMD": { - "command": "cmd /c $env$cmd" if sys.platform == "win32" else "$env$cmd", - "description": "Building $out", - "pool": "local_pool", - }, - # We add the deps processing variables to this below. We - # don't pipe these through cmd.exe on Windows because we - # use this to generate a compile_commands.json database - # which can't use the shell command as it's compile - # command. - "CC": { - "command": "$env$CC @$out.rsp", - "description": "Compiling $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - }, - "CXX": { - "command": "$env$CXX @$out.rsp", - "description": "Compiling $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - }, - "LINK": { - "command": "$env$LINK @$out.rsp", - "description": "Linking $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - "pool": "local_pool", - }, - # Ninja does not automatically delete the archive before - # invoking ar. The ar utility will append to an existing archive, which - # can cause duplicate symbols if the symbols moved between object files. - # Native SCons will perform this operation so we need to force ninja - # to do the same. See related for more info: - # https://jira.mongodb.org/browse/SERVER-49457 - "AR": { - "command": "{}$env$AR @$out.rsp".format( - '' if sys.platform == "win32" else "rm -f $out && " - ), - "description": "Archiving $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - "pool": "local_pool", - }, - "SYMLINK": { - "command": ( - "cmd /c mklink $out $in" - if sys.platform == "win32" - else "ln -s $in $out" - ), - "description": "Symlink $in -> $out", - }, - "INSTALL": { - "command": "$COPY $in $out", - "description": "Install $out", - "pool": "install_pool", - # On Windows cmd.exe /c copy does not always correctly - # update the timestamp on the output file. This leads - # to a stuck constant timestamp in the Ninja database - # and needless rebuilds. - # - # Adding restat here ensures that Ninja always checks - # the copy updated the timestamp and that Ninja has - # the correct information. - "restat": 1, - }, - "TEMPLATE": { - "command": "$SCONS_INVOCATION $out", - "description": "Rendering $SCONS_INVOCATION $out", - "pool": "scons_pool", - "restat": 1, - }, - "SCONS": { - "command": "$SCONS_INVOCATION $out", - "description": "$SCONS_INVOCATION $out", - "pool": "scons_pool", - # restat - # if present, causes Ninja to re-stat the command's outputs - # after execution of the command. Each output whose - # modification time the command did not change will be - # treated as though it had never needed to be built. This - # may cause the output's reverse dependencies to be removed - # from the list of pending build actions. - # - # We use restat any time we execute SCons because - # SCons calls in Ninja typically create multiple - # targets. But since SCons is doing it's own up to - # date-ness checks it may only update say one of - # them. Restat will find out which of the multiple - # build targets did actually change then only rebuild - # those targets which depend specifically on that - # output. - "restat": 1, - }, - "REGENERATE": { - "command": "$SCONS_INVOCATION_W_TARGETS", - "description": "Regenerating $out", - "generator": 1, - "depfile": os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile'), - # Console pool restricts to 1 job running at a time, - # it additionally has some special handling about - # passing stdin, stdout, etc to process in this pool - # that we need for SCons to behave correctly when - # regenerating Ninja - "pool": "console", - # Again we restat in case Ninja thought the - # build.ninja should be regenerated but SCons knew - # better. - "restat": 1, - }, - } - - if env['PLATFORM'] == 'darwin' and env['AR'] == 'ar': - self.rules["AR"] = { - "command": "rm -f $out && $env$AR $rspc", - "description": "Archiving $out", - "pool": "local_pool", - } - - self.pools = { - "local_pool": self.env.GetOption("num_jobs"), - "install_pool": self.env.GetOption("num_jobs") / 2, - "scons_pool": 1, - } - - for rule in ["CC", "CXX"]: - if env["PLATFORM"] == "win32": - self.rules[rule]["deps"] = "msvc" - else: - self.rules[rule]["deps"] = "gcc" - self.rules[rule]["depfile"] = "$out.d" - - def add_build(self, node): - if not node.has_builder(): - return False - - if isinstance(node, SCons.Node.Alias.Alias): - build = alias_to_ninja_build(node) - else: - build = self.translator.action_to_ninja_build(node) - - # Some things are unbuild-able or need not be built in Ninja - if build is None: - return False - - node_string = str(node) - if node_string in self.builds: - raise Exception("Node {} added to ninja build state more than once".format(node_string)) - self.builds[node_string] = build - self.built.update(build["outputs"]) - return True - - # TODO: rely on SCons to tell us what is generated source - # or some form of user scanner maybe (Github Issue #3624) - def is_generated_source(self, output): - """Check if output ends with a known generated suffix.""" - _, suffix = splitext(output) - return suffix in self.generated_suffixes - - def has_generated_sources(self, output): - """ - Determine if output indicates this is a generated header file. - """ - for generated in output: - if self.is_generated_source(generated): - return True - return False - - # pylint: disable=too-many-branches,too-many-locals - def generate(self): - """ - Generate the build.ninja. - - This should only be called once for the lifetime of this object. - """ - if self.__generated: - return - - self.rules.update(self.env.get(NINJA_RULES, {})) - self.pools.update(self.env.get(NINJA_POOLS, {})) - - content = io.StringIO() - ninja = self.writer_class(content, width=100) - - ninja.comment("Generated by scons. DO NOT EDIT.") - - ninja.variable("builddir", get_path(self.env['NINJA_BUILDDIR'])) - - for pool_name, size in self.pools.items(): - ninja.pool(pool_name, size) - - for var, val in self.variables.items(): - ninja.variable(var, val) - - for rule, kwargs in self.rules.items(): - ninja.rule(rule, **kwargs) - - generated_source_files = sorted({ - output - # First find builds which have header files in their outputs. - for build in self.builds.values() - if self.has_generated_sources(build["outputs"]) - for output in build["outputs"] - # Collect only the header files from the builds with them - # in their output. We do this because is_generated_source - # returns True if it finds a header in any of the outputs, - # here we need to filter so we only have the headers and - # not the other outputs. - if self.is_generated_source(output) - }) - - if generated_source_files: - ninja.build( - outputs="_generated_sources", - rule="phony", - implicit=generated_source_files - ) - - template_builders = [] - - for build in [self.builds[key] for key in sorted(self.builds.keys())]: - if build["rule"] == "TEMPLATE": - template_builders.append(build) - continue - - if "implicit" in build: - build["implicit"].sort() - - # Don't make generated sources depend on each other. We - # have to check that none of the outputs are generated - # sources and none of the direct implicit dependencies are - # generated sources or else we will create a dependency - # cycle. - if ( - generated_source_files - and not build["rule"] == "INSTALL" - and set(build["outputs"]).isdisjoint(generated_source_files) - and set(build.get("implicit", [])).isdisjoint(generated_source_files) - ): - - # Make all non-generated source targets depend on - # _generated_sources. We use order_only for generated - # sources so that we don't rebuild the world if one - # generated source was rebuilt. We just need to make - # sure that all of these sources are generated before - # other builds. - order_only = build.get("order_only", []) - order_only.append("_generated_sources") - build["order_only"] = order_only - if "order_only" in build: - build["order_only"].sort() - - # When using a depfile Ninja can only have a single output - # but SCons will usually have emitted an output for every - # thing a command will create because it's caching is much - # more complex than Ninja's. This includes things like DWO - # files. Here we make sure that Ninja only ever sees one - # target when using a depfile. It will still have a command - # that will create all of the outputs but most targets don't - # depend directly on DWO files and so this assumption is safe - # to make. - rule = self.rules.get(build["rule"]) - - # Some rules like 'phony' and other builtins we don't have - # listed in self.rules so verify that we got a result - # before trying to check if it has a deps key. - # - # Anything using deps or rspfile in Ninja can only have a single - # output, but we may have a build which actually produces - # multiple outputs which other targets can depend on. Here we - # slice up the outputs so we have a single output which we will - # use for the "real" builder and multiple phony targets that - # match the file names of the remaining outputs. This way any - # build can depend on any output from any build. - # - # We assume that the first listed output is the 'key' - # output and is stably presented to us by SCons. For - # instance if -gsplit-dwarf is in play and we are - # producing foo.o and foo.dwo, we expect that outputs[0] - # from SCons will be the foo.o file and not the dwo - # file. If instead we just sorted the whole outputs array, - # we would find that the dwo file becomes the - # first_output, and this breaks, for instance, header - # dependency scanning. - if rule is not None and (rule.get("deps") or rule.get("rspfile")): - first_output, remaining_outputs = ( - build["outputs"][0], - build["outputs"][1:], - ) - - if remaining_outputs: - ninja.build( - outputs=sorted(remaining_outputs), rule="phony", implicit=first_output, - ) - - build["outputs"] = first_output - - # Optionally a rule can specify a depfile, and SCons can generate implicit - # dependencies into the depfile. This allows for dependencies to come and go - # without invalidating the ninja file. The depfile was created in ninja specifically - # for dealing with header files appearing and disappearing across rebuilds, but it can - # be repurposed for anything, as long as you have a way to regenerate the depfile. - # More specific info can be found here: https://ninja-build.org/manual.html#_depfile - if rule is not None and rule.get('depfile') and build.get('deps_files'): - path = build['outputs'] if SCons.Util.is_List(build['outputs']) else [build['outputs']] - generate_depfile(self.env, path[0], build.pop('deps_files', [])) - - if "inputs" in build: - build["inputs"].sort() - - ninja.build(**build) - - template_builds = dict() - for template_builder in template_builders: - - # Special handling for outputs and implicit since we need to - # aggregate not replace for each builder. - for agg_key in ["outputs", "implicit", "inputs"]: - new_val = template_builds.get(agg_key, []) - - # Use pop so the key is removed and so the update - # below will not overwrite our aggregated values. - cur_val = template_builder.pop(agg_key, []) - if is_List(cur_val): - new_val += cur_val - else: - new_val.append(cur_val) - template_builds[agg_key] = new_val - - # Collect all other keys - template_builds.update(template_builder) - - if template_builds.get("outputs", []): - ninja.build(**template_builds) - - # We have to glob the SCons files here to teach the ninja file - # how to regenerate itself. We'll never see ourselves in the - # DAG walk so we can't rely on action_to_ninja_build to - # generate this rule even though SCons should know we're - # dependent on SCons files. - # - # The REGENERATE rule uses depfile, so we need to generate the depfile - # in case any of the SConscripts have changed. The depfile needs to be - # path with in the build and the passed ninja file is an abspath, so - # we will use SCons to give us the path within the build. Normally - # generate_depfile should not be called like this, but instead be called - # through the use of custom rules, and filtered out in the normal - # list of build generation about. However, because the generate rule - # is hardcoded here, we need to do this generate_depfile call manually. - ninja_file_path = self.env.File(self.ninja_file).path - generate_depfile( - self.env, - ninja_file_path, - self.env['NINJA_REGENERATE_DEPS'] - ) - - ninja.build( - ninja_file_path, - rule="REGENERATE", - implicit=[__file__], - ) - - # If we ever change the name/s of the rules that include - # compile commands (i.e. something like CC) we will need to - # update this build to reflect that complete list. - ninja.build( - "compile_commands.json", - rule="CMD", - pool="console", - implicit=[str(self.ninja_file)], - variables={ - "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( - # NINJA_COMPDB_EXPAND - should only be true for ninja - # This was added to ninja's compdb tool in version 1.9.0 (merged April 2018) - # https://github.com/ninja-build/ninja/pull/1223 - # TODO: add check in generate to check version and enable this by default if it's available. - self.ninja_bin_path, str(self.ninja_file), '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' - ) - }, - ) - - ninja.build( - "compiledb", rule="phony", implicit=["compile_commands.json"], - ) - - # Look in SCons's list of DEFAULT_TARGETS, find the ones that - # we generated a ninja build rule for. - scons_default_targets = [ - get_path(tgt) - for tgt in SCons.Script.DEFAULT_TARGETS - if get_path(tgt) in self.built - ] - - # If we found an overlap between SCons's list of default - # targets and the targets we created ninja builds for then use - # those as ninja's default as well. - if scons_default_targets: - ninja.default(" ".join(scons_default_targets)) - - with open(str(self.ninja_file), "w") as build_ninja: - build_ninja.write(content.getvalue()) - - self.__generated = True - - - - - -def get_comstr(env, action, targets, sources): - """Get the un-substituted string for action.""" - # Despite being having "list" in it's name this member is not - # actually a list. It's the pre-subst'd string of the command. We - # use it to determine if the command we're about to generate needs - # to use a custom Ninja rule. By default this redirects CC, CXX, - # AR, SHLINK, and LINK commands to their respective rules but the - # user can inject custom Ninja rules and tie them to commands by - # using their pre-subst'd string. - if hasattr(action, "process"): - return action.cmd_list - - return action.genstring(targets, sources, env) - - -def get_command_env(env): - """ - Return a string that sets the environment for any environment variables that - differ between the OS environment and the SCons command ENV. - - It will be compatible with the default shell of the operating system. - """ - try: - return env["NINJA_ENV_VAR_CACHE"] - except KeyError: - pass - - # Scan the ENV looking for any keys which do not exist in - # os.environ or differ from it. We assume if it's a new or - # differing key from the process environment then it's - # important to pass down to commands in the Ninja file. - ENV = get_default_ENV(env) - scons_specified_env = { - key: value - for key, value in ENV.items() - # TODO: Remove this filter, unless there's a good reason to keep. SCons's behavior shouldn't depend on shell's. - if key not in os.environ or os.environ.get(key, None) != value - } - - windows = env["PLATFORM"] == "win32" - command_env = "" - for key, value in scons_specified_env.items(): - # Ensure that the ENV values are all strings: - if is_List(value): - # If the value is a list, then we assume it is a - # path list, because that's a pretty common list-like - # value to stick in an environment variable: - value = flatten_sequence(value) - value = joinpath(map(str, value)) - else: - # If it isn't a string or a list, then we just coerce - # it to a string, which is the proper way to handle - # Dir and File instances and will produce something - # reasonable for just about everything else: - value = str(value) - - if windows: - command_env += "set '{}={}' && ".format(key, value) - else: - # We address here *only* the specific case that a user might have - # an environment variable which somehow gets included and has - # spaces in the value. These are escapes that Ninja handles. This - # doesn't make builds on paths with spaces (Ninja and SCons issues) - # nor expanding response file paths with spaces (Ninja issue) work. - value = value.replace(r' ', r'$ ') - command_env += "export {}='{}';".format(key, value) - - env["NINJA_ENV_VAR_CACHE"] = command_env - return command_env def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom_env={}): @@ -891,143 +109,6 @@ def get_response_file_command(env, node, action, targets, sources, executor=None return get_response_file_command -def generate_command(env, node, action, targets, sources, executor=None): - # Actions like CommandAction have a method called process that is - # used by SCons to generate the cmd_line they need to run. So - # check if it's a thing like CommandAction and call it if we can. - if hasattr(action, "process"): - cmd_list, _, _ = action.process(targets, sources, env, executor=executor) - cmd = _string_from_cmd_list(cmd_list[0]) - else: - # Anything else works with genstring, this is most commonly hit by - # ListActions which essentially call process on all of their - # commands and concatenate it for us. - genstring = action.genstring(targets, sources, env) - if executor is not None: - cmd = env.subst(genstring, executor=executor) - else: - cmd = env.subst(genstring, targets, sources) - - cmd = cmd.replace("\n", " && ").strip() - if cmd.endswith("&&"): - cmd = cmd[0:-2].strip() - - # Escape dollars as necessary - return cmd.replace("$", "$$") - - -def get_generic_shell_command(env, node, action, targets, sources, executor=None): - return ( - "CMD", - { - # TODO: Why is executor passed in and then ignored below? (bdbaddog) - "cmd": generate_command(env, node, action, targets, sources, executor=None), - "env": get_command_env(env), - }, - # Since this function is a rule mapping provider, it must return a list of dependencies, - # and usually this would be the path to a tool, such as a compiler, used for this rule. - # However this function is to generic to be able to reliably extract such deps - # from the command, so we return a placeholder empty list. It should be noted that - # generally this function will not be used solely and is more like a template to generate - # the basics for a custom provider which may have more specific options for a provider - # function for a custom NinjaRuleMapping. - [] - ) - - -def get_command(env, node, action): # pylint: disable=too-many-branches - """Get the command to execute for node.""" - if node.env: - sub_env = node.env - else: - sub_env = env - executor = node.get_executor() - tlist, slist = get_targets_sources(node) - - # Generate a real CommandAction - if isinstance(action, SCons.Action.CommandGeneratorAction): - # pylint: disable=protected-access - action = action._generate(tlist, slist, sub_env, 1, executor=executor) - - variables = {} - - comstr = get_comstr(sub_env, action, tlist, slist) - if not comstr: - return None - - provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) - rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) - - # Get the dependencies for all targets - implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) - - # Now add in the other dependencies related to the command, - # e.g. the compiler binary. The ninja rule can be user provided so - # we must do some validation to resolve the dependency path for ninja. - for provider_dep in provider_deps: - - provider_dep = sub_env.subst(provider_dep) - if not provider_dep: - continue - - # If the tool is a node, then SCons will resolve the path later, if its not - # a node then we assume it generated from build and make sure it is existing. - if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep): - implicit.append(provider_dep) - continue - - # in some case the tool could be in the local directory and be suppled without the ext - # such as in windows, so append the executable suffix and check. - prog_suffix = sub_env.get('PROGSUFFIX', '') - provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix - if os.path.exists(provider_dep_ext): - implicit.append(provider_dep_ext) - continue - - # Many commands will assume the binary is in the path, so - # we accept this as a possible input from a given command. - - provider_dep_abspath = sub_env.WhereIs(provider_dep) or sub_env.WhereIs(provider_dep, path=os.environ["PATH"]) - if provider_dep_abspath: - implicit.append(provider_dep_abspath) - continue - - # Possibly these could be ignore and the build would still work, however it may not always - # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided - # ninja rule. - raise Exception("Could not resolve path for %s dependency on node '%s'" % (provider_dep, node)) - - ninja_build = { - "order_only": get_order_only(node), - "outputs": get_outputs(node), - "inputs": get_inputs(node), - "implicit": implicit, - "rule": get_rule(node, rule), - "variables": variables, - } - - # Don't use sub_env here because we require that NINJA_POOL be set - # on a per-builder call basis to prevent accidental strange - # behavior like env['NINJA_POOL'] = 'console' and sub_env can be - # the global Environment object if node.env is None. - # Example: - # - # Allowed: - # - # env.Command("ls", NINJA_POOL="ls_pool") - # - # Not allowed and ignored: - # - # env["NINJA_POOL"] = "ls_pool" - # env.Command("ls") - # - # TODO: Why not alloe env['NINJA_POOL'] ? (bdbaddog) - if node.env and node.env.get("NINJA_POOL", None) is not None: - ninja_build["pool"] = node.env["NINJA_POOL"] - - return ninja_build - - def ninja_builder(env, target, source): """Generate a build.ninja for source.""" if not isinstance(source, list): @@ -1109,8 +190,7 @@ def register_custom_handler(env, name, handler): def register_custom_rule_mapping(env, pre_subst_string, rule): """Register a function to call for a given rule.""" - global __NINJA_RULE_MAPPING - __NINJA_RULE_MAPPING[pre_subst_string] = rule + SCons.Tool.ninjaCommon.Globals.__NINJA_RULE_MAPPING[pre_subst_string] = rule def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): @@ -1190,6 +270,7 @@ def CheckNinjaCompdbExpand(env, context): context.Result(result) return result + def ninja_stat(_self, path): """ Eternally memoized stat call. @@ -1200,42 +281,29 @@ def ninja_stat(_self, path): change. For these reasons we patch SCons.Node.FS.LocalFS.stat to use our eternal memoized dictionary. """ - global NINJA_STAT_MEMO try: - return NINJA_STAT_MEMO[path] + return SCons.Tool.ninjaCommon.Globals.NINJA_STAT_MEMO[path] except KeyError: try: result = os.stat(path) except os.error: result = None - NINJA_STAT_MEMO[path] = result + SCons.Tool.ninjaCommon.Globals.NINJA_STAT_MEMO[path] = result return result -def ninja_noop(*_args, **_kwargs): - """ - A general purpose no-op function. - - There are many things that happen in SCons that we don't need and - also don't return anything. We use this to disable those functions - instead of creating multiple definitions of the same thing. - """ - return None - - def ninja_whereis(thing, *_args, **_kwargs): """Replace env.WhereIs with a much faster version""" - global NINJA_WHEREIS_MEMO # Optimize for success, this gets called significantly more often # when the value is already memoized than when it's not. try: - return NINJA_WHEREIS_MEMO[thing] + return SCons.Tool.ninjaCommon.Globals.NINJA_WHEREIS_MEMO[thing] except KeyError: # We do not honor any env['ENV'] or env[*] variables in the - # generated ninja ile. Ninja passes your raw shell environment + # generated ninja file. Ninja passes your raw shell environment # down to it's subprocess so the only sane option is to do the # same during generation. At some point, if and when we try to # upstream this, I'm sure a sticking point will be respecting @@ -1244,7 +312,7 @@ def ninja_whereis(thing, *_args, **_kwargs): # with shell quoting is nigh impossible. So I've decided to # cross that bridge when it's absolutely required. path = shutil.which(thing) - NINJA_WHEREIS_MEMO[thing] = path + SCons.Tool.ninjaCommon.Globals.NINJA_WHEREIS_MEMO[thing] = path return path @@ -1329,9 +397,10 @@ def ninja_emitter(target, source, env): def generate(env): """Generate the NINJA builders.""" - global ninja_builder_initialized - if not ninja_builder_initialized: - ninja_builder_initialized = True + global NINJA_STATE + + if not SCons.Tool.ninjaCommon.Globals.ninja_builder_initialized: + SCons.Tool.ninjaCommon.Globals.ninja_builder_initialized = True ninja_add_command_line_options() @@ -1343,8 +412,6 @@ def generate(env): env["DISABLE_AUTO_NINJA"] = GetOption('disable_execute_ninja') - global NINJA_STATE - env["NINJA_FILE_NAME"] = env.get("NINJA_FILE_NAME", "build.ninja") # Add the Ninja builder. @@ -1353,7 +420,6 @@ def generate(env): emitter=ninja_emitter) env.Append(BUILDERS={"Ninja": ninja_builder_obj}) - env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") env['NINJA_BUILDDIR'] = env.get("NINJA_BUILDDIR", env.Dir(".ninja").path) @@ -1561,20 +627,7 @@ def robust_rule_mapping(var, rule, tool): if NINJA_STATE is None: NINJA_STATE = NinjaState(env, ninja_file[0], ninja_syntax.Writer) - NINJA_STATE.ninja_bin_path = env.get('NINJA') - if not NINJA_STATE.ninja_bin_path: - # default to using ninja installed with python module - ninja_bin = 'ninja.exe' if env["PLATFORM"] == "win32" else 'ninja' - NINJA_STATE.ninja_bin_path = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - ninja_bin)) - if not os.path.exists(NINJA_STATE.ninja_bin_path): - # couldn't find it, just give the bin name and hope - # its in the path later - NINJA_STATE.ninja_bin_path = ninja_bin + # TODO: this is hacking into scons, preferable if there were a less intrusive way # We will subvert the normal builder execute to make sure all the ninja file is dependent @@ -1607,7 +660,6 @@ def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): # file once we have the upstream support for referencing SConscripts as File # nodes. def ninja_execute(self): - global NINJA_STATE target = self.targets[0] if target.check_attributes('ninja_file') is None or not target.is_conftest: From 07119cd6aa3a34813214d95b26745182929cd296 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 10 Apr 2021 16:59:22 -0700 Subject: [PATCH 140/163] Address sider issues --- SCons/Tool/ninjaCommon/Util.py | 2 +- SCons/Tool/ninjaCommon/__init__.py | 12 ++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/SCons/Tool/ninjaCommon/Util.py b/SCons/Tool/ninjaCommon/Util.py index bb20d80a3f..cc30552b50 100644 --- a/SCons/Tool/ninjaCommon/Util.py +++ b/SCons/Tool/ninjaCommon/Util.py @@ -26,7 +26,7 @@ import SCons from SCons.Action import get_default_ENV, _string_from_cmd_list from SCons.Script import AddOption -from SCons.Tool.ninjaCommon import __NINJA_RULE_MAPPING +from SCons.Tool.ninjaCommon.Globals import __NINJA_RULE_MAPPING from SCons.Util import is_List, flatten_sequence diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninjaCommon/__init__.py index a2745c5827..4c5b08eb2c 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninjaCommon/__init__.py @@ -35,16 +35,12 @@ import SCons import SCons.Tool.ninjaCommon.Globals -from SCons.Action import _string_from_cmd_list from SCons.Script import GetOption -from SCons.Util import is_List -import SCons.Tool.ninjaCommon.Globals -from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS, __NINJA_RULE_MAPPING +from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS from .NinjaState import NinjaState -from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function -from .Util import ninja_add_command_line_options, alias_to_ninja_build, \ - get_targets_sources, get_path, ninja_noop, get_command, get_command_env, get_comstr, get_generic_shell_command, \ +from .Util import ninja_add_command_line_options, \ + get_path, ninja_noop, get_command, get_command_env, get_comstr, get_generic_shell_command, \ generate_command NINJA_STATE = None @@ -405,7 +401,7 @@ def generate(env): ninja_add_command_line_options() try: - import ninja + import ninja # noqa: F401 except ImportError: SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return From 1d456a8feb5fffff54364ab61b576e81d743687e Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 10 Apr 2021 17:09:03 -0700 Subject: [PATCH 141/163] Fix ninja.xml syntax error --- SCons/Tool/ninjaCommon/ninja.xml | 2 + doc/generated/builders.gen | 53 +++++++++ doc/generated/builders.mod | 4 + doc/generated/functions.gen | 4 +- doc/generated/tools.gen | 7 ++ doc/generated/tools.mod | 2 + doc/generated/variables.gen | 189 ++++++++++++++++++++++++++++--- doc/generated/variables.mod | 26 +++++ 8 files changed, 267 insertions(+), 20 deletions(-) diff --git a/SCons/Tool/ninjaCommon/ninja.xml b/SCons/Tool/ninjaCommon/ninja.xml index 8f7de362bf..c28d697953 100644 --- a/SCons/Tool/ninjaCommon/ninja.xml +++ b/SCons/Tool/ninjaCommon/ninja.xml @@ -176,9 +176,11 @@ See its __doc__ string for a discussion of the format. This propagates directly into the generated Ninja.build file. From Ninja's docs:
+ builddir A directory for some Ninja output files. ... (You can also store other build output in this directory.) +
diff --git a/doc/generated/builders.gen b/doc/generated/builders.gen index 14508ea96f..d3490f0ca0 100644 --- a/doc/generated/builders.gen +++ b/doc/generated/builders.gen @@ -1221,6 +1221,59 @@ env.MSVSSolution( + + Ninja() + env.Ninja() + + &b-Ninja; is a special builder which + adds a target to create a ninja build file. + The builder does not require any source files to be specified, + + + + If called with no arguments, + the builder will default to a target name of + ninja.build. + + + If called with a single positional argument, + &scons; will "deduce" the target name from that source + argument, giving it the same name, and then + ignore the source. + This is the usual way to call the builder if a + non-default target name is wanted. + + + If called with either the + target= + or source= keyword arguments, + the value of the argument is taken as the target name. + If called with both, the + target= + value is used and source= is ignored. + If called with multiple sources, + the source list will be ignored, + since there is no way to deduce what the intent was; + in this case the default target name will be used. + + + + You must load the &t-ninja; tool prior to specifying + any part of your build or some source/output + files will not show up in the compilation database. + + + To use this tool you must install pypi's ninja + package. + This can be done via + pip install ninja + + + + Available since &scons; 4.2. + + + Object() env.Object() diff --git a/doc/generated/builders.mod b/doc/generated/builders.mod index 4835118f69..3542b99110 100644 --- a/doc/generated/builders.mod +++ b/doc/generated/builders.mod @@ -37,6 +37,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. MOFiles"> MSVSProject"> MSVSSolution"> +Ninja"> Object"> Package"> PCH"> @@ -94,6 +95,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. env.MOFiles"> env.MSVSProject"> env.MSVSSolution"> +env.Ninja"> env.Object"> env.Package"> env.PCH"> @@ -157,6 +159,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. MOFiles"> MSVSProject"> MSVSSolution"> +Ninja"> Object"> Package"> PCH"> @@ -214,6 +217,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. env.MOFiles"> env.MSVSProject"> env.MSVSSolution"> +env.Ninja"> env.Object"> env.Package"> env.PCH"> diff --git a/doc/generated/functions.gen b/doc/generated/functions.gen index 1af4baeac8..cd71ee9a8c 100644 --- a/doc/generated/functions.gen +++ b/doc/generated/functions.gen @@ -548,8 +548,8 @@ do not make sense and a Python exception will be raised. -When using &f-env-Append; to modify &consvars; -which are path specifications (normally, +When using &f-env-Append; to modify &consvars; +which are path specifications (normally, those names which end in PATH), it is recommended to add the values as a list of strings, even if there is only a single string to add. diff --git a/doc/generated/tools.gen b/doc/generated/tools.gen index dc15b6e5bc..bdb353ce67 100644 --- a/doc/generated/tools.gen +++ b/doc/generated/tools.gen @@ -811,6 +811,13 @@ Sets construction variables for the Sets: &cv-link-AS;, &cv-link-ASCOM;, &cv-link-ASFLAGS;, &cv-link-ASPPCOM;, &cv-link-ASPPFLAGS;.Uses: &cv-link-ASCOMSTR;, &cv-link-ASPPCOMSTR;. + + ninja + + Sets up &b-link-Ninja; builder which generates a ninja build file, and then optionally runs ninja. + + Sets: &cv-link-DISABLE_AUTO_NINJA;, &cv-link-NINJA_ALIAS_NAME;, &cv-link-NINJA_BUILDDIR;, &cv-link-NINJA_COMPDB_EXPAND;, &cv-link-NINJA_ENV_VAR_CACHE;, &cv-link-NINJA_FILE_NAME;, &cv-link-NINJA_GENERATED_SOURCE_SUFFIXES;, &cv-link-NINJA_MSVC_DEPS_PREFIX;, &cv-link-NINJA_POOL;, &cv-link-NINJA_REGENERATE_DEPS;, &cv-link-NINJA_SYNTAX;, &cv-link-_NINJA_REGENERATE_DEPS_FUNC;, &cv-link-__NINJA_NO;.Uses: &cv-link-AR;, &cv-link-ARCOM;, &cv-link-ARFLAGS;, &cv-link-CC;, &cv-link-CCCOM;, &cv-link-CXX;, &cv-link-CXXCOM;, &cv-link-ESCAPE;, &cv-link-LINK;, &cv-link-LINKCOM;, &cv-link-PLATFORM;, &cv-link-RANLIB;, &cv-link-RANLIBCOM;, &cv-link-SHCXXCOM;, &cv-link-SHLINK;, &cv-link-SHLINKCOM;. + packaging diff --git a/doc/generated/tools.mod b/doc/generated/tools.mod index 78aa9ef982..35eea5e2b4 100644 --- a/doc/generated/tools.mod +++ b/doc/generated/tools.mod @@ -79,6 +79,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. mwcc"> mwld"> nasm"> +ninja"> packaging"> pdf"> pdflatex"> @@ -184,6 +185,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. mwcc"> mwld"> nasm"> +ninja"> packaging"> pdf"> pdflatex"> diff --git a/doc/generated/variables.gen b/doc/generated/variables.gen index a24fd34ae9..4d5f92f26d 100644 --- a/doc/generated/variables.gen +++ b/doc/generated/variables.gen @@ -22,6 +22,16 @@ if &cv-link-LDMODULEVERSION; is set. Othervise it evaluates to an empty string. + + + __NINJA_NO + + + Internal flag. Used to tell SCons whether or not to try to import pypi's ninja python package. + This is set to True when being called by Ninja? + + + __SHLIBVERSIONFLAGS @@ -703,8 +713,8 @@ the type of value of &cv-CPPDEFINES;: If &cv-CPPDEFINES; is a string, the values of the &cv-link-CPPDEFPREFIX; and &cv-link-CPPDEFSUFFIX; &consvars; -will be respectively prepended and appended to -each definition in &cv-link-CPPDEFINES;. +will be respectively prepended and appended to +each definition in &cv-CPPDEFINES;. @@ -716,8 +726,8 @@ env = Environment(CPPDEFINES='xyz') If &cv-CPPDEFINES; is a list, the values of the -&cv-CPPDEFPREFIX; and &cv-CPPDEFSUFFIX; &consvars; -will be respectively prepended and appended to +&cv-link-CPPDEFPREFIX; and &cv-link-CPPDEFSUFFIX; &consvars; +will be respectively prepended and appended to each element in the list. If any element is a list or tuple, then the first item is the name being @@ -733,7 +743,7 @@ env = Environment(CPPDEFINES=[('B', 2), 'A']) If &cv-CPPDEFINES; is a dictionary, the values of the -&cv-CPPDEFPREFIX; and &cv-CPPDEFSUFFIX; &consvars; +&cv-link-CPPDEFPREFIX; and &cv-link-CPPDEFSUFFIX; &consvars; will be respectively prepended and appended to each item from the dictionary. The key of each dictionary item @@ -835,7 +845,7 @@ to each directory in &cv-link-CPPPATH;. The list of directories that the C preprocessor will search for include directories. The C/C++ implicit dependency scanner will search these -directories for include files. +directories for include files. In general it's not advised to put include directory directives directly into &cv-link-CCFLAGS; or &cv-link-CXXFLAGS; as the result will be non-portable @@ -847,11 +857,11 @@ Python's os.sep. Note: -directory names in &cv-CPPPATH; +directory names in &cv-link-CPPPATH; will be looked-up relative to the directory of the SConscript file -when they are used in a command. +when they are used in a command. To force &scons; -to look-up a directory relative to the root of the source tree use +to look-up a directory relative to the root of the source tree use the # prefix: @@ -1162,6 +1172,17 @@ into a list of Dir instances relative to the target being built. + + + DISABLE_AUTO_NINJA + + + Boolean (True|False). Default: False + When True, SCons will not run ninja automatically after creating the ninja.build file. + + + + DLIB @@ -3137,7 +3158,7 @@ The default list is: IMPLIBNOVERSIONSYMLINKS -Used to override &cv-link-SHLIBNOVERSIONSYMLINKS;/&cv-link-LDMODULENOVERSIONSYMLINKS; when +Used to override &cv-link-SHLIBNOVERSIONSYMLINKS;/&cv-link-LDMODULENOVERSIONSYMLINKS; when creating versioned import library for a shared library/loadable module. If not defined, then &cv-link-SHLIBNOVERSIONSYMLINKS;/&cv-link-LDMODULENOVERSIONSYMLINKS; is used to determine whether to disable symlink generation or not. @@ -4053,9 +4074,9 @@ as the result will be non-portable. Note: directory names in &cv-LIBPATH; will be looked-up relative to the directory of the SConscript file -when they are used in a command. +when they are used in a command. To force &scons; -to look-up a directory relative to the root of the source tree use +to look-up a directory relative to the root of the source tree use the # prefix: @@ -4127,7 +4148,7 @@ and suffixes from the &cv-link-LIBSUFFIXES; list. A list of one or more libraries that will be added to the link line -for linking with any executable program, shared library, or loadable module +for linking with any executable program, shared library, or loadable module created by the &consenv; or override. @@ -4493,7 +4514,7 @@ See &t-link-msgfmt; tool and &b-link-MOFiles; builder. MSGFMTCOMSTR -String to display when msgfmt(1) is invoked +String to display when msgfmt(1) is invoked (default: '', which means ``print &cv-link-MSGFMTCOM;''). See &t-link-msgfmt; tool and &b-link-MOFiles; builder. @@ -4535,7 +4556,7 @@ See &t-link-msginit; tool and &b-link-POInit; builder. MSGINITCOMSTR -String to display when msginit(1) is invoked +String to display when msginit(1) is invoked (default: '', which means ``print &cv-link-MSGINITCOM;''). See &t-link-msginit; tool and &b-link-POInit; builder. @@ -4631,7 +4652,7 @@ Supported versions include 6.0A, 6.0, 2003R2 -and +and 2003R1. @@ -5156,6 +5177,138 @@ Specfies the name of the project to package. See the &b-link-Package; builder. + + + NINJA_ALIAS_NAME + + + Name of the Alias() which is will cause SCons to create the ninja.build file, and + then (optionally) run ninja. + + + + + + NINJA_BUILDDIR + + + This propagates directly into the generated Ninja.build file. + From Ninja's docs: +
+ + builddir + A directory for some Ninja output files. ... (You can also store other build output in this + directory.) + +
+ + + + + + NINJA_COMPDB_EXPAND + + + Boolean value (True|False) to instruct ninja to expand the command line arguments normally put into + response files. + This prevents lines in the compilation database like gcc @rsp_file and instead yields + gcc -c -o myfile.o myfile.c -Ia -DXYZ + + + Ninja's compdb tool added the -x flag in Ninja V1.9.0 + + + + + + NINJA_ENV_VAR_CACHE + + + A string that sets the environment for any environment variables that + differ between the OS environment and the SCons command ENV. + + It will be compatible with the default shell of the operating system. + + If not explicitly specified, SCons will generate this dynamically from the Environment()'s 'ENV' + env['ENV'] + where those values differ from the existing shell.. + + + + + + NINJA_FILE_NAME + + + The filename for the generated Ninja build file defaults to ninja.build + + + + + + NINJA_GENERATED_SOURCE_SUFFIXES + + + The list of source file suffixes which are generated by SCons build steps. + All source files which match these suffixes will be added to the _generated_sources alias in the output + ninja.build file. + Then all other source files will be made to depend on this in the Ninja.build file, forcing the + generated sources to be built first. + + + + + + NINJA_MSVC_DEPS_PREFIX + + + This propagates directly into the generated Ninja.build file. + From Ninja's docs + defines the string which should be stripped from msvc’s /showIncludes output + + + + + + NINJA_POOL + + + Set the ninja_pool for this or all targets in scope for this env var. + + + + + + NINJA_REGENERATE_DEPS + + + A generator function used to create a ninja depsfile which includes all the files which would require + SCons to be invoked if they change. + Or a list of said files. + + + + + + _NINJA_REGENERATE_DEPS_FUNC + + + Internal value used to specify the function to call with argument env to generate the list of files + which if changed would require the ninja file to be regenerated. + + + + + + NINJA_SYNTAX + + + Theres also NINJA_SYNTAX which is the path to a custom ninja_syntax.py file which is used in generation. + The tool currently assumes you have ninja installed through pip, and grabs the syntax file from that + installation if none specified. + + + no_import_lib @@ -7357,7 +7510,7 @@ for more information). SOVERSION -This will construct the SONAME using on the base library name +This will construct the SONAME using on the base library name (test in the example below) and use specified SOVERSION to create SONAME. @@ -7366,7 +7519,7 @@ env.SharedLibrary('test', 'test.c', SHLIBVERSION='0.1.2', SOVERSION='2') The variable is used, for example, by &t-link-gnulink; linker tool. -In the example above SONAME would be libtest.so.2 +In the example above SONAME would be libtest.so.2 which would be a symlink and point to libtest.so.0.1.2 diff --git a/doc/generated/variables.mod b/doc/generated/variables.mod index a3dbc0d8f2..86fc604b45 100644 --- a/doc/generated/variables.mod +++ b/doc/generated/variables.mod @@ -9,6 +9,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. --> $__LDMODULEVERSIONFLAGS"> +$__NINJA_NO"> $__SHLIBVERSIONFLAGS"> $APPLELINK_COMPATIBILITY_VERSION"> $_APPLELINK_COMPATIBILITY_VERSION"> @@ -83,6 +84,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $DINCSUFFIX"> $Dir"> $Dirs"> +$DISABLE_AUTO_NINJA"> $DLIB"> $DLIBCOM"> $DLIBDIRPREFIX"> @@ -349,6 +351,17 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $MWCW_VERSION"> $MWCW_VERSIONS"> $NAME"> +$NINJA_ALIAS_NAME"> +$NINJA_BUILDDIR"> +$NINJA_COMPDB_EXPAND"> +$NINJA_ENV_VAR_CACHE"> +$NINJA_FILE_NAME"> +$NINJA_GENERATED_SOURCE_SUFFIXES"> +$NINJA_MSVC_DEPS_PREFIX"> +$NINJA_POOL"> +$NINJA_REGENERATE_DEPS"> +$_NINJA_REGENERATE_DEPS_FUNC"> +$NINJA_SYNTAX"> $no_import_lib"> $OBJPREFIX"> $OBJSUFFIX"> @@ -649,6 +662,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. --> $__LDMODULEVERSIONFLAGS"> +$__NINJA_NO"> $__SHLIBVERSIONFLAGS"> $APPLELINK_COMPATIBILITY_VERSION"> $_APPLELINK_COMPATIBILITY_VERSION"> @@ -723,6 +737,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $DINCSUFFIX"> $Dir"> $Dirs"> +$DISABLE_AUTO_NINJA"> $DLIB"> $DLIBCOM"> $DLIBDIRPREFIX"> @@ -989,6 +1004,17 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $MWCW_VERSION"> $MWCW_VERSIONS"> $NAME"> +$NINJA_ALIAS_NAME"> +$NINJA_BUILDDIR"> +$NINJA_COMPDB_EXPAND"> +$NINJA_ENV_VAR_CACHE"> +$NINJA_FILE_NAME"> +$NINJA_GENERATED_SOURCE_SUFFIXES"> +$NINJA_MSVC_DEPS_PREFIX"> +$NINJA_POOL"> +$NINJA_REGENERATE_DEPS"> +$_NINJA_REGENERATE_DEPS_FUNC"> +$NINJA_SYNTAX"> $no_import_lib"> $OBJPREFIX"> $OBJSUFFIX"> From 9651189675aeecf8884f1aee46d739bc75c30907 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 12 Apr 2021 17:57:47 -0700 Subject: [PATCH 142/163] No need to have ninja.py which just references ninjaCommon, rename package to ninja and get rid of ninja.py --- SCons/Tool/ninja.py | 26 ------------------- SCons/Tool/{ninjaCommon => ninja}/Globals.py | 0 .../Tool/{ninjaCommon => ninja}/NinjaState.py | 5 ++-- .../Tool/{ninjaCommon => ninja}/Overrides.py | 0 SCons/Tool/{ninjaCommon => ninja}/Rules.py | 0 SCons/Tool/{ninjaCommon => ninja}/Util.py | 2 +- SCons/Tool/{ninjaCommon => ninja}/__init__.py | 16 ++++++------ SCons/Tool/{ninjaCommon => ninja}/ninja.xml | 0 8 files changed, 12 insertions(+), 37 deletions(-) delete mode 100644 SCons/Tool/ninja.py rename SCons/Tool/{ninjaCommon => ninja}/Globals.py (100%) rename SCons/Tool/{ninjaCommon => ninja}/NinjaState.py (99%) rename SCons/Tool/{ninjaCommon => ninja}/Overrides.py (100%) rename SCons/Tool/{ninjaCommon => ninja}/Rules.py (100%) rename SCons/Tool/{ninjaCommon => ninja}/Util.py (99%) rename SCons/Tool/{ninjaCommon => ninja}/__init__.py (98%) rename SCons/Tool/{ninjaCommon => ninja}/ninja.xml (100%) diff --git a/SCons/Tool/ninja.py b/SCons/Tool/ninja.py deleted file mode 100644 index 88b7d48f09..0000000000 --- a/SCons/Tool/ninja.py +++ /dev/null @@ -1,26 +0,0 @@ -# MIT License -# -# Copyright The SCons Foundation -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - - -from .ninjaCommon import generate, exists # noqa: F401 diff --git a/SCons/Tool/ninjaCommon/Globals.py b/SCons/Tool/ninja/Globals.py similarity index 100% rename from SCons/Tool/ninjaCommon/Globals.py rename to SCons/Tool/ninja/Globals.py diff --git a/SCons/Tool/ninjaCommon/NinjaState.py b/SCons/Tool/ninja/NinjaState.py similarity index 99% rename from SCons/Tool/ninjaCommon/NinjaState.py rename to SCons/Tool/ninja/NinjaState.py index 4e41934aa3..a7c3584582 100644 --- a/SCons/Tool/ninjaCommon/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -5,6 +5,7 @@ # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including +# "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to @@ -30,7 +31,7 @@ import SCons from SCons.Script import COMMAND_LINE_TARGETS from SCons.Util import is_List -import SCons.Tool.ninjaCommon.Globals +import SCons.Tool.ninja.Globals from .Globals import COMMAND_TYPES, NINJA_RULES, NINJA_POOLS, \ NINJA_CUSTOM_HANDLERS from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function @@ -562,7 +563,7 @@ def action_to_ninja_build(self, node, action=None): # Ninja builders out of being sources of ninja builders but I # can't fix every DAG problem so we just skip ninja_builders # if we find one - if SCons.Tool.ninjaCommon.NINJA_STATE.ninja_file == str(node): + if SCons.Tool.ninja.NINJA_STATE.ninja_file == str(node): build = None elif isinstance(action, SCons.Action.FunctionAction): build = self.handle_func_action(node, action) diff --git a/SCons/Tool/ninjaCommon/Overrides.py b/SCons/Tool/ninja/Overrides.py similarity index 100% rename from SCons/Tool/ninjaCommon/Overrides.py rename to SCons/Tool/ninja/Overrides.py diff --git a/SCons/Tool/ninjaCommon/Rules.py b/SCons/Tool/ninja/Rules.py similarity index 100% rename from SCons/Tool/ninjaCommon/Rules.py rename to SCons/Tool/ninja/Rules.py diff --git a/SCons/Tool/ninjaCommon/Util.py b/SCons/Tool/ninja/Util.py similarity index 99% rename from SCons/Tool/ninjaCommon/Util.py rename to SCons/Tool/ninja/Util.py index cc30552b50..80d1b16d75 100644 --- a/SCons/Tool/ninjaCommon/Util.py +++ b/SCons/Tool/ninja/Util.py @@ -26,7 +26,7 @@ import SCons from SCons.Action import get_default_ENV, _string_from_cmd_list from SCons.Script import AddOption -from SCons.Tool.ninjaCommon.Globals import __NINJA_RULE_MAPPING +from SCons.Tool.ninja.Globals import __NINJA_RULE_MAPPING from SCons.Util import is_List, flatten_sequence diff --git a/SCons/Tool/ninjaCommon/__init__.py b/SCons/Tool/ninja/__init__.py similarity index 98% rename from SCons/Tool/ninjaCommon/__init__.py rename to SCons/Tool/ninja/__init__.py index 4c5b08eb2c..9cc8b2e5f7 100644 --- a/SCons/Tool/ninjaCommon/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -34,7 +34,7 @@ from glob import glob import SCons -import SCons.Tool.ninjaCommon.Globals +import SCons.Tool.ninja.Globals from SCons.Script import GetOption from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS @@ -186,7 +186,7 @@ def register_custom_handler(env, name, handler): def register_custom_rule_mapping(env, pre_subst_string, rule): """Register a function to call for a given rule.""" - SCons.Tool.ninjaCommon.Globals.__NINJA_RULE_MAPPING[pre_subst_string] = rule + SCons.Tool.ninja.Globals.__NINJA_RULE_MAPPING[pre_subst_string] = rule def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): @@ -279,14 +279,14 @@ def ninja_stat(_self, path): """ try: - return SCons.Tool.ninjaCommon.Globals.NINJA_STAT_MEMO[path] + return SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] except KeyError: try: result = os.stat(path) except os.error: result = None - SCons.Tool.ninjaCommon.Globals.NINJA_STAT_MEMO[path] = result + SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] = result return result @@ -296,7 +296,7 @@ def ninja_whereis(thing, *_args, **_kwargs): # Optimize for success, this gets called significantly more often # when the value is already memoized than when it's not. try: - return SCons.Tool.ninjaCommon.Globals.NINJA_WHEREIS_MEMO[thing] + return SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] except KeyError: # We do not honor any env['ENV'] or env[*] variables in the # generated ninja file. Ninja passes your raw shell environment @@ -308,7 +308,7 @@ def ninja_whereis(thing, *_args, **_kwargs): # with shell quoting is nigh impossible. So I've decided to # cross that bridge when it's absolutely required. path = shutil.which(thing) - SCons.Tool.ninjaCommon.Globals.NINJA_WHEREIS_MEMO[thing] = path + SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] = path return path @@ -395,8 +395,8 @@ def generate(env): """Generate the NINJA builders.""" global NINJA_STATE - if not SCons.Tool.ninjaCommon.Globals.ninja_builder_initialized: - SCons.Tool.ninjaCommon.Globals.ninja_builder_initialized = True + if not SCons.Tool.ninja.Globals.ninja_builder_initialized: + SCons.Tool.ninja.Globals.ninja_builder_initialized = True ninja_add_command_line_options() diff --git a/SCons/Tool/ninjaCommon/ninja.xml b/SCons/Tool/ninja/ninja.xml similarity index 100% rename from SCons/Tool/ninjaCommon/ninja.xml rename to SCons/Tool/ninja/ninja.xml From a9783b084bafea76f53ef5c77047442ad4172936 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 12 Apr 2021 21:21:39 -0700 Subject: [PATCH 143/163] Updated from MongoDB commit: https://github.com/mongodb/mongo/commit/59965cf5430b8132074771c0d9278a3a4b1e6730. Better handling of config test output --- SCons/Tool/ninja/__init__.py | 9 ++++++++- test/ninja/iterative_speedup.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 9cc8b2e5f7..ee059ced1d 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -340,6 +340,13 @@ def ninja_hack_linkcom(env): ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' +def ninja_print_conf_log(s, target, source, env): + """Command line print only for conftest to generate a correct conf log.""" + if target and target[0].is_conftest(): + action = SCons.Action._ActionAction() + action.print_cmd_line(s, target, source, env) + + class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" @@ -600,7 +607,7 @@ def robust_rule_mapping(var, rule, tool): SCons.Executor.Executor._get_unchanged_targets = SCons.Executor.Executor._get_targets # Replace false action messages with nothing. - env["PRINT_CMD_LINE_FUNC"] = ninja_noop + env["PRINT_CMD_LINE_FUNC"] = ninja_print_conf_log # This reduces unnecessary subst_list calls to add the compiler to # the implicit dependencies of targets. Since we encode full paths diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index df010f4d5b..9823550b59 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -225,7 +225,7 @@ def mod_source_orig(test_num): full_build_print = True for ninja_time, scons_time in zip(ninja_times, scons_times): if ninja_time > scons_time: - test.fail_test() + test.fail_test(message="Ninja was slower than SCons: SCons: {:.3f}s Ninja: {:.3f}s".format(scons_time, ninja_time)) if full_build_print: full_build_print = False print("Clean build {} files - SCons: {:.3f}s Ninja: {:.3f}s".format(num_source, scons_time, ninja_time)) From f0035f7c24875b9a35b9191d30feb7b305e2c55f Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 13 Apr 2021 14:28:01 -0700 Subject: [PATCH 144/163] Add --experimental=ninja --- SCons/Script/SConsOptions.py | 4 ++-- test/option/option--experimental.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index 4988a8d96e..def866303f 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -39,7 +39,7 @@ diskcheck_all = SCons.Node.FS.diskcheck_types() -experimental_features = {'warp_speed', 'transporter'} +experimental_features = {'warp_speed', 'transporter', 'ninja'} def diskcheck_convert(value): @@ -749,7 +749,7 @@ def experimental_callback(option, opt, value, parser): op.add_option('--experimental', dest='experimental', action='callback', - default={}, # empty set + default=set(), # empty set type='str', # choices=experimental_options+experimental_features, callback =experimental_callback, diff --git a/test/option/option--experimental.py b/test/option/option--experimental.py index 0f0e9ec677..2e06dc3777 100644 --- a/test/option/option--experimental.py +++ b/test/option/option--experimental.py @@ -35,12 +35,13 @@ tests = [ ('.', []), - ('--experimental=all', ['transporter', 'warp_speed']), + ('--experimental=ninja', ['ninja']), + ('--experimental=all', ['ninja', 'transporter', 'warp_speed']), ('--experimental=none', []), ] for args, exper in tests: - read_string = """All Features=transporter,warp_speed + read_string = """All Features=ninja,transporter,warp_speed Experimental=%s """ % (exper) test.run(arguments=args, @@ -49,7 +50,7 @@ test.run(arguments='--experimental=warp_drive', stderr="""usage: scons [OPTION] [TARGET] ... -SCons Error: option --experimental: invalid choice: 'warp_drive' (choose from 'all','none','transporter','warp_speed') +SCons Error: option --experimental: invalid choice: 'warp_drive' (choose from 'all','none','ninja','transporter','warp_speed') """, status=2) From 181044983e6d026c8c19bef5b12203b74848d8ad Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 13 Apr 2021 14:28:37 -0700 Subject: [PATCH 145/163] Gate ninja tool being initialized by --experimental=ninja --- SCons/Tool/ninja/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index ee059ced1d..37e2cf8933 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -361,6 +361,9 @@ def _print_cmd_str(*_args, **_kwargs): def exists(env): """Enable if called.""" + if 'ninja' not in GetOption('experimental'): + return False + # This variable disables the tool when storing the SCons command in the # generated ninja file to ensure that the ninja tool is not loaded when # SCons should do actual work as a subprocess of a ninja build. The ninja @@ -402,6 +405,9 @@ def generate(env): """Generate the NINJA builders.""" global NINJA_STATE + if not 'ninja' in GetOption('experimental'): + return + if not SCons.Tool.ninja.Globals.ninja_builder_initialized: SCons.Tool.ninja.Globals.ninja_builder_initialized = True From b20646de49dbb586371a40ac27cbfe1b4b48aa6a Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 13 Apr 2021 14:28:53 -0700 Subject: [PATCH 146/163] update tests to use SetOption('experimental','ninja') --- test/ninja/build_libraries.py | 2 ++ test/ninja/copy_function_command.py | 2 ++ test/ninja/generate_source.py | 3 +++ test/ninja/iterative_speedup.py | 3 +++ test/ninja/multi_env.py | 3 +++ .../ninja/ninja_test_sconscripts/sconstruct_generate_and_build | 1 + .../ninja_test_sconscripts/sconstruct_generate_and_build_cxx | 1 + test/ninja/shell_command.py | 3 +++ 8 files changed, 18 insertions(+) diff --git a/test/ninja/build_libraries.py b/test/ninja/build_libraries.py index 198dcced99..eb8eb74590 100644 --- a/test/ninja/build_libraries.py +++ b/test/ninja/build_libraries.py @@ -66,6 +66,8 @@ lib_suffix = '.dylib' test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) env = Environment() env.Tool('ninja') env['NINJA'] = "%(ninja_bin)s" diff --git a/test/ninja/copy_function_command.py b/test/ninja/copy_function_command.py index 37af62f52e..7e999b33e5 100644 --- a/test/ninja/copy_function_command.py +++ b/test/ninja/copy_function_command.py @@ -47,6 +47,8 @@ test.dir_fixture('ninja-fixture') test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) env = Environment() env.Tool('ninja') env.Command('foo2.c', ['foo.c'], Copy('$TARGET','$SOURCE')) diff --git a/test/ninja/generate_source.py b/test/ninja/generate_source.py index 18b53f249b..8300176c2e 100644 --- a/test/ninja/generate_source.py +++ b/test/ninja/generate_source.py @@ -49,6 +49,9 @@ shell = '' if IS_WINDOWS else './' test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + env = Environment() env.Tool('ninja') prog = env.Program(target = 'generate_source', source = 'generate_source.c') diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index 9823550b59..05e372cf56 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -166,6 +166,9 @@ def mod_source_orig(test_num): """ % locals()) test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + env = Environment() env.Tool('ninja') sources = ['main.c'] + env.Glob('source*.c') diff --git a/test/ninja/multi_env.py b/test/ninja/multi_env.py index d14588876b..e5da6cf885 100644 --- a/test/ninja/multi_env.py +++ b/test/ninja/multi_env.py @@ -49,6 +49,9 @@ test.dir_fixture('ninja-fixture') test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + env = Environment() env.Tool('ninja') env.Program(target = 'foo', source = 'foo.c') diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build index 242eb76265..81a4366755 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build @@ -1,3 +1,4 @@ +SetOption('experimental','ninja') DefaultEnvironment(tools=[]) env = Environment() diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx index 51ca6c356d..f7137dfb6e 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generate_and_build_cxx @@ -1,3 +1,4 @@ +SetOption('experimental','ninja') DefaultEnvironment(tools=[]) env = Environment() env.Tool('ninja') diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index f0450d9cef..a6926c778d 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -49,6 +49,9 @@ shell = '' if IS_WINDOWS else './' test.write('SConstruct', """ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + env = Environment() env.Tool('ninja') prog = env.Program(target = 'foo', source = 'foo.c') From f8415bafaa37ea1ddc665721202a2406fc8ffaf9 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 13 Apr 2021 14:32:49 -0700 Subject: [PATCH 147/163] fix sider complaint --- SCons/Tool/ninja/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 37e2cf8933..62c16cd3f7 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -405,7 +405,7 @@ def generate(env): """Generate the NINJA builders.""" global NINJA_STATE - if not 'ninja' in GetOption('experimental'): + if 'ninja' not in GetOption('experimental'): return if not SCons.Tool.ninja.Globals.ninja_builder_initialized: From 3ed0272404e86f852772b51f6f888f35cac0bafd Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 13 Apr 2021 16:33:22 -0700 Subject: [PATCH 148/163] switch to using SCons.Node.SConscriptNodes to get list of sconscripts vs using Glob()'s --- SCons/Tool/ninja/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 62c16cd3f7..b239c379a4 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -443,10 +443,14 @@ def generate(env): SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] - # TODO: API for getting the SConscripts programmatically - # exists upstream: https://github.com/SCons/scons/issues/3625 def ninja_generate_deps(env): - return sorted([env.File("#SConstruct").path] + glob("**/SConscript", recursive=True)) + """Return a list of SConscripts + TODO: Should we also include files loaded from site_scons/*** + or even all loaded modules? https://stackoverflow.com/questions/4858100/how-to-list-imported-modules + TODO: Do we want this to be Nodes? + """ + return sorted([str(s) for s in SCons.Node.SConscriptNodes]) + env['_NINJA_REGENERATE_DEPS_FUNC'] = ninja_generate_deps env['NINJA_REGENERATE_DEPS'] = env.get('NINJA_REGENERATE_DEPS', '${_NINJA_REGENERATE_DEPS_FUNC(__env__)}') From 0a430bb9fbbaf345c3d89bf2df5bc9c8d7d9e74a Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 18 May 2021 15:49:21 -0700 Subject: [PATCH 149/163] Quiet sider complaint --- SCons/Tool/ninja/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index b239c379a4..fd9c13c7e3 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -414,7 +414,7 @@ def generate(env): ninja_add_command_line_options() try: - import ninja # noqa: F401 + import ninja # noqa: F401 except ImportError: SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return From 31e1abfd9db0834d0c210733a47acebb6e589764 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 24 May 2021 16:01:47 -0700 Subject: [PATCH 150/163] Continue refactor. Simplify __init__, and move logic to reasonably named files --- SCons/Tool/ninja/Methods.py | 269 ++++++++++++++++++++++++ SCons/Tool/ninja/NinjaState.py | 4 +- SCons/Tool/ninja/Overrides.py | 96 +++++++++ SCons/Tool/ninja/Rules.py | 2 +- SCons/Tool/ninja/{Util.py => Utils.py} | 191 +++++++---------- SCons/Tool/ninja/__init__.py | 276 +------------------------ 6 files changed, 454 insertions(+), 384 deletions(-) create mode 100644 SCons/Tool/ninja/Methods.py rename SCons/Tool/ninja/{Util.py => Utils.py} (72%) diff --git a/SCons/Tool/ninja/Methods.py b/SCons/Tool/ninja/Methods.py new file mode 100644 index 0000000000..3612236ed7 --- /dev/null +++ b/SCons/Tool/ninja/Methods.py @@ -0,0 +1,269 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import os +import shlex +import textwrap + +import SCons +from SCons.Tool.ninja import NINJA_CUSTOM_HANDLERS, NINJA_RULES, NINJA_POOLS +from SCons.Tool.ninja.Globals import __NINJA_RULE_MAPPING +from SCons.Tool.ninja.Utils import get_targets_sources, get_dependencies, get_order_only, get_outputs, get_inputs, \ + get_rule, get_path, generate_command, get_command_env, get_comstr + + +def register_custom_handler(env, name, handler): + """Register a custom handler for SCons function actions.""" + env[NINJA_CUSTOM_HANDLERS][name] = handler + + +def register_custom_rule_mapping(env, pre_subst_string, rule): + """Register a function to call for a given rule.""" + SCons.Tool.ninja.Globals.__NINJA_RULE_MAPPING[pre_subst_string] = rule + + +def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): + """Allows specification of Ninja rules from inside SCons files.""" + rule_obj = { + "command": command, + "description": description if description else "{} $out".format(rule), + } + + if use_depfile: + rule_obj["depfile"] = os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile') + + if deps is not None: + rule_obj["deps"] = deps + + if pool is not None: + rule_obj["pool"] = pool + + if use_response_file: + rule_obj["rspfile"] = "$out.rsp" + rule_obj["rspfile_content"] = response_file_content + + env[NINJA_RULES][rule] = rule_obj + + +def register_custom_pool(env, pool, size): + """Allows the creation of custom Ninja pools""" + env[NINJA_POOLS][pool] = size + + +def set_build_node_callback(env, node, callback): + if not node.is_conftest(): + node.attributes.ninja_build_callback = callback + + +def get_generic_shell_command(env, node, action, targets, sources, executor=None): + return ( + "CMD", + { + # TODO: Why is executor passed in and then ignored below? (bdbaddog) + "cmd": generate_command(env, node, action, targets, sources, executor=None), + "env": get_command_env(env), + }, + # Since this function is a rule mapping provider, it must return a list of dependencies, + # and usually this would be the path to a tool, such as a compiler, used for this rule. + # However this function is to generic to be able to reliably extract such deps + # from the command, so we return a placeholder empty list. It should be noted that + # generally this function will not be used solely and is more like a template to generate + # the basics for a custom provider which may have more specific options for a provider + # function for a custom NinjaRuleMapping. + [] + ) + + +def CheckNinjaCompdbExpand(env, context): + """ Configure check testing if ninja's compdb can expand response files""" + + # TODO: When would this be false? + context.Message('Checking if ninja compdb can expand response files... ') + ret, output = context.TryAction( + action='ninja -f $SOURCE -t compdb -x CMD_RSP > $TARGET', + extension='.ninja', + text=textwrap.dedent(""" + rule CMD_RSP + command = $cmd @$out.rsp > fake_output.txt + description = Building $out + rspfile = $out.rsp + rspfile_content = $rspc + build fake_output.txt: CMD_RSP fake_input.txt + cmd = echo + pool = console + rspc = "test" + """)) + result = '@fake_output.txt.rsp' not in output + context.Result(result) + return result + + +def get_command(env, node, action): # pylint: disable=too-many-branches + """Get the command to execute for node.""" + if node.env: + sub_env = node.env + else: + sub_env = env + executor = node.get_executor() + tlist, slist = get_targets_sources(node) + + # Generate a real CommandAction + if isinstance(action, SCons.Action.CommandGeneratorAction): + # pylint: disable=protected-access + action = action._generate(tlist, slist, sub_env, 1, executor=executor) + + variables = {} + + comstr = get_comstr(sub_env, action, tlist, slist) + if not comstr: + return None + + provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) + rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) + + # Get the dependencies for all targets + implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) + + # Now add in the other dependencies related to the command, + # e.g. the compiler binary. The ninja rule can be user provided so + # we must do some validation to resolve the dependency path for ninja. + for provider_dep in provider_deps: + + provider_dep = sub_env.subst(provider_dep) + if not provider_dep: + continue + + # If the tool is a node, then SCons will resolve the path later, if its not + # a node then we assume it generated from build and make sure it is existing. + if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep): + implicit.append(provider_dep) + continue + + # in some case the tool could be in the local directory and be suppled without the ext + # such as in windows, so append the executable suffix and check. + prog_suffix = sub_env.get('PROGSUFFIX', '') + provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix + if os.path.exists(provider_dep_ext): + implicit.append(provider_dep_ext) + continue + + # Many commands will assume the binary is in the path, so + # we accept this as a possible input from a given command. + + provider_dep_abspath = sub_env.WhereIs(provider_dep) or sub_env.WhereIs(provider_dep, path=os.environ["PATH"]) + if provider_dep_abspath: + implicit.append(provider_dep_abspath) + continue + + # Possibly these could be ignore and the build would still work, however it may not always + # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided + # ninja rule. + raise Exception("Could not resolve path for %s dependency on node '%s'" % (provider_dep, node)) + + ninja_build = { + "order_only": get_order_only(node), + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "implicit": implicit, + "rule": get_rule(node, rule), + "variables": variables, + } + + # Don't use sub_env here because we require that NINJA_POOL be set + # on a per-builder call basis to prevent accidental strange + # behavior like env['NINJA_POOL'] = 'console' and sub_env can be + # the global Environment object if node.env is None. + # Example: + # + # Allowed: + # + # env.Command("ls", NINJA_POOL="ls_pool") + # + # Not allowed and ignored: + # + # env["NINJA_POOL"] = "ls_pool" + # env.Command("ls") + # + # TODO: Why not alloe env['NINJA_POOL'] ? (bdbaddog) + if node.env and node.env.get("NINJA_POOL", None) is not None: + ninja_build["pool"] = node.env["NINJA_POOL"] + + return ninja_build + + +def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom_env={}): + """Generate a response file command provider for rule name.""" + + # If win32 using the environment with a response file command will cause + # ninja to fail to create the response file. Additionally since these rules + # generally are not piping through cmd.exe /c any environment variables will + # make CreateProcess fail to start. + # + # On POSIX we can still set environment variables even for compile + # commands so we do so. + use_command_env = not env["PLATFORM"] == "win32" + if "$" in tool: + tool_is_dynamic = True + + def get_response_file_command(env, node, action, targets, sources, executor=None): + if hasattr(action, "process"): + cmd_list, _, _ = action.process(targets, sources, env, executor=executor) + cmd_list = [str(c).replace("$", "$$") for c in cmd_list[0]] + else: + command = generate_command( + env, node, action, targets, sources, executor=executor + ) + cmd_list = shlex.split(command) + + if tool_is_dynamic: + tool_command = env.subst( + tool, target=targets, source=sources, executor=executor + ) + else: + tool_command = tool + + try: + # Add 1 so we always keep the actual tool inside of cmd + tool_idx = cmd_list.index(tool_command) + 1 + except ValueError: + raise Exception( + "Could not find tool {} in {} generated from {}".format( + tool, cmd_list, get_comstr(env, action, targets, sources) + ) + ) + + cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] + rsp_content = ['"' + rsp_content_item + '"' for rsp_content_item in rsp_content] + rsp_content = " ".join(rsp_content) + + variables = {"rspc": rsp_content, rule: cmd} + if use_command_env: + variables["env"] = get_command_env(env) + + for key, value in custom_env.items(): + variables["env"] += env.subst( + "export %s=%s;" % (key, value), target=targets, source=sources, executor=executor + ) + " " + return rule, variables, [tool_command] + + return get_response_file_command \ No newline at end of file diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index a7c3584582..f906b33fb7 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -35,9 +35,9 @@ from .Globals import COMMAND_TYPES, NINJA_RULES, NINJA_POOLS, \ NINJA_CUSTOM_HANDLERS from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function -from .Util import get_path, alias_to_ninja_build, generate_depfile, ninja_noop, get_command, get_order_only, \ +from .Utils import get_path, alias_to_ninja_build, generate_depfile, ninja_noop, get_order_only, \ get_outputs, get_inputs, get_dependencies, get_rule, get_command_env - +from . import get_command # pylint: disable=too-many-instance-attributes diff --git a/SCons/Tool/ninja/Overrides.py b/SCons/Tool/ninja/Overrides.py index e69de29bb2..80516a26ea 100644 --- a/SCons/Tool/ninja/Overrides.py +++ b/SCons/Tool/ninja/Overrides.py @@ -0,0 +1,96 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +This module is to hold logic which overrides default SCons behavoirs to enable +ninja file generation +""" +import SCons + + +def ninja_hack_linkcom(env): + # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks + # without relying on the SCons hacks that SCons uses by default. + if env["PLATFORM"] == "win32": + from SCons.Tool.mslink import compositeLinkAction + + if env.get("LINKCOM", None) == compositeLinkAction: + env[ + "LINKCOM" + ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' + env[ + "SHLINKCOM" + ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' + + +def ninja_hack_arcom(env): + """ + Force ARCOM so use 's' flag on ar instead of separately running ranlib + """ + if env["PLATFORM"] != "win32" and env.get("RANLIBCOM"): + # There is no way to translate the ranlib list action into + # Ninja so add the s flag and disable ranlib. + # + # This is equivalent to Meson. + # https://github.com/mesonbuild/meson/blob/master/mesonbuild/linkers.py#L143 + old_arflags = str(env["ARFLAGS"]) + if "s" not in old_arflags: + old_arflags += "s" + + env["ARFLAGS"] = SCons.Util.CLVar([old_arflags]) + + # Disable running ranlib, since we added 's' above + env["RANLIBCOM"] = "" + + +class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): + """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" + + def __call__(self, target, source, env, for_signature): + return self.cmd + + def _print_cmd_str(*_args, **_kwargs): + """Disable this method""" + pass + + +def ninja_always_serial(self, num, taskmaster): + """Replacement for SCons.Job.Jobs constructor which always uses the Serial Job class.""" + # We still set self.num_jobs to num even though it's a lie. The + # only consumer of this attribute is the Parallel Job class AND + # the Main.py function which instantiates a Jobs class. It checks + # if Jobs.num_jobs is equal to options.num_jobs, so if the user + # provides -j12 but we set self.num_jobs = 1 they get an incorrect + # warning about this version of Python not supporting parallel + # builds. So here we lie so the Main.py will not give a false + # warning to users. + self.num_jobs = num + self.job = SCons.Job.Serial(taskmaster) + + +# pylint: disable=too-few-public-methods +class AlwaysExecAction(SCons.Action.FunctionAction): + """Override FunctionAction.__call__ to always execute.""" + + def __call__(self, *args, **kwargs): + kwargs["execute"] = 1 + return super().__call__(*args, **kwargs) \ No newline at end of file diff --git a/SCons/Tool/ninja/Rules.py b/SCons/Tool/ninja/Rules.py index c1c238e1d7..a2f6bc5456 100644 --- a/SCons/Tool/ninja/Rules.py +++ b/SCons/Tool/ninja/Rules.py @@ -21,7 +21,7 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from .Util import get_outputs, get_rule, get_inputs, get_dependencies +from .Utils import get_outputs, get_rule, get_inputs, get_dependencies def _install_action_function(_env, node): diff --git a/SCons/Tool/ninja/Util.py b/SCons/Tool/ninja/Utils.py similarity index 72% rename from SCons/Tool/ninja/Util.py rename to SCons/Tool/ninja/Utils.py index 80d1b16d75..18d54dcdf2 100644 --- a/SCons/Tool/ninja/Util.py +++ b/SCons/Tool/ninja/Utils.py @@ -21,12 +21,12 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import os +import shutil from os.path import join as joinpath import SCons from SCons.Action import get_default_ENV, _string_from_cmd_list from SCons.Script import AddOption -from SCons.Tool.ninja.Globals import __NINJA_RULE_MAPPING from SCons.Util import is_List, flatten_sequence @@ -249,99 +249,6 @@ def ninja_noop(*_args, **_kwargs): return None -def get_command(env, node, action): # pylint: disable=too-many-branches - """Get the command to execute for node.""" - if node.env: - sub_env = node.env - else: - sub_env = env - executor = node.get_executor() - tlist, slist = get_targets_sources(node) - - # Generate a real CommandAction - if isinstance(action, SCons.Action.CommandGeneratorAction): - # pylint: disable=protected-access - action = action._generate(tlist, slist, sub_env, 1, executor=executor) - - variables = {} - - comstr = get_comstr(sub_env, action, tlist, slist) - if not comstr: - return None - - provider = __NINJA_RULE_MAPPING.get(comstr, get_generic_shell_command) - rule, variables, provider_deps = provider(sub_env, node, action, tlist, slist, executor=executor) - - # Get the dependencies for all targets - implicit = list({dep for tgt in tlist for dep in get_dependencies(tgt)}) - - # Now add in the other dependencies related to the command, - # e.g. the compiler binary. The ninja rule can be user provided so - # we must do some validation to resolve the dependency path for ninja. - for provider_dep in provider_deps: - - provider_dep = sub_env.subst(provider_dep) - if not provider_dep: - continue - - # If the tool is a node, then SCons will resolve the path later, if its not - # a node then we assume it generated from build and make sure it is existing. - if isinstance(provider_dep, SCons.Node.Node) or os.path.exists(provider_dep): - implicit.append(provider_dep) - continue - - # in some case the tool could be in the local directory and be suppled without the ext - # such as in windows, so append the executable suffix and check. - prog_suffix = sub_env.get('PROGSUFFIX', '') - provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix - if os.path.exists(provider_dep_ext): - implicit.append(provider_dep_ext) - continue - - # Many commands will assume the binary is in the path, so - # we accept this as a possible input from a given command. - - provider_dep_abspath = sub_env.WhereIs(provider_dep) or sub_env.WhereIs(provider_dep, path=os.environ["PATH"]) - if provider_dep_abspath: - implicit.append(provider_dep_abspath) - continue - - # Possibly these could be ignore and the build would still work, however it may not always - # rebuild correctly, so we hard stop, and force the user to fix the issue with the provided - # ninja rule. - raise Exception("Could not resolve path for %s dependency on node '%s'" % (provider_dep, node)) - - ninja_build = { - "order_only": get_order_only(node), - "outputs": get_outputs(node), - "inputs": get_inputs(node), - "implicit": implicit, - "rule": get_rule(node, rule), - "variables": variables, - } - - # Don't use sub_env here because we require that NINJA_POOL be set - # on a per-builder call basis to prevent accidental strange - # behavior like env['NINJA_POOL'] = 'console' and sub_env can be - # the global Environment object if node.env is None. - # Example: - # - # Allowed: - # - # env.Command("ls", NINJA_POOL="ls_pool") - # - # Not allowed and ignored: - # - # env["NINJA_POOL"] = "ls_pool" - # env.Command("ls") - # - # TODO: Why not alloe env['NINJA_POOL'] ? (bdbaddog) - if node.env and node.env.get("NINJA_POOL", None) is not None: - ninja_build["pool"] = node.env["NINJA_POOL"] - - return ninja_build - - def get_command_env(env): """ Return a string that sets the environment for any environment variables that @@ -413,25 +320,6 @@ def get_comstr(env, action, targets, sources): return action.genstring(targets, sources, env) -def get_generic_shell_command(env, node, action, targets, sources, executor=None): - return ( - "CMD", - { - # TODO: Why is executor passed in and then ignored below? (bdbaddog) - "cmd": generate_command(env, node, action, targets, sources, executor=None), - "env": get_command_env(env), - }, - # Since this function is a rule mapping provider, it must return a list of dependencies, - # and usually this would be the path to a tool, such as a compiler, used for this rule. - # However this function is to generic to be able to reliably extract such deps - # from the command, so we return a placeholder empty list. It should be noted that - # generally this function will not be used solely and is more like a template to generate - # the basics for a custom provider which may have more specific options for a provider - # function for a custom NinjaRuleMapping. - [] - ) - - def generate_command(env, node, action, targets, sources, executor=None): # Actions like CommandAction have a method called process that is # used by SCons to generate the cmd_line they need to run. So @@ -454,4 +342,79 @@ def generate_command(env, node, action, targets, sources, executor=None): cmd = cmd[0:-2].strip() # Escape dollars as necessary - return cmd.replace("$", "$$") \ No newline at end of file + return cmd.replace("$", "$$") + + +def ninja_csig(original): + """Return a dummy csig""" + + def wrapper(self): + if isinstance(self, SCons.Node.Node) and self.is_sconscript(): + return original(self) + return "dummy_ninja_csig" + + return wrapper + + +def ninja_contents(original): + """Return a dummy content without doing IO""" + + def wrapper(self): + if isinstance(self, SCons.Node.Node) and self.is_sconscript(): + return original(self) + return bytes("dummy_ninja_contents", encoding="utf-8") + + return wrapper + + +def ninja_stat(_self, path): + """ + Eternally memoized stat call. + + SCons is very aggressive about clearing out cached values. For our + purposes everything should only ever call stat once since we're + running in a no_exec build the file system state should not + change. For these reasons we patch SCons.Node.FS.LocalFS.stat to + use our eternal memoized dictionary. + """ + + try: + return SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] + except KeyError: + try: + result = os.stat(path) + except os.error: + result = None + + SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] = result + return result + + +def ninja_whereis(thing, *_args, **_kwargs): + """Replace env.WhereIs with a much faster version""" + + # Optimize for success, this gets called significantly more often + # when the value is already memoized than when it's not. + try: + return SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] + except KeyError: + # TODO: Fix this to respect env['ENV']['PATH']... WPD + # We do not honor any env['ENV'] or env[*] variables in the + # generated ninja file. Ninja passes your raw shell environment + # down to it's subprocess so the only sane option is to do the + # same during generation. At some point, if and when we try to + # upstream this, I'm sure a sticking point will be respecting + # env['ENV'] variables and such but it's actually quite + # complicated. I have a naive version but making it always work + # with shell quoting is nigh impossible. So I've decided to + # cross that bridge when it's absolutely required. + path = shutil.which(thing) + SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] = path + return path + + +def ninja_print_conf_log(s, target, source, env): + """Command line print only for conftest to generate a correct conf log.""" + if target and target[0].is_conftest(): + action = SCons.Action._ActionAction() + action.print_cmd_line(s, target, source, env) \ No newline at end of file diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index fd9c13c7e3..ee852a704a 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -26,85 +26,25 @@ import importlib import os -import shlex -import shutil import subprocess import sys -import textwrap -from glob import glob import SCons import SCons.Tool.ninja.Globals from SCons.Script import GetOption from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS +from .Methods import register_custom_handler, register_custom_rule_mapping, register_custom_rule, register_custom_pool, \ + set_build_node_callback, get_generic_shell_command, CheckNinjaCompdbExpand, get_command, \ + gen_get_response_file_command from .NinjaState import NinjaState -from .Util import ninja_add_command_line_options, \ - get_path, ninja_noop, get_command, get_command_env, get_comstr, get_generic_shell_command, \ - generate_command +from .Overrides import ninja_hack_linkcom, ninja_hack_arcom, NinjaNoResponseFiles, ninja_always_serial, AlwaysExecAction +from .Utils import ninja_add_command_line_options, \ + get_path, ninja_noop, ninja_print_conf_log, get_command_env, get_comstr, generate_command, ninja_csig, ninja_contents, ninja_stat, ninja_whereis, ninja_csig, ninja_contents NINJA_STATE = None -def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom_env={}): - """Generate a response file command provider for rule name.""" - - # If win32 using the environment with a response file command will cause - # ninja to fail to create the response file. Additionally since these rules - # generally are not piping through cmd.exe /c any environment variables will - # make CreateProcess fail to start. - # - # On POSIX we can still set environment variables even for compile - # commands so we do so. - use_command_env = not env["PLATFORM"] == "win32" - if "$" in tool: - tool_is_dynamic = True - - def get_response_file_command(env, node, action, targets, sources, executor=None): - if hasattr(action, "process"): - cmd_list, _, _ = action.process(targets, sources, env, executor=executor) - cmd_list = [str(c).replace("$", "$$") for c in cmd_list[0]] - else: - command = generate_command( - env, node, action, targets, sources, executor=executor - ) - cmd_list = shlex.split(command) - - if tool_is_dynamic: - tool_command = env.subst( - tool, target=targets, source=sources, executor=executor - ) - else: - tool_command = tool - - try: - # Add 1 so we always keep the actual tool inside of cmd - tool_idx = cmd_list.index(tool_command) + 1 - except ValueError: - raise Exception( - "Could not find tool {} in {} generated from {}".format( - tool, cmd_list, get_comstr(env, action, targets, sources) - ) - ) - - cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] - rsp_content = ['"' + rsp_content_item + '"' for rsp_content_item in rsp_content] - rsp_content = " ".join(rsp_content) - - variables = {"rspc": rsp_content} - variables[rule] = cmd - if use_command_env: - variables["env"] = get_command_env(env) - - for key, value in custom_env.items(): - variables["env"] += env.subst( - "export %s=%s;" % (key, value), target=targets, source=sources, executor=executor - ) + " " - return rule, variables, [tool_command] - - return get_response_file_command - - def ninja_builder(env, target, source): """Generate a build.ninja for source.""" if not isinstance(source, list): @@ -120,6 +60,7 @@ def ninja_builder(env, target, source): NINJA_STATE.generate() if env["PLATFORM"] == "win32": + # TODO: Is this necessary as you set env variable in the ninja build file per target? # this is not great, its doesn't consider specific # node environments, which means on linux the build could # behave differently, because on linux you can set the environment @@ -170,193 +111,6 @@ def execute_ninja(): # prone to failure with such a simple check erase_previous = output.startswith('[') -# pylint: disable=too-few-public-methods -class AlwaysExecAction(SCons.Action.FunctionAction): - """Override FunctionAction.__call__ to always execute.""" - - def __call__(self, *args, **kwargs): - kwargs["execute"] = 1 - return super().__call__(*args, **kwargs) - - -def register_custom_handler(env, name, handler): - """Register a custom handler for SCons function actions.""" - env[NINJA_CUSTOM_HANDLERS][name] = handler - - -def register_custom_rule_mapping(env, pre_subst_string, rule): - """Register a function to call for a given rule.""" - SCons.Tool.ninja.Globals.__NINJA_RULE_MAPPING[pre_subst_string] = rule - - -def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): - """Allows specification of Ninja rules from inside SCons files.""" - rule_obj = { - "command": command, - "description": description if description else "{} $out".format(rule), - } - - if use_depfile: - rule_obj["depfile"] = os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile') - - if deps is not None: - rule_obj["deps"] = deps - - if pool is not None: - rule_obj["pool"] = pool - - if use_response_file: - rule_obj["rspfile"] = "$out.rsp" - rule_obj["rspfile_content"] = response_file_content - - env[NINJA_RULES][rule] = rule_obj - - -def register_custom_pool(env, pool, size): - """Allows the creation of custom Ninja pools""" - env[NINJA_POOLS][pool] = size - - -def set_build_node_callback(env, node, callback): - if not node.is_conftest(): - node.attributes.ninja_build_callback = callback - - -def ninja_csig(original): - """Return a dummy csig""" - - def wrapper(self): - if isinstance(self, SCons.Node.Node) and self.is_sconscript(): - return original(self) - return "dummy_ninja_csig" - - return wrapper - - -def ninja_contents(original): - """Return a dummy content without doing IO""" - - def wrapper(self): - if isinstance(self, SCons.Node.Node) and self.is_sconscript(): - return original(self) - return bytes("dummy_ninja_contents", encoding="utf-8") - - return wrapper - - -def CheckNinjaCompdbExpand(env, context): - """ Configure check testing if ninja's compdb can expand response files""" - - context.Message('Checking if ninja compdb can expand response files... ') - ret, output = context.TryAction( - action='ninja -f $SOURCE -t compdb -x CMD_RSP > $TARGET', - extension='.ninja', - text=textwrap.dedent(""" - rule CMD_RSP - command = $cmd @$out.rsp > fake_output.txt - description = Building $out - rspfile = $out.rsp - rspfile_content = $rspc - build fake_output.txt: CMD_RSP fake_input.txt - cmd = echo - pool = console - rspc = "test" - """)) - result = '@fake_output.txt.rsp' not in output - context.Result(result) - return result - - -def ninja_stat(_self, path): - """ - Eternally memoized stat call. - - SCons is very aggressive about clearing out cached values. For our - purposes everything should only ever call stat once since we're - running in a no_exec build the file system state should not - change. For these reasons we patch SCons.Node.FS.LocalFS.stat to - use our eternal memoized dictionary. - """ - - try: - return SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] - except KeyError: - try: - result = os.stat(path) - except os.error: - result = None - - SCons.Tool.ninja.Globals.NINJA_STAT_MEMO[path] = result - return result - - -def ninja_whereis(thing, *_args, **_kwargs): - """Replace env.WhereIs with a much faster version""" - - # Optimize for success, this gets called significantly more often - # when the value is already memoized than when it's not. - try: - return SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] - except KeyError: - # We do not honor any env['ENV'] or env[*] variables in the - # generated ninja file. Ninja passes your raw shell environment - # down to it's subprocess so the only sane option is to do the - # same during generation. At some point, if and when we try to - # upstream this, I'm sure a sticking point will be respecting - # env['ENV'] variables and such but it's actually quite - # complicated. I have a naive version but making it always work - # with shell quoting is nigh impossible. So I've decided to - # cross that bridge when it's absolutely required. - path = shutil.which(thing) - SCons.Tool.ninja.Globals.NINJA_WHEREIS_MEMO[thing] = path - return path - - -def ninja_always_serial(self, num, taskmaster): - """Replacement for SCons.Job.Jobs constructor which always uses the Serial Job class.""" - # We still set self.num_jobs to num even though it's a lie. The - # only consumer of this attribute is the Parallel Job class AND - # the Main.py function which instantiates a Jobs class. It checks - # if Jobs.num_jobs is equal to options.num_jobs, so if the user - # provides -j12 but we set self.num_jobs = 1 they get an incorrect - # warning about this version of Python not supporting parallel - # builds. So here we lie so the Main.py will not give a false - # warning to users. - self.num_jobs = num - self.job = SCons.Job.Serial(taskmaster) - -def ninja_hack_linkcom(env): - # TODO: change LINKCOM and SHLINKCOM to handle embedding manifest exe checks - # without relying on the SCons hacks that SCons uses by default. - if env["PLATFORM"] == "win32": - from SCons.Tool.mslink import compositeLinkAction - - if env.get("LINKCOM", None) == compositeLinkAction: - env[ - "LINKCOM" - ] = '${TEMPFILE("$LINK $LINKFLAGS /OUT:$TARGET.windows $_LIBDIRFLAGS $_LIBFLAGS $_PDB $SOURCES.windows", "$LINKCOMSTR")}' - env[ - "SHLINKCOM" - ] = '${TEMPFILE("$SHLINK $SHLINKFLAGS $_SHLINK_TARGETS $_LIBDIRFLAGS $_LIBFLAGS $_PDB $_SHLINK_SOURCES", "$SHLINKCOMSTR")}' - - -def ninja_print_conf_log(s, target, source, env): - """Command line print only for conftest to generate a correct conf log.""" - if target and target[0].is_conftest(): - action = SCons.Action._ActionAction() - action.print_cmd_line(s, target, source, env) - - -class NinjaNoResponseFiles(SCons.Platform.TempFileMunge): - """Overwrite the __call__ method of SCons' TempFileMunge to not delete.""" - - def __call__(self, target, source, env, for_signature): - return self.cmd - - def _print_cmd_str(*_args, **_kwargs): - """Disable this method""" - pass - def exists(env): """Enable if called.""" @@ -529,20 +283,8 @@ def robust_rule_mapping(var, rule, tool): # TODO: switch to using SCons to help determine this (Github Issue #3624) env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] - if env["PLATFORM"] != "win32" and env.get("RANLIBCOM"): - # There is no way to translate the ranlib list action into - # Ninja so add the s flag and disable ranlib. - # - # This is equivalent to Meson. - # https://github.com/mesonbuild/meson/blob/master/mesonbuild/linkers.py#L143 - old_arflags = str(env["ARFLAGS"]) - if "s" not in old_arflags: - old_arflags += "s" - - env["ARFLAGS"] = SCons.Util.CLVar([old_arflags]) - - # Disable running ranlib, since we added 's' above - env["RANLIBCOM"] = "" + # Force ARCOM so use 's' flag on ar instead of separately running ranlib + ninja_hack_arcom(env) if GetOption('disable_ninja'): return env From 764ddca95223d7915ac1406bcd1fed8c0e3078b0 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 24 May 2021 16:11:14 -0700 Subject: [PATCH 151/163] Fix sider complaints --- SCons/Tool/ninja/Overrides.py | 2 +- SCons/Tool/ninja/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/ninja/Overrides.py b/SCons/Tool/ninja/Overrides.py index 80516a26ea..c6f12074b7 100644 --- a/SCons/Tool/ninja/Overrides.py +++ b/SCons/Tool/ninja/Overrides.py @@ -21,7 +21,7 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -This module is to hold logic which overrides default SCons behavoirs to enable +This module is to hold logic which overrides default SCons behaviors to enable ninja file generation """ import SCons diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index ee852a704a..8207b15860 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -40,7 +40,7 @@ from .NinjaState import NinjaState from .Overrides import ninja_hack_linkcom, ninja_hack_arcom, NinjaNoResponseFiles, ninja_always_serial, AlwaysExecAction from .Utils import ninja_add_command_line_options, \ - get_path, ninja_noop, ninja_print_conf_log, get_command_env, get_comstr, generate_command, ninja_csig, ninja_contents, ninja_stat, ninja_whereis, ninja_csig, ninja_contents + ninja_noop, ninja_print_conf_log, ninja_csig, ninja_contents, ninja_stat, ninja_whereis NINJA_STATE = None From 208eb11f5e0a6e976258e8430692dd53b849775f Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 30 May 2021 14:42:27 -0700 Subject: [PATCH 152/163] Address mwichmann's comments on PR. Mostly doc and a few import related code changes --- SCons/Tool/ninja/NinjaState.py | 6 +- SCons/Tool/ninja/ninja.xml | 5 +- SCons/__init__.py | 6 +- doc/generated/builders.gen | 4 +- .../examples/caching_ex-random_1.xml | 2 +- .../examples/troubleshoot_Dump_1.xml | 4 +- .../examples/troubleshoot_Dump_2.xml | 4 +- .../examples/troubleshoot_explain1_3.xml | 2 +- .../examples/troubleshoot_stacktrace_2.xml | 4 +- doc/generated/functions.gen | 764 +++++++++--------- doc/generated/tools.gen | 2 +- doc/user/external.xml | 521 ++++++------ 12 files changed, 693 insertions(+), 631 deletions(-) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index f906b33fb7..14704eb10a 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -31,13 +31,13 @@ import SCons from SCons.Script import COMMAND_LINE_TARGETS from SCons.Util import is_List -import SCons.Tool.ninja.Globals +from SCons.Errors import InternalError from .Globals import COMMAND_TYPES, NINJA_RULES, NINJA_POOLS, \ NINJA_CUSTOM_HANDLERS from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function from .Utils import get_path, alias_to_ninja_build, generate_depfile, ninja_noop, get_order_only, \ get_outputs, get_inputs, get_dependencies, get_rule, get_command_env -from . import get_command +from .Methods import get_command # pylint: disable=too-many-instance-attributes @@ -259,7 +259,7 @@ def add_build(self, node): node_string = str(node) if node_string in self.builds: - raise Exception("Node {} added to ninja build state more than once".format(node_string)) + raise InternalError("Node {} added to ninja build state more than once".format(node_string)) self.builds[node_string] = build self.built.update(build["outputs"]) return True diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index c28d697953..2ea9fd67ac 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -74,6 +74,7 @@ See its __doc__ string for a discussion of the format. CCCOM CXXCOM SHCXXCOM + SHCCCOM CC CXX @@ -130,9 +131,7 @@ See its __doc__ string for a discussion of the format. - You must load the &t-ninja; tool prior to specifying - any part of your build or some source/output - files will not show up in the compilation database. + You must specify scons --experimental=ninja to enable this tool. To use this tool you must install pypi's ninja diff --git a/SCons/__init__.py b/SCons/__init__.py index bc0f428f53..cd521d1f1c 100644 --- a/SCons/__init__.py +++ b/SCons/__init__.py @@ -1,9 +1,9 @@ __version__="4.1.1a" __copyright__="Copyright (c) 2001 - 2021 The SCons Foundation" __developer__="bdbaddog" -__date__="2021-04-18 22:22:37" +__date__="2021-05-30 21:33:41" __buildsys__="ProDog2020" -__revision__="1850bdd13c7668aac493b9f5b896578813e60ca9" -__build__="1850bdd13c7668aac493b9f5b896578813e60ca9" +__revision__="b4c7a0966c36d37d0c2ea125dd98bff0061e28f2" +__build__="b4c7a0966c36d37d0c2ea125dd98bff0061e28f2" # make sure compatibility is always in place import SCons.compat # noqa \ No newline at end of file diff --git a/doc/generated/builders.gen b/doc/generated/builders.gen index d3490f0ca0..ba90e6cfbd 100644 --- a/doc/generated/builders.gen +++ b/doc/generated/builders.gen @@ -1258,9 +1258,7 @@ env.MSVSSolution( - You must load the &t-ninja; tool prior to specifying - any part of your build or some source/output - files will not show up in the compilation database. + You must specify scons --experimental=ninja to enable this tool. To use this tool you must install pypi's ninja diff --git a/doc/generated/examples/caching_ex-random_1.xml b/doc/generated/examples/caching_ex-random_1.xml index a5593d302d..1570d7df30 100644 --- a/doc/generated/examples/caching_ex-random_1.xml +++ b/doc/generated/examples/caching_ex-random_1.xml @@ -1,8 +1,8 @@ % scons -Q +cc -o f4.o -c f4.c cc -o f2.o -c f2.c cc -o f1.o -c f1.c cc -o f3.o -c f3.c -cc -o f4.o -c f4.c cc -o f5.o -c f5.c cc -o prog f1.o f2.o f3.o f4.o f5.o diff --git a/doc/generated/examples/troubleshoot_Dump_1.xml b/doc/generated/examples/troubleshoot_Dump_1.xml index 8f4e624d94..2b4178bd8d 100644 --- a/doc/generated/examples/troubleshoot_Dump_1.xml +++ b/doc/generated/examples/troubleshoot_Dump_1.xml @@ -62,8 +62,8 @@ scons: Reading SConscript files ... 'TEMPFILEARGJOIN': ' ', 'TEMPFILEPREFIX': '@', 'TOOLS': ['install', 'install'], - '_CPPDEFFLAGS': '${_defines(CPPDEFPREFIX, CPPDEFINES, CPPDEFSUFFIX, ' - '__env__)}', + '_CPPDEFFLAGS': '${_defines(CPPDEFPREFIX, CPPDEFINES, CPPDEFSUFFIX, __env__, ' + 'TARGET, SOURCE)}', '_CPPINCFLAGS': '$( ${_concat(INCPREFIX, CPPPATH, INCSUFFIX, __env__, RDirs, ' 'TARGET, SOURCE)} $)', '_LIBDIRFLAGS': '$( ${_concat(LIBDIRPREFIX, LIBPATH, LIBDIRSUFFIX, __env__, ' diff --git a/doc/generated/examples/troubleshoot_Dump_2.xml b/doc/generated/examples/troubleshoot_Dump_2.xml index f2fc5e6f6f..c2fced13c1 100644 --- a/doc/generated/examples/troubleshoot_Dump_2.xml +++ b/doc/generated/examples/troubleshoot_Dump_2.xml @@ -107,8 +107,8 @@ scons: Reading SConscript files ... 'TOOLS': ['msvc', 'install', 'install'], 'VSWHERE': None, '_CCCOMCOM': '$CPPFLAGS $_CPPDEFFLAGS $_CPPINCFLAGS $CCPCHFLAGS $CCPDBFLAGS', - '_CPPDEFFLAGS': '${_defines(CPPDEFPREFIX, CPPDEFINES, CPPDEFSUFFIX, ' - '__env__)}', + '_CPPDEFFLAGS': '${_defines(CPPDEFPREFIX, CPPDEFINES, CPPDEFSUFFIX, __env__, ' + 'TARGET, SOURCE)}', '_CPPINCFLAGS': '$( ${_concat(INCPREFIX, CPPPATH, INCSUFFIX, __env__, RDirs, ' 'TARGET, SOURCE)} $)', '_LIBDIRFLAGS': '$( ${_concat(LIBDIRPREFIX, LIBPATH, LIBDIRSUFFIX, __env__, ' diff --git a/doc/generated/examples/troubleshoot_explain1_3.xml b/doc/generated/examples/troubleshoot_explain1_3.xml index 6b90e1cabe..14a105e048 100644 --- a/doc/generated/examples/troubleshoot_explain1_3.xml +++ b/doc/generated/examples/troubleshoot_explain1_3.xml @@ -2,5 +2,5 @@ cp file.in file.oout scons: warning: Cannot find target file.out after building -File "/Users/bdbaddog/devel/scons/git/as_scons/scripts/scons.py", line 96, in <module> +File "/Users/bdbaddog/devel/scons/git/scons-bugfixes-4/scripts/scons.py", line 96, in <module> diff --git a/doc/generated/examples/troubleshoot_stacktrace_2.xml b/doc/generated/examples/troubleshoot_stacktrace_2.xml index 8e8809c461..75dd6fba03 100644 --- a/doc/generated/examples/troubleshoot_stacktrace_2.xml +++ b/doc/generated/examples/troubleshoot_stacktrace_2.xml @@ -3,10 +3,10 @@ scons: *** [prog.o] Source `prog.c' not found, needed by target `prog.o'. scons: internal stack trace: File "SCons/Job.py", line 195, in start task.prepare() - File "SCons/Script/Main.py", line 179, in prepare + File "SCons/Script/Main.py", line 178, in prepare return SCons.Taskmaster.OutOfDateTask.prepare(self) File "SCons/Taskmaster.py", line 186, in prepare executor.prepare() - File "SCons/Executor.py", line 424, in prepare + File "SCons/Executor.py", line 418, in prepare raise SCons.Errors.StopError(msg % (s, self.batches[0].targets[0])) diff --git a/doc/generated/functions.gen b/doc/generated/functions.gen index cd71ee9a8c..53c35837a7 100644 --- a/doc/generated/functions.gen +++ b/doc/generated/functions.gen @@ -13,8 +13,8 @@ - Action(action, [cmd/str/fun, [var, ...]] [option=value, ...]) - env.Action(action, [cmd/str/fun, [var, ...]] [option=value, ...]) + Action(action, [output, [var, ...]] [key=value, ...]) + env.Action(action, [output, [var, ...]] [key=value, ...]) A factory function to create an Action object for the specified @@ -2124,261 +2124,242 @@ file is found. env.GetOption(name) This function provides a way to query the value of -SCons options set on scons command line -(or set using the -&f-link-SetOption; -function). -The options supported are: +options which can be set via the command line or using the +&f-link-SetOption; function. - - - - -cache_debug - - -which corresponds to ; - - - - -cache_disable - - -which corresponds to ; - - - - -cache_force - - -which corresponds to ; - - - - -cache_show - - -which corresponds to ; - - - - -clean - - -which corresponds to , -and ; - - - - -config - - -which corresponds to ; - - - - -directory - - -which corresponds to and ; - - - - -diskcheck - - -which corresponds to ; - - - - -duplicate - - -which corresponds to ; - - - - -file - - -which corresponds to , , and ; - - - - -help - - -which corresponds to and ; - - - - -ignore_errors - - -which corresponds to ; - - - - -implicit_cache - - -which corresponds to ; - - - - -implicit_deps_changed - - -which corresponds to ; - - - - -implicit_deps_unchanged - - -which corresponds to ; - - - - -interactive - - -which corresponds to and ; - - - - -keep_going - - -which corresponds to and ; - - - - -max_drift - - -which corresponds to ; - - - - -no_exec - - -which corresponds to , -, , - and ; - - - - -no_site_dir - - -which corresponds to ; - - - - -num_jobs - - -which corresponds to and ; - - - - -profile_file - - -which corresponds to ; - - - - -question - - -which corresponds to and ; - - - - -random - - -which corresponds to ; - - - - -repository - - -which corresponds to , and ; - - - - -silent - - -which corresponds to , and ; - - - - -site_dir - - -which corresponds to ; - - - - -stack_size - - -which corresponds to ; - - - - -taskmastertrace_file - -which corresponds to ; and - - - - -warn - - -which corresponds to and . - - - - +name can be an entry from the following table, +which shows the corresponding command line arguments +that could affect the value. +name can be also be the destination +variable name from a project-specific option added using the +&f-link-AddOption; function, as long as the addition +happens prior to the &f-GetOption; call in the SConscript files. + + + + + Query name + Command-line options + Notes + + + + + + cache_debug + + + + cache_disable + + , + + + + + cache_force + + , + + + + + cache_readonly + + + + cache_show + + + + clean + + , + , + + + + + climb_up + + + + + + + + + + config + + + + debug + + + + directory + , + + + diskcheck + + + + duplicate + + + + enable_virtualenv + + + + experimental + + since 4.2 + + + file + + , + , + , + + + + + hash_format + + since 4.2 + + + help + , + + + ignore_errors + , + + + ignore_virtualenv + + + + implicit_cache + + + + implicit_deps_changed + + + + implicit_deps_unchanged + + + + include_dir + , + + + install_sandbox + + Available only if the &t-link-install; tool has been called + + + keep_going + , + + + max_drift + + + + md5_chunksize + + , + + + since 4.2 + + + no_exec + + , + , + , + , + + + + + no_progress + + + + num_jobs + , + + + package_type + + Available only if the &t-link-packaging; tool has been called + + + profile_file + + + + question + , + + + random + + + + repository + + , + , + + + + + silent + + , + , + + + + + site_dir + , + + + stack_size + + + + taskmastertrace_file + + + + tree_printers + + + + warn + , + + + + + + See the documentation for the corresponding command line option for information about each specific @@ -3879,15 +3860,33 @@ if 'FOO' not in env: Sets &scons; option variable name to value. These options are all also settable via -&scons; command-line options but the variable name -may differ from the command-line option name (see table). +command-line options but the variable name +may differ from the command-line option name - +see the table for correspondences. A value set via command-line option will take precedence over one set with &f-SetOption;, which allows setting a project default in the scripts and temporarily overriding it via command line. +&f-SetOption; calls can also be placed in the +site_init.py file. + + + +See the documentation in the manpage for the +corresponding command line option for information about each specific option. +The value parameter is mandatory, +for option values which are boolean in nature +(that is, the command line option does not take an argument) +use a value +which evaluates to true (e.g. True, +1) or false (e.g. False, +0). + + + Options which affect the reading and processing of SConscript files -are not settable this way, since those files must -be read in order to find the &f-SetOption; call. +are not settable using &f-SetOption; since those files must +be read in order to find the &f-SetOption; call in the first place. @@ -3895,126 +3894,161 @@ The settable variables with their associated command-line options are: - + -VariableCommand-line options + + Settable name + Command-line options + Notes + + - -clean - -, , - - -diskcheck - - - - - -duplicate - - - - - - - - experimental - - - - - - - - -help - -, - - -implicit_cache - - - - - -max_drift - - - - -md5_chunksize - - - - -no_exec - -, , -, , - - - -no_progress - - - - -num_jobs - -, - - -random - - - - -silent - -. - - -stack_size - - - - -warn - -. - + + clean + + , + , + + + + + + diskcheck + + + + + duplicate + + + + + experimental + + since 4.2 + + + + hash_chunksize + + + Actually sets md5_chunksize. + since 4.2 + + + + + hash_format + + since 4.2 + + + + help + , + + + + implicit_cache + + + + + implicit_deps_changed + + + Also sets implicit_cache. + (settable since 4.2) + + + + + implicit_deps_unchanged + + + Also sets implicit_cache. + (settable since 4.2) + + + + + max_drift + + + + + md5_chunksize + + + + + no_exec + + , + , + , + , + + + + + + no_progress + + See + + If no_progress is set via &f-SetOption; + in an SConscript file + (but not if set in a site_init.py file) + there will still be an initial status message about + reading SConscript files since &SCons; has + to start reading them before it can see the + &f-SetOption;. + + + + + + + num_jobs + , + + + + random + + + + + silent + + , + , + + + + + + stack_size + + + + + warn + + + - -See the documentation in the manpage for the -corresponding command line option for information about each specific option. -Option values which are boolean in nature (that is, they are -either on or off) should be set to a true value (True, -1) or a false value (False, -0). - - - - -If no_progress is set via &f-SetOption; -there will still be initial progress output as &SCons; has -to start reading SConscript files before it can see the -&f-SetOption; in an SConscript file: -scons: Reading SConscript files ... - - - Example: -SetOption('max_drift', True) +SetOption('max_drift', 0) diff --git a/doc/generated/tools.gen b/doc/generated/tools.gen index bdb353ce67..01b69f593e 100644 --- a/doc/generated/tools.gen +++ b/doc/generated/tools.gen @@ -816,7 +816,7 @@ Sets construction variables for the Sets up &b-link-Ninja; builder which generates a ninja build file, and then optionally runs ninja. - Sets: &cv-link-DISABLE_AUTO_NINJA;, &cv-link-NINJA_ALIAS_NAME;, &cv-link-NINJA_BUILDDIR;, &cv-link-NINJA_COMPDB_EXPAND;, &cv-link-NINJA_ENV_VAR_CACHE;, &cv-link-NINJA_FILE_NAME;, &cv-link-NINJA_GENERATED_SOURCE_SUFFIXES;, &cv-link-NINJA_MSVC_DEPS_PREFIX;, &cv-link-NINJA_POOL;, &cv-link-NINJA_REGENERATE_DEPS;, &cv-link-NINJA_SYNTAX;, &cv-link-_NINJA_REGENERATE_DEPS_FUNC;, &cv-link-__NINJA_NO;.Uses: &cv-link-AR;, &cv-link-ARCOM;, &cv-link-ARFLAGS;, &cv-link-CC;, &cv-link-CCCOM;, &cv-link-CXX;, &cv-link-CXXCOM;, &cv-link-ESCAPE;, &cv-link-LINK;, &cv-link-LINKCOM;, &cv-link-PLATFORM;, &cv-link-RANLIB;, &cv-link-RANLIBCOM;, &cv-link-SHCXXCOM;, &cv-link-SHLINK;, &cv-link-SHLINKCOM;. + Sets: &cv-link-DISABLE_AUTO_NINJA;, &cv-link-NINJA_ALIAS_NAME;, &cv-link-NINJA_BUILDDIR;, &cv-link-NINJA_COMPDB_EXPAND;, &cv-link-NINJA_ENV_VAR_CACHE;, &cv-link-NINJA_FILE_NAME;, &cv-link-NINJA_GENERATED_SOURCE_SUFFIXES;, &cv-link-NINJA_MSVC_DEPS_PREFIX;, &cv-link-NINJA_POOL;, &cv-link-NINJA_REGENERATE_DEPS;, &cv-link-NINJA_SYNTAX;, &cv-link-_NINJA_REGENERATE_DEPS_FUNC;, &cv-link-__NINJA_NO;.Uses: &cv-link-AR;, &cv-link-ARCOM;, &cv-link-ARFLAGS;, &cv-link-CC;, &cv-link-CCCOM;, &cv-link-CXX;, &cv-link-CXXCOM;, &cv-link-ESCAPE;, &cv-link-LINK;, &cv-link-LINKCOM;, &cv-link-PLATFORM;, &cv-link-RANLIB;, &cv-link-RANLIBCOM;, &cv-link-SHCCCOM;, &cv-link-SHCXXCOM;, &cv-link-SHLINK;, &cv-link-SHLINKCOM;. packaging diff --git a/doc/user/external.xml b/doc/user/external.xml index f580d3ff46..78e0a7d488 100644 --- a/doc/user/external.xml +++ b/doc/user/external.xml @@ -1,290 +1,305 @@ - %scons; + + %scons; - - %builders-mod; - - %functions-mod; - - %tools-mod; - - %variables-mod; + + %builders-mod; + + %functions-mod; + + %tools-mod; + + %variables-mod; -]> + ]> -Using SCons with other build tools + Using SCons with other build tools - + --> - + - Sometimes a project needs to interact with other projects - in various ways. For example, many open source projects - make use of components from other open source projects, - and want to use those in their released form, not recode - their builds into &SCons;. As another example, sometimes - the flexibility and power of &SCons; is useful for managing the - overall project, but developers might like faster incremental - builds when making small changes by using a different tool. + Sometimes a project needs to interact with other projects + in various ways. For example, many open source projects + make use of components from other open source projects, + and want to use those in their released form, not recode + their builds into &SCons;. As another example, sometimes + the flexibility and power of &SCons; is useful for managing the + overall project, but developers might like faster incremental + builds when making small changes by using a different tool. - + - + - This chapter shows some techniques for interacting with other - projects and tools effectively from within &SCons;. + This chapter shows some techniques for interacting with other + projects and tools effectively from within &SCons;. - + -
- Creating a Compilation Database +
+ Creating a Compilation Database - + - Tooling to perform analysis and modification - of source code often needs to know not only the source code - itself, but also how it will be compiled, as the compilation line - affects the behavior of macros, includes, etc. &SCons; has a - record of this information once it has run, in the form of - Actions associated with the sources, and can emit this information - so tools can use it. + Tooling to perform analysis and modification + of source code often needs to know not only the source code + itself, but also how it will be compiled, as the compilation line + affects the behavior of macros, includes, etc. &SCons; has a + record of this information once it has run, in the form of + Actions associated with the sources, and can emit this information + so tools can use it. - + - + - The Clang project has defined a JSON Compilation Database. - This database is in common use as input into Clang tools - and many IDEs and editors as well. - See - - JSON Compilation Database Format Specification - - for complete information. &SCons; can emit a - compilation database in this format - by enabling the &t-link-compilation_db; tool - and calling the &b-link-CompilationDatabase; builder - (available since &scons; 4.0). + The Clang project has defined a JSON Compilation Database. + This database is in common use as input into Clang tools + and many IDEs and editors as well. + See + + JSON Compilation Database Format Specification + + for complete information. &SCons; can emit a + compilation database in this format + by enabling the &t-link-compilation_db; tool + and calling the &b-link-CompilationDatabase; builder + (available since &scons; 4.0). - + - + - The compilation database can be populated with - source and output files either with paths relative - to the top of the build, or using absolute paths. - This is controlled by - COMPILATIONDB_USE_ABSPATH=(True|False) - which defaults to False. - The entries in this file can be filtered by using - - COMPILATIONDB_PATH_FILTER='pattern' - where the filter pattern is a string following the Python - - fnmatch - - syntax. - This filtering can be used for outputting different - build variants to different compilation database files. + The compilation database can be populated with + source and output files either with paths relative + to the top of the build, or using absolute paths. + This is controlled by + COMPILATIONDB_USE_ABSPATH=(True|False) + which defaults to False. + The entries in this file can be filtered by using + + COMPILATIONDB_PATH_FILTER='pattern' + where the filter pattern is a string following the Python + + fnmatch + + syntax. + This filtering can be used for outputting different + build variants to different compilation database files. - + - + - The following example illustrates generating a compilation - database containing absolute paths: + The following example illustrates generating a compilation + database containing absolute paths: - + - - -env = Environment(COMPILATIONDB_USE_ABSPATH=True) -env.Tool('compilation_db') -env.CompilationDatabase() -env.Program('hello.c') - - -int main( int argc, char* argv[] ) -{ - return 0; -} - - - - - scons -Q - - - compile_commands.json contains: - - -[ - { - "command": "gcc -o hello.o -c hello.c", - "directory": "/home/user/sandbox", - "file": "/home/user/sandbox/hello.c", - "output": "/home/user/sandbox/hello.o" - } -] - + + + env = Environment(COMPILATIONDB_USE_ABSPATH=True) + env.Tool('compilation_db') + env.CompilationDatabase() + env.Program('hello.c') + + + int main( int argc, char* argv[] ) + { + return 0; + } + + + + + scons -Q + - + + compile_commands.json + contains: + - Notice that the generated database contains only an entry for - the hello.c/hello.o pairing, - and nothing for the generation of the final executable - hello - the transformation of - hello.o to hello - does not have any information that affects interpretation - of the source code, - so it is not interesting to the compilation database. + + [ + { + "command": "gcc -o hello.o -c hello.c", + "directory": "/home/user/sandbox", + "file": "/home/user/sandbox/hello.c", + "output": "/home/user/sandbox/hello.o" + } + ] + - + - + Notice that the generated database contains only an entry for + the hello.c/hello.o pairing, + and nothing for the generation of the final executable + hello + - the transformation of + hello.o + to + hello + does not have any information that affects interpretation + of the source code, + so it is not interesting to the compilation database. - Although it can be a little surprising at first glance, - a compilation database target is, like any other target, - subject to &scons; target selection rules. - This means if you set a default target (that does not - include the compilation database), or use command-line - targets, it might not be selected for building. - This can actually be an advantage, since you don't - necessarily want to regenerate the compilation database - every build. - The following example - shows selecting relative paths (the default) - for output and source, - and also giving a non-default name to the database. - In order to be able to generate the database separately from building, - an alias is set referring to the database, - which can then be used as a target - here we are only - building the compilation database target, not the code. + - + - - -env = Environment() -env.Tool('compilation_db') -cdb = env.CompilationDatabase('compile_database.json') -Alias('cdb', cdb) -env.Program('test_main.c') - - -#include "test_main.h" -int main( int argc, char* argv[] ) -{ - return 0; -} - - -/* dummy include file */ - - - - - scons -Q cdb - - - compile_database.json contains: - - -[ - { - "command": "gcc -o test_main.o -c test_main.c", - "directory": "/home/user/sandbox", - "file": "test_main.c", - "output": "test_main.o" - } -] - + Although it can be a little surprising at first glance, + a compilation database target is, like any other target, + subject to &scons; target selection rules. + This means if you set a default target (that does not + include the compilation database), or use command-line + targets, it might not be selected for building. + This can actually be an advantage, since you don't + necessarily want to regenerate the compilation database + every build. + The following example + shows selecting relative paths (the default) + for output and source, + and also giving a non-default name to the database. + In order to be able to generate the database separately from building, + an alias is set referring to the database, + which can then be used as a target - here we are only + building the compilation database target, not the code. - + - The following (incomplete) example shows using filtering - to separate build variants. - In the case of using variants, - you want different compilation databases for each, - since the build parameters differ, so the code analysis - needs to see the correct build lines for the 32-bit build - and 64-bit build hinted at here. - For simplicity of presentation, - the example omits the setup details of the variant directories: + + + env = Environment() + env.Tool('compilation_db') + cdb = env.CompilationDatabase('compile_database.json') + Alias('cdb', cdb) + env.Program('test_main.c') + + + #include "test_main.h" + int main( int argc, char* argv[] ) + { + return 0; + } + + + /* dummy include file */ + + + + + scons -Q cdb + - + + compile_database.json + contains: + + + + [ + { + "command": "gcc -o test_main.o -c test_main.c", + "directory": "/home/user/sandbox", + "file": "test_main.c", + "output": "test_main.o" + } + ] + + + + + The following (incomplete) example shows using filtering + to separate build variants. + In the case of using variants, + you want different compilation databases for each, + since the build parameters differ, so the code analysis + needs to see the correct build lines for the 32-bit build + and 64-bit build hinted at here. + For simplicity of presentation, + the example omits the setup details of the variant directories: + + + + + env = Environment() + env.Tool('compilation_db') + + env1 = env.Clone() + env1['COMPILATIONDB_PATH_FILTER'] = 'build/linux32/*' + env1.CompilationDatabase('compile_commands-linux32.json') - -env = Environment() -env.Tool('compilation_db') - -env1 = env.Clone() -env1['COMPILATIONDB_PATH_FILTER'] = 'build/linux32/*' -env1.CompilationDatabase('compile_commands-linux32.json') - -env2 = env.Clone() -env2['COMPILATIONDB_PATH_FILTER'] = 'build/linux64/*' -env2.CompilationDatabase('compile_commands-linux64.json') - - - compile_commands-linux32.json contains: - - -[ - { - "command": "gcc -m32 -o build/linux32/test_main.o -c test_main.c", - "directory": "/home/user/sandbox", - "file": "test_main.c", - "output": "build/linux32/test_main.o" - } -] - - - compile_commands-linux64.json contains: - - -[ - { - "command": "gcc -m64 -o build/linux64/test_main.o -c test_main.c", - "directory": "/home/user/sandbox", - "file": "test_main.c", - "output": "build/linux64/test_main.o" - } -] - - -
+ env2 = env.Clone() + env2['COMPILATIONDB_PATH_FILTER'] = 'build/linux64/*' + env2.CompilationDatabase('compile_commands-linux64.json') + + + + compile_commands-linux32.json + contains: + + + + [ + { + "command": "gcc -m32 -o build/linux32/test_main.o -c test_main.c", + "directory": "/home/user/sandbox", + "file": "test_main.c", + "output": "build/linux32/test_main.o" + } + ] + + + + compile_commands-linux64.json + contains: + + + + [ + { + "command": "gcc -m64 -o build/linux64/test_main.o -c test_main.c", + "directory": "/home/user/sandbox", + "file": "test_main.c", + "output": "build/linux64/test_main.o" + } + ] + + +
Ninja Build Generator @@ -293,16 +308,32 @@ env2.CompilationDatabase('compile_commands-linux64.json') This is an experimental new feature. - Using the + This tool will enabled creating a ninja build file from your SCons based build system. It can then invoke + ninja to run your build. For most builds ninja will be significantly faster, but you may have to give up + some accuracy. You are NOT advised to use this for production builds. It can however significantly speed up + your build/debug/compile iterations. + + + It's not expected that the ninja builder will work for all builds at this point. It's still under active + development. If you find that your build doesn't work with ninja please bring this to the users mailing list + or devel channel on our discord server. + + + Specifically if your build has many (or even any) python function actions you may find that the ninja build + will be slower as it will run ninja, which will then run SCons for each target created by a python action. + To alleviate some of these, especially those python based actions built into SCons there is special logic to + implement those actions via shell commands in the ninja build file. - Ninja Build System + + Ninja Build System + - - Ninja File Format Specification - + + Ninja File Format Specification +
From a20a886e5ab90f32d17e98a67d761bb859bc9e73 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 2 Jun 2021 01:09:05 -0500 Subject: [PATCH 153/163] updates to get working with mongodb build --- SCons/SConf.py | 3 ++- SCons/Tool/ninja/Methods.py | 5 ++--- SCons/Tool/ninja/NinjaState.py | 2 +- SCons/Tool/ninja/Utils.py | 7 +++---- SCons/Tool/ninja/__init__.py | 2 ++ 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/SCons/SConf.py b/SCons/SConf.py index 68e890a69b..38a2b945ac 100644 --- a/SCons/SConf.py +++ b/SCons/SConf.py @@ -606,7 +606,7 @@ def TryBuild(self, builder, text=None, extension=""): f = "_".join([f, textSig, textSigCounter]) textFile = self.confdir.File(f + extension) - self._set_conftest_node(sourcetext) + self._set_conftest_node(textFile) textFileNode = self.env.SConfSourceBuilder(target=textFile, source=sourcetext) nodesToBeBuilt.extend(textFileNode) @@ -625,6 +625,7 @@ def TryBuild(self, builder, text=None, extension=""): pref = self.env.subst( builder.builder.prefix ) suff = self.env.subst( builder.builder.suffix ) target = self.confdir.File(pref + f + suff) + self._set_conftest_node(target) try: # Slide our wrapper into the construction environment as diff --git a/SCons/Tool/ninja/Methods.py b/SCons/Tool/ninja/Methods.py index 3612236ed7..6f24dff95c 100644 --- a/SCons/Tool/ninja/Methods.py +++ b/SCons/Tool/ninja/Methods.py @@ -77,10 +77,9 @@ def set_build_node_callback(env, node, callback): def get_generic_shell_command(env, node, action, targets, sources, executor=None): return ( - "CMD", + "GENERATED_CMD", { - # TODO: Why is executor passed in and then ignored below? (bdbaddog) - "cmd": generate_command(env, node, action, targets, sources, executor=None), + "cmd": generate_command(env, node, action, targets, sources, executor=executor), "env": get_command_env(env), }, # Since this function is a rule mapping provider, it must return a list of dependencies, diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index 14704eb10a..e834d61c95 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -95,7 +95,7 @@ def __init__(self, env, ninja_file, writer_class): [escape(arg) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] ), ), - "SCONS_INVOCATION_W_TARGETS": "{} {} --disable-ninja".format( + "SCONS_INVOCATION_W_TARGETS": "{} {}".format( python_bin, " ".join([escape(arg) for arg in sys.argv]) ), # This must be set to a global default per: diff --git a/SCons/Tool/ninja/Utils.py b/SCons/Tool/ninja/Utils.py index 18d54dcdf2..fba5a636d7 100644 --- a/SCons/Tool/ninja/Utils.py +++ b/SCons/Tool/ninja/Utils.py @@ -64,10 +64,9 @@ def is_valid_dependent_node(node): if isinstance(node, SCons.Node.Alias.Alias): return node.children() - if not node.env: - return True + return not node.get_env().get("NINJA_SKIP") + - return not node.env.get("NINJA_SKIP") def alias_to_ninja_build(node): @@ -88,7 +87,7 @@ def check_invalid_ninja_node(node): def filter_ninja_nodes(node_list): ninja_nodes = [] for node in node_list: - if isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)): + if isinstance(node, (SCons.Node.FS.Base, SCons.Node.Alias.Alias)) and not node.get_env().get('NINJA_SKIP'): ninja_nodes.append(node) else: continue diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 8207b15860..e346e7d449 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -417,6 +417,8 @@ def NinjaBuilderExecute(self, env, target, source, overwarn={}, executor_kw={}): def ninja_execute(self): target = self.targets[0] + if target.get_env().get('NINJA_SKIP'): + return if target.check_attributes('ninja_file') is None or not target.is_conftest: NINJA_STATE.add_build(target) else: From a9bf0d4276d983fb99b0e3357a01d15b8225d1bc Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 2 Jun 2021 01:16:23 -0500 Subject: [PATCH 154/163] update from mongodb for limiting ninja max jobs from commit: 0b620c24c55859f325ce23daa90b3c1c55bc76cb --- SCons/Tool/ninja/NinjaState.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index e834d61c95..1fd6e839ba 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -231,9 +231,10 @@ def __init__(self, env, ninja_file, writer_class): "pool": "local_pool", } + num_jobs = self.env.get('NINJA_MAX_JOBS', self.env.GetOption("num_jobs")) self.pools = { - "local_pool": self.env.GetOption("num_jobs"), - "install_pool": self.env.GetOption("num_jobs") / 2, + "local_pool": num_jobs, + "install_pool": num_jobs / 2, "scons_pool": 1, } @@ -301,12 +302,14 @@ def generate(self): ninja.variable("builddir", get_path(self.env['NINJA_BUILDDIR'])) for pool_name, size in self.pools.items(): - ninja.pool(pool_name, size) + ninja.pool(pool_name, min(self.env.get('NINJA_MAX_JOBS', size), size)) for var, val in self.variables.items(): ninja.variable(var, val) for rule, kwargs in self.rules.items(): + if self.env.get('NINJA_MAX_JOBS') is not None and 'pool' not in kwargs: + kwargs['pool'] = 'local_pool' ninja.rule(rule, **kwargs) generated_source_files = sorted({ From 76aa3ea564236954514341369300efcf5489e6e8 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 6 Jun 2021 21:01:26 -0700 Subject: [PATCH 155/163] resolve sider complaint --- SCons/Tool/ninja/NinjaState.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index 1fd6e839ba..3facce2cb2 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -31,7 +31,7 @@ import SCons from SCons.Script import COMMAND_LINE_TARGETS from SCons.Util import is_List -from SCons.Errors import InternalError +from SCons.Errors import InternalError from .Globals import COMMAND_TYPES, NINJA_RULES, NINJA_POOLS, \ NINJA_CUSTOM_HANDLERS from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function From c80bd2fdd664dd47a96acbe0d03033b28a317529 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 24 Jun 2021 12:38:37 -0700 Subject: [PATCH 156/163] rename NINJA_BUILDDIR -> NINJA_DIR --- SCons/Tool/ninja/Methods.py | 4 +-- SCons/Tool/ninja/NinjaState.py | 4 +-- SCons/Tool/ninja/Utils.py | 2 +- SCons/Tool/ninja/__init__.py | 4 +-- SCons/Tool/ninja/ninja.xml | 65 +++++++++++++++++++++------------- 5 files changed, 47 insertions(+), 32 deletions(-) diff --git a/SCons/Tool/ninja/Methods.py b/SCons/Tool/ninja/Methods.py index 6f24dff95c..ef8e44f6a9 100644 --- a/SCons/Tool/ninja/Methods.py +++ b/SCons/Tool/ninja/Methods.py @@ -50,7 +50,7 @@ def register_custom_rule(env, rule, command, description="", deps=None, pool=Non } if use_depfile: - rule_obj["depfile"] = os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile') + rule_obj["depfile"] = os.path.join(get_path(env['NINJA_DIR']), '$out.depfile') if deps is not None: rule_obj["deps"] = deps @@ -158,7 +158,7 @@ def get_command(env, node, action): # pylint: disable=too-many-branches implicit.append(provider_dep) continue - # in some case the tool could be in the local directory and be suppled without the ext + # in some case the tool could be in the local directory and be supplied without the ext # such as in windows, so append the executable suffix and check. prog_suffix = sub_env.get('PROGSUFFIX', '') provider_dep_ext = provider_dep if provider_dep.endswith(prog_suffix) else provider_dep + prog_suffix diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index 3facce2cb2..e7d9882a1b 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -210,7 +210,7 @@ def __init__(self, env, ninja_file, writer_class): "command": "$SCONS_INVOCATION_W_TARGETS", "description": "Regenerating $out", "generator": 1, - "depfile": os.path.join(get_path(env['NINJA_BUILDDIR']), '$out.depfile'), + "depfile": os.path.join(get_path(env['NINJA_DIR']), '$out.depfile'), # Console pool restricts to 1 job running at a time, # it additionally has some special handling about # passing stdin, stdout, etc to process in this pool @@ -299,7 +299,7 @@ def generate(self): ninja.comment("Generated by scons. DO NOT EDIT.") - ninja.variable("builddir", get_path(self.env['NINJA_BUILDDIR'])) + ninja.variable("builddir", get_path(self.env['NINJA_DIR'])) for pool_name, size in self.pools.items(): ninja.pool(pool_name, min(self.env.get('NINJA_MAX_JOBS', size), size)) diff --git a/SCons/Tool/ninja/Utils.py b/SCons/Tool/ninja/Utils.py index fba5a636d7..3bdefe8207 100644 --- a/SCons/Tool/ninja/Utils.py +++ b/SCons/Tool/ninja/Utils.py @@ -209,7 +209,7 @@ def generate_depfile(env, node, dependencies): dependencies arg can be a list or a subst generator which returns a list. """ - depfile = os.path.join(get_path(env['NINJA_BUILDDIR']), str(node) + '.depfile') + depfile = os.path.join(get_path(env['NINJA_DIR']), str(node) + '.depfile') # subst_list will take in either a raw list or a subst callable which generates # a list, and return a list of CmdStringHolders which can be converted into raw strings. diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index e346e7d449..73152a2653 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -184,7 +184,7 @@ def generate(env): env.Append(BUILDERS={"Ninja": ninja_builder_obj}) env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") - env['NINJA_BUILDDIR'] = env.get("NINJA_BUILDDIR", env.Dir(".ninja").path) + env['NINJA_DIR'] = env.get("NINJA_DIR", env.Dir(".ninja").path) # here we allow multiple environments to construct rules and builds # into the same ninja file @@ -437,7 +437,7 @@ def ninja_execute(self): # Set all three environment variables that Python's # tempfile.mkstemp looks at as it behaves differently on different # platforms and versions of Python. - build_dir = env.subst("$BUILD_DIR") + build_dir = env.subst("$NINJA_DIR") if build_dir == "": build_dir = "." os.environ["TMPDIR"] = env.Dir("{}/.response_files".format(build_dir)).get_abspath() diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index 2ea9fd67ac..94d40406f9 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -52,45 +52,60 @@ See its __doc__ string for a discussion of the format. - NINJA_GENERATED_SOURCE_SUFFIXES - NINJA_MSVC_DEPS_PREFIX - NINJA_BUILDDIR - NINJA_REGENERATE_DEPS + DISABLE_AUTO_NINJA + NINJA_ALIAS_NAME + NINJA_COMPDB_EXPAND NINJA_ENV_VAR_CACHE + NINJA_FILE_NAME + NINJA_GENERATED_SOURCE_SUFFIXES + NINJA_MSVC_DEPS_PREFIX NINJA_POOL - DISABLE_AUTO_NINJA + NINJA_REGENERATE_DEPS + NINJA_SYNTAX + _NINJA_REGENERATE_DEPS_FUNC __NINJA_NO + IMPLICIT_COMMAND_DEPENDENCIES - NINJA_FILE_NAME - NINJA_ALIAS_NAME - NINJA_SYNTAX + + NINJA_RULES + NINJA_POOLS + NINJA + GENERATING_NINJA + NINJA_DIR - _NINJA_REGENERATE_DEPS_FUNC - CCCOM - CXXCOM - SHCXXCOM - SHCCCOM + AR + ARCOM + ARFLAGS CC + CCCOM + CCFLAGS CXX - - LINKCOM + CXXCOM + ESCAPE LINK - - SHLINKCOM - SHLINK - - ARCOM - AR + LINKCOM + PLATFORM RANLIB - ARFLAGS RANLIBCOM - PLATFORM - ESCAPE + SHCCCOM + SHCXXCOM + SHLINK + SHLINKCOM + PROGSUFFIX + PRINT_CMD_LINE_FUNC + TEMPFILE + + + NINJA_MAX_JOBS + NINJA_SKIP + NINJA_CUSTOM_HANDLERS + BUILD_DIR + @@ -169,7 +184,7 @@ See its __doc__ string for a discussion of the format. - + This propagates directly into the generated Ninja.build file. From a262a13096a6c6783887f9b4a1ab0700588a7504 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 24 Jun 2021 12:44:05 -0700 Subject: [PATCH 157/163] Rename DISABLE_AUTO_NINJA -> NINJA_DISABLE_AUTO_RUN --- SCons/Tool/ninja/__init__.py | 4 ++-- SCons/Tool/ninja/ninja.xml | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 73152a2653..eb661987d7 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -76,7 +76,7 @@ def ninja_builder(env, target, source): else: cmd = [NINJA_STATE.ninja_bin_path, '-f', generated_build_ninja] - if not env.get("DISABLE_AUTO_NINJA"): + if not env.get("NINJA_DISABLE_AUTO_RUN"): print("Executing:", str(' '.join(cmd))) # execute the ninja build at the end of SCons, trying to @@ -173,7 +173,7 @@ def generate(env): SCons.Warnings.SConsWarning("Failed to import ninja, attempt normal SCons build.") return - env["DISABLE_AUTO_NINJA"] = GetOption('disable_execute_ninja') + env["NINJA_DISABLE_AUTO_RUN"] = GetOption('disable_execute_ninja') env["NINJA_FILE_NAME"] = env.get("NINJA_FILE_NAME", "build.ninja") diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index 94d40406f9..9e3b04151e 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -52,9 +52,9 @@ See its __doc__ string for a discussion of the format. - DISABLE_AUTO_NINJA + NINJA_DISABLE_AUTO_RUN NINJA_ALIAS_NAME - + NINJA_DIR NINJA_COMPDB_EXPAND NINJA_ENV_VAR_CACHE NINJA_FILE_NAME @@ -104,7 +104,6 @@ See its __doc__ string for a discussion of the format. NINJA_MAX_JOBS NINJA_SKIP NINJA_CUSTOM_HANDLERS - BUILD_DIR @@ -247,7 +246,7 @@ See its __doc__ string for a discussion of the format. - + Boolean (True|False). Default: False From 2889a57a42038445ed7ca4f4e89edfb788c4ba91 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 7 Jul 2021 15:58:38 -0700 Subject: [PATCH 158/163] incremental doc updates to indicate experimental state of ninja builder --- SCons/Tool/ninja/ninja.xml | 47 +++++++++++++++++++++++++------------- doc/user/external.xml | 13 ++++++++++- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index 9e3b04151e..8bcfe1c475 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -50,6 +50,10 @@ See its __doc__ string for a discussion of the format. Sets up &b-link-Ninja; builder which generates a ninja build file, and then optionally runs ninja. + NOTE: This is an experimental feature. You'll need to pass scons --experimental=ninja to enable. + + NOTE: This functionality is subject to change and/or removal without deprecation cycle. + NINJA_DISABLE_AUTO_RUN @@ -115,8 +119,29 @@ See its __doc__ string for a discussion of the format. &b-Ninja; is a special builder which adds a target to create a ninja build file. The builder does not require any source files to be specified, - + + This is an experimental feature. You'll need to pass scons --experimental=ninja to enable. + + This functionality is subject to change and/or removal without deprecation cycle. + + + # On the command line + --experimental=ninja + + # Or in your SConstruct + SetOption('experimental', 'ninja') + + + + To use this tool you must install pypi's ninja + package. + This can be done via + pip install ninja + + + + If called with no arguments, the builder will default to a target name of @@ -143,17 +168,6 @@ See its __doc__ string for a discussion of the format. since there is no way to deduce what the intent was; in this case the default target name will be used. - - - You must specify scons --experimental=ninja to enable this tool. - - - To use this tool you must install pypi's ninja - package. - This can be done via - pip install ninja - - Available since &scons; 4.2. @@ -190,9 +204,9 @@ See its __doc__ string for a discussion of the format. From Ninja's docs:
- builddir - A directory for some Ninja output files. ... (You can also store other build output in this - directory.) + builddir + A directory for some Ninja output files. ... (You can also store other build output in this + directory.)
@@ -269,7 +283,8 @@ See its __doc__ string for a discussion of the format. - The filename for the generated Ninja build file defaults to ninja.build + The filename for the generated Ninja build file defaults to + ninja.build diff --git a/doc/user/external.xml b/doc/user/external.xml index 78e0a7d488..f9cc7bf4bb 100644 --- a/doc/user/external.xml +++ b/doc/user/external.xml @@ -305,8 +305,19 @@ Ninja Build Generator - This is an experimental new feature. + NOTE: This is an experimental new feature. It is subject to change and/or removal without depreciation cycle + + To enable this feature you'll need to use one of the following + + + # On the command line + --experimental=ninja + + # Or in your SConstruct + SetOption('experimental', 'ninja') + + This tool will enabled creating a ninja build file from your SCons based build system. It can then invoke ninja to run your build. For most builds ninja will be significantly faster, but you may have to give up From f359308f045555963d59e08a4901ace7e07833c5 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 7 Jul 2021 20:42:24 -0700 Subject: [PATCH 159/163] Updates to docs to indicate experimental status --- SCons/Tool/ninja/__init__.py | 19 ++-- SCons/Tool/ninja/ninja.xml | 30 ++++--- bin/SConsDoc.py | 9 ++ doc/generated/builders.gen | 36 +++++--- .../examples/caching_ex-random_1.xml | 4 +- doc/generated/functions.gen | 6 +- doc/generated/tools.gen | 8 +- doc/generated/variables.gen | 89 ++++++++++--------- doc/generated/variables.mod | 8 +- doc/user/external.xml | 35 +++++--- 10 files changed, 143 insertions(+), 101 deletions(-) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index eb661987d7..d940bb1636 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -437,13 +437,14 @@ def ninja_execute(self): # Set all three environment variables that Python's # tempfile.mkstemp looks at as it behaves differently on different # platforms and versions of Python. - build_dir = env.subst("$NINJA_DIR") - if build_dir == "": - build_dir = "." - os.environ["TMPDIR"] = env.Dir("{}/.response_files".format(build_dir)).get_abspath() - os.environ["TEMP"] = os.environ["TMPDIR"] - os.environ["TMP"] = os.environ["TMPDIR"] - if not os.path.isdir(os.environ["TMPDIR"]): - env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) - + # build_dir = env.subst("$NINJA_DIR") + # if build_dir == "": + # build_dir = "." + # os.environ["TMPDIR"] = env.Dir("{}/.response_files".format(build_dir)).get_abspath() + # os.environ["TEMP"] = os.environ["TMPDIR"] + # os.environ["TMP"] = os.environ["TMPDIR"] + # if not os.path.isdir(os.environ["TMPDIR"]): + # env.Execute(SCons.Defaults.Mkdir(os.environ["TMPDIR"])) + + env['TEMPFILEDIR'] = "$NINJA_DIR/.response_files" env["TEMPFILE"] = NinjaNoResponseFiles diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index 8bcfe1c475..eaed82a644 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -50,9 +50,11 @@ See its __doc__ string for a discussion of the format. Sets up &b-link-Ninja; builder which generates a ninja build file, and then optionally runs ninja. - NOTE: This is an experimental feature. You'll need to pass scons --experimental=ninja to enable. - - NOTE: This functionality is subject to change and/or removal without deprecation cycle. + + This is an experimental feature. + + This functionality is subject to change and/or removal without deprecation cycle. + @@ -73,11 +75,11 @@ See its __doc__ string for a discussion of the format. - NINJA_RULES - NINJA_POOLS - NINJA - GENERATING_NINJA - NINJA_DIR + + + + + @@ -102,13 +104,13 @@ See its __doc__ string for a discussion of the format. SHLINKCOM PROGSUFFIX PRINT_CMD_LINE_FUNC - TEMPFILE + - + @@ -121,9 +123,8 @@ See its __doc__ string for a discussion of the format. The builder does not require any source files to be specified, - This is an experimental feature. You'll need to pass scons --experimental=ninja to enable. + This is an experimental feature. To enable it you must use one of the following methods - This functionality is subject to change and/or removal without deprecation cycle. # On the command line @@ -133,6 +134,9 @@ See its __doc__ string for a discussion of the format. SetOption('experimental', 'ninja') + This functionality is subject to change and/or removal without deprecation cycle. + + To use this tool you must install pypi's ninja package. diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py index baffbdcc20..f36964209b 100644 --- a/bin/SConsDoc.py +++ b/bin/SConsDoc.py @@ -364,9 +364,18 @@ def validateXml(fpath, xmlschema_context): doc.xinclude() try: TreeFactory.xmlschema.assertValid(doc) + except etree.XMLSchemaValidateError as e: + print("ERROR: %s fails to validate:" % fpath) + print(e) + print(e.error_log.last_error.message) + print("In file: [%s]"%e.error_log.last_error.filename) + print("Line : %d"%e.error_log.last_error.line) + return False + except Exception as e: print("ERROR: %s fails to validate:" % fpath) print(e) + return False return True diff --git a/doc/generated/builders.gen b/doc/generated/builders.gen index ba90e6cfbd..ce50a97f9e 100644 --- a/doc/generated/builders.gen +++ b/doc/generated/builders.gen @@ -1228,8 +1228,31 @@ env.MSVSSolution( &b-Ninja; is a special builder which adds a target to create a ninja build file. The builder does not require any source files to be specified, - + + This is an experimental feature. To enable it you must use one of the following methods + + + + # On the command line + --experimental=ninja + + # Or in your SConstruct + SetOption('experimental', 'ninja') + + + This functionality is subject to change and/or removal without deprecation cycle. + + + + To use this tool you must install pypi's ninja + package. + This can be done via + pip install ninja + + + + If called with no arguments, the builder will default to a target name of @@ -1256,17 +1279,6 @@ env.MSVSSolution( since there is no way to deduce what the intent was; in this case the default target name will be used. - - - You must specify scons --experimental=ninja to enable this tool. - - - To use this tool you must install pypi's ninja - package. - This can be done via - pip install ninja - - Available since &scons; 4.2. diff --git a/doc/generated/examples/caching_ex-random_1.xml b/doc/generated/examples/caching_ex-random_1.xml index 1570d7df30..0cdf9d73ee 100644 --- a/doc/generated/examples/caching_ex-random_1.xml +++ b/doc/generated/examples/caching_ex-random_1.xml @@ -1,8 +1,8 @@ % scons -Q -cc -o f4.o -c f4.c +cc -o f5.o -c f5.c cc -o f2.o -c f2.c cc -o f1.o -c f1.c cc -o f3.o -c f3.c -cc -o f5.o -c f5.c +cc -o f4.o -c f4.c cc -o prog f1.o f2.o f3.o f4.o f5.o diff --git a/doc/generated/functions.gen b/doc/generated/functions.gen index 53c35837a7..fe2f516b5d 100644 --- a/doc/generated/functions.gen +++ b/doc/generated/functions.gen @@ -548,8 +548,8 @@ do not make sense and a Python exception will be raised. -When using &f-env-Append; to modify &consvars; -which are path specifications (normally, +When using &f-env-Append; to modify &consvars; +which are path specifications (normally, those names which end in PATH), it is recommended to add the values as a list of strings, even if there is only a single string to add. @@ -3861,7 +3861,7 @@ Sets &scons; option variable name to value. These options are all also settable via command-line options but the variable name -may differ from the command-line option name - +may differ from the command-line option name - see the table for correspondences. A value set via command-line option will take precedence over one set with &f-SetOption;, which diff --git a/doc/generated/tools.gen b/doc/generated/tools.gen index 01b69f593e..d28084d3ac 100644 --- a/doc/generated/tools.gen +++ b/doc/generated/tools.gen @@ -816,7 +816,13 @@ Sets construction variables for the Sets up &b-link-Ninja; builder which generates a ninja build file, and then optionally runs ninja. - Sets: &cv-link-DISABLE_AUTO_NINJA;, &cv-link-NINJA_ALIAS_NAME;, &cv-link-NINJA_BUILDDIR;, &cv-link-NINJA_COMPDB_EXPAND;, &cv-link-NINJA_ENV_VAR_CACHE;, &cv-link-NINJA_FILE_NAME;, &cv-link-NINJA_GENERATED_SOURCE_SUFFIXES;, &cv-link-NINJA_MSVC_DEPS_PREFIX;, &cv-link-NINJA_POOL;, &cv-link-NINJA_REGENERATE_DEPS;, &cv-link-NINJA_SYNTAX;, &cv-link-_NINJA_REGENERATE_DEPS_FUNC;, &cv-link-__NINJA_NO;.Uses: &cv-link-AR;, &cv-link-ARCOM;, &cv-link-ARFLAGS;, &cv-link-CC;, &cv-link-CCCOM;, &cv-link-CXX;, &cv-link-CXXCOM;, &cv-link-ESCAPE;, &cv-link-LINK;, &cv-link-LINKCOM;, &cv-link-PLATFORM;, &cv-link-RANLIB;, &cv-link-RANLIBCOM;, &cv-link-SHCCCOM;, &cv-link-SHCXXCOM;, &cv-link-SHLINK;, &cv-link-SHLINKCOM;. + + This is an experimental feature. + + This functionality is subject to change and/or removal without deprecation cycle. + + + Sets: &cv-link-IMPLICIT_COMMAND_DEPENDENCIES;, &cv-link-NINJA_ALIAS_NAME;, &cv-link-NINJA_COMPDB_EXPAND;, &cv-link-NINJA_DIR;, &cv-link-NINJA_DISABLE_AUTO_RUN;, &cv-link-NINJA_ENV_VAR_CACHE;, &cv-link-NINJA_FILE_NAME;, &cv-link-NINJA_GENERATED_SOURCE_SUFFIXES;, &cv-link-NINJA_MSVC_DEPS_PREFIX;, &cv-link-NINJA_POOL;, &cv-link-NINJA_REGENERATE_DEPS;, &cv-link-NINJA_SYNTAX;, &cv-link-_NINJA_REGENERATE_DEPS_FUNC;, &cv-link-__NINJA_NO;.Uses: &cv-link-AR;, &cv-link-ARCOM;, &cv-link-ARFLAGS;, &cv-link-CC;, &cv-link-CCCOM;, &cv-link-CCFLAGS;, &cv-link-CXX;, &cv-link-CXXCOM;, &cv-link-ESCAPE;, &cv-link-LINK;, &cv-link-LINKCOM;, &cv-link-PLATFORM;, &cv-link-PRINT_CMD_LINE_FUNC;, &cv-link-PROGSUFFIX;, &cv-link-RANLIB;, &cv-link-RANLIBCOM;, &cv-link-SHCCCOM;, &cv-link-SHCXXCOM;, &cv-link-SHLINK;, &cv-link-SHLINKCOM;. packaging diff --git a/doc/generated/variables.gen b/doc/generated/variables.gen index 4d5f92f26d..e6fec13693 100644 --- a/doc/generated/variables.gen +++ b/doc/generated/variables.gen @@ -713,8 +713,8 @@ the type of value of &cv-CPPDEFINES;: If &cv-CPPDEFINES; is a string, the values of the &cv-link-CPPDEFPREFIX; and &cv-link-CPPDEFSUFFIX; &consvars; -will be respectively prepended and appended to -each definition in &cv-CPPDEFINES;. +will be respectively prepended and appended to +each definition in &cv-link-CPPDEFINES;. @@ -726,8 +726,8 @@ env = Environment(CPPDEFINES='xyz') If &cv-CPPDEFINES; is a list, the values of the -&cv-link-CPPDEFPREFIX; and &cv-link-CPPDEFSUFFIX; &consvars; -will be respectively prepended and appended to +&cv-CPPDEFPREFIX; and &cv-CPPDEFSUFFIX; &consvars; +will be respectively prepended and appended to each element in the list. If any element is a list or tuple, then the first item is the name being @@ -743,7 +743,7 @@ env = Environment(CPPDEFINES=[('B', 2), 'A']) If &cv-CPPDEFINES; is a dictionary, the values of the -&cv-link-CPPDEFPREFIX; and &cv-link-CPPDEFSUFFIX; &consvars; +&cv-CPPDEFPREFIX; and &cv-CPPDEFSUFFIX; &consvars; will be respectively prepended and appended to each item from the dictionary. The key of each dictionary item @@ -845,7 +845,7 @@ to each directory in &cv-link-CPPPATH;. The list of directories that the C preprocessor will search for include directories. The C/C++ implicit dependency scanner will search these -directories for include files. +directories for include files. In general it's not advised to put include directory directives directly into &cv-link-CCFLAGS; or &cv-link-CXXFLAGS; as the result will be non-portable @@ -857,11 +857,11 @@ Python's os.sep. Note: -directory names in &cv-link-CPPPATH; +directory names in &cv-CPPPATH; will be looked-up relative to the directory of the SConscript file -when they are used in a command. +when they are used in a command. To force &scons; -to look-up a directory relative to the root of the source tree use +to look-up a directory relative to the root of the source tree use the # prefix: @@ -1172,17 +1172,6 @@ into a list of Dir instances relative to the target being built. - - - DISABLE_AUTO_NINJA - - - Boolean (True|False). Default: False - When True, SCons will not run ninja automatically after creating the ninja.build file. - - - - DLIB @@ -3158,7 +3147,7 @@ The default list is: IMPLIBNOVERSIONSYMLINKS -Used to override &cv-link-SHLIBNOVERSIONSYMLINKS;/&cv-link-LDMODULENOVERSIONSYMLINKS; when +Used to override &cv-link-SHLIBNOVERSIONSYMLINKS;/&cv-link-LDMODULENOVERSIONSYMLINKS; when creating versioned import library for a shared library/loadable module. If not defined, then &cv-link-SHLIBNOVERSIONSYMLINKS;/&cv-link-LDMODULENOVERSIONSYMLINKS; is used to determine whether to disable symlink generation or not. @@ -4074,9 +4063,9 @@ as the result will be non-portable. Note: directory names in &cv-LIBPATH; will be looked-up relative to the directory of the SConscript file -when they are used in a command. +when they are used in a command. To force &scons; -to look-up a directory relative to the root of the source tree use +to look-up a directory relative to the root of the source tree use the # prefix: @@ -4148,7 +4137,7 @@ and suffixes from the &cv-link-LIBSUFFIXES; list. A list of one or more libraries that will be added to the link line -for linking with any executable program, shared library, or loadable module +for linking with any executable program, shared library, or loadable module created by the &consenv; or override. @@ -4514,7 +4503,7 @@ See &t-link-msgfmt; tool and &b-link-MOFiles; builder. MSGFMTCOMSTR -String to display when msgfmt(1) is invoked +String to display when msgfmt(1) is invoked (default: '', which means ``print &cv-link-MSGFMTCOM;''). See &t-link-msgfmt; tool and &b-link-MOFiles; builder. @@ -4556,7 +4545,7 @@ See &t-link-msginit; tool and &b-link-POInit; builder. MSGINITCOMSTR -String to display when msginit(1) is invoked +String to display when msginit(1) is invoked (default: '', which means ``print &cv-link-MSGINITCOM;''). See &t-link-msginit; tool and &b-link-POInit; builder. @@ -4652,7 +4641,7 @@ Supported versions include 6.0A, 6.0, 2003R2 -and +and 2003R1. @@ -5187,35 +5176,46 @@ Specfies the name of the project to package. - + - NINJA_BUILDDIR + NINJA_COMPDB_EXPAND + + + Boolean value (True|False) to instruct ninja to expand the command line arguments normally put into + response files. + This prevents lines in the compilation database like gcc @rsp_file and instead yields + gcc -c -o myfile.o myfile.c -Ia -DXYZ + + + Ninja's compdb tool added the -x flag in Ninja V1.9.0 + + + + + + NINJA_DIR This propagates directly into the generated Ninja.build file. From Ninja's docs:
- builddir - A directory for some Ninja output files. ... (You can also store other build output in this - directory.) + builddir + A directory for some Ninja output files. ... (You can also store other build output in this + directory.)
- + - NINJA_COMPDB_EXPAND + NINJA_DISABLE_AUTO_RUN - Boolean value (True|False) to instruct ninja to expand the command line arguments normally put into - response files. - This prevents lines in the compilation database like gcc @rsp_file and instead yields - gcc -c -o myfile.o myfile.c -Ia -DXYZ - - - Ninja's compdb tool added the -x flag in Ninja V1.9.0 + Boolean (True|False). Default: False + When True, SCons will not run ninja automatically after creating the ninja.build file. + @@ -5240,7 +5240,8 @@ Specfies the name of the project to package. NINJA_FILE_NAME - The filename for the generated Ninja build file defaults to ninja.build + The filename for the generated Ninja build file defaults to + ninja.build @@ -7510,7 +7511,7 @@ for more information). SOVERSION -This will construct the SONAME using on the base library name +This will construct the SONAME using on the base library name (test in the example below) and use specified SOVERSION to create SONAME. @@ -7519,7 +7520,7 @@ env.SharedLibrary('test', 'test.c', SHLIBVERSION='0.1.2', SOVERSION='2') The variable is used, for example, by &t-link-gnulink; linker tool. -In the example above SONAME would be libtest.so.2 +In the example above SONAME would be libtest.so.2 which would be a symlink and point to libtest.so.0.1.2 diff --git a/doc/generated/variables.mod b/doc/generated/variables.mod index 86fc604b45..08aa21dbf3 100644 --- a/doc/generated/variables.mod +++ b/doc/generated/variables.mod @@ -84,7 +84,6 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $DINCSUFFIX"> $Dir"> $Dirs"> -$DISABLE_AUTO_NINJA"> $DLIB"> $DLIBCOM"> $DLIBDIRPREFIX"> @@ -352,8 +351,9 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $MWCW_VERSIONS"> $NAME"> $NINJA_ALIAS_NAME"> -$NINJA_BUILDDIR"> $NINJA_COMPDB_EXPAND"> +$NINJA_DIR"> +$NINJA_DISABLE_AUTO_RUN"> $NINJA_ENV_VAR_CACHE"> $NINJA_FILE_NAME"> $NINJA_GENERATED_SOURCE_SUFFIXES"> @@ -737,7 +737,6 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $DINCSUFFIX"> $Dir"> $Dirs"> -$DISABLE_AUTO_NINJA"> $DLIB"> $DLIBCOM"> $DLIBDIRPREFIX"> @@ -1005,8 +1004,9 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $MWCW_VERSIONS"> $NAME"> $NINJA_ALIAS_NAME"> -$NINJA_BUILDDIR"> $NINJA_COMPDB_EXPAND"> +$NINJA_DIR"> +$NINJA_DISABLE_AUTO_RUN"> $NINJA_ENV_VAR_CACHE"> $NINJA_FILE_NAME"> $NINJA_GENERATED_SOURCE_SUFFIXES"> diff --git a/doc/user/external.xml b/doc/user/external.xml index f9cc7bf4bb..5f88f5adbb 100644 --- a/doc/user/external.xml +++ b/doc/user/external.xml @@ -304,19 +304,28 @@
Ninja Build Generator - - NOTE: This is an experimental new feature. It is subject to change and/or removal without depreciation cycle - - - To enable this feature you'll need to use one of the following - - - # On the command line - --experimental=ninja - - # Or in your SConstruct - SetOption('experimental', 'ninja') - + + + This is an experimental new feature. It is subject to change and/or removal without depreciation cycle + + + To use this tool you must install pypi's ninja + package. + This can be done via + pip install ninja + + + To enable this feature you'll need to use one of the following + + + # On the command line + --experimental=ninja + + # Or in your SConstruct + SetOption('experimental', 'ninja') + + + This tool will enabled creating a ninja build file from your SCons based build system. It can then invoke From 24ec1b346df1ba9479c40d43e5e1a2f71c31c5f5 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 9 Jul 2021 11:17:13 -0700 Subject: [PATCH 160/163] Resolve sider issue --- bin/SConsDoc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py index f36964209b..a46505785c 100644 --- a/bin/SConsDoc.py +++ b/bin/SConsDoc.py @@ -368,8 +368,8 @@ def validateXml(fpath, xmlschema_context): print("ERROR: %s fails to validate:" % fpath) print(e) print(e.error_log.last_error.message) - print("In file: [%s]"%e.error_log.last_error.filename) - print("Line : %d"%e.error_log.last_error.line) + print("In file: [%s]" % e.error_log.last_error.filename) + print("Line : %d" % e.error_log.last_error.line) return False except Exception as e: From 91364b39b3709f6742b0a97aaa5b0bfc681a2f8f Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 9 Jul 2021 11:17:47 -0700 Subject: [PATCH 161/163] add ninja entity and wrap all ninja.build with tags --- SCons/Tool/ninja/ninja.xml | 14 +++++++------- doc/scons.mod | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index eaed82a644..5413e85cc9 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -48,7 +48,7 @@ See its __doc__ string for a discussion of the format. - Sets up &b-link-Ninja; builder which generates a ninja build file, and then optionally runs ninja. + Sets up &b-link-Ninja; builder which generates a ninja build file, and then optionally runs &ninja. This is an experimental feature. @@ -120,7 +120,7 @@ See its __doc__ string for a discussion of the format. &b-Ninja; is a special builder which adds a target to create a ninja build file. - The builder does not require any source files to be specified, + The builder does not require any source files to be specified. This is an experimental feature. To enable it you must use one of the following methods @@ -184,8 +184,8 @@ See its __doc__ string for a discussion of the format. The list of source file suffixes which are generated by SCons build steps. All source files which match these suffixes will be added to the _generated_sources alias in the output - ninja.build file. - Then all other source files will be made to depend on this in the Ninja.build file, forcing the + ninja.build file. + Then all other source files will be made to depend on this in the ninja.build file, forcing the generated sources to be built first. @@ -194,7 +194,7 @@ See its __doc__ string for a discussion of the format. - This propagates directly into the generated Ninja.build file. + This propagates directly into the generated ninja.build file. From Ninja's docs defines the string which should be stripped from msvc’s /showIncludes output @@ -204,7 +204,7 @@ See its __doc__ string for a discussion of the format. - This propagates directly into the generated Ninja.build file. + This propagates directly into the generated ninja.build file. From Ninja's docs:
@@ -268,7 +268,7 @@ See its __doc__ string for a discussion of the format. Boolean (True|False). Default: False - When True, SCons will not run ninja automatically after creating the ninja.build file. + When True, SCons will not run ninja automatically after creating the ninja.build file. diff --git a/doc/scons.mod b/doc/scons.mod index 57004549f6..72adf68240 100644 --- a/doc/scons.mod +++ b/doc/scons.mod @@ -57,6 +57,7 @@ m4"> Make"> Make++"> +ninja"> pdflatex"> pdftex"> Python"> From 20b5f2482d33fa52a6c360c0872bcd3dcd459206 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 9 Jul 2021 11:27:48 -0700 Subject: [PATCH 162/163] Fix broken markup, regenerate files --- SCons/Tool/ninja/ninja.xml | 2 +- doc/generated/builders.gen | 2 +- doc/generated/examples/caching_ex-random_1.xml | 4 ++-- doc/generated/tools.gen | 2 +- doc/generated/variables.gen | 10 +++++----- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index 5413e85cc9..0ec1708917 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -48,7 +48,7 @@ See its __doc__ string for a discussion of the format. - Sets up &b-link-Ninja; builder which generates a ninja build file, and then optionally runs &ninja. + Sets up &b-link-Ninja; builder which generates a ninja build file, and then optionally runs &ninja;. This is an experimental feature. diff --git a/doc/generated/builders.gen b/doc/generated/builders.gen index ce50a97f9e..6499ef38b4 100644 --- a/doc/generated/builders.gen +++ b/doc/generated/builders.gen @@ -1227,7 +1227,7 @@ env.MSVSSolution( &b-Ninja; is a special builder which adds a target to create a ninja build file. - The builder does not require any source files to be specified, + The builder does not require any source files to be specified. This is an experimental feature. To enable it you must use one of the following methods diff --git a/doc/generated/examples/caching_ex-random_1.xml b/doc/generated/examples/caching_ex-random_1.xml index 0cdf9d73ee..2990d7477d 100644 --- a/doc/generated/examples/caching_ex-random_1.xml +++ b/doc/generated/examples/caching_ex-random_1.xml @@ -1,8 +1,8 @@ % scons -Q -cc -o f5.o -c f5.c -cc -o f2.o -c f2.c cc -o f1.o -c f1.c cc -o f3.o -c f3.c +cc -o f5.o -c f5.c cc -o f4.o -c f4.c +cc -o f2.o -c f2.c cc -o prog f1.o f2.o f3.o f4.o f5.o diff --git a/doc/generated/tools.gen b/doc/generated/tools.gen index d28084d3ac..16d13e3cfa 100644 --- a/doc/generated/tools.gen +++ b/doc/generated/tools.gen @@ -814,7 +814,7 @@ Sets construction variables for the ninja - Sets up &b-link-Ninja; builder which generates a ninja build file, and then optionally runs ninja. + Sets up &b-link-Ninja; builder which generates a ninja build file, and then optionally runs &ninja;. This is an experimental feature. diff --git a/doc/generated/variables.gen b/doc/generated/variables.gen index e6fec13693..c56217be7b 100644 --- a/doc/generated/variables.gen +++ b/doc/generated/variables.gen @@ -5196,7 +5196,7 @@ Specfies the name of the project to package. NINJA_DIR - This propagates directly into the generated Ninja.build file. + This propagates directly into the generated ninja.build file. From Ninja's docs:
@@ -5214,7 +5214,7 @@ Specfies the name of the project to package. Boolean (True|False). Default: False - When True, SCons will not run ninja automatically after creating the ninja.build file. + When True, SCons will not run ninja automatically after creating the ninja.build file. @@ -5252,8 +5252,8 @@ Specfies the name of the project to package. The list of source file suffixes which are generated by SCons build steps. All source files which match these suffixes will be added to the _generated_sources alias in the output - ninja.build file. - Then all other source files will be made to depend on this in the Ninja.build file, forcing the + ninja.build file. + Then all other source files will be made to depend on this in the ninja.build file, forcing the generated sources to be built first. @@ -5263,7 +5263,7 @@ Specfies the name of the project to package. NINJA_MSVC_DEPS_PREFIX - This propagates directly into the generated Ninja.build file. + This propagates directly into the generated ninja.build file. From Ninja's docs defines the string which should be stripped from msvc’s /showIncludes output From 6c08f310bbbb21bf304862cd541858e2aa793dda Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 9 Jul 2021 12:04:54 -0700 Subject: [PATCH 163/163] Add info to CHANGES.txt on ninja --- CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.txt b/CHANGES.txt index 6a817badd2..2b0de50916 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -29,6 +29,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Fix versioned shared library naming for MacOS platform. (Previously was libxyz.dylib.1.2.3, has been fixed to libxyz.1.2.3.dylib. Additionally the sonamed symlink had the same issue, that is now resolved as well) + - Add experimental ninja builder. (Contributed by MongoDB, Daniel Moody and many others). From David H: - Fix Issue #3906 - `IMPLICIT_COMMAND_DEPENDENCIES` was not properly disabled when