From 359a3c127ff7f467f64edc64bf584fdf894cb104 Mon Sep 17 00:00:00 2001 From: Tony Allevato Date: Thu, 7 Jul 2022 15:15:02 -0700 Subject: [PATCH] Add index-while-building support (back) to the Swift build rules When the `swift.index_while_building` feature is set, targets will emit an indexstore directory to the "indexstore" output group. Unlike the original implementation, this one is focused on the remote indexing use case, so we don't support the old behavior meant for local builds of scanning `--swiftcopt` for a manually-specified path that would be the destination for all index data. Users who want to do that should just pass _all_ of the indexing flags to `--swiftcopt` instead of using this feature. PiperOrigin-RevId: 459613898 (cherry picked from commit 8150427e09350638381c3fd7a7b21b3783a8ad9e) --- Cherry-pick notes: we never removed this. This is mainly just moving things around to make future cherry-picks easier. Though, we did take the `--swiftcopt` change by removing `_is_index_store_path_overridden` (which is a breaking change). Signed-off-by: Brentley Jones --- doc/api.md | 43 +++++++++------ proto/swift_proto_utils.bzl | 8 ++- swift/internal/compiling.bzl | 61 +++++++++++++--------- swift/internal/feature_names.bzl | 20 +++---- swift/internal/output_groups.bzl | 4 +- swift/swift_binary.bzl | 21 ++++---- swift/swift_compiler_plugin.bzl | 16 +++--- swift/swift_library.bzl | 14 +++-- swift/swift_module_alias.bzl | 12 +++-- swift/swift_test.bzl | 15 +++--- swift/toolchains/config/compile_config.bzl | 21 +------- 11 files changed, 126 insertions(+), 109 deletions(-) diff --git a/doc/api.md b/doc/api.md index c852d8b93..26247a3b5 100644 --- a/doc/api.md +++ b/doc/api.md @@ -122,28 +122,39 @@ Compiles a Swift module. **RETURNS** -A tuple containing three elements: - - 1. A Swift module context (as returned by `swift_common.create_module`) - that contains the Swift (and potentially C/Objective-C) compilation - prerequisites of the compiled module. This should typically be - propagated by a `SwiftInfo` provider of the calling rule, and the - `CcCompilationContext` inside the Clang module substructure should - be propagated by the `CcInfo` provider of the calling rule. - 2. A `CcCompilationOutputs` object (as returned by - `cc_common.create_compilation_outputs`) that contains the compiled - object files. - 3. A struct containing: +A `struct` with the following fields: + + * `module_context`: A Swift module context (as returned by + `swift_common.create_module`) that contains the Swift (and + potentially C/Objective-C) compilation prerequisites of the compiled + module. This should typically be propagated by a `SwiftInfo` + provider of the calling rule, and the `CcCompilationContext` inside + the Clang module substructure should be propagated by the `CcInfo` + provider of the calling rule. + + * `compilation_outputs`: A `CcCompilationOutputs` object (as returned + by `cc_common.create_compilation_outputs`) that contains the + compiled object files. + + * `supplemental_outputs`: A `struct` representing supplemental, + optional outputs. Its fields are: + * `ast_files`: A list of `File`s output from the `DUMP_AST` action. - * `indexstore`: A `File` representing the directory that contains - the index store data generated by the compiler if the - `"swift.index_while_building"` feature is enabled, otherwise - this will be `None`. + * `const_values_files`: A list of `File`s that contains JSON representations of constant values extracted from the source files, if requested via a direct dependency. + * `indexstore_directory`: A directory-type `File` that represents + the indexstore output files created when the feature + `swift.index_while_building` is enabled. + + * `macro_expansion_directory`: A directory-type `File` that + represents the location where macro expansion files were written + (only in debug/fastbuild and only when the toolchain supports + macros). + diff --git a/proto/swift_proto_utils.bzl b/proto/swift_proto_utils.bzl index 5fc86e10e..a2bb0338e 100644 --- a/proto/swift_proto_utils.bzl +++ b/proto/swift_proto_utils.bzl @@ -267,7 +267,7 @@ def compile_swift_protos_for_target( # Compile the generated Swift source files as a module: include_dev_srch_paths = include_developer_search_paths(attr) - module_context, cc_compilation_outputs, supplemental_outputs = swift_common.compile( + compile_result = swift_common.compile( actions = ctx.actions, cc_infos = get_providers(compiler_deps, CcInfo), copts = ["-parse-as-library"], @@ -283,11 +283,15 @@ def compile_swift_protos_for_target( workspace_name = ctx.workspace_name, ) + module_context = compile_result.module_context + compilation_outputs = compile_result.compilation_outputs + supplemental_outputs = compile_result.supplemental_outputs + # Create the linking context from the compilation outputs: linking_context, linking_output = ( swift_common.create_linking_context_from_compilation_outputs( actions = ctx.actions, - compilation_outputs = cc_compilation_outputs, + compilation_outputs = compilation_outputs, feature_configuration = feature_configuration, include_dev_srch_paths = include_dev_srch_paths, label = target_label, diff --git a/swift/internal/compiling.bzl b/swift/internal/compiling.bzl index f816e8333..85fcb2a21 100644 --- a/swift/internal/compiling.bzl +++ b/swift/internal/compiling.bzl @@ -439,27 +439,38 @@ def compile( outputs. Returns: - A tuple containing three elements: - - 1. A Swift module context (as returned by `swift_common.create_module`) - that contains the Swift (and potentially C/Objective-C) compilation - prerequisites of the compiled module. This should typically be - propagated by a `SwiftInfo` provider of the calling rule, and the - `CcCompilationContext` inside the Clang module substructure should - be propagated by the `CcInfo` provider of the calling rule. - 2. A `CcCompilationOutputs` object (as returned by - `cc_common.create_compilation_outputs`) that contains the compiled - object files. - 3. A struct containing: + A `struct` with the following fields: + + * `module_context`: A Swift module context (as returned by + `swift_common.create_module`) that contains the Swift (and + potentially C/Objective-C) compilation prerequisites of the compiled + module. This should typically be propagated by a `SwiftInfo` + provider of the calling rule, and the `CcCompilationContext` inside + the Clang module substructure should be propagated by the `CcInfo` + provider of the calling rule. + + * `compilation_outputs`: A `CcCompilationOutputs` object (as returned + by `cc_common.create_compilation_outputs`) that contains the + compiled object files. + + * `supplemental_outputs`: A `struct` representing supplemental, + optional outputs. Its fields are: + * `ast_files`: A list of `File`s output from the `DUMP_AST` action. - * `indexstore`: A `File` representing the directory that contains - the index store data generated by the compiler if the - `"swift.index_while_building"` feature is enabled, otherwise - this will be `None`. + * `const_values_files`: A list of `File`s that contains JSON representations of constant values extracted from the source files, if requested via a direct dependency. + + * `indexstore_directory`: A directory-type `File` that represents + the indexstore output files created when the feature + `swift.index_while_building` is enabled. + + * `macro_expansion_directory`: A directory-type `File` that + represents the location where macro expansion files were written + (only in debug/fastbuild and only when the toolchain supports + macros). """ # Collect the `SwiftInfo` providers that represent the dependencies of the @@ -815,20 +826,22 @@ to use swift_common.compile(include_dev_srch_paths = ...) instead.\ ), ) - cc_compilation_outputs = cc_common.create_compilation_outputs( + compilation_outputs = cc_common.create_compilation_outputs( objects = depset(compile_outputs.object_files), pic_objects = depset(compile_outputs.object_files), ) - other_compilation_outputs = struct( - ast_files = compile_outputs.ast_files, - indexstore = compile_outputs.indexstore_directory, - macro_expansion_directory = compile_outputs.macro_expansion_directory, - const_values_files = compile_outputs.const_values_files, + return struct( + module_context = module_context, + compilation_outputs = compilation_outputs, + supplemental_outputs = struct( + ast_files = compile_outputs.ast_files, + const_values_files = compile_outputs.const_values_files, + indexstore_directory = compile_outputs.indexstore_directory, + macro_expansion_directory = compile_outputs.macro_expansion_directory, + ), ) - return module_context, cc_compilation_outputs, other_compilation_outputs - def precompile_clang_module( *, actions, diff --git a/swift/internal/feature_names.bzl b/swift/internal/feature_names.bzl index 571fa2bc8..5a2cf43d2 100644 --- a/swift/internal/feature_names.bzl +++ b/swift/internal/feature_names.bzl @@ -69,6 +69,16 @@ SWIFT_FEATURE_FILE_PREFIX_MAP = "swift.file_prefix_map" # graph. SWIFT_FEATURE_EMIT_C_MODULE = "swift.emit_c_module" +# If enabled, the compilation action for a target will produce an index store. +# https://docs.google.com/document/d/1cH2sTpgSnJZCkZtJl1aY-rzy4uGPcrI-6RrUpdATO2Q/ +SWIFT_FEATURE_INDEX_WHILE_BUILDING = "swift.index_while_building" + +# If enabled the compilation action will not produce indexes for system modules. +SWIFT_FEATURE_DISABLE_SYSTEM_INDEX = "swift.disable_system_index" + +# Index while building - using a global index store cache +SWIFT_FEATURE_USE_GLOBAL_INDEX_STORE = "swift.use_global_index_store" + # If enabled, when compiling an explicit C or Objectve-C module, every header # included by the module being compiled must belong to one of the modules listed # in its dependencies. This is ignored for system modules. @@ -107,16 +117,6 @@ SWIFT_FEATURE_FULL_DEBUG_INFO = "swift.full_debug_info" # Use CodeView debug information, which enables generation of PDBs for debugging. SWIFT_FEATURE_CODEVIEW_DEBUG_INFO = "swift.codeview_debug_info" -# If enabled, the compilation action for a target will produce an index store. -# https://docs.google.com/document/d/1cH2sTpgSnJZCkZtJl1aY-rzy4uGPcrI-6RrUpdATO2Q/ -SWIFT_FEATURE_INDEX_WHILE_BUILDING = "swift.index_while_building" - -# If enabled the compilation action will not produce indexes for system modules. -SWIFT_FEATURE_DISABLE_SYSTEM_INDEX = "swift.disable_system_index" - -# Index while building - using a global index store cache -SWIFT_FEATURE_USE_GLOBAL_INDEX_STORE = "swift.use_global_index_store" - # If enabled, compilation actions and module map generation will assume that the # header paths in module maps are relative to the current working directory # (i.e., the workspace root); if disabled, header paths in module maps are diff --git a/swift/internal/output_groups.bzl b/swift/internal/output_groups.bzl index 26ef79a15..9334d7134 100644 --- a/swift/internal/output_groups.bzl +++ b/swift/internal/output_groups.bzl @@ -41,8 +41,8 @@ def supplemental_compilation_output_groups(*supplemental_outputs): ast_files.extend(outputs.ast_files) if outputs.const_values_files: const_values_files.extend(outputs.const_values_files) - if outputs.indexstore: - indexstore_files.append(outputs.indexstore) + if outputs.indexstore_directory: + indexstore_files.append(outputs.indexstore_directory) if outputs.macro_expansion_directory: macro_expansions_files.append(outputs.macro_expansion_directory) diff --git a/swift/swift_binary.bzl b/swift/swift_binary.bzl index ef2a0bdf4..5b2aa09e8 100644 --- a/swift/swift_binary.bzl +++ b/swift/swift_binary.bzl @@ -69,9 +69,9 @@ def _swift_binary_impl(ctx): ) srcs = ctx.files.srcs + output_groups = {} module_contexts = [] additional_linking_contexts = [] - all_supplemental_outputs = [] # If the binary has sources, compile those first and collect the outputs to # be passed to the linker. @@ -82,7 +82,7 @@ def _swift_binary_impl(ctx): include_dev_srch_paths = include_developer_search_paths(ctx.attr) - module_context, cc_compilation_outputs, supplemental_outputs = swift_common.compile( + compile_result = swift_common.compile( actions = ctx.actions, additional_inputs = ctx.files.swiftc_inputs, cc_infos = get_providers(ctx.attr.deps, CcInfo), @@ -104,8 +104,13 @@ def _swift_binary_impl(ctx): target_name = ctx.label.name, workspace_name = ctx.workspace_name, ) + module_context = compile_result.module_context module_contexts.append(module_context) - all_supplemental_outputs.append(supplemental_outputs) + compilation_outputs = compile_result.compilation_outputs + supplemental_outputs = compile_result.supplemental_outputs + output_groups = supplemental_compilation_output_groups( + supplemental_outputs, + ) # Unlike `upstream`, we create a linking_context in order to support # `autolink-extract`. See `a1395155c6a27d76aab5e1a93455259a0ac10b2f` and @@ -113,7 +118,7 @@ def _swift_binary_impl(ctx): linking_context, _ = swift_common.create_linking_context_from_compilation_outputs( actions = ctx.actions, alwayslink = True, - compilation_outputs = cc_compilation_outputs, + compilation_outputs = compilation_outputs, feature_configuration = feature_configuration, include_dev_srch_paths = include_dev_srch_paths, label = ctx.label, @@ -126,8 +131,6 @@ def _swift_binary_impl(ctx): swift_toolchain = swift_toolchain, ) additional_linking_contexts.append(linking_context) - else: - cc_compilation_outputs = cc_common.create_compilation_outputs() additional_linking_contexts.append(malloc_linking_context(ctx)) @@ -172,11 +175,7 @@ def _swift_binary_impl(ctx): files = ctx.files.data, ), ), - OutputGroupInfo( - **supplemental_compilation_output_groups( - *all_supplemental_outputs - ) - ), + OutputGroupInfo(**output_groups), swift_common.create_swift_info( modules = [ swift_common.create_module( diff --git a/swift/swift_compiler_plugin.bzl b/swift/swift_compiler_plugin.bzl index c1128660e..4e8ba6b8a 100644 --- a/swift/swift_compiler_plugin.bzl +++ b/swift/swift_compiler_plugin.bzl @@ -70,7 +70,7 @@ def _swift_compiler_plugin_impl(ctx): module_name = derive_swift_module_name(ctx.label) entry_point_function_name = "{}_main".format(module_name) - module_context, cc_compilation_outputs, supplemental_outputs = swift_common.compile( + compile_result = swift_common.compile( actions = ctx.actions, additional_inputs = ctx.files.swiftc_inputs, cc_infos = get_providers(deps, CcInfo), @@ -103,9 +103,9 @@ def _swift_compiler_plugin_impl(ctx): target_name = ctx.label.name, workspace_name = ctx.workspace_name, ) - output_groups = supplemental_compilation_output_groups( - supplemental_outputs, - ) + module_context = compile_result.module_context + compilation_outputs = compile_result.compilation_outputs + supplemental_outputs = compile_result.supplemental_outputs cc_feature_configuration = swift_common.cc_feature_configuration( feature_configuration = feature_configuration, @@ -116,7 +116,7 @@ def _swift_compiler_plugin_impl(ctx): additional_inputs = ctx.files.swiftc_inputs, additional_linking_contexts = [malloc_linking_context(ctx)], cc_feature_configuration = cc_feature_configuration, - compilation_outputs = cc_compilation_outputs, + compilation_outputs = compilation_outputs, deps = deps, name = ctx.label.name, output_type = "executable", @@ -141,7 +141,7 @@ def _swift_compiler_plugin_impl(ctx): actions = ctx.actions, additional_inputs = ctx.files.swiftc_inputs, alwayslink = True, - compilation_outputs = cc_compilation_outputs, + compilation_outputs = compilation_outputs, feature_configuration = feature_configuration, label = ctx.label, include_dev_srch_paths = ctx.attr.testonly, @@ -166,7 +166,9 @@ def _swift_compiler_plugin_impl(ctx): files = ctx.files.data, ), ), - OutputGroupInfo(**output_groups), + OutputGroupInfo( + **supplemental_compilation_output_groups(supplemental_outputs) + ), SwiftCompilerPluginInfo( cc_info = CcInfo( compilation_context = module_context.clang.compilation_context, diff --git a/swift/swift_library.bzl b/swift/swift_library.bzl index 15d44d808..79102b83d 100644 --- a/swift/swift_library.bzl +++ b/swift/swift_library.bzl @@ -183,7 +183,7 @@ def _swift_library_impl(ctx): include_dev_srch_paths = include_developer_search_paths(ctx.attr) - module_context, cc_compilation_outputs, supplemental_outputs = swift_common.compile( + compile_result = swift_common.compile( actions = ctx.actions, additional_inputs = additional_inputs, cc_infos = get_providers(ctx.attr.deps, CcInfo), @@ -204,12 +204,16 @@ def _swift_library_impl(ctx): workspace_name = ctx.workspace_name, ) + module_context = compile_result.module_context + compilation_outputs = compile_result.compilation_outputs + supplemental_outputs = compile_result.supplemental_outputs + linking_context, linking_output = ( swift_common.create_linking_context_from_compilation_outputs( actions = ctx.actions, additional_inputs = additional_inputs, alwayslink = ctx.attr.alwayslink, - compilation_outputs = cc_compilation_outputs, + compilation_outputs = compilation_outputs, feature_configuration = feature_configuration, include_dev_srch_paths = include_dev_srch_paths, label = ctx.label, @@ -255,9 +259,6 @@ def _swift_library_impl(ctx): files = ctx.files.data, ), ), - OutputGroupInfo(**supplemental_compilation_output_groups( - supplemental_outputs, - )), CcInfo( compilation_context = module_context.clang.compilation_context, linking_context = linking_context, @@ -274,6 +275,9 @@ def _swift_library_impl(ctx): # not propagate. swift_infos = swift_infos, ), + OutputGroupInfo( + **supplemental_compilation_output_groups(supplemental_outputs) + ), ] # Propagate an `apple_common.Objc` provider with linking info about the diff --git a/swift/swift_module_alias.bzl b/swift/swift_module_alias.bzl index 28a2d25c1..d72c6430a 100644 --- a/swift/swift_module_alias.bzl +++ b/swift/swift_module_alias.bzl @@ -58,7 +58,7 @@ def _swift_module_alias_impl(ctx): swift_infos = get_providers(deps, SwiftInfo) - module_context, compilation_outputs, supplemental_outputs = swift_common.compile( + compile_result = swift_common.compile( actions = ctx.actions, cc_infos = get_providers(ctx.attr.deps, CcInfo), copts = ["-parse-as-library"], @@ -74,6 +74,10 @@ def _swift_module_alias_impl(ctx): workspace_name = ctx.workspace_name, ) + module_context = compile_result.module_context + compilation_outputs = compile_result.compilation_outputs + supplemental_outputs = compile_result.supplemental_outputs + linking_context, linking_output = ( swift_common.create_linking_context_from_compilation_outputs( actions = ctx.actions, @@ -102,9 +106,9 @@ def _swift_module_alias_impl(ctx): linking_output.library_to_link.static_library, ])), ), - OutputGroupInfo(**supplemental_compilation_output_groups( - supplemental_outputs, - )), + OutputGroupInfo( + **supplemental_compilation_output_groups(supplemental_outputs) + ), coverage_common.instrumented_files_info( ctx, dependency_attributes = ["deps"], diff --git a/swift/swift_test.bzl b/swift/swift_test.bzl index 8ebd3a121..21ca4ef11 100644 --- a/swift/swift_test.bzl +++ b/swift/swift_test.bzl @@ -283,7 +283,7 @@ def _swift_test_impl(ctx): include_dev_srch_paths = include_developer_search_paths(ctx.attr) - module_context, cc_compilation_outputs, supplemental_outputs = swift_common.compile( + compile_result = swift_common.compile( actions = ctx.actions, additional_inputs = ctx.files.swiftc_inputs, cc_infos = get_providers(ctx.attr.deps + extra_deps, CcInfo), @@ -309,8 +309,11 @@ def _swift_test_impl(ctx): target_name = ctx.label.name, workspace_name = ctx.workspace_name, ) + + module_context = compile_result.module_context module_contexts.append(module_context) - all_supplemental_outputs.append(supplemental_outputs) + compilation_outputs = compile_result.compilation_outputs + all_supplemental_outputs.append(compile_result.supplemental_outputs) # Unlike `upstream`, we create a linking_context in order to support # `autolink-extract`. See `a1395155c6a27d76aab5e1a93455259a0ac10b2f` and @@ -318,7 +321,7 @@ def _swift_test_impl(ctx): linking_context, _ = swift_common.create_linking_context_from_compilation_outputs( actions = ctx.actions, alwayslink = True, - compilation_outputs = cc_compilation_outputs, + compilation_outputs = compilation_outputs, feature_configuration = feature_configuration, include_dev_srch_paths = include_dev_srch_paths, label = ctx.label, @@ -331,8 +334,6 @@ def _swift_test_impl(ctx): swift_toolchain = swift_toolchain, ) additional_linking_contexts.append(linking_context) - else: - cc_compilation_outputs = cc_common.create_compilation_outputs() additional_linking_contexts.append(malloc_linking_context(ctx)) @@ -399,9 +400,7 @@ def _swift_test_impl(ctx): ), ), OutputGroupInfo( - **supplemental_compilation_output_groups( - *all_supplemental_outputs - ) + **supplemental_compilation_output_groups(*all_supplemental_outputs) ), coverage_common.instrumented_files_info( ctx, diff --git a/swift/toolchains/config/compile_config.bzl b/swift/toolchains/config/compile_config.bzl index 23faf6e59..226b6fe4c 100644 --- a/swift/toolchains/config/compile_config.bzl +++ b/swift/toolchains/config/compile_config.bzl @@ -1761,8 +1761,7 @@ def _package_name_configurator(prerequisites, args): def _index_while_building_configurator(prerequisites, args): """Adds flags for indexstore generation to the command line.""" - if not _is_index_store_path_overridden(prerequisites.user_compile_flags): - args.add("-index-store-path", prerequisites.indexstore_directory.path) + args.add("-index-store-path", prerequisites.indexstore_directory.path) def _global_index_store_configurator(prerequisites, args): """Adds flags for index-store generation to the command line.""" @@ -1828,24 +1827,6 @@ def _is_wmo_manually_requested(user_compile_flags): return True return False -def _is_index_store_path_overridden(user_compile_flags): - """Returns `True` if `-index-store-path` is in the given list of compiler flags. - - Index while building is disabled when the copts include a custom - `-index-store-path`. - - Args: - user_compile_flags: A list of compiler flags to scan for index store - usage. - - Returns: - True if `-index-store-path` is set in the given list of flags. - """ - for opt in user_compile_flags: - if opt == "-index-store-path": - return True - return False - def _exclude_swift_incompatible_define(define): """A `map_each` helper that excludes a define if it is not Swift-compatible.