Skip to content

Commit

Permalink
Multiple header units per lib
Browse files Browse the repository at this point in the history 
Summary:
This provides some flexibility to modularise existing libraries in a more fine-grained fashion. Ideally, libraries should be buckified along the lines of the modules we want to generate (one header unit per target), but this is not always readily feasible. In order, to aid with migration and performance, we introduce the ability to partition the headers into different header units.

`export_header_unit_filter` now accepts a list of regexes, each defining a different group of headers to turn into a unit. Headers will be intercepted and load the corresponding header unit. The list is built sequentially so that headers in subsequent groups can depend on headers in earlier groups.

Reviewed By: apolloww

Differential Revision: D64062362

fbshipit-source-id: e6493ec9e34fbca28a3184f32c05d469c5e40705
  • Loading branch information
@facebook-github-bot
Nhat Minh Le authored and facebook-github-bot committed Oct 10, 2024
1 parent 34ec237 commit afcbcb9
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 67 deletions.
146 changes: 107 additions & 39 deletions cxx/compile.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,18 @@ CxxSrcCompileCommand = record(
error_handler = field([typing.Callable, None], None),
)

_CxxSrcPrecompileCommand = record(
# Source file to compile.
src = field(Artifact),
# The CxxCompileCommand to use to compile this file.
cxx_compile_cmd = field(_CxxCompileCommand),
# Arguments specific to the source file.
args = field(list[typing.Any]),
# Extra argsfile to include after any other header units argsfile but before the
# main argsfiles.
extra_argsfile = field([CompileArgsfile, None], None),
)

# Output of creating compile commands for Cxx source files.
CxxCompileCommandOutput = record(
# List of compile commands for each source file.
Expand Down Expand Up @@ -700,17 +712,17 @@ def _compiler_supports_header_units(compiler_info: typing.Any):
return (compiler_info.compiler_type == "clang" and
compiler_info.supports_two_phase_compilation)

def _get_module_name(ctx: AnalysisContext) -> str:
def _get_module_name(ctx: AnalysisContext, group_name: str) -> str:
return paths.normalize(paths.join(
"__header_units__",
ctx.label.package,
"{}.h".format(ctx.label.name),
"{}{}.h".format(ctx.label.name, group_name),
))

def _get_import_filename(ctx: AnalysisContext) -> str:
def _get_import_filename(ctx: AnalysisContext, group_name: str) -> str:
return paths.normalize(paths.join(
ctx.label.package,
"__import__{}.h".format(ctx.label.name),
"__import__{}{}.h".format(ctx.label.name, group_name),
))

def _is_standalone_header(header: CHeader) -> bool:
Expand Down Expand Up @@ -746,24 +758,14 @@ def _convert_raw_header(
named = False,
)

def create_precompile_cmd(
def _create_precompile_cmd(
ctx: AnalysisContext,
compiler_info: typing.Any,
preprocessors: list[CPreprocessor],
include_headers: str | None,
compile_cmd_output: CxxCompileCommandOutput) -> CxxSrcCompileCommand | None:
"""
Forms the CxxSrcCompileCommand to use for all headers. Returns CxxCompileCommandOutput
containing a pair of the generated compile command and argsfile output.
"""
toolchain = get_cxx_toolchain_info(ctx)
if not _compiler_supports_header_units(toolchain.cxx_compiler_info):
return None

ext = CxxExtension(".cpp")
if ext not in compile_cmd_output.base_compile_cmds:
return None
cmd = compile_cmd_output.base_compile_cmds[ext]

header_group: str | None,
group_name: str,
extra_preprocessors: list[CPreprocessor],
cmd: _CxxCompileCommand) -> _CxxSrcPrecompileCommand:
include_dirs = flatten([x.include_dirs for x in preprocessors])
converted_headers = [
_convert_raw_header(ctx, raw_header, include_dirs)
Expand All @@ -772,13 +774,11 @@ def create_precompile_cmd(
headers = [
header
for header in flatten([x.headers for x in preprocessors]) + converted_headers
if (_is_standalone_header(header) if include_headers == None else regex_match(include_headers, header.name))
if (_is_standalone_header(header) if header_group == None else regex_match(header_group, header.name))
]
if not headers:
return None

module_name = _get_module_name(ctx)
import_name = _get_import_filename(ctx)
module_name = _get_module_name(ctx, group_name)
import_name = _get_import_filename(ctx, group_name)
input_header = ctx.actions.write(module_name, "")

import_stub = ctx.actions.write(
Expand All @@ -804,10 +804,10 @@ module "{}" {{
export *
}}
""".format(module_name, module_name)
modulemap_file = ctx.actions.write("module.modulemap", modulemap_content)
modulemap_file = ctx.actions.write("module.modulemap" + group_name, modulemap_content)

src_dir = ctx.actions.symlinked_dir(
"header-unit",
"header-unit" + group_name,
symlinked_files | {
module_name: input_header,
import_name: import_stub,
Expand All @@ -825,31 +825,45 @@ module "{}" {{
"-Wno-c++98-compat-extra-semi",
])

extra_argsfile = None
if extra_preprocessors:
extra_argsfile = _mk_header_units_argsfile(
ctx = ctx,
compiler_info = compiler_info,
preprocessor = cxx_merge_cpreprocessors(ctx, extra_preprocessors, []),
name = "export" + group_name,
ext = CxxExtension(".cpp"),
)

for header in headers:
args.extend(["-include", paths.join(header.namespace, header.name)])
args.extend(["-xc++-user-header", "-fmodule-header"])
args.extend(["-fmodule-name={}".format(module_name)])
args.extend(["-Xclang", cmd_args(input_header, format = "-fmodules-embed-file={}")])
args.extend(["--precompile", input_header])

return CxxSrcCompileCommand(
return _CxxSrcPrecompileCommand(
src = src_dir,
cxx_compile_cmd = cmd,
args = args,
is_header = True,
extra_argsfile = extra_argsfile,
)

def precompile_cxx(
def _precompile_single_cxx(
ctx: AnalysisContext,
impl_params: CxxRuleConstructorParams,
src_compile_cmd: CxxSrcCompileCommand) -> HeaderUnit:
group_name: str,
src_compile_cmd: _CxxSrcPrecompileCommand) -> HeaderUnit:
identifier = src_compile_cmd.src.short_path

filename = "{}.pcm".format(identifier)
module = ctx.actions.declare_output("__pcm_files__", filename)

cmd = cmd_args(src_compile_cmd.cxx_compile_cmd.base_compile_cmd)
cmd.add(src_compile_cmd.cxx_compile_cmd.header_units_argsfile.cmd_form)
if src_compile_cmd.cxx_compile_cmd.header_units_argsfile:
cmd.add(src_compile_cmd.cxx_compile_cmd.header_units_argsfile.cmd_form)
if src_compile_cmd.extra_argsfile:
cmd.add(src_compile_cmd.extra_argsfile.cmd_form)
cmd.add(src_compile_cmd.cxx_compile_cmd.argsfile.cmd_form)
cmd.add(src_compile_cmd.args)
cmd.add(["-o", module.as_output()])
Expand Down Expand Up @@ -890,12 +904,66 @@ def precompile_cxx(
)

return HeaderUnit(
name = _get_module_name(ctx),
name = _get_module_name(ctx, group_name),
module = module,
include_dir = src_compile_cmd.src,
import_include = _get_import_filename(ctx) if impl_params.export_header_unit == "preload" else None,
import_include = _get_import_filename(ctx, group_name) if impl_params.export_header_unit == "preload" else None,
)

def precompile_cxx(
ctx: AnalysisContext,
impl_params: CxxRuleConstructorParams,
preprocessors: list[CPreprocessor],
compile_cmd_output: CxxCompileCommandOutput) -> list[CPreprocessor]:
"""
Produces header units for the target and returns a list of preprocessors enabling
them; depending on those preprocessors will allow the corresponding module to load.
"""
toolchain = get_cxx_toolchain_info(ctx)
if not _compiler_supports_header_units(toolchain.cxx_compiler_info):
return []

ext = CxxExtension(".cpp")
if ext not in compile_cmd_output.base_compile_cmds:
return []
cmd = compile_cmd_output.base_compile_cmds[ext]

header_unit_preprocessors = []
if len(impl_params.export_header_unit_filter) <= 1:
group = None
if impl_params.export_header_unit_filter:
group = impl_params.export_header_unit_filter[0]
precompile_cmd = _create_precompile_cmd(
ctx = ctx,
compiler_info = toolchain.cxx_compiler_info,
preprocessors = preprocessors,
header_group = group,
group_name = "",
extra_preprocessors = [],
cmd = cmd,
)
header_unit = _precompile_single_cxx(ctx, impl_params, "", precompile_cmd)
header_unit_preprocessors.append(CPreprocessor(header_units = [header_unit]))
else:
# Chain preprocessors in order.
i = 0
for header_group in impl_params.export_header_unit_filter:
name = ".{}".format(i)
precompile_cmd = _create_precompile_cmd(
ctx = ctx,
compiler_info = toolchain.cxx_compiler_info,
preprocessors = preprocessors,
header_group = header_group,
group_name = name,
extra_preprocessors = header_unit_preprocessors,
cmd = cmd,
)
header_unit = _precompile_single_cxx(ctx, impl_params, name, precompile_cmd)
header_unit_preprocessors.append(CPreprocessor(header_units = [header_unit]))
i += 1

return header_unit_preprocessors

def cxx_objects_sub_targets(outs: list[CxxCompileOutput]) -> dict[str, list[Provider]]:
objects_sub_targets = {}
for obj in outs:
Expand Down Expand Up @@ -1146,8 +1214,8 @@ def _mk_header_units_argsfile(
ctx: AnalysisContext,
compiler_info: typing.Any,
preprocessor: CPreprocessorInfo,
ext: CxxExtension,
private: bool) -> CompileArgsfile | None:
name: str,
ext: CxxExtension) -> CompileArgsfile | None:
"""
Generate and return an argsfile artifact containing all header unit options, and
command args that utilize the argsfile.
Expand All @@ -1159,7 +1227,7 @@ def _mk_header_units_argsfile(
if not _compiler_supports_header_units(compiler_info):
return None

file_name = "{}.{}header_units.argsfile".format(ext.value, "private_" if private else "")
file_name = "{}.{}.header_units_args".format(ext.value, name)
args = cmd_args()
args.add([
# TODO(nml): We only support Clang 17+, which don't need/want the extra -f
Expand Down Expand Up @@ -1228,8 +1296,8 @@ def _generate_base_compile_command(

argsfile = _mk_argsfiles(ctx, impl_params, compiler_info, pre, ext, headers_tag, False)
xcode_argsfile = _mk_argsfiles(ctx, impl_params, compiler_info, pre, ext, headers_tag, True)
header_units_argsfile = _mk_header_units_argsfile(ctx, compiler_info, header_pre, ext, False)
private_header_units_argsfile = _mk_header_units_argsfile(ctx, compiler_info, pre, ext, True)
header_units_argsfile = _mk_header_units_argsfile(ctx, compiler_info, header_pre, "public", ext)
private_header_units_argsfile = _mk_header_units_argsfile(ctx, compiler_info, pre, "private", ext)

allow_cache_upload = cxx_attrs_get_allow_cache_upload(ctx.attrs, default = compiler_info.allow_cache_upload)
return _CxxCompileCommand(
Expand Down
53 changes: 28 additions & 25 deletions cxx/cxx_library.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,6 @@ load(
"CxxSrcCompileCommand",
"compile_cxx",
"create_compile_cmds",
"create_precompile_cmd",
"cxx_objects_sub_targets",
"precompile_cxx",
)
Expand Down Expand Up @@ -209,7 +208,6 @@ load(
"CPreprocessor", # @unused Used as a type
"CPreprocessorForTestsInfo",
"CPreprocessorInfo", # @unused Used as a type
"HeaderUnit", # @unused Used as a type
"cxx_exported_preprocessor_info",
"cxx_inherited_preprocessor_infos",
"cxx_merge_cpreprocessors",
Expand Down Expand Up @@ -329,8 +327,7 @@ _CxxCompiledSourcesOutput = record(
# Non PIC compile outputs
non_pic = field([_CxxLibraryCompileOutput, None]),
# Header unit outputs
header_unit = field([HeaderUnit, None]),
header_unit_preprocessor = field([CPreprocessor, None]),
header_unit_preprocessors = field(list[CPreprocessor]),
)

# The outputs of a cxx_library_parameterized rule.
Expand Down Expand Up @@ -695,17 +692,35 @@ def cxx_library_parameterized(ctx: AnalysisContext, impl_params: CxxRuleConstruc

# Header unit PCM.
if impl_params.generate_sub_targets.header_unit:
if compiled_srcs.header_unit and compiled_srcs.header_unit_preprocessor:
if compiled_srcs.header_unit_preprocessors:
header_unit_preprocessors = []
header_unit_sub_targets = []
for x in compiled_srcs.header_unit_preprocessors:
header_unit_preprocessors.append(x)
header_unit_sub_targets.append([
DefaultInfo(default_outputs = [h.module for h in x.header_units]),
cxx_merge_cpreprocessors(
ctx,
own_exported_preprocessors + header_unit_preprocessors,
propagated_preprocessor_merge_list,
),
])
sub_targets["header-unit"] = [
DefaultInfo(default_output = compiled_srcs.header_unit.module),
cxx_merge_cpreprocessors(
ctx,
own_exported_preprocessors + [compiled_srcs.header_unit_preprocessor],
propagated_preprocessor_merge_list,
DefaultInfo(
default_outputs = [
h.module
for x in header_unit_preprocessors
for h in x.header_units
],
sub_targets = {
str(i): x
for i, x in enumerate(header_unit_sub_targets)
},
),
header_unit_sub_targets[-1][1],
]
if impl_params.export_header_unit:
own_exported_preprocessors.append(compiled_srcs.header_unit_preprocessor)
own_exported_preprocessors.extend(header_unit_preprocessors)
else:
sub_targets["header-unit"] = [
DefaultInfo(),
Expand Down Expand Up @@ -1086,18 +1101,7 @@ def cxx_compile_srcs(
)

# Define header unit.
precompile_cmd = create_precompile_cmd(
ctx = ctx,
preprocessors = own_exported_preprocessors,
include_headers = impl_params.export_header_unit_filter,
compile_cmd_output = compile_cmd_output,
)
if precompile_cmd:
header_unit = precompile_cxx(ctx, impl_params, precompile_cmd)
header_unit_preprocessor = CPreprocessor(header_units = [header_unit])
else:
header_unit = None
header_unit_preprocessor = None
header_unit_preprocessors = precompile_cxx(ctx, impl_params, own_exported_preprocessors, compile_cmd_output)

# Define object files.
pic_cxx_outs = compile_cxx(
Expand Down Expand Up @@ -1154,8 +1158,7 @@ def cxx_compile_srcs(
pic = pic,
pic_optimized = pic_optimized,
non_pic = non_pic,
header_unit = header_unit,
header_unit_preprocessor = header_unit_preprocessor,
header_unit_preprocessors = header_unit_preprocessors,
)

def _form_library_outputs(
Expand Down
4 changes: 2 additions & 2 deletions cxx/cxx_types.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,6 @@ CxxRuleConstructorParams = record(
use_header_units = field(bool, False),
# Whether to export a header unit to all dependents.
export_header_unit = field([str, None], None),
# Filter what headers to include in the header unit.
export_header_unit_filter = field([str, None], None),
# Filter what headers to include in header units.
export_header_unit_filter = field(list[str], []),
)
2 changes: 1 addition & 1 deletion decls/cxx_rules.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -593,7 +593,7 @@ cxx_library = prelude_rule(
"weak_framework_names": attrs.list(attrs.string(), default = []),
"use_header_units": attrs.bool(default = False),
"export_header_unit": attrs.option(attrs.enum(["include", "preload"]), default = None),
"export_header_unit_filter": attrs.option(attrs.string(), default = None),
"export_header_unit_filter": attrs.list(attrs.string(), default = []),
} |
buck.allow_cache_upload_arg()
),
Expand Down

0 comments on commit afcbcb9

Please sign in to comment.