diff --git a/docs/BUILD b/docs/BUILD index ee56918102..38350e5631 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -19,6 +19,7 @@ bzl_library( "@io_bazel_rules_rust//proto:rules", "@io_bazel_rules_rust//rust:rules", "@io_bazel_rules_rust//wasm_bindgen:rules", + "@io_bazel_rules_rust//worker:rules", "@rules_proto//proto:rules", ], ) diff --git a/docs/index.md b/docs/index.md index 2035bc3d18..c7022f4ee3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -69,6 +69,15 @@ Similarly, `rustfmt_version` may also be configured: ```python rust_repositories(rustfmt_version = "1.4.20") ``` +# Using Bazel Persistent Workers + +Iterating on Rust code may benefit from [incremental compilation](https://doc.rust-lang.org/edition-guide/rust-2018/the-compiler/incremental-compilation-for-faster-compiles.html). This is supported by using a [Bazel Persistent Worker](https://docs.bazel.build/versions/master/persistent-workers.html). While Bazel can determine what needs to be rebuilt at a crate level, the compiler can speed up building a single crate by sharing information across runs. It does this by storing intermediate information in a directory across invocations. This is enabled by default in Cargo. Persistent workers bring this feature to Bazel. + +The Rust rules have preliminary support for workers. Pass `use_worker = True` to enable this when available. + +```python +rust_repositories(use_worker = True) +``` ## External Dependencies diff --git a/docs/rust_repositories.md b/docs/rust_repositories.md index d38d5e8b12..82471c564b 100644 --- a/docs/rust_repositories.md +++ b/docs/rust_repositories.md @@ -139,7 +139,7 @@ Generates a toolchain-bearing repository that declares the toolchains from some ## rust_repositories 
-rust_repositories(version, iso_date, rustfmt_version, edition, dev_components, sha256s)
+rust_repositories(version, iso_date, rustfmt_version, edition, dev_components, sha256s, use_worker)
Emits a default set of toolchains for Linux, OSX, and Freebsd @@ -158,6 +158,8 @@ This would match for `exec_triple = "x86_64-unknown-linux-gnu"`. If not specifi See `load_arbitrary_tool` in `@io_bazel_rules_rust//rust:repositories.bzl` for more details. +The `use_worker` boolean enables [Bazel Persistent Workers] to be used when available. Not all execution platforms have a pre-built binary worker available. In such a case, the rules will fall back to invoking `rustc` directly. Supported platforms: +- linux-x86_64 **PARAMETERS** @@ -170,6 +172,7 @@ See `load_arbitrary_tool` in `@io_bazel_rules_rust//rust:repositories.bzl` for m | edition | The rust edition to be used by default (2015 (default) or 2018) | None | | dev_components | Whether to download the rustc-dev components (defaults to False). Requires version to be "nightly". | False | | sha256s | A dict associating tool subdirectories to sha256 hashes. | None | +| use_worker | Set this to True to enable worker. | False | diff --git a/examples/examples_deps.bzl b/examples/examples_deps.bzl index 0aa569e2a2..cf4145553c 100644 --- a/examples/examples_deps.bzl +++ b/examples/examples_deps.bzl @@ -14,7 +14,7 @@ load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependen def deps(): """Define dependencies for `rules_rust` examples""" - rust_repositories() + rust_repositories(use_worker = True) rust_bindgen_repositories() diff --git a/proto/proto.bzl b/proto/proto.bzl index 797bb741b2..390c25b5dc 100644 --- a/proto/proto.bzl +++ b/proto/proto.bzl @@ -305,6 +305,7 @@ rust_proto_library = rule( "@io_bazel_rules_rust//proto:toolchain", "@io_bazel_rules_rust//rust:toolchain", "@bazel_tools//tools/cpp:toolchain_type", + "@io_bazel_rules_rust//worker:toolchain_type", ], doc = """\ Builds a Rust library crate from a set of `proto_library`s. @@ -384,6 +385,7 @@ rust_grpc_library = rule( "@io_bazel_rules_rust//proto:toolchain", "@io_bazel_rules_rust//rust:toolchain", "@bazel_tools//tools/cpp:toolchain_type", + "@io_bazel_rules_rust//worker:toolchain_type", ], doc = """\ Builds a Rust library crate from a set of `proto_library`s suitable for gRPC. diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index 718e5c574e..5f9c2e015a 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -495,6 +495,7 @@ rust_library = rule( toolchains = [ "@io_bazel_rules_rust//rust:toolchain", "@bazel_tools//tools/cpp:toolchain_type", + "@io_bazel_rules_rust//worker:toolchain_type", ], doc = """\ Builds a Rust library crate. @@ -585,6 +586,7 @@ rust_binary = rule( toolchains = [ "@io_bazel_rules_rust//rust:toolchain", "@bazel_tools//tools/cpp:toolchain_type", + "@io_bazel_rules_rust//worker:toolchain_type", ], doc = """\ Builds a Rust binary crate. @@ -683,6 +685,7 @@ rust_test = rule( toolchains = [ "@io_bazel_rules_rust//rust:toolchain", "@bazel_tools//tools/cpp:toolchain_type", + "@io_bazel_rules_rust//worker:toolchain_type", ], doc = """\ Builds a Rust test crate. @@ -831,6 +834,7 @@ rust_test_binary = rule( toolchains = [ "@io_bazel_rules_rust//rust:toolchain", "@bazel_tools//tools/cpp:toolchain_type", + "@io_bazel_rules_rust//worker:toolchain_type", ], doc = """\ Builds a Rust test binary, without marking this rule as a Bazel test. @@ -854,6 +858,7 @@ rust_benchmark = rule( toolchains = [ "@io_bazel_rules_rust//rust:toolchain", "@bazel_tools//tools/cpp:toolchain_type", + "@io_bazel_rules_rust//worker:toolchain_type", ], doc = """\ Builds a Rust benchmark test. diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index fd1754bcdd..2073df9162 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -412,7 +412,8 @@ def construct_arguments( build_env_file, build_flags_files, maker_path = None, - aspect = False): + aspect = False, + use_worker = False): """Builds an Args object containing common rustc flags Args: @@ -431,6 +432,7 @@ def construct_arguments( build_flags_files (list): The output files of a `cargo_build_script` actions containing rustc build flags maker_path (File): An optional clippy marker file aspect (bool): True if called in an aspect context. + use_worker (bool): If True, sets up the arguments in a worker-compatible fashion Returns: tuple: A tuple of the following items @@ -445,6 +447,10 @@ def construct_arguments( # Wrapper args first args = ctx.actions.args() + if use_worker: + # Write the args to a param file that will be used by Bazel to send messages to the worker. + args.set_param_file_format("multiline") + args.use_param_file("@%s", use_always = True) if build_env_file != None: args.add("--env-file", build_env_file) @@ -589,6 +595,8 @@ def rustc_compile_action( - (DepInfo): The transitive dependencies of this crate. - (DefaultInfo): The output file for this crate, and its runfiles. """ + worker_binary = ctx.toolchains["@io_bazel_rules_rust//worker:toolchain_type"].worker_binary + dep_info, build_info = collect_deps( ctx.label, crate_info.deps, @@ -623,6 +631,7 @@ def rustc_compile_action( out_dir, build_env_file, build_flags_files, + use_worker = worker_binary != None, ) if hasattr(ctx.attr, "version") and ctx.attr.version != "0.0.0": @@ -630,13 +639,27 @@ def rustc_compile_action( else: formatted_version = "" + if worker_binary != None: + executable = worker_binary + tools = [ctx.executable._process_wrapper] + arguments = [ctx.executable._process_wrapper.path, toolchain.rustc.path, ctx.var["COMPILATION_MODE"], args] + execution_requirements = {"supports-workers": "1"} + else: + # Not all execution platforms support a worker. + executable = ctx.executable._process_wrapper + tools = [] + arguments = [args] + execution_requirements = {} + ctx.actions.run( - executable = ctx.executable._process_wrapper, + executable = executable, inputs = compile_inputs, outputs = [crate_info.output], + tools = tools, env = env, - arguments = [args], + arguments = arguments, mnemonic = "Rustc", + execution_requirements = execution_requirements, progress_message = "Compiling Rust {} {}{} ({} files)".format( crate_info.type, ctx.label.name, diff --git a/rust/repositories.bzl b/rust/repositories.bzl index 2bdec46dd4..8dc29b91ec 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -2,6 +2,7 @@ load(":known_shas.bzl", "FILE_KEY_TO_SHA") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("//worker:repositories.bzl", "rust_worker_repositories", "rust_worker_toolchains") load( "//rust/platform:triple_mappings.bzl", "system_to_binary_ext", @@ -21,7 +22,8 @@ def rust_repositories( rustfmt_version = "1.4.20", edition = None, dev_components = False, - sha256s = None): + sha256s = None, + use_worker = False): """Emits a default set of toolchains for Linux, OSX, and Freebsd Skip this macro and call the `rust_repository_set` macros directly if you need a compiler for \ @@ -47,6 +49,7 @@ def rust_repositories( edition: The rust edition to be used by default (2015 (default) or 2018) dev_components: Whether to download the rustc-dev components (defaults to False). Requires version to be "nightly". sha256s: A dict associating tool subdirectories to sha256 hashes. + use_worker: boolean. Set to True to use Bazel workers for Rust. This downloads binaries for https://github.com/nikhilm/rustc-worker. """ if dev_components and version != "nightly": @@ -142,6 +145,17 @@ def rust_repositories( edition = edition, ) + rust_worker_repositories() + + # Register the real toolchains. + if use_worker: + rust_worker_toolchains() + + # Register a fallback for when workers are not enabled or not available for the execution platform. + native.register_toolchains( + "@io_bazel_rules_rust//worker:dummy", + ) + def _check_version_valid(version, iso_date, param_prefix = ""): """Verifies that the provided rust version and iso_date make sense.""" diff --git a/worker/BUILD b/worker/BUILD new file mode 100644 index 0000000000..c4b813fd6e --- /dev/null +++ b/worker/BUILD @@ -0,0 +1,40 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("//worker:toolchain.bzl", "worker_toolchain") + +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "repositories.bzl", +]) + +bzl_library( + name = "rules", + srcs = glob(["**/*.bzl"]), +) + +toolchain_type(name = "toolchain_type") + +worker_toolchain(name = "worker_dummy") + +toolchain( + name = "dummy", + toolchain = ":worker_dummy", + toolchain_type = ":toolchain_type", +) + +# These toolchains are only registered if workers are enabled in rust_repositories(). + +worker_toolchain( + name = "worker_linux_x86_64", + worker_binary = "@rust_worker_linux_x86_64//file", +) + +toolchain( + name = "linux_x86_64", + exec_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + toolchain = ":worker_linux_x86_64", + toolchain_type = ":toolchain_type", +) diff --git a/worker/README.md b/worker/README.md new file mode 100644 index 0000000000..3cdb08ac48 --- /dev/null +++ b/worker/README.md @@ -0,0 +1,14 @@ + +# Rust Persistent Worker + +The Rust Persistent Worker is itself implemented in Rust. It is built via Cargo and distributed as binaries. Source and release binaries are maintained at the [rustc-worker project](https://github.com/nikhilm/rustc-worker). Contributions should be submitted there and then the version of the binaries updated in `worker/repositories.bzl`. + +## Why isn't this built via Bazel? + +Bootstrapping the worker using these same rules (e.g. `rust_binary`) may be possible, but is not supported right now. There are a couple of challenges: +1. Since the worker has dependencies on various crates, it uses cargo-raze, which generates relevant rules. This means "don't use the worker to build this target" is transitive and such information needs to be propagated down the tree in a way that works with restrictions in Bazel. Initial experiments repeatedly encountered failures due to Bazel treating the rule dependency on the worker executable target as a cycle, even when building in non-worker mode. This may be user error or a restriction in Bazel. Until that is addressed, the easiest way to fix this is to change cargo-raze to customize what rules are used, and provide `rust_no_worker_binary` and `rust_no_worker_library` rules that do not have this cycle. +2. Figuring out a good strategy for dependencies. Since Bazel doesn't really have transitive dependencies, attempts to build this worker from source necessarily require users of these rules to register all the external repositories for the worker in their `WORKSPACE`. This could cause collisions with other dependencies. In addition, if the `_no_worker_` approach above is adopted, users will lose the benefits of workers for those dependencies (like `protobuf`) shared between the worker and their code. There is no satisfactory solution for this right now. + +## How about rewriting the worker in C++? + +That is certainly an option! diff --git a/worker/repositories.bzl b/worker/repositories.bzl new file mode 100644 index 0000000000..cceaed11a0 --- /dev/null +++ b/worker/repositories.bzl @@ -0,0 +1,18 @@ +# buildifier: disable=module-docstring +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") + +def rust_worker_repositories(): + """Registers rustc-worker binary archives.""" + http_file( + name = "rust_worker_linux_x86_64", + executable = True, + sha256 = "c7ee178d658a9ff9c9b10f7acce48af57227170d454741072aff5fabf923f8fb", + urls = ["https://github.com/nikhilm/rustc-worker/releases/download/v0.2.0/rustc-worker-0.2.0-linux-x86_64"], + ) + +# buildifier: disable=unnamed-macro +def rust_worker_toolchains(): + """Registers worker toolchains for supported platforms.""" + native.register_toolchains( + "@io_bazel_rules_rust//worker:linux_x86_64", + ) diff --git a/worker/toolchain.bzl b/worker/toolchain.bzl new file mode 100644 index 0000000000..4a30ab7db5 --- /dev/null +++ b/worker/toolchain.bzl @@ -0,0 +1,13 @@ +# buildifier: disable=module-docstring +def _worker_toolchain_impl(ctx): + toolchain_info = platform_common.ToolchainInfo( + worker_binary = ctx.executable.worker_binary, + ) + return [toolchain_info] + +worker_toolchain = rule( + implementation = _worker_toolchain_impl, + attrs = { + "worker_binary": attr.label(allow_single_file = True, executable = True, cfg = "exec"), + }, +)