Persistent worker support

docs/BUILD

@@ -19,6 +19,7 @@ bzl_library(
         "@io_bazel_rules_rust//proto:rules",
         "@io_bazel_rules_rust//rust:rules",
         "@io_bazel_rules_rust//wasm_bindgen:rules",
+        "@io_bazel_rules_rust//worker:rules",
         "@rules_proto//proto:rules",
     ],
 )

docs/index.md

@@ -69,6 +69,15 @@ Similarly, `rustfmt_version` may also be configured:
 ```python
 rust_repositories(rustfmt_version = "1.4.20")
 ```
+# Using Bazel Persistent Workers
+
+Iterating on Rust code may benefit from [incremental compilation](https://doc.rust-lang.org/edition-guide/rust-2018/the-compiler/incremental-compilation-for-faster-compiles.html). This is supported by using a [Bazel Persistent Worker](https://docs.bazel.build/versions/master/persistent-workers.html). While Bazel can determine what needs to be rebuilt at a crate level, the compiler can speed up building a single crate by sharing information across runs. It does this by storing intermediate information in a directory across invocations. This is enabled by default in Cargo. Persistent workers bring this feature to Bazel.
+
+The Rust rules have preliminary support for workers. Pass `use_worker = True` to enable this when available.
+
+```python
+rust_repositories(use_worker = True)
+```
 
 ## External Dependencies
 

docs/rust_repositories.md

@@ -139,7 +139,7 @@ Generates a toolchain-bearing repository that declares the toolchains from some
 ## rust_repositories
 
 <pre>
-rust_repositories(<a href="#rust_repositories-version">version</a>, <a href="#rust_repositories-iso_date">iso_date</a>, <a href="#rust_repositories-rustfmt_version">rustfmt_version</a>, <a href="#rust_repositories-edition">edition</a>, <a href="#rust_repositories-dev_components">dev_components</a>, <a href="#rust_repositories-sha256s">sha256s</a>)
+rust_repositories(<a href="#rust_repositories-version">version</a>, <a href="#rust_repositories-iso_date">iso_date</a>, <a href="#rust_repositories-rustfmt_version">rustfmt_version</a>, <a href="#rust_repositories-edition">edition</a>, <a href="#rust_repositories-dev_components">dev_components</a>, <a href="#rust_repositories-sha256s">sha256s</a>, <a href="#rust_repositories-use_worker">use_worker</a>)
 </pre>
 
 Emits a default set of toolchains for Linux, OSX, and Freebsd
@@ -158,6 +158,8 @@ This would match for `exec_triple = "x86_64-unknown-linux-gnu"`.  If not specifi
 
 See `load_arbitrary_tool` in `@io_bazel_rules_rust//rust:repositories.bzl` for more details.
 
+The `use_worker` boolean enables [Bazel Persistent Workers] to be used when available. Not all execution platforms have a pre-built binary worker available. In such a case, the rules will fall back to invoking `rustc` directly. Supported platforms:
+- linux-x86_64
 
 **PARAMETERS**
 
@@ -170,6 +172,7 @@ See `load_arbitrary_tool` in `@io_bazel_rules_rust//rust:repositories.bzl` for m
 | <a id="rust_repositories-edition"></a>edition |  The rust edition to be used by default (2015 (default) or 2018)   |  <code>None</code> |
 | <a id="rust_repositories-dev_components"></a>dev_components |  Whether to download the rustc-dev components (defaults to False). Requires version to be "nightly".   |  <code>False</code> |
 | <a id="rust_repositories-sha256s"></a>sha256s |  A dict associating tool subdirectories to sha256 hashes.   |  <code>None</code> |
+| <a id="rust_repositories-use_worker"></a>use_worker |  Set this to True to enable worker.   |  <code>False</code> |
 
 
 <a id="#rust_repository_set"></a>

examples/examples_deps.bzl

@@ -14,7 +14,7 @@ load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependen
 def deps():
     """Define dependencies for `rules_rust` examples"""
 
-    rust_repositories()
+    rust_repositories(use_worker = True)
 
     rust_bindgen_repositories()
 

proto/proto.bzl

@@ -305,6 +305,7 @@ rust_proto_library = rule(
         "@io_bazel_rules_rust//proto:toolchain",
         "@io_bazel_rules_rust//rust:toolchain",
         "@bazel_tools//tools/cpp:toolchain_type",
+        "@io_bazel_rules_rust//worker:toolchain_type",
     ],
     doc = """\
 Builds a Rust library crate from a set of `proto_library`s.
@@ -384,6 +385,7 @@ rust_grpc_library = rule(
         "@io_bazel_rules_rust//proto:toolchain",
         "@io_bazel_rules_rust//rust:toolchain",
         "@bazel_tools//tools/cpp:toolchain_type",
+        "@io_bazel_rules_rust//worker:toolchain_type",
     ],
     doc = """\
 Builds a Rust library crate from a set of `proto_library`s suitable for gRPC.

rust/private/rust.bzl

@@ -495,6 +495,7 @@ rust_library = rule(
     toolchains = [
         "@io_bazel_rules_rust//rust:toolchain",
         "@bazel_tools//tools/cpp:toolchain_type",
+        "@io_bazel_rules_rust//worker:toolchain_type",
     ],
     doc = """\
 Builds a Rust library crate.
@@ -585,6 +586,7 @@ rust_binary = rule(
     toolchains = [
         "@io_bazel_rules_rust//rust:toolchain",
         "@bazel_tools//tools/cpp:toolchain_type",
+        "@io_bazel_rules_rust//worker:toolchain_type",
     ],
     doc = """\
 Builds a Rust binary crate.
@@ -683,6 +685,7 @@ rust_test = rule(
     toolchains = [
         "@io_bazel_rules_rust//rust:toolchain",
         "@bazel_tools//tools/cpp:toolchain_type",
+        "@io_bazel_rules_rust//worker:toolchain_type",
     ],
     doc = """\
 Builds a Rust test crate.
@@ -831,6 +834,7 @@ rust_test_binary = rule(
     toolchains = [
         "@io_bazel_rules_rust//rust:toolchain",
         "@bazel_tools//tools/cpp:toolchain_type",
+        "@io_bazel_rules_rust//worker:toolchain_type",
     ],
     doc = """\
 Builds a Rust test binary, without marking this rule as a Bazel test.
@@ -854,6 +858,7 @@ rust_benchmark = rule(
     toolchains = [
         "@io_bazel_rules_rust//rust:toolchain",
         "@bazel_tools//tools/cpp:toolchain_type",
+        "@io_bazel_rules_rust//worker:toolchain_type",
     ],
     doc = """\
 Builds a Rust benchmark test.

rust/private/rustc.bzl

@@ -412,7 +412,8 @@ def construct_arguments(
         build_env_file,
         build_flags_files,
         maker_path = None,
-        aspect = False):
+        aspect = False,
+        use_worker = False):
     """Builds an Args object containing common rustc flags
 
     Args:
@@ -431,6 +432,7 @@ def construct_arguments(
         build_flags_files (list): The output files of a `cargo_build_script` actions containing rustc build flags
         maker_path (File): An optional clippy marker file
         aspect (bool): True if called in an aspect context.
+        use_worker (bool): If True, sets up the arguments in a worker-compatible fashion
 
     Returns:
         tuple: A tuple of the following items
@@ -445,6 +447,10 @@ def construct_arguments(
 
     # Wrapper args first
     args = ctx.actions.args()
+    if use_worker:
+        # Write the args to a param file that will be used by Bazel to send messages to the worker.
+        args.set_param_file_format("multiline")
+        args.use_param_file("@%s", use_always = True)
 
     if build_env_file != None:
         args.add("--env-file", build_env_file)
@@ -589,6 +595,8 @@ def rustc_compile_action(
             - (DepInfo): The transitive dependencies of this crate.
             - (DefaultInfo): The output file for this crate, and its runfiles.
     """
+    worker_binary = ctx.toolchains["@io_bazel_rules_rust//worker:toolchain_type"].worker_binary
+
     dep_info, build_info = collect_deps(
         ctx.label,
         crate_info.deps,
@@ -623,20 +631,35 @@ def rustc_compile_action(
         out_dir,
         build_env_file,
         build_flags_files,
+        use_worker = worker_binary != None,
     )
 
     if hasattr(ctx.attr, "version") and ctx.attr.version != "0.0.0":
         formatted_version = " v{}".format(ctx.attr.version)
     else:
         formatted_version = ""
 
+    if worker_binary != None:
+        executable = worker_binary
+        tools = [ctx.executable._process_wrapper]
+        arguments = [ctx.executable._process_wrapper.path, toolchain.rustc.path, ctx.var["COMPILATION_MODE"], args]
+        execution_requirements = {"supports-workers": "1"}
+    else:
+        # Not all execution platforms support a worker.
+        executable = ctx.executable._process_wrapper
+        tools = []
+        arguments = [args]
+        execution_requirements = {}
+
     ctx.actions.run(
-        executable = ctx.executable._process_wrapper,
+        executable = executable,
         inputs = compile_inputs,
         outputs = [crate_info.output],
+        tools = tools,
         env = env,
-        arguments = [args],
+        arguments = arguments,
         mnemonic = "Rustc",
+        execution_requirements = execution_requirements,
         progress_message = "Compiling Rust {} {}{} ({} files)".format(
             crate_info.type,
             ctx.label.name,

rust/repositories.bzl

@@ -2,6 +2,7 @@
 load(":known_shas.bzl", "FILE_KEY_TO_SHA")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+load("//worker:repositories.bzl", "rust_worker_repositories", "rust_worker_toolchains")
 load(
     "//rust/platform:triple_mappings.bzl",
     "system_to_binary_ext",
@@ -21,7 +22,8 @@ def rust_repositories(
         rustfmt_version = "1.4.20",
         edition = None,
         dev_components = False,
-        sha256s = None):
+        sha256s = None,
+        use_worker = False):
     """Emits a default set of toolchains for Linux, OSX, and Freebsd
 
     Skip this macro and call the `rust_repository_set` macros directly if you need a compiler for \
@@ -47,6 +49,7 @@ def rust_repositories(
         edition: The rust edition to be used by default (2015 (default) or 2018)
         dev_components: Whether to download the rustc-dev components (defaults to False). Requires version to be "nightly".
         sha256s: A dict associating tool subdirectories to sha256 hashes.
+        use_worker: boolean. Set to True to use Bazel workers for Rust. This downloads binaries for https://github.com/nikhilm/rustc-worker.
     """
 
     if dev_components and version != "nightly":
@@ -142,6 +145,17 @@ def rust_repositories(
         edition = edition,
     )
 
+    rust_worker_repositories()
+
+    # Register the real toolchains.
+    if use_worker:
+        rust_worker_toolchains()
+
+    # Register a fallback for when workers are not enabled or not available for the execution platform.
+    native.register_toolchains(
+        "@io_bazel_rules_rust//worker:dummy",
+    )
+
 def _check_version_valid(version, iso_date, param_prefix = ""):
     """Verifies that the provided rust version and iso_date make sense."""
 

worker/BUILD

@@ -0,0 +1,40 @@
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+load("//worker:toolchain.bzl", "worker_toolchain")
+
+package(default_visibility = ["//visibility:public"])
+
+exports_files([
+    "repositories.bzl",
+])
+
+bzl_library(
+    name = "rules",
+    srcs = glob(["**/*.bzl"]),
+)
+
+toolchain_type(name = "toolchain_type")
+
+worker_toolchain(name = "worker_dummy")
+
+toolchain(
+    name = "dummy",
+    toolchain = ":worker_dummy",
+    toolchain_type = ":toolchain_type",
+)
+
+# These toolchains are only registered if workers are enabled in rust_repositories().
+
+worker_toolchain(
+    name = "worker_linux_x86_64",
+    worker_binary = "@rust_worker_linux_x86_64//file",
+)
+
+toolchain(
+    name = "linux_x86_64",
+    exec_compatible_with = [
+        "@platforms//os:linux",
+        "@platforms//cpu:x86_64",
+    ],
+    toolchain = ":worker_linux_x86_64",
+    toolchain_type = ":toolchain_type",
+)

worker/README.md

@@ -0,0 +1,14 @@
+
+# Rust Persistent Worker
+
+The Rust Persistent Worker is itself implemented in Rust. It is built via Cargo and distributed as binaries. Source and release binaries are maintained at the [rustc-worker project](https://github.com/nikhilm/rustc-worker). Contributions should be submitted there and then the version of the binaries updated in `worker/repositories.bzl`.
+
+## Why isn't this built via Bazel?
+
+Bootstrapping the worker using these same rules (e.g. `rust_binary`) may be possible, but is not supported right now. There are a couple of challenges:
+1. Since the worker has dependencies on various crates, it uses cargo-raze, which generates relevant rules. This means "don't use the worker to build this target" is transitive and such information needs to be propagated down the tree in a way that works with restrictions in Bazel. Initial experiments repeatedly encountered failures due to Bazel treating the rule dependency on the worker executable target as a cycle, even when building in non-worker mode. This may be user error or a restriction in Bazel. Until that is addressed, the easiest way to fix this is to change cargo-raze to customize what rules are used, and provide `rust_no_worker_binary` and `rust_no_worker_library` rules that do not have this cycle.
+2. Figuring out a good strategy for dependencies. Since Bazel doesn't really have transitive dependencies, attempts to build this worker from source necessarily require users of these rules to register all the external repositories for the worker in their `WORKSPACE`. This could cause collisions with other dependencies. In addition, if the `_no_worker_` approach above is adopted, users will lose the benefits of workers for those dependencies (like `protobuf`) shared between the worker and their code. There is no satisfactory solution for this right now.
+
+## How about rewriting the worker in C++?
+
+That is certainly an option!

worker/repositories.bzl

@@ -0,0 +1,18 @@
+# buildifier: disable=module-docstring
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")
+
+def rust_worker_repositories():
+    """Registers rustc-worker binary archives."""
+    http_file(
+        name = "rust_worker_linux_x86_64",
+        executable = True,
+        sha256 = "c7ee178d658a9ff9c9b10f7acce48af57227170d454741072aff5fabf923f8fb",
+        urls = ["https://github.com/nikhilm/rustc-worker/releases/download/v0.2.0/rustc-worker-0.2.0-linux-x86_64"],
+    )
+
+# buildifier: disable=unnamed-macro
+def rust_worker_toolchains():
+    """Registers worker toolchains for supported platforms."""
+    native.register_toolchains(
+        "@io_bazel_rules_rust//worker:linux_x86_64",
+    )

worker/toolchain.bzl

@@ -0,0 +1,13 @@
+# buildifier: disable=module-docstring
+def _worker_toolchain_impl(ctx):
+    toolchain_info = platform_common.ToolchainInfo(
+        worker_binary = ctx.executable.worker_binary,
+    )
+    return [toolchain_info]
+
+worker_toolchain = rule(
+    implementation = _worker_toolchain_impl,
+    attrs = {
+        "worker_binary": attr.label(allow_single_file = True, executable = True, cfg = "exec"),
+    },
+)