From a88defb969576bc1e2fd353039d39475f308d086 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Wed, 16 Sep 2020 22:21:05 -0700 Subject: [PATCH 01/24] tweak to use a pretend persistent worker executable --- rust/private/rust.bzl | 7 +++++++ rust/private/rustc.bzl | 7 +++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index dfa2fc3a3e..ecc983ab66 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -454,6 +454,13 @@ _rust_common_attrs = { allow_single_file = True, cfg = "exec", ), + "_process_wrapper_worker": attr.label( + default = "@rustc_worker//file", + # Actually we may want the persistent worker to be built by the rust rules themselves, but right now we are just going to use a binary. + executable = True, + allow_single_file = True, + cfg = "exec", + ), } _rust_library_attrs = { diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 23f0a3b0df..a0a53cd517 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -432,6 +432,8 @@ def construct_arguments( # Wrapper args first args = ctx.actions.args() + args.set_param_file_format("multiline") + args.use_param_file("@%s", use_always=True) if build_env_file != None: args.add("--env-file", build_env_file) @@ -611,12 +613,13 @@ def rustc_compile_action( formatted_version = "" ctx.actions.run( - executable = ctx.executable._process_wrapper, + executable = ctx.executable._process_wrapper_worker, inputs = compile_inputs, outputs = [crate_info.output], env = env, - arguments = [args], + arguments = [ctx.executable._process_wrapper.path, args], mnemonic = "Rustc", + execution_requirements = { "supports-workers": "1" }, progress_message = "Compiling Rust {} {}{} ({} files)".format( crate_info.type, ctx.label.name, From ec046ba245e9bae5b0da572d1098dcaa67955a43 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Thu, 17 Sep 2020 08:17:15 -0700 Subject: [PATCH 02/24] add program path to args as well. --- rust/private/rustc.bzl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index a0a53cd517..6a6287dce9 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -434,6 +434,9 @@ def construct_arguments( args = ctx.actions.args() args.set_param_file_format("multiline") args.use_param_file("@%s", use_always=True) + # Pass the actual executable as an arg to the worker. + # TODO: This is also used by clippy, so do the relevant gating later. + args.add("--program_path", ctx.executable._process_wrapper) if build_env_file != None: args.add("--env-file", build_env_file) @@ -617,7 +620,7 @@ def rustc_compile_action( inputs = compile_inputs, outputs = [crate_info.output], env = env, - arguments = [ctx.executable._process_wrapper.path, args], + arguments = [args], mnemonic = "Rustc", execution_requirements = { "supports-workers": "1" }, progress_message = "Compiling Rust {} {}{} ({} files)".format( From 7cdb1694f300191fab68a747857a39a9927e63c7 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Thu, 17 Sep 2020 08:43:25 -0700 Subject: [PATCH 03/24] actually, the program path should be separate to make it easy in the non-persistent case --- rust/private/rustc.bzl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 6a6287dce9..a0a53cd517 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -434,9 +434,6 @@ def construct_arguments( args = ctx.actions.args() args.set_param_file_format("multiline") args.use_param_file("@%s", use_always=True) - # Pass the actual executable as an arg to the worker. - # TODO: This is also used by clippy, so do the relevant gating later. - args.add("--program_path", ctx.executable._process_wrapper) if build_env_file != None: args.add("--env-file", build_env_file) @@ -620,7 +617,7 @@ def rustc_compile_action( inputs = compile_inputs, outputs = [crate_info.output], env = env, - arguments = [args], + arguments = [ctx.executable._process_wrapper.path, args], mnemonic = "Rustc", execution_requirements = { "supports-workers": "1" }, progress_message = "Compiling Rust {} {}{} ({} files)".format( From 12f8abc256e312764d3cdb0c11d707488ad9185d Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Thu, 17 Sep 2020 08:59:59 -0700 Subject: [PATCH 04/24] add original process wrapper to tools depset --- rust/private/rustc.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index a0a53cd517..bc696ee7a2 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -616,6 +616,7 @@ def rustc_compile_action( executable = ctx.executable._process_wrapper_worker, inputs = compile_inputs, outputs = [crate_info.output], + tools = [ctx.executable._process_wrapper], env = env, arguments = [ctx.executable._process_wrapper.path, args], mnemonic = "Rustc", From 55ee86accea98a9673adbc8341def369dae6e5e5 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Thu, 17 Sep 2020 09:57:24 -0700 Subject: [PATCH 05/24] enable incremental compilation --- rust/private/rustc.bzl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index bc696ee7a2..f57dd0c9ef 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -487,6 +487,9 @@ def construct_arguments( args.add("--crate-name=" + crate_info.name) args.add("--crate-type=" + crate_info.type) + # Enable incremental compilation. + args.add("--codegen=incremental=incremental/") + # Mangle symbols to disambiguate crates with the same name extra_filename = "-" + output_hash if output_hash else "" args.add("--codegen=metadata=" + extra_filename) From 8fa4e5481a0b8cf3d97b2c5c9a7f977d7f341be6 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Fri, 18 Sep 2020 10:36:01 -0700 Subject: [PATCH 06/24] remove incremental. rustc-worker should pass it and point at a shared dir. --- rust/private/rustc.bzl | 3 --- 1 file changed, 3 deletions(-) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index f57dd0c9ef..bc696ee7a2 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -487,9 +487,6 @@ def construct_arguments( args.add("--crate-name=" + crate_info.name) args.add("--crate-type=" + crate_info.type) - # Enable incremental compilation. - args.add("--codegen=incremental=incremental/") - # Mangle symbols to disambiguate crates with the same name extra_filename = "-" + output_hash if output_hash else "" args.add("--codegen=metadata=" + extra_filename) From 6b9ee799de81b15cbf0288ce6078639ba7fb9e29 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Sun, 20 Sep 2020 13:07:46 -0700 Subject: [PATCH 07/24] pass workspace name to create cache dir --- rust/private/rustc.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index bc696ee7a2..6df25c03c1 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -618,7 +618,7 @@ def rustc_compile_action( outputs = [crate_info.output], tools = [ctx.executable._process_wrapper], env = env, - arguments = [ctx.executable._process_wrapper.path, args], + arguments = [ctx.executable._process_wrapper.path, ctx.workspace_name, args], mnemonic = "Rustc", execution_requirements = { "supports-workers": "1" }, progress_message = "Compiling Rust {} {}{} ({} files)".format( From 715794b20eccceb73a5e0a72215794d2a0821737 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Sun, 20 Sep 2020 15:38:59 -0700 Subject: [PATCH 08/24] clean up --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 6123310f87..033adfaec0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -78,4 +78,4 @@ docs_deps() load("@docs//:docs_transitive_deps.bzl", docs_transitive_deps = "transitive_deps") -docs_transitive_deps(is_top_level = True) +docs_transitive_deps(is_top_level = True) \ No newline at end of file From b693c68ae109bf3e43252e0e369d407e9bd23c68 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Mon, 21 Sep 2020 21:28:16 -0700 Subject: [PATCH 09/24] pass compilation mode and rustc path to worker to namespace the cache --- rust/private/rustc.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 6df25c03c1..13bfca9301 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -618,7 +618,7 @@ def rustc_compile_action( outputs = [crate_info.output], tools = [ctx.executable._process_wrapper], env = env, - arguments = [ctx.executable._process_wrapper.path, ctx.workspace_name, args], + arguments = [ctx.executable._process_wrapper.path, toolchain.rustc.path, ctx.var["COMPILATION_MODE"], args], mnemonic = "Rustc", execution_requirements = { "supports-workers": "1" }, progress_message = "Compiling Rust {} {}{} ({} files)".format( From da0c8b820c994b25b75fe4cf7821f0191ccc4412 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Tue, 6 Oct 2020 19:09:41 -0700 Subject: [PATCH 10/24] add the worker as a toolchain --- rust/private/rust.bzl | 12 +++++------- rust/private/rustc.bzl | 33 ++++++++++++++++++++++++++------- rust/repositories.bzl | 3 +++ worker/BUILD | 26 ++++++++++++++++++++++++++ worker/repositories.bzl | 14 ++++++++++++++ worker/toolchain.bzl | 12 ++++++++++++ 6 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 worker/BUILD create mode 100644 worker/repositories.bzl create mode 100644 worker/toolchain.bzl diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index ecc983ab66..748c92e639 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -454,13 +454,6 @@ _rust_common_attrs = { allow_single_file = True, cfg = "exec", ), - "_process_wrapper_worker": attr.label( - default = "@rustc_worker//file", - # Actually we may want the persistent worker to be built by the rust rules themselves, but right now we are just going to use a binary. - executable = True, - allow_single_file = True, - cfg = "exec", - ), } _rust_library_attrs = { @@ -496,6 +489,7 @@ rust_library = rule( toolchains = [ "@io_bazel_rules_rust//rust:toolchain", "@bazel_tools//tools/cpp:toolchain_type", + "@io_bazel_rules_rust//worker:toolchain_type", ], doc = """\ Builds a Rust library crate. @@ -586,6 +580,7 @@ rust_binary = rule( toolchains = [ "@io_bazel_rules_rust//rust:toolchain", "@bazel_tools//tools/cpp:toolchain_type", + "@io_bazel_rules_rust//worker:toolchain_type", ], doc = """\ Builds a Rust binary crate. @@ -684,6 +679,7 @@ rust_test = rule( toolchains = [ "@io_bazel_rules_rust//rust:toolchain", "@bazel_tools//tools/cpp:toolchain_type", + "@io_bazel_rules_rust//worker:toolchain_type", ], doc = """\ Builds a Rust test crate. @@ -832,6 +828,7 @@ rust_test_binary = rule( toolchains = [ "@io_bazel_rules_rust//rust:toolchain", "@bazel_tools//tools/cpp:toolchain_type", + "@io_bazel_rules_rust//worker:toolchain_type", ], doc = """\ Builds a Rust test binary, without marking this rule as a Bazel test. @@ -855,6 +852,7 @@ rust_benchmark = rule( toolchains = [ "@io_bazel_rules_rust//rust:toolchain", "@bazel_tools//tools/cpp:toolchain_type", + "@io_bazel_rules_rust//worker:toolchain_type", ], doc = """\ Builds a Rust benchmark test. diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 13bfca9301..8e156142ca 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -400,7 +400,8 @@ def construct_arguments( out_dir, build_env_file, build_flags_files, - maker_path = None): + maker_path, + use_worker=False): """Builds an Args object containing common rustc flags Args: @@ -432,8 +433,10 @@ def construct_arguments( # Wrapper args first args = ctx.actions.args() - args.set_param_file_format("multiline") - args.use_param_file("@%s", use_always=True) + if use_worker: + # Write the args to a param file that will be used by Bazel to send messages to the worker. + args.set_param_file_format("multiline") + args.use_param_file("@%s", use_always=True) if build_env_file != None: args.add("--env-file", build_env_file) @@ -571,6 +574,8 @@ def rustc_compile_action( - (DepInfo): The transitive dependencies of this crate. - (DefaultInfo): The output file for this crate, and its runfiles. """ + worker_binary = ctx.toolchains["@io_bazel_rules_rust//worker:toolchain_type"].worker_binary + dep_info, build_info = collect_deps( ctx.label, crate_info.deps, @@ -605,6 +610,7 @@ def rustc_compile_action( out_dir, build_env_file, build_flags_files, + use_worker = worker_binary != None, ) if hasattr(ctx.attr, "version") and ctx.attr.version != "0.0.0": @@ -612,15 +618,28 @@ def rustc_compile_action( else: formatted_version = "" + if worker_binary != None: + executable = worker_binary + tools = [ctx.executable._process_wrapper] + arguments = [ctx.executable._process_wrapper.path, toolchain.rustc.path, ctx.var["COMPILATION_MODE"], args] + execution_requirements = { "supports-workers": "1" } + else: + # Not all execution platforms support a worker. + executable = ctx.executable._process_wrapper + tools = [] + arguments = [args] + execution_requirements = {} + # TODO: Select if worker is to be enabled. + ctx.actions.run( - executable = ctx.executable._process_wrapper_worker, + executable = executable, inputs = compile_inputs, outputs = [crate_info.output], - tools = [ctx.executable._process_wrapper], + tools = tools, env = env, - arguments = [ctx.executable._process_wrapper.path, toolchain.rustc.path, ctx.var["COMPILATION_MODE"], args], + arguments = arguments, mnemonic = "Rustc", - execution_requirements = { "supports-workers": "1" }, + execution_requirements = execution_requirements, progress_message = "Compiling Rust {} {}{} ({} files)".format( crate_info.type, ctx.label.name, diff --git a/rust/repositories.bzl b/rust/repositories.bzl index 2332f3c651..e2570f6c33 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -2,6 +2,7 @@ load(":known_shas.bzl", "FILE_KEY_TO_SHA") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("//worker:repositories.bzl", "rust_worker_repositories") load( "//rust/platform:triple_mappings.bzl", "system_to_binary_ext", @@ -118,6 +119,8 @@ def rust_repositories( edition = edition, ) + rust_worker_repositories() + def _check_version_valid(version, iso_date, param_prefix = ""): """Verifies that the provided rust version and iso_date make sense.""" diff --git a/worker/BUILD b/worker/BUILD new file mode 100644 index 0000000000..ddd6198602 --- /dev/null +++ b/worker/BUILD @@ -0,0 +1,26 @@ +load("//worker:toolchain.bzl", "worker_toolchain") + +toolchain_type(name = "toolchain_type") + +worker_toolchain( + name = "worker_linux_x86_64", + worker_binary = "@rust_worker_linux_x86_64//file", +) + +worker_toolchain(name = "worker_dummy") + +toolchain( + name = "linux_x86_64", + exec_compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + toolchain = ":worker_linux_x86_64", + toolchain_type = ":toolchain_type", +) + +toolchain( + name = "dummy", + toolchain = ":worker_dummy", + toolchain_type = ":toolchain_type", +) diff --git a/worker/repositories.bzl b/worker/repositories.bzl new file mode 100644 index 0000000000..a1bc7348f2 --- /dev/null +++ b/worker/repositories.bzl @@ -0,0 +1,14 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") + +def rust_worker_repositories(): + http_file( + name = "rust_worker_linux_x86_64", + executable = True, + sha256 = "e3383d062752bcbeeb8a97d36bba0dea244db77b45a083bee718978a0deb5d37", + urls = ["https://github.com/nikhilm/rustc-worker/releases/download/v0.2.0/rustc-worker-linux_x86_64"], + ) + + native.register_toolchains( + "@io_bazel_rules_rust//worker:linux_x86_64", + "@io_bazel_rules_rust//worker:dummy", + ) diff --git a/worker/toolchain.bzl b/worker/toolchain.bzl new file mode 100644 index 0000000000..063b1880d4 --- /dev/null +++ b/worker/toolchain.bzl @@ -0,0 +1,12 @@ +def _worker_toolchain_impl(ctx): + toolchain_info = platform_common.ToolchainInfo( + worker_binary = ctx.executable.worker_binary, + ) + return [toolchain_info] + +worker_toolchain = rule( + implementation = _worker_toolchain_impl, + attrs = { + "worker_binary": attr.label(allow_single_file = True, executable = True, cfg = "exec"), + }, +) From 801e2ae6887d46616a1f6ae7d9dba4438ac6656c Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Tue, 6 Oct 2020 20:20:39 -0700 Subject: [PATCH 11/24] make workers optional --- rust/repositories.bzl | 11 +++++++++-- worker/BUILD | 18 ++++++++++-------- worker/repositories.bzl | 1 - 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/rust/repositories.bzl b/rust/repositories.bzl index e2570f6c33..6dc4bcc55d 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -22,7 +22,8 @@ def rust_repositories( rustfmt_version = "1.4.20", edition = None, dev_components = False, - sha256s = None): + sha256s = None, + use_worker = False): """Emits a default set of toolchains for Linux, OSX, and Freebsd Skip this macro and call the `rust_repository_set` macros directly if you need a compiler for \ @@ -48,6 +49,7 @@ def rust_repositories( edition: The rust edition to be used by default (2015 (default) or 2018) dev_components: Whether to download the rustc-dev components (defaults to False). Requires version to be "nightly". sha256s: A dict associating tool subdirectories to sha256 hashes. + use_worker: boolean. Set to True to use Bazel workers for Rust. This downloads binaries for https://github.com/nikhilm/rustc-worker. """ if dev_components and version != "nightly": @@ -119,7 +121,12 @@ def rust_repositories( edition = edition, ) - rust_worker_repositories() + if use_worker: + rust_worker_repositories() + # Register a fallback for when workers are not enabled or not available for the execution platform. + native.register_toolchains( + "@io_bazel_rules_rust//worker:dummy", + ) def _check_version_valid(version, iso_date, param_prefix = ""): """Verifies that the provided rust version and iso_date make sense.""" diff --git a/worker/BUILD b/worker/BUILD index ddd6198602..af4424b45a 100644 --- a/worker/BUILD +++ b/worker/BUILD @@ -2,13 +2,21 @@ load("//worker:toolchain.bzl", "worker_toolchain") toolchain_type(name = "toolchain_type") +worker_toolchain(name = "worker_dummy") + +toolchain( + name = "dummy", + toolchain = ":worker_dummy", + toolchain_type = ":toolchain_type", +) + +# These toolchains are only registered if workers are enabled in rust_repositories(). + worker_toolchain( name = "worker_linux_x86_64", worker_binary = "@rust_worker_linux_x86_64//file", ) -worker_toolchain(name = "worker_dummy") - toolchain( name = "linux_x86_64", exec_compatible_with = [ @@ -18,9 +26,3 @@ toolchain( toolchain = ":worker_linux_x86_64", toolchain_type = ":toolchain_type", ) - -toolchain( - name = "dummy", - toolchain = ":worker_dummy", - toolchain_type = ":toolchain_type", -) diff --git a/worker/repositories.bzl b/worker/repositories.bzl index a1bc7348f2..6cbd539e4a 100644 --- a/worker/repositories.bzl +++ b/worker/repositories.bzl @@ -10,5 +10,4 @@ def rust_worker_repositories(): native.register_toolchains( "@io_bazel_rules_rust//worker:linux_x86_64", - "@io_bazel_rules_rust//worker:dummy", ) From 77fe9c07c1e3f0246ca124c91840a2722af2dd5f Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Wed, 21 Oct 2020 19:28:24 -0700 Subject: [PATCH 12/24] Fix toolchain registration Repositories should always be registered. --- rust/repositories.bzl | 9 ++++++--- worker/repositories.bzl | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/rust/repositories.bzl b/rust/repositories.bzl index 6dc4bcc55d..75dd2975b7 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -2,7 +2,7 @@ load(":known_shas.bzl", "FILE_KEY_TO_SHA") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") -load("//worker:repositories.bzl", "rust_worker_repositories") +load("//worker:repositories.bzl", "rust_worker_repositories", "rust_worker_toolchains") load( "//rust/platform:triple_mappings.bzl", "system_to_binary_ext", @@ -121,12 +121,15 @@ def rust_repositories( edition = edition, ) - if use_worker: - rust_worker_repositories() + rust_worker_repositories() + # Register a fallback for when workers are not enabled or not available for the execution platform. native.register_toolchains( "@io_bazel_rules_rust//worker:dummy", ) + # Register the real toolchains. + if use_worker: + rust_worker_toolchains() def _check_version_valid(version, iso_date, param_prefix = ""): """Verifies that the provided rust version and iso_date make sense.""" diff --git a/worker/repositories.bzl b/worker/repositories.bzl index 6cbd539e4a..acaadea410 100644 --- a/worker/repositories.bzl +++ b/worker/repositories.bzl @@ -8,6 +8,7 @@ def rust_worker_repositories(): urls = ["https://github.com/nikhilm/rustc-worker/releases/download/v0.2.0/rustc-worker-linux_x86_64"], ) +def rust_worker_toolchains(): native.register_toolchains( "@io_bazel_rules_rust//worker:linux_x86_64", ) From 8a7b18a86e4c79f623a92c786fd05fa00b492961 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Wed, 21 Oct 2020 19:34:42 -0700 Subject: [PATCH 13/24] oops! Toolchain order matters. --- rust/repositories.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/repositories.bzl b/rust/repositories.bzl index 75dd2975b7..f80e55f5ef 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -123,13 +123,13 @@ def rust_repositories( rust_worker_repositories() + # Register the real toolchains. + if use_worker: + rust_worker_toolchains() # Register a fallback for when workers are not enabled or not available for the execution platform. native.register_toolchains( "@io_bazel_rules_rust//worker:dummy", ) - # Register the real toolchains. - if use_worker: - rust_worker_toolchains() def _check_version_valid(version, iso_date, param_prefix = ""): """Verifies that the provided rust version and iso_date make sense.""" From e4d92e34ce50ab2e48ebea33c7b34f6f1144cc3d Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Wed, 21 Oct 2020 20:03:03 -0700 Subject: [PATCH 14/24] update proto rules toolchains --- proto/proto.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proto/proto.bzl b/proto/proto.bzl index 797bb741b2..390c25b5dc 100644 --- a/proto/proto.bzl +++ b/proto/proto.bzl @@ -305,6 +305,7 @@ rust_proto_library = rule( "@io_bazel_rules_rust//proto:toolchain", "@io_bazel_rules_rust//rust:toolchain", "@bazel_tools//tools/cpp:toolchain_type", + "@io_bazel_rules_rust//worker:toolchain_type", ], doc = """\ Builds a Rust library crate from a set of `proto_library`s. @@ -384,6 +385,7 @@ rust_grpc_library = rule( "@io_bazel_rules_rust//proto:toolchain", "@io_bazel_rules_rust//rust:toolchain", "@bazel_tools//tools/cpp:toolchain_type", + "@io_bazel_rules_rust//worker:toolchain_type", ], doc = """\ Builds a Rust library crate from a set of `proto_library`s suitable for gRPC. From 68c326c61fa2b2692d0e3a7f7a56b493632d6968 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Wed, 21 Oct 2020 20:04:49 -0700 Subject: [PATCH 15/24] Remove TODO --- rust/private/rustc.bzl | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 8e156142ca..d4c9e5d3d5 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -629,7 +629,6 @@ def rustc_compile_action( tools = [] arguments = [args] execution_requirements = {} - # TODO: Select if worker is to be enabled. ctx.actions.run( executable = executable, From 068b6c9faa6fe756ae378e1786794f0693e2e0d9 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Wed, 21 Oct 2020 20:30:58 -0700 Subject: [PATCH 16/24] Fix rebase issues --- WORKSPACE | 2 +- rust/private/rustc.bzl | 3 ++- rust/repositories.bzl | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 033adfaec0..6123310f87 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -78,4 +78,4 @@ docs_deps() load("@docs//:docs_transitive_deps.bzl", docs_transitive_deps = "transitive_deps") -docs_transitive_deps(is_top_level = True) \ No newline at end of file +docs_transitive_deps(is_top_level = True) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index d4c9e5d3d5..76a6606ddb 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -400,7 +400,7 @@ def construct_arguments( out_dir, build_env_file, build_flags_files, - maker_path, + maker_path=None, use_worker=False): """Builds an Args object containing common rustc flags @@ -419,6 +419,7 @@ def construct_arguments( build_env_file (str): The output file of a `cargo_build_script` action containing rustc environment variables build_flags_files (list): The output files of a `cargo_build_script` actions containing rustc build flags maker_path (File): An optional clippy marker file + use_worker (bool): If True, sets up the arguments in a worker-compatible fashion Returns: tuple: A tuple of the following items diff --git a/rust/repositories.bzl b/rust/repositories.bzl index f80e55f5ef..c102b86eb3 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -49,7 +49,7 @@ def rust_repositories( edition: The rust edition to be used by default (2015 (default) or 2018) dev_components: Whether to download the rustc-dev components (defaults to False). Requires version to be "nightly". sha256s: A dict associating tool subdirectories to sha256 hashes. - use_worker: boolean. Set to True to use Bazel workers for Rust. This downloads binaries for https://github.com/nikhilm/rustc-worker. + use_worker: boolean. Set to True to use Bazel workers for Rust. This downloads binaries for https://github.com/nikhilm/rustc-worker. """ if dev_components and version != "nightly": From 3c2184b24bdcfd0e97ba6803be39e5c0574f72a1 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Wed, 21 Oct 2020 20:42:35 -0700 Subject: [PATCH 17/24] Run buildifier --- rust/private/rustc.bzl | 8 ++++---- rust/repositories.bzl | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 76a6606ddb..669363123a 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -400,8 +400,8 @@ def construct_arguments( out_dir, build_env_file, build_flags_files, - maker_path=None, - use_worker=False): + maker_path = None, + use_worker = False): """Builds an Args object containing common rustc flags Args: @@ -437,7 +437,7 @@ def construct_arguments( if use_worker: # Write the args to a param file that will be used by Bazel to send messages to the worker. args.set_param_file_format("multiline") - args.use_param_file("@%s", use_always=True) + args.use_param_file("@%s", use_always = True) if build_env_file != None: args.add("--env-file", build_env_file) @@ -623,7 +623,7 @@ def rustc_compile_action( executable = worker_binary tools = [ctx.executable._process_wrapper] arguments = [ctx.executable._process_wrapper.path, toolchain.rustc.path, ctx.var["COMPILATION_MODE"], args] - execution_requirements = { "supports-workers": "1" } + execution_requirements = {"supports-workers": "1"} else: # Not all execution platforms support a worker. executable = ctx.executable._process_wrapper diff --git a/rust/repositories.bzl b/rust/repositories.bzl index c102b86eb3..12f6cf830b 100644 --- a/rust/repositories.bzl +++ b/rust/repositories.bzl @@ -126,6 +126,7 @@ def rust_repositories( # Register the real toolchains. if use_worker: rust_worker_toolchains() + # Register a fallback for when workers are not enabled or not available for the execution platform. native.register_toolchains( "@io_bazel_rules_rust//worker:dummy", From 1f79d092273006bd411a440adf67eefa25b3dd30 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Wed, 21 Oct 2020 21:44:03 -0700 Subject: [PATCH 18/24] Fix documentation generation --- docs/BUILD | 1 + worker/BUILD | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/docs/BUILD b/docs/BUILD index 3b92ea5341..081c80200b 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -19,6 +19,7 @@ bzl_library( "@io_bazel_rules_rust//proto:rules", "@io_bazel_rules_rust//rust:rules", "@io_bazel_rules_rust//wasm_bindgen:rules", + "@io_bazel_rules_rust//worker:rules", "@rules_proto//proto:rules", ], ) diff --git a/worker/BUILD b/worker/BUILD index af4424b45a..c4b813fd6e 100644 --- a/worker/BUILD +++ b/worker/BUILD @@ -1,5 +1,17 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("//worker:toolchain.bzl", "worker_toolchain") +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "repositories.bzl", +]) + +bzl_library( + name = "rules", + srcs = glob(["**/*.bzl"]), +) + toolchain_type(name = "toolchain_type") worker_toolchain(name = "worker_dummy") From a3e0ebb51ddee2dcba245264c902eac5cc9e4521 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Wed, 21 Oct 2020 21:55:55 -0700 Subject: [PATCH 19/24] Change examples to use the worker --- examples/examples_deps.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/examples_deps.bzl b/examples/examples_deps.bzl index b45d3eb5c6..5cbfffefe7 100644 --- a/examples/examples_deps.bzl +++ b/examples/examples_deps.bzl @@ -12,7 +12,7 @@ load("@io_bazel_rules_rust//wasm_bindgen:repositories.bzl", "rust_wasm_bindgen_r def deps(): """Define dependencies for `rules_rust` examples""" - rust_repositories() + rust_repositories(use_worker = True) rust_bindgen_repositories() From a68c7e24d43033400614ca6f282cb7a9f50967c9 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Wed, 21 Oct 2020 22:16:01 -0700 Subject: [PATCH 20/24] Fix buildifier lints --- worker/repositories.bzl | 3 +++ worker/toolchain.bzl | 1 + 2 files changed, 4 insertions(+) diff --git a/worker/repositories.bzl b/worker/repositories.bzl index acaadea410..132e957808 100644 --- a/worker/repositories.bzl +++ b/worker/repositories.bzl @@ -1,6 +1,8 @@ +# buildifier: disable=module-docstring load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") def rust_worker_repositories(): + """Registers rustc-worker binary archives.""" http_file( name = "rust_worker_linux_x86_64", executable = True, @@ -9,6 +11,7 @@ def rust_worker_repositories(): ) def rust_worker_toolchains(): + """Registers worker toolchains for supported platforms.""" native.register_toolchains( "@io_bazel_rules_rust//worker:linux_x86_64", ) diff --git a/worker/toolchain.bzl b/worker/toolchain.bzl index 063b1880d4..4a30ab7db5 100644 --- a/worker/toolchain.bzl +++ b/worker/toolchain.bzl @@ -1,3 +1,4 @@ +# buildifier: disable=module-docstring def _worker_toolchain_impl(ctx): toolchain_info = platform_common.ToolchainInfo( worker_binary = ctx.executable.worker_binary, From 917bee9e968dd9c56ecefa97738c627b426f0206 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Wed, 21 Oct 2020 22:21:44 -0700 Subject: [PATCH 21/24] Switch to a musl linked static library so it works without glibc --- worker/repositories.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/worker/repositories.bzl b/worker/repositories.bzl index 132e957808..d9a17f31fc 100644 --- a/worker/repositories.bzl +++ b/worker/repositories.bzl @@ -6,8 +6,8 @@ def rust_worker_repositories(): http_file( name = "rust_worker_linux_x86_64", executable = True, - sha256 = "e3383d062752bcbeeb8a97d36bba0dea244db77b45a083bee718978a0deb5d37", - urls = ["https://github.com/nikhilm/rustc-worker/releases/download/v0.2.0/rustc-worker-linux_x86_64"], + sha256 = "c7ee178d658a9ff9c9b10f7acce48af57227170d454741072aff5fabf923f8fb", + urls = ["https://github.com/nikhilm/rustc-worker/releases/download/v0.2.0/rustc-worker-linux_x86_64-static"], ) def rust_worker_toolchains(): From 81353d6215d9e6ec2a6bc446c6e28c09327ab1ae Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Wed, 21 Oct 2020 22:25:54 -0700 Subject: [PATCH 22/24] Final buildifier fix --- worker/repositories.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/worker/repositories.bzl b/worker/repositories.bzl index d9a17f31fc..0c9af921e8 100644 --- a/worker/repositories.bzl +++ b/worker/repositories.bzl @@ -10,6 +10,7 @@ def rust_worker_repositories(): urls = ["https://github.com/nikhilm/rustc-worker/releases/download/v0.2.0/rustc-worker-linux_x86_64-static"], ) +# buildifier: disable=unnamed-macro def rust_worker_toolchains(): """Registers worker toolchains for supported platforms.""" native.register_toolchains( From 7eba058a51f748ed831b4de5cca82f4076fd06ed Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Thu, 22 Oct 2020 15:18:55 -0700 Subject: [PATCH 23/24] Add documentation about using the worker --- docs/index.md | 9 +++++++++ docs/rust_repositories.md | 5 ++++- worker/README.md | 14 ++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 worker/README.md diff --git a/docs/index.md b/docs/index.md index 40ad8986cb..349a8cbec8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -69,6 +69,15 @@ Similarly, `rustfmt_version` may also be configured: ```python rust_repositories(rustfmt_version = "1.4.8") ``` +# Using Bazel Persistent Workers + +Iterating on Rust code may benefit from [incremental compilation](https://doc.rust-lang.org/edition-guide/rust-2018/the-compiler/incremental-compilation-for-faster-compiles.html). This is supported by using a [Bazel Persistent Worker](https://docs.bazel.build/versions/master/persistent-workers.html). While Bazel can determine what needs to be rebuilt at a crate level, the compiler can speed up building a single crate by sharing information across runs. It does this by storing intermediate information in a directory across invocations. This is enabled by default in Cargo. Persistent workers bring this feature to Bazel. + +The Rust rules have preliminary support for workers. Pass `use_worker = True` to enable this when available. + +```python +rust_repositories(use_worker = True) +``` ## External Dependencies diff --git a/docs/rust_repositories.md b/docs/rust_repositories.md index d38d5e8b12..82471c564b 100644 --- a/docs/rust_repositories.md +++ b/docs/rust_repositories.md @@ -139,7 +139,7 @@ Generates a toolchain-bearing repository that declares the toolchains from some ## rust_repositories 
-rust_repositories(version, iso_date, rustfmt_version, edition, dev_components, sha256s)
+rust_repositories(version, iso_date, rustfmt_version, edition, dev_components, sha256s, use_worker)
Emits a default set of toolchains for Linux, OSX, and Freebsd @@ -158,6 +158,8 @@ This would match for `exec_triple = "x86_64-unknown-linux-gnu"`. If not specifi See `load_arbitrary_tool` in `@io_bazel_rules_rust//rust:repositories.bzl` for more details. +The `use_worker` boolean enables [Bazel Persistent Workers] to be used when available. Not all execution platforms have a pre-built binary worker available. In such a case, the rules will fall back to invoking `rustc` directly. Supported platforms: +- linux-x86_64 **PARAMETERS** @@ -170,6 +172,7 @@ See `load_arbitrary_tool` in `@io_bazel_rules_rust//rust:repositories.bzl` for m | edition | The rust edition to be used by default (2015 (default) or 2018) | None | | dev_components | Whether to download the rustc-dev components (defaults to False). Requires version to be "nightly". | False | | sha256s | A dict associating tool subdirectories to sha256 hashes. | None | +| use_worker | Set this to True to enable worker. | False | diff --git a/worker/README.md b/worker/README.md new file mode 100644 index 0000000000..3cdb08ac48 --- /dev/null +++ b/worker/README.md @@ -0,0 +1,14 @@ + +# Rust Persistent Worker + +The Rust Persistent Worker is itself implemented in Rust. It is built via Cargo and distributed as binaries. Source and release binaries are maintained at the [rustc-worker project](https://github.com/nikhilm/rustc-worker). Contributions should be submitted there and then the version of the binaries updated in `worker/repositories.bzl`. + +## Why isn't this built via Bazel? + +Bootstrapping the worker using these same rules (e.g. `rust_binary`) may be possible, but is not supported right now. There are a couple of challenges: +1. Since the worker has dependencies on various crates, it uses cargo-raze, which generates relevant rules. This means "don't use the worker to build this target" is transitive and such information needs to be propagated down the tree in a way that works with restrictions in Bazel. Initial experiments repeatedly encountered failures due to Bazel treating the rule dependency on the worker executable target as a cycle, even when building in non-worker mode. This may be user error or a restriction in Bazel. Until that is addressed, the easiest way to fix this is to change cargo-raze to customize what rules are used, and provide `rust_no_worker_binary` and `rust_no_worker_library` rules that do not have this cycle. +2. Figuring out a good strategy for dependencies. Since Bazel doesn't really have transitive dependencies, attempts to build this worker from source necessarily require users of these rules to register all the external repositories for the worker in their `WORKSPACE`. This could cause collisions with other dependencies. In addition, if the `_no_worker_` approach above is adopted, users will lose the benefits of workers for those dependencies (like `protobuf`) shared between the worker and their code. There is no satisfactory solution for this right now. + +## How about rewriting the worker in C++? + +That is certainly an option! From 96a0d34abeb586ae288df3ca981d2f4869db57f3 Mon Sep 17 00:00:00 2001 From: Nikhil Marathe Date: Fri, 6 Nov 2020 17:16:36 -0800 Subject: [PATCH 24/24] Fix linux version URL --- worker/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/repositories.bzl b/worker/repositories.bzl index 0c9af921e8..cceaed11a0 100644 --- a/worker/repositories.bzl +++ b/worker/repositories.bzl @@ -7,7 +7,7 @@ def rust_worker_repositories(): name = "rust_worker_linux_x86_64", executable = True, sha256 = "c7ee178d658a9ff9c9b10f7acce48af57227170d454741072aff5fabf923f8fb", - urls = ["https://github.com/nikhilm/rustc-worker/releases/download/v0.2.0/rustc-worker-linux_x86_64-static"], + urls = ["https://github.com/nikhilm/rustc-worker/releases/download/v0.2.0/rustc-worker-0.2.0-linux-x86_64"], ) # buildifier: disable=unnamed-macro