feat: compile source files at build time

.bazelignore

@@ -25,4 +25,5 @@ examples/pip_parse_vendored/bazel-pip_parse_vendored
 examples/py_proto_library/bazel-py_proto_library
 tests/integration/compile_pip_requirements/bazel-compile_pip_requirements
 tests/integration/ignore_root_user_error/bazel-ignore_root_user_error
+tests/integration/local_toolchains/bazel-local_toolchains
 tests/integration/pip_repository_entry_points/bazel-pip_repository_entry_points

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
 
 test --test_output=errors
 

CHANGELOG.md

@@ -22,6 +22,9 @@ A brief description of the categories of changes:
 [x.x.x]: https://github.com/bazelbuild/rules_python/releases/tag/x.x.x
 
 ### Changed
+* (toolchains) Optional toolchain dependency: `py_binary`, `py_test`, and
+  `py_library` now depend on the `//python:exec_tools_toolchain_type` for build
+  tools.
 
 * (deps): Bumped `bazel_skylib` to 1.6.1.
 * (bzlmod): The `python` and internal `rules_python` extensions have been
@@ -37,6 +40,28 @@ A brief description of the categories of changes:
   for this package before will be deleted automatically.
 
 ### Added
+* (rules) Precompiling Python source at build time is available. but is
+  disabled by default, for now. Set
+  `@rules_python//python/config_settings:precompile=enabled` to enable it
+  by default. A subsequent release will enable it by default. See the
+  [Precompiling docs][precompile-docs] and API reference docs for more
+  information on precompiling. Note this requires Bazel 7+ and the Pystar rule
+  implementation enabled.
+  ([#1761](https://github.com/bazelbuild/rules_python/issues/1761))
+* (rules) Attributes and flags to control precompile behavior: `precompile`,
+  `precompile_optimize_level`, `precompile_source_retention`,
+  `precompile_invalidation_mode`, and `pyc_collection`
+* (toolchains) The target runtime toolchain (`//python:toolchain_type`) has
+  two new optional attributes: `pyc_tag` (tells the pyc filename infix to use) and
+  `implementation_name` (tells the Python implementation name).
+* (toolchains) A toolchain type for build tools has been added:
+  `//python:exec_tools_toolchain_type`.
+* (providers) `PyInfo` has two new attributes: `direct_pyc_files` and
+  `transitive_pyc_files`, which tell the pyc files a target makes available
+  directly and transitively, respectively.
+* `//python:features.bzl` added to allow easy feature-detection in the future.
+
+[precompile-docs]: /precompiling
 
 ## [0.32.2] - 2024-05-14
 

docs/sphinx/index.md

@@ -60,6 +60,7 @@ pypi-dependencies
 toolchains
 pip
 coverage
+precompiling
 gazelle
 Contributing <contributing>
 support

docs/sphinx/precompiling.md

@@ -0,0 +1,95 @@
+# Precompiling
+
+Precompiling is compiling Python source files (`.py` files) into byte code (`.pyc`
+files) at build
+time instead of runtime. Doing it at build time can improve performance by
+skipping that work at runtime.
+
+Precompiling is enabled by default, so there typically isn't anything special
+you must do to use it.
+
+
+## Overhead of precompiling
+
+While precompiling helps runtime performance, it has two main costs:
+1. Increasing the size (count and disk usage) of runfiles. It approximately
+   double the count of the runfiles because for every `.py` file, there is also
+   a `.pyc` file. Compiled files are generally around the same size as the
+   source files, so it approximately doubles the disk usage.
+2. Precompiling requires running an extra action at build time. While
+   compiling itself isn't that expensive, the overhead can become noticable
+   as more files need to be compiled.
+
+## Binary-level opt-in
+
+Because of the costs of precompiling, it may not be feasible to globally enable it
+for your repo for everything. For example, some binaries may be
+particularly large, and doubling the number of runfiles isn't doable.
+
+If this is the case, there's an alternative way to more selectively and
+incrementally control precompiling on a per-binry basis.
+
+To use this approach, the two basic steps are:
+1. Disable pyc files from being automatically added to runfiles:
+   `--@rules_python//python/config_settings:precompile_add_to_runfiles=decided_elsewhere`,
+2. Set the `pyc_collection` attribute on the binaries/tests that should or should
+   not use precompiling.
+
+The default for the `pyc_collection` attribute is controlled by a flag, so you
+can use an opt-in or opt-out approach by setting the flag:
+* targets must opt-out: `--@rules_python//python/config_settings:pyc_collection=include_pyc`,
+* targets must opt-in: `--@rules_python//python/config_settings:pyc_collection=disabled`,
+
+## Advanced precompiler customization
+
+The default implementation of the precompiler is a persistent, multiplexed,
+sandbox-aware, cancellation-enabled, json-protocol worker that uses the same
+interpreter as the target toolchain. This works well for local builds, but may
+not work as well for remote execution builds. To customize the precompiler, two
+mechanisms are available:
+
+* The exec tools toolchain allows customizing the precompiler binary used with
+  the `precompiler` attribute. Arbitrary binaries are supported.
+* The execution requirements can be customized using
+  `--@rules_python//tools/precompiler:execution_requirements`. This is a list
+  flag that can be repeated. Each entry is a key=value that is added to the
+  execution requirements of the `PyPrecompile` action. Note that this flag
+  is specific to the rules_python precompiler. If a custom binary is used,
+  this flag will have to be propagated from the custom binary using the
+  `testing.ExecutionInfo` provider; refer to the `py_interpreter_program` an
+
+The default precompiler implementation is an asynchronous/concurrent
+implementation. If you find it has bugs or hangs, please report them. In the
+meantime, the flag `--worker_extra_flag=PyPrecompile=--worker_impl=serial` can
+be used to switch to a synchronous/serial implementation that may not perform
+as well, but is less likely to have issues.
+
+The `execution_requirements` keys of most relevance are:
+* `supports-workers`: 1 or 0, to indicate if a regular persistent worker is
+  desired.
+* `supports-multiplex-workers`: 1 o 0, to indicate if a multiplexed persistent
+  worker is desired.
+* `requires-worker-protocol`: json or proto; the rules_python precompiler
+  currently only supports json.
+* `supports-multiplex-sandboxing`: 1 or 0, to indicate if sanboxing is of the
+  worker is supported.
+* `supports-worker-cancellation`: 1 or 1, to indicate if requests to the worker
+  can be cancelled.
+
+Note that any execution requirements values can be specified in the flag.
+
+## Known issues, caveats, and idiosyncracies
+
+* Precompiling requires Bazel 7+ with the Pystar rule implementation enabled.
+* Mixing rules_python PyInfo with Bazel builtin PyInfo will result in pyc files
+  being dropped.
+* Precompiled files may not be used in certain cases prior to Python 3.11. This
+  occurs due Python adding the directory of the binary's main `.py` file, which
+  causes the module to be found in the workspace source directory instead of
+  within the binary's runfiles directory (where the pyc files are). This can
+  usually be worked around by removing `sys.path[0]` (or otherwise ensuring the
+  runfiles directory comes before the repos source directory in `sys.path`).
+* The pyc filename does not include the optimization level (e.g.
+  `foo.cpython-39.opt-2.pyc`). This works fine (it's all byte code), but also
+  means the interpreter `-O` argument can't be used -- doing so will cause the
+  interpreter to look for the non-existent `opt-N` named files.

python/BUILD.bazel

@@ -71,6 +71,11 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "features_bzl",
+    srcs = ["features.bzl"],
+)
+
 bzl_library(
     name = "packaging_bzl",
     srcs = ["packaging.bzl"],
@@ -292,6 +297,11 @@ alias(
     actual = "@bazel_tools//tools/python:toolchain_type",
 )
 
+toolchain_type(
+    name = "exec_tools_toolchain_type",
+    visibility = ["//visibility:public"],
+)
+
 # Definitions for a Python toolchain that, at execution time, attempts to detect
 # a platform runtime having the appropriate major Python version. Consider this
 # a toolchain of last resort.

python/config_settings/BUILD.bazel

@@ -1,3 +1,11 @@
+load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
+load(
+    "//python/private:flags.bzl",
+    "PrecompileAddToRunfilesFlag",
+    "PrecompileFlag",
+    "PrecompileSourceRetentionFlag",
+    "PycCollectionFlag",
+)
 load(":config_settings.bzl", "construct_config_settings")
 
 filegroup(
@@ -12,3 +20,35 @@ filegroup(
 construct_config_settings(
     name = "construct_config_settings",
 )
+
+string_flag(
+    name = "precompile",
+    build_setting_default = PrecompileFlag.AUTO,
+    values = sorted(PrecompileFlag.__members__.values()),
+    # NOTE: Only public because its an implicit dependency
+    visibility = ["//visibility:public"],
+)
+
+string_flag(
+    name = "precompile_source_retention",
+    build_setting_default = PrecompileSourceRetentionFlag.KEEP_SOURCE,
+    values = sorted(PrecompileSourceRetentionFlag.__members__.values()),
+    # NOTE: Only public because its an implicit dependency
+    visibility = ["//visibility:public"],
+)
+
+string_flag(
+    name = "precompile_add_to_runfiles",
+    build_setting_default = PrecompileAddToRunfilesFlag.ALWAYS,
+    values = sorted(PrecompileAddToRunfilesFlag.__members__.values()),
+    # NOTE: Only public because its an implicit dependency
+    visibility = ["//visibility:public"],
+)
+
+string_flag(
+    name = "pyc_collection",
+    build_setting_default = PycCollectionFlag.DISABLED,
+    values = sorted(PycCollectionFlag.__members__.values()),
+    # NOTE: Only public because its an implicit dependency
+    visibility = ["//visibility:public"],
+)

tests/support/test_platforms.bzl → python/features.bzl

@@ -1,4 +1,4 @@
-# Copyright 2023 The Bazel Authors. All rights reserved.
+# Copyright 2024 The Bazel Authors. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,10 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Constants for referring to platforms."""
+"""Allows detecting of rules_python features that aren't easily detected."""
 
-# Explicit Label() calls are required so that it resolves in @rules_python
-# context instead of e.g. the @rules_testing context.
-MAC = Label("//tests/support:mac")
-LINUX = Label("//tests/support:linux")
-WINDOWS = Label("//tests/support:windows")
+features = struct(
+    precompile = True,
+)

python/private/BUILD.bazel

@@ -86,11 +86,25 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "enum_bzl",
+    srcs = ["enum.bzl"],
+)
+
 bzl_library(
     name = "envsubst_bzl",
     srcs = ["envsubst.bzl"],
 )
 
+bzl_library(
+    name = "flags_bzl",
+    srcs = ["flags.bzl"],
+    deps = [
+        ":enum_bzl",
+        "@bazel_skylib//rules:common_settings",
+    ],
+)
+
 bzl_library(
     name = "full_version_bzl",
     srcs = ["full_version.bzl"],
@@ -166,6 +180,18 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "py_exec_tools_toolchain_bzl",
+    srcs = ["py_exec_tools_toolchain.bzl"],
+    deps = ["//python/private/common:providers_bzl"],
+)
+
+bzl_library(
+    name = "py_interpreter_program_bzl",
+    srcs = ["py_interpreter_program.bzl"],
+    deps = ["@bazel_skylib//rules:common_settings"],
+)
+
 bzl_library(
     name = "py_package_bzl",
     srcs = ["py_package.bzl"],
@@ -192,6 +218,7 @@ bzl_library(
     name = "py_toolchain_suite_bzl",
     srcs = ["py_toolchain_suite.bzl"],
     deps = [
+        ":toolchain_types_bzl",
         "@bazel_skylib//lib:selects",
     ],
 )
@@ -256,6 +283,11 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "toolchain_types_bzl",
+    srcs = ["toolchain_types.bzl"],
+)
+
 bzl_library(
     name = "util_bzl",
     srcs = ["util.bzl"],

python/private/common/BUILD.bazel

@@ -32,7 +32,10 @@ bzl_library(
         ":providers_bzl",
         ":py_internal_bzl",
         ":semantics_bzl",
+        "//python/private:enum_bzl",
+        "//python/private:flags_bzl",
         "//python/private:reexports_bzl",
+        "@bazel_skylib//rules:common_settings",
     ],
 )
 
@@ -46,9 +49,12 @@ bzl_library(
     name = "common_bazel_bzl",
     srcs = ["common_bazel.bzl"],
     deps = [
+        ":attributes_bzl",
         ":common_bzl",
         ":providers_bzl",
         ":py_internal_bzl",
+        "//python/private:py_interpreter_program_bzl",
+        "//python/private:toolchain_types_bzl",
         "@bazel_skylib//lib:paths",
     ],
 )
@@ -124,8 +130,11 @@ bzl_library(
         ":common_bzl",
         ":providers_bzl",
         ":py_internal_bzl",
+        "//python/private:flags_bzl",
         "//python/private:rules_cc_srcs_bzl",
+        "//python/private:toolchain_types_bzl",
         "@bazel_skylib//lib:dicts",
+        "@bazel_skylib//rules:common_settings",
     ],
 )
 
@@ -143,7 +152,10 @@ bzl_library(
         ":common_bzl",
         ":providers_bzl",
         ":py_internal_bzl",
+        "//python/private:flags_bzl",
+        "//python/private:toolchain_types_bzl",
         "@bazel_skylib//lib:dicts",
+        "@bazel_skylib//rules:common_settings",
     ],
 )