Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Introduce an experimental uv toolchain #1989

Merged
merged 74 commits into from
Jul 12, 2024
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
74 commits
Select commit Hold shift + click to select a range
4a816ca
Spike uv
groodt Jun 18, 2024
6d0bdeb
Spike uv
groodt Jun 18, 2024
c5c3f88
Adds Windows
groodt Jun 18, 2024
f038ce8
buildifier
groodt Jun 18, 2024
8086333
buildifier
groodt Jun 18, 2024
2591363
chore: add uv to rules_python bzlmod
aignas May 6, 2024
2797a46
adapt the uv directly from http_archive
aignas Jun 19, 2024
308fe32
bump uv version
aignas Jun 19, 2024
04e5e6c
move the usage of the uv extension around
aignas Jun 19, 2024
0fcdf65
fixup the buildozer fixup command
aignas Jun 19, 2024
6d1743f
Merge branch 'main' into groodt-uv-hacks
groodt Jun 20, 2024
aea2ef9
Merge branch 'groodt-uv-hacks' of github.com:bazelbuild/rules_python …
groodt Jun 20, 2024
9b1f5fd
Hacky toolchain. Needs py interpreter.
groodt Jun 22, 2024
abbcc97
Hacky toolchain. Needs py interpreter.
groodt Jun 22, 2024
4a01495
Kinda works
groodt Jun 22, 2024
850f254
Merge branch 'main' into groodt-uv-hacks
groodt Jun 23, 2024
c6bddea
.
groodt Jun 23, 2024
750d8da
Init toolchain
groodt Jun 23, 2024
3a1f889
Init toolchain
groodt Jun 23, 2024
37859b8
Init toolchain
groodt Jun 23, 2024
0ff0003
Init toolchain
groodt Jun 23, 2024
fb73a9f
Init toolchain
groodt Jun 23, 2024
aefd672
Init toolchain
groodt Jun 23, 2024
8002c0e
Working toolchain
groodt Jun 24, 2024
1c8bf77
Working toolchain
groodt Jun 24, 2024
4356066
Working executable rule
groodt Jun 26, 2024
26a87f2
Working executable rule
groodt Jun 26, 2024
eea3f34
Working executable rule
groodt Jun 26, 2024
2cede41
Fix copyright date
rickeylev Jun 28, 2024
76b381a
address some nit/style type of comments
rickeylev Jun 28, 2024
c4b01c7
Merge branch 'main' into groodt-uv-hacks
groodt Jun 28, 2024
6041414
.
groodt Jun 28, 2024
ba09edc
Address review comments
groodt Jun 29, 2024
ec0f6fa
Address review comments
groodt Jun 29, 2024
4cb451a
Address review comments
groodt Jun 29, 2024
07c1ff6
Address review comments
groodt Jun 29, 2024
a60a308
Address review comments
groodt Jun 29, 2024
ec0f9f4
Address review comments
groodt Jun 29, 2024
014afbb
Address review comments
groodt Jun 29, 2024
e8fbfa1
Address review comments
groodt Jun 29, 2024
23628ea
Address review comments
groodt Jun 29, 2024
c2d7694
Address review comments
groodt Jun 29, 2024
fd9eebf
Address review comments
groodt Jun 29, 2024
a59ab24
Address review comments
groodt Jun 29, 2024
431c79f
Address review comments
groodt Jun 29, 2024
e67fdc8
Address review comments
groodt Jun 29, 2024
d141195
Address review comments
groodt Jun 29, 2024
3a0441b
Address review comments
groodt Jun 29, 2024
40ccfb0
Address review comments
groodt Jun 29, 2024
7750195
Address review comments
groodt Jun 29, 2024
8342931
Address review comments
groodt Jun 30, 2024
d98ba65
Address review comments
groodt Jun 30, 2024
dc0df0b
Address review comments
groodt Jun 30, 2024
4c8c490
Address review comments
groodt Jun 30, 2024
ae45048
Address review comments
groodt Jun 30, 2024
0b0d49c
Address review comments
groodt Jun 30, 2024
93e81c6
Address review comments
groodt Jun 30, 2024
5ecc345
Address review comments
groodt Jun 30, 2024
5733118
Address review comments
groodt Jun 30, 2024
ec10150
Address review comments
groodt Jul 1, 2024
6adac07
Merge branch 'main' into groodt-uv-hacks
groodt Jul 9, 2024
8e3cbe9
Bump to uv 0.2.23
groodt Jul 9, 2024
7dd729c
Address review comments
groodt Jul 9, 2024
bb2996c
Address review comments
groodt Jul 9, 2024
c31f6f9
Address review comments
groodt Jul 9, 2024
6f22655
Add EXPERIMENTAL comments and visibility restrictions
groodt Jul 9, 2024
22ad69f
Add EXPERIMENTAL comments and visibility restrictions
groodt Jul 9, 2024
f4c8790
Address review comments
groodt Jul 9, 2024
02c537d
Address review comments
groodt Jul 9, 2024
dfa24fe
Address review comments
groodt Jul 9, 2024
5c11f75
Address review comments
groodt Jul 9, 2024
93829ca
Address review comments
groodt Jul 10, 2024
a0f3d5c
Address review comments
groodt Jul 10, 2024
7fb7af1
Address review comments
groodt Jul 10, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions python/private/bzlmod/pin.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#!/usr/bin/env bash

set -euo pipefail

uv_path=$1
shift

$uv_path pip compile \
"$@"
57 changes: 57 additions & 0 deletions python/private/bzlmod/pip.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
"pip module extension for use with bzlmod"

load("@bazel_features//:features.bzl", "bazel_features")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS")
load(
"//python/pip_install:pip_repository.bzl",
Expand Down Expand Up @@ -323,6 +325,56 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s

return is_hub_reproducible

# TODO(groodt): Add more platforms
_UV_DEPS = {
"linux_aarch64": (
"https://files.pythonhosted.org/packages/00/82/3d0acad7ebd4098aaa23f163b492b47fd9321b2681c55918fe5f18856513/uv-0.2.12-py3-none-manylinux_2_28_aarch64.whl",
"05d809516b651997a151585c9bbd150d888e71976a734781a3dde95430c1cab2",
),
"linux_x86_64": (
"https://files.pythonhosted.org/packages/d9/85/e608bfe78772afa3f9807a074203982131ff9c12e279bb3bc502e14d51ab/uv-0.2.12-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"78d5329094de1f6b503bffe00abcdf1aaa7b3e4aa2aa373e128d7faf39b98de9",
),
"osx_x86_64": (
"https://files.pythonhosted.org/packages/ff/f5/df24800e195f10e41aeb4a89f38488aca76c22d0f87c3eb8bf309f940aa3/uv-0.2.12-py3-none-macosx_10_12_x86_64.whl",
"85d2b9649c14014d6c54ecfb5d36f57e5c0e890957078ed4937e03569141d99b",
),
"windows_x86_64": (
"https://files.pythonhosted.org/packages/c8/bc/a905f3d4b961781ead58243bd4f3c6cd2fcb2deaa55c565a64ba7d06cd20/uv-0.2.12-py3-none-win_amd64.whl",
"ca95af397769422e11e87dc147be1262a54d3a13d8837b69c80fad20f1ff9af4",
groodt marked this conversation as resolved.
Show resolved Hide resolved
),
}

# Repository to retrieve tools for pinning dependencies.
# We currently use uv as a dependency for the pinning tool.
def _fetch_pin_dependencies(python_version, platform):
url, sha256 = _UV_DEPS.get(platform)
py_version_label = "cp" + version_label(python_version)

name = "{py_version_label}_{platform}_uv".format(
py_version_label = py_version_label,
platform = platform,
)
maybe(
http_archive,
name = name,
build_file_content = """
filegroup(
name = "uv",
srcs = glob(["*/scripts/uv"]),
visibility = ["//visibility:public"],
)
""",
sha256 = sha256,
type = "zip",
urls = [url],
)

return "@{py_version_label}_{platform}_uv//:uv".format(
py_version_label = py_version_label,
platform = platform,
)

def _pip_impl(module_ctx):
"""Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories.

Expand Down Expand Up @@ -472,6 +524,10 @@ def _pip_impl(module_ctx):
is_hub_reproducible = _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache)
is_extension_reproducible = is_extension_reproducible and is_hub_reproducible

# Register repository for pin tooling
repository_platform = host_platform(module_ctx.os)
pin_tool_label = _fetch_pin_dependencies(_major_minor_version(DEFAULT_PYTHON_VERSION), repository_platform)

for hub_name, whl_map in hub_whl_map.items():
pip_repository(
name = hub_name,
Expand All @@ -482,6 +538,7 @@ def _pip_impl(module_ctx):
},
default_version = _major_minor_version(DEFAULT_PYTHON_VERSION),
groups = hub_group_map.get(hub_name),
pin_tool_label = pin_tool_label,
)

if bazel_features.external_deps.extension_metadata_has_reproducible:
Expand Down
21 changes: 20 additions & 1 deletion python/private/bzlmod/pip_repository.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,18 @@ package(default_visibility = ["//visibility:public"])

# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it
exports_files(["requirements.bzl"])

sh_binary(
name = "pin",
srcs = ["pin.sh"],
data = [
"{pin_tool_label}",
],
args = [
"$(location {pin_tool_label})",
],
visibility = ["//visibility:public"],
)
"""

def _pip_repository_impl(rctx):
Expand All @@ -48,7 +60,10 @@ def _pip_repository_impl(rctx):
# `requirement`, et al. macros.
macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name)

rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS)
rctx.template("pin.sh", rctx.attr._pin)
rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS.format(
pin_tool_label = rctx.attr.pin_tool_label,
))
rctx.template("requirements.bzl", rctx.attr._template, substitutions = {
"%%ALL_DATA_REQUIREMENTS%%": render.list([
macro_tmpl.format(p, "data")
Expand Down Expand Up @@ -77,6 +92,9 @@ setting.""",
"groups": attr.string_list_dict(
mandatory = False,
),
"pin_tool_label": attr.string(
mandatory = True,
),
"repo_name": attr.string(
mandatory = True,
doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.",
Expand All @@ -88,6 +106,7 @@ The wheel map where values are json.encoded strings of the whl_map constructed
in the pip.parse tag class.
""",
),
"_pin": attr.label(default = ":pin.sh"),
"_template": attr.label(
default = ":requirements.bzl.tmpl",
),
Expand Down