Skip to content

Commit

Permalink
refactor(bzlmod)!: simplify pip.parse repository layout (#1395)
Browse files Browse the repository at this point in the history 
Before this PR we would generate extra `alias` repos and the extra `hub`
repo
for the `entry_point` macro usage. This PR removes the extras and
delegates the
creation of version-aware aliases to the `render_pkg_aliases` internal
function. This reduces the number of repositories created by the
`pip.parse`
extension.

Fixes #1255.

BREAKING CHANGE:

Note that this only affects bzlmod support, which is still beta.

* Bzlmod `pip.parse` no longer generates `{hub_name}_{py_version}` hub
repos.
* Bzlmod `pip.parse` no longer generates `{hub_name}_{distribution}` hub
repos.

These repos aren't part of a public API, but were typically used for the
`entry_point`
macros. Instead, use `py_console_script_binary`, which is the supported
replacement
for entry points under bzlmod. Directly referencing the underlying
distribution
repos remains unsupported.
  • Loading branch information
@aignas
aignas authored Sep 27, 2023
1 parent 7bb1f4a commit f2a4dd5
Show file tree
Hide file tree
Showing 6 changed files with 72 additions and 209 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@ A brief description of the categories of changes:
* (bzlmod) The `entry_point` macro is no longer supported and has been removed
in favour of the `py_console_script_binary` macro for `bzlmod` users.

* (bzlmod) The `pip.parse` no longer generates `{hub_name}_{py_version}` hub repos
as the `entry_point` macro has been superseded by `py_console_script_binary`.

* (bzlmod) The `pip.parse` no longer generates `{hub_name}_{distribution}` hub repos.

### Fixed

* (whl_library) No longer restarts repository rule when fetching external
Expand Down
30 changes: 3 additions & 27 deletions docs/pip_repository.md
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

136 changes: 44 additions & 92 deletions python/extensions/pip.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,16 @@
"pip module extension for use with bzlmod"

load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS")
load("//python:pip.bzl", "whl_library_alias")
load(
"//python/pip_install:pip_repository.bzl",
"locked_requirements_label",
"pip_hub_repository_bzlmod",
"pip_repository_attrs",
"pip_repository_bzlmod",
"use_isolated",
"whl_library",
)
load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse")
load("//python/private:full_version.bzl", "full_version")
load("//python/private:normalize_name.bzl", "normalize_name")
load("//python/private:version_label.bzl", "version_label")

Expand Down Expand Up @@ -78,11 +77,11 @@ You cannot use both the additive_build_content and additive_build_content_file a
whl_mods = whl_mods,
)

def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map):
def _create_whl_repos(module_ctx, pip_attr, whl_map):
python_interpreter_target = pip_attr.python_interpreter_target

# if we do not have the python_interpreter set in the attributes
# we programtically find it.
# we programmatically find it.
hub_name = pip_attr.hub_name
if python_interpreter_target == None:
python_name = "python_" + version_label(pip_attr.python_version, sep = "_")
Expand All @@ -104,23 +103,12 @@ def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map):
requrements_lock = locked_requirements_label(module_ctx, pip_attr)

# Parse the requirements file directly in starlark to get the information
# needed for the whl_libary declarations below. This is needed to contain
# the pip_repository logic to a single module extension.
# needed for the whl_libary declarations below.
requirements_lock_content = module_ctx.read(requrements_lock)
parse_result = parse_requirements(requirements_lock_content)
requirements = parse_result.requirements
extra_pip_args = pip_attr.extra_pip_args + parse_result.options

# Create the repository where users load the `requirement` macro. Under bzlmod
# this does not create the install_deps() macro.
# TODO: we may not need this repository once we have entry points
# supported. For now a user can access this repository and use
# the entrypoint functionality.
pip_repository_bzlmod(
name = pip_name,
repo_name = pip_name,
requirements_lock = pip_attr.requirements_lock,
)
if hub_name not in whl_map:
whl_map[hub_name] = {}

Expand Down Expand Up @@ -157,12 +145,12 @@ def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map):
if whl_name not in whl_map[hub_name]:
whl_map[hub_name][whl_name] = {}

whl_map[hub_name][whl_name][pip_attr.python_version] = pip_name + "_"
whl_map[hub_name][whl_name][full_version(pip_attr.python_version)] = pip_name + "_"

def _pip_impl(module_ctx):
"""Implementation of a class tag that creates the pip hub(s) and corresponding pip spoke, alias and whl repositories.
"""Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories.
This implmentation iterates through all of the `pip.parse` calls and creates
This implementation iterates through all of the `pip.parse` calls and creates
different pip hub repositories based on the "hub_name". Each of the
pip calls create spoke repos that uses a specific Python interpreter.
Expand Down Expand Up @@ -196,52 +184,33 @@ def _pip_impl(module_ctx):
Both of these pip spokes contain requirements files that includes websocket
and its dependencies.
Two different repositories are created for the two spokes:
- @@rules_python~override~pip~pip_39
- @@rules_python~override~pip~pip_310
The different spoke names are a combination of the hub_name and the Python version.
In the future we may remove this repository, but we do not support entry points.
yet, and that functionality exists in these repos.
We also need repositories for the wheels that the different pip spokes contain.
For each Python version a different wheel repository is created. In our example
each pip spoke had a requirments file that contained websockets. We
each pip spoke had a requirements file that contained websockets. We
then create two different wheel repositories that are named the following.
- @@rules_python~override~pip~pip_39_websockets
- @@rules_python~override~pip~pip_310_websockets
And if the wheel has any other dependies subsequest wheels are created in the same fashion.
We also create a repository for the wheel alias. We want to just use the syntax
'requirement("websockets")' we need to have an alias repository that is named:
And if the wheel has any other dependencies subsequent wheels are created in the same fashion.
- @@rules_python~override~pip~pip_websockets
This repository contains alias statements for the different wheel components (pkg, data, etc).
Each of those aliases has a select that resolves to a spoke repository depending on
the Python version.
The hub repository has aliases for `pkg`, `data`, etc, which have a select that resolves to
a spoke repository depending on the Python version.
Also we may have more than one hub as defined in a MODULES.bazel file. So we could have multiple
hubs pointing to various different pip spokes.
Some other business rules notes. A hub can only have one spoke per Python version. We cannot
Some other business rules notes. A hub can only have one spoke per Python version. We cannot
have a hub named "pip" that has two spokes that use the Python 3.9 interpreter. Second
we cannot have the same hub name used in submodules. The hub name has to be globally
we cannot have the same hub name used in sub-modules. The hub name has to be globally
unique.
This implementation reuses elements of non-bzlmod code and also reuses the first implementation
of pip bzlmod, but adds the capability to have multiple pip.parse calls.
This implementation also handles the creation of whl_modification JSON files that are used
during the creation of wheel libraries. These JSON files used via the annotations argument
during the creation of wheel libraries. These JSON files used via the annotations argument
when calling wheel_installer.py.
Args:
module_ctx: module contents
"""

# Build all of the wheel modifications if the tag class is called.
Expand All @@ -259,63 +228,46 @@ def _pip_impl(module_ctx):
for mod in module_ctx.modules:
for pip_attr in mod.tags.parse:
hub_name = pip_attr.hub_name
if hub_name in pip_hub_map:
# We cannot have two hubs with the same name in different
# modules.
if pip_hub_map[hub_name].module_name != mod.name:
fail((
"Duplicate cross-module pip hub named '{hub}': pip hub " +
"names must be unique across modules. First defined " +
"by module '{first_module}', second attempted by " +
"module '{second_module}'"
).format(
hub = hub_name,
first_module = pip_hub_map[hub_name].module_name,
second_module = mod.name,
))

if pip_attr.python_version in pip_hub_map[hub_name].python_versions:
fail((
"Duplicate pip python version '{version}' for hub " +
"'{hub}' in module '{module}': the Python versions " +
"used for a hub must be unique"
).format(
hub = hub_name,
module = mod.name,
version = pip_attr.python_version,
))
else:
pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version)
else:
if hub_name not in pip_hub_map:
pip_hub_map[pip_attr.hub_name] = struct(
module_name = mod.name,
python_versions = [pip_attr.python_version],
)
elif pip_hub_map[hub_name].module_name != mod.name:
# We cannot have two hubs with the same name in different
# modules.
fail((
"Duplicate cross-module pip hub named '{hub}': pip hub " +
"names must be unique across modules. First defined " +
"by module '{first_module}', second attempted by " +
"module '{second_module}'"
).format(
hub = hub_name,
first_module = pip_hub_map[hub_name].module_name,
second_module = mod.name,
))

_create_versioned_pip_and_whl_repos(module_ctx, pip_attr, hub_whl_map)
elif pip_attr.python_version in pip_hub_map[hub_name].python_versions:
fail((
"Duplicate pip python version '{version}' for hub " +
"'{hub}' in module '{module}': the Python versions " +
"used for a hub must be unique"
).format(
hub = hub_name,
module = mod.name,
version = pip_attr.python_version,
))
else:
pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version)

_create_whl_repos(module_ctx, pip_attr, hub_whl_map)

for hub_name, whl_map in hub_whl_map.items():
for whl_name, version_map in whl_map.items():
if DEFAULT_PYTHON_VERSION in version_map:
whl_default_version = DEFAULT_PYTHON_VERSION
else:
whl_default_version = None

# Create the alias repositories which contains different select
# statements These select statements point to the different pip
# whls that are based on a specific version of Python.
whl_library_alias(
name = hub_name + "_" + whl_name,
wheel_name = whl_name,
default_version = whl_default_version,
version_map = version_map,
)

# Create the hub repository for pip.
pip_hub_repository_bzlmod(
name = hub_name,
repo_name = hub_name,
whl_library_alias_names = whl_map.keys(),
whl_map = whl_map,
default_version = full_version(DEFAULT_PYTHON_VERSION),
)

def _pip_parse_ext_attrs():
Expand Down
29 changes: 0 additions & 29 deletions python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl
View file

This file was deleted.

Loading

0 comments on commit f2a4dd5

Please sign in to comment.