diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3449bcfac0..a75b5a91a3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,5 +11,6 @@ /python/private/toolchains_repo.bzl @f0rmiga /python/tests/toolchains/ @f0rmiga -# pip_parse related code -/python/pip_install/ @hrfuller +# PyPI integration related code +/python/private/pypi/ @aignas +/tests/pypi/ @aignas diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index 8298f49cf2..50bfe9fe8e 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -6,16 +6,54 @@ load("@rules_python//python:pip.bzl", "pip_utils") load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library") -all_requirements = ["@my_project_pip_deps_vendored_certifi//:pkg", "@my_project_pip_deps_vendored_charset_normalizer//:pkg", "@my_project_pip_deps_vendored_idna//:pkg", "@my_project_pip_deps_vendored_requests//:pkg", "@my_project_pip_deps_vendored_urllib3//:pkg"] - -all_whl_requirements_by_package = {"certifi": "@my_project_pip_deps_vendored_certifi//:whl", "charset_normalizer": "@my_project_pip_deps_vendored_charset_normalizer//:whl", "idna": "@my_project_pip_deps_vendored_idna//:whl", "requests": "@my_project_pip_deps_vendored_requests//:whl", "urllib3": "@my_project_pip_deps_vendored_urllib3//:whl"} +all_requirements = [ + "@my_project_pip_deps_vendored_certifi//:pkg", + "@my_project_pip_deps_vendored_charset_normalizer//:pkg", + "@my_project_pip_deps_vendored_idna//:pkg", + "@my_project_pip_deps_vendored_requests//:pkg", + "@my_project_pip_deps_vendored_urllib3//:pkg", +] + +all_whl_requirements_by_package = { + "certifi": "@my_project_pip_deps_vendored_certifi//:whl", + "charset_normalizer": "@my_project_pip_deps_vendored_charset_normalizer//:whl", + "idna": "@my_project_pip_deps_vendored_idna//:whl", + "requests": "@my_project_pip_deps_vendored_requests//:whl", + "urllib3": "@my_project_pip_deps_vendored_urllib3//:whl", +} all_whl_requirements = all_whl_requirements_by_package.values() -all_data_requirements = ["@my_project_pip_deps_vendored_certifi//:data", "@my_project_pip_deps_vendored_charset_normalizer//:data", "@my_project_pip_deps_vendored_idna//:data", "@my_project_pip_deps_vendored_requests//:data", "@my_project_pip_deps_vendored_urllib3//:data"] - -_packages = [("my_project_pip_deps_vendored_certifi", "certifi==2023.7.22 --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"), ("my_project_pip_deps_vendored_charset_normalizer", "charset-normalizer==2.1.1 --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), ("my_project_pip_deps_vendored_idna", "idna==3.4 --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), ("my_project_pip_deps_vendored_requests", "requests==2.28.1 --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), ("my_project_pip_deps_vendored_urllib3", "urllib3==1.26.13 --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8")] -_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "envsubst": ["PIP_RETRIES"], "extra_pip_args": ["--retries=${PIP_RETRIES:-5}"], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": "@python39_host//:python", "quiet": True, "repo": "my_project_pip_deps_vendored", "repo_prefix": "my_project_pip_deps_vendored_", "timeout": 600} +all_data_requirements = [ + "@my_project_pip_deps_vendored_certifi//:data", + "@my_project_pip_deps_vendored_charset_normalizer//:data", + "@my_project_pip_deps_vendored_idna//:data", + "@my_project_pip_deps_vendored_requests//:data", + "@my_project_pip_deps_vendored_urllib3//:data", +] + +_packages = [ + ("my_project_pip_deps_vendored_certifi", "certifi==2023.7.22 --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"), + ("my_project_pip_deps_vendored_charset_normalizer", "charset-normalizer==2.1.1 --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), + ("my_project_pip_deps_vendored_idna", "idna==3.4 --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), + ("my_project_pip_deps_vendored_requests", "requests==2.28.1 --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), + ("my_project_pip_deps_vendored_urllib3", "urllib3==1.26.13 --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"), +] +_config = { + "download_only": False, + "enable_implicit_namespace_pkgs": False, + "environment": {}, + "envsubst": ["PIP_RETRIES"], + "extra_pip_args": ["--retries=${PIP_RETRIES:-5}"], + "isolated": True, + "pip_data_exclude": [], + "python_interpreter": "python3", + "python_interpreter_target": "@python39_host//:python", + "quiet": True, + "repo": "my_project_pip_deps_vendored", + "repo_prefix": "my_project_pip_deps_vendored_", + "timeout": 600, +} _annotations = {} def requirement(name): diff --git a/python/BUILD.bazel b/python/BUILD.bazel index cbf29964fb..29b495bf90 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -101,7 +101,7 @@ bzl_library( "//python/pip_install:requirements_bzl", "//python/private:bzlmod_enabled_bzl", "//python/private:full_version_bzl", - "//python/private:render_pkg_aliases_bzl", + "//python/private/pypi:render_pkg_aliases_bzl", "//python/private/whl_filegroup:whl_filegroup_bzl", ], ) diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index 1003efb8c2..f6f46db770 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -9,7 +9,7 @@ load( "PycCollectionFlag", ) load( - "//python/private:pip_flags.bzl", + "//python/private/pypi:flags.bzl", "INTERNAL_FLAGS", "UniversalWhlFlag", "UseWhlFlag", diff --git a/python/extensions/BUILD.bazel b/python/extensions/BUILD.bazel index eb095ab746..1bc2a71192 100644 --- a/python/extensions/BUILD.bazel +++ b/python/extensions/BUILD.bazel @@ -36,8 +36,6 @@ bzl_library( srcs = ["python.bzl"], visibility = ["//:__subpackages__"], deps = [ - "//python/private:util_bzl", - "//python/private/bzlmod:bazel_features_bzl", "//python/private/bzlmod:python_bzl", ], ) diff --git a/python/pip.bzl b/python/pip.bzl index 8cc091d3cb..f1c74dd964 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -19,12 +19,13 @@ symbols should not be used and they are either undocumented here or marked as for internal use only. """ -load("//python/pip_install:pip_repository.bzl", "pip_repository", _package_annotation = "package_annotation") load("//python/pip_install:requirements.bzl", _compile_pip_requirements = "compile_pip_requirements") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/private:full_version.bzl", "full_version") load("//python/private:normalize_name.bzl", "normalize_name") -load("//python/private:render_pkg_aliases.bzl", "NO_MATCH_ERROR_MESSAGE_TEMPLATE") +load("//python/private/pypi:package_annotation.bzl", _package_annotation = "package_annotation") +load("//python/private/pypi:pip_repository.bzl", "pip_repository") +load("//python/private/pypi:render_pkg_aliases.bzl", "NO_MATCH_ERROR_MESSAGE_TEMPLATE") load("//python/private/whl_filegroup:whl_filegroup.bzl", _whl_filegroup = "whl_filegroup") compile_pip_requirements = _compile_pip_requirements diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel index 91f2ec7b59..1894c4d915 100644 --- a/python/pip_install/BUILD.bazel +++ b/python/pip_install/BUILD.bazel @@ -22,23 +22,10 @@ bzl_library( name = "pip_repository_bzl", srcs = ["pip_repository.bzl"], deps = [ - ":repositories_bzl", - "//python:repositories_bzl", - "//python:versions_bzl", - "//python/pip_install/private:generate_group_library_build_bazel_bzl", - "//python/pip_install/private:generate_whl_library_build_bazel_bzl", - "//python/pip_install/private:srcs_bzl", - "//python/private:bzlmod_enabled_bzl", - "//python/private:envsubst_bzl", - "//python/private:normalize_name_bzl", - "//python/private:parse_requirements_bzl", - "//python/private:parse_whl_name_bzl", - "//python/private:patch_whl_bzl", - "//python/private:render_pkg_aliases_bzl", - "//python/private:repo_utils_bzl", - "//python/private:toolchains_repo_bzl", - "//python/private:whl_target_platforms_bzl", - "@bazel_skylib//lib:sets", + "//python/private/pypi:group_library_bzl", + "//python/private/pypi:package_annotation_bzl", + "//python/private/pypi:pip_repository_bzl", + "//python/private/pypi:whl_library_bzl", ], ) @@ -51,11 +38,6 @@ bzl_library( ], ) -bzl_library( - name = "requirements_parser_bzl", - srcs = ["requirements_parser.bzl"], -) - bzl_library( name = "repositories_bzl", srcs = ["repositories.bzl"], @@ -71,7 +53,6 @@ filegroup( srcs = glob(["*.bzl"]) + [ "BUILD.bazel", "pip_repository_requirements.bzl.tmpl", - "//python/pip_install/private:distribution", "//python/pip_install/tools/dependency_resolver:distribution", "//python/pip_install/tools/wheel_installer:distribution", ], @@ -92,23 +73,10 @@ filegroup( filegroup( name = "bzl", - srcs = glob(["*.bzl"]) + [ - "//python/pip_install/private:bzl_srcs", - ], + srcs = glob(["*.bzl"]), visibility = ["//:__subpackages__"], ) -filegroup( - name = "py_srcs", - srcs = [ - "//python/pip_install/tools/dependency_resolver:py_srcs", - "//python/pip_install/tools/wheel_installer:py_srcs", - "//python/private:repack_whl.py", - "//tools:wheelmaker.py", - ], - visibility = ["//python/pip_install/private:__pkg__"], -) - exports_files( glob(["*.bzl"]), visibility = ["//docs:__pkg__"], diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index d6c8d9149c..18deee1993 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -14,1099 +14,13 @@ "" -load("@bazel_skylib//lib:sets.bzl", "sets") -load("//python:repositories.bzl", "is_standalone_interpreter") -load("//python:versions.bzl", "WINDOWS_NAME") -load("//python/pip_install:repositories.bzl", "all_requirements") -load("//python/pip_install/private:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel") -load("//python/pip_install/private:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") -load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS") -load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") -load("//python/private:envsubst.bzl", "envsubst") -load("//python/private:normalize_name.bzl", "normalize_name") -load("//python/private:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") -load("//python/private:parse_whl_name.bzl", "parse_whl_name") -load("//python/private:patch_whl.bzl", "patch_whl") -load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias") -load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") -load("//python/private:toolchains_repo.bzl", "get_host_os_arch") -load("//python/private:whl_target_platforms.bzl", "whl_target_platforms") - -CPPFLAGS = "CPPFLAGS" - -COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools" - -_WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point" - -def _construct_pypath(rctx): - """Helper function to construct a PYTHONPATH. - - Contains entries for code in this repo as well as packages downloaded from //python/pip_install:repositories.bzl. - This allows us to run python code inside repository rule implementations. - - Args: - rctx: Handle to the repository_context. - - Returns: String of the PYTHONPATH. - """ - - separator = ":" if not "windows" in rctx.os.name.lower() else ";" - pypath = separator.join([ - str(rctx.path(entry).dirname) - for entry in rctx.attr._python_path_entries - ]) - return pypath - -def _get_python_interpreter_attr(rctx): - """A helper function for getting the `python_interpreter` attribute or it's default - - Args: - rctx (repository_ctx): Handle to the rule repository context. - - Returns: - str: The attribute value or it's default - """ - if rctx.attr.python_interpreter: - return rctx.attr.python_interpreter - - if "win" in rctx.os.name: - return "python.exe" - else: - return "python3" - -def _resolve_python_interpreter(rctx): - """Helper function to find the python interpreter from the common attributes - - Args: - rctx: Handle to the rule repository context. - - Returns: - `path` object, for the resolved path to the Python interpreter. - """ - python_interpreter = _get_python_interpreter_attr(rctx) - - if rctx.attr.python_interpreter_target != None: - python_interpreter = rctx.path(rctx.attr.python_interpreter_target) - - (os, _) = get_host_os_arch(rctx) - - # On Windows, the symlink doesn't work because Windows attempts to find - # Python DLLs where the symlink is, not where the symlink points. - if os == WINDOWS_NAME: - python_interpreter = python_interpreter.realpath - elif "/" not in python_interpreter: - # It's a plain command, e.g. "python3", to look up in the environment. - found_python_interpreter = rctx.which(python_interpreter) - if not found_python_interpreter: - fail("python interpreter `{}` not found in PATH".format(python_interpreter)) - python_interpreter = found_python_interpreter - else: - python_interpreter = rctx.path(python_interpreter) - return python_interpreter - -def _get_xcode_location_cflags(rctx): - """Query the xcode sdk location to update cflags - - Figure out if this interpreter target comes from rules_python, and patch the xcode sdk location if so. - Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg - otherwise. See https://github.com/indygreg/python-build-standalone/issues/103 - """ - - # Only run on MacOS hosts - if not rctx.os.name.lower().startswith("mac os"): - return [] - - xcode_sdk_location = repo_utils.execute_unchecked( - rctx, - op = "GetXcodeLocation", - arguments = [repo_utils.which_checked(rctx, "xcode-select"), "--print-path"], - ) - if xcode_sdk_location.return_code != 0: - return [] - - xcode_root = xcode_sdk_location.stdout.strip() - if COMMAND_LINE_TOOLS_PATH_SLUG not in xcode_root.lower(): - # This is a full xcode installation somewhere like /Applications/Xcode13.0.app/Contents/Developer - # so we need to change the path to to the macos specific tools which are in a different relative - # path than xcode installed command line tools. - xcode_root = "{}/Platforms/MacOSX.platform/Developer".format(xcode_root) - return [ - "-isysroot {}/SDKs/MacOSX.sdk".format(xcode_root), - ] - -def _get_toolchain_unix_cflags(rctx, python_interpreter): - """Gather cflags from a standalone toolchain for unix systems. - - Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg - otherwise. See https://github.com/indygreg/python-build-standalone/issues/103 - """ - - # Only run on Unix systems - if not rctx.os.name.lower().startswith(("mac os", "linux")): - return [] - - # Only update the location when using a standalone toolchain. - if not is_standalone_interpreter(rctx, python_interpreter): - return [] - - stdout = repo_utils.execute_checked_stdout( - rctx, - op = "GetPythonVersionForUnixCflags", - arguments = [ - python_interpreter, - "-c", - "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')", - ], - ) - _python_version = stdout - include_path = "{}/include/python{}".format( - python_interpreter.dirname, - _python_version, - ) - - return ["-isystem {}".format(include_path)] - -def use_isolated(ctx, attr): - """Determine whether or not to pass the pip `--isolated` flag to the pip invocation. - - Args: - ctx: repository or module context - attr: attributes for the repo rule or tag extension - - Returns: - True if --isolated should be passed - """ - use_isolated = attr.isolated - - # The environment variable will take precedence over the attribute - isolated_env = ctx.os.environ.get("RULES_PYTHON_PIP_ISOLATED", None) - if isolated_env != None: - if isolated_env.lower() in ("0", "false"): - use_isolated = False - else: - use_isolated = True - - return use_isolated - -def _parse_optional_attrs(rctx, args, extra_pip_args = None): - """Helper function to parse common attributes of pip_repository and whl_library repository rules. - - This function also serializes the structured arguments as JSON - so they can be passed on the command line to subprocesses. - - Args: - rctx: Handle to the rule repository context. - args: A list of parsed args for the rule. - extra_pip_args: The pip args to pass. - Returns: Augmented args list. - """ - - if use_isolated(rctx, rctx.attr): - args.append("--isolated") - - # Bazel version 7.1.0 and later (and rolling releases from version 8.0.0-pre.20240128.3) - # support rctx.getenv(name, default): When building incrementally, any change to the value of - # the variable named by name will cause this repository to be re-fetched. - if "getenv" in dir(rctx): - getenv = rctx.getenv - else: - getenv = rctx.os.environ.get - - # Check for None so we use empty default types from our attrs. - # Some args want to be list, and some want to be dict. - if extra_pip_args != None: - args += [ - "--extra_pip_args", - json.encode(struct(arg = [ - envsubst(pip_arg, rctx.attr.envsubst, getenv) - for pip_arg in rctx.attr.extra_pip_args - ])), - ] - - if rctx.attr.download_only: - args.append("--download_only") - - if rctx.attr.pip_data_exclude != None: - args += [ - "--pip_data_exclude", - json.encode(struct(arg = rctx.attr.pip_data_exclude)), - ] - - if rctx.attr.enable_implicit_namespace_pkgs: - args.append("--enable_implicit_namespace_pkgs") - - if rctx.attr.environment != None: - args += [ - "--environment", - json.encode(struct(arg = rctx.attr.environment)), - ] - - return args - -def _create_repository_execution_environment(rctx, python_interpreter): - """Create a environment dictionary for processes we spawn with rctx.execute. - - Args: - rctx (repository_ctx): The repository context. - python_interpreter (path): The resolved python interpreter. - Returns: - Dictionary of environment variable suitable to pass to rctx.execute. - """ - - # Gather any available CPPFLAGS values - cppflags = [] - cppflags.extend(_get_xcode_location_cflags(rctx)) - cppflags.extend(_get_toolchain_unix_cflags(rctx, python_interpreter)) - - env = { - "PYTHONPATH": _construct_pypath(rctx), - CPPFLAGS: " ".join(cppflags), - } - - return env - -_BUILD_FILE_CONTENTS = """\ -package(default_visibility = ["//visibility:public"]) - -# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it -exports_files(["requirements.bzl"]) -""" - -def _pip_repository_impl(rctx): - requirements_by_platform = parse_requirements( - rctx, - requirements_by_platform = rctx.attr.requirements_by_platform, - requirements_linux = rctx.attr.requirements_linux, - requirements_lock = rctx.attr.requirements_lock, - requirements_osx = rctx.attr.requirements_darwin, - requirements_windows = rctx.attr.requirements_windows, - extra_pip_args = rctx.attr.extra_pip_args, - ) - selected_requirements = {} - options = None - repository_platform = host_platform(rctx.os) - for name, requirements in requirements_by_platform.items(): - r = select_requirement( - requirements, - platform = repository_platform, - ) - if not r: - continue - options = options or r.extra_pip_args - selected_requirements[name] = r.requirement_line - - bzl_packages = sorted(selected_requirements.keys()) - - # Normalize cycles first - requirement_cycles = { - name: sorted(sets.to_list(sets.make(deps))) - for name, deps in rctx.attr.experimental_requirement_cycles.items() - } - - # Check for conflicts between cycles _before_ we normalize package names so - # that reported errors use the names the user specified - for i in range(len(requirement_cycles)): - left_group = requirement_cycles.keys()[i] - left_deps = requirement_cycles.values()[i] - for j in range(len(requirement_cycles) - (i + 1)): - right_deps = requirement_cycles.values()[1 + i + j] - right_group = requirement_cycles.keys()[1 + i + j] - for d in left_deps: - if d in right_deps: - fail("Error: Requirement %s cannot be repeated between cycles %s and %s; please merge the cycles." % (d, left_group, right_group)) - - # And normalize the names as used in the cycle specs - # - # NOTE: We must check that a listed dependency is actually in the actual - # requirements set for the current platform so that we can support cycles in - # platform-conditional requirements. Otherwise we'll blindly generate a - # label referencing a package which may not be installed on the current - # platform. - requirement_cycles = { - normalize_name(name): sorted([normalize_name(d) for d in group if normalize_name(d) in bzl_packages]) - for name, group in requirement_cycles.items() - } - - imports = [ - # NOTE: Maintain the order consistent with `buildifier` - 'load("@rules_python//python:pip.bzl", "pip_utils")', - 'load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library")', - ] - - annotations = {} - for pkg, annotation in rctx.attr.annotations.items(): - filename = "{}.annotation.json".format(normalize_name(pkg)) - rctx.file(filename, json.encode_indent(json.decode(annotation))) - annotations[pkg] = "@{name}//:{filename}".format(name = rctx.attr.name, filename = filename) - - config = { - "download_only": rctx.attr.download_only, - "enable_implicit_namespace_pkgs": rctx.attr.enable_implicit_namespace_pkgs, - "environment": rctx.attr.environment, - "envsubst": rctx.attr.envsubst, - "extra_pip_args": options, - "isolated": use_isolated(rctx, rctx.attr), - "pip_data_exclude": rctx.attr.pip_data_exclude, - "python_interpreter": _get_python_interpreter_attr(rctx), - "quiet": rctx.attr.quiet, - "repo": rctx.attr.name, - "timeout": rctx.attr.timeout, - } - if rctx.attr.use_hub_alias_dependencies: - config["dep_template"] = "@{}//{{name}}:{{target}}".format(rctx.attr.name) - else: - config["repo_prefix"] = "{}_".format(rctx.attr.name) - - if rctx.attr.python_interpreter_target: - config["python_interpreter_target"] = str(rctx.attr.python_interpreter_target) - if rctx.attr.experimental_target_platforms: - config["experimental_target_platforms"] = rctx.attr.experimental_target_platforms - - macro_tmpl = "@%s//{}:{}" % rctx.attr.name - - aliases = render_pkg_aliases( - aliases = { - pkg: [whl_alias(repo = rctx.attr.name + "_" + pkg)] - for pkg in bzl_packages or [] - }, - ) - for path, contents in aliases.items(): - rctx.file(path, contents) - - rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) - rctx.template("requirements.bzl", rctx.attr._template, substitutions = { - " # %%GROUP_LIBRARY%%": """\ - group_repo = "{name}__groups" - group_library( - name = group_repo, - repo_prefix = "{name}_", - groups = all_requirement_groups, - )""".format(name = rctx.attr.name) if not rctx.attr.use_hub_alias_dependencies else "", - "%%ALL_DATA_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, "data") - for p in bzl_packages - ]), - "%%ALL_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, "pkg") - for p in bzl_packages - ]), - "%%ALL_REQUIREMENT_GROUPS%%": _format_dict(_repr_dict(requirement_cycles)), - "%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": _format_dict(_repr_dict({ - p: macro_tmpl.format(p, "whl") - for p in bzl_packages - })), - "%%ANNOTATIONS%%": _format_dict(_repr_dict(annotations)), - "%%CONFIG%%": _format_dict(_repr_dict(config)), - "%%EXTRA_PIP_ARGS%%": json.encode(options), - "%%IMPORTS%%": "\n".join(imports), - "%%MACRO_TMPL%%": macro_tmpl, - "%%NAME%%": rctx.attr.name, - "%%PACKAGES%%": _format_repr_list( - [ - ("{}_{}".format(rctx.attr.name, p), r) - for p, r in sorted(selected_requirements.items()) - ], - ), - }) - - return - -common_env = [ - "RULES_PYTHON_PIP_ISOLATED", - REPO_DEBUG_ENV_VAR, -] - -common_attrs = { - "download_only": attr.bool( - doc = """ -Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of ---platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different -platform from the host platform. - """, - ), - "enable_implicit_namespace_pkgs": attr.bool( - default = False, - doc = """ -If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary -and py_test targets must specify either `legacy_create_init=False` or the global Bazel option -`--incompatible_default_to_explicit_init_py` to prevent `__init__.py` being automatically generated in every directory. - -This option is required to support some packages which cannot handle the conversion to pkg-util style. - """, - ), - "environment": attr.string_dict( - doc = """ -Environment variables to set in the pip subprocess. -Can be used to set common variables such as `http_proxy`, `https_proxy` and `no_proxy` -Note that pip is run with "--isolated" on the CLI so `PIP__` -style env vars are ignored, but env vars that control requests and urllib3 -can be passed. If you need `PIP__`, take a look at `extra_pip_args` -and `envsubst`. - """, - default = {}, - ), - "envsubst": attr.string_list( - mandatory = False, - doc = """\ -A list of environment variables to substitute (e.g. `["PIP_INDEX_URL", -"PIP_RETRIES"]`). The corresponding variables are expanded in `extra_pip_args` -using the syntax `$VARNAME` or `${VARNAME}` (expanding to empty string if unset) -or `${VARNAME:-default}` (expanding to default if the variable is unset or empty -in the environment). Note: On Bazel 6 and Bazel 7.0 changes to the variables named -here do not cause packages to be re-fetched. Don't fetch different things based -on the value of these variables. -""", - ), - "experimental_requirement_cycles": attr.string_list_dict( - default = {}, - doc = """\ -A mapping of dependency cycle names to a list of requirements which form that cycle. - -Requirements which form cycles will be installed together and taken as -dependencies together in order to ensure that the cycle is always satisified. - -Example: - `sphinx` depends on `sphinxcontrib-serializinghtml` - When listing both as requirements, ala - - ``` - py_binary( - name = "doctool", - ... - deps = [ - "@pypi//sphinx:pkg", - "@pypi//sphinxcontrib_serializinghtml", - ] - ) - ``` - - Will produce a Bazel error such as - - ``` - ERROR: .../external/pypi_sphinxcontrib_serializinghtml/BUILD.bazel:44:6: in alias rule @pypi_sphinxcontrib_serializinghtml//:pkg: cycle in dependency graph: - //:doctool (...) - @pypi//sphinxcontrib_serializinghtml:pkg (...) - .-> @pypi_sphinxcontrib_serializinghtml//:pkg (...) - | @pypi_sphinxcontrib_serializinghtml//:_pkg (...) - | @pypi_sphinx//:pkg (...) - | @pypi_sphinx//:_pkg (...) - `-- @pypi_sphinxcontrib_serializinghtml//:pkg (...) - ``` - - Which we can resolve by configuring these two requirements to be installed together as a cycle - - ``` - pip_parse( - ... - experimental_requirement_cycles = { - "sphinx": [ - "sphinx", - "sphinxcontrib-serializinghtml", - ] - }, - ) - ``` - -Warning: - If a dependency participates in multiple cycles, all of those cycles must be - collapsed down to one. For instance `a <-> b` and `a <-> c` cannot be listed - as two separate cycles. -""", - ), - "experimental_target_platforms": attr.string_list( - default = [], - doc = """\ -A list of platforms that we will generate the conditional dependency graph for -cross platform wheels by parsing the wheel metadata. This will generate the -correct dependencies for packages like `sphinx` or `pylint`, which include -`colorama` when installed and used on Windows platforms. - -An empty list means falling back to the legacy behaviour where the host -platform is the target platform. - -WARNING: It may not work as expected in cases where the python interpreter -implementation that is being used at runtime is different between different platforms. -This has been tested for CPython only. - -For specific target platforms use values of the form `_` where `` -is one of `linux`, `osx`, `windows` and arch is one of `x86_64`, `x86_32`, -`aarch64`, `s390x` and `ppc64le`. - -You can also target a specific Python version by using `cp3__`. -If multiple python versions are specified as target platforms, then select statements -of the `lib` and `whl` targets will include usage of version aware toolchain config -settings like `@rules_python//python/config_settings:is_python_3.y`. - -Special values: `host` (for generating deps for the host platform only) and -`_*` values. For example, `cp39_*`, `linux_*`, `cp39_linux_*`. - -NOTE: this is not for cross-compiling Python wheels but rather for parsing the `whl` METADATA correctly. -""", - ), - "extra_pip_args": attr.string_list( - doc = """Extra arguments to pass on to pip. Must not contain spaces. - -Supports environment variables using the syntax `$VARNAME` or -`${VARNAME}` (expanding to empty string if unset) or -`${VARNAME:-default}` (expanding to default if the variable is unset -or empty in the environment), if `"VARNAME"` is listed in the -`envsubst` attribute. See also `envsubst`. -""", - ), - "isolated": attr.bool( - doc = """\ -Whether or not to pass the [--isolated](https://pip.pypa.io/en/stable/cli/pip/#cmdoption-isolated) flag to -the underlying pip command. Alternatively, the `RULES_PYTHON_PIP_ISOLATED` environment variable can be used -to control this flag. -""", - default = True, - ), - "pip_data_exclude": attr.string_list( - doc = "Additional data exclusion parameters to add to the pip packages BUILD file.", - ), - "python_interpreter": attr.string( - doc = """\ -The python interpreter to use. This can either be an absolute path or the name -of a binary found on the host's `PATH` environment variable. If no value is set -`python3` is defaulted for Unix systems and `python.exe` for Windows. -""", - # NOTE: This attribute should not have a default. See `_get_python_interpreter_attr` - # default = "python3" - ), - "python_interpreter_target": attr.label( - allow_single_file = True, - doc = """ -If you are using a custom python interpreter built by another repository rule, -use this attribute to specify its BUILD target. This allows pip_repository to invoke -pip using the same interpreter as your toolchain. If set, takes precedence over -python_interpreter. An example value: "@python3_x86_64-unknown-linux-gnu//:python". -""", - ), - "quiet": attr.bool( - default = True, - doc = """\ -If True, suppress printing stdout and stderr output to the terminal. - -If you would like to get more diagnostic output, please use: - - RULES_PYTHON_REPO_DEBUG=1 - -or - - RULES_PYTHON_REPO_DEBUG_VERBOSITY= -""", - ), - "repo_prefix": attr.string( - doc = """ -Prefix for the generated packages will be of the form `@//...` - -DEPRECATED. Only left for people who vendor requirements.bzl. -""", - ), - # 600 is documented as default here: https://docs.bazel.build/versions/master/skylark/lib/repository_ctx.html#execute - "timeout": attr.int( - default = 600, - doc = "Timeout (in seconds) on the rule's execution duration.", - ), - "_py_srcs": attr.label_list( - doc = "Python sources used in the repository rule", - allow_files = True, - default = PIP_INSTALL_PY_SRCS, - ), -} - -pip_repository_attrs = { - "annotations": attr.string_dict( - doc = "Optional annotations to apply to packages", - ), - "requirements_by_platform": attr.label_keyed_string_dict( - doc = """\ -The requirements files and the comma delimited list of target platforms as values. - -The keys are the requirement files and the values are comma-separated platform -identifiers. For now we only support `_` values that are present in -`@platforms//os` and `@platforms//cpu` packages respectively. -""", - ), - "requirements_darwin": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Mac OS", - ), - "requirements_linux": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Linux", - ), - "requirements_lock": attr.label( - allow_single_file = True, - doc = """\ -A fully resolved 'requirements.txt' pip requirement file containing the -transitive set of your dependencies. If this file is passed instead of -'requirements' no resolve will take place and pip_repository will create -individual repositories for each of your dependencies so that wheels are -fetched/built only for the targets specified by 'build/run/test'. Note that if -your lockfile is platform-dependent, you can use the `requirements_[platform]` -attributes. - -Note, that in general requirements files are compiled for a specific platform, -but sometimes they can work for multiple platforms. `rules_python` right now -supports requirements files that are created for a particular platform without -platform markers. -""", - ), - "requirements_windows": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Windows", - ), - "use_hub_alias_dependencies": attr.bool( - default = False, - doc = """\ -Controls if the hub alias dependencies are used. If set to true, then the -group_library will be included in the hub repo. - -True will become default in a subsequent release. -""", - ), - "_template": attr.label( - default = ":pip_repository_requirements.bzl.tmpl", - ), -} - -pip_repository_attrs.update(**common_attrs) - -pip_repository = repository_rule( - attrs = pip_repository_attrs, - doc = """Accepts a locked/compiled requirements file and installs the dependencies listed within. - -Those dependencies become available in a generated `requirements.bzl` file. -You can instead check this `requirements.bzl` file into your repo, see the "vendoring" section below. - -In your WORKSPACE file: - -```starlark -load("@rules_python//python:pip.bzl", "pip_parse") - -pip_parse( - name = "pypi", - requirements_lock = ":requirements.txt", -) - -load("@pypi//:requirements.bzl", "install_deps") - -install_deps() -``` - -You can then reference installed dependencies from a `BUILD` file with the alias targets generated in the same repo, for example, for `PyYAML` we would have the following: -- `@pypi//pyyaml` and `@pypi//pyyaml:pkg` both point to the `py_library` - created after extracting the `PyYAML` package. -- `@pypi//pyyaml:data` points to the extra data included in the package. -- `@pypi//pyyaml:dist_info` points to the `dist-info` files in the package. -- `@pypi//pyyaml:whl` points to the wheel file that was extracted. - -```starlark -py_library( - name = "bar", - ... - deps = [ - "//my/other:dep", - "@pypi//numpy", - "@pypi//requests", - ], -) -``` - -or - -```starlark -load("@pypi//:requirements.bzl", "requirement") - -py_library( - name = "bar", - ... - deps = [ - "//my/other:dep", - requirement("numpy"), - requirement("requests"), - ], -) -``` - -In addition to the `requirement` macro, which is used to access the generated `py_library` -target generated from a package's wheel, The generated `requirements.bzl` file contains -functionality for exposing [entry points][whl_ep] as `py_binary` targets as well. - -[whl_ep]: https://packaging.python.org/specifications/entry-points/ - -```starlark -load("@pypi//:requirements.bzl", "entry_point") - -alias( - name = "pip-compile", - actual = entry_point( - pkg = "pip-tools", - script = "pip-compile", - ), -) -``` - -Note that for packages whose name and script are the same, only the name of the package -is needed when calling the `entry_point` macro. - -```starlark -load("@pip//:requirements.bzl", "entry_point") - -alias( - name = "flake8", - actual = entry_point("flake8"), -) -``` - -### Vendoring the requirements.bzl file - -In some cases you may not want to generate the requirements.bzl file as a repository rule -while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module -such as a ruleset, you may want to include the requirements.bzl file rather than make your users -install the WORKSPACE setup to generate it. -See https://github.com/bazelbuild/rules_python/issues/608 - -This is the same workflow as Gazelle, which creates `go_repository` rules with -[`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos) - -To do this, use the "write to source file" pattern documented in -https://blog.aspect.dev/bazel-can-write-to-the-source-folder -to put a copy of the generated requirements.bzl into your project. -Then load the requirements.bzl file directly rather than from the generated repository. -See the example in rules_python/examples/pip_parse_vendored. -""", - implementation = _pip_repository_impl, - environ = common_env, -) - -def _whl_library_impl(rctx): - python_interpreter = _resolve_python_interpreter(rctx) - args = [ - python_interpreter, - "-m", - "python.pip_install.tools.wheel_installer.wheel_installer", - "--requirement", - rctx.attr.requirement, - ] - extra_pip_args = [] - extra_pip_args.extend(rctx.attr.extra_pip_args) - - # Manually construct the PYTHONPATH since we cannot use the toolchain here - environment = _create_repository_execution_environment(rctx, python_interpreter) - - whl_path = None - if rctx.attr.whl_file: - whl_path = rctx.path(rctx.attr.whl_file) - - # Simulate the behaviour where the whl is present in the current directory. - rctx.symlink(whl_path, whl_path.basename) - whl_path = rctx.path(whl_path.basename) - elif rctx.attr.urls: - filename = rctx.attr.filename - urls = rctx.attr.urls - if not filename: - _, _, filename = urls[0].rpartition("/") - - if not (filename.endswith(".whl") or filename.endswith("tar.gz") or filename.endswith(".zip")): - if rctx.attr.filename: - msg = "got '{}'".format(filename) - else: - msg = "detected '{}' from url:\n{}".format(filename, urls[0]) - fail("Only '.whl', '.tar.gz' or '.zip' files are supported, {}".format(msg)) - - result = rctx.download( - url = urls, - output = filename, - sha256 = rctx.attr.sha256, - auth = get_auth(rctx, urls), - ) - - if not result.success: - fail("could not download the '{}' from {}:\n{}".format(filename, urls, result)) - - if filename.endswith(".whl"): - whl_path = rctx.path(rctx.attr.filename) - else: - # It is an sdist and we need to tell PyPI to use a file in this directory - # and not use any indexes. - extra_pip_args.extend(["--no-index", "--find-links", "."]) - - args = _parse_optional_attrs(rctx, args, extra_pip_args) - - if not whl_path: - repo_utils.execute_checked( - rctx, - op = "whl_library.ResolveRequirement({}, {})".format(rctx.attr.name, rctx.attr.requirement), - arguments = args, - environment = environment, - quiet = rctx.attr.quiet, - timeout = rctx.attr.timeout, - ) - - whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"]) - if not rctx.delete("whl_file.json"): - fail("failed to delete the whl_file.json file") - - if rctx.attr.whl_patches: - patches = {} - for patch_file, json_args in rctx.attr.whl_patches.items(): - patch_dst = struct(**json.decode(json_args)) - if whl_path.basename in patch_dst.whls: - patches[patch_file] = patch_dst.patch_strip - - whl_path = patch_whl( - rctx, - python_interpreter = python_interpreter, - whl_path = whl_path, - patches = patches, - quiet = rctx.attr.quiet, - timeout = rctx.attr.timeout, - ) - - target_platforms = rctx.attr.experimental_target_platforms - if target_platforms: - parsed_whl = parse_whl_name(whl_path.basename) - if parsed_whl.platform_tag != "any": - # NOTE @aignas 2023-12-04: if the wheel is a platform specific - # wheel, we only include deps for that target platform - target_platforms = [ - p.target_platform - for p in whl_target_platforms( - platform_tag = parsed_whl.platform_tag, - abi_tag = parsed_whl.abi_tag, - ) - ] - - repo_utils.execute_checked( - rctx, - op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path), - arguments = args + [ - "--whl-file", - whl_path, - ] + ["--platform={}".format(p) for p in target_platforms], - environment = environment, - quiet = rctx.attr.quiet, - timeout = rctx.attr.timeout, - ) - - metadata = json.decode(rctx.read("metadata.json")) - rctx.delete("metadata.json") - - entry_points = {} - for item in metadata["entry_points"]: - name = item["name"] - module = item["module"] - attribute = item["attribute"] - - # There is an extreme edge-case with entry_points that end with `.py` - # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174 - entry_point_without_py = name[:-3] + "_py" if name.endswith(".py") else name - entry_point_target_name = ( - _WHEEL_ENTRY_POINT_PREFIX + "_" + entry_point_without_py - ) - entry_point_script_name = entry_point_target_name + ".py" - - rctx.file( - entry_point_script_name, - _generate_entry_point_contents(module, attribute), - ) - entry_points[entry_point_without_py] = entry_point_script_name - - build_file_contents = generate_whl_library_build_bazel( - dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix), - whl_name = whl_path.basename, - dependencies = metadata["deps"], - dependencies_by_platform = metadata["deps_by_platform"], - group_name = rctx.attr.group_name, - group_deps = rctx.attr.group_deps, - data_exclude = rctx.attr.pip_data_exclude, - tags = [ - "pypi_name=" + metadata["name"], - "pypi_version=" + metadata["version"], - ], - entry_points = entry_points, - annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))), - ) - rctx.file("BUILD.bazel", build_file_contents) - - return - -def _generate_entry_point_contents( - module, - attribute, - shebang = "#!/usr/bin/env python3"): - """Generate the contents of an entry point script. - - Args: - module (str): The name of the module to use. - attribute (str): The name of the attribute to call. - shebang (str, optional): The shebang to use for the entry point python - file. - - Returns: - str: A string of python code. - """ - contents = """\ -{shebang} -import sys -from {module} import {attribute} -if __name__ == "__main__": - sys.exit({attribute}()) -""".format( - shebang = shebang, - module = module, - attribute = attribute, - ) - return contents - -# NOTE @aignas 2024-03-21: The usage of dict({}, **common) ensures that all args to `dict` are unique -whl_library_attrs = dict({ - "annotation": attr.label( - doc = ( - "Optional json encoded file containing annotation to apply to the extracted wheel. " + - "See `package_annotation`" - ), - allow_files = True, - ), - "dep_template": attr.string( - doc = """ -The dep template to use for referencing the dependencies. It should have `{name}` -and `{target}` tokens that will be replaced with the normalized distribution name -and the target that we need respectively. -""", - ), - "filename": attr.string( - doc = "Download the whl file to this filename. Only used when the `urls` is passed. If not specified, will be auto-detected from the `urls`.", - ), - "group_deps": attr.string_list( - doc = "List of dependencies to skip in order to break the cycles within a dependency group.", - default = [], - ), - "group_name": attr.string( - doc = "Name of the group, if any.", - ), - "repo": attr.string( - mandatory = True, - doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.", - ), - "requirement": attr.string( - mandatory = True, - doc = "Python requirement string describing the package to make available, if 'urls' or 'whl_file' is given, then this only needs to include foo[any_extras] as a bare minimum.", - ), - "sha256": attr.string( - doc = "The sha256 of the downloaded whl. Only used when the `urls` is passed.", - ), - "urls": attr.string_list( - doc = """\ -The list of urls of the whl to be downloaded using bazel downloader. Using this -attr makes `extra_pip_args` and `download_only` ignored.""", - ), - "whl_file": attr.label( - doc = "The whl file that should be used instead of downloading or building the whl.", - ), - "whl_patches": attr.label_keyed_string_dict( - doc = """a label-keyed-string dict that has - json.encode(struct([whl_file], patch_strip]) as values. This - is to maintain flexibility and correct bzlmod extension interface - until we have a better way to define whl_library and move whl - patching to a separate place. INTERNAL USE ONLY.""", - ), - "_python_path_entries": attr.label_list( - # Get the root directory of these rules and keep them as a default attribute - # in order to avoid unnecessary repository fetching restarts. - # - # This is very similar to what was done in https://github.com/bazelbuild/rules_go/pull/3478 - default = [ - Label("//:BUILD.bazel"), - ] + [ - # Includes all the external dependencies from repositories.bzl - Label("@" + repo + "//:BUILD.bazel") - for repo in all_requirements - ], - ), -}, **common_attrs) -whl_library_attrs.update(AUTH_ATTRS) - -whl_library = repository_rule( - attrs = whl_library_attrs, - doc = """ -Download and extracts a single wheel based into a bazel repo based on the requirement string passed in. -Instantiated from pip_repository and inherits config options from there.""", - implementation = _whl_library_impl, - environ = common_env, -) - -def package_annotation( - additive_build_content = None, - copy_files = {}, - copy_executables = {}, - data = [], - data_exclude_glob = [], - srcs_exclude_glob = []): - """Annotations to apply to the BUILD file content from package generated from a `pip_repository` rule. - - [cf]: https://github.com/bazelbuild/bazel-skylib/blob/main/docs/copy_file_doc.md - - Args: - additive_build_content (str, optional): Raw text to add to the generated `BUILD` file of a package. - copy_files (dict, optional): A mapping of `src` and `out` files for [@bazel_skylib//rules:copy_file.bzl][cf] - copy_executables (dict, optional): A mapping of `src` and `out` files for - [@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as - executable. - data (list, optional): A list of labels to add as `data` dependencies to the generated `py_library` target. - data_exclude_glob (list, optional): A list of exclude glob patterns to add as `data` to the generated - `py_library` target. - srcs_exclude_glob (list, optional): A list of labels to add as `srcs` to the generated `py_library` target. - - Returns: - str: A json encoded string of the provided content. - """ - return json.encode(struct( - additive_build_content = additive_build_content, - copy_files = copy_files, - copy_executables = copy_executables, - data = data, - data_exclude_glob = data_exclude_glob, - srcs_exclude_glob = srcs_exclude_glob, - )) - -def _group_library_impl(rctx): - build_file_contents = generate_group_library_build_bazel( - repo_prefix = rctx.attr.repo_prefix, - groups = rctx.attr.groups, - ) - rctx.file("BUILD.bazel", build_file_contents) - -group_library = repository_rule( - attrs = { - "groups": attr.string_list_dict( - doc = "A mapping of group names to requirements within that group.", - ), - "repo_prefix": attr.string( - doc = "Prefix used for the whl_library created components of each group", - ), - }, - implementation = _group_library_impl, - doc = """ -Create a package containing only wrapper py_library and whl_library rules for implementing dependency groups. -This is an implementation detail of dependency groups and should not be used alone. - """, -) - -# pip_repository implementation - -def _format_list(items): - return "[{}]".format(", ".join(items)) - -def _format_repr_list(strings): - return _format_list( - [repr(s) for s in strings], - ) - -def _repr_dict(items): - return {k: repr(v) for k, v in items.items()} - -def _format_dict(items): - return "{{{}}}".format(", ".join(sorted(['"{}": {}'.format(k, v) for k, v in items.items()]))) +load("//python/private/pypi:group_library.bzl", _group_library = "group_library") +load("//python/private/pypi:package_annotation.bzl", _package_annotation = "package_annotation") +load("//python/private/pypi:pip_repository.bzl", _pip_repository = "pip_repository") +load("//python/private/pypi:whl_library.bzl", _whl_library = "whl_library") + +# Re-exports for backwards compatibility +group_library = _group_library +pip_repository = _pip_repository +whl_library = _whl_library +package_annotation = _package_annotation diff --git a/python/pip_install/private/BUILD.bazel b/python/pip_install/private/BUILD.bazel deleted file mode 100644 index 887d2d3468..0000000000 --- a/python/pip_install/private/BUILD.bazel +++ /dev/null @@ -1,48 +0,0 @@ -load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load(":pip_install_utils.bzl", "srcs_module") - -package(default_visibility = ["//:__subpackages__"]) - -exports_files([ - "srcs.bzl", -]) - -filegroup( - name = "distribution", - srcs = glob(["*"]), - visibility = ["//python/pip_install:__subpackages__"], -) - -filegroup( - name = "bzl_srcs", - srcs = glob(["*.bzl"]), -) - -srcs_module( - name = "srcs_module", - srcs = "//python/pip_install:py_srcs", - dest = ":srcs.bzl", -) - -bzl_library( - name = "generate_whl_library_build_bazel_bzl", - srcs = ["generate_whl_library_build_bazel.bzl"], - deps = [ - "//python/private:labels_bzl", - "//python/private:normalize_name_bzl", - ], -) - -bzl_library( - name = "generate_group_library_build_bazel_bzl", - srcs = ["generate_group_library_build_bazel.bzl"], - deps = [ - "//python/private:labels_bzl", - "//python/private:normalize_name_bzl", - ], -) - -bzl_library( - name = "srcs_bzl", - srcs = ["srcs.bzl"], -) diff --git a/python/pip_install/private/pip_install_utils.bzl b/python/pip_install/private/pip_install_utils.bzl deleted file mode 100644 index 488583dcb8..0000000000 --- a/python/pip_install/private/pip_install_utils.bzl +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utilities for `rules_python` pip rules""" - -_SRCS_TEMPLATE = """\ -\"\"\"A generated file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules - -This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.update` target. Please -`bazel run` this target to apply any updates. Note that doing so will discard any local modifications. -"\"\" - -# Each source file is tracked as a target so `pip_repository` rules will know to automatically rebuild if any of the -# sources changed. -PIP_INSTALL_PY_SRCS = [ - {srcs} -] -""" - -def _src_label(file): - dir_path, file_name = file.short_path.rsplit("/", 1) - - return "@rules_python//{}:{}".format( - dir_path, - file_name, - ) - -def _srcs_module_impl(ctx): - srcs = [_src_label(src) for src in ctx.files.srcs] - if not srcs: - fail("`srcs` cannot be empty") - output = ctx.actions.declare_file(ctx.label.name) - - ctx.actions.write( - output = output, - content = _SRCS_TEMPLATE.format( - srcs = "\n ".join(["\"{}\",".format(src) for src in srcs]), - ), - ) - - return DefaultInfo( - files = depset([output]), - ) - -_srcs_module = rule( - doc = "A rule for writing a list of sources to a templated file", - implementation = _srcs_module_impl, - attrs = { - "srcs": attr.label( - doc = "A filegroup of source files", - allow_files = True, - ), - }, -) - -_INSTALLER_TEMPLATE = """\ -#!/bin/bash -set -euo pipefail -cp -f "{path}" "${{BUILD_WORKSPACE_DIRECTORY}}/{dest}" -""" - -def _srcs_updater_impl(ctx): - output = ctx.actions.declare_file(ctx.label.name + ".sh") - target_file = ctx.file.input - dest = ctx.file.dest.short_path - - ctx.actions.write( - output = output, - content = _INSTALLER_TEMPLATE.format( - path = target_file.short_path, - dest = dest, - ), - is_executable = True, - ) - - return DefaultInfo( - files = depset([output]), - runfiles = ctx.runfiles(files = [target_file]), - executable = output, - ) - -_srcs_updater = rule( - doc = "A rule for writing a `srcs.bzl` file back to the repository", - implementation = _srcs_updater_impl, - attrs = { - "dest": attr.label( - doc = "The target file to write the new `input` to.", - allow_single_file = ["srcs.bzl"], - mandatory = True, - ), - "input": attr.label( - doc = "The file to write back to the repository", - allow_single_file = True, - mandatory = True, - ), - }, - executable = True, -) - -def srcs_module(name, dest, **kwargs): - """A helper rule to ensure `pip_repository` rules are always up to date - - Args: - name (str): The name of the sources module - dest (str): The filename the module should be written as in the current package. - **kwargs (dict): Additional keyword arguments - """ - tags = kwargs.pop("tags", []) - - _srcs_module( - name = name, - tags = tags, - **kwargs - ) - - _srcs_updater( - name = name + ".update", - input = name, - dest = dest, - tags = tags, - ) diff --git a/python/pip_install/private/srcs.bzl b/python/pip_install/private/srcs.bzl deleted file mode 100644 index e92e49fc5f..0000000000 --- a/python/pip_install/private/srcs.bzl +++ /dev/null @@ -1,18 +0,0 @@ -"""A generated file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules - -This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.update` target. Please -`bazel run` this target to apply any updates. Note that doing so will discard any local modifications. -""" - -# Each source file is tracked as a target so `pip_repository` rules will know to automatically rebuild if any of the -# sources changed. -PIP_INSTALL_PY_SRCS = [ - "@rules_python//python/pip_install/tools/dependency_resolver:__init__.py", - "@rules_python//python/pip_install/tools/dependency_resolver:dependency_resolver.py", - "@rules_python//python/pip_install/tools/wheel_installer:arguments.py", - "@rules_python//python/pip_install/tools/wheel_installer:namespace_pkgs.py", - "@rules_python//python/pip_install/tools/wheel_installer:wheel.py", - "@rules_python//python/pip_install/tools/wheel_installer:wheel_installer.py", - "@rules_python//python/private:repack_whl.py", - "@rules_python//tools:wheelmaker.py", -] diff --git a/python/pip_install/requirements_parser.bzl b/python/pip_install/requirements_parser.bzl index 3b49fdf181..82ec1b946c 100644 --- a/python/pip_install/requirements_parser.bzl +++ b/python/pip_install/requirements_parser.bzl @@ -14,120 +14,6 @@ "Pip requirements parser for Starlark" -_STATE = struct( - # Consume extraneous whitespace - ConsumeSpace = 0, - # Consume a comment - ConsumeComment = 1, - # Parse the name of a pip package - ParseDependency = 2, - # Parse a full requirement line - ParseRequirement = 3, - # Parse a pip option - ParseOption = 4, -) +load("//python/private/pypi:parse_requirements_txt.bzl", "parse_requirements_txt") -EOF = {} - -def parse(content): - """A simplistic (and incomplete) pip requirements lockfile parser. - - Parses package names and their full requirement lines, as well pip - options. - - Args: - content: lockfile content as a string - - Returns: - Struct with fields `requirements` and `options`. - - requirements: List of requirements, where each requirement is a 2-element - tuple containing the package name and the requirement line. - E.g., [(certifi', 'certifi==2021.10.8 --hash=sha256:7888...'), ...] - - options: List of pip option lines - """ - content = content.replace("\r", "") - - result = struct( - requirements = [], - options = [], - ) - state = _STATE.ConsumeSpace - buffer = "" - - inputs = content.elems()[:] - inputs.append(EOF) - - for input in inputs: - if state == _STATE.ConsumeSpace: - (state, buffer) = _handleConsumeSpace(input) - elif state == _STATE.ConsumeComment: - (state, buffer) = _handleConsumeComment(input, buffer, result) - elif state == _STATE.ParseDependency: - (state, buffer) = _handleParseDependency(input, buffer, result) - elif state == _STATE.ParseOption: - (state, buffer) = _handleParseOption(input, buffer, result) - elif state == _STATE.ParseRequirement: - (state, buffer) = _handleParseRequirement(input, buffer, result) - else: - fail("Unknown state %d" % state) - - return result - -def _handleConsumeSpace(input): - if input == EOF: - return (_STATE.ConsumeSpace, "") - if input.isspace(): - return (_STATE.ConsumeSpace, "") - elif input == "#": - return (_STATE.ConsumeComment, "") - elif input == "-": - return (_STATE.ParseOption, input) - - return (_STATE.ParseDependency, input) - -def _handleConsumeComment(input, buffer, result): - if input == "\n": - if len(result.requirements) > 0 and len(result.requirements[-1]) == 1: - result.requirements[-1] = (result.requirements[-1][0], buffer.rstrip(" \n")) - return (_STATE.ConsumeSpace, "") - elif len(buffer) > 0: - result.options.append(buffer.rstrip(" \n")) - return (_STATE.ConsumeSpace, "") - return (_STATE.ConsumeSpace, "") - return (_STATE.ConsumeComment, buffer) - -def _handleParseDependency(input, buffer, result): - if input == EOF: - fail("Enountered unexpected end of file while parsing requirement") - elif input.isspace() or input in [">", "<", "~", "=", ";", "["]: - result.requirements.append((buffer,)) - return (_STATE.ParseRequirement, buffer + input) - - return (_STATE.ParseDependency, buffer + input) - -def _handleParseOption(input, buffer, result): - if input == "\n" and buffer.endswith("\\"): - return (_STATE.ParseOption, buffer[0:-1]) - elif input == " ": - result.options.append(buffer.rstrip("\n")) - return (_STATE.ParseOption, "") - elif input == "\n" or input == EOF: - result.options.append(buffer.rstrip("\n")) - return (_STATE.ConsumeSpace, "") - elif input == "#" and (len(buffer) == 0 or buffer[-1].isspace()): - return (_STATE.ConsumeComment, buffer) - - return (_STATE.ParseOption, buffer + input) - -def _handleParseRequirement(input, buffer, result): - if input == "\n" and buffer.endswith("\\"): - return (_STATE.ParseRequirement, buffer[0:-1]) - elif input == "\n" or input == EOF: - result.requirements[-1] = (result.requirements[-1][0], buffer.rstrip(" \n")) - return (_STATE.ConsumeSpace, "") - elif input == "#" and (len(buffer) == 0 or buffer[-1].isspace()): - return (_STATE.ConsumeComment, buffer) - - return (_STATE.ParseRequirement, buffer + input) +parse = parse_requirements_txt diff --git a/python/pip_install/tools/dependency_resolver/BUILD.bazel b/python/pip_install/tools/dependency_resolver/BUILD.bazel index c2cfb39509..467b009332 100644 --- a/python/pip_install/tools/dependency_resolver/BUILD.bazel +++ b/python/pip_install/tools/dependency_resolver/BUILD.bazel @@ -15,5 +15,5 @@ filegroup( include = ["**/*.py"], exclude = ["**/*_test.py"], ), - visibility = ["//python/pip_install:__subpackages__"], + visibility = ["//:__subpackages__"], ) diff --git a/python/pip_install/tools/wheel_installer/BUILD.bazel b/python/pip_install/tools/wheel_installer/BUILD.bazel index a396488d3d..0c24d5a489 100644 --- a/python/pip_install/tools/wheel_installer/BUILD.bazel +++ b/python/pip_install/tools/wheel_installer/BUILD.bazel @@ -87,5 +87,5 @@ filegroup( include = ["**/*.py"], exclude = ["**/*_test.py"], ), - visibility = ["//python/pip_install:__subpackages__"], + visibility = ["//:__subpackages__"], ) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index cd385e3700..ccc6acdcbf 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -31,6 +31,7 @@ filegroup( "//python/private/bzlmod:distribution", "//python/private/common:distribution", "//python/private/proto:distribution", + "//python/private/pypi:distribution", "//python/private/whl_filegroup:distribution", "//tools/build_defs/python/private:distribution", ], @@ -125,62 +126,6 @@ bzl_library( srcs = ["normalize_name.bzl"], ) -bzl_library( - name = "patch_whl_bzl", - srcs = ["patch_whl.bzl"], - deps = [":parse_whl_name_bzl"], -) - -bzl_library( - name = "parse_requirements_bzl", - srcs = ["parse_requirements.bzl"], - deps = [ - ":normalize_name_bzl", - ":pypi_index_sources_bzl", - ":whl_target_platforms_bzl", - "//python/pip_install:requirements_parser_bzl", - ], -) - -bzl_library( - name = "parse_whl_name_bzl", - srcs = ["parse_whl_name.bzl"], -) - -bzl_library( - name = "pip_flags_bzl", - srcs = ["pip_flags.bzl"], - deps = [ - ":enum_bzl", - ], -) - -bzl_library( - name = "pip_repo_name_bzl", - srcs = ["pip_repo_name.bzl"], - deps = [ - ":normalize_name_bzl", - ":parse_whl_name_bzl", - ], -) - -bzl_library( - name = "pypi_index_bzl", - srcs = ["pypi_index.bzl"], - deps = [ - ":auth_bzl", - ":normalize_name_bzl", - ":text_util_bzl", - "//python/pip_install:requirements_parser_bzl", - "//python/private/bzlmod:bazel_features_bzl", - ], -) - -bzl_library( - name = "pypi_index_sources_bzl", - srcs = ["pypi_index_sources.bzl"], -) - bzl_library( name = "py_cc_toolchain_bzl", srcs = [ @@ -285,17 +230,6 @@ bzl_library( srcs = ["register_extension_info.bzl"], ) -bzl_library( - name = "render_pkg_aliases_bzl", - srcs = ["render_pkg_aliases.bzl"], - deps = [ - ":normalize_name_bzl", - ":text_util_bzl", - ":version_label_bzl", - "//python/pip_install/private:generate_group_library_build_bazel_bzl", - ], -) - bzl_library( name = "repo_utils_bzl", srcs = ["repo_utils.bzl"], @@ -340,20 +274,6 @@ bzl_library( srcs = ["version_label.bzl"], ) -bzl_library( - name = "whl_target_platforms_bzl", - srcs = ["whl_target_platforms.bzl"], - visibility = ["//:__subpackages__"], - deps = [ - "parse_whl_name_bzl", - ], -) - -bzl_library( - name = "labels_bzl", - srcs = ["labels.bzl"], -) - # @bazel_tools can't define bzl_library itself, so we just put a wrapper around it. bzl_library( name = "bazel_tools_bzl", diff --git a/python/private/bzlmod/BUILD.bazel b/python/private/bzlmod/BUILD.bazel index 3362f34ffd..2cb35fc03e 100644 --- a/python/private/bzlmod/BUILD.bazel +++ b/python/private/bzlmod/BUILD.bazel @@ -13,7 +13,6 @@ # limitations under the License. load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") package(default_visibility = ["//:__subpackages__"]) @@ -29,33 +28,7 @@ bzl_library( name = "pip_bzl", srcs = ["pip.bzl"], deps = [ - ":pip_repository_bzl", - "//python/pip_install:pip_repository_bzl", - "//python/private:pypi_index_bzl", - "//python/private:full_version_bzl", - "//python/private:normalize_name_bzl", - "//python/private:parse_requirements_bzl", - "//python/private:parse_whl_name_bzl", - "//python/private:pip_repo_name_bzl", - "//python/private:version_label_bzl", - ":bazel_features_bzl", - ] + [ - "@pythons_hub//:interpreters_bzl", - ] if BZLMOD_ENABLED else [], -) - -bzl_library( - name = "bazel_features_bzl", - deps = ["@bazel_features//:features"], -) - -bzl_library( - name = "pip_repository_bzl", - srcs = ["pip_repository.bzl"], - visibility = ["//:__subpackages__"], - deps = [ - "//python/private:render_pkg_aliases_bzl", - "//python/private:text_util_bzl", + "//python/private/pypi:bzlmod_bzl", ], ) @@ -66,6 +39,8 @@ bzl_library( ":pythons_hub_bzl", "//python:repositories_bzl", "//python/private:toolchains_repo_bzl", + "//python/private:util_bzl", + "@bazel_features//:features", ], ) diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl index 2ded3a4acd..ecf94b69d5 100644 --- a/python/private/bzlmod/pip.bzl +++ b/python/private/bzlmod/pip.bzl @@ -12,817 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"pip module extension for use with bzlmod" +"pip module extensions for use with bzlmod." -load("@bazel_features//:features.bzl", "bazel_features") -load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS") -load( - "//python/pip_install:pip_repository.bzl", - "pip_repository_attrs", - "use_isolated", - "whl_library", -) -load("//python/private:auth.bzl", "AUTH_ATTRS") -load("//python/private:normalize_name.bzl", "normalize_name") -load("//python/private:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") -load("//python/private:parse_whl_name.bzl", "parse_whl_name") -load("//python/private:pip_repo_name.bzl", "pip_repo_name") -load("//python/private:pypi_index.bzl", "simpleapi_download") -load("//python/private:render_pkg_aliases.bzl", "whl_alias") -load("//python/private:repo_utils.bzl", "repo_utils") -load("//python/private:version_label.bzl", "version_label") -load(":pip_repository.bzl", "pip_repository") +load("//python/private/pypi:bzlmod.bzl", "pypi", "pypi_internal") -def _parse_version(version): - major, _, version = version.partition(".") - minor, _, version = version.partition(".") - patch, _, version = version.partition(".") - build, _, version = version.partition(".") - - return struct( - # use semver vocabulary here - major = major, - minor = minor, - patch = patch, # this is called `micro` in the Python interpreter versioning scheme - build = build, - ) - -def _major_minor_version(version): - version = _parse_version(version) - return "{}.{}".format(version.major, version.minor) - -def _whl_mods_impl(mctx): - """Implementation of the pip.whl_mods tag class. - - This creates the JSON files used to modify the creation of different wheels. -""" - whl_mods_dict = {} - for mod in mctx.modules: - for whl_mod_attr in mod.tags.whl_mods: - if whl_mod_attr.hub_name not in whl_mods_dict.keys(): - whl_mods_dict[whl_mod_attr.hub_name] = {whl_mod_attr.whl_name: whl_mod_attr} - elif whl_mod_attr.whl_name in whl_mods_dict[whl_mod_attr.hub_name].keys(): - # We cannot have the same wheel name in the same hub, as we - # will create the same JSON file name. - fail("""\ -Found same whl_name '{}' in the same hub '{}', please use a different hub_name.""".format( - whl_mod_attr.whl_name, - whl_mod_attr.hub_name, - )) - else: - whl_mods_dict[whl_mod_attr.hub_name][whl_mod_attr.whl_name] = whl_mod_attr - - for hub_name, whl_maps in whl_mods_dict.items(): - whl_mods = {} - - # create a struct that we can pass to the _whl_mods_repo rule - # to create the different JSON files. - for whl_name, mods in whl_maps.items(): - build_content = mods.additive_build_content - if mods.additive_build_content_file != None and mods.additive_build_content != "": - fail("""\ -You cannot use both the additive_build_content and additive_build_content_file arguments at the same time. -""") - elif mods.additive_build_content_file != None: - build_content = mctx.read(mods.additive_build_content_file) - - whl_mods[whl_name] = json.encode(struct( - additive_build_content = build_content, - copy_files = mods.copy_files, - copy_executables = mods.copy_executables, - data = mods.data, - data_exclude_glob = mods.data_exclude_glob, - srcs_exclude_glob = mods.srcs_exclude_glob, - )) - - _whl_mods_repo( - name = hub_name, - whl_mods = whl_mods, - ) - -def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, simpleapi_cache): - logger = repo_utils.logger(module_ctx) - python_interpreter_target = pip_attr.python_interpreter_target - is_hub_reproducible = True - - # if we do not have the python_interpreter set in the attributes - # we programmatically find it. - hub_name = pip_attr.hub_name - if python_interpreter_target == None and not pip_attr.python_interpreter: - python_name = "python_{}_host".format( - pip_attr.python_version.replace(".", "_"), - ) - if python_name not in INTERPRETER_LABELS: - fail(( - "Unable to find interpreter for pip hub '{hub_name}' for " + - "python_version={version}: Make sure a corresponding " + - '`python.toolchain(python_version="{version}")` call exists.' + - "Expected to find {python_name} among registered versions:\n {labels}" - ).format( - hub_name = hub_name, - version = pip_attr.python_version, - python_name = python_name, - labels = " \n".join(INTERPRETER_LABELS), - )) - python_interpreter_target = INTERPRETER_LABELS[python_name] - - pip_name = "{}_{}".format( - hub_name, - version_label(pip_attr.python_version), - ) - major_minor = _major_minor_version(pip_attr.python_version) - - if hub_name not in whl_map: - whl_map[hub_name] = {} - - whl_modifications = {} - if pip_attr.whl_modifications != None: - for mod, whl_name in pip_attr.whl_modifications.items(): - whl_modifications[whl_name] = mod - - if pip_attr.experimental_requirement_cycles: - requirement_cycles = { - name: [normalize_name(whl_name) for whl_name in whls] - for name, whls in pip_attr.experimental_requirement_cycles.items() - } - - whl_group_mapping = { - whl_name: group_name - for group_name, group_whls in requirement_cycles.items() - for whl_name in group_whls - } - - # TODO @aignas 2024-04-05: how do we support different requirement - # cycles for different abis/oses? For now we will need the users to - # assume the same groups across all versions/platforms until we start - # using an alternative cycle resolution strategy. - group_map[hub_name] = pip_attr.experimental_requirement_cycles - else: - whl_group_mapping = {} - requirement_cycles = {} - - # Create a new wheel library for each of the different whls - - get_index_urls = None - if pip_attr.experimental_index_url: - if pip_attr.download_only: - fail("Currently unsupported to use `download_only` and `experimental_index_url`") - - get_index_urls = lambda ctx, distributions: simpleapi_download( - ctx, - attr = struct( - index_url = pip_attr.experimental_index_url, - extra_index_urls = pip_attr.experimental_extra_index_urls or [], - index_url_overrides = pip_attr.experimental_index_url_overrides or {}, - sources = distributions, - envsubst = pip_attr.envsubst, - # Auth related info - netrc = pip_attr.netrc, - auth_patterns = pip_attr.auth_patterns, - ), - cache = simpleapi_cache, - parallel_download = pip_attr.parallel_download, - ) - - requirements_by_platform = parse_requirements( - module_ctx, - requirements_by_platform = pip_attr.requirements_by_platform, - requirements_linux = pip_attr.requirements_linux, - requirements_lock = pip_attr.requirements_lock, - requirements_osx = pip_attr.requirements_darwin, - requirements_windows = pip_attr.requirements_windows, - extra_pip_args = pip_attr.extra_pip_args, - get_index_urls = get_index_urls, - python_version = major_minor, - logger = logger, - ) - - repository_platform = host_platform(module_ctx.os) - for whl_name, requirements in requirements_by_platform.items(): - # We are not using the "sanitized name" because the user - # would need to guess what name we modified the whl name - # to. - annotation = whl_modifications.get(whl_name) - whl_name = normalize_name(whl_name) - - group_name = whl_group_mapping.get(whl_name) - group_deps = requirement_cycles.get(group_name, []) - - # Construct args separately so that the lock file can be smaller and does not include unused - # attrs. - whl_library_args = dict( - repo = pip_name, - dep_template = "@{}//{{name}}:{{target}}".format(hub_name), - ) - maybe_args = dict( - # The following values are safe to omit if they have false like values - annotation = annotation, - download_only = pip_attr.download_only, - enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs, - environment = pip_attr.environment, - envsubst = pip_attr.envsubst, - experimental_target_platforms = pip_attr.experimental_target_platforms, - group_deps = group_deps, - group_name = group_name, - pip_data_exclude = pip_attr.pip_data_exclude, - python_interpreter = pip_attr.python_interpreter, - python_interpreter_target = python_interpreter_target, - whl_patches = { - p: json.encode(args) - for p, args in whl_overrides.get(whl_name, {}).items() - }, - ) - whl_library_args.update({k: v for k, v in maybe_args.items() if v}) - maybe_args_with_default = dict( - # The following values have defaults next to them - isolated = (use_isolated(module_ctx, pip_attr), True), - quiet = (pip_attr.quiet, True), - timeout = (pip_attr.timeout, 600), - ) - whl_library_args.update({ - k: v - for k, (v, default) in maybe_args_with_default.items() - if v != default - }) - - if get_index_urls: - # TODO @aignas 2024-05-26: move to a separate function - found_something = False - for requirement in requirements: - for distribution in requirement.whls + [requirement.sdist]: - if not distribution: - # sdist may be None - continue - - found_something = True - is_hub_reproducible = False - - if pip_attr.netrc: - whl_library_args["netrc"] = pip_attr.netrc - if pip_attr.auth_patterns: - whl_library_args["auth_patterns"] = pip_attr.auth_patterns - - # pip is not used to download wheels and the python `whl_library` helpers are only extracting things - whl_library_args.pop("extra_pip_args", None) - - # This is no-op because pip is not used to download the wheel. - whl_library_args.pop("download_only", None) - - repo_name = pip_repo_name(pip_name, distribution.filename, distribution.sha256) - whl_library_args["requirement"] = requirement.srcs.requirement - whl_library_args["urls"] = [distribution.url] - whl_library_args["sha256"] = distribution.sha256 - whl_library_args["filename"] = distribution.filename - whl_library_args["experimental_target_platforms"] = requirement.target_platforms - - # Pure python wheels or sdists may need to have a platform here - target_platforms = None - if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"): - if len(requirements) > 1: - target_platforms = requirement.target_platforms - - whl_library(name = repo_name, **dict(sorted(whl_library_args.items()))) - - whl_map[hub_name].setdefault(whl_name, []).append( - whl_alias( - repo = repo_name, - version = major_minor, - filename = distribution.filename, - target_platforms = target_platforms, - ), - ) - - if found_something: - continue - - requirement = select_requirement( - requirements, - platform = repository_platform, - ) - if not requirement: - # Sometimes the package is not present for host platform if there - # are whls specified only in particular requirements files, in that - # case just continue, however, if the download_only flag is set up, - # then the user can also specify the target platform of the wheel - # packages they want to download, in that case there will be always - # a requirement here, so we will not be in this code branch. - continue - elif get_index_urls: - logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line)) - - whl_library_args["requirement"] = requirement.requirement_line - if requirement.extra_pip_args: - whl_library_args["extra_pip_args"] = requirement.extra_pip_args - - # We sort so that the lock-file remains the same no matter the order of how the - # args are manipulated in the code going before. - repo_name = "{}_{}".format(pip_name, whl_name) - whl_library(name = repo_name, **dict(sorted(whl_library_args.items()))) - whl_map[hub_name].setdefault(whl_name, []).append( - whl_alias( - repo = repo_name, - version = major_minor, - ), - ) - - return is_hub_reproducible - -def _pip_impl(module_ctx): - """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories. - - This implementation iterates through all of the `pip.parse` calls and creates - different pip hub repositories based on the "hub_name". Each of the - pip calls create spoke repos that uses a specific Python interpreter. - - In a MODULES.bazel file we have: - - pip.parse( - hub_name = "pip", - python_version = 3.9, - requirements_lock = "//:requirements_lock_3_9.txt", - requirements_windows = "//:requirements_windows_3_9.txt", - ) - pip.parse( - hub_name = "pip", - python_version = 3.10, - requirements_lock = "//:requirements_lock_3_10.txt", - requirements_windows = "//:requirements_windows_3_10.txt", - ) - - For instance, we have a hub with the name of "pip". - A repository named the following is created. It is actually called last when - all of the pip spokes are collected. - - - @@rules_python~override~pip~pip - - As shown in the example code above we have the following. - Two different pip.parse statements exist in MODULE.bazel provide the hub_name "pip". - These definitions create two different pip spoke repositories that are - related to the hub "pip". - One spoke uses Python 3.9 and the other uses Python 3.10. This code automatically - determines the Python version and the interpreter. - Both of these pip spokes contain requirements files that includes websocket - and its dependencies. - - We also need repositories for the wheels that the different pip spokes contain. - For each Python version a different wheel repository is created. In our example - each pip spoke had a requirements file that contained websockets. We - then create two different wheel repositories that are named the following. - - - @@rules_python~override~pip~pip_39_websockets - - @@rules_python~override~pip~pip_310_websockets - - And if the wheel has any other dependencies subsequent wheels are created in the same fashion. - - The hub repository has aliases for `pkg`, `data`, etc, which have a select that resolves to - a spoke repository depending on the Python version. - - Also we may have more than one hub as defined in a MODULES.bazel file. So we could have multiple - hubs pointing to various different pip spokes. - - Some other business rules notes. A hub can only have one spoke per Python version. We cannot - have a hub named "pip" that has two spokes that use the Python 3.9 interpreter. Second - we cannot have the same hub name used in sub-modules. The hub name has to be globally - unique. - - This implementation also handles the creation of whl_modification JSON files that are used - during the creation of wheel libraries. These JSON files used via the annotations argument - when calling wheel_installer.py. - - Args: - module_ctx: module contents - """ - - # Build all of the wheel modifications if the tag class is called. - _whl_mods_impl(module_ctx) - - _overriden_whl_set = {} - whl_overrides = {} - - for module in module_ctx.modules: - for attr in module.tags.override: - if not module.is_root: - fail("overrides are only supported in root modules") - - if not attr.file.endswith(".whl"): - fail("Only whl overrides are supported at this time") - - whl_name = normalize_name(parse_whl_name(attr.file).distribution) - - if attr.file in _overriden_whl_set: - fail("Duplicate module overrides for '{}'".format(attr.file)) - _overriden_whl_set[attr.file] = None - - for patch in attr.patches: - if whl_name not in whl_overrides: - whl_overrides[whl_name] = {} - - if patch not in whl_overrides[whl_name]: - whl_overrides[whl_name][patch] = struct( - patch_strip = attr.patch_strip, - whls = [], - ) - - whl_overrides[whl_name][patch].whls.append(attr.file) - - # Used to track all the different pip hubs and the spoke pip Python - # versions. - pip_hub_map = {} - - # Keeps track of all the hub's whl repos across the different versions. - # dict[hub, dict[whl, dict[version, str pip]]] - # Where hub, whl, and pip are the repo names - hub_whl_map = {} - hub_group_map = {} - - simpleapi_cache = {} - is_extension_reproducible = True - - for mod in module_ctx.modules: - for pip_attr in mod.tags.parse: - hub_name = pip_attr.hub_name - if hub_name not in pip_hub_map: - pip_hub_map[pip_attr.hub_name] = struct( - module_name = mod.name, - python_versions = [pip_attr.python_version], - ) - elif pip_hub_map[hub_name].module_name != mod.name: - # We cannot have two hubs with the same name in different - # modules. - fail(( - "Duplicate cross-module pip hub named '{hub}': pip hub " + - "names must be unique across modules. First defined " + - "by module '{first_module}', second attempted by " + - "module '{second_module}'" - ).format( - hub = hub_name, - first_module = pip_hub_map[hub_name].module_name, - second_module = mod.name, - )) - - elif pip_attr.python_version in pip_hub_map[hub_name].python_versions: - fail(( - "Duplicate pip python version '{version}' for hub " + - "'{hub}' in module '{module}': the Python versions " + - "used for a hub must be unique" - ).format( - hub = hub_name, - module = mod.name, - version = pip_attr.python_version, - )) - else: - pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version) - - is_hub_reproducible = _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache) - is_extension_reproducible = is_extension_reproducible and is_hub_reproducible - - for hub_name, whl_map in hub_whl_map.items(): - pip_repository( - name = hub_name, - repo_name = hub_name, - whl_map = { - key: json.encode(value) - for key, value in whl_map.items() - }, - default_version = _major_minor_version(DEFAULT_PYTHON_VERSION), - groups = hub_group_map.get(hub_name), - ) - - if bazel_features.external_deps.extension_metadata_has_reproducible: - # If we are not using the `experimental_index_url feature, the extension is fully - # deterministic and we don't need to create a lock entry for it. - # - # In order to be able to dogfood the `experimental_index_url` feature before it gets - # stabilized, we have created the `_pip_non_reproducible` function, that will result - # in extra entries in the lock file. - return module_ctx.extension_metadata(reproducible = is_extension_reproducible) - else: - return None - -def _pip_non_reproducible(module_ctx): - _pip_impl(module_ctx) - - # We default to calling the PyPI index and that will go into the - # MODULE.bazel.lock file, hence return nothing here. - return None - -def _pip_parse_ext_attrs(**kwargs): - """Get the attributes for the pip extension. - - Args: - **kwargs: A kwarg for setting defaults for the specific attributes. The - key is expected to be the same as the attribute key. - - Returns: - A dict of attributes. - """ - attrs = dict({ - "experimental_extra_index_urls": attr.string_list( - doc = """\ -The extra index URLs to use for downloading wheels using bazel downloader. -Each value is going to be subject to `envsubst` substitutions if necessary. - -The indexes must support Simple API as described here: -https://packaging.python.org/en/latest/specifications/simple-repository-api/ - -This is equivalent to `--extra-index-urls` `pip` option. -""", - default = [], - ), - "experimental_index_url": attr.string( - default = kwargs.get("experimental_index_url", ""), - doc = """\ -The index URL to use for downloading wheels using bazel downloader. This value is going -to be subject to `envsubst` substitutions if necessary. - -The indexes must support Simple API as described here: -https://packaging.python.org/en/latest/specifications/simple-repository-api/ - -In the future this could be defaulted to `https://pypi.org` when this feature becomes -stable. - -This is equivalent to `--index-url` `pip` option. -""", - ), - "experimental_index_url_overrides": attr.string_dict( - doc = """\ -The index URL overrides for each package to use for downloading wheels using -bazel downloader. This value is going to be subject to `envsubst` substitutions -if necessary. - -The key is the package name (will be normalized before usage) and the value is the -index URL. - -This design pattern has been chosen in order to be fully deterministic about which -packages come from which source. We want to avoid issues similar to what happened in -https://pytorch.org/blog/compromised-nightly-dependency/. - -The indexes must support Simple API as described here: -https://packaging.python.org/en/latest/specifications/simple-repository-api/ -""", - ), - "hub_name": attr.string( - mandatory = True, - doc = """ -The name of the repo pip dependencies will be accessible from. - -This name must be unique between modules; unless your module is guaranteed to -always be the root module, it's highly recommended to include your module name -in the hub name. Repo mapping, `use_repo(..., pip="my_modules_pip_deps")`, can -be used for shorter local names within your module. - -Within a module, the same `hub_name` can be specified to group different Python -versions of pip dependencies under one repository name. This allows using a -Python version-agnostic name when referring to pip dependencies; the -correct version will be automatically selected. - -Typically, a module will only have a single hub of pip dependencies, but this -is not required. Each hub is a separate resolution of pip dependencies. This -means if different programs need different versions of some library, separate -hubs can be created, and each program can use its respective hub's targets. -Targets from different hubs should not be used together. -""", - ), - "parallel_download": attr.bool( - doc = """\ -The flag allows to make use of parallel downloading feature in bazel 7.1 and above -when the bazel downloader is used. This is by default enabled as it improves the -performance by a lot, but in case the queries to the simple API are very expensive -or when debugging authentication issues one may want to disable this feature. - -NOTE, This will download (potentially duplicate) data for multiple packages if -there is more than one index available, but in general this should be negligible -because the simple API calls are very cheap and the user should not notice any -extra overhead. - -If we are in synchronous mode, then we will use the first result that we -find in case extra indexes are specified. -""", - default = True, - ), - "python_version": attr.string( - mandatory = True, - doc = """ -The Python version the dependencies are targetting, in Major.Minor format -(e.g., "3.11") or patch level granularity (e.g. "3.11.1"). - -If an interpreter isn't explicitly provided (using `python_interpreter` or -`python_interpreter_target`), then the version specified here must have -a corresponding `python.toolchain()` configured. -""", - ), - "whl_modifications": attr.label_keyed_string_dict( - mandatory = False, - doc = """\ -A dict of labels to wheel names that is typically generated by the whl_modifications. -The labels are JSON config files describing the modifications. -""", - ), - }, **pip_repository_attrs) - attrs.update(AUTH_ATTRS) - - # Like the pip_repository rule, we end up setting this manually so - # don't allow users to override it. - attrs.pop("repo_prefix") - - # annotations has been replaced with whl_modifications in bzlmod - attrs.pop("annotations") - - return attrs - -def _whl_mod_attrs(): - attrs = { - "additive_build_content": attr.string( - doc = "(str, optional): Raw text to add to the generated `BUILD` file of a package.", - ), - "additive_build_content_file": attr.label( - doc = """\ -(label, optional): path to a BUILD file to add to the generated -`BUILD` file of a package. You cannot use both additive_build_content and additive_build_content_file -arguments at the same time.""", - ), - "copy_executables": attr.string_dict( - doc = """\ -(dict, optional): A mapping of `src` and `out` files for -[@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as -executable.""", - ), - "copy_files": attr.string_dict( - doc = """\ -(dict, optional): A mapping of `src` and `out` files for -[@bazel_skylib//rules:copy_file.bzl][cf]""", - ), - "data": attr.string_list( - doc = """\ -(list, optional): A list of labels to add as `data` dependencies to -the generated `py_library` target.""", - ), - "data_exclude_glob": attr.string_list( - doc = """\ -(list, optional): A list of exclude glob patterns to add as `data` to -the generated `py_library` target.""", - ), - "hub_name": attr.string( - doc = """\ -Name of the whl modification, hub we use this name to set the modifications for -pip.parse. If you have different pip hubs you can use a different name, -otherwise it is best practice to just use one. - -You cannot have the same `hub_name` in different modules. You can reuse the same -name in the same module for different wheels that you put in the same hub, but you -cannot have a child module that uses the same `hub_name`. -""", - mandatory = True, - ), - "srcs_exclude_glob": attr.string_list( - doc = """\ -(list, optional): A list of labels to add as `srcs` to the generated -`py_library` target.""", - ), - "whl_name": attr.string( - doc = "The whl name that the modifications are used for.", - mandatory = True, - ), - } - return attrs - -# NOTE: the naming of 'override' is taken from the bzlmod native -# 'archive_override', 'git_override' bzlmod functions. -_override_tag = tag_class( - attrs = { - "file": attr.string( - doc = """\ -The Python distribution file name which needs to be patched. This will be -applied to all repositories that setup this distribution via the pip.parse tag -class.""", - mandatory = True, - ), - "patch_strip": attr.int( - default = 0, - doc = """\ -The number of leading path segments to be stripped from the file name in the -patches.""", - ), - "patches": attr.label_list( - doc = """\ -A list of patches to apply to the repository *after* 'whl_library' is extracted -and BUILD.bazel file is generated.""", - mandatory = True, - ), - }, - doc = """\ -Apply any overrides (e.g. patches) to a given Python distribution defined by -other tags in this extension.""", -) - -pip = module_extension( - doc = """\ -This extension is used to make dependencies from pip available. - -pip.parse: -To use, call `pip.parse()` and specify `hub_name` and your requirements file. -Dependencies will be downloaded and made available in a repo named after the -`hub_name` argument. - -Each `pip.parse()` call configures a particular Python version. Multiple calls -can be made to configure different Python versions, and will be grouped by -the `hub_name` argument. This allows the same logical name, e.g. `@pip//numpy` -to automatically resolve to different, Python version-specific, libraries. - -pip.whl_mods: -This tag class is used to help create JSON files to describe modifications to -the BUILD files for wheels. -""", - implementation = _pip_impl, - tag_classes = { - "override": _override_tag, - "parse": tag_class( - attrs = _pip_parse_ext_attrs(), - doc = """\ -This tag class is used to create a pip hub and all of the spokes that are part of that hub. -This tag class reuses most of the pip attributes that are found in -@rules_python//python/pip_install:pip_repository.bzl. -The exception is it does not use the arg 'repo_prefix'. We set the repository -prefix for the user and the alias arg is always True in bzlmod. -""", - ), - "whl_mods": tag_class( - attrs = _whl_mod_attrs(), - doc = """\ -This tag class is used to create JSON file that are used when calling wheel_builder.py. These -JSON files contain instructions on how to modify a wheel's project. Each of the attributes -create different modifications based on the type of attribute. Previously to bzlmod these -JSON files where referred to as annotations, and were renamed to whl_modifications in this -extension. -""", - ), - }, -) - -pip_internal = module_extension( - doc = """\ -This extension is used to make dependencies from pypi available. - -For now this is intended to be used internally so that usage of the `pip` -extension in `rules_python` does not affect the evaluations of the extension -for the consumers. - -pip.parse: -To use, call `pip.parse()` and specify `hub_name` and your requirements file. -Dependencies will be downloaded and made available in a repo named after the -`hub_name` argument. - -Each `pip.parse()` call configures a particular Python version. Multiple calls -can be made to configure different Python versions, and will be grouped by -the `hub_name` argument. This allows the same logical name, e.g. `@pypi//numpy` -to automatically resolve to different, Python version-specific, libraries. - -pip.whl_mods: -This tag class is used to help create JSON files to describe modifications to -the BUILD files for wheels. -""", - implementation = _pip_non_reproducible, - tag_classes = { - "override": _override_tag, - "parse": tag_class( - attrs = _pip_parse_ext_attrs( - experimental_index_url = "https://pypi.org/simple", - ), - doc = """\ -This tag class is used to create a pypi hub and all of the spokes that are part of that hub. -This tag class reuses most of the pypi attributes that are found in -@rules_python//python/pip_install:pip_repository.bzl. -The exception is it does not use the arg 'repo_prefix'. We set the repository -prefix for the user and the alias arg is always True in bzlmod. -""", - ), - "whl_mods": tag_class( - attrs = _whl_mod_attrs(), - doc = """\ -This tag class is used to create JSON file that are used when calling wheel_builder.py. These -JSON files contain instructions on how to modify a wheel's project. Each of the attributes -create different modifications based on the type of attribute. Previously to bzlmod these -JSON files where referred to as annotations, and were renamed to whl_modifications in this -extension. -""", - ), - }, -) - -def _whl_mods_repo_impl(rctx): - rctx.file("BUILD.bazel", "") - for whl_name, mods in rctx.attr.whl_mods.items(): - rctx.file("{}.json".format(whl_name), mods) - -_whl_mods_repo = repository_rule( - doc = """\ -This rule creates json files based on the whl_mods attribute. -""", - implementation = _whl_mods_repo_impl, - attrs = { - "whl_mods": attr.string_dict( - mandatory = True, - doc = "JSON endcoded string that is provided to wheel_builder.py", - ), - }, -) +pip = pypi +pip_internal = pypi_internal diff --git a/python/private/normalize_platform.bzl b/python/private/normalize_platform.bzl deleted file mode 100644 index 633062f399..0000000000 --- a/python/private/normalize_platform.bzl +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel new file mode 100644 index 0000000000..1530837f7d --- /dev/null +++ b/python/private/pypi/BUILD.bazel @@ -0,0 +1,238 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") + +package(default_visibility = ["//:__subpackages__"]) + +licenses(["notice"]) + +filegroup( + name = "distribution", + srcs = glob(["**"]), + visibility = ["//python/private:__pkg__"], +) + +# Filegroup of bzl files that can be used by downstream rules for documentation generation +filegroup( + name = "bzl", + srcs = glob(["**/*.bzl"]), + visibility = ["//python/private:__pkg__"], +) + +# Keep sorted by library name and keep the files named by the main symbol they export + +bzl_library( + name = "attrs_bzl", + srcs = ["attrs.bzl"], +) + +bzl_library( + name = "bzlmod_bzl", + srcs = ["bzlmod.bzl"], + deps = [ + ":attrs_bzl", + ":hub_repository_bzl", + ":parse_requirements_bzl", + ":parse_whl_name_bzl", + ":pip_repository_attrs_bzl", + ":simpleapi_download_bzl", + ":whl_library_bzl", + ":whl_repo_name_bzl", + "//python/private:full_version_bzl", + "//python/private:normalize_name_bzl", + "//python/private:version_label_bzl", + "@bazel_features//:features", + ] + [ + "@pythons_hub//:interpreters_bzl", + ] if BZLMOD_ENABLED else [], +) + +bzl_library( + name = "config_settings_bzl", + srcs = ["config_settings.bzl"], + deps = ["flags_bzl"], +) + +bzl_library( + name = "flags_bzl", + srcs = ["flags.bzl"], + deps = ["//python/private:enum_bzl"], +) + +bzl_library( + name = "generate_whl_library_build_bazel_bzl", + srcs = ["generate_whl_library_build_bazel.bzl"], + deps = [ + ":labels_bzl", + "//python/private:normalize_name_bzl", + ], +) + +bzl_library( + name = "generate_group_library_build_bazel_bzl", + srcs = ["generate_group_library_build_bazel.bzl"], + deps = [ + ":labels_bzl", + "//python/private:normalize_name_bzl", + ], +) + +bzl_library( + name = "group_library_bzl", + srcs = ["group_library.bzl"], + deps = [ + ":generate_group_library_build_bazel_bzl", + ], +) + +bzl_library( + name = "hub_repository_bzl", + srcs = ["hub_repository.bzl"], + visibility = ["//:__subpackages__"], + deps = [ + ":render_pkg_aliases_bzl", + "//python/private:text_util_bzl", + ], +) + +bzl_library( + name = "index_sources_bzl", + srcs = ["index_sources.bzl"], +) + +bzl_library( + name = "labels_bzl", + srcs = ["labels.bzl"], +) + +bzl_library( + name = "package_annotation_bzl", + srcs = ["package_annotation.bzl"], +) + +bzl_library( + name = "parse_requirements_bzl", + srcs = ["parse_requirements.bzl"], + deps = [ + ":index_sources_bzl", + ":parse_requirements_txt_bzl", + ":whl_target_platforms_bzl", + "//python/private:normalize_name_bzl", + ], +) + +bzl_library( + name = "parse_requirements_txt_bzl", + srcs = ["parse_requirements_txt.bzl"], +) + +bzl_library( + name = "parse_simpleapi_html_bzl", + srcs = ["parse_simpleapi_html.bzl"], +) + +bzl_library( + name = "parse_whl_name_bzl", + srcs = ["parse_whl_name.bzl"], +) + +bzl_library( + name = "patch_whl_bzl", + srcs = ["patch_whl.bzl"], + deps = [ + ":parse_whl_name_bzl", + "//python/private:repo_utils_bzl", + ], +) + +bzl_library( + name = "pip_repository_bzl", + srcs = ["pip_repository.bzl"], + deps = [ + ":attrs_bzl", + ":parse_requirements_bzl", + ":pip_repository_attrs_bzl", + ":render_pkg_aliases_bzl", + "//python/private:normalize_name_bzl", + "//python/private:repo_utils_bzl", + "//python/private:text_util_bzl", + "@bazel_skylib//lib:sets", + ], +) + +bzl_library( + name = "pip_repository_attrs_bzl", + srcs = ["pip_repository_attrs.bzl"], +) + +bzl_library( + name = "render_pkg_aliases_bzl", + srcs = ["render_pkg_aliases.bzl"], + deps = [ + ":generate_group_library_build_bazel_bzl", + ":labels_bzl", + ":parse_whl_name_bzl", + ":whl_target_platforms_bzl", + "//python/private:normalize_name_bzl", + "//python/private:text_util_bzl", + ], +) + +bzl_library( + name = "simpleapi_download_bzl", + srcs = ["simpleapi_download.bzl"], + deps = [ + ":parse_simpleapi_html_bzl", + "//python/private:auth_bzl", + "//python/private:normalize_name_bzl", + "//python/private:text_util_bzl", + "@bazel_features//:features", + ], +) + +bzl_library( + name = "whl_library_bzl", + srcs = ["whl_library.bzl"], + deps = [ + ":attrs_bzl", + ":generate_whl_library_build_bazel_bzl", + ":parse_whl_name_bzl", + ":patch_whl_bzl", + ":whl_target_platforms_bzl", + "//python:repositories_bzl", + "//python:versions_bzl", + "//python/pip_install:repositories_bzl", + "//python/private:auth_bzl", + "//python/private:envsubst_bzl", + "//python/private:repo_utils_bzl", + "//python/private:toolchains_repo_bzl", + ], +) + +bzl_library( + name = "whl_repo_name_bzl", + srcs = ["whl_repo_name.bzl"], + deps = [ + ":parse_whl_name_bzl", + "//python/private:normalize_name_bzl", + ], +) + +bzl_library( + name = "whl_target_platforms_bzl", + srcs = ["whl_target_platforms.bzl"], + deps = [":parse_whl_name_bzl"], +) diff --git a/python/private/pypi/README.md b/python/private/pypi/README.md new file mode 100644 index 0000000000..6be5703912 --- /dev/null +++ b/python/private/pypi/README.md @@ -0,0 +1,9 @@ +# PyPI integration code + +This code is for integrating with PyPI and other compatible indexes. At the +moment we have code for: +* Downloading packages using `pip` or `repository_ctx.download`. +* Interacting with PyPI compatible indexes via [SimpleAPI] spec. +* Locking a `requirements.in` or [PEP621] compliant `pyproject.toml`. + +[PEP621]: https://peps.python.org/pep-0621/ diff --git a/python/private/pypi/attrs.bzl b/python/private/pypi/attrs.bzl new file mode 100644 index 0000000000..79ffea54a1 --- /dev/null +++ b/python/private/pypi/attrs.bzl @@ -0,0 +1,224 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"common attributes for whl_library and pip_repository" + +ATTRS = { + "download_only": attr.bool( + doc = """ +Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of +--platform, --python-version, --implementation, and --abi in --extra_pip_args to download wheels for a different +platform from the host platform. + """, + ), + "enable_implicit_namespace_pkgs": attr.bool( + default = False, + doc = """ +If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary +and py_test targets must specify either `legacy_create_init=False` or the global Bazel option +`--incompatible_default_to_explicit_init_py` to prevent `__init__.py` being automatically generated in every directory. + +This option is required to support some packages which cannot handle the conversion to pkg-util style. + """, + ), + "environment": attr.string_dict( + doc = """ +Environment variables to set in the pip subprocess. +Can be used to set common variables such as `http_proxy`, `https_proxy` and `no_proxy` +Note that pip is run with "--isolated" on the CLI so `PIP__` +style env vars are ignored, but env vars that control requests and urllib3 +can be passed. If you need `PIP__`, take a look at `extra_pip_args` +and `envsubst`. + """, + default = {}, + ), + "envsubst": attr.string_list( + mandatory = False, + doc = """\ +A list of environment variables to substitute (e.g. `["PIP_INDEX_URL", +"PIP_RETRIES"]`). The corresponding variables are expanded in `extra_pip_args` +using the syntax `$VARNAME` or `${VARNAME}` (expanding to empty string if unset) +or `${VARNAME:-default}` (expanding to default if the variable is unset or empty +in the environment). Note: On Bazel 6 and Bazel 7.0 changes to the variables named +here do not cause packages to be re-fetched. Don't fetch different things based +on the value of these variables. +""", + ), + "experimental_requirement_cycles": attr.string_list_dict( + default = {}, + doc = """\ +A mapping of dependency cycle names to a list of requirements which form that cycle. + +Requirements which form cycles will be installed together and taken as +dependencies together in order to ensure that the cycle is always satisified. + +Example: + `sphinx` depends on `sphinxcontrib-serializinghtml` + When listing both as requirements, ala + + ``` + py_binary( + name = "doctool", + ... + deps = [ + "@pypi//sphinx:pkg", + "@pypi//sphinxcontrib_serializinghtml", + ] + ) + ``` + + Will produce a Bazel error such as + + ``` + ERROR: .../external/pypi_sphinxcontrib_serializinghtml/BUILD.bazel:44:6: in alias rule @pypi_sphinxcontrib_serializinghtml//:pkg: cycle in dependency graph: + //:doctool (...) + @pypi//sphinxcontrib_serializinghtml:pkg (...) + .-> @pypi_sphinxcontrib_serializinghtml//:pkg (...) + | @pypi_sphinxcontrib_serializinghtml//:_pkg (...) + | @pypi_sphinx//:pkg (...) + | @pypi_sphinx//:_pkg (...) + `-- @pypi_sphinxcontrib_serializinghtml//:pkg (...) + ``` + + Which we can resolve by configuring these two requirements to be installed together as a cycle + + ``` + pip_parse( + ... + experimental_requirement_cycles = { + "sphinx": [ + "sphinx", + "sphinxcontrib-serializinghtml", + ] + }, + ) + ``` + +Warning: + If a dependency participates in multiple cycles, all of those cycles must be + collapsed down to one. For instance `a <-> b` and `a <-> c` cannot be listed + as two separate cycles. +""", + ), + "experimental_target_platforms": attr.string_list( + default = [], + doc = """\ +A list of platforms that we will generate the conditional dependency graph for +cross platform wheels by parsing the wheel metadata. This will generate the +correct dependencies for packages like `sphinx` or `pylint`, which include +`colorama` when installed and used on Windows platforms. + +An empty list means falling back to the legacy behaviour where the host +platform is the target platform. + +WARNING: It may not work as expected in cases where the python interpreter +implementation that is being used at runtime is different between different platforms. +This has been tested for CPython only. + +For specific target platforms use values of the form `_` where `` +is one of `linux`, `osx`, `windows` and arch is one of `x86_64`, `x86_32`, +`aarch64`, `s390x` and `ppc64le`. + +You can also target a specific Python version by using `cp3__`. +If multiple python versions are specified as target platforms, then select statements +of the `lib` and `whl` targets will include usage of version aware toolchain config +settings like `@rules_python//python/config_settings:is_python_3.y`. + +Special values: `host` (for generating deps for the host platform only) and +`_*` values. For example, `cp39_*`, `linux_*`, `cp39_linux_*`. + +NOTE: this is not for cross-compiling Python wheels but rather for parsing the `whl` METADATA correctly. +""", + ), + "extra_pip_args": attr.string_list( + doc = """Extra arguments to pass on to pip. Must not contain spaces. + +Supports environment variables using the syntax `$VARNAME` or +`${VARNAME}` (expanding to empty string if unset) or +`${VARNAME:-default}` (expanding to default if the variable is unset +or empty in the environment), if `"VARNAME"` is listed in the +`envsubst` attribute. See also `envsubst`. +""", + ), + "isolated": attr.bool( + doc = """\ +Whether or not to pass the [--isolated](https://pip.pypa.io/en/stable/cli/pip/#cmdoption-isolated) flag to +the underlying pip command. Alternatively, the `RULES_PYTHON_PIP_ISOLATED` environment variable can be used +to control this flag. +""", + default = True, + ), + "pip_data_exclude": attr.string_list( + doc = "Additional data exclusion parameters to add to the pip packages BUILD file.", + ), + "python_interpreter": attr.string( + doc = """\ +The python interpreter to use. This can either be an absolute path or the name +of a binary found on the host's `PATH` environment variable. If no value is set +`python3` is defaulted for Unix systems and `python.exe` for Windows. +""", + # NOTE: This attribute should not have a default. See `_get_python_interpreter_attr` + # default = "python3" + ), + "python_interpreter_target": attr.label( + allow_single_file = True, + doc = """ +If you are using a custom python interpreter built by another repository rule, +use this attribute to specify its BUILD target. This allows pip_repository to invoke +pip using the same interpreter as your toolchain. If set, takes precedence over +python_interpreter. An example value: "@python3_x86_64-unknown-linux-gnu//:python". +""", + ), + "quiet": attr.bool( + default = True, + doc = """\ +If True, suppress printing stdout and stderr output to the terminal. + +If you would like to get more diagnostic output, please use: + + RULES_PYTHON_REPO_DEBUG=1 + +or + + RULES_PYTHON_REPO_DEBUG_VERBOSITY= +""", + ), + # 600 is documented as default here: https://docs.bazel.build/versions/master/skylark/lib/repository_ctx.html#execute + "timeout": attr.int( + default = 600, + doc = "Timeout (in seconds) on the rule's execution duration.", + ), +} + +def use_isolated(ctx, attr): + """Determine whether or not to pass the pip `--isolated` flag to the pip invocation. + + Args: + ctx: repository or module context + attr: attributes for the repo rule or tag extension + + Returns: + True if --isolated should be passed + """ + use_isolated = attr.isolated + + # The environment variable will take precedence over the attribute + isolated_env = ctx.os.environ.get("RULES_PYTHON_PIP_ISOLATED", None) + if isolated_env != None: + if isolated_env.lower() in ("0", "false"): + use_isolated = False + else: + use_isolated = True + + return use_isolated diff --git a/python/private/pypi/bzlmod.bzl b/python/private/pypi/bzlmod.bzl new file mode 100644 index 0000000000..e98208a2a6 --- /dev/null +++ b/python/private/pypi/bzlmod.bzl @@ -0,0 +1,818 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"pip module extension for use with bzlmod" + +load("@bazel_features//:features.bzl", "bazel_features") +load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS") +load("//python/private:auth.bzl", "AUTH_ATTRS") +load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:repo_utils.bzl", "repo_utils") +load("//python/private:version_label.bzl", "version_label") +load(":attrs.bzl", "use_isolated") +load(":hub_repository.bzl", "hub_repository") +load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") +load(":parse_whl_name.bzl", "parse_whl_name") +load(":pip_repository_attrs.bzl", "ATTRS") +load(":render_pkg_aliases.bzl", "whl_alias") +load(":simpleapi_download.bzl", "simpleapi_download") +load(":whl_library.bzl", "whl_library") +load(":whl_repo_name.bzl", "whl_repo_name") + +def _parse_version(version): + major, _, version = version.partition(".") + minor, _, version = version.partition(".") + patch, _, version = version.partition(".") + build, _, version = version.partition(".") + + return struct( + # use semver vocabulary here + major = major, + minor = minor, + patch = patch, # this is called `micro` in the Python interpreter versioning scheme + build = build, + ) + +def _major_minor_version(version): + version = _parse_version(version) + return "{}.{}".format(version.major, version.minor) + +def _whl_mods_impl(mctx): + """Implementation of the pip.whl_mods tag class. + + This creates the JSON files used to modify the creation of different wheels. +""" + whl_mods_dict = {} + for mod in mctx.modules: + for whl_mod_attr in mod.tags.whl_mods: + if whl_mod_attr.hub_name not in whl_mods_dict.keys(): + whl_mods_dict[whl_mod_attr.hub_name] = {whl_mod_attr.whl_name: whl_mod_attr} + elif whl_mod_attr.whl_name in whl_mods_dict[whl_mod_attr.hub_name].keys(): + # We cannot have the same wheel name in the same hub, as we + # will create the same JSON file name. + fail("""\ +Found same whl_name '{}' in the same hub '{}', please use a different hub_name.""".format( + whl_mod_attr.whl_name, + whl_mod_attr.hub_name, + )) + else: + whl_mods_dict[whl_mod_attr.hub_name][whl_mod_attr.whl_name] = whl_mod_attr + + for hub_name, whl_maps in whl_mods_dict.items(): + whl_mods = {} + + # create a struct that we can pass to the _whl_mods_repo rule + # to create the different JSON files. + for whl_name, mods in whl_maps.items(): + build_content = mods.additive_build_content + if mods.additive_build_content_file != None and mods.additive_build_content != "": + fail("""\ +You cannot use both the additive_build_content and additive_build_content_file arguments at the same time. +""") + elif mods.additive_build_content_file != None: + build_content = mctx.read(mods.additive_build_content_file) + + whl_mods[whl_name] = json.encode(struct( + additive_build_content = build_content, + copy_files = mods.copy_files, + copy_executables = mods.copy_executables, + data = mods.data, + data_exclude_glob = mods.data_exclude_glob, + srcs_exclude_glob = mods.srcs_exclude_glob, + )) + + _whl_mods_repo( + name = hub_name, + whl_mods = whl_mods, + ) + +def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, simpleapi_cache): + logger = repo_utils.logger(module_ctx) + python_interpreter_target = pip_attr.python_interpreter_target + is_hub_reproducible = True + + # if we do not have the python_interpreter set in the attributes + # we programmatically find it. + hub_name = pip_attr.hub_name + if python_interpreter_target == None and not pip_attr.python_interpreter: + python_name = "python_{}_host".format( + pip_attr.python_version.replace(".", "_"), + ) + if python_name not in INTERPRETER_LABELS: + fail(( + "Unable to find interpreter for pip hub '{hub_name}' for " + + "python_version={version}: Make sure a corresponding " + + '`python.toolchain(python_version="{version}")` call exists.' + + "Expected to find {python_name} among registered versions:\n {labels}" + ).format( + hub_name = hub_name, + version = pip_attr.python_version, + python_name = python_name, + labels = " \n".join(INTERPRETER_LABELS), + )) + python_interpreter_target = INTERPRETER_LABELS[python_name] + + pip_name = "{}_{}".format( + hub_name, + version_label(pip_attr.python_version), + ) + major_minor = _major_minor_version(pip_attr.python_version) + + if hub_name not in whl_map: + whl_map[hub_name] = {} + + whl_modifications = {} + if pip_attr.whl_modifications != None: + for mod, whl_name in pip_attr.whl_modifications.items(): + whl_modifications[whl_name] = mod + + if pip_attr.experimental_requirement_cycles: + requirement_cycles = { + name: [normalize_name(whl_name) for whl_name in whls] + for name, whls in pip_attr.experimental_requirement_cycles.items() + } + + whl_group_mapping = { + whl_name: group_name + for group_name, group_whls in requirement_cycles.items() + for whl_name in group_whls + } + + # TODO @aignas 2024-04-05: how do we support different requirement + # cycles for different abis/oses? For now we will need the users to + # assume the same groups across all versions/platforms until we start + # using an alternative cycle resolution strategy. + group_map[hub_name] = pip_attr.experimental_requirement_cycles + else: + whl_group_mapping = {} + requirement_cycles = {} + + # Create a new wheel library for each of the different whls + + get_index_urls = None + if pip_attr.experimental_index_url: + if pip_attr.download_only: + fail("Currently unsupported to use `download_only` and `experimental_index_url`") + + get_index_urls = lambda ctx, distributions: simpleapi_download( + ctx, + attr = struct( + index_url = pip_attr.experimental_index_url, + extra_index_urls = pip_attr.experimental_extra_index_urls or [], + index_url_overrides = pip_attr.experimental_index_url_overrides or {}, + sources = distributions, + envsubst = pip_attr.envsubst, + # Auth related info + netrc = pip_attr.netrc, + auth_patterns = pip_attr.auth_patterns, + ), + cache = simpleapi_cache, + parallel_download = pip_attr.parallel_download, + ) + + requirements_by_platform = parse_requirements( + module_ctx, + requirements_by_platform = pip_attr.requirements_by_platform, + requirements_linux = pip_attr.requirements_linux, + requirements_lock = pip_attr.requirements_lock, + requirements_osx = pip_attr.requirements_darwin, + requirements_windows = pip_attr.requirements_windows, + extra_pip_args = pip_attr.extra_pip_args, + get_index_urls = get_index_urls, + python_version = major_minor, + logger = logger, + ) + + repository_platform = host_platform(module_ctx.os) + for whl_name, requirements in requirements_by_platform.items(): + # We are not using the "sanitized name" because the user + # would need to guess what name we modified the whl name + # to. + annotation = whl_modifications.get(whl_name) + whl_name = normalize_name(whl_name) + + group_name = whl_group_mapping.get(whl_name) + group_deps = requirement_cycles.get(group_name, []) + + # Construct args separately so that the lock file can be smaller and does not include unused + # attrs. + whl_library_args = dict( + repo = pip_name, + dep_template = "@{}//{{name}}:{{target}}".format(hub_name), + ) + maybe_args = dict( + # The following values are safe to omit if they have false like values + annotation = annotation, + download_only = pip_attr.download_only, + enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs, + environment = pip_attr.environment, + envsubst = pip_attr.envsubst, + experimental_target_platforms = pip_attr.experimental_target_platforms, + group_deps = group_deps, + group_name = group_name, + pip_data_exclude = pip_attr.pip_data_exclude, + python_interpreter = pip_attr.python_interpreter, + python_interpreter_target = python_interpreter_target, + whl_patches = { + p: json.encode(args) + for p, args in whl_overrides.get(whl_name, {}).items() + }, + ) + whl_library_args.update({k: v for k, v in maybe_args.items() if v}) + maybe_args_with_default = dict( + # The following values have defaults next to them + isolated = (use_isolated(module_ctx, pip_attr), True), + quiet = (pip_attr.quiet, True), + timeout = (pip_attr.timeout, 600), + ) + whl_library_args.update({ + k: v + for k, (v, default) in maybe_args_with_default.items() + if v != default + }) + + if get_index_urls: + # TODO @aignas 2024-05-26: move to a separate function + found_something = False + for requirement in requirements: + for distribution in requirement.whls + [requirement.sdist]: + if not distribution: + # sdist may be None + continue + + found_something = True + is_hub_reproducible = False + + if pip_attr.netrc: + whl_library_args["netrc"] = pip_attr.netrc + if pip_attr.auth_patterns: + whl_library_args["auth_patterns"] = pip_attr.auth_patterns + + # pip is not used to download wheels and the python `whl_library` helpers are only extracting things + whl_library_args.pop("extra_pip_args", None) + + # This is no-op because pip is not used to download the wheel. + whl_library_args.pop("download_only", None) + + repo_name = whl_repo_name(pip_name, distribution.filename, distribution.sha256) + whl_library_args["requirement"] = requirement.srcs.requirement + whl_library_args["urls"] = [distribution.url] + whl_library_args["sha256"] = distribution.sha256 + whl_library_args["filename"] = distribution.filename + whl_library_args["experimental_target_platforms"] = requirement.target_platforms + + # Pure python wheels or sdists may need to have a platform here + target_platforms = None + if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"): + if len(requirements) > 1: + target_platforms = requirement.target_platforms + + whl_library(name = repo_name, **dict(sorted(whl_library_args.items()))) + + whl_map[hub_name].setdefault(whl_name, []).append( + whl_alias( + repo = repo_name, + version = major_minor, + filename = distribution.filename, + target_platforms = target_platforms, + ), + ) + + if found_something: + continue + + requirement = select_requirement( + requirements, + platform = repository_platform, + ) + if not requirement: + # Sometimes the package is not present for host platform if there + # are whls specified only in particular requirements files, in that + # case just continue, however, if the download_only flag is set up, + # then the user can also specify the target platform of the wheel + # packages they want to download, in that case there will be always + # a requirement here, so we will not be in this code branch. + continue + elif get_index_urls: + logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line)) + + whl_library_args["requirement"] = requirement.requirement_line + if requirement.extra_pip_args: + whl_library_args["extra_pip_args"] = requirement.extra_pip_args + + # We sort so that the lock-file remains the same no matter the order of how the + # args are manipulated in the code going before. + repo_name = "{}_{}".format(pip_name, whl_name) + whl_library(name = repo_name, **dict(sorted(whl_library_args.items()))) + whl_map[hub_name].setdefault(whl_name, []).append( + whl_alias( + repo = repo_name, + version = major_minor, + ), + ) + + return is_hub_reproducible + +def _pip_impl(module_ctx): + """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories. + + This implementation iterates through all of the `pip.parse` calls and creates + different pip hub repositories based on the "hub_name". Each of the + pip calls create spoke repos that uses a specific Python interpreter. + + In a MODULES.bazel file we have: + + pip.parse( + hub_name = "pip", + python_version = 3.9, + requirements_lock = "//:requirements_lock_3_9.txt", + requirements_windows = "//:requirements_windows_3_9.txt", + ) + pip.parse( + hub_name = "pip", + python_version = 3.10, + requirements_lock = "//:requirements_lock_3_10.txt", + requirements_windows = "//:requirements_windows_3_10.txt", + ) + + For instance, we have a hub with the name of "pip". + A repository named the following is created. It is actually called last when + all of the pip spokes are collected. + + - @@rules_python~override~pip~pip + + As shown in the example code above we have the following. + Two different pip.parse statements exist in MODULE.bazel provide the hub_name "pip". + These definitions create two different pip spoke repositories that are + related to the hub "pip". + One spoke uses Python 3.9 and the other uses Python 3.10. This code automatically + determines the Python version and the interpreter. + Both of these pip spokes contain requirements files that includes websocket + and its dependencies. + + We also need repositories for the wheels that the different pip spokes contain. + For each Python version a different wheel repository is created. In our example + each pip spoke had a requirements file that contained websockets. We + then create two different wheel repositories that are named the following. + + - @@rules_python~override~pip~pip_39_websockets + - @@rules_python~override~pip~pip_310_websockets + + And if the wheel has any other dependencies subsequent wheels are created in the same fashion. + + The hub repository has aliases for `pkg`, `data`, etc, which have a select that resolves to + a spoke repository depending on the Python version. + + Also we may have more than one hub as defined in a MODULES.bazel file. So we could have multiple + hubs pointing to various different pip spokes. + + Some other business rules notes. A hub can only have one spoke per Python version. We cannot + have a hub named "pip" that has two spokes that use the Python 3.9 interpreter. Second + we cannot have the same hub name used in sub-modules. The hub name has to be globally + unique. + + This implementation also handles the creation of whl_modification JSON files that are used + during the creation of wheel libraries. These JSON files used via the annotations argument + when calling wheel_installer.py. + + Args: + module_ctx: module contents + """ + + # Build all of the wheel modifications if the tag class is called. + _whl_mods_impl(module_ctx) + + _overriden_whl_set = {} + whl_overrides = {} + + for module in module_ctx.modules: + for attr in module.tags.override: + if not module.is_root: + fail("overrides are only supported in root modules") + + if not attr.file.endswith(".whl"): + fail("Only whl overrides are supported at this time") + + whl_name = normalize_name(parse_whl_name(attr.file).distribution) + + if attr.file in _overriden_whl_set: + fail("Duplicate module overrides for '{}'".format(attr.file)) + _overriden_whl_set[attr.file] = None + + for patch in attr.patches: + if whl_name not in whl_overrides: + whl_overrides[whl_name] = {} + + if patch not in whl_overrides[whl_name]: + whl_overrides[whl_name][patch] = struct( + patch_strip = attr.patch_strip, + whls = [], + ) + + whl_overrides[whl_name][patch].whls.append(attr.file) + + # Used to track all the different pip hubs and the spoke pip Python + # versions. + pip_hub_map = {} + + # Keeps track of all the hub's whl repos across the different versions. + # dict[hub, dict[whl, dict[version, str pip]]] + # Where hub, whl, and pip are the repo names + hub_whl_map = {} + hub_group_map = {} + + simpleapi_cache = {} + is_extension_reproducible = True + + for mod in module_ctx.modules: + for pip_attr in mod.tags.parse: + hub_name = pip_attr.hub_name + if hub_name not in pip_hub_map: + pip_hub_map[pip_attr.hub_name] = struct( + module_name = mod.name, + python_versions = [pip_attr.python_version], + ) + elif pip_hub_map[hub_name].module_name != mod.name: + # We cannot have two hubs with the same name in different + # modules. + fail(( + "Duplicate cross-module pip hub named '{hub}': pip hub " + + "names must be unique across modules. First defined " + + "by module '{first_module}', second attempted by " + + "module '{second_module}'" + ).format( + hub = hub_name, + first_module = pip_hub_map[hub_name].module_name, + second_module = mod.name, + )) + + elif pip_attr.python_version in pip_hub_map[hub_name].python_versions: + fail(( + "Duplicate pip python version '{version}' for hub " + + "'{hub}' in module '{module}': the Python versions " + + "used for a hub must be unique" + ).format( + hub = hub_name, + module = mod.name, + version = pip_attr.python_version, + )) + else: + pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version) + + is_hub_reproducible = _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache) + is_extension_reproducible = is_extension_reproducible and is_hub_reproducible + + for hub_name, whl_map in hub_whl_map.items(): + hub_repository( + name = hub_name, + repo_name = hub_name, + whl_map = { + key: json.encode(value) + for key, value in whl_map.items() + }, + default_version = _major_minor_version(DEFAULT_PYTHON_VERSION), + groups = hub_group_map.get(hub_name), + ) + + if bazel_features.external_deps.extension_metadata_has_reproducible: + # If we are not using the `experimental_index_url feature, the extension is fully + # deterministic and we don't need to create a lock entry for it. + # + # In order to be able to dogfood the `experimental_index_url` feature before it gets + # stabilized, we have created the `_pip_non_reproducible` function, that will result + # in extra entries in the lock file. + return module_ctx.extension_metadata(reproducible = is_extension_reproducible) + else: + return None + +def _pip_non_reproducible(module_ctx): + _pip_impl(module_ctx) + + # We default to calling the PyPI index and that will go into the + # MODULE.bazel.lock file, hence return nothing here. + return None + +def _pip_parse_ext_attrs(**kwargs): + """Get the attributes for the pip extension. + + Args: + **kwargs: A kwarg for setting defaults for the specific attributes. The + key is expected to be the same as the attribute key. + + Returns: + A dict of attributes. + """ + attrs = dict({ + "experimental_extra_index_urls": attr.string_list( + doc = """\ +The extra index URLs to use for downloading wheels using bazel downloader. +Each value is going to be subject to `envsubst` substitutions if necessary. + +The indexes must support Simple API as described here: +https://packaging.python.org/en/latest/specifications/simple-repository-api/ + +This is equivalent to `--extra-index-urls` `pip` option. +""", + default = [], + ), + "experimental_index_url": attr.string( + default = kwargs.get("experimental_index_url", ""), + doc = """\ +The index URL to use for downloading wheels using bazel downloader. This value is going +to be subject to `envsubst` substitutions if necessary. + +The indexes must support Simple API as described here: +https://packaging.python.org/en/latest/specifications/simple-repository-api/ + +In the future this could be defaulted to `https://pypi.org` when this feature becomes +stable. + +This is equivalent to `--index-url` `pip` option. +""", + ), + "experimental_index_url_overrides": attr.string_dict( + doc = """\ +The index URL overrides for each package to use for downloading wheels using +bazel downloader. This value is going to be subject to `envsubst` substitutions +if necessary. + +The key is the package name (will be normalized before usage) and the value is the +index URL. + +This design pattern has been chosen in order to be fully deterministic about which +packages come from which source. We want to avoid issues similar to what happened in +https://pytorch.org/blog/compromised-nightly-dependency/. + +The indexes must support Simple API as described here: +https://packaging.python.org/en/latest/specifications/simple-repository-api/ +""", + ), + "hub_name": attr.string( + mandatory = True, + doc = """ +The name of the repo pip dependencies will be accessible from. + +This name must be unique between modules; unless your module is guaranteed to +always be the root module, it's highly recommended to include your module name +in the hub name. Repo mapping, `use_repo(..., pip="my_modules_pip_deps")`, can +be used for shorter local names within your module. + +Within a module, the same `hub_name` can be specified to group different Python +versions of pip dependencies under one repository name. This allows using a +Python version-agnostic name when referring to pip dependencies; the +correct version will be automatically selected. + +Typically, a module will only have a single hub of pip dependencies, but this +is not required. Each hub is a separate resolution of pip dependencies. This +means if different programs need different versions of some library, separate +hubs can be created, and each program can use its respective hub's targets. +Targets from different hubs should not be used together. +""", + ), + "parallel_download": attr.bool( + doc = """\ +The flag allows to make use of parallel downloading feature in bazel 7.1 and above +when the bazel downloader is used. This is by default enabled as it improves the +performance by a lot, but in case the queries to the simple API are very expensive +or when debugging authentication issues one may want to disable this feature. + +NOTE, This will download (potentially duplicate) data for multiple packages if +there is more than one index available, but in general this should be negligible +because the simple API calls are very cheap and the user should not notice any +extra overhead. + +If we are in synchronous mode, then we will use the first result that we +find in case extra indexes are specified. +""", + default = True, + ), + "python_version": attr.string( + mandatory = True, + doc = """ +The Python version the dependencies are targetting, in Major.Minor format +(e.g., "3.11") or patch level granularity (e.g. "3.11.1"). + +If an interpreter isn't explicitly provided (using `python_interpreter` or +`python_interpreter_target`), then the version specified here must have +a corresponding `python.toolchain()` configured. +""", + ), + "whl_modifications": attr.label_keyed_string_dict( + mandatory = False, + doc = """\ +A dict of labels to wheel names that is typically generated by the whl_modifications. +The labels are JSON config files describing the modifications. +""", + ), + }, **ATTRS) + attrs.update(AUTH_ATTRS) + + return attrs + +def _whl_mod_attrs(): + attrs = { + "additive_build_content": attr.string( + doc = "(str, optional): Raw text to add to the generated `BUILD` file of a package.", + ), + "additive_build_content_file": attr.label( + doc = """\ +(label, optional): path to a BUILD file to add to the generated +`BUILD` file of a package. You cannot use both additive_build_content and additive_build_content_file +arguments at the same time.""", + ), + "copy_executables": attr.string_dict( + doc = """\ +(dict, optional): A mapping of `src` and `out` files for +[@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as +executable.""", + ), + "copy_files": attr.string_dict( + doc = """\ +(dict, optional): A mapping of `src` and `out` files for +[@bazel_skylib//rules:copy_file.bzl][cf]""", + ), + "data": attr.string_list( + doc = """\ +(list, optional): A list of labels to add as `data` dependencies to +the generated `py_library` target.""", + ), + "data_exclude_glob": attr.string_list( + doc = """\ +(list, optional): A list of exclude glob patterns to add as `data` to +the generated `py_library` target.""", + ), + "hub_name": attr.string( + doc = """\ +Name of the whl modification, hub we use this name to set the modifications for +pip.parse. If you have different pip hubs you can use a different name, +otherwise it is best practice to just use one. + +You cannot have the same `hub_name` in different modules. You can reuse the same +name in the same module for different wheels that you put in the same hub, but you +cannot have a child module that uses the same `hub_name`. +""", + mandatory = True, + ), + "srcs_exclude_glob": attr.string_list( + doc = """\ +(list, optional): A list of labels to add as `srcs` to the generated +`py_library` target.""", + ), + "whl_name": attr.string( + doc = "The whl name that the modifications are used for.", + mandatory = True, + ), + } + return attrs + +# NOTE: the naming of 'override' is taken from the bzlmod native +# 'archive_override', 'git_override' bzlmod functions. +_override_tag = tag_class( + attrs = { + "file": attr.string( + doc = """\ +The Python distribution file name which needs to be patched. This will be +applied to all repositories that setup this distribution via the pip.parse tag +class.""", + mandatory = True, + ), + "patch_strip": attr.int( + default = 0, + doc = """\ +The number of leading path segments to be stripped from the file name in the +patches.""", + ), + "patches": attr.label_list( + doc = """\ +A list of patches to apply to the repository *after* 'whl_library' is extracted +and BUILD.bazel file is generated.""", + mandatory = True, + ), + }, + doc = """\ +Apply any overrides (e.g. patches) to a given Python distribution defined by +other tags in this extension.""", +) + +pypi = module_extension( + doc = """\ +This extension is used to make dependencies from pip available. + +pip.parse: +To use, call `pip.parse()` and specify `hub_name` and your requirements file. +Dependencies will be downloaded and made available in a repo named after the +`hub_name` argument. + +Each `pip.parse()` call configures a particular Python version. Multiple calls +can be made to configure different Python versions, and will be grouped by +the `hub_name` argument. This allows the same logical name, e.g. `@pip//numpy` +to automatically resolve to different, Python version-specific, libraries. + +pip.whl_mods: +This tag class is used to help create JSON files to describe modifications to +the BUILD files for wheels. +""", + implementation = _pip_impl, + tag_classes = { + "override": _override_tag, + "parse": tag_class( + attrs = _pip_parse_ext_attrs(), + doc = """\ +This tag class is used to create a pip hub and all of the spokes that are part of that hub. +This tag class reuses most of the pip attributes that are found in +@rules_python//python/pip_install:pip_repository.bzl. +The exception is it does not use the arg 'repo_prefix'. We set the repository +prefix for the user and the alias arg is always True in bzlmod. +""", + ), + "whl_mods": tag_class( + attrs = _whl_mod_attrs(), + doc = """\ +This tag class is used to create JSON file that are used when calling wheel_builder.py. These +JSON files contain instructions on how to modify a wheel's project. Each of the attributes +create different modifications based on the type of attribute. Previously to bzlmod these +JSON files where referred to as annotations, and were renamed to whl_modifications in this +extension. +""", + ), + }, +) + +pypi_internal = module_extension( + doc = """\ +This extension is used to make dependencies from pypi available. + +For now this is intended to be used internally so that usage of the `pip` +extension in `rules_python` does not affect the evaluations of the extension +for the consumers. + +pip.parse: +To use, call `pip.parse()` and specify `hub_name` and your requirements file. +Dependencies will be downloaded and made available in a repo named after the +`hub_name` argument. + +Each `pip.parse()` call configures a particular Python version. Multiple calls +can be made to configure different Python versions, and will be grouped by +the `hub_name` argument. This allows the same logical name, e.g. `@pypi//numpy` +to automatically resolve to different, Python version-specific, libraries. + +pip.whl_mods: +This tag class is used to help create JSON files to describe modifications to +the BUILD files for wheels. +""", + implementation = _pip_non_reproducible, + tag_classes = { + "override": _override_tag, + "parse": tag_class( + attrs = _pip_parse_ext_attrs( + experimental_index_url = "https://pypi.org/simple", + ), + doc = """\ +This tag class is used to create a pypi hub and all of the spokes that are part of that hub. +This tag class reuses most of the pypi attributes that are found in +@rules_python//python/pip_install:pip_repository.bzl. +The exception is it does not use the arg 'repo_prefix'. We set the repository +prefix for the user and the alias arg is always True in bzlmod. +""", + ), + "whl_mods": tag_class( + attrs = _whl_mod_attrs(), + doc = """\ +This tag class is used to create JSON file that are used when calling wheel_builder.py. These +JSON files contain instructions on how to modify a wheel's project. Each of the attributes +create different modifications based on the type of attribute. Previously to bzlmod these +JSON files where referred to as annotations, and were renamed to whl_modifications in this +extension. +""", + ), + }, +) + +def _whl_mods_repo_impl(rctx): + rctx.file("BUILD.bazel", "") + for whl_name, mods in rctx.attr.whl_mods.items(): + rctx.file("{}.json".format(whl_name), mods) + +_whl_mods_repo = repository_rule( + doc = """\ +This rule creates json files based on the whl_mods attribute. +""", + implementation = _whl_mods_repo_impl, + attrs = { + "whl_mods": attr.string_dict( + mandatory = True, + doc = "JSON endcoded string that is provided to wheel_builder.py", + ), + }, +) diff --git a/python/private/pip_config_settings.bzl b/python/private/pypi/config_settings.bzl similarity index 98% rename from python/private/pip_config_settings.bzl rename to python/private/pypi/config_settings.bzl index b13b39b725..974121782f 100644 --- a/python/private/pip_config_settings.bzl +++ b/python/private/pypi/config_settings.bzl @@ -39,12 +39,7 @@ Note, that here the specialization of musl vs manylinux wheels is the same in order to ensure that the matching fails if the user requests for `musl` and we don't have it or vice versa. """ -load( - ":pip_flags.bzl", - "INTERNAL_FLAGS", - "UniversalWhlFlag", - "WhlLibcFlag", -) +load(":flags.bzl", "INTERNAL_FLAGS", "UniversalWhlFlag", "WhlLibcFlag") FLAGS = struct( **{ @@ -73,7 +68,7 @@ _flags = struct( } ) -def pip_config_settings( +def config_settings( *, python_versions = [], glibc_versions = [], diff --git a/python/private/pip_flags.bzl b/python/private/pypi/flags.bzl similarity index 98% rename from python/private/pip_flags.bzl rename to python/private/pypi/flags.bzl index 1d271c7318..d834be8cc6 100644 --- a/python/private/pip_flags.bzl +++ b/python/private/pypi/flags.bzl @@ -18,7 +18,7 @@ NOTE: The transitive loads of this should be kept minimal. This avoids loading unnecessary files when all that are needed are flag definitions. """ -load(":enum.bzl", "enum") +load("//python/private:enum.bzl", "enum") # Determines if we should use whls for third party # diff --git a/python/pip_install/private/generate_group_library_build_bazel.bzl b/python/private/pypi/generate_group_library_build_bazel.bzl similarity index 99% rename from python/pip_install/private/generate_group_library_build_bazel.bzl rename to python/private/pypi/generate_group_library_build_bazel.bzl index 5fa93e22b7..54da066b42 100644 --- a/python/pip_install/private/generate_group_library_build_bazel.bzl +++ b/python/private/pypi/generate_group_library_build_bazel.bzl @@ -14,15 +14,15 @@ """Generate the BUILD.bazel contents for a repo defined by a group_library.""" +load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:text_util.bzl", "render") load( - "//python/private:labels.bzl", + ":labels.bzl", "PY_LIBRARY_IMPL_LABEL", "PY_LIBRARY_PUBLIC_LABEL", "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", ) -load("//python/private:normalize_name.bzl", "normalize_name") -load("//python/private:text_util.bzl", "render") _PRELUDE = """\ load("@rules_python//python:defs.bzl", "py_library") diff --git a/python/pip_install/private/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl similarity index 99% rename from python/pip_install/private/generate_whl_library_build_bazel.bzl rename to python/private/pypi/generate_whl_library_build_bazel.bzl index f3ddd3bcab..d25f73a049 100644 --- a/python/pip_install/private/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -14,8 +14,10 @@ """Generate the BUILD.bazel contents for a repo defined by a whl_library.""" +load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:text_util.bzl", "render") load( - "//python/private:labels.bzl", + ":labels.bzl", "DATA_LABEL", "DIST_INFO_LABEL", "PY_LIBRARY_IMPL_LABEL", @@ -24,8 +26,6 @@ load( "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", ) -load("//python/private:normalize_name.bzl", "normalize_name") -load("//python/private:text_util.bzl", "render") _COPY_FILE_TEMPLATE = """\ copy_file( diff --git a/python/private/pypi/group_library.bzl b/python/private/pypi/group_library.bzl new file mode 100644 index 0000000000..ff800e2f18 --- /dev/null +++ b/python/private/pypi/group_library.bzl @@ -0,0 +1,40 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""group_library implementation for WORKSPACE setups.""" + +load(":generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel") + +def _group_library_impl(rctx): + build_file_contents = generate_group_library_build_bazel( + repo_prefix = rctx.attr.repo_prefix, + groups = rctx.attr.groups, + ) + rctx.file("BUILD.bazel", build_file_contents) + +group_library = repository_rule( + attrs = { + "groups": attr.string_list_dict( + doc = "A mapping of group names to requirements within that group.", + ), + "repo_prefix": attr.string( + doc = "Prefix used for the whl_library created components of each group", + ), + }, + implementation = _group_library_impl, + doc = """ +Create a package containing only wrapper py_library and whl_library rules for implementing dependency groups. +This is an implementation detail of dependency groups and should not be used alone. + """, +) diff --git a/python/private/bzlmod/pip_repository.bzl b/python/private/pypi/hub_repository.bzl similarity index 79% rename from python/private/bzlmod/pip_repository.bzl rename to python/private/pypi/hub_repository.bzl index d42eb8b056..5e209d6f9f 100644 --- a/python/private/bzlmod/pip_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -14,12 +14,12 @@ "" +load("//python/private:text_util.bzl", "render") load( - "//python/private:render_pkg_aliases.bzl", + ":render_pkg_aliases.bzl", "render_multiplatform_pkg_aliases", "whl_alias", ) -load("//python/private:text_util.bzl", "render") _BUILD_FILE_CONTENTS = """\ package(default_visibility = ["//visibility:public"]) @@ -28,7 +28,7 @@ package(default_visibility = ["//visibility:public"]) exports_files(["requirements.bzl"]) """ -def _pip_repository_impl(rctx): +def _impl(rctx): bzl_packages = rctx.attr.whl_map.keys() aliases = render_multiplatform_pkg_aliases( aliases = { @@ -66,35 +66,33 @@ def _pip_repository_impl(rctx): "%%NAME%%": rctx.attr.repo_name, }) -pip_repository_attrs = { - "default_version": attr.string( - mandatory = True, - doc = """\ +hub_repository = repository_rule( + attrs = { + "default_version": attr.string( + mandatory = True, + doc = """\ This is the default python version in the format of X.Y. This should match what is setup by the 'python' extension using the 'is_default = True' setting.""", - ), - "groups": attr.string_list_dict( - mandatory = False, - ), - "repo_name": attr.string( - mandatory = True, - doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", - ), - "whl_map": attr.string_dict( - mandatory = True, - doc = """\ + ), + "groups": attr.string_list_dict( + mandatory = False, + ), + "repo_name": attr.string( + mandatory = True, + doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", + ), + "whl_map": attr.string_dict( + mandatory = True, + doc = """\ The wheel map where values are json.encoded strings of the whl_map constructed in the pip.parse tag class. """, - ), - "_template": attr.label( - default = ":requirements.bzl.tmpl", - ), -} - -pip_repository = repository_rule( - attrs = pip_repository_attrs, + ), + "_template": attr.label( + default = ":requirements.bzl.tmpl.bzlmod", + ), + }, doc = """A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY.""", - implementation = _pip_repository_impl, + implementation = _impl, ) diff --git a/python/private/pypi_index_sources.bzl b/python/private/pypi/index_sources.bzl similarity index 98% rename from python/private/pypi_index_sources.bzl rename to python/private/pypi/index_sources.bzl index 470a8c9f5a..21660141db 100644 --- a/python/private/pypi_index_sources.bzl +++ b/python/private/pypi/index_sources.bzl @@ -16,7 +16,7 @@ A file that houses private functions used in the `bzlmod` extension with the same name. """ -def get_simpleapi_sources(line): +def index_sources(line): """Get PyPI sources from a requirements.txt line. We interpret the spec described in diff --git a/python/private/labels.bzl b/python/private/pypi/labels.bzl similarity index 100% rename from python/private/labels.bzl rename to python/private/pypi/labels.bzl diff --git a/python/private/pypi/package_annotation.bzl b/python/private/pypi/package_annotation.bzl new file mode 100644 index 0000000000..4a54703ac4 --- /dev/null +++ b/python/private/pypi/package_annotation.bzl @@ -0,0 +1,49 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Package annotation API for WORKSPACE setups.""" + +def package_annotation( + additive_build_content = None, + copy_files = {}, + copy_executables = {}, + data = [], + data_exclude_glob = [], + srcs_exclude_glob = []): + """Annotations to apply to the BUILD file content from package generated from a `pip_repository` rule. + + [cf]: https://github.com/bazelbuild/bazel-skylib/blob/main/docs/copy_file_doc.md + + Args: + additive_build_content (str, optional): Raw text to add to the generated `BUILD` file of a package. + copy_files (dict, optional): A mapping of `src` and `out` files for [@bazel_skylib//rules:copy_file.bzl][cf] + copy_executables (dict, optional): A mapping of `src` and `out` files for + [@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as + executable. + data (list, optional): A list of labels to add as `data` dependencies to the generated `py_library` target. + data_exclude_glob (list, optional): A list of exclude glob patterns to add as `data` to the generated + `py_library` target. + srcs_exclude_glob (list, optional): A list of labels to add as `srcs` to the generated `py_library` target. + + Returns: + str: A json encoded string of the provided content. + """ + return json.encode(struct( + additive_build_content = additive_build_content, + copy_files = copy_files, + copy_executables = copy_executables, + data = data, + data_exclude_glob = data_exclude_glob, + srcs_exclude_glob = srcs_exclude_glob, + )) diff --git a/python/private/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl similarity index 98% rename from python/private/parse_requirements.bzl rename to python/private/pypi/parse_requirements.bzl index 21e132b4f8..22a6f0a875 100644 --- a/python/private/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -26,9 +26,9 @@ file for the host platform to be backwards compatible with the legacy behavior. """ -load("//python/pip_install:requirements_parser.bzl", "parse") -load(":normalize_name.bzl", "normalize_name") -load(":pypi_index_sources.bzl", "get_simpleapi_sources") +load("//python/private:normalize_name.bzl", "normalize_name") +load(":index_sources.bzl", "index_sources") +load(":parse_requirements_txt.bzl", "parse_requirements_txt") load(":whl_target_platforms.bzl", "select_whls", "whl_target_platforms") # This includes the vendored _translate_cpu and _translate_os from @@ -271,7 +271,7 @@ def parse_requirements( # Parse the requirements file directly in starlark to get the information # needed for the whl_library declarations later. - parse_result = parse(contents) + parse_result = parse_requirements_txt(contents) # Replicate a surprising behavior that WORKSPACE builds allowed: # Defining a repo with the same name multiple times, but only the last @@ -317,7 +317,7 @@ def parse_requirements( (requirement_line, ",".join(extra_pip_args)), struct( distribution = distribution, - srcs = get_simpleapi_sources(requirement_line), + srcs = index_sources(requirement_line), requirement_line = requirement_line, target_platforms = [], extra_pip_args = extra_pip_args, diff --git a/python/private/pypi/parse_requirements_txt.bzl b/python/private/pypi/parse_requirements_txt.bzl new file mode 100644 index 0000000000..6f51d034da --- /dev/null +++ b/python/private/pypi/parse_requirements_txt.bzl @@ -0,0 +1,133 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pip requirements parser for Starlark.""" + +_STATE = struct( + # Consume extraneous whitespace + ConsumeSpace = 0, + # Consume a comment + ConsumeComment = 1, + # Parse the name of a pip package + ParseDependency = 2, + # Parse a full requirement line + ParseRequirement = 3, + # Parse a pip option + ParseOption = 4, +) + +EOF = {} + +def parse_requirements_txt(content): + """A simplistic (and incomplete) pip requirements lockfile parser. + + Parses package names and their full requirement lines, as well pip + options. + + Args: + content: lockfile content as a string + + Returns: + Struct with fields `requirements` and `options`. + + requirements: List of requirements, where each requirement is a 2-element + tuple containing the package name and the requirement line. + E.g., [(certifi', 'certifi==2021.10.8 --hash=sha256:7888...'), ...] + + options: List of pip option lines + """ + content = content.replace("\r", "") + + result = struct( + requirements = [], + options = [], + ) + state = _STATE.ConsumeSpace + buffer = "" + + inputs = content.elems()[:] + inputs.append(EOF) + + for input in inputs: + if state == _STATE.ConsumeSpace: + (state, buffer) = _handleConsumeSpace(input) + elif state == _STATE.ConsumeComment: + (state, buffer) = _handleConsumeComment(input, buffer, result) + elif state == _STATE.ParseDependency: + (state, buffer) = _handleParseDependency(input, buffer, result) + elif state == _STATE.ParseOption: + (state, buffer) = _handleParseOption(input, buffer, result) + elif state == _STATE.ParseRequirement: + (state, buffer) = _handleParseRequirement(input, buffer, result) + else: + fail("Unknown state %d" % state) + + return result + +def _handleConsumeSpace(input): + if input == EOF: + return (_STATE.ConsumeSpace, "") + if input.isspace(): + return (_STATE.ConsumeSpace, "") + elif input == "#": + return (_STATE.ConsumeComment, "") + elif input == "-": + return (_STATE.ParseOption, input) + + return (_STATE.ParseDependency, input) + +def _handleConsumeComment(input, buffer, result): + if input == "\n": + if len(result.requirements) > 0 and len(result.requirements[-1]) == 1: + result.requirements[-1] = (result.requirements[-1][0], buffer.rstrip(" \n")) + return (_STATE.ConsumeSpace, "") + elif len(buffer) > 0: + result.options.append(buffer.rstrip(" \n")) + return (_STATE.ConsumeSpace, "") + return (_STATE.ConsumeSpace, "") + return (_STATE.ConsumeComment, buffer) + +def _handleParseDependency(input, buffer, result): + if input == EOF: + fail("Enountered unexpected end of file while parsing requirement") + elif input.isspace() or input in [">", "<", "~", "=", ";", "["]: + result.requirements.append((buffer,)) + return (_STATE.ParseRequirement, buffer + input) + + return (_STATE.ParseDependency, buffer + input) + +def _handleParseOption(input, buffer, result): + if input == "\n" and buffer.endswith("\\"): + return (_STATE.ParseOption, buffer[0:-1]) + elif input == " ": + result.options.append(buffer.rstrip("\n")) + return (_STATE.ParseOption, "") + elif input == "\n" or input == EOF: + result.options.append(buffer.rstrip("\n")) + return (_STATE.ConsumeSpace, "") + elif input == "#" and (len(buffer) == 0 or buffer[-1].isspace()): + return (_STATE.ConsumeComment, buffer) + + return (_STATE.ParseOption, buffer + input) + +def _handleParseRequirement(input, buffer, result): + if input == "\n" and buffer.endswith("\\"): + return (_STATE.ParseRequirement, buffer[0:-1]) + elif input == "\n" or input == EOF: + result.requirements[-1] = (result.requirements[-1][0], buffer.rstrip(" \n")) + return (_STATE.ConsumeSpace, "") + elif input == "#" and (len(buffer) == 0 or buffer[-1].isspace()): + return (_STATE.ConsumeComment, buffer) + + return (_STATE.ParseRequirement, buffer + input) diff --git a/python/private/pypi/parse_simpleapi_html.bzl b/python/private/pypi/parse_simpleapi_html.bzl new file mode 100644 index 0000000000..f7cd032aca --- /dev/null +++ b/python/private/pypi/parse_simpleapi_html.bzl @@ -0,0 +1,106 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Parse SimpleAPI HTML in Starlark. +""" + +def parse_simpleapi_html(*, url, content): + """Get the package URLs for given shas by parsing the Simple API HTML. + + Args: + url(str): The URL that the HTML content can be downloaded from. + content(str): The Simple API HTML content. + + Returns: + A list of structs with: + * filename: The filename of the artifact. + * url: The URL to download the artifact. + * sha256: The sha256 of the artifact. + * metadata_sha256: The whl METADATA sha256 if we can download it. If this is + present, then the 'metadata_url' is also present. Defaults to "". + * metadata_url: The URL for the METADATA if we can download it. Defaults to "". + """ + sdists = {} + whls = {} + lines = content.split("= (2, 0): + # We don't expect to have version 2.0 here, but have this check in place just in case. + # https://packaging.python.org/en/latest/specifications/simple-repository-api/#versioning-pypi-s-simple-api + fail("Unsupported API version: {}".format(api_version)) + + for line in lines[1:]: + dist_url, _, tail = line.partition("#sha256=") + sha256, _, tail = tail.partition("\"") + + # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api + yanked = "data-yanked" in line + + maybe_metadata, _, tail = tail.partition(">") + filename, _, tail = tail.partition("<") + + metadata_sha256 = "" + metadata_url = "" + for metadata_marker in ["data-core-metadata", "data-dist-info-metadata"]: + metadata_marker = metadata_marker + "=\"sha256=" + if metadata_marker in maybe_metadata: + # Implement https://peps.python.org/pep-0714/ + _, _, tail = maybe_metadata.partition(metadata_marker) + metadata_sha256, _, _ = tail.partition("\"") + metadata_url = dist_url + ".metadata" + break + + if filename.endswith(".whl"): + whls[sha256] = struct( + filename = filename, + url = _absolute_url(url, dist_url), + sha256 = sha256, + metadata_sha256 = metadata_sha256, + metadata_url = _absolute_url(url, metadata_url), + yanked = yanked, + ) + else: + sdists[sha256] = struct( + filename = filename, + url = _absolute_url(url, dist_url), + sha256 = sha256, + metadata_sha256 = "", + metadata_url = "", + yanked = yanked, + ) + + return struct( + sdists = sdists, + whls = whls, + ) + +def _absolute_url(index_url, candidate): + if not candidate.startswith(".."): + return candidate + + candidate_parts = candidate.split("..") + last = candidate_parts[-1] + for _ in range(len(candidate_parts) - 1): + index_url, _, _ = index_url.rstrip("/").rpartition("/") + + return "{}/{}".format(index_url, last.strip("/")) diff --git a/python/private/parse_whl_name.bzl b/python/private/pypi/parse_whl_name.bzl similarity index 100% rename from python/private/parse_whl_name.bzl rename to python/private/pypi/parse_whl_name.bzl diff --git a/python/private/patch_whl.bzl b/python/private/pypi/patch_whl.bzl similarity index 87% rename from python/private/patch_whl.bzl rename to python/private/pypi/patch_whl.bzl index 9e3119f744..c2c633da7f 100644 --- a/python/private/patch_whl.bzl +++ b/python/private/pypi/patch_whl.bzl @@ -27,7 +27,8 @@ other patches ensures that the users have overview on exactly what has changed within the wheel. """ -load("//python/private:parse_whl_name.bzl", "parse_whl_name") +load("//python/private:repo_utils.bzl", "repo_utils") +load(":parse_whl_name.bzl", "parse_whl_name") _rules_python_root = Label("//:BUILD.bazel") @@ -40,7 +41,7 @@ def patch_whl(rctx, *, python_interpreter, whl_path, patches, **kwargs): whl_path: The whl file name to be patched. patches: a label-keyed-int dict that has the patch files as keys and the patch_strip as the value. - **kwargs: extras passed to rctx.execute. + **kwargs: extras passed to repo_utils.execute_checked. Returns: value of the repackaging action. @@ -75,11 +76,12 @@ def patch_whl(rctx, *, python_interpreter, whl_path, patches, **kwargs): record_patch = rctx.path("RECORD.patch") - result = rctx.execute( - [ + repo_utils.execute_checked( + rctx, + arguments = [ python_interpreter, "-m", - "python.private.repack_whl", + "python.private.pypi.repack_whl", "--record-patch", record_patch, whl_input, @@ -91,16 +93,6 @@ def patch_whl(rctx, *, python_interpreter, whl_path, patches, **kwargs): **kwargs ) - if result.return_code: - fail( - "repackaging .whl {whl} failed: with exit code '{return_code}':\n{stdout}\n\nstderr:\n{stderr}".format( - whl = whl_input.basename, - stdout = result.stdout, - stderr = result.stderr, - return_code = result.return_code, - ), - ) - if record_patch.exists: record_patch_contents = rctx.read(record_patch) warning_msg = """WARNING: the resultant RECORD file of the patch wheel is different diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl new file mode 100644 index 0000000000..a22f4d9d2c --- /dev/null +++ b/python/private/pypi/pip_repository.bzl @@ -0,0 +1,327 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("@bazel_skylib//lib:sets.bzl", "sets") +load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR") +load("//python/private:text_util.bzl", "render") +load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") +load(":pip_repository_attrs.bzl", "ATTRS") +load(":render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias") + +def _get_python_interpreter_attr(rctx): + """A helper function for getting the `python_interpreter` attribute or it's default + + Args: + rctx (repository_ctx): Handle to the rule repository context. + + Returns: + str: The attribute value or it's default + """ + if rctx.attr.python_interpreter: + return rctx.attr.python_interpreter + + if "win" in rctx.os.name: + return "python.exe" + else: + return "python3" + +def use_isolated(ctx, attr): + """Determine whether or not to pass the pip `--isolated` flag to the pip invocation. + + Args: + ctx: repository or module context + attr: attributes for the repo rule or tag extension + + Returns: + True if --isolated should be passed + """ + use_isolated = attr.isolated + + # The environment variable will take precedence over the attribute + isolated_env = ctx.os.environ.get("RULES_PYTHON_PIP_ISOLATED", None) + if isolated_env != None: + if isolated_env.lower() in ("0", "false"): + use_isolated = False + else: + use_isolated = True + + return use_isolated + +_BUILD_FILE_CONTENTS = """\ +package(default_visibility = ["//visibility:public"]) + +# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it +exports_files(["requirements.bzl"]) +""" + +def _pip_repository_impl(rctx): + requirements_by_platform = parse_requirements( + rctx, + requirements_by_platform = rctx.attr.requirements_by_platform, + requirements_linux = rctx.attr.requirements_linux, + requirements_lock = rctx.attr.requirements_lock, + requirements_osx = rctx.attr.requirements_darwin, + requirements_windows = rctx.attr.requirements_windows, + extra_pip_args = rctx.attr.extra_pip_args, + ) + selected_requirements = {} + options = None + repository_platform = host_platform(rctx.os) + for name, requirements in requirements_by_platform.items(): + r = select_requirement( + requirements, + platform = repository_platform, + ) + if not r: + continue + options = options or r.extra_pip_args + selected_requirements[name] = r.requirement_line + + bzl_packages = sorted(selected_requirements.keys()) + + # Normalize cycles first + requirement_cycles = { + name: sorted(sets.to_list(sets.make(deps))) + for name, deps in rctx.attr.experimental_requirement_cycles.items() + } + + # Check for conflicts between cycles _before_ we normalize package names so + # that reported errors use the names the user specified + for i in range(len(requirement_cycles)): + left_group = requirement_cycles.keys()[i] + left_deps = requirement_cycles.values()[i] + for j in range(len(requirement_cycles) - (i + 1)): + right_deps = requirement_cycles.values()[1 + i + j] + right_group = requirement_cycles.keys()[1 + i + j] + for d in left_deps: + if d in right_deps: + fail("Error: Requirement %s cannot be repeated between cycles %s and %s; please merge the cycles." % (d, left_group, right_group)) + + # And normalize the names as used in the cycle specs + # + # NOTE: We must check that a listed dependency is actually in the actual + # requirements set for the current platform so that we can support cycles in + # platform-conditional requirements. Otherwise we'll blindly generate a + # label referencing a package which may not be installed on the current + # platform. + requirement_cycles = { + normalize_name(name): sorted([normalize_name(d) for d in group if normalize_name(d) in bzl_packages]) + for name, group in requirement_cycles.items() + } + + imports = [ + # NOTE: Maintain the order consistent with `buildifier` + 'load("@rules_python//python:pip.bzl", "pip_utils")', + 'load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library")', + ] + + annotations = {} + for pkg, annotation in rctx.attr.annotations.items(): + filename = "{}.annotation.json".format(normalize_name(pkg)) + rctx.file(filename, json.encode_indent(json.decode(annotation))) + annotations[pkg] = "@{name}//:{filename}".format(name = rctx.attr.name, filename = filename) + + config = { + "download_only": rctx.attr.download_only, + "enable_implicit_namespace_pkgs": rctx.attr.enable_implicit_namespace_pkgs, + "environment": rctx.attr.environment, + "envsubst": rctx.attr.envsubst, + "extra_pip_args": options, + "isolated": use_isolated(rctx, rctx.attr), + "pip_data_exclude": rctx.attr.pip_data_exclude, + "python_interpreter": _get_python_interpreter_attr(rctx), + "quiet": rctx.attr.quiet, + "repo": rctx.attr.name, + "timeout": rctx.attr.timeout, + } + if rctx.attr.use_hub_alias_dependencies: + config["dep_template"] = "@{}//{{name}}:{{target}}".format(rctx.attr.name) + else: + config["repo_prefix"] = "{}_".format(rctx.attr.name) + + if rctx.attr.python_interpreter_target: + config["python_interpreter_target"] = str(rctx.attr.python_interpreter_target) + if rctx.attr.experimental_target_platforms: + config["experimental_target_platforms"] = rctx.attr.experimental_target_platforms + + macro_tmpl = "@%s//{}:{}" % rctx.attr.name + + aliases = render_pkg_aliases( + aliases = { + pkg: [whl_alias(repo = rctx.attr.name + "_" + pkg)] + for pkg in bzl_packages or [] + }, + ) + for path, contents in aliases.items(): + rctx.file(path, contents) + + rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) + rctx.template("requirements.bzl", rctx.attr._template, substitutions = { + " # %%GROUP_LIBRARY%%": """\ + group_repo = "{name}__groups" + group_library( + name = group_repo, + repo_prefix = "{name}_", + groups = all_requirement_groups, + )""".format(name = rctx.attr.name) if not rctx.attr.use_hub_alias_dependencies else "", + "%%ALL_DATA_REQUIREMENTS%%": render.list([ + macro_tmpl.format(p, "data") + for p in bzl_packages + ]), + "%%ALL_REQUIREMENTS%%": render.list([ + macro_tmpl.format(p, "pkg") + for p in bzl_packages + ]), + "%%ALL_REQUIREMENT_GROUPS%%": render.dict(requirement_cycles), + "%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": render.dict({ + p: macro_tmpl.format(p, "whl") + for p in bzl_packages + }), + "%%ANNOTATIONS%%": render.dict(dict(sorted(annotations.items()))), + "%%CONFIG%%": render.dict(dict(sorted(config.items()))), + "%%EXTRA_PIP_ARGS%%": json.encode(options), + "%%IMPORTS%%": "\n".join(imports), + "%%MACRO_TMPL%%": macro_tmpl, + "%%NAME%%": rctx.attr.name, + "%%PACKAGES%%": render.list( + [ + ("{}_{}".format(rctx.attr.name, p), r) + for p, r in sorted(selected_requirements.items()) + ], + ), + }) + + return + +pip_repository = repository_rule( + attrs = dict( + annotations = attr.string_dict( + doc = "Optional annotations to apply to packages", + ), + _template = attr.label( + default = ":requirements.bzl.tmpl.workspace", + ), + **ATTRS + ), + doc = """Accepts a locked/compiled requirements file and installs the dependencies listed within. + +Those dependencies become available in a generated `requirements.bzl` file. +You can instead check this `requirements.bzl` file into your repo, see the "vendoring" section below. + +In your WORKSPACE file: + +```starlark +load("@rules_python//python:pip.bzl", "pip_parse") + +pip_parse( + name = "pypi", + requirements_lock = ":requirements.txt", +) + +load("@pypi//:requirements.bzl", "install_deps") + +install_deps() +``` + +You can then reference installed dependencies from a `BUILD` file with the alias targets generated in the same repo, for example, for `PyYAML` we would have the following: +- `@pypi//pyyaml` and `@pypi//pyyaml:pkg` both point to the `py_library` + created after extracting the `PyYAML` package. +- `@pypi//pyyaml:data` points to the extra data included in the package. +- `@pypi//pyyaml:dist_info` points to the `dist-info` files in the package. +- `@pypi//pyyaml:whl` points to the wheel file that was extracted. + +```starlark +py_library( + name = "bar", + ... + deps = [ + "//my/other:dep", + "@pypi//numpy", + "@pypi//requests", + ], +) +``` + +or + +```starlark +load("@pypi//:requirements.bzl", "requirement") + +py_library( + name = "bar", + ... + deps = [ + "//my/other:dep", + requirement("numpy"), + requirement("requests"), + ], +) +``` + +In addition to the `requirement` macro, which is used to access the generated `py_library` +target generated from a package's wheel, The generated `requirements.bzl` file contains +functionality for exposing [entry points][whl_ep] as `py_binary` targets as well. + +[whl_ep]: https://packaging.python.org/specifications/entry-points/ + +```starlark +load("@pypi//:requirements.bzl", "entry_point") + +alias( + name = "pip-compile", + actual = entry_point( + pkg = "pip-tools", + script = "pip-compile", + ), +) +``` + +Note that for packages whose name and script are the same, only the name of the package +is needed when calling the `entry_point` macro. + +```starlark +load("@pip//:requirements.bzl", "entry_point") + +alias( + name = "flake8", + actual = entry_point("flake8"), +) +``` + +### Vendoring the requirements.bzl file + +In some cases you may not want to generate the requirements.bzl file as a repository rule +while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module +such as a ruleset, you may want to include the requirements.bzl file rather than make your users +install the WORKSPACE setup to generate it. +See https://github.com/bazelbuild/rules_python/issues/608 + +This is the same workflow as Gazelle, which creates `go_repository` rules with +[`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos) + +To do this, use the "write to source file" pattern documented in +https://blog.aspect.dev/bazel-can-write-to-the-source-folder +to put a copy of the generated requirements.bzl into your project. +Then load the requirements.bzl file directly rather than from the generated repository. +See the example in rules_python/examples/pip_parse_vendored. +""", + implementation = _pip_repository_impl, + environ = [ + "RULES_PYTHON_PIP_ISOLATED", + REPO_DEBUG_ENV_VAR, + ], +) diff --git a/python/private/pypi/pip_repository_attrs.bzl b/python/private/pypi/pip_repository_attrs.bzl new file mode 100644 index 0000000000..23000869e9 --- /dev/null +++ b/python/private/pypi/pip_repository_attrs.bzl @@ -0,0 +1,73 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Common attributes between bzlmod pip.parse and workspace pip_parse. + +A common attributes shared between bzlmod and workspace implementations +stored in a separate file to avoid unnecessary refetching of the +repositories.""" + +load(":attrs.bzl", COMMON_ATTRS = "ATTRS") + +ATTRS = { + "requirements_by_platform": attr.label_keyed_string_dict( + doc = """\ +The requirements files and the comma delimited list of target platforms as values. + +The keys are the requirement files and the values are comma-separated platform +identifiers. For now we only support `_` values that are present in +`@platforms//os` and `@platforms//cpu` packages respectively. +""", + ), + "requirements_darwin": attr.label( + allow_single_file = True, + doc = "Override the requirements_lock attribute when the host platform is Mac OS", + ), + "requirements_linux": attr.label( + allow_single_file = True, + doc = "Override the requirements_lock attribute when the host platform is Linux", + ), + "requirements_lock": attr.label( + allow_single_file = True, + doc = """\ +A fully resolved 'requirements.txt' pip requirement file containing the +transitive set of your dependencies. If this file is passed instead of +'requirements' no resolve will take place and pip_repository will create +individual repositories for each of your dependencies so that wheels are +fetched/built only for the targets specified by 'build/run/test'. Note that if +your lockfile is platform-dependent, you can use the `requirements_[platform]` +attributes. + +Note, that in general requirements files are compiled for a specific platform, +but sometimes they can work for multiple platforms. `rules_python` right now +supports requirements files that are created for a particular platform without +platform markers. +""", + ), + "requirements_windows": attr.label( + allow_single_file = True, + doc = "Override the requirements_lock attribute when the host platform is Windows", + ), + "use_hub_alias_dependencies": attr.bool( + default = False, + doc = """\ +Controls if the hub alias dependencies are used. If set to true, then the +group_library will be included in the hub repo. + +True will become default in a subsequent release. +""", + ), +} + +ATTRS.update(**COMMON_ATTRS) diff --git a/python/private/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl similarity index 97% rename from python/private/render_pkg_aliases.bzl rename to python/private/pypi/render_pkg_aliases.bzl index 82ac764f00..eb907fee0f 100644 --- a/python/private/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -16,8 +16,10 @@ This is used in bzlmod and non-bzlmod setups.""" +load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:text_util.bzl", "render") load( - "//python/pip_install/private:generate_group_library_build_bazel.bzl", + ":generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel", ) # buildifier: disable=bzl-visibility load( @@ -29,9 +31,7 @@ load( "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", ) -load(":normalize_name.bzl", "normalize_name") load(":parse_whl_name.bzl", "parse_whl_name") -load(":text_util.bzl", "render") load(":whl_target_platforms.bzl", "whl_target_platforms") NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\ @@ -309,7 +309,7 @@ def render_multiplatform_pkg_aliases(*, aliases, default_version = None, **kwarg aliases = config_setting_aliases, **kwargs ) - contents["_config/BUILD.bazel"] = _render_pip_config_settings(**flag_versions) + contents["_config/BUILD.bazel"] = _render_config_settings(**flag_versions) return contents def multiplatform_whl_aliases(*, aliases, default_version = None, **kwargs): @@ -398,12 +398,12 @@ def multiplatform_whl_aliases(*, aliases, default_version = None, **kwargs): ret.extend(versioned.values()) return ret -def _render_pip_config_settings(python_versions = [], target_platforms = [], osx_versions = [], glibc_versions = [], muslc_versions = []): +def _render_config_settings(python_versions = [], target_platforms = [], osx_versions = [], glibc_versions = [], muslc_versions = []): return """\ -load("@rules_python//python/private:pip_config_settings.bzl", "pip_config_settings") +load("@rules_python//python/private/pypi:config_settings.bzl", "config_settings") -pip_config_settings( - name = "pip_config_settings", +config_settings( + name = "config_settings", glibc_versions = {glibc_versions}, muslc_versions = {muslc_versions}, osx_versions = {osx_versions}, diff --git a/python/private/repack_whl.py b/python/private/pypi/repack_whl.py similarity index 100% rename from python/private/repack_whl.py rename to python/private/pypi/repack_whl.py diff --git a/python/private/bzlmod/requirements.bzl.tmpl b/python/private/pypi/requirements.bzl.tmpl.bzlmod similarity index 100% rename from python/private/bzlmod/requirements.bzl.tmpl rename to python/private/pypi/requirements.bzl.tmpl.bzlmod diff --git a/python/private/pypi/requirements.bzl.tmpl.workspace b/python/private/pypi/requirements.bzl.tmpl.workspace new file mode 100644 index 0000000000..2f4bcd6916 --- /dev/null +++ b/python/private/pypi/requirements.bzl.tmpl.workspace @@ -0,0 +1,72 @@ +"""Starlark representation of locked requirements. + +@generated by rules_python pip_parse repository rule. +""" + +%%IMPORTS%% + +all_requirements = %%ALL_REQUIREMENTS%% + +all_whl_requirements_by_package = %%ALL_WHL_REQUIREMENTS_BY_PACKAGE%% + +all_whl_requirements = all_whl_requirements_by_package.values() + +all_data_requirements = %%ALL_DATA_REQUIREMENTS%% + +_packages = %%PACKAGES%% +_config = %%CONFIG%% +_annotations = %%ANNOTATIONS%% + +def requirement(name): + return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "pkg") + +def whl_requirement(name): + return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "whl") + +def data_requirement(name): + return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "data") + +def dist_info_requirement(name): + return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "dist_info") + +def _get_annotation(requirement): + # This expects to parse `setuptools==58.2.0 --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11` + # down to `setuptools`. + name = requirement.split(" ")[0].split("=")[0].split("[")[0] + return _annotations.get(name) + +def install_deps(**whl_library_kwargs): + """Repository rule macro. Install dependencies from `pip_parse`. + + Args: + **whl_library_kwargs: Additional arguments which will flow to underlying + `whl_library` calls. See pip_repository.bzl for details. + """ + + # Set up the requirement groups + all_requirement_groups = %%ALL_REQUIREMENT_GROUPS%% + + requirement_group_mapping = { + requirement: group_name + for group_name, group_requirements in all_requirement_groups.items() + for requirement in group_requirements + } + + # %%GROUP_LIBRARY%% + + # Install wheels which may be participants in a group + whl_config = dict(_config) + whl_config.update(whl_library_kwargs) + + for name, requirement in _packages: + group_name = requirement_group_mapping.get(name.replace("%%NAME%%_", "")) + group_deps = all_requirement_groups.get(group_name, []) + + whl_library( + name = name, + requirement = requirement, + group_name = group_name, + group_deps = group_deps, + annotation = _get_annotation(requirement), + **whl_config + ) diff --git a/python/private/pypi_index.bzl b/python/private/pypi/simpleapi_download.bzl similarity index 67% rename from python/private/pypi_index.bzl rename to python/private/pypi/simpleapi_download.bzl index 64d908e32b..b258fef07a 100644 --- a/python/private/pypi_index.bzl +++ b/python/private/pypi/simpleapi_download.bzl @@ -17,9 +17,10 @@ A file that houses private functions used in the `bzlmod` extension with the sam """ load("@bazel_features//:features.bzl", "bazel_features") -load(":auth.bzl", "get_auth") -load(":envsubst.bzl", "envsubst") -load(":normalize_name.bzl", "normalize_name") +load("//python/private:auth.bzl", "get_auth") +load("//python/private:envsubst.bzl", "envsubst") +load("//python/private:normalize_name.bzl", "normalize_name") +load(":parse_simpleapi_html.bzl", "parse_simpleapi_html") def simpleapi_download(ctx, *, attr, cache, parallel_download = True): """Download Simple API HTML. @@ -71,7 +72,7 @@ def simpleapi_download(ctx, *, attr, cache, parallel_download = True): success = False for index_url in index_urls: - result = read_simple_api( + result = _read_simpleapi( ctx = ctx, url = "{}/{}/".format( index_url_overrides.get(pkg_normalized, index_url).rstrip("/"), @@ -122,7 +123,7 @@ def simpleapi_download(ctx, *, attr, cache, parallel_download = True): return contents -def read_simple_api(ctx, url, attr, cache, **download_kwargs): +def _read_simpleapi(ctx, url, attr, cache, **download_kwargs): """Read SimpleAPI. Args: @@ -195,98 +196,9 @@ def _read_index_result(ctx, result, output, url, cache, cache_key): content = ctx.read(output) - output = parse_simple_api_html(url = url, content = content) + output = parse_simpleapi_html(url = url, content = content) if output: cache.setdefault(cache_key, output) return struct(success = True, output = output, cache_key = cache_key) else: return struct(success = False) - -def parse_simple_api_html(*, url, content): - """Get the package URLs for given shas by parsing the Simple API HTML. - - Args: - url(str): The URL that the HTML content can be downloaded from. - content(str): The Simple API HTML content. - - Returns: - A list of structs with: - * filename: The filename of the artifact. - * url: The URL to download the artifact. - * sha256: The sha256 of the artifact. - * metadata_sha256: The whl METADATA sha256 if we can download it. If this is - present, then the 'metadata_url' is also present. Defaults to "". - * metadata_url: The URL for the METADATA if we can download it. Defaults to "". - """ - sdists = {} - whls = {} - lines = content.split("= (2, 0): - # We don't expect to have version 2.0 here, but have this check in place just in case. - # https://packaging.python.org/en/latest/specifications/simple-repository-api/#versioning-pypi-s-simple-api - fail("Unsupported API version: {}".format(api_version)) - - for line in lines[1:]: - dist_url, _, tail = line.partition("#sha256=") - sha256, _, tail = tail.partition("\"") - - # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api - yanked = "data-yanked" in line - - maybe_metadata, _, tail = tail.partition(">") - filename, _, tail = tail.partition("<") - - metadata_sha256 = "" - metadata_url = "" - for metadata_marker in ["data-core-metadata", "data-dist-info-metadata"]: - metadata_marker = metadata_marker + "=\"sha256=" - if metadata_marker in maybe_metadata: - # Implement https://peps.python.org/pep-0714/ - _, _, tail = maybe_metadata.partition(metadata_marker) - metadata_sha256, _, _ = tail.partition("\"") - metadata_url = dist_url + ".metadata" - break - - if filename.endswith(".whl"): - whls[sha256] = struct( - filename = filename, - url = _absolute_url(url, dist_url), - sha256 = sha256, - metadata_sha256 = metadata_sha256, - metadata_url = _absolute_url(url, metadata_url), - yanked = yanked, - ) - else: - sdists[sha256] = struct( - filename = filename, - url = _absolute_url(url, dist_url), - sha256 = sha256, - metadata_sha256 = "", - metadata_url = "", - yanked = yanked, - ) - - return struct( - sdists = sdists, - whls = whls, - ) - -def _absolute_url(index_url, candidate): - if not candidate.startswith(".."): - return candidate - - candidate_parts = candidate.split("..") - last = candidate_parts[-1] - for _ in range(len(candidate_parts) - 1): - index_url, _, _ = index_url.rstrip("/").rpartition("/") - - return "{}/{}".format(index_url, last.strip("/")) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl new file mode 100644 index 0000000000..cae0db3e2b --- /dev/null +++ b/python/private/pypi/whl_library.bzl @@ -0,0 +1,509 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("//python:repositories.bzl", "is_standalone_interpreter") +load("//python:versions.bzl", "WINDOWS_NAME") +load("//python/pip_install:repositories.bzl", "all_requirements") +load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") +load("//python/private:envsubst.bzl", "envsubst") +load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") +load("//python/private:toolchains_repo.bzl", "get_host_os_arch") +load(":attrs.bzl", "ATTRS", "use_isolated") +load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") +load(":parse_whl_name.bzl", "parse_whl_name") +load(":patch_whl.bzl", "patch_whl") +load(":whl_target_platforms.bzl", "whl_target_platforms") + +_CPPFLAGS = "CPPFLAGS" +_COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools" +_WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point" + +def _construct_pypath(rctx): + """Helper function to construct a PYTHONPATH. + + Contains entries for code in this repo as well as packages downloaded from //python/pip_install:repositories.bzl. + This allows us to run python code inside repository rule implementations. + + Args: + rctx: Handle to the repository_context. + + Returns: String of the PYTHONPATH. + """ + + separator = ":" if not "windows" in rctx.os.name.lower() else ";" + pypath = separator.join([ + str(rctx.path(entry).dirname) + for entry in rctx.attr._python_path_entries + ]) + return pypath + +def _get_python_interpreter_attr(rctx): + """A helper function for getting the `python_interpreter` attribute or it's default + + Args: + rctx (repository_ctx): Handle to the rule repository context. + + Returns: + str: The attribute value or it's default + """ + if rctx.attr.python_interpreter: + return rctx.attr.python_interpreter + + if "win" in rctx.os.name: + return "python.exe" + else: + return "python3" + +def _resolve_python_interpreter(rctx): + """Helper function to find the python interpreter from the common attributes + + Args: + rctx: Handle to the rule repository context. + + Returns: + `path` object, for the resolved path to the Python interpreter. + """ + python_interpreter = _get_python_interpreter_attr(rctx) + + if rctx.attr.python_interpreter_target != None: + python_interpreter = rctx.path(rctx.attr.python_interpreter_target) + + (os, _) = get_host_os_arch(rctx) + + # On Windows, the symlink doesn't work because Windows attempts to find + # Python DLLs where the symlink is, not where the symlink points. + if os == WINDOWS_NAME: + python_interpreter = python_interpreter.realpath + elif "/" not in python_interpreter: + # It's a plain command, e.g. "python3", to look up in the environment. + found_python_interpreter = rctx.which(python_interpreter) + if not found_python_interpreter: + fail("python interpreter `{}` not found in PATH".format(python_interpreter)) + python_interpreter = found_python_interpreter + else: + python_interpreter = rctx.path(python_interpreter) + return python_interpreter + +def _get_xcode_location_cflags(rctx): + """Query the xcode sdk location to update cflags + + Figure out if this interpreter target comes from rules_python, and patch the xcode sdk location if so. + Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg + otherwise. See https://github.com/indygreg/python-build-standalone/issues/103 + """ + + # Only run on MacOS hosts + if not rctx.os.name.lower().startswith("mac os"): + return [] + + xcode_sdk_location = repo_utils.execute_unchecked( + rctx, + op = "GetXcodeLocation", + arguments = [repo_utils.which_checked(rctx, "xcode-select"), "--print-path"], + ) + if xcode_sdk_location.return_code != 0: + return [] + + xcode_root = xcode_sdk_location.stdout.strip() + if _COMMAND_LINE_TOOLS_PATH_SLUG not in xcode_root.lower(): + # This is a full xcode installation somewhere like /Applications/Xcode13.0.app/Contents/Developer + # so we need to change the path to to the macos specific tools which are in a different relative + # path than xcode installed command line tools. + xcode_root = "{}/Platforms/MacOSX.platform/Developer".format(xcode_root) + return [ + "-isysroot {}/SDKs/MacOSX.sdk".format(xcode_root), + ] + +def _get_toolchain_unix_cflags(rctx, python_interpreter): + """Gather cflags from a standalone toolchain for unix systems. + + Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg + otherwise. See https://github.com/indygreg/python-build-standalone/issues/103 + """ + + # Only run on Unix systems + if not rctx.os.name.lower().startswith(("mac os", "linux")): + return [] + + # Only update the location when using a standalone toolchain. + if not is_standalone_interpreter(rctx, python_interpreter): + return [] + + stdout = repo_utils.execute_checked_stdout( + rctx, + op = "GetPythonVersionForUnixCflags", + arguments = [ + python_interpreter, + "-c", + "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')", + ], + ) + _python_version = stdout + include_path = "{}/include/python{}".format( + python_interpreter.dirname, + _python_version, + ) + + return ["-isystem {}".format(include_path)] + +def _parse_optional_attrs(rctx, args, extra_pip_args = None): + """Helper function to parse common attributes of pip_repository and whl_library repository rules. + + This function also serializes the structured arguments as JSON + so they can be passed on the command line to subprocesses. + + Args: + rctx: Handle to the rule repository context. + args: A list of parsed args for the rule. + extra_pip_args: The pip args to pass. + Returns: Augmented args list. + """ + + if use_isolated(rctx, rctx.attr): + args.append("--isolated") + + # Bazel version 7.1.0 and later (and rolling releases from version 8.0.0-pre.20240128.3) + # support rctx.getenv(name, default): When building incrementally, any change to the value of + # the variable named by name will cause this repository to be re-fetched. + if "getenv" in dir(rctx): + getenv = rctx.getenv + else: + getenv = rctx.os.environ.get + + # Check for None so we use empty default types from our attrs. + # Some args want to be list, and some want to be dict. + if extra_pip_args != None: + args += [ + "--extra_pip_args", + json.encode(struct(arg = [ + envsubst(pip_arg, rctx.attr.envsubst, getenv) + for pip_arg in rctx.attr.extra_pip_args + ])), + ] + + if rctx.attr.download_only: + args.append("--download_only") + + if rctx.attr.pip_data_exclude != None: + args += [ + "--pip_data_exclude", + json.encode(struct(arg = rctx.attr.pip_data_exclude)), + ] + + if rctx.attr.enable_implicit_namespace_pkgs: + args.append("--enable_implicit_namespace_pkgs") + + if rctx.attr.environment != None: + args += [ + "--environment", + json.encode(struct(arg = rctx.attr.environment)), + ] + + return args + +def _create_repository_execution_environment(rctx, python_interpreter): + """Create a environment dictionary for processes we spawn with rctx.execute. + + Args: + rctx (repository_ctx): The repository context. + python_interpreter (path): The resolved python interpreter. + Returns: + Dictionary of environment variable suitable to pass to rctx.execute. + """ + + # Gather any available CPPFLAGS values + cppflags = [] + cppflags.extend(_get_xcode_location_cflags(rctx)) + cppflags.extend(_get_toolchain_unix_cflags(rctx, python_interpreter)) + + env = { + "PYTHONPATH": _construct_pypath(rctx), + _CPPFLAGS: " ".join(cppflags), + } + + return env + +def _whl_library_impl(rctx): + python_interpreter = _resolve_python_interpreter(rctx) + args = [ + python_interpreter, + "-m", + "python.pip_install.tools.wheel_installer.wheel_installer", + "--requirement", + rctx.attr.requirement, + ] + extra_pip_args = [] + extra_pip_args.extend(rctx.attr.extra_pip_args) + + # Manually construct the PYTHONPATH since we cannot use the toolchain here + environment = _create_repository_execution_environment(rctx, python_interpreter) + + whl_path = None + if rctx.attr.whl_file: + whl_path = rctx.path(rctx.attr.whl_file) + + # Simulate the behaviour where the whl is present in the current directory. + rctx.symlink(whl_path, whl_path.basename) + whl_path = rctx.path(whl_path.basename) + elif rctx.attr.urls: + filename = rctx.attr.filename + urls = rctx.attr.urls + if not filename: + _, _, filename = urls[0].rpartition("/") + + if not (filename.endswith(".whl") or filename.endswith("tar.gz") or filename.endswith(".zip")): + if rctx.attr.filename: + msg = "got '{}'".format(filename) + else: + msg = "detected '{}' from url:\n{}".format(filename, urls[0]) + fail("Only '.whl', '.tar.gz' or '.zip' files are supported, {}".format(msg)) + + result = rctx.download( + url = urls, + output = filename, + sha256 = rctx.attr.sha256, + auth = get_auth(rctx, urls), + ) + + if not result.success: + fail("could not download the '{}' from {}:\n{}".format(filename, urls, result)) + + if filename.endswith(".whl"): + whl_path = rctx.path(rctx.attr.filename) + else: + # It is an sdist and we need to tell PyPI to use a file in this directory + # and not use any indexes. + extra_pip_args.extend(["--no-index", "--find-links", "."]) + + args = _parse_optional_attrs(rctx, args, extra_pip_args) + + if not whl_path: + repo_utils.execute_checked( + rctx, + op = "whl_library.ResolveRequirement({}, {})".format(rctx.attr.name, rctx.attr.requirement), + arguments = args, + environment = environment, + quiet = rctx.attr.quiet, + timeout = rctx.attr.timeout, + ) + + whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"]) + if not rctx.delete("whl_file.json"): + fail("failed to delete the whl_file.json file") + + if rctx.attr.whl_patches: + patches = {} + for patch_file, json_args in rctx.attr.whl_patches.items(): + patch_dst = struct(**json.decode(json_args)) + if whl_path.basename in patch_dst.whls: + patches[patch_file] = patch_dst.patch_strip + + whl_path = patch_whl( + rctx, + op = "whl_library.PatchWhl({}, {})".format(rctx.attr.name, rctx.attr.requirement), + python_interpreter = python_interpreter, + whl_path = whl_path, + patches = patches, + quiet = rctx.attr.quiet, + timeout = rctx.attr.timeout, + ) + + target_platforms = rctx.attr.experimental_target_platforms + if target_platforms: + parsed_whl = parse_whl_name(whl_path.basename) + if parsed_whl.platform_tag != "any": + # NOTE @aignas 2023-12-04: if the wheel is a platform specific + # wheel, we only include deps for that target platform + target_platforms = [ + p.target_platform + for p in whl_target_platforms( + platform_tag = parsed_whl.platform_tag, + abi_tag = parsed_whl.abi_tag, + ) + ] + + repo_utils.execute_checked( + rctx, + op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path), + arguments = args + [ + "--whl-file", + whl_path, + ] + ["--platform={}".format(p) for p in target_platforms], + environment = environment, + quiet = rctx.attr.quiet, + timeout = rctx.attr.timeout, + ) + + metadata = json.decode(rctx.read("metadata.json")) + rctx.delete("metadata.json") + + # NOTE @aignas 2024-06-22: this has to live on until we stop supporting + # passing `twine` as a `:pkg` library via the `WORKSPACE` builds. + # + # See ../../packaging.bzl line 190 + entry_points = {} + for item in metadata["entry_points"]: + name = item["name"] + module = item["module"] + attribute = item["attribute"] + + # There is an extreme edge-case with entry_points that end with `.py` + # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174 + entry_point_without_py = name[:-3] + "_py" if name.endswith(".py") else name + entry_point_target_name = ( + _WHEEL_ENTRY_POINT_PREFIX + "_" + entry_point_without_py + ) + entry_point_script_name = entry_point_target_name + ".py" + + rctx.file( + entry_point_script_name, + _generate_entry_point_contents(module, attribute), + ) + entry_points[entry_point_without_py] = entry_point_script_name + + build_file_contents = generate_whl_library_build_bazel( + dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix), + whl_name = whl_path.basename, + dependencies = metadata["deps"], + dependencies_by_platform = metadata["deps_by_platform"], + group_name = rctx.attr.group_name, + group_deps = rctx.attr.group_deps, + data_exclude = rctx.attr.pip_data_exclude, + tags = [ + "pypi_name=" + metadata["name"], + "pypi_version=" + metadata["version"], + ], + entry_points = entry_points, + annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))), + ) + rctx.file("BUILD.bazel", build_file_contents) + + return + +def _generate_entry_point_contents( + module, + attribute, + shebang = "#!/usr/bin/env python3"): + """Generate the contents of an entry point script. + + Args: + module (str): The name of the module to use. + attribute (str): The name of the attribute to call. + shebang (str, optional): The shebang to use for the entry point python + file. + + Returns: + str: A string of python code. + """ + contents = """\ +{shebang} +import sys +from {module} import {attribute} +if __name__ == "__main__": + sys.exit({attribute}()) +""".format( + shebang = shebang, + module = module, + attribute = attribute, + ) + return contents + +# NOTE @aignas 2024-03-21: The usage of dict({}, **common) ensures that all args to `dict` are unique +whl_library_attrs = dict({ + "annotation": attr.label( + doc = ( + "Optional json encoded file containing annotation to apply to the extracted wheel. " + + "See `package_annotation`" + ), + allow_files = True, + ), + "dep_template": attr.string( + doc = """ +The dep template to use for referencing the dependencies. It should have `{name}` +and `{target}` tokens that will be replaced with the normalized distribution name +and the target that we need respectively. +""", + ), + "filename": attr.string( + doc = "Download the whl file to this filename. Only used when the `urls` is passed. If not specified, will be auto-detected from the `urls`.", + ), + "group_deps": attr.string_list( + doc = "List of dependencies to skip in order to break the cycles within a dependency group.", + default = [], + ), + "group_name": attr.string( + doc = "Name of the group, if any.", + ), + "repo": attr.string( + mandatory = True, + doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.", + ), + "repo_prefix": attr.string( + doc = """ +Prefix for the generated packages will be of the form `@//...` + +DEPRECATED. Only left for people who vendor requirements.bzl. +""", + ), + "requirement": attr.string( + mandatory = True, + doc = "Python requirement string describing the package to make available, if 'urls' or 'whl_file' is given, then this only needs to include foo[any_extras] as a bare minimum.", + ), + "sha256": attr.string( + doc = "The sha256 of the downloaded whl. Only used when the `urls` is passed.", + ), + "urls": attr.string_list( + doc = """\ +The list of urls of the whl to be downloaded using bazel downloader. Using this +attr makes `extra_pip_args` and `download_only` ignored.""", + ), + "whl_file": attr.label( + doc = "The whl file that should be used instead of downloading or building the whl.", + ), + "whl_patches": attr.label_keyed_string_dict( + doc = """a label-keyed-string dict that has + json.encode(struct([whl_file], patch_strip]) as values. This + is to maintain flexibility and correct bzlmod extension interface + until we have a better way to define whl_library and move whl + patching to a separate place. INTERNAL USE ONLY.""", + ), + "_python_path_entries": attr.label_list( + # Get the root directory of these rules and keep them as a default attribute + # in order to avoid unnecessary repository fetching restarts. + # + # This is very similar to what was done in https://github.com/bazelbuild/rules_go/pull/3478 + default = [ + Label("//:BUILD.bazel"), + ] + [ + # Includes all the external dependencies from repositories.bzl + Label("@" + repo + "//:BUILD.bazel") + for repo in all_requirements + ], + ), +}, **ATTRS) +whl_library_attrs.update(AUTH_ATTRS) + +whl_library = repository_rule( + attrs = whl_library_attrs, + doc = """ +Download and extracts a single wheel based into a bazel repo based on the requirement string passed in. +Instantiated from pip_repository and inherits config options from there.""", + implementation = _whl_library_impl, + environ = [ + "RULES_PYTHON_PIP_ISOLATED", + REPO_DEBUG_ENV_VAR, + ], +) diff --git a/python/private/pip_repo_name.bzl b/python/private/pypi/whl_repo_name.bzl similarity index 94% rename from python/private/pip_repo_name.bzl rename to python/private/pypi/whl_repo_name.bzl index bef4304e15..295f5a45c4 100644 --- a/python/private/pip_repo_name.bzl +++ b/python/private/pypi/whl_repo_name.bzl @@ -15,10 +15,10 @@ """A function to convert a dist name to a valid bazel repo name. """ -load(":normalize_name.bzl", "normalize_name") +load("//python/private:normalize_name.bzl", "normalize_name") load(":parse_whl_name.bzl", "parse_whl_name") -def pip_repo_name(prefix, filename, sha256): +def whl_repo_name(prefix, filename, sha256): """Return a valid whl_library repo name given a distribution filename. Args: diff --git a/python/private/whl_target_platforms.bzl b/python/private/pypi/whl_target_platforms.bzl similarity index 100% rename from python/private/whl_target_platforms.bzl rename to python/private/pypi/whl_target_platforms.bzl diff --git a/python/private/text_util.bzl b/python/private/text_util.bzl index dade9cba9b..702a08e281 100644 --- a/python/private/text_util.bzl +++ b/python/private/text_util.bzl @@ -36,6 +36,9 @@ def _render_alias(name, actual, *, visibility = None): ]) def _render_dict(d, *, key_repr = repr, value_repr = repr): + if not d: + return "{}" + return "\n".join([ "{", _indent("\n".join([ diff --git a/tests/private/envsubst/BUILD.bazel b/tests/envsubst/BUILD.bazel similarity index 100% rename from tests/private/envsubst/BUILD.bazel rename to tests/envsubst/BUILD.bazel diff --git a/tests/private/envsubst/envsubst_tests.bzl b/tests/envsubst/envsubst_tests.bzl similarity index 100% rename from tests/private/envsubst/envsubst_tests.bzl rename to tests/envsubst/envsubst_tests.bzl diff --git a/tests/pip_hub_repository/normalize_name/BUILD.bazel b/tests/normalize_name/BUILD.bazel similarity index 100% rename from tests/pip_hub_repository/normalize_name/BUILD.bazel rename to tests/normalize_name/BUILD.bazel diff --git a/tests/pip_hub_repository/normalize_name/normalize_name_tests.bzl b/tests/normalize_name/normalize_name_tests.bzl similarity index 100% rename from tests/pip_hub_repository/normalize_name/normalize_name_tests.bzl rename to tests/normalize_name/normalize_name_tests.bzl diff --git a/tests/pip_install/BUILD.bazel b/tests/pip_install/BUILD.bazel deleted file mode 100644 index 60d25de7df..0000000000 --- a/tests/pip_install/BUILD.bazel +++ /dev/null @@ -1,17 +0,0 @@ -load("@bazel_skylib//rules:diff_test.bzl", "diff_test") - -diff_test( - name = "srcs_diff_test", - failure_message = ( - "Please run 'bazel run //python/pip_install/private:srcs_module.update' " + - "to update the 'srcs.bzl' module found in the same package." - ), - file1 = "//python/pip_install/private:srcs_module", - file2 = "//python/pip_install/private:srcs.bzl", - # TODO: The diff_test here fails on Windows. As does the - # install script. This should be fixed. - target_compatible_with = select({ - "@platforms//os:windows": ["@platforms//:incompatible"], - "//conditions:default": [], - }), -) diff --git a/tests/pip_install/group_library/BUILD.bazel b/tests/pip_install/group_library/BUILD.bazel deleted file mode 100644 index 5a27e112db..0000000000 --- a/tests/pip_install/group_library/BUILD.bazel +++ /dev/null @@ -1,3 +0,0 @@ -load(":generate_build_bazel_tests.bzl", "generate_build_bazel_test_suite") - -generate_build_bazel_test_suite(name = "generate_build_bazel_tests") diff --git a/tests/pip_install/requirements_parser/BUILD.bazel b/tests/pip_install/requirements_parser/BUILD.bazel deleted file mode 100644 index 2787f16f94..0000000000 --- a/tests/pip_install/requirements_parser/BUILD.bazel +++ /dev/null @@ -1,3 +0,0 @@ -load(":requirements_parser_tests.bzl", parse_requirements_tests = "parse_tests") - -parse_requirements_tests(name = "test_parse_requirements") diff --git a/tests/pip_install/whl_library/BUILD.bazel b/tests/pip_install/whl_library/BUILD.bazel deleted file mode 100644 index 5a27e112db..0000000000 --- a/tests/pip_install/whl_library/BUILD.bazel +++ /dev/null @@ -1,3 +0,0 @@ -load(":generate_build_bazel_tests.bzl", "generate_build_bazel_test_suite") - -generate_build_bazel_test_suite(name = "generate_build_bazel_tests") diff --git a/tests/private/pip_config_settings/BUILD.bazel b/tests/private/pip_config_settings/BUILD.bazel deleted file mode 100644 index c3752e0ac3..0000000000 --- a/tests/private/pip_config_settings/BUILD.bazel +++ /dev/null @@ -1,5 +0,0 @@ -load(":pip_config_settings_tests.bzl", "pip_config_settings_test_suite") - -pip_config_settings_test_suite( - name = "pip_config_settings", -) diff --git a/tests/private/pip_repo_name/BUILD.bazel b/tests/private/pip_repo_name/BUILD.bazel deleted file mode 100644 index 7c6782daaf..0000000000 --- a/tests/private/pip_repo_name/BUILD.bazel +++ /dev/null @@ -1,3 +0,0 @@ -load(":pip_repo_name_tests.bzl", "pip_repo_name_test_suite") - -pip_repo_name_test_suite(name = "pip_repo_name_tests") diff --git a/tests/private/pypi_index/BUILD.bazel b/tests/private/pypi_index/BUILD.bazel deleted file mode 100644 index d365896cd3..0000000000 --- a/tests/private/pypi_index/BUILD.bazel +++ /dev/null @@ -1,3 +0,0 @@ -load(":pypi_index_tests.bzl", "pypi_index_test_suite") - -pypi_index_test_suite(name = "pypi_index_tests") diff --git a/tests/private/pypi_index_sources/BUILD.bazel b/tests/private/pypi_index_sources/BUILD.bazel deleted file mode 100644 index 212615f480..0000000000 --- a/tests/private/pypi_index_sources/BUILD.bazel +++ /dev/null @@ -1,3 +0,0 @@ -load(":pypi_index_sources_tests.bzl", "pypi_index_sources_test_suite") - -pypi_index_sources_test_suite(name = "pypi_index_sources_tests") diff --git a/tests/pypi/config_settings/BUILD.bazel b/tests/pypi/config_settings/BUILD.bazel new file mode 100644 index 0000000000..15dbd7f70e --- /dev/null +++ b/tests/pypi/config_settings/BUILD.bazel @@ -0,0 +1,5 @@ +load(":config_settings_tests.bzl", "config_settings_test_suite") + +config_settings_test_suite( + name = "config_settings_tests", +) diff --git a/tests/private/pip_config_settings/pip_config_settings_tests.bzl b/tests/pypi/config_settings/config_settings_tests.bzl similarity index 98% rename from tests/private/pip_config_settings/pip_config_settings_tests.bzl rename to tests/pypi/config_settings/config_settings_tests.bzl index a66e7f47d5..87e18b412f 100644 --- a/tests/private/pip_config_settings/pip_config_settings_tests.bzl +++ b/tests/pypi/config_settings/config_settings_tests.bzl @@ -17,7 +17,7 @@ load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:test_suite.bzl", "test_suite") load("@rules_testing//lib:truth.bzl", "subjects") load("@rules_testing//lib:util.bzl", test_util = "util") -load("//python/private:pip_config_settings.bzl", "pip_config_settings") # buildifier: disable=bzl-visibility +load("//python/private/pypi:config_settings.bzl", "config_settings") # buildifier: disable=bzl-visibility def _subject_impl(ctx): _ = ctx # @unused @@ -520,13 +520,13 @@ def _test_all(name): _tests.append(_test_all) -def pip_config_settings_test_suite(name): # buildifier: disable=function-docstring +def config_settings_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, tests = _tests, ) - pip_config_settings( + config_settings( name = "dummy", python_versions = ["3.8", "3.9", "3.10"], glibc_versions = [(2, 14), (2, 17)], diff --git a/tests/pypi/generate_group_library_build_bazel/BUILD.bazel b/tests/pypi/generate_group_library_build_bazel/BUILD.bazel new file mode 100644 index 0000000000..df5ab82320 --- /dev/null +++ b/tests/pypi/generate_group_library_build_bazel/BUILD.bazel @@ -0,0 +1,3 @@ +load(":generate_group_library_build_bazel_tests.bzl", "generate_group_library_build_bazel_test_suite") + +generate_group_library_build_bazel_test_suite(name = "generate_group_library_build_bazel_tests") diff --git a/tests/pip_install/group_library/generate_build_bazel_tests.bzl b/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl similarity index 91% rename from tests/pip_install/group_library/generate_build_bazel_tests.bzl rename to tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl index cf082c2990..a46aa413a3 100644 --- a/tests/pip_install/group_library/generate_build_bazel_tests.bzl +++ b/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl @@ -15,7 +15,7 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/pip_install/private:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel") # buildifier: disable=bzl-visibility +load("//python/private/pypi:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel") # buildifier: disable=bzl-visibility _tests = [] @@ -95,7 +95,7 @@ py_library( _tests.append(_test_in_hub) -def generate_build_bazel_test_suite(name): +def generate_group_library_build_bazel_test_suite(name): """Create the test suite. Args: diff --git a/tests/pypi/generate_whl_library_build_bazel/BUILD.bazel b/tests/pypi/generate_whl_library_build_bazel/BUILD.bazel new file mode 100644 index 0000000000..bea8e82ce3 --- /dev/null +++ b/tests/pypi/generate_whl_library_build_bazel/BUILD.bazel @@ -0,0 +1,3 @@ +load(":generate_whl_library_build_bazel_tests.bzl", "generate_whl_library_build_bazel_test_suite") + +generate_whl_library_build_bazel_test_suite(name = "generate_whl_library_build_bazel_tests") diff --git a/tests/pip_install/whl_library/generate_build_bazel_tests.bzl b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl similarity index 98% rename from tests/pip_install/whl_library/generate_build_bazel_tests.bzl rename to tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl index 62858afc94..3d4df14b5b 100644 --- a/tests/pip_install/whl_library/generate_build_bazel_tests.bzl +++ b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl @@ -15,7 +15,7 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/pip_install/private:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") # buildifier: disable=bzl-visibility +load("//python/private/pypi:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") # buildifier: disable=bzl-visibility _tests = [] @@ -569,7 +569,7 @@ config_setting( _tests.append(_test_group_member_deps_to_hub) -def generate_build_bazel_test_suite(name): +def generate_whl_library_build_bazel_test_suite(name): """Create the test suite. Args: diff --git a/tests/pypi/index_sources/BUILD.bazel b/tests/pypi/index_sources/BUILD.bazel new file mode 100644 index 0000000000..7cd327abef --- /dev/null +++ b/tests/pypi/index_sources/BUILD.bazel @@ -0,0 +1,3 @@ +load(":index_sources_tests.bzl", "index_sources_test_suite") + +index_sources_test_suite(name = "index_sources_tests") diff --git a/tests/private/pypi_index_sources/pypi_index_sources_tests.bzl b/tests/pypi/index_sources/index_sources_tests.bzl similarity index 88% rename from tests/private/pypi_index_sources/pypi_index_sources_tests.bzl rename to tests/pypi/index_sources/index_sources_tests.bzl index 48d790fc68..0a767078ba 100644 --- a/tests/private/pypi_index_sources/pypi_index_sources_tests.bzl +++ b/tests/pypi/index_sources/index_sources_tests.bzl @@ -15,7 +15,7 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:pypi_index_sources.bzl", "get_simpleapi_sources") # buildifier: disable=bzl-visibility +load("//python/private/pypi:index_sources.bzl", "index_sources") # buildifier: disable=bzl-visibility _tests = [] @@ -27,7 +27,7 @@ def _test_no_simple_api_sources(env): "foo==0.0.1 @ https://someurl.org; python_version < 2.7 --hash=sha256:deadbeef", ] for input in inputs: - got = get_simpleapi_sources(input) + got = index_sources(input) env.expect.that_collection(got.shas).contains_exactly([]) env.expect.that_str(got.version).equals("0.0.1") @@ -45,13 +45,13 @@ def _test_simple_api_sources(env): ], } for input, want_shas in tests.items(): - got = get_simpleapi_sources(input) + got = index_sources(input) env.expect.that_collection(got.shas).contains_exactly(want_shas) env.expect.that_str(got.version).equals("0.0.2") _tests.append(_test_simple_api_sources) -def pypi_index_sources_test_suite(name): +def index_sources_test_suite(name): """Create the test suite. Args: diff --git a/tests/private/parse_requirements/BUILD.bazel b/tests/pypi/parse_requirements/BUILD.bazel similarity index 100% rename from tests/private/parse_requirements/BUILD.bazel rename to tests/pypi/parse_requirements/BUILD.bazel diff --git a/tests/private/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl similarity index 99% rename from tests/private/parse_requirements/parse_requirements_tests.bzl rename to tests/pypi/parse_requirements/parse_requirements_tests.bzl index 81cf523460..13ce8f44ed 100644 --- a/tests/private/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -15,7 +15,7 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:parse_requirements.bzl", "parse_requirements", "select_requirement") # buildifier: disable=bzl-visibility +load("//python/private/pypi:parse_requirements.bzl", "parse_requirements", "select_requirement") # buildifier: disable=bzl-visibility def _mock_ctx(): testdata = { diff --git a/tests/pypi/parse_requirements_txt/BUILD.bazel b/tests/pypi/parse_requirements_txt/BUILD.bazel new file mode 100644 index 0000000000..526fa73d4b --- /dev/null +++ b/tests/pypi/parse_requirements_txt/BUILD.bazel @@ -0,0 +1,3 @@ +load(":parse_requirements_txt_tests.bzl", "parse_requirements_txt_test_suite") + +parse_requirements_txt_test_suite(name = "parse_requirements_txt_tests") diff --git a/tests/pip_install/requirements_parser/requirements_parser_tests.bzl b/tests/pypi/parse_requirements_txt/parse_requirements_txt_tests.bzl similarity index 88% rename from tests/pip_install/requirements_parser/requirements_parser_tests.bzl rename to tests/pypi/parse_requirements_txt/parse_requirements_txt_tests.bzl index 5ea742e70d..f4e899054a 100644 --- a/tests/pip_install/requirements_parser/requirements_parser_tests.bzl +++ b/tests/pypi/parse_requirements_txt/parse_requirements_txt_tests.bzl @@ -15,83 +15,83 @@ "Unit tests for yaml.bzl" load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") -load("//python/pip_install:requirements_parser.bzl", "parse") +load("//python/private/pypi:parse_requirements_txt.bzl", "parse_requirements_txt") # buildifier: disable=bzl-visibility def _parse_basic_test_impl(ctx): env = unittest.begin(ctx) # Base cases - asserts.equals(env, [], parse("").requirements) - asserts.equals(env, [], parse("\n").requirements) + asserts.equals(env, [], parse_requirements_txt("").requirements) + asserts.equals(env, [], parse_requirements_txt("\n").requirements) # Various requirement specifiers (https://pip.pypa.io/en/stable/reference/requirement-specifiers/#requirement-specifiers) - asserts.equals(env, [("SomeProject", "SomeProject")], parse("SomeProject\n").requirements) - asserts.equals(env, [("SomeProject", "SomeProject == 1.3")], parse("SomeProject == 1.3\n").requirements) - asserts.equals(env, [("SomeProject", "SomeProject >= 1.2, < 2.0")], parse("SomeProject >= 1.2, < 2.0\n").requirements) - asserts.equals(env, [("SomeProject", "SomeProject[foo, bar]")], parse("SomeProject[foo, bar]\n").requirements) - asserts.equals(env, [("SomeProject", "SomeProject ~= 1.4.2")], parse("SomeProject ~= 1.4.2\n").requirements) - asserts.equals(env, [("SomeProject", "SomeProject == 5.4 ; python_version < '3.8'")], parse("SomeProject == 5.4 ; python_version < '3.8'\n").requirements) - asserts.equals(env, [("SomeProject", "SomeProject ; sys_platform == 'win32'")], parse("SomeProject ; sys_platform == 'win32'\n").requirements) - asserts.equals(env, [("requests", "requests [security] >= 2.8.1, == 2.8.* ; python_version < 2.7")], parse("requests [security] >= 2.8.1, == 2.8.* ; python_version < 2.7\n").requirements) + asserts.equals(env, [("SomeProject", "SomeProject")], parse_requirements_txt("SomeProject\n").requirements) + asserts.equals(env, [("SomeProject", "SomeProject == 1.3")], parse_requirements_txt("SomeProject == 1.3\n").requirements) + asserts.equals(env, [("SomeProject", "SomeProject >= 1.2, < 2.0")], parse_requirements_txt("SomeProject >= 1.2, < 2.0\n").requirements) + asserts.equals(env, [("SomeProject", "SomeProject[foo, bar]")], parse_requirements_txt("SomeProject[foo, bar]\n").requirements) + asserts.equals(env, [("SomeProject", "SomeProject ~= 1.4.2")], parse_requirements_txt("SomeProject ~= 1.4.2\n").requirements) + asserts.equals(env, [("SomeProject", "SomeProject == 5.4 ; python_version < '3.8'")], parse_requirements_txt("SomeProject == 5.4 ; python_version < '3.8'\n").requirements) + asserts.equals(env, [("SomeProject", "SomeProject ; sys_platform == 'win32'")], parse_requirements_txt("SomeProject ; sys_platform == 'win32'\n").requirements) + asserts.equals(env, [("requests", "requests [security] >= 2.8.1, == 2.8.* ; python_version < 2.7")], parse_requirements_txt("requests [security] >= 2.8.1, == 2.8.* ; python_version < 2.7\n").requirements) # Multiple requirements - asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse("""\ + asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse_requirements_txt("""\ FooProject==1.0.0 BarProject==2.0.0 """).requirements) - asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse("""\ + asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse_requirements_txt("""\ FooProject==1.0.0 BarProject==2.0.0 """).requirements) # Comments - asserts.equals(env, [("SomeProject", "SomeProject")], parse("""\ + asserts.equals(env, [("SomeProject", "SomeProject")], parse_requirements_txt("""\ # This is a comment SomeProject """).requirements) - asserts.equals(env, [("SomeProject", "SomeProject")], parse("""\ + asserts.equals(env, [("SomeProject", "SomeProject")], parse_requirements_txt("""\ SomeProject # This is a comment """).requirements) - asserts.equals(env, [("SomeProject", "SomeProject == 1.3")], parse("""\ + asserts.equals(env, [("SomeProject", "SomeProject == 1.3")], parse_requirements_txt("""\ SomeProject == 1.3 # This is a comment """).requirements) - asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse("""\ + asserts.equals(env, [("FooProject", "FooProject==1.0.0"), ("BarProject", "BarProject==2.0.0")], parse_requirements_txt("""\ FooProject==1.0.0 # Comment BarProject==2.0.0 #Comment """).requirements) - asserts.equals(env, [("requests", "requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49")], parse("""\ + asserts.equals(env, [("requests", "requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49")], parse_requirements_txt("""\ requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49 """).requirements) # Multiline - asserts.equals(env, [("certifi", "certifi==2021.10.8 --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569")], parse("""\ + asserts.equals(env, [("certifi", "certifi==2021.10.8 --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569")], parse_requirements_txt("""\ certifi==2021.10.8 \ --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 \ --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569 # via requests """).requirements) - asserts.equals(env, [("requests", "requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49 --hash=sha256:eca58eb564b134e4ff521a02aa6f566c653835753e1fc8a50a20cb6bee4673cd")], parse("""\ + asserts.equals(env, [("requests", "requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49 --hash=sha256:eca58eb564b134e4ff521a02aa6f566c653835753e1fc8a50a20cb6bee4673cd")], parse_requirements_txt("""\ requests @ https://github.com/psf/requests/releases/download/v2.29.0/requests-2.29.0.tar.gz#sha1=3897c249b51a1a405d615a8c9cb92e5fdbf0dd49 \ --hash=sha256:eca58eb564b134e4ff521a02aa6f566c653835753e1fc8a50a20cb6bee4673cd # via requirements.txt """).requirements) # Options - asserts.equals(env, ["--pre"], parse("--pre\n").options) - asserts.equals(env, ["--find-links", "/my/local/archives"], parse("--find-links /my/local/archives\n").options) - asserts.equals(env, ["--pre", "--find-links", "/my/local/archives"], parse("""\ + asserts.equals(env, ["--pre"], parse_requirements_txt("--pre\n").options) + asserts.equals(env, ["--find-links", "/my/local/archives"], parse_requirements_txt("--find-links /my/local/archives\n").options) + asserts.equals(env, ["--pre", "--find-links", "/my/local/archives"], parse_requirements_txt("""\ --pre --find-links /my/local/archives """).options) - asserts.equals(env, ["--pre", "--find-links", "/my/local/archives"], parse("""\ + asserts.equals(env, ["--pre", "--find-links", "/my/local/archives"], parse_requirements_txt("""\ --pre # Comment --find-links /my/local/archives """).options) - asserts.equals(env, struct(requirements = [("FooProject", "FooProject==1.0.0")], options = ["--pre", "--find-links", "/my/local/archives"]), parse("""\ + asserts.equals(env, struct(requirements = [("FooProject", "FooProject==1.0.0")], options = ["--pre", "--find-links", "/my/local/archives"]), parse_requirements_txt("""\ --pre # Comment FooProject==1.0.0 --find-links /my/local/archives @@ -116,7 +116,7 @@ def _parse_requirements_lockfile_test_impl(ctx): ("urllib3", "urllib3==1.26.7 --hash=sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece --hash=sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"), ("yamllint", "yamllint==1.26.3 --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e"), ("setuptools", "setuptools==59.6.0 --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e"), - ], parse("""\ + ], parse_requirements_txt("""\ # # This file is autogenerated by pip-compile with python 3.9 # To update, run: @@ -220,5 +220,5 @@ parse_requirements_lockfile_test = unittest.make( attrs = {}, ) -def parse_tests(name): +def parse_requirements_txt_test_suite(name): unittest.suite(name, parse_basic_test, parse_requirements_lockfile_test) diff --git a/tests/pypi/parse_simpleapi_html/BUILD.bazel b/tests/pypi/parse_simpleapi_html/BUILD.bazel new file mode 100644 index 0000000000..e63ef0d5fa --- /dev/null +++ b/tests/pypi/parse_simpleapi_html/BUILD.bazel @@ -0,0 +1,3 @@ +load(":parse_simpleapi_html_tests.bzl", "parse_simpleapi_html_test_suite") + +parse_simpleapi_html_test_suite(name = "parse_simpleapi_html_tests") diff --git a/tests/private/pypi_index/pypi_index_tests.bzl b/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl similarity index 95% rename from tests/private/pypi_index/pypi_index_tests.bzl rename to tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl index fa381065b1..a60bb1f330 100644 --- a/tests/private/pypi_index/pypi_index_tests.bzl +++ b/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl @@ -16,7 +16,7 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("@rules_testing//lib:truth.bzl", "subjects") -load("//python/private:pypi_index.bzl", "parse_simple_api_html") # buildifier: disable=bzl-visibility +load("//python/private/pypi:parse_simpleapi_html.bzl", "parse_simpleapi_html") # buildifier: disable=bzl-visibility _tests = [] @@ -42,7 +42,7 @@ def _generate_html(*items): ]), ) -def _test_parse_simple_api_html(env): +def _test_sdist(env): # buildifier: disable=unsorted-dict-items tests = [ ( @@ -65,7 +65,7 @@ def _test_parse_simple_api_html(env): for (input, want) in tests: html = _generate_html(input) - got = parse_simple_api_html(url = input.url, content = html) + got = parse_simpleapi_html(url = input.url, content = html) env.expect.that_collection(got.sdists).has_size(1) env.expect.that_collection(got.whls).has_size(0) if not got: @@ -85,9 +85,9 @@ def _test_parse_simple_api_html(env): actual.url().equals(want.url) actual.yanked().equals(want.yanked) -_tests.append(_test_parse_simple_api_html) +_tests.append(_test_sdist) -def _test_parse_simple_api_html_whls(env): +def _test_whls(env): # buildifier: disable=unsorted-dict-items tests = [ ( @@ -189,7 +189,7 @@ def _test_parse_simple_api_html_whls(env): for (input, want) in tests: html = _generate_html(input) - got = parse_simple_api_html(url = input.url, content = html) + got = parse_simpleapi_html(url = input.url, content = html) env.expect.that_collection(got.sdists).has_size(0) env.expect.that_collection(got.whls).has_size(1) if not got: @@ -213,9 +213,9 @@ def _test_parse_simple_api_html_whls(env): actual.url().equals(want.url) actual.yanked().equals(want.yanked) -_tests.append(_test_parse_simple_api_html_whls) +_tests.append(_test_whls) -def pypi_index_test_suite(name): +def parse_simpleapi_html_test_suite(name): """Create the test suite. Args: diff --git a/tests/private/parse_whl_name/BUILD.bazel b/tests/pypi/parse_whl_name/BUILD.bazel similarity index 100% rename from tests/private/parse_whl_name/BUILD.bazel rename to tests/pypi/parse_whl_name/BUILD.bazel diff --git a/tests/private/parse_whl_name/parse_whl_name_tests.bzl b/tests/pypi/parse_whl_name/parse_whl_name_tests.bzl similarity index 96% rename from tests/private/parse_whl_name/parse_whl_name_tests.bzl rename to tests/pypi/parse_whl_name/parse_whl_name_tests.bzl index c249f9fb1a..4a88a6e7c5 100644 --- a/tests/private/parse_whl_name/parse_whl_name_tests.bzl +++ b/tests/pypi/parse_whl_name/parse_whl_name_tests.bzl @@ -15,7 +15,7 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:parse_whl_name.bzl", "parse_whl_name") # buildifier: disable=bzl-visibility +load("//python/private/pypi:parse_whl_name.bzl", "parse_whl_name") # buildifier: disable=bzl-visibility _tests = [] diff --git a/tests/pip_hub_repository/render_pkg_aliases/BUILD.bazel b/tests/pypi/render_pkg_aliases/BUILD.bazel similarity index 100% rename from tests/pip_hub_repository/render_pkg_aliases/BUILD.bazel rename to tests/pypi/render_pkg_aliases/BUILD.bazel diff --git a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl similarity index 98% rename from tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl rename to tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index da8a918aed..0d4c75e3c2 100644 --- a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -16,12 +16,9 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility +load("//python/private/pypi:config_settings.bzl", "config_settings") # buildifier: disable=bzl-visibility load( - "//python/private:pip_config_settings.bzl", - "pip_config_settings", -) # buildifier: disable=bzl-visibility -load( - "//python/private:render_pkg_aliases.bzl", + "//python/private/pypi:render_pkg_aliases.bzl", "get_filename_config_settings", "get_whl_flag_versions", "multiplatform_whl_aliases", @@ -182,10 +179,10 @@ alias( env.expect.that_str(actual.pop("_config/BUILD.bazel")).equals( """\ -load("@rules_python//python/private:pip_config_settings.bzl", "pip_config_settings") +load("@rules_python//python/private/pypi:config_settings.bzl", "config_settings") -pip_config_settings( - name = "pip_config_settings", +config_settings( + name = "config_settings", glibc_versions = [], muslc_versions = [], osx_versions = [], @@ -722,7 +719,6 @@ def _test_cp37_abi3_linux_x86_64(env): python_default = True, want = [ ":is_cp3x_abi3_linux_x86_64", - # TODO @aignas 2024-05-29: update the pip_config_settings to generate this ":is_cp3.2_cp3x_abi3_linux_x86_64", ], ) @@ -924,7 +920,7 @@ def _test_config_settings_exist(env): ] available_config_settings = [] mock_rule = lambda name, **kwargs: available_config_settings.append(name) - pip_config_settings( + config_settings( python_versions = ["3.11"], native = struct( alias = mock_rule, diff --git a/tests/pypi/whl_repo_name/BUILD.bazel b/tests/pypi/whl_repo_name/BUILD.bazel new file mode 100644 index 0000000000..8671dd7754 --- /dev/null +++ b/tests/pypi/whl_repo_name/BUILD.bazel @@ -0,0 +1,3 @@ +load(":whl_repo_name_tests.bzl", "whl_repo_name_test_suite") + +whl_repo_name_test_suite(name = "whl_repo_name_tests") diff --git a/tests/private/pip_repo_name/pip_repo_name_tests.bzl b/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl similarity index 82% rename from tests/private/pip_repo_name/pip_repo_name_tests.bzl rename to tests/pypi/whl_repo_name/whl_repo_name_tests.bzl index 574d558277..8b7df83530 100644 --- a/tests/private/pip_repo_name/pip_repo_name_tests.bzl +++ b/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl @@ -15,24 +15,24 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:pip_repo_name.bzl", "pip_repo_name") # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_repo_name.bzl", "whl_repo_name") # buildifier: disable=bzl-visibility _tests = [] def _test_simple(env): - got = pip_repo_name("prefix", "foo-1.2.3-py3-none-any.whl", "deadbeef") + got = whl_repo_name("prefix", "foo-1.2.3-py3-none-any.whl", "deadbeef") env.expect.that_str(got).equals("prefix_foo_py3_none_any_deadbeef") _tests.append(_test_simple) def _test_sdist(env): - got = pip_repo_name("prefix", "foo-1.2.3.tar.gz", "deadbeef000deadbeef") + got = whl_repo_name("prefix", "foo-1.2.3.tar.gz", "deadbeef000deadbeef") env.expect.that_str(got).equals("prefix_foo_sdist_deadbeef") _tests.append(_test_sdist) def _test_platform_whl(env): - got = pip_repo_name( + got = whl_repo_name( "prefix", "foo-1.2.3-cp39.cp310-abi3-manylinux1_x86_64.manylinux_2_17_x86_64.whl", "deadbeef000deadbeef", @@ -43,7 +43,7 @@ def _test_platform_whl(env): _tests.append(_test_platform_whl) -def pip_repo_name_test_suite(name): +def whl_repo_name_test_suite(name): """Create the test suite. Args: diff --git a/tests/private/whl_target_platforms/BUILD.bazel b/tests/pypi/whl_target_platforms/BUILD.bazel similarity index 100% rename from tests/private/whl_target_platforms/BUILD.bazel rename to tests/pypi/whl_target_platforms/BUILD.bazel diff --git a/tests/private/whl_target_platforms/select_whl_tests.bzl b/tests/pypi/whl_target_platforms/select_whl_tests.bzl similarity index 98% rename from tests/private/whl_target_platforms/select_whl_tests.bzl rename to tests/pypi/whl_target_platforms/select_whl_tests.bzl index 59e9d77918..3fd80e3ee3 100644 --- a/tests/private/whl_target_platforms/select_whl_tests.bzl +++ b/tests/pypi/whl_target_platforms/select_whl_tests.bzl @@ -15,7 +15,7 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:whl_target_platforms.bzl", "select_whls") # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_target_platforms.bzl", "select_whls") # buildifier: disable=bzl-visibility WHL_LIST = [ "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", diff --git a/tests/private/whl_target_platforms/whl_target_platforms_tests.bzl b/tests/pypi/whl_target_platforms/whl_target_platforms_tests.bzl similarity index 97% rename from tests/private/whl_target_platforms/whl_target_platforms_tests.bzl rename to tests/pypi/whl_target_platforms/whl_target_platforms_tests.bzl index 07f3158b31..a72bdc275f 100644 --- a/tests/private/whl_target_platforms/whl_target_platforms_tests.bzl +++ b/tests/pypi/whl_target_platforms/whl_target_platforms_tests.bzl @@ -15,7 +15,7 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:whl_target_platforms.bzl", "whl_target_platforms") # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_target_platforms.bzl", "whl_target_platforms") # buildifier: disable=bzl-visibility _tests = [] diff --git a/tests/private/text_util/BUILD.bazel b/tests/text_util/BUILD.bazel similarity index 100% rename from tests/private/text_util/BUILD.bazel rename to tests/text_util/BUILD.bazel diff --git a/tests/private/text_util/render_tests.bzl b/tests/text_util/render_tests.bzl similarity index 100% rename from tests/private/text_util/render_tests.bzl rename to tests/text_util/render_tests.bzl