From ea49937782fb0b969c72625f3b397f4bab53d412 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 23 Jun 2024 21:25:59 +0900 Subject: [PATCH] refactor: move the remaining PyPI related functions to private/pypi (#2006) A continuation of #2003, this time we finish moving the `pip_compile` stuff. Summary: - move multi_pip_parse to private/pypi and re-export - remove unused files leftover from #2003 - move repositories.bzl to private/pypi/deps.bzl - move pip_compile to private/pypi - move the pip_install tools to private --- BUILD.bazel | 1 - DEVELOPING.md | 2 +- MODULE.bazel | 4 +- examples/multi_python_versions/WORKSPACE | 4 - internal_setup.bzl | 4 +- python/BUILD.bazel | 15 +- python/pip.bzl | 245 +----------------- python/pip_install/BUILD.bazel | 30 +-- .../pip_repository_requirements.bzl.tmpl | 72 ----- python/pip_install/repositories.bzl | 129 +-------- python/pip_install/requirements.bzl | 147 +---------- .../tools/dependency_resolver/BUILD.bazel | 19 -- .../tools/wheel_installer/BUILD.bazel | 91 ------- python/private/bzlmod/internal_deps.bzl | 4 +- python/private/normalize_name.bzl | 1 - python/private/pypi/BUILD.bazel | 53 +++- .../pypi/dependency_resolver/BUILD.bazel | 7 + .../pypi}/dependency_resolver/__init__.py | 0 .../dependency_resolver.py | 0 python/private/pypi/deps.bzl | 143 ++++++++++ python/private/pypi/multi_pip_parse.bzl | 160 ++++++++++++ python/private/pypi/pip_compile.bzl | 167 ++++++++++++ .../tools => private/pypi}/requirements.txt | 0 python/private/pypi/whl_installer/BUILD.bazel | 36 +++ .../pypi/whl_installer}/arguments.py | 2 +- .../pypi/whl_installer}/namespace_pkgs.py | 0 .../pypi/whl_installer}/wheel.py | 0 .../pypi/whl_installer}/wheel_installer.py | 2 +- python/private/pypi/whl_library.bzl | 4 +- python/private/pypi/whl_library_alias.bzl | 99 +++++++ python/repositories.bzl | 4 +- .../compile_pip_requirements/WORKSPACE | 4 - .../WORKSPACE | 4 - tests/integration/pip_parse/WORKSPACE | 5 + tests/pypi/whl_installer/BUILD.bazel | 52 ++++ .../pypi/whl_installer}/arguments_test.py | 5 +- .../whl_installer}/namespace_pkgs_test.py | 2 +- .../whl_installer}/wheel_installer_test.py | 2 +- .../pypi/whl_installer}/wheel_test.py | 10 +- .../pycross/private/tools/BUILD.bazel | 2 +- .../pycross/private/tools/wheel_installer.py | 2 +- tools/private/update_deps/BUILD.bazel | 10 +- tools/private/update_deps/update_pip_deps.py | 24 +- 43 files changed, 777 insertions(+), 790 deletions(-) delete mode 100644 python/pip_install/pip_repository_requirements.bzl.tmpl delete mode 100644 python/pip_install/tools/dependency_resolver/BUILD.bazel delete mode 100644 python/pip_install/tools/wheel_installer/BUILD.bazel create mode 100644 python/private/pypi/dependency_resolver/BUILD.bazel rename python/{pip_install/tools => private/pypi}/dependency_resolver/__init__.py (100%) rename python/{pip_install/tools => private/pypi}/dependency_resolver/dependency_resolver.py (100%) create mode 100644 python/private/pypi/deps.bzl create mode 100644 python/private/pypi/multi_pip_parse.bzl create mode 100644 python/private/pypi/pip_compile.bzl rename python/{pip_install/tools => private/pypi}/requirements.txt (100%) create mode 100644 python/private/pypi/whl_installer/BUILD.bazel rename python/{pip_install/tools/wheel_installer => private/pypi/whl_installer}/arguments.py (98%) rename python/{pip_install/tools/wheel_installer => private/pypi/whl_installer}/namespace_pkgs.py (100%) rename python/{pip_install/tools/wheel_installer => private/pypi/whl_installer}/wheel.py (100%) rename python/{pip_install/tools/wheel_installer => private/pypi/whl_installer}/wheel_installer.py (98%) create mode 100644 python/private/pypi/whl_library_alias.bzl create mode 100644 tests/pypi/whl_installer/BUILD.bazel rename {python/pip_install/tools/wheel_installer => tests/pypi/whl_installer}/arguments_test.py (94%) rename {python/pip_install/tools/wheel_installer => tests/pypi/whl_installer}/namespace_pkgs_test.py (98%) rename {python/pip_install/tools/wheel_installer => tests/pypi/whl_installer}/wheel_installer_test.py (97%) rename {python/pip_install/tools/wheel_installer => tests/pypi/whl_installer}/wheel_test.py (97%) diff --git a/BUILD.bazel b/BUILD.bazel index c97f41dee..038b56a0c 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -40,7 +40,6 @@ filegroup( "internal_setup.bzl", "version.bzl", "//python:distribution", - "//python/pip_install:distribution", "//tools:distribution", "@rules_python_gazelle_plugin//:distribution", ], diff --git a/DEVELOPING.md b/DEVELOPING.md index 3c9e89d1d..a70d3b171 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -2,7 +2,7 @@ ## Updating internal dependencies -1. Modify the `./python/pip_install/tools/requirements.txt` file and run: +1. Modify the `./python/private/pypi/requirements.txt` file and run: ``` bazel run //tools/private/update_deps:update_pip_deps ``` diff --git a/MODULE.bazel b/MODULE.bazel index 3cdc47fb7..735600b98 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -16,8 +16,6 @@ bazel_dep(name = "protobuf", version = "24.4", repo_name = "com_google_protobuf" internal_deps = use_extension("//python/private/bzlmod:internal_deps.bzl", "internal_deps") use_repo( internal_deps, - "rules_python_internal", - # START: maintained by 'bazel run //tools/private/update_deps:update_pip_deps' "pypi__build", "pypi__click", "pypi__colorama", @@ -33,7 +31,7 @@ use_repo( "pypi__tomli", "pypi__wheel", "pypi__zipp", - # END: maintained by 'bazel run //tools/private/update_deps:update_pip_deps' + "rules_python_internal", ) # We need to do another use_extension call to expose the "pythons_hub" diff --git a/examples/multi_python_versions/WORKSPACE b/examples/multi_python_versions/WORKSPACE index f3a69ce76..4f731d95a 100644 --- a/examples/multi_python_versions/WORKSPACE +++ b/examples/multi_python_versions/WORKSPACE @@ -9,10 +9,6 @@ load("@rules_python//python:repositories.bzl", "py_repositories", "python_regist py_repositories() -load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies") - -pip_install_dependencies() - default_python_version = "3.9" python_register_multi_toolchains( diff --git a/internal_setup.bzl b/internal_setup.bzl index bb6261121..1967c0e56 100644 --- a/internal_setup.bzl +++ b/internal_setup.bzl @@ -22,8 +22,8 @@ load("@rules_bazel_integration_test//bazel_integration_test:deps.bzl", "bazel_in load("@rules_bazel_integration_test//bazel_integration_test:repo_defs.bzl", "bazel_binaries") load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") load("//:version.bzl", "SUPPORTED_BAZEL_VERSIONS") -load("//python/pip_install:repositories.bzl", "pip_install_dependencies") load("//python/private:internal_config_repo.bzl", "internal_config_repo") # buildifier: disable=bzl-visibility +load("//python/private/pypi:deps.bzl", "pypi_deps") # buildifier: disable=bzl-visibility def rules_python_internal_setup(): """Setup for rules_python tests and tools.""" @@ -31,7 +31,7 @@ def rules_python_internal_setup(): internal_config_repo(name = "rules_python_internal") # Because we don't use the pip_install rule, we have to call this to fetch its deps - pip_install_dependencies() + pypi_deps() bazel_skylib_workspace() diff --git a/python/BUILD.bazel b/python/BUILD.bazel index 29b495bf9..96b228222 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -39,6 +39,7 @@ filegroup( "//python/constraints:distribution", "//python/entry_points:distribution", "//python/extensions:distribution", + "//python/pip_install:distribution", "//python/private:distribution", "//python/runfiles:distribution", ], @@ -96,12 +97,12 @@ bzl_library( name = "pip_bzl", srcs = ["pip.bzl"], deps = [ - "//python/pip_install:pip_repository_bzl", - "//python/pip_install:repositories_bzl", - "//python/pip_install:requirements_bzl", - "//python/private:bzlmod_enabled_bzl", - "//python/private:full_version_bzl", - "//python/private/pypi:render_pkg_aliases_bzl", + "//python/private:normalize_name_bzl", + "//python/private/pypi:multi_pip_parse_bzl", + "//python/private/pypi:package_annotation_bzl", + "//python/private/pypi:pip_compile_bzl", + "//python/private/pypi:pip_repository_bzl", + "//python/private/pypi:whl_library_alias_bzl", "//python/private/whl_filegroup:whl_filegroup_bzl", ], ) @@ -210,7 +211,6 @@ bzl_library( srcs = ["repositories.bzl"], deps = [ ":versions_bzl", - "//python/pip_install:repositories_bzl", "//python/private:auth_bzl", "//python/private:bazel_tools_bzl", "//python/private:bzlmod_enabled_bzl", @@ -219,6 +219,7 @@ bzl_library( "//python/private:internal_config_repo_bzl", "//python/private:repo_utils_bzl", "//python/private:toolchains_repo_bzl", + "//python/private/pypi:deps_bzl", ], ) diff --git a/python/pip.bzl b/python/pip.bzl index f1c74dd96..a1a67200b 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -19,250 +19,27 @@ symbols should not be used and they are either undocumented here or marked as for internal use only. """ -load("//python/pip_install:requirements.bzl", _compile_pip_requirements = "compile_pip_requirements") -load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") -load("//python/private:full_version.bzl", "full_version") load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private/pypi:multi_pip_parse.bzl", _multi_pip_parse = "multi_pip_parse") load("//python/private/pypi:package_annotation.bzl", _package_annotation = "package_annotation") +load("//python/private/pypi:pip_compile.bzl", "pip_compile") load("//python/private/pypi:pip_repository.bzl", "pip_repository") -load("//python/private/pypi:render_pkg_aliases.bzl", "NO_MATCH_ERROR_MESSAGE_TEMPLATE") +load("//python/private/pypi:whl_library_alias.bzl", _whl_library_alias = "whl_library_alias") load("//python/private/whl_filegroup:whl_filegroup.bzl", _whl_filegroup = "whl_filegroup") -compile_pip_requirements = _compile_pip_requirements +compile_pip_requirements = pip_compile package_annotation = _package_annotation pip_parse = pip_repository whl_filegroup = _whl_filegroup -def _multi_pip_parse_impl(rctx): - rules_python = rctx.attr._rules_python_workspace.workspace_name - load_statements = [] - install_deps_calls = [] - process_requirements_calls = [] - for python_version, pypi_repository in rctx.attr.pip_parses.items(): - sanitized_python_version = python_version.replace(".", "_") - load_statement = """\ -load( - "@{pypi_repository}//:requirements.bzl", - _{sanitized_python_version}_install_deps = "install_deps", - _{sanitized_python_version}_all_requirements = "all_requirements", -)""".format( - pypi_repository = pypi_repository, - sanitized_python_version = sanitized_python_version, - ) - load_statements.append(load_statement) - process_requirements_call = """\ -_process_requirements( - pkg_labels = _{sanitized_python_version}_all_requirements, - python_version = "{python_version}", - repo_prefix = "{pypi_repository}_", -)""".format( - pypi_repository = pypi_repository, - python_version = python_version, - sanitized_python_version = sanitized_python_version, - ) - process_requirements_calls.append(process_requirements_call) - install_deps_call = """ _{sanitized_python_version}_install_deps(**whl_library_kwargs)""".format( - sanitized_python_version = sanitized_python_version, - ) - install_deps_calls.append(install_deps_call) - - # NOTE @aignas 2023-10-31: I am not sure it is possible to render aliases - # for all of the packages using the `render_pkg_aliases` function because - # we need to know what the list of packages for each version is and then - # we would be creating directories for each. - macro_tmpl = "@%s_{}//:{}" % rctx.attr.name - - requirements_bzl = """\ -# Generated by python/pip.bzl - -load("@{rules_python}//python:pip.bzl", "whl_library_alias", "pip_utils") -{load_statements} - -_wheel_names = [] -_version_map = dict() -def _process_requirements(pkg_labels, python_version, repo_prefix): - for pkg_label in pkg_labels: - wheel_name = Label(pkg_label).package - if not wheel_name: - # We are dealing with the cases where we don't have aliases. - workspace_name = Label(pkg_label).workspace_name - wheel_name = workspace_name[len(repo_prefix):] - - _wheel_names.append(wheel_name) - if not wheel_name in _version_map: - _version_map[wheel_name] = dict() - _version_map[wheel_name][python_version] = repo_prefix - -{process_requirements_calls} - -def requirement(name): - return "{macro_tmpl}".format(pip_utils.normalize_name(name), "pkg") - -def whl_requirement(name): - return "{macro_tmpl}".format(pip_utils.normalize_name(name), "whl") - -def data_requirement(name): - return "{macro_tmpl}".format(pip_utils.normalize_name(name), "data") - -def dist_info_requirement(name): - return "{macro_tmpl}".format(pip_utils.normalize_name(name), "dist_info") - -def install_deps(**whl_library_kwargs): -{install_deps_calls} - for wheel_name in _wheel_names: - whl_library_alias( - name = "{name}_" + wheel_name, - wheel_name = wheel_name, - default_version = "{default_version}", - version_map = _version_map[wheel_name], - ) -""".format( - name = rctx.attr.name, - install_deps_calls = "\n".join(install_deps_calls), - load_statements = "\n".join(load_statements), - macro_tmpl = macro_tmpl, - process_requirements_calls = "\n".join(process_requirements_calls), - rules_python = rules_python, - default_version = rctx.attr.default_version, - ) - rctx.file("requirements.bzl", requirements_bzl) - rctx.file("BUILD.bazel", "exports_files(['requirements.bzl'])") - -_multi_pip_parse = repository_rule( - _multi_pip_parse_impl, - attrs = { - "default_version": attr.string(), - "pip_parses": attr.string_dict(), - "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")), - }, -) - -def _whl_library_alias_impl(rctx): - rules_python = rctx.attr._rules_python_workspace.workspace_name - if rctx.attr.default_version: - default_repo_prefix = rctx.attr.version_map[rctx.attr.default_version] - else: - default_repo_prefix = None - version_map = rctx.attr.version_map.items() - build_content = ["# Generated by python/pip.bzl"] - for alias_name in ["pkg", "whl", "data", "dist_info"]: - build_content.append(_whl_library_render_alias_target( - alias_name = alias_name, - default_repo_prefix = default_repo_prefix, - rules_python = rules_python, - version_map = version_map, - wheel_name = rctx.attr.wheel_name, - )) - rctx.file("BUILD.bazel", "\n".join(build_content)) - -def _whl_library_render_alias_target( - alias_name, - default_repo_prefix, - rules_python, - version_map, - wheel_name): - # The template below adds one @, but under bzlmod, the name - # is canonical, so we have to add a second @. - if BZLMOD_ENABLED: - rules_python = "@" + rules_python - - alias = ["""\ -alias( - name = "{alias_name}", - actual = select({{""".format(alias_name = alias_name)] - for [python_version, repo_prefix] in version_map: - alias.append("""\ - "@{rules_python}//python/config_settings:is_python_{full_python_version}": "{actual}",""".format( - full_python_version = full_version(python_version), - actual = "@{repo_prefix}{wheel_name}//:{alias_name}".format( - repo_prefix = repo_prefix, - wheel_name = wheel_name, - alias_name = alias_name, - ), - rules_python = rules_python, - )) - if default_repo_prefix: - default_actual = "@{repo_prefix}{wheel_name}//:{alias_name}".format( - repo_prefix = default_repo_prefix, - wheel_name = wheel_name, - alias_name = alias_name, - ) - alias.append(' "//conditions:default": "{default_actual}",'.format( - default_actual = default_actual, - )) - - alias.append(" },") # Close select expression condition dict - if not default_repo_prefix: - supported_versions = sorted([python_version for python_version, _ in version_map]) - alias.append(' no_match_error="""{}""",'.format( - NO_MATCH_ERROR_MESSAGE_TEMPLATE.format( - supported_versions = ", ".join(supported_versions), - rules_python = rules_python, - ), - )) - alias.append(" ),") # Close the select expression - alias.append(' visibility = ["//visibility:public"],') - alias.append(")") # Close the alias() expression - return "\n".join(alias) - -whl_library_alias = repository_rule( - _whl_library_alias_impl, - attrs = { - "default_version": attr.string( - mandatory = False, - doc = "Optional Python version in major.minor format, e.g. '3.10'." + - "The Python version of the wheel to use when the versions " + - "from `version_map` don't match. This allows the default " + - "(version unaware) rules to match and select a wheel. If " + - "not specified, then the default rules won't be able to " + - "resolve a wheel and an error will occur.", - ), - "version_map": attr.string_dict(mandatory = True), - "wheel_name": attr.string(mandatory = True), - "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")), - }, -) - -def multi_pip_parse(name, default_version, python_versions, python_interpreter_target, requirements_lock, **kwargs): - """NOT INTENDED FOR DIRECT USE! - - This is intended to be used by the multi_pip_parse implementation in the template of the - multi_toolchain_aliases repository rule. - - Args: - name: the name of the multi_pip_parse repository. - default_version: the default Python version. - python_versions: all Python toolchain versions currently registered. - python_interpreter_target: a dictionary which keys are Python versions and values are resolved host interpreters. - requirements_lock: a dictionary which keys are Python versions and values are locked requirements files. - **kwargs: extra arguments passed to all wrapped pip_parse. - - Returns: - The internal implementation of multi_pip_parse repository rule. - """ - pip_parses = {} - for python_version in python_versions: - if not python_version in python_interpreter_target: - fail("Missing python_interpreter_target for Python version %s in '%s'" % (python_version, name)) - if not python_version in requirements_lock: - fail("Missing requirements_lock for Python version %s in '%s'" % (python_version, name)) - - pip_parse_name = name + "_" + python_version.replace(".", "_") - pip_parse( - name = pip_parse_name, - python_interpreter_target = python_interpreter_target[python_version], - requirements_lock = requirements_lock[python_version], - **kwargs - ) - pip_parses[python_version] = pip_parse_name - - return _multi_pip_parse( - name = name, - default_version = default_version, - pip_parses = pip_parses, - ) - # Extra utilities visible to rules_python users. pip_utils = struct( normalize_name = normalize_name, ) + +# The following are only exported here because they are used from +# multi_toolchain_aliases repository_rule, not intended for public use. +# +# See ./private/toolchains_repo.bzl +multi_pip_parse = _multi_pip_parse +whl_library_alias = _whl_library_alias diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel index 1894c4d91..683199f80 100644 --- a/python/pip_install/BUILD.bazel +++ b/python/pip_install/BUILD.bazel @@ -32,43 +32,21 @@ bzl_library( bzl_library( name = "requirements_bzl", srcs = ["requirements.bzl"], - deps = [ - ":repositories_bzl", - "//python:defs_bzl", - ], + deps = ["//python/private/pypi:pip_compile_bzl"], ) bzl_library( name = "repositories_bzl", srcs = ["repositories.bzl"], deps = [ - "//:version_bzl", - "//python/private:bazel_tools_bzl", - "@bazel_skylib//lib:versions", + "//python/private/pypi:deps_bzl", ], ) filegroup( name = "distribution", - srcs = glob(["*.bzl"]) + [ - "BUILD.bazel", - "pip_repository_requirements.bzl.tmpl", - "//python/pip_install/tools/dependency_resolver:distribution", - "//python/pip_install/tools/wheel_installer:distribution", - ], - visibility = ["//:__pkg__"], -) - -filegroup( - name = "repositories", - srcs = ["repositories.bzl"], - visibility = ["//tools/private/update_deps:__pkg__"], -) - -filegroup( - name = "requirements_txt", - srcs = ["tools/requirements.txt"], - visibility = ["//tools/private/update_deps:__pkg__"], + srcs = glob(["**"]), + visibility = ["//python:__pkg__"], ) filegroup( diff --git a/python/pip_install/pip_repository_requirements.bzl.tmpl b/python/pip_install/pip_repository_requirements.bzl.tmpl deleted file mode 100644 index 2f4bcd691..000000000 --- a/python/pip_install/pip_repository_requirements.bzl.tmpl +++ /dev/null @@ -1,72 +0,0 @@ -"""Starlark representation of locked requirements. - -@generated by rules_python pip_parse repository rule. -""" - -%%IMPORTS%% - -all_requirements = %%ALL_REQUIREMENTS%% - -all_whl_requirements_by_package = %%ALL_WHL_REQUIREMENTS_BY_PACKAGE%% - -all_whl_requirements = all_whl_requirements_by_package.values() - -all_data_requirements = %%ALL_DATA_REQUIREMENTS%% - -_packages = %%PACKAGES%% -_config = %%CONFIG%% -_annotations = %%ANNOTATIONS%% - -def requirement(name): - return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "pkg") - -def whl_requirement(name): - return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "whl") - -def data_requirement(name): - return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "data") - -def dist_info_requirement(name): - return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "dist_info") - -def _get_annotation(requirement): - # This expects to parse `setuptools==58.2.0 --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11` - # down to `setuptools`. - name = requirement.split(" ")[0].split("=")[0].split("[")[0] - return _annotations.get(name) - -def install_deps(**whl_library_kwargs): - """Repository rule macro. Install dependencies from `pip_parse`. - - Args: - **whl_library_kwargs: Additional arguments which will flow to underlying - `whl_library` calls. See pip_repository.bzl for details. - """ - - # Set up the requirement groups - all_requirement_groups = %%ALL_REQUIREMENT_GROUPS%% - - requirement_group_mapping = { - requirement: group_name - for group_name, group_requirements in all_requirement_groups.items() - for requirement in group_requirements - } - - # %%GROUP_LIBRARY%% - - # Install wheels which may be participants in a group - whl_config = dict(_config) - whl_config.update(whl_library_kwargs) - - for name, requirement in _packages: - group_name = requirement_group_mapping.get(name.replace("%%NAME%%_", "")) - group_deps = all_requirement_groups.get(group_name, []) - - whl_library( - name = name, - requirement = requirement, - group_name = group_name, - group_deps = group_deps, - annotation = _get_annotation(requirement), - **whl_config - ) diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index 3f91860b8..5231d1f0a 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -14,131 +14,6 @@ "" -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("//python/private/pypi:deps.bzl", "pypi_deps") -_RULE_DEPS = [ - # START: maintained by 'bazel run //tools/private/update_deps:update_pip_deps' - ( - "pypi__build", - "https://files.pythonhosted.org/packages/e2/03/f3c8ba0a6b6e30d7d18c40faab90807c9bb5e9a1e3b2fe2008af624a9c97/build-1.2.1-py3-none-any.whl", - "75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4", - ), - ( - "pypi__click", - "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", - "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", - ), - ( - "pypi__colorama", - "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", - "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", - ), - ( - "pypi__importlib_metadata", - "https://files.pythonhosted.org/packages/2d/0a/679461c511447ffaf176567d5c496d1de27cbe34a87df6677d7171b2fbd4/importlib_metadata-7.1.0-py3-none-any.whl", - "30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570", - ), - ( - "pypi__installer", - "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", - "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", - ), - ( - "pypi__more_itertools", - "https://files.pythonhosted.org/packages/50/e2/8e10e465ee3987bb7c9ab69efb91d867d93959095f4807db102d07995d94/more_itertools-10.2.0-py3-none-any.whl", - "686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684", - ), - ( - "pypi__packaging", - "https://files.pythonhosted.org/packages/49/df/1fceb2f8900f8639e278b056416d49134fb8d84c5942ffaa01ad34782422/packaging-24.0-py3-none-any.whl", - "2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", - ), - ( - "pypi__pep517", - "https://files.pythonhosted.org/packages/25/6e/ca4a5434eb0e502210f591b97537d322546e4833dcb4d470a48c375c5540/pep517-0.13.1-py3-none-any.whl", - "31b206f67165b3536dd577c5c3f1518e8fbaf38cbc57efff8369a392feff1721", - ), - ( - "pypi__pip", - "https://files.pythonhosted.org/packages/8a/6a/19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b/pip-24.0-py3-none-any.whl", - "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc", - ), - ( - "pypi__pip_tools", - "https://files.pythonhosted.org/packages/0d/dc/38f4ce065e92c66f058ea7a368a9c5de4e702272b479c0992059f7693941/pip_tools-7.4.1-py3-none-any.whl", - "4c690e5fbae2f21e87843e89c26191f0d9454f362d8acdbd695716493ec8b3a9", - ), - ( - "pypi__pyproject_hooks", - "https://files.pythonhosted.org/packages/ae/f3/431b9d5fe7d14af7a32340792ef43b8a714e7726f1d7b69cc4e8e7a3f1d7/pyproject_hooks-1.1.0-py3-none-any.whl", - "7ceeefe9aec63a1064c18d939bdc3adf2d8aa1988a510afec15151578b232aa2", - ), - ( - "pypi__setuptools", - "https://files.pythonhosted.org/packages/de/88/70c5767a0e43eb4451c2200f07d042a4bcd7639276003a9c54a68cfcc1f8/setuptools-70.0.0-py3-none-any.whl", - "54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4", - ), - ( - "pypi__tomli", - "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", - "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", - ), - ( - "pypi__wheel", - "https://files.pythonhosted.org/packages/7d/cd/d7460c9a869b16c3dd4e1e403cce337df165368c71d6af229a74699622ce/wheel-0.43.0-py3-none-any.whl", - "55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81", - ), - ( - "pypi__zipp", - "https://files.pythonhosted.org/packages/da/55/a03fd7240714916507e1fcf7ae355bd9d9ed2e6db492595f1a67f61681be/zipp-3.18.2-py3-none-any.whl", - "dce197b859eb796242b0622af1b8beb0a722d52aa2f57133ead08edd5bf5374e", - ), - # END: maintained by 'bazel run //tools/private/update_deps:update_pip_deps' -] - -_GENERIC_WHEEL = """\ -package(default_visibility = ["//visibility:public"]) - -load("@rules_python//python:defs.bzl", "py_library") - -py_library( - name = "lib", - srcs = glob(["**/*.py"]), - data = glob(["**/*"], exclude=[ - # These entries include those put into user-installed dependencies by - # data_exclude in /python/pip_install/tools/bazel.py - # to avoid non-determinism following pip install's behavior. - "**/*.py", - "**/*.pyc", - "**/*.pyc.*", # During pyc creation, temp files named *.pyc.NNN are created - "**/* *", - "**/*.dist-info/RECORD", - "BUILD", - "WORKSPACE", - ]), - # This makes this directory a top-level in the python import - # search path for anything that depends on this. - imports = ["."], -) -""" - -# Collate all the repository names so they can be easily consumed -all_requirements = [name for (name, _, _) in _RULE_DEPS] - -def requirement(pkg): - return Label("@pypi__" + pkg + "//:lib") - -def pip_install_dependencies(): - """ - Fetch dependencies these rules depend on. Workspaces that use the pip_parse rule can call this. - """ - for (name, url, sha256) in _RULE_DEPS: - maybe( - http_archive, - name, - url = url, - sha256 = sha256, - type = "zip", - build_file_content = _GENERIC_WHEEL, - ) +pip_install_dependencies = pypi_deps diff --git a/python/pip_install/requirements.bzl b/python/pip_install/requirements.bzl index 5caf7629f..6ae3f8fef 100644 --- a/python/pip_install/requirements.bzl +++ b/python/pip_install/requirements.bzl @@ -14,149 +14,6 @@ """Rules to verify and update pip-compile locked requirements.txt""" -load("//python:defs.bzl", _py_binary = "py_binary", _py_test = "py_test") -load("//python/pip_install:repositories.bzl", "requirement") +load("//python/private/pypi:pip_compile.bzl", "pip_compile") -def compile_pip_requirements( - name, - src = None, - extra_args = [], - extra_deps = [], - generate_hashes = True, - py_binary = _py_binary, - py_test = _py_test, - requirements_in = None, - requirements_txt = None, - requirements_darwin = None, - requirements_linux = None, - requirements_windows = None, - visibility = ["//visibility:private"], - tags = None, - **kwargs): - """Generates targets for managing pip dependencies with pip-compile. - - By default this rules generates a filegroup named "[name]" which can be included in the data - of some other compile_pip_requirements rule that references these requirements - (e.g. with `-r ../other/requirements.txt`). - - It also generates two targets for running pip-compile: - - - validate with `bazel test [name]_test` - - update with `bazel run [name].update` - - If you are using a version control system, the requirements.txt generated by this rule should - be checked into it to ensure that all developers/users have the same dependency versions. - - Args: - name: base name for generated targets, typically "requirements". - src: file containing inputs to dependency resolution. If not specified, - defaults to `pyproject.toml`. Supported formats are: - * a requirements text file, usually named `requirements.in` - * A `.toml` file, where the `project.dependencies` list is used as per - [PEP621](https://peps.python.org/pep-0621/). - extra_args: passed to pip-compile. - extra_deps: extra dependencies passed to pip-compile. - generate_hashes: whether to put hashes in the requirements_txt file. - py_binary: the py_binary rule to be used. - py_test: the py_test rule to be used. - requirements_in: file expressing desired dependencies. Deprecated, use src instead. - requirements_txt: result of "compiling" the requirements.in file. - requirements_linux: File of linux specific resolve output to check validate if requirement.in has changes. - requirements_darwin: File of darwin specific resolve output to check validate if requirement.in has changes. - requirements_windows: File of windows specific resolve output to check validate if requirement.in has changes. - tags: tagging attribute common to all build rules, passed to both the _test and .update rules. - visibility: passed to both the _test and .update rules. - **kwargs: other bazel attributes passed to the "_test" rule. - """ - if requirements_in and src: - fail("Only one of 'src' and 'requirements_in' attributes can be used") - else: - src = requirements_in or src or "pyproject.toml" - - requirements_txt = name + ".txt" if requirements_txt == None else requirements_txt - - # "Default" target produced by this macro - # Allow a compile_pip_requirements rule to include another one in the data - # for a requirements file that does `-r ../other/requirements.txt` - native.filegroup( - name = name, - srcs = kwargs.pop("data", []) + [requirements_txt], - visibility = visibility, - ) - - data = [name, requirements_txt, src] + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None] - - # Use the Label constructor so this is expanded in the context of the file - # where it appears, which is to say, in @rules_python - pip_compile = Label("//python/pip_install/tools/dependency_resolver:dependency_resolver.py") - - loc = "$(rlocationpath {})" - - args = [ - loc.format(src), - loc.format(requirements_txt), - "//%s:%s.update" % (native.package_name(), name), - "--resolver=backtracking", - "--allow-unsafe", - ] - if generate_hashes: - args.append("--generate-hashes") - if requirements_linux: - args.append("--requirements-linux={}".format(loc.format(requirements_linux))) - if requirements_darwin: - args.append("--requirements-darwin={}".format(loc.format(requirements_darwin))) - if requirements_windows: - args.append("--requirements-windows={}".format(loc.format(requirements_windows))) - args.extend(extra_args) - - deps = [ - requirement("build"), - requirement("click"), - requirement("colorama"), - requirement("importlib_metadata"), - requirement("more_itertools"), - requirement("packaging"), - requirement("pep517"), - requirement("pip"), - requirement("pip_tools"), - requirement("pyproject_hooks"), - requirement("setuptools"), - requirement("tomli"), - requirement("zipp"), - Label("//python/runfiles:runfiles"), - ] + extra_deps - - tags = tags or [] - tags.append("requires-network") - tags.append("no-remote-exec") - tags.append("no-sandbox") - attrs = { - "args": args, - "data": data, - "deps": deps, - "main": pip_compile, - "srcs": [pip_compile], - "tags": tags, - "visibility": visibility, - } - - # cheap way to detect the bazel version - _bazel_version_4_or_greater = "propeller_optimize" in dir(native) - - # Bazel 4.0 added the "env" attribute to py_test/py_binary - if _bazel_version_4_or_greater: - attrs["env"] = kwargs.pop("env", {}) - - py_binary( - name = name + ".update", - **attrs - ) - - timeout = kwargs.pop("timeout", "short") - - py_test( - name = name + "_test", - timeout = timeout, - # kwargs could contain test-specific attributes like size or timeout - **dict(attrs, **kwargs) - ) +compile_pip_requirements = pip_compile diff --git a/python/pip_install/tools/dependency_resolver/BUILD.bazel b/python/pip_install/tools/dependency_resolver/BUILD.bazel deleted file mode 100644 index 467b00933..000000000 --- a/python/pip_install/tools/dependency_resolver/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -exports_files(["dependency_resolver.py"]) - -filegroup( - name = "distribution", - srcs = glob( - ["*"], - exclude = ["*_test.py"], - ), - visibility = ["//python/pip_install:__subpackages__"], -) - -filegroup( - name = "py_srcs", - srcs = glob( - include = ["**/*.py"], - exclude = ["**/*_test.py"], - ), - visibility = ["//:__subpackages__"], -) diff --git a/python/pip_install/tools/wheel_installer/BUILD.bazel b/python/pip_install/tools/wheel_installer/BUILD.bazel deleted file mode 100644 index 0c24d5a48..000000000 --- a/python/pip_install/tools/wheel_installer/BUILD.bazel +++ /dev/null @@ -1,91 +0,0 @@ -load("//python:defs.bzl", "py_binary", "py_library", "py_test") -load("//python/pip_install:repositories.bzl", "requirement") - -py_library( - name = "lib", - srcs = [ - "arguments.py", - "namespace_pkgs.py", - "wheel.py", - "wheel_installer.py", - ], - visibility = ["//third_party/rules_pycross/pycross/private:__subpackages__"], - deps = [ - requirement("installer"), - requirement("pip"), - requirement("packaging"), - requirement("setuptools"), - ], -) - -py_binary( - name = "wheel_installer", - srcs = [ - "wheel_installer.py", - ], - deps = [":lib"], -) - -py_test( - name = "arguments_test", - size = "small", - srcs = [ - "arguments_test.py", - ], - deps = [ - ":lib", - ], -) - -py_test( - name = "namespace_pkgs_test", - size = "small", - srcs = [ - "namespace_pkgs_test.py", - ], - deps = [ - ":lib", - ], -) - -py_test( - name = "wheel_test", - size = "small", - srcs = [ - "wheel_test.py", - ], - data = ["//examples/wheel:minimal_with_py_package"], - deps = [ - ":lib", - ], -) - -py_test( - name = "wheel_installer_test", - size = "small", - srcs = [ - "wheel_installer_test.py", - ], - data = ["//examples/wheel:minimal_with_py_package"], - deps = [ - ":lib", - ], -) - -filegroup( - name = "distribution", - srcs = glob( - ["*"], - exclude = ["*_test.py"], - ), - visibility = ["//python/pip_install:__subpackages__"], -) - -filegroup( - name = "py_srcs", - srcs = glob( - include = ["**/*.py"], - exclude = ["**/*_test.py"], - ), - visibility = ["//:__subpackages__"], -) diff --git a/python/private/bzlmod/internal_deps.bzl b/python/private/bzlmod/internal_deps.bzl index 62ca71fec..e0eca9ef3 100644 --- a/python/private/bzlmod/internal_deps.bzl +++ b/python/private/bzlmod/internal_deps.bzl @@ -9,12 +9,12 @@ "Python toolchain module extension for internal rule use" load("@bazel_skylib//lib:modules.bzl", "modules") -load("//python/pip_install:repositories.bzl", "pip_install_dependencies") load("//python/private:internal_config_repo.bzl", "internal_config_repo") +load("//python/private/pypi:deps.bzl", "pypi_deps") def _internal_deps(): internal_config_repo(name = "rules_python_internal") - pip_install_dependencies() + pypi_deps() internal_deps = modules.as_extension( _internal_deps, diff --git a/python/private/normalize_name.bzl b/python/private/normalize_name.bzl index aaeca803b..7898222e4 100644 --- a/python/private/normalize_name.bzl +++ b/python/private/normalize_name.bzl @@ -38,7 +38,6 @@ Also see Python spec on normalizing package names: https://packaging.python.org/en/latest/specifications/name-normalization/ """ -# Keep in sync with ../pip_install/tools/lib/bazel.py def normalize_name(name): """normalize a PyPI package name and return a valid bazel label. diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 1530837f7..e7ae73553 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -21,7 +21,13 @@ licenses(["notice"]) filegroup( name = "distribution", - srcs = glob(["**"]), + srcs = glob( + ["**"], + exclude = ["requirements.txt"], + ) + [ + "//python/private/pypi/dependency_resolver:distribution", + "//python/private/pypi/whl_installer:distribution", + ], visibility = ["//python/private:__pkg__"], ) @@ -29,7 +35,16 @@ filegroup( filegroup( name = "bzl", srcs = glob(["**/*.bzl"]), - visibility = ["//python/private:__pkg__"], + visibility = [ + "//python/private:__pkg__", + "//tools/private:__pkg__", + ], +) + +filegroup( + name = "requirements_txt", + srcs = ["requirements.txt"], + visibility = ["//tools/private/update_deps:__pkg__"], ) # Keep sorted by library name and keep the files named by the main symbol they export @@ -66,6 +81,14 @@ bzl_library( deps = ["flags_bzl"], ) +bzl_library( + name = "deps_bzl", + srcs = ["deps.bzl"], + deps = [ + "//python/private:bazel_tools_bzl", + ], +) + bzl_library( name = "flags_bzl", srcs = ["flags.bzl"], @@ -118,6 +141,12 @@ bzl_library( srcs = ["labels.bzl"], ) +bzl_library( + name = "multi_pip_parse_bzl", + srcs = ["multi_pip_parse.bzl"], + deps = ["pip_repository_bzl"], +) + bzl_library( name = "package_annotation_bzl", srcs = ["package_annotation.bzl"], @@ -158,6 +187,15 @@ bzl_library( ], ) +bzl_library( + name = "pip_compile_bzl", + srcs = ["pip_compile.bzl"], + deps = [ + ":deps_bzl", + "//python:defs_bzl", + ], +) + bzl_library( name = "pip_repository_bzl", srcs = ["pip_repository.bzl"], @@ -203,18 +241,27 @@ bzl_library( ], ) +bzl_library( + name = "whl_library_alias_bzl", + srcs = ["whl_library_alias.bzl"], + deps = [ + ":render_pkg_aliases_bzl", + "//python/private:full_version_bzl", + ], +) + bzl_library( name = "whl_library_bzl", srcs = ["whl_library.bzl"], deps = [ ":attrs_bzl", + ":deps_bzl", ":generate_whl_library_build_bazel_bzl", ":parse_whl_name_bzl", ":patch_whl_bzl", ":whl_target_platforms_bzl", "//python:repositories_bzl", "//python:versions_bzl", - "//python/pip_install:repositories_bzl", "//python/private:auth_bzl", "//python/private:envsubst_bzl", "//python/private:repo_utils_bzl", diff --git a/python/private/pypi/dependency_resolver/BUILD.bazel b/python/private/pypi/dependency_resolver/BUILD.bazel new file mode 100644 index 000000000..9531b5555 --- /dev/null +++ b/python/private/pypi/dependency_resolver/BUILD.bazel @@ -0,0 +1,7 @@ +exports_files(["dependency_resolver.py"]) + +filegroup( + name = "distribution", + srcs = glob(["**"]), + visibility = ["//python/private/pypi:__subpackages__"], +) diff --git a/python/pip_install/tools/dependency_resolver/__init__.py b/python/private/pypi/dependency_resolver/__init__.py similarity index 100% rename from python/pip_install/tools/dependency_resolver/__init__.py rename to python/private/pypi/dependency_resolver/__init__.py diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/private/pypi/dependency_resolver/dependency_resolver.py similarity index 100% rename from python/pip_install/tools/dependency_resolver/dependency_resolver.py rename to python/private/pypi/dependency_resolver/dependency_resolver.py diff --git a/python/private/pypi/deps.bzl b/python/private/pypi/deps.bzl new file mode 100644 index 000000000..81bef7aaa --- /dev/null +++ b/python/private/pypi/deps.bzl @@ -0,0 +1,143 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +_RULE_DEPS = [ + # START: maintained by 'bazel run //tools/private/update_deps:update_pip_deps' + ( + "pypi__build", + "https://files.pythonhosted.org/packages/e2/03/f3c8ba0a6b6e30d7d18c40faab90807c9bb5e9a1e3b2fe2008af624a9c97/build-1.2.1-py3-none-any.whl", + "75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4", + ), + ( + "pypi__click", + "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", + "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", + ), + ( + "pypi__colorama", + "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", + "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", + ), + ( + "pypi__importlib_metadata", + "https://files.pythonhosted.org/packages/2d/0a/679461c511447ffaf176567d5c496d1de27cbe34a87df6677d7171b2fbd4/importlib_metadata-7.1.0-py3-none-any.whl", + "30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570", + ), + ( + "pypi__installer", + "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", + "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", + ), + ( + "pypi__more_itertools", + "https://files.pythonhosted.org/packages/50/e2/8e10e465ee3987bb7c9ab69efb91d867d93959095f4807db102d07995d94/more_itertools-10.2.0-py3-none-any.whl", + "686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684", + ), + ( + "pypi__packaging", + "https://files.pythonhosted.org/packages/49/df/1fceb2f8900f8639e278b056416d49134fb8d84c5942ffaa01ad34782422/packaging-24.0-py3-none-any.whl", + "2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", + ), + ( + "pypi__pep517", + "https://files.pythonhosted.org/packages/25/6e/ca4a5434eb0e502210f591b97537d322546e4833dcb4d470a48c375c5540/pep517-0.13.1-py3-none-any.whl", + "31b206f67165b3536dd577c5c3f1518e8fbaf38cbc57efff8369a392feff1721", + ), + ( + "pypi__pip", + "https://files.pythonhosted.org/packages/8a/6a/19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b/pip-24.0-py3-none-any.whl", + "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc", + ), + ( + "pypi__pip_tools", + "https://files.pythonhosted.org/packages/0d/dc/38f4ce065e92c66f058ea7a368a9c5de4e702272b479c0992059f7693941/pip_tools-7.4.1-py3-none-any.whl", + "4c690e5fbae2f21e87843e89c26191f0d9454f362d8acdbd695716493ec8b3a9", + ), + ( + "pypi__pyproject_hooks", + "https://files.pythonhosted.org/packages/ae/f3/431b9d5fe7d14af7a32340792ef43b8a714e7726f1d7b69cc4e8e7a3f1d7/pyproject_hooks-1.1.0-py3-none-any.whl", + "7ceeefe9aec63a1064c18d939bdc3adf2d8aa1988a510afec15151578b232aa2", + ), + ( + "pypi__setuptools", + "https://files.pythonhosted.org/packages/de/88/70c5767a0e43eb4451c2200f07d042a4bcd7639276003a9c54a68cfcc1f8/setuptools-70.0.0-py3-none-any.whl", + "54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4", + ), + ( + "pypi__tomli", + "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", + "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + ), + ( + "pypi__wheel", + "https://files.pythonhosted.org/packages/7d/cd/d7460c9a869b16c3dd4e1e403cce337df165368c71d6af229a74699622ce/wheel-0.43.0-py3-none-any.whl", + "55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81", + ), + ( + "pypi__zipp", + "https://files.pythonhosted.org/packages/da/55/a03fd7240714916507e1fcf7ae355bd9d9ed2e6db492595f1a67f61681be/zipp-3.18.2-py3-none-any.whl", + "dce197b859eb796242b0622af1b8beb0a722d52aa2f57133ead08edd5bf5374e", + ), + # END: maintained by 'bazel run //tools/private/update_deps:update_pip_deps' +] + +_GENERIC_WHEEL = """\ +package(default_visibility = ["//visibility:public"]) + +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "lib", + srcs = glob(["**/*.py"]), + data = glob(["**/*"], exclude=[ + # These entries include those put into user-installed dependencies by + # data_exclude to avoid non-determinism. + "**/*.py", + "**/*.pyc", + "**/*.pyc.*", # During pyc creation, temp files named *.pyc.NNN are created + "**/* *", + "**/*.dist-info/RECORD", + "BUILD", + "WORKSPACE", + ]), + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["."], +) +""" + +# Collate all the repository names so they can be easily consumed +all_requirements = [name for (name, _, _) in _RULE_DEPS] + +def requirement(pkg): + return Label("@pypi__" + pkg + "//:lib") + +def pypi_deps(): + """ + Fetch dependencies these rules depend on. Workspaces that use the pip_parse rule can call this. + """ + for (name, url, sha256) in _RULE_DEPS: + maybe( + http_archive, + name, + url = url, + sha256 = sha256, + type = "zip", + build_file_content = _GENERIC_WHEEL, + ) diff --git a/python/private/pypi/multi_pip_parse.bzl b/python/private/pypi/multi_pip_parse.bzl new file mode 100644 index 000000000..fe9e2db82 --- /dev/null +++ b/python/private/pypi/multi_pip_parse.bzl @@ -0,0 +1,160 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A pip_parse implementation for version aware toolchains in WORKSPACE.""" + +load(":pip_repository.bzl", pip_parse = "pip_repository") + +def _multi_pip_parse_impl(rctx): + rules_python = rctx.attr._rules_python_workspace.workspace_name + load_statements = [] + install_deps_calls = [] + process_requirements_calls = [] + for python_version, pypi_repository in rctx.attr.pip_parses.items(): + sanitized_python_version = python_version.replace(".", "_") + load_statement = """\ +load( + "@{pypi_repository}//:requirements.bzl", + _{sanitized_python_version}_install_deps = "install_deps", + _{sanitized_python_version}_all_requirements = "all_requirements", +)""".format( + pypi_repository = pypi_repository, + sanitized_python_version = sanitized_python_version, + ) + load_statements.append(load_statement) + process_requirements_call = """\ +_process_requirements( + pkg_labels = _{sanitized_python_version}_all_requirements, + python_version = "{python_version}", + repo_prefix = "{pypi_repository}_", +)""".format( + pypi_repository = pypi_repository, + python_version = python_version, + sanitized_python_version = sanitized_python_version, + ) + process_requirements_calls.append(process_requirements_call) + install_deps_call = """ _{sanitized_python_version}_install_deps(**whl_library_kwargs)""".format( + sanitized_python_version = sanitized_python_version, + ) + install_deps_calls.append(install_deps_call) + + # NOTE @aignas 2023-10-31: I am not sure it is possible to render aliases + # for all of the packages using the `render_pkg_aliases` function because + # we need to know what the list of packages for each version is and then + # we would be creating directories for each. + macro_tmpl = "@%s_{}//:{}" % rctx.attr.name + + requirements_bzl = """\ +# Generated by python/pip.bzl + +load("@{rules_python}//python:pip.bzl", "whl_library_alias", "pip_utils") +{load_statements} + +_wheel_names = [] +_version_map = dict() +def _process_requirements(pkg_labels, python_version, repo_prefix): + for pkg_label in pkg_labels: + wheel_name = Label(pkg_label).package + if not wheel_name: + # We are dealing with the cases where we don't have aliases. + workspace_name = Label(pkg_label).workspace_name + wheel_name = workspace_name[len(repo_prefix):] + + _wheel_names.append(wheel_name) + if not wheel_name in _version_map: + _version_map[wheel_name] = dict() + _version_map[wheel_name][python_version] = repo_prefix + +{process_requirements_calls} + +def requirement(name): + return "{macro_tmpl}".format(pip_utils.normalize_name(name), "pkg") + +def whl_requirement(name): + return "{macro_tmpl}".format(pip_utils.normalize_name(name), "whl") + +def data_requirement(name): + return "{macro_tmpl}".format(pip_utils.normalize_name(name), "data") + +def dist_info_requirement(name): + return "{macro_tmpl}".format(pip_utils.normalize_name(name), "dist_info") + +def install_deps(**whl_library_kwargs): +{install_deps_calls} + for wheel_name in _wheel_names: + whl_library_alias( + name = "{name}_" + wheel_name, + wheel_name = wheel_name, + default_version = "{default_version}", + version_map = _version_map[wheel_name], + ) +""".format( + name = rctx.attr.name, + install_deps_calls = "\n".join(install_deps_calls), + load_statements = "\n".join(load_statements), + macro_tmpl = macro_tmpl, + process_requirements_calls = "\n".join(process_requirements_calls), + rules_python = rules_python, + default_version = rctx.attr.default_version, + ) + rctx.file("requirements.bzl", requirements_bzl) + rctx.file("BUILD.bazel", "exports_files(['requirements.bzl'])") + +_multi_pip_parse = repository_rule( + _multi_pip_parse_impl, + attrs = { + "default_version": attr.string(), + "pip_parses": attr.string_dict(), + "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")), + }, +) + +def multi_pip_parse(name, default_version, python_versions, python_interpreter_target, requirements_lock, **kwargs): + """NOT INTENDED FOR DIRECT USE! + + This is intended to be used by the multi_pip_parse implementation in the template of the + multi_toolchain_aliases repository rule. + + Args: + name: the name of the multi_pip_parse repository. + default_version: the default Python version. + python_versions: all Python toolchain versions currently registered. + python_interpreter_target: a dictionary which keys are Python versions and values are resolved host interpreters. + requirements_lock: a dictionary which keys are Python versions and values are locked requirements files. + **kwargs: extra arguments passed to all wrapped pip_parse. + + Returns: + The internal implementation of multi_pip_parse repository rule. + """ + pip_parses = {} + for python_version in python_versions: + if not python_version in python_interpreter_target: + fail("Missing python_interpreter_target for Python version %s in '%s'" % (python_version, name)) + if not python_version in requirements_lock: + fail("Missing requirements_lock for Python version %s in '%s'" % (python_version, name)) + + pip_parse_name = name + "_" + python_version.replace(".", "_") + pip_parse( + name = pip_parse_name, + python_interpreter_target = python_interpreter_target[python_version], + requirements_lock = requirements_lock[python_version], + **kwargs + ) + pip_parses[python_version] = pip_parse_name + + return _multi_pip_parse( + name = name, + default_version = default_version, + pip_parses = pip_parses, + ) diff --git a/python/private/pypi/pip_compile.bzl b/python/private/pypi/pip_compile.bzl new file mode 100644 index 000000000..7389e7212 --- /dev/null +++ b/python/private/pypi/pip_compile.bzl @@ -0,0 +1,167 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Rules to verify and update pip-compile locked requirements.txt. + +NOTE @aignas 2024-06-23: We are using the implementation specific name here to +make it possible to have multiple tools inside the `pypi` directory +""" + +load("//python:defs.bzl", _py_binary = "py_binary", _py_test = "py_test") +load(":deps.bzl", "requirement") + +def pip_compile( + name, + src = None, + extra_args = [], + extra_deps = [], + generate_hashes = True, + py_binary = _py_binary, + py_test = _py_test, + requirements_in = None, + requirements_txt = None, + requirements_darwin = None, + requirements_linux = None, + requirements_windows = None, + visibility = ["//visibility:private"], + tags = None, + **kwargs): + """Generates targets for managing pip dependencies with pip-compile. + + By default this rules generates a filegroup named "[name]" which can be included in the data + of some other compile_pip_requirements rule that references these requirements + (e.g. with `-r ../other/requirements.txt`). + + It also generates two targets for running pip-compile: + + - validate with `bazel test [name]_test` + - update with `bazel run [name].update` + + If you are using a version control system, the requirements.txt generated by this rule should + be checked into it to ensure that all developers/users have the same dependency versions. + + Args: + name: base name for generated targets, typically "requirements". + src: file containing inputs to dependency resolution. If not specified, + defaults to `pyproject.toml`. Supported formats are: + * a requirements text file, usually named `requirements.in` + * A `.toml` file, where the `project.dependencies` list is used as per + [PEP621](https://peps.python.org/pep-0621/). + extra_args: passed to pip-compile. + extra_deps: extra dependencies passed to pip-compile. + generate_hashes: whether to put hashes in the requirements_txt file. + py_binary: the py_binary rule to be used. + py_test: the py_test rule to be used. + requirements_in: file expressing desired dependencies. Deprecated, use src instead. + requirements_txt: result of "compiling" the requirements.in file. + requirements_linux: File of linux specific resolve output to check validate if requirement.in has changes. + requirements_darwin: File of darwin specific resolve output to check validate if requirement.in has changes. + requirements_windows: File of windows specific resolve output to check validate if requirement.in has changes. + tags: tagging attribute common to all build rules, passed to both the _test and .update rules. + visibility: passed to both the _test and .update rules. + **kwargs: other bazel attributes passed to the "_test" rule. + """ + if requirements_in and src: + fail("Only one of 'src' and 'requirements_in' attributes can be used") + else: + src = requirements_in or src or "pyproject.toml" + + requirements_txt = name + ".txt" if requirements_txt == None else requirements_txt + + # "Default" target produced by this macro + # Allow a compile_pip_requirements rule to include another one in the data + # for a requirements file that does `-r ../other/requirements.txt` + native.filegroup( + name = name, + srcs = kwargs.pop("data", []) + [requirements_txt], + visibility = visibility, + ) + + data = [name, requirements_txt, src] + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None] + + # Use the Label constructor so this is expanded in the context of the file + # where it appears, which is to say, in @rules_python + pip_compile = Label("//python/private/pypi/dependency_resolver:dependency_resolver.py") + + loc = "$(rlocationpath {})" + + args = [ + loc.format(src), + loc.format(requirements_txt), + "//%s:%s.update" % (native.package_name(), name), + "--resolver=backtracking", + "--allow-unsafe", + ] + if generate_hashes: + args.append("--generate-hashes") + if requirements_linux: + args.append("--requirements-linux={}".format(loc.format(requirements_linux))) + if requirements_darwin: + args.append("--requirements-darwin={}".format(loc.format(requirements_darwin))) + if requirements_windows: + args.append("--requirements-windows={}".format(loc.format(requirements_windows))) + args.extend(extra_args) + + deps = [ + requirement("build"), + requirement("click"), + requirement("colorama"), + requirement("importlib_metadata"), + requirement("more_itertools"), + requirement("packaging"), + requirement("pep517"), + requirement("pip"), + requirement("pip_tools"), + requirement("pyproject_hooks"), + requirement("setuptools"), + requirement("tomli"), + requirement("zipp"), + Label("//python/runfiles:runfiles"), + ] + extra_deps + + tags = tags or [] + tags.append("requires-network") + tags.append("no-remote-exec") + tags.append("no-sandbox") + attrs = { + "args": args, + "data": data, + "deps": deps, + "main": pip_compile, + "srcs": [pip_compile], + "tags": tags, + "visibility": visibility, + } + + # cheap way to detect the bazel version + _bazel_version_4_or_greater = "propeller_optimize" in dir(native) + + # Bazel 4.0 added the "env" attribute to py_test/py_binary + if _bazel_version_4_or_greater: + attrs["env"] = kwargs.pop("env", {}) + + py_binary( + name = name + ".update", + **attrs + ) + + timeout = kwargs.pop("timeout", "short") + + py_test( + name = name + "_test", + timeout = timeout, + # kwargs could contain test-specific attributes like size or timeout + **dict(attrs, **kwargs) + ) diff --git a/python/pip_install/tools/requirements.txt b/python/private/pypi/requirements.txt similarity index 100% rename from python/pip_install/tools/requirements.txt rename to python/private/pypi/requirements.txt diff --git a/python/private/pypi/whl_installer/BUILD.bazel b/python/private/pypi/whl_installer/BUILD.bazel new file mode 100644 index 000000000..58231ceb0 --- /dev/null +++ b/python/private/pypi/whl_installer/BUILD.bazel @@ -0,0 +1,36 @@ +load("//python:defs.bzl", "py_binary", "py_library") +load("//python/private/pypi:deps.bzl", "requirement") + +py_library( + name = "lib", + srcs = [ + "arguments.py", + "namespace_pkgs.py", + "wheel.py", + "wheel_installer.py", + ], + visibility = [ + "//tests:__subpackages__", + "//third_party/rules_pycross/pycross/private:__subpackages__", + ], + deps = [ + requirement("installer"), + requirement("pip"), + requirement("packaging"), + requirement("setuptools"), + ], +) + +py_binary( + name = "wheel_installer", + srcs = [ + "wheel_installer.py", + ], + deps = [":lib"], +) + +filegroup( + name = "distribution", + srcs = glob(["*"]), + visibility = ["//python/private/pypi:__subpackages__"], +) diff --git a/python/pip_install/tools/wheel_installer/arguments.py b/python/private/pypi/whl_installer/arguments.py similarity index 98% rename from python/pip_install/tools/wheel_installer/arguments.py rename to python/private/pypi/whl_installer/arguments.py index 71133c29c..173d3a39a 100644 --- a/python/pip_install/tools/wheel_installer/arguments.py +++ b/python/private/pypi/whl_installer/arguments.py @@ -17,7 +17,7 @@ import pathlib from typing import Any, Dict, Set -from python.pip_install.tools.wheel_installer import wheel +from python.private.pypi.whl_installer import wheel def parser(**kwargs: Any) -> argparse.ArgumentParser: diff --git a/python/pip_install/tools/wheel_installer/namespace_pkgs.py b/python/private/pypi/whl_installer/namespace_pkgs.py similarity index 100% rename from python/pip_install/tools/wheel_installer/namespace_pkgs.py rename to python/private/pypi/whl_installer/namespace_pkgs.py diff --git a/python/pip_install/tools/wheel_installer/wheel.py b/python/private/pypi/whl_installer/wheel.py similarity index 100% rename from python/pip_install/tools/wheel_installer/wheel.py rename to python/private/pypi/whl_installer/wheel.py diff --git a/python/pip_install/tools/wheel_installer/wheel_installer.py b/python/private/pypi/whl_installer/wheel_installer.py similarity index 98% rename from python/pip_install/tools/wheel_installer/wheel_installer.py rename to python/private/pypi/whl_installer/wheel_installer.py index 801ef959f..ef8181c30 100644 --- a/python/pip_install/tools/wheel_installer/wheel_installer.py +++ b/python/private/pypi/whl_installer/wheel_installer.py @@ -27,7 +27,7 @@ from pip._vendor.packaging.utils import canonicalize_name -from python.pip_install.tools.wheel_installer import arguments, namespace_pkgs, wheel +from python.private.pypi.whl_installer import arguments, namespace_pkgs, wheel def _configure_reproducible_wheels() -> None: diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index cae0db3e2..77cbd4e29 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -16,12 +16,12 @@ load("//python:repositories.bzl", "is_standalone_interpreter") load("//python:versions.bzl", "WINDOWS_NAME") -load("//python/pip_install:repositories.bzl", "all_requirements") load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") load("//python/private:envsubst.bzl", "envsubst") load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load("//python/private:toolchains_repo.bzl", "get_host_os_arch") load(":attrs.bzl", "ATTRS", "use_isolated") +load(":deps.bzl", "all_requirements") load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") load(":parse_whl_name.bzl", "parse_whl_name") load(":patch_whl.bzl", "patch_whl") @@ -241,7 +241,7 @@ def _whl_library_impl(rctx): args = [ python_interpreter, "-m", - "python.pip_install.tools.wheel_installer.wheel_installer", + "python.private.pypi.whl_installer.wheel_installer", "--requirement", rctx.attr.requirement, ] diff --git a/python/private/pypi/whl_library_alias.bzl b/python/private/pypi/whl_library_alias.bzl new file mode 100644 index 000000000..263d7ec0e --- /dev/null +++ b/python/private/pypi/whl_library_alias.bzl @@ -0,0 +1,99 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""whl_library aliases for multi_pip_parse.""" + +load("//python/private:full_version.bzl", "full_version") +load(":render_pkg_aliases.bzl", "NO_MATCH_ERROR_MESSAGE_TEMPLATE") + +def _whl_library_alias_impl(rctx): + rules_python = rctx.attr._rules_python_workspace.workspace_name + if rctx.attr.default_version: + default_repo_prefix = rctx.attr.version_map[rctx.attr.default_version] + else: + default_repo_prefix = None + version_map = rctx.attr.version_map.items() + build_content = ["# Generated by python/pip.bzl"] + for alias_name in ["pkg", "whl", "data", "dist_info"]: + build_content.append(_whl_library_render_alias_target( + alias_name = alias_name, + default_repo_prefix = default_repo_prefix, + rules_python = rules_python, + version_map = version_map, + wheel_name = rctx.attr.wheel_name, + )) + rctx.file("BUILD.bazel", "\n".join(build_content)) + +def _whl_library_render_alias_target( + alias_name, + default_repo_prefix, + rules_python, + version_map, + wheel_name): + alias = ["""\ +alias( + name = "{alias_name}", + actual = select({{""".format(alias_name = alias_name)] + for [python_version, repo_prefix] in version_map: + alias.append("""\ + "@{rules_python}//python/config_settings:is_python_{full_python_version}": "{actual}",""".format( + full_python_version = full_version(python_version), + actual = "@{repo_prefix}{wheel_name}//:{alias_name}".format( + repo_prefix = repo_prefix, + wheel_name = wheel_name, + alias_name = alias_name, + ), + rules_python = rules_python, + )) + if default_repo_prefix: + default_actual = "@{repo_prefix}{wheel_name}//:{alias_name}".format( + repo_prefix = default_repo_prefix, + wheel_name = wheel_name, + alias_name = alias_name, + ) + alias.append(' "//conditions:default": "{default_actual}",'.format( + default_actual = default_actual, + )) + + alias.append(" },") # Close select expression condition dict + if not default_repo_prefix: + supported_versions = sorted([python_version for python_version, _ in version_map]) + alias.append(' no_match_error="""{}""",'.format( + NO_MATCH_ERROR_MESSAGE_TEMPLATE.format( + supported_versions = ", ".join(supported_versions), + rules_python = rules_python, + ), + )) + alias.append(" ),") # Close the select expression + alias.append(' visibility = ["//visibility:public"],') + alias.append(")") # Close the alias() expression + return "\n".join(alias) + +whl_library_alias = repository_rule( + _whl_library_alias_impl, + attrs = { + "default_version": attr.string( + mandatory = False, + doc = "Optional Python version in major.minor format, e.g. '3.10'." + + "The Python version of the wheel to use when the versions " + + "from `version_map` don't match. This allows the default " + + "(version unaware) rules to match and select a wheel. If " + + "not specified, then the default rules won't be able to " + + "resolve a wheel and an error will occur.", + ), + "version_map": attr.string_dict(mandatory = True), + "wheel_name": attr.string(mandatory = True), + "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")), + }, +) diff --git a/python/repositories.bzl b/python/repositories.bzl index 245aae254..d58feefd3 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -19,7 +19,6 @@ For historic reasons, pip_repositories() is defined in //python:pip.bzl. load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") -load("//python/pip_install:repositories.bzl", "pip_install_dependencies") load("//python/private:auth.bzl", "get_auth") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/private:coverage_deps.bzl", "coverage_dep") @@ -33,6 +32,7 @@ load( "toolchain_aliases", "toolchains_repo", ) +load("//python/private/pypi:deps.bzl", "pypi_deps") load( ":versions.bzl", "DEFAULT_RELEASE_BASE_URL", @@ -68,7 +68,7 @@ def py_repositories(): sha256 = "2037875b9a4456dce4a79d112a8ae885bbc4aad968e6587dca6e64f3a0900cdf", strip_prefix = "rules_cc-0.0.9", ) - pip_install_dependencies() + pypi_deps() ######## # Remaining content of the file is only used to support toolchains. diff --git a/tests/integration/compile_pip_requirements/WORKSPACE b/tests/integration/compile_pip_requirements/WORKSPACE index 5a2204bc3..0eeab2067 100644 --- a/tests/integration/compile_pip_requirements/WORKSPACE +++ b/tests/integration/compile_pip_requirements/WORKSPACE @@ -7,10 +7,6 @@ load("@rules_python//python:repositories.bzl", "py_repositories", "python_regist py_repositories() -load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies") - -pip_install_dependencies() - python_register_toolchains( name = "python39", python_version = "3.9", diff --git a/tests/integration/compile_pip_requirements_test_from_external_repo/WORKSPACE b/tests/integration/compile_pip_requirements_test_from_external_repo/WORKSPACE index 9a08c285e..48caeb442 100644 --- a/tests/integration/compile_pip_requirements_test_from_external_repo/WORKSPACE +++ b/tests/integration/compile_pip_requirements_test_from_external_repo/WORKSPACE @@ -7,10 +7,6 @@ load("@rules_python//python:repositories.bzl", "py_repositories", "python_regist py_repositories() -load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies") - -pip_install_dependencies() - python_register_toolchains( name = "python39", python_version = "3.9", diff --git a/tests/integration/pip_parse/WORKSPACE b/tests/integration/pip_parse/WORKSPACE index 4b4f41dda..db0cd0c7c 100644 --- a/tests/integration/pip_parse/WORKSPACE +++ b/tests/integration/pip_parse/WORKSPACE @@ -7,6 +7,11 @@ load("@rules_python//python:repositories.bzl", "py_repositories", "python_regist py_repositories() +# This call is included in `py_repositories` and we are calling +# `pip_install_dependencies` only to ensure that we are not breaking really old +# code. +# +# TODO @aignas 2024-06-23: remove this before 1.0.0 load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies") pip_install_dependencies() diff --git a/tests/pypi/whl_installer/BUILD.bazel b/tests/pypi/whl_installer/BUILD.bazel new file mode 100644 index 000000000..048a877b6 --- /dev/null +++ b/tests/pypi/whl_installer/BUILD.bazel @@ -0,0 +1,52 @@ +load("//python:defs.bzl", "py_test") + +alias( + name = "lib", + actual = "//python/private/pypi/whl_installer:lib", +) + +py_test( + name = "arguments_test", + size = "small", + srcs = [ + "arguments_test.py", + ], + deps = [ + ":lib", + ], +) + +py_test( + name = "namespace_pkgs_test", + size = "small", + srcs = [ + "namespace_pkgs_test.py", + ], + deps = [ + ":lib", + ], +) + +py_test( + name = "wheel_installer_test", + size = "small", + srcs = [ + "wheel_installer_test.py", + ], + data = ["//examples/wheel:minimal_with_py_package"], + deps = [ + ":lib", + ], +) + +py_test( + name = "wheel_test", + size = "small", + srcs = [ + "wheel_test.py", + ], + data = ["//examples/wheel:minimal_with_py_package"], + deps = [ + ":lib", + ], +) diff --git a/python/pip_install/tools/wheel_installer/arguments_test.py b/tests/pypi/whl_installer/arguments_test.py similarity index 94% rename from python/pip_install/tools/wheel_installer/arguments_test.py rename to tests/pypi/whl_installer/arguments_test.py index fa018da40..5538054a5 100644 --- a/python/pip_install/tools/wheel_installer/arguments_test.py +++ b/tests/pypi/whl_installer/arguments_test.py @@ -12,18 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import argparse import json import unittest -from python.pip_install.tools.wheel_installer import arguments, wheel +from python.private.pypi.whl_installer import arguments, wheel class ArgumentsTestCase(unittest.TestCase): def test_arguments(self) -> None: parser = arguments.parser() - repo_name = "foo" - repo_prefix = "pypi_" index_url = "--index_url=pypi.org/simple" extra_pip_args = [index_url] requirement = "foo==1.0.0 --hash=sha256:deadbeef" diff --git a/python/pip_install/tools/wheel_installer/namespace_pkgs_test.py b/tests/pypi/whl_installer/namespace_pkgs_test.py similarity index 98% rename from python/pip_install/tools/wheel_installer/namespace_pkgs_test.py rename to tests/pypi/whl_installer/namespace_pkgs_test.py index 4aa0fea97..fbbd50926 100644 --- a/python/pip_install/tools/wheel_installer/namespace_pkgs_test.py +++ b/tests/pypi/whl_installer/namespace_pkgs_test.py @@ -19,7 +19,7 @@ import unittest from typing import Optional, Set -from python.pip_install.tools.wheel_installer import namespace_pkgs +from python.private.pypi.whl_installer import namespace_pkgs class TempDir: diff --git a/python/pip_install/tools/wheel_installer/wheel_installer_test.py b/tests/pypi/whl_installer/wheel_installer_test.py similarity index 97% rename from python/pip_install/tools/wheel_installer/wheel_installer_test.py rename to tests/pypi/whl_installer/wheel_installer_test.py index 74b9c305f..7139779c3 100644 --- a/python/pip_install/tools/wheel_installer/wheel_installer_test.py +++ b/tests/pypi/whl_installer/wheel_installer_test.py @@ -19,7 +19,7 @@ import unittest from pathlib import Path -from python.pip_install.tools.wheel_installer import wheel, wheel_installer +from python.private.pypi.whl_installer import wheel_installer class TestRequirementExtrasParsing(unittest.TestCase): diff --git a/python/pip_install/tools/wheel_installer/wheel_test.py b/tests/pypi/whl_installer/wheel_test.py similarity index 97% rename from python/pip_install/tools/wheel_installer/wheel_test.py rename to tests/pypi/whl_installer/wheel_test.py index 3ddfaf7f2..9b27205ac 100644 --- a/python/pip_install/tools/wheel_installer/wheel_test.py +++ b/tests/pypi/whl_installer/wheel_test.py @@ -2,7 +2,7 @@ from random import shuffle from unittest import mock -from python.pip_install.tools.wheel_installer import wheel +from python.private.pypi.whl_installer import wheel class DepsTest(unittest.TestCase): @@ -218,7 +218,7 @@ def test_can_get_deps_based_on_specific_python_version(self): self.assertEqual({"@platforms//os:linux": ["posix_dep"]}, py38_deps.deps_select) @mock.patch( - "python.pip_install.tools.wheel_installer.wheel.host_interpreter_minor_version" + "python.private.pypi.whl_installer.wheel.host_interpreter_minor_version" ) def test_can_get_version_select(self, mock_host_interpreter_version): requires_dist = [ @@ -267,7 +267,7 @@ def test_can_get_version_select(self, mock_host_interpreter_version): ) @mock.patch( - "python.pip_install.tools.wheel_installer.wheel.host_interpreter_minor_version" + "python.private.pypi.whl_installer.wheel.host_interpreter_minor_version" ) def test_deps_spanning_all_target_py_versions_are_added_to_common( self, mock_host_version @@ -290,7 +290,7 @@ def test_deps_spanning_all_target_py_versions_are_added_to_common( self.assertEqual({}, got.deps_select) @mock.patch( - "python.pip_install.tools.wheel_installer.wheel.host_interpreter_minor_version" + "python.private.pypi.whl_installer.wheel.host_interpreter_minor_version" ) def test_deps_are_not_duplicated(self, mock_host_version): mock_host_version.return_value = 7 @@ -319,7 +319,7 @@ def test_deps_are_not_duplicated(self, mock_host_version): self.assertEqual({}, got.deps_select) @mock.patch( - "python.pip_install.tools.wheel_installer.wheel.host_interpreter_minor_version" + "python.private.pypi.whl_installer.wheel.host_interpreter_minor_version" ) def test_deps_are_not_duplicated_when_encountering_platform_dep_first( self, mock_host_version diff --git a/third_party/rules_pycross/pycross/private/tools/BUILD.bazel b/third_party/rules_pycross/pycross/private/tools/BUILD.bazel index a87e6aa67..41485c18a 100644 --- a/third_party/rules_pycross/pycross/private/tools/BUILD.bazel +++ b/third_party/rules_pycross/pycross/private/tools/BUILD.bazel @@ -20,7 +20,7 @@ py_binary( srcs = ["wheel_installer.py"], visibility = ["//visibility:public"], deps = [ - "//python/pip_install/tools/wheel_installer:lib", + "//python/private/pypi/whl_installer:lib", "@pypi__installer//:lib", ], ) diff --git a/third_party/rules_pycross/pycross/private/tools/wheel_installer.py b/third_party/rules_pycross/pycross/private/tools/wheel_installer.py index 0c352cf12..c03c4c252 100644 --- a/third_party/rules_pycross/pycross/private/tools/wheel_installer.py +++ b/third_party/rules_pycross/pycross/private/tools/wheel_installer.py @@ -30,7 +30,7 @@ from installer.destinations import SchemeDictionaryDestination from installer.sources import WheelFile -from python.pip_install.tools.wheel_installer import namespace_pkgs +from python.private.pypi.whl_installer import namespace_pkgs def setup_namespace_pkg_compatibility(wheel_dir: Path) -> None: diff --git a/tools/private/update_deps/BUILD.bazel b/tools/private/update_deps/BUILD.bazel index 2ab7cc73a..c83deb03d 100644 --- a/tools/private/update_deps/BUILD.bazel +++ b/tools/private/update_deps/BUILD.bazel @@ -50,14 +50,12 @@ py_binary( name = "update_pip_deps", srcs = ["update_pip_deps.py"], data = [ - "//:MODULE.bazel", - "//python/pip_install:repositories", - "//python/pip_install:requirements_txt", + "//python/private/pypi:deps.bzl", + "//python/private/pypi:requirements_txt", ], env = { - "MODULE_BAZEL": "$(rlocationpath //:MODULE.bazel)", - "REPOSITORIES_BZL": "$(rlocationpath //python/pip_install:repositories)", - "REQUIREMENTS_TXT": "$(rlocationpath //python/pip_install:requirements_txt)", + "DEPS_BZL": "$(rlocationpath //python/private/pypi:deps.bzl)", + "REQUIREMENTS_TXT": "$(rlocationpath //python/private/pypi:requirements_txt)", }, imports = ["../../.."], deps = [ diff --git a/tools/private/update_deps/update_pip_deps.py b/tools/private/update_deps/update_pip_deps.py index 3c4b46ecf..1034382f0 100755 --- a/tools/private/update_deps/update_pip_deps.py +++ b/tools/private/update_deps/update_pip_deps.py @@ -129,19 +129,13 @@ def main(): "--requirements-txt", type=path_from_runfiles, default=os.environ.get("REQUIREMENTS_TXT"), - help="The requirements.txt path for the pip_install tools, defaults to the value taken from REQUIREMENTS_TXT", + help="The requirements.txt path for the pypi tools, defaults to the value taken from REQUIREMENTS_TXT", ) parser.add_argument( - "--module-bazel", + "--deps-bzl", type=path_from_runfiles, - default=os.environ.get("MODULE_BAZEL"), - help="The path for the file to be updated, defaults to the value taken from MODULE_BAZEL", - ) - parser.add_argument( - "--repositories-bzl", - type=path_from_runfiles, - default=os.environ.get("REPOSITORIES_BZL"), - help="The path for the file to be updated, defaults to the value taken from REPOSITORIES_BZL", + default=os.environ.get("DEPS_BZL"), + help="The path for the file to be updated, defaults to the value taken from DEPS_BZL", ) args = parser.parse_args() @@ -149,21 +143,13 @@ def main(): deps = _get_deps(report) update_file( - path=args.repositories_bzl, + path=args.deps_bzl, snippet=_dep_snippet(deps), start_marker=args.start, end_marker=args.end, dry_run=args.dry_run, ) - update_file( - path=args.module_bazel, - snippet=_module_snippet(deps), - start_marker=args.start, - end_marker=args.end, - dry_run=args.dry_run, - ) - if __name__ == "__main__": main()