refactor: move PyPI related extension and repository_rule setup to its own dir

.github/CODEOWNERS

@@ -11,5 +11,6 @@
 /python/private/toolchains_repo.bzl @f0rmiga
 /python/tests/toolchains/ @f0rmiga
 
-# pip_parse related code
-/python/pip_install/ @hrfuller
+# PyPI integration related code
+/python/private/pypi/ @aignas
+/tests/pypi/ @aignas

examples/pip_parse_vendored/requirements.bzl

@@ -6,16 +6,54 @@
 load("@rules_python//python:pip.bzl", "pip_utils")
 load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library")
 
-all_requirements = ["@my_project_pip_deps_vendored_certifi//:pkg", "@my_project_pip_deps_vendored_charset_normalizer//:pkg", "@my_project_pip_deps_vendored_idna//:pkg", "@my_project_pip_deps_vendored_requests//:pkg", "@my_project_pip_deps_vendored_urllib3//:pkg"]
-
-all_whl_requirements_by_package = {"certifi": "@my_project_pip_deps_vendored_certifi//:whl", "charset_normalizer": "@my_project_pip_deps_vendored_charset_normalizer//:whl", "idna": "@my_project_pip_deps_vendored_idna//:whl", "requests": "@my_project_pip_deps_vendored_requests//:whl", "urllib3": "@my_project_pip_deps_vendored_urllib3//:whl"}
+all_requirements = [
+    "@my_project_pip_deps_vendored_certifi//:pkg",
+    "@my_project_pip_deps_vendored_charset_normalizer//:pkg",
+    "@my_project_pip_deps_vendored_idna//:pkg",
+    "@my_project_pip_deps_vendored_requests//:pkg",
+    "@my_project_pip_deps_vendored_urllib3//:pkg",
+]
+
+all_whl_requirements_by_package = {
+    "certifi": "@my_project_pip_deps_vendored_certifi//:whl",
+    "charset_normalizer": "@my_project_pip_deps_vendored_charset_normalizer//:whl",
+    "idna": "@my_project_pip_deps_vendored_idna//:whl",
+    "requests": "@my_project_pip_deps_vendored_requests//:whl",
+    "urllib3": "@my_project_pip_deps_vendored_urllib3//:whl",
+}
 
 all_whl_requirements = all_whl_requirements_by_package.values()
 
-all_data_requirements = ["@my_project_pip_deps_vendored_certifi//:data", "@my_project_pip_deps_vendored_charset_normalizer//:data", "@my_project_pip_deps_vendored_idna//:data", "@my_project_pip_deps_vendored_requests//:data", "@my_project_pip_deps_vendored_urllib3//:data"]
-
-_packages = [("my_project_pip_deps_vendored_certifi", "certifi==2023.7.22     --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082     --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"), ("my_project_pip_deps_vendored_charset_normalizer", "charset-normalizer==2.1.1     --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845     --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), ("my_project_pip_deps_vendored_idna", "idna==3.4     --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4     --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), ("my_project_pip_deps_vendored_requests", "requests==2.28.1     --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983     --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), ("my_project_pip_deps_vendored_urllib3", "urllib3==1.26.13     --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc     --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8")]
-_config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "envsubst": ["PIP_RETRIES"], "extra_pip_args": ["--retries=${PIP_RETRIES:-5}"], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": "@python39_host//:python", "quiet": True, "repo": "my_project_pip_deps_vendored", "repo_prefix": "my_project_pip_deps_vendored_", "timeout": 600}
+all_data_requirements = [
+    "@my_project_pip_deps_vendored_certifi//:data",
+    "@my_project_pip_deps_vendored_charset_normalizer//:data",
+    "@my_project_pip_deps_vendored_idna//:data",
+    "@my_project_pip_deps_vendored_requests//:data",
+    "@my_project_pip_deps_vendored_urllib3//:data",
+]
+
+_packages = [
+    ("my_project_pip_deps_vendored_certifi", "certifi==2023.7.22     --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082     --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"),
+    ("my_project_pip_deps_vendored_charset_normalizer", "charset-normalizer==2.1.1     --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845     --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"),
+    ("my_project_pip_deps_vendored_idna", "idna==3.4     --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4     --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"),
+    ("my_project_pip_deps_vendored_requests", "requests==2.28.1     --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983     --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"),
+    ("my_project_pip_deps_vendored_urllib3", "urllib3==1.26.13     --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc     --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"),
+]
+_config = {
+    "download_only": False,
+    "enable_implicit_namespace_pkgs": False,
+    "environment": {},
+    "envsubst": ["PIP_RETRIES"],
+    "extra_pip_args": ["--retries=${PIP_RETRIES:-5}"],
+    "isolated": True,
+    "pip_data_exclude": [],
+    "python_interpreter": "python3",
+    "python_interpreter_target": "@python39_host//:python",
+    "quiet": True,
+    "repo": "my_project_pip_deps_vendored",
+    "repo_prefix": "my_project_pip_deps_vendored_",
+    "timeout": 600,
+}
 _annotations = {}
 
 def requirement(name):

python/BUILD.bazel

@@ -101,7 +101,7 @@ bzl_library(
         "//python/pip_install:requirements_bzl",
         "//python/private:bzlmod_enabled_bzl",
         "//python/private:full_version_bzl",
-        "//python/private:render_pkg_aliases_bzl",
+        "//python/private/pypi:render_pkg_aliases_bzl",
         "//python/private/whl_filegroup:whl_filegroup_bzl",
     ],
 )

python/config_settings/BUILD.bazel

@@ -9,7 +9,7 @@ load(
     "PycCollectionFlag",
 )
 load(
-    "//python/private:pip_flags.bzl",
+    "//python/private/pypi:flags.bzl",
     "INTERNAL_FLAGS",
     "UniversalWhlFlag",
     "UseWhlFlag",

python/extensions/BUILD.bazel

@@ -36,8 +36,6 @@ bzl_library(
     srcs = ["python.bzl"],
     visibility = ["//:__subpackages__"],
     deps = [
-        "//python/private:util_bzl",
-        "//python/private/bzlmod:bazel_features_bzl",
         "//python/private/bzlmod:python_bzl",
     ],
 )

python/pip.bzl

@@ -19,12 +19,13 @@ symbols should not be used and they are either undocumented here or marked as
 for internal use only.
 """
 
-load("//python/pip_install:pip_repository.bzl", "pip_repository", _package_annotation = "package_annotation")
 load("//python/pip_install:requirements.bzl", _compile_pip_requirements = "compile_pip_requirements")
 load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")
 load("//python/private:full_version.bzl", "full_version")
 load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:render_pkg_aliases.bzl", "NO_MATCH_ERROR_MESSAGE_TEMPLATE")
+load("//python/private/pypi:package_annotation.bzl", _package_annotation = "package_annotation")
+load("//python/private/pypi:pip_repository.bzl", "pip_repository")
+load("//python/private/pypi:render_pkg_aliases.bzl", "NO_MATCH_ERROR_MESSAGE_TEMPLATE")
 load("//python/private/whl_filegroup:whl_filegroup.bzl", _whl_filegroup = "whl_filegroup")
 
 compile_pip_requirements = _compile_pip_requirements

python/pip_install/BUILD.bazel

@@ -22,23 +22,10 @@ bzl_library(
     name = "pip_repository_bzl",
     srcs = ["pip_repository.bzl"],
     deps = [
-        ":repositories_bzl",
-        "//python:repositories_bzl",
-        "//python:versions_bzl",
-        "//python/pip_install/private:generate_group_library_build_bazel_bzl",
-        "//python/pip_install/private:generate_whl_library_build_bazel_bzl",
-        "//python/pip_install/private:srcs_bzl",
-        "//python/private:bzlmod_enabled_bzl",
-        "//python/private:envsubst_bzl",
-        "//python/private:normalize_name_bzl",
-        "//python/private:parse_requirements_bzl",
-        "//python/private:parse_whl_name_bzl",
-        "//python/private:patch_whl_bzl",
-        "//python/private:render_pkg_aliases_bzl",
-        "//python/private:repo_utils_bzl",
-        "//python/private:toolchains_repo_bzl",
-        "//python/private:whl_target_platforms_bzl",
-        "@bazel_skylib//lib:sets",
+        "//python/private/pypi:group_library_bzl",
+        "//python/private/pypi:package_annotation_bzl",
+        "//python/private/pypi:pip_repository_bzl",
+        "//python/private/pypi:whl_library_bzl",
     ],
 )
 
@@ -51,11 +38,6 @@ bzl_library(
     ],
 )
 
-bzl_library(
-    name = "requirements_parser_bzl",
-    srcs = ["requirements_parser.bzl"],
-)
-
 bzl_library(
     name = "repositories_bzl",
     srcs = ["repositories.bzl"],
@@ -71,7 +53,6 @@ filegroup(
     srcs = glob(["*.bzl"]) + [
         "BUILD.bazel",
         "pip_repository_requirements.bzl.tmpl",
-        "//python/pip_install/private:distribution",
         "//python/pip_install/tools/dependency_resolver:distribution",
         "//python/pip_install/tools/wheel_installer:distribution",
     ],
@@ -92,23 +73,10 @@ filegroup(
 
 filegroup(
     name = "bzl",
-    srcs = glob(["*.bzl"]) + [
-        "//python/pip_install/private:bzl_srcs",
-    ],
+    srcs = glob(["*.bzl"]),
     visibility = ["//:__subpackages__"],
 )
 
-filegroup(
-    name = "py_srcs",
-    srcs = [
-        "//python/pip_install/tools/dependency_resolver:py_srcs",
-        "//python/pip_install/tools/wheel_installer:py_srcs",
-        "//python/private:repack_whl.py",
-        "//tools:wheelmaker.py",
-    ],
-    visibility = ["//python/pip_install/private:__pkg__"],
-)
-
 exports_files(
     glob(["*.bzl"]),
     visibility = ["//docs:__pkg__"],