tooling: Update pip repos and shift py bazel deps (envoyproxy#19575)

Signed-off-by: Ryan Northey <ryan@synca.io>
keith · Jan 19, 2022 · e845ee8 · e845ee8
1 parent 4f43564
commit e845ee8
Show file tree

Hide file tree

Showing 18 changed files with 37 additions and 143 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -6,16 +6,6 @@ updates:
   schedule:
     interval: "daily"
 
-- package-ecosystem: "pip"
-  directory: "/configs"
-  schedule:
-    interval: "daily"
-
-- package-ecosystem: "pip"
-  directory: "/test/extensions/filters/network/thrift_proxy"
-  schedule:
-    interval: "daily"
-
 - package-ecosystem: "pip"
   directory: "/examples/grpc-bridge/client"
   schedule:

diff --git a/WORKSPACE b/WORKSPACE
@@ -16,10 +16,6 @@ load("//bazel:repositories_extra.bzl", "envoy_dependencies_extra")
 
 envoy_dependencies_extra()
 
-load("@base_pip3//:requirements.bzl", "install_deps")
-
-install_deps()
-
 load("//bazel:dependency_imports.bzl", "envoy_dependency_imports")
 
 envoy_dependency_imports()
diff --git a/bazel/dependency_imports.bzl b/bazel/dependency_imports.bzl
@@ -9,6 +9,7 @@ load("@rules_rust//rust:repositories.bzl", "rust_repositories")
 load("@rules_antlr//antlr:deps.bzl", "antlr_dependencies")
 load("@proxy_wasm_rust_sdk//bazel:dependencies.bzl", "proxy_wasm_rust_sdk_dependencies")
 load("@rules_cc//cc:repositories.bzl", "rules_cc_dependencies", "rules_cc_toolchains")
+load("@base_pip3//:requirements.bzl", pip_dependencies = "install_deps")
 
 # go version for rules_go
 GO_VERSION = "1.17.5"
@@ -20,6 +21,7 @@ def envoy_dependency_imports(go_version = GO_VERSION):
     go_register_toolchains(go_version)
     gazelle_dependencies()
     apple_rules_dependencies()
+    pip_dependencies()
     rust_repositories()
     upb_deps()
     antlr_dependencies(472)

diff --git a/bazel/external/twitter_common_rpc.BUILD b/bazel/external/twitter_common_rpc.BUILD
diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
@@ -100,22 +100,6 @@ def envoy_repo():
     if "envoy_repo" not in native.existing_rules().keys():
         _envoy_repo(name = "envoy_repo")
 
-# Python dependencies.
-def _python_deps():
-    # TODO(htuch): convert these to pip3_import.
-    external_http_archive(
-        name = "com_github_twitter_common_lang",
-        build_file = "@envoy//bazel/external:twitter_common_lang.BUILD",
-    )
-    external_http_archive(
-        name = "com_github_twitter_common_rpc",
-        build_file = "@envoy//bazel/external:twitter_common_rpc.BUILD",
-    )
-    external_http_archive(
-        name = "com_github_twitter_common_finagle_thrift",
-        build_file = "@envoy//bazel/external:twitter_common_finagle_thrift.BUILD",
-    )
-
 # Bazel native C++ dependencies. For the dependencies that doesn't provide autoconf/automake builds.
 def _cc_deps():
     external_http_archive("grpc_httpjson_transcoding")
@@ -236,7 +220,6 @@ def envoy_dependencies(skip_targets = []):
     # Unconditional, since we use this only for compiler-agnostic fuzzing utils.
     _org_llvm_releases_compiler_rt()
 
-    _python_deps()
     _cc_deps()
     _go_deps(skip_targets)
     _rust_deps()

diff --git a/bazel/repositories_extra.bzl b/bazel/repositories_extra.bzl
@@ -9,21 +9,13 @@ def _python_deps():
         requirements_lock = "@envoy//tools/base:requirements.txt",
         extra_pip_args = ["--require-hashes"],
     )
-    pip_install(
-        name = "configs_pip3",
-        requirements = "@envoy//configs:requirements.txt",
-        extra_pip_args = ["--require-hashes"],
-    )
+
+    # These need to use `pip_install`
     pip_install(
         # Note: dev requirements do *not* check hashes
         name = "dev_pip3",
         requirements = "@envoy//tools/dev:requirements.txt",
     )
-    pip_install(
-        name = "thrift_pip3",
-        requirements = "@envoy//test/extensions/filters/network/thrift_proxy:requirements.txt",
-        extra_pip_args = ["--require-hashes"],
-    )
     pip_install(
         name = "fuzzing_pip3",
         requirements = "@rules_fuzzing//fuzzing:requirements.txt",

diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl
@@ -589,39 +589,6 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         release_date = "2019-09-18",
         cpe = "N/A",
     ),
-    com_github_twitter_common_lang = dict(
-        project_name = "twitter.common.lang (Thrift)",
-        project_desc = "twitter.common Python language and compatibility facilities",
-        project_url = "https://pypi.org/project/twitter.common.lang",
-        version = "0.3.9",
-        sha256 = "56d1d266fd4767941d11c27061a57bc1266a3342e551bde3780f9e9eb5ad0ed1",
-        strip_prefix = "twitter.common.lang-{version}/src",
-        urls = ["https://files.pythonhosted.org/packages/08/bc/d6409a813a9dccd4920a6262eb6e5889e90381453a5f58938ba4cf1d9420/twitter.common.lang-{version}.tar.gz"],
-        release_date = "2016-10-17",
-        use_category = ["test_only"],
-    ),
-    com_github_twitter_common_rpc = dict(
-        project_name = "twitter.common.rpc (Thrift)",
-        project_desc = "twitter.common Thrift helpers including Finagle and SSL transports",
-        project_url = "https://pypi.org/project/twitter.common.rpc",
-        version = "0.3.9",
-        sha256 = "0792b63fb2fb32d970c2e9a409d3d00633190a22eb185145fe3d9067fdaa4514",
-        strip_prefix = "twitter.common.rpc-{version}/src",
-        urls = ["https://files.pythonhosted.org/packages/be/97/f5f701b703d0f25fbf148992cd58d55b4d08d3db785aad209255ee67e2d0/twitter.common.rpc-{version}.tar.gz"],
-        release_date = "2016-10-17",
-        use_category = ["test_only"],
-    ),
-    com_github_twitter_common_finagle_thrift = dict(
-        project_name = "twitter.common.finagle-thrift",
-        project_desc = "twitter.common Thrift stubs for Zipkin RPC tracing support in Finagle",
-        project_url = "https://pypi.org/project/twitter.common.finagle-thrift",
-        version = "0.3.9",
-        sha256 = "1e3a57d11f94f58745e6b83348ecd4fa74194618704f45444a15bc391fde497a",
-        strip_prefix = "twitter.common.finagle-thrift-{version}/src",
-        urls = ["https://files.pythonhosted.org/packages/f9/e7/4f80d582578f8489226370762d2cf6bc9381175d1929eba1754e03f70708/twitter.common.finagle-thrift-{version}.tar.gz"],
-        release_date = "2016-10-17",
-        use_category = ["test_only"],
-    ),
     com_google_googletest = dict(
         project_name = "Google Test",
         project_desc = "Google's C++ test framework",

diff --git a/configs/BUILD b/configs/BUILD
@@ -3,7 +3,7 @@ load(
     "envoy_package",
 )
 load("@rules_python//python:defs.bzl", "py_binary")
-load("@configs_pip3//:requirements.bzl", "requirement")
+load("@base_pip3//:requirements.bzl", "requirement")
 
 licenses(["notice"])  # Apache 2
 

diff --git a/configs/requirements.txt b/configs/requirements.txt
diff --git a/test/extensions/filters/network/thrift_proxy/driver/BUILD b/test/extensions/filters/network/thrift_proxy/driver/BUILD
@@ -1,6 +1,6 @@
 load("@rules_python//python:defs.bzl", "py_binary")
 load("//bazel:envoy_build_system.bzl", "envoy_package")
-load("@thrift_pip3//:requirements.bzl", "requirement")
+load("@base_pip3//:requirements.bzl", "requirement")
 
 licenses(["notice"])  # Apache 2
 
@@ -23,7 +23,7 @@ py_binary(
         "//test/extensions/filters/network/thrift_proxy/driver/fbthrift:fbthrift_lib",
         "//test/extensions/filters/network/thrift_proxy/driver/finagle:finagle_lib",
         "//test/extensions/filters/network/thrift_proxy/driver/generated/example:example_lib",
-        "@com_github_twitter_common_rpc//:twitter_common_rpc",
+        requirement("twitter.common.rpc"),
         requirement("six"),
     ],
 )
@@ -36,7 +36,7 @@ py_binary(
         "//test/extensions/filters/network/thrift_proxy/driver/fbthrift:fbthrift_lib",
         "//test/extensions/filters/network/thrift_proxy/driver/finagle:finagle_lib",
         "//test/extensions/filters/network/thrift_proxy/driver/generated/example:example_lib",
-        "@com_github_twitter_common_rpc//:twitter_common_rpc",
+        requirement("twitter.common.rpc"),
         requirement("six"),
     ],
 )
diff --git a/test/extensions/filters/network/thrift_proxy/driver/fbthrift/BUILD b/test/extensions/filters/network/thrift_proxy/driver/fbthrift/BUILD
@@ -1,6 +1,6 @@
 load("@rules_python//python:defs.bzl", "py_library")
 load("//bazel:envoy_build_system.bzl", "envoy_package")
-load("@thrift_pip3//:requirements.bzl", "requirement")
+load("@base_pip3//:requirements.bzl", "requirement")
 
 licenses(["notice"])  # Apache 2
 

diff --git a/test/extensions/filters/network/thrift_proxy/driver/finagle/BUILD b/test/extensions/filters/network/thrift_proxy/driver/finagle/BUILD
@@ -1,6 +1,6 @@
 load("@rules_python//python:defs.bzl", "py_library")
 load("//bazel:envoy_build_system.bzl", "envoy_package")
-load("@thrift_pip3//:requirements.bzl", "requirement")
+load("@base_pip3//:requirements.bzl", "requirement")
 
 licenses(["notice"])  # Apache 2
 
@@ -14,8 +14,8 @@ py_library(
         "__init__.py",
     ],
     deps = [
-        "@com_github_twitter_common_finagle_thrift//:twitter_common_finagle_thrift",
-        "@com_github_twitter_common_rpc//:twitter_common_rpc",
         requirement("thrift"),
+        requirement("twitter.common.finagle-thrift"),
+        requirement("twitter.common.rpc"),
     ],
 )
diff --git a/test/extensions/filters/network/thrift_proxy/driver/generated/example/BUILD b/test/extensions/filters/network/thrift_proxy/driver/generated/example/BUILD
@@ -1,6 +1,6 @@
 load("@rules_python//python:defs.bzl", "py_library")
 load("//bazel:envoy_build_system.bzl", "envoy_package")
-load("@thrift_pip3//:requirements.bzl", "requirement")
+load("@base_pip3//:requirements.bzl", "requirement")
 
 licenses(["notice"])  # Apache 2
 

diff --git a/test/extensions/filters/network/thrift_proxy/requirements.in b/test/extensions/filters/network/thrift_proxy/requirements.in
diff --git a/test/extensions/filters/network/thrift_proxy/requirements.txt b/test/extensions/filters/network/thrift_proxy/requirements.txt
diff --git a/tools/base/requirements.in b/tools/base/requirements.in
@@ -37,5 +37,8 @@ sphinx-copybutton
 sphinxext-rediraffe
 sphinx-rtd-theme
 sphinx-tabs
+thrift
+twitter.common.rpc==0.3.9
+twitter.common.finagle-thrift
 verboselogs
 yapf
diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt
@@ -673,6 +673,7 @@ six==1.16.0 \
     # via
     #   pynacl
     #   sphinxcontrib-httpdomain
+    #   thrift
 slackclient==2.9.3 \
     --hash=sha256:07ec8fa76f6aa64852210ae235ff9e637ba78124e06c0b07a7eeea4abb955965 \
     --hash=sha256:2d68d668c02f4038299897e5c4723ab85dd40a3548354924b24f333a435856f8
@@ -753,6 +754,11 @@ sphinxext-rediraffe==0.2.7 \
     # via
     #   -r requirements.in
     #   envoy.docs.sphinx-runner
+thrift==0.15.0 \
+    --hash=sha256:87c8205a71cf8bbb111cb99b1f7495070fbc9cabb671669568854210da5b3e29
+    # via
+    #   -r requirements.in
+    #   twitter.common.rpc
 toml==0.10.2 \
     --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
     --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f
@@ -765,6 +771,17 @@ trycast==0.3.0 \
     --hash=sha256:1b7b4c0d4b0d674770a53f34a762e52a6cd6879eb251ab21625602699920080d \
     --hash=sha256:687185b812e8d1c45f2ba841e8de7bdcdee0695dcf3464f206800505d4c65f26
     # via envoy.base.utils
+twitter.common.finagle-thrift==0.3.9 \
+    --hash=sha256:1e3a57d11f94f58745e6b83348ecd4fa74194618704f45444a15bc391fde497a
+    # via
+    #   -r requirements.in
+    #   twitter.common.rpc
+twitter.common.lang==0.3.9 \
+    --hash=sha256:56d1d266fd4767941d11c27061a57bc1266a3342e551bde3780f9e9eb5ad0ed1
+    # via twitter.common.rpc
+twitter.common.rpc==0.3.9 \
+    --hash=sha256:0792b63fb2fb32d970c2e9a409d3d00633190a22eb185145fe3d9067fdaa4514
+    # via -r requirements.in
 typing-extensions==3.10.0.2 \
     --hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \
     --hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \

diff --git a/tools/code_format/requirements.in b/tools/code_format/requirements.in
@@ -0,0 +1,4 @@
+flake8
+pep8-naming
+pyflakes==2.4.0
+yapf