py/tools: Namespace repo packages to prevent conflict (#29787)

Signed-off-by: Ryan Northey <ryan@synca.io>

BUILD

@@ -1,5 +1,12 @@
+load("//bazel:envoy_build_system.bzl", "envoy_package")
+load("//tools/base:envoy_python.bzl", "envoy_py_namespace")
+
 licenses(["notice"])  # Apache 2
 
+envoy_package()
+
+envoy_py_namespace()
+
 exports_files([
     "VERSION.txt",
     "API_VERSION.txt",

bazel/repositories.bzl

@@ -115,6 +115,7 @@ envoy_entry_point(
     name = "get_project_json",
     pkg = "envoy.base.utils",
     script = "envoy.project_data",
+    init_data = [":__init__.py"],
 )
 
 genrule(
@@ -139,6 +140,7 @@ envoy_entry_point(
     ],
     pkg = "envoy.base.utils",
     script = "envoy.project",
+    init_data = [":__init__.py"],
 )
 
 envoy_entry_point(
@@ -149,6 +151,7 @@ envoy_entry_point(
     ],
     pkg = "envoy.base.utils",
     script = "envoy.project",
+    init_data = [":__init__.py"],
 )
 
 envoy_entry_point(
@@ -159,6 +162,7 @@ envoy_entry_point(
     ],
     pkg = "envoy.base.utils",
     script = "envoy.project",
+    init_data = [":__init__.py"],
 )
 
 envoy_entry_point(
@@ -169,6 +173,7 @@ envoy_entry_point(
     ],
     pkg = "envoy.base.utils",
     script = "envoy.project",
+    init_data = [":__init__.py"],
 )
 
 envoy_entry_point(
@@ -179,6 +184,7 @@ envoy_entry_point(
     ],
     pkg = "envoy.base.utils",
     script = "envoy.project",
+    init_data = [":__init__.py"],
 )
 
 ''')

distribution/dockerhub/BUILD

@@ -1,10 +1,12 @@
 load("//bazel:envoy_build_system.bzl", "envoy_package")
-load("//tools/base:envoy_python.bzl", "envoy_gencontent")
+load("//tools/base:envoy_python.bzl", "envoy_gencontent", "envoy_py_namespace")
 
 licenses(["notice"])  # Apache 2
 
 envoy_package()
 
+envoy_py_namespace()
+
 envoy_gencontent(
     name = "readme",
     srcs = ["@envoy_repo//:project"],

mobile/BUILD

@@ -7,11 +7,14 @@ load(
     "xcode_schemes",
     "xcodeproj",
 )
+load("@envoy//tools/base:envoy_python.bzl", "envoy_py_namespace")
 load("@io_bazel_rules_kotlin//kotlin/internal:toolchains.bzl", "define_kt_toolchain")
 load("//bazel:framework_imports_extractor.bzl", "framework_imports_extractor")
 
 licenses(["notice"])  # Apache 2
 
+envoy_py_namespace()
+
 alias(
     name = "ios_xcframework",
     actual = "//library/swift:Envoy",

mobile/docs/BUILD

@@ -1,15 +1,21 @@
 load("@base_pip3//:requirements.bzl", "requirement")
 load("@envoy//bazel:envoy_build_system.bzl", "envoy_package")
-load("@envoy//tools/base:envoy_python.bzl", "envoy_entry_point")
+load("@envoy//tools/base:envoy_python.bzl", "envoy_entry_point", "envoy_py_namespace")
 load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files")
 load("@rules_pkg//pkg:pkg.bzl", "pkg_tar")
 
 licenses(["notice"])  # Apache 2
 
 envoy_package()
 
+envoy_py_namespace()
+
 envoy_entry_point(
     name = "sphinx",
+    init_data = [
+        "//:py-init",
+        ":py-init",
+    ],
     pkg = "sphinx",
     script = "sphinx-build",
     deps = [

tools/BUILD

@@ -5,11 +5,14 @@ load(
     "envoy_package",
     "envoy_py_test_binary",
 )
+load("//tools/base:envoy_python.bzl", "envoy_py_namespace")
 
 licenses(["notice"])  # Apache 2
 
 envoy_package()
 
+envoy_py_namespace()
+
 exports_files([
     "gen_git_sha.sh",
     "check_repositories.sh",

tools/base/envoy_python.bzl

@@ -4,13 +4,64 @@ load("@base_pip3//:requirements.bzl", "requirement", base_entry_point = "entry_p
 load("@envoy_toolshed//py:macros.bzl", "entry_point")
 load("@rules_python//python:defs.bzl", "py_binary", "py_library")
 
+ENVOY_PYTOOL_NAMESPACE = [
+    ":py-init",
+    "//:py-init",
+    "//tools:py-init",
+]
+
+def envoy_py_namespace():
+    """Adding this to a build, injects a namespaced __init__.py, this allows namespaced
+    packages - eg envoy.base.utils to co-exist with packages created from the repo."""
+    native.genrule(
+        name = "py-init-file",
+        outs = ["__init__.py"],
+        cmd = """
+        echo "__path__ = __import__('pkgutil').extend_path(__path__, __name__)" > $@
+        """,
+    )
+    py_library(
+        name = "py-init",
+        srcs = [":py-init-file"],
+        visibility = ["//visibility:public"],
+    )
+
+def envoy_pytool_binary(
+        name,
+        data = None,
+        init_data = ENVOY_PYTOOL_NAMESPACE,
+        **kwargs):
+    """Wraps py_binary with envoy namespaced __init__.py files.
+
+    If used outside of tools/${toolname}/BUILD you must specify the init_data."""
+    py_binary(
+        name = name,
+        data = init_data + (data or []),
+        **kwargs
+    )
+
+def envoy_pytool_library(
+        name,
+        data = None,
+        init_data = ENVOY_PYTOOL_NAMESPACE,
+        **kwargs):
+    """Wraps py_library with envoy namespaced __init__.py files.
+
+    If used outside of tools/${toolname}/BUILD you must specify the init_data."""
+    py_library(
+        name = name,
+        data = init_data + (data or []),
+        **kwargs
+    )
+
 def envoy_entry_point(
         name,
         pkg,
         entry_point_script = "@envoy//tools/base:entry_point.py",
         entry_point_alias = base_entry_point,
         script = None,
         data = None,
+        init_data = ENVOY_PYTOOL_NAMESPACE,
         deps = None,
         args = None,
         visibility = ["//visibility:public"]):
@@ -20,7 +71,7 @@ def envoy_entry_point(
         script = script,
         entry_point_script = entry_point_script,
         entry_point_alias = entry_point_alias,
-        data = data,
+        data = (data or []) + init_data,
         deps = deps,
         args = args,
         visibility = visibility,
@@ -31,6 +82,8 @@ def envoy_jinja_env(
         templates,
         filters = {},
         env_kwargs = {},
+        init_data = ENVOY_PYTOOL_NAMESPACE,
+        data = [],
         deps = [],
         entry_point_alias = base_entry_point):
     """This provides a prebuilt jinja environment that can be imported as a module.
@@ -152,9 +205,10 @@ def envoy_jinja_env(
         tools = [name_templates],
     )
 
-    py_library(
+    envoy_pytool_library(
         name = name,
         srcs = [name_env_py],
+        init_data = init_data,
         data = [name_templates],
         deps = [name_entry_point],
     )
@@ -212,7 +266,12 @@ def envoy_genjson(name, srcs = [], yaml_srcs = [], filter = None, args = None):
         filter = filter,
     )
 
-def envoy_py_data(name, src, format = None, entry_point_alias = base_entry_point):
+def envoy_py_data(
+        name,
+        src,
+        init_data = ENVOY_PYTOOL_NAMESPACE,
+        format = None,
+        entry_point_alias = base_entry_point):
     """Preload JSON/YAML data as a python lib.
 
     Data is loaded to python and then dumped to a pickle file.
@@ -293,9 +352,10 @@ def envoy_py_data(name, src, format = None, entry_point_alias = base_entry_point
         tools = [name_pickle],
     )
 
-    py_library(
+    envoy_pytool_library(
         name = name,
         srcs = [name_env_py],
+        init_data = init_data,
         data = [name_pickle],
         deps = [name_entry_point, requirement("envoy.base.utils")],
     )
@@ -306,6 +366,7 @@ def envoy_gencontent(
         output,
         srcs = [],
         yaml_srcs = [],
+        init_data = ENVOY_PYTOOL_NAMESPACE,
         json_kwargs = {},
         template_name = None,
         template_filters = {},
@@ -353,10 +414,12 @@ def envoy_gencontent(
     envoy_py_data(
         name = "%s_data" % name,
         src = ":%s_json" % name,
+        init_data = init_data,
         entry_point_alias = entry_point_alias,
     )
     envoy_jinja_env(
         name = name_tpl,
+        init_data = init_data,
         env_kwargs = template_kwargs,
         templates = [template],
         filters = template_filters,
@@ -377,10 +440,11 @@ def envoy_gencontent(
         outs = ["%s_generate_content.py" % name],
         tools = [":%s" % name_data, name_tpl, template],
     )
-    py_binary(
+    envoy_pytool_binary(
         name = "%s_generate_content" % name,
         main = ":%s_generate_content.py" % name,
         srcs = [":%s_generate_content.py" % name],
+        init_data = init_data,
         deps = [
             ":%s" % name_data,
             name_tpl,

tools/code/BUILD

@@ -6,12 +6,14 @@ load(
     "READFILTER_FUZZ_FILTERS",
     "READFILTER_NOFUZZ_FILTERS",
 )
-load("//tools/base:envoy_python.bzl", "envoy_entry_point")
+load("//tools/base:envoy_python.bzl", "envoy_entry_point", "envoy_py_namespace")
 
 licenses(["notice"])  # Apache 2
 
 envoy_package()
 
+envoy_py_namespace()
+
 FUZZ_FILTER_COUNT = (
     len(READFILTER_FUZZ_FILTERS) +
     len(READFILTER_NOFUZZ_FILTERS)

tools/dependency/BUILD

@@ -1,13 +1,14 @@
 load("@base_pip3//:requirements.bzl", "requirement")
 load("@envoy_repo//:path.bzl", "PATH")
-load("@rules_python//python:defs.bzl", "py_binary")
 load("//bazel:envoy_build_system.bzl", "envoy_package")
-load("//tools/base:envoy_python.bzl", "envoy_entry_point")
+load("//tools/base:envoy_python.bzl", "envoy_entry_point", "envoy_py_namespace", "envoy_pytool_binary")
 
 licenses(["notice"])  # Apache 2
 
 envoy_package()
 
+envoy_py_namespace()
+
 envoy_entry_point(
     name = "check",
     args = [
@@ -30,7 +31,7 @@ envoy_entry_point(
     pkg = "dependatool",
 )
 
-py_binary(
+envoy_pytool_binary(
     name = "validate",
     srcs = ["validate.py"],
     args = [

tools/distribution/BUILD

@@ -1,11 +1,13 @@
 load("@base_pip3//:requirements.bzl", "requirement")
 load("//bazel:envoy_build_system.bzl", "envoy_package")
-load("//tools/base:envoy_python.bzl", "envoy_entry_point")
+load("//tools/base:envoy_python.bzl", "envoy_entry_point", "envoy_py_namespace", "envoy_pytool_binary")
 
 licenses(["notice"])  # Apache 2
 
 envoy_package()
 
+envoy_py_namespace()
+
 envoy_entry_point(
     name = "release",
     pkg = "envoy.distribution.release",
@@ -21,7 +23,7 @@ envoy_entry_point(
     pkg = "envoy.distribution.verify",
 )
 
-py_binary(
+envoy_pytool_binary(
     name = "update_dockerhub_repository",
     srcs = ["update_dockerhub_repository.py"],
     data = ["//distribution/dockerhub:readme.md"],