api: shadowing for API protos. (#9429)

This is intended to simplify the internal handling of deprecations during API boosting.

See https://docs.google.com/document/d/1mGO9LtVo7t4Lph7WlmyGCxXye3h6j29z3JZvIBbs_D0/edit

Ultimately the plan is to hide this all as a build artifact in Bazel cache, but due to the technical complexity of the pure Bazel solution (involving changes spanning PGV, protoxform, API build rules), we will use checked-in artifacts for 1.13.0.

Risk level: Low
Testing: Additional API test and protoxform golden test.

Part of #8082

This should be unwound in the future with #9479

Signed-off-by: Harvey Tuch <htuch@google.com>

.gitattributes

@@ -1,6 +1,8 @@
 /docs/root/intro/version_history.rst merge=union
 *.generated.pb_text linguist-generated=true
 *.generated.pb_text -diff -merge
+/generated_api_shadow/envoy/** linguist-generated=true
+/generated_api_shadow/envoy/** -diff -merge
 *.svg binary
 /test/**/*_corpus/* linguist-generated=true
 /test/**/*_corpus/* -diff -merge

api/bazel/api_build_system.bzl

@@ -149,11 +149,15 @@ def api_go_test(name, **kwargs):
         **kwargs
     )
 
-def api_proto_package(srcs = [], deps = [], has_services = False, visibility = ["//visibility:public"]):
+def api_proto_package(
+        name = "pkg",
+        srcs = [],
+        deps = [],
+        has_services = False,
+        visibility = ["//visibility:public"]):
     if srcs == []:
         srcs = native.glob(["*.proto"])
 
-    name = "pkg"
     api_cc_py_proto_library(
         name = name,
         visibility = visibility,
@@ -162,9 +166,9 @@ def api_proto_package(srcs = [], deps = [], has_services = False, visibility = [
         has_services = has_services,
     )
 
-    compilers = ["@io_bazel_rules_go//proto:go_proto", "//bazel:pgv_plugin_go"]
+    compilers = ["@io_bazel_rules_go//proto:go_proto", "@envoy_api//bazel:pgv_plugin_go"]
     if has_services:
-        compilers = ["@io_bazel_rules_go//proto:go_grpc", "//bazel:pgv_plugin_go"]
+        compilers = ["@io_bazel_rules_go//proto:go_grpc", "@envoy_api//bazel:pgv_plugin_go"]
 
     go_proto_library(
         name = name + _GO_PROTO_SUFFIX,

api/tools/generate_listeners_test.py

@@ -5,7 +5,7 @@
 import generate_listeners
 
 if __name__ == "__main__":
-  srcdir = os.path.join(os.getenv("TEST_SRCDIR"), 'envoy_api')
+  srcdir = os.path.join(os.getenv("TEST_SRCDIR"), 'envoy_api_canonical')
   generate_listeners.GenerateListeners(
       os.path.join(srcdir, "examples/service_envoy/listeners.pb"), "/dev/stdout", "/dev/stdout",
       iter([os.path.join(srcdir, "examples/service_envoy/http_connection_manager.pb")]))

api/tools/tap2pcap.py

@@ -8,7 +8,7 @@
 
 Usage:
 
-bazel run @envoy_api//tools:tap2pcap <tap .pb/.pb_text> <pcap path>
+bazel run @envoy_api_canonical//tools:tap2pcap <tap .pb/.pb_text> <pcap path>
 
 Known issues:
 - IPv6 PCAP generation has malformed TCP packets. This appears to be a text2pcap

api/tools/tap2pcap_test.py

@@ -11,7 +11,7 @@
 # a golden output file for the tshark dump. Since we run tap2pcap in a
 # subshell with a limited environment, the inferred time zone should be UTC.
 if __name__ == '__main__':
-  srcdir = os.path.join(os.getenv('TEST_SRCDIR'), 'envoy_api')
+  srcdir = os.path.join(os.getenv('TEST_SRCDIR'), 'envoy_api_canonical')
   tap_path = os.path.join(srcdir, 'tools/data/tap2pcap_h2_ipv4.pb_text')
   expected_path = os.path.join(srcdir, 'tools/data/tap2pcap_h2_ipv4.txt')
   pcap_path = os.path.join(os.getenv('TEST_TMPDIR'), 'generated.pcap')

bazel/api_binding.bzl

@@ -10,20 +10,26 @@ def _default_envoy_api_impl(ctx):
         "tools",
     ]
     for d in api_dirs:
-        ctx.symlink(ctx.path(ctx.attr.envoy_root).dirname.get_child("api").get_child(d), d)
+        ctx.symlink(ctx.path(ctx.attr.envoy_root).dirname.get_child(ctx.attr.reldir).get_child(d), d)
 
 _default_envoy_api = repository_rule(
     implementation = _default_envoy_api_impl,
     attrs = {
         "envoy_root": attr.label(default = "@envoy//:BUILD"),
+        "reldir": attr.string(),
     },
 )
 
 def envoy_api_binding():
-    # Treat the data plane API as an external repo, this simplifies exporting the API to
-    # https://github.com/envoyproxy/data-plane-api.
+    # Treat the data plane API as an external repo, this simplifies exporting
+    # the API to https://github.com/envoyproxy/data-plane-api. This is the
+    # shadow API for Envoy internal use, see #9479.
     if "envoy_api" not in native.existing_rules().keys():
-        _default_envoy_api(name = "envoy_api")
+        _default_envoy_api(name = "envoy_api", reldir = "generated_api_shadow")
+
+    # We also provide the non-shadowed API for developer use (see #9479).
+    if "envoy_api_raw" not in native.existing_rules().keys():
+        _default_envoy_api(name = "envoy_api_canonical", reldir = "api")
 
     # TODO(https://github.com/envoyproxy/envoy/issues/7719) need to remove both bindings and use canonical rules
     native.bind(

ci/do_ci.sh

@@ -235,10 +235,10 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then
 elif [[ "$CI_TARGET" == "bazel.api" ]]; then
   setup_clang_toolchain
   echo "Building API..."
-  bazel build ${BAZEL_BUILD_OPTIONS} -c fastbuild @envoy_api//envoy/...
+  bazel build ${BAZEL_BUILD_OPTIONS} -c fastbuild @envoy_api_canonical//envoy/...
   echo "Testing API..."
-  bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c fastbuild @envoy_api//test/... @envoy_api//tools/... \
-    @envoy_api//tools:tap2pcap_test
+  bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c fastbuild @envoy_api_canonical//test/... @envoy_api_canonical//tools/... \
+    @envoy_api_canonical//tools:tap2pcap_test
   echo "Testing API boosting (unit tests)..."
   bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c fastbuild @envoy_dev//clang_tools/api_booster/...
   echo "Testing API boosting (golden C++ tests)..."

docs/build.sh

@@ -48,7 +48,7 @@ pip3 install -r "${SCRIPT_DIR}"/requirements.txt
 
 # Clean up any stale files in the API tree output. Bazel remembers valid cached
 # files still.
-rm -rf bazel-bin/external/envoy_api
+rm -rf bazel-bin/external/envoy_api_canonical
 
 # This is for local RBE setup, should be no-op for builds without RBE setting in bazelrc files.
 BAZEL_BUILD_OPTIONS+=" --remote_download_outputs=all --strategy=protodoc=sandboxed,local"
@@ -65,7 +65,7 @@ mkdir -p "${GENERATED_RST_DIR}"/intro/arch_overview/security
 ./docs/generate_extension_rst.py "${EXTENSION_DB_PATH}" "${GENERATED_RST_DIR}"/intro/arch_overview/security
 
 # Generate the extensions docs
-bazel build ${BAZEL_BUILD_OPTIONS} @envoy_api//docs:protos --aspects \
+bazel build ${BAZEL_BUILD_OPTIONS} @envoy_api_canonical//docs:protos --aspects \
   tools/protodoc/protodoc.bzl%protodoc_aspect --output_groups=rst --action_env=CPROFILE_ENABLED=1 \
   --action_env=ENVOY_BLOB_SHA --action_env=EXTENSION_DB_PATH="${EXTENSION_DB_PATH}" --host_force_python=PY3
 
@@ -79,19 +79,19 @@ bazel run ${BAZEL_BUILD_OPTIONS} //tools/protodoc:generate_empty -- \
 shopt -s globstar
 
 # Find all source protos.
-declare -r PROTO_TARGET=$(bazel query "labels(srcs, labels(deps, @envoy_api//docs:protos))")
+declare -r PROTO_TARGET=$(bazel query "labels(srcs, labels(deps, @envoy_api_canonical//docs:protos))")
 
 # Only copy in the protos we care about and know how to deal with in protodoc.
 for p in ${PROTO_TARGET}
 do
-  declare PROTO_FILE_WITHOUT_PREFIX="${p#@envoy_api//}"
+  declare PROTO_FILE_WITHOUT_PREFIX="${p#@envoy_api_canonical//}"
   declare PROTO_FILE_CANONICAL="${PROTO_FILE_WITHOUT_PREFIX/://}"
   # We use ** glob matching here to deal with the fact that we have something
   # like
-  # bazel-bin/external/envoy_api/envoy/admin/v2alpha/pkg/envoy/admin/v2alpha/certs.proto.proto
+  # bazel-bin/external/envoy_api_canonical/envoy/admin/v2alpha/pkg/envoy/admin/v2alpha/certs.proto.proto
   # and we don't want to have to do a nested loop and slow bazel query to
   # recover the canonical package part of the path.
-  declare SRCS=(bazel-bin/external/envoy_api/**/"${PROTO_FILE_CANONICAL}.rst")
+  declare SRCS=(bazel-bin/external/envoy_api_canonical/**/"${PROTO_FILE_CANONICAL}.rst")
   # While we may have reformatted the file multiple times due to the transitive
   # dependencies in the aspect above, they all look the same. So, just pick an
   # arbitrary match and we're done.

docs/root/operations/traffic_tapping.rst

@@ -111,7 +111,7 @@ analysis with tools such as `Wireshark <https://www.wireshark.org/>`_ with the
 
 .. code-block:: bash
 
-  bazel run @envoy_api//tools:tap2pcap /some/tap/path_0.pb path_0.pcap
+  bazel run @envoy_api_canonical//tools:tap2pcap /some/tap/path_0.pb path_0.pcap
   tshark -r path_0.pcap -d "tcp.port==10000,http2" -P
     1   0.000000    127.0.0.1 → 127.0.0.1    HTTP2 157 Magic, SETTINGS, WINDOW_UPDATE, HEADERS
     2   0.013713    127.0.0.1 → 127.0.0.1    HTTP2 91 SETTINGS, SETTINGS, WINDOW_UPDATE

generated_api_shadow/BUILD

@@ -0,0 +1,157 @@
+# DO NOT EDIT. This file is generated by tools/proto_sync.py.
+
+load("@rules_proto//proto:defs.bzl", "proto_library")
+
+licenses(["notice"])  # Apache 2
+
+proto_library(
+    name = "protos",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//envoy/admin/v2alpha:pkg",
+        "//envoy/admin/v3alpha:pkg",
+        "//envoy/api/v2:pkg",
+        "//envoy/api/v2/auth:pkg",
+        "//envoy/api/v2/cluster:pkg",
+        "//envoy/api/v2/core:pkg",
+        "//envoy/api/v2/endpoint:pkg",
+        "//envoy/api/v2/listener:pkg",
+        "//envoy/api/v2/ratelimit:pkg",
+        "//envoy/api/v2/route:pkg",
+        "//envoy/api/v3alpha:pkg",
+        "//envoy/api/v3alpha/auth:pkg",
+        "//envoy/api/v3alpha/cluster:pkg",
+        "//envoy/api/v3alpha/core:pkg",
+        "//envoy/api/v3alpha/endpoint:pkg",
+        "//envoy/api/v3alpha/listener:pkg",
+        "//envoy/api/v3alpha/route:pkg",
+        "//envoy/config/accesslog/v2:pkg",
+        "//envoy/config/accesslog/v3alpha:pkg",
+        "//envoy/config/bootstrap/v2:pkg",
+        "//envoy/config/bootstrap/v3alpha:pkg",
+        "//envoy/config/cluster/aggregate/v2alpha:pkg",
+        "//envoy/config/cluster/dynamic_forward_proxy/v2alpha:pkg",
+        "//envoy/config/cluster/dynamic_forward_proxy/v3alpha:pkg",
+        "//envoy/config/cluster/redis:pkg",
+        "//envoy/config/common/dynamic_forward_proxy/v2alpha:pkg",
+        "//envoy/config/common/dynamic_forward_proxy/v3alpha:pkg",
+        "//envoy/config/common/tap/v2alpha:pkg",
+        "//envoy/config/common/tap/v3alpha:pkg",
+        "//envoy/config/filter/accesslog/v2:pkg",
+        "//envoy/config/filter/accesslog/v3alpha:pkg",
+        "//envoy/config/filter/dubbo/router/v2alpha1:pkg",
+        "//envoy/config/filter/fault/v2:pkg",
+        "//envoy/config/filter/fault/v3alpha:pkg",
+        "//envoy/config/filter/http/adaptive_concurrency/v2alpha:pkg",
+        "//envoy/config/filter/http/adaptive_concurrency/v3alpha:pkg",
+        "//envoy/config/filter/http/buffer/v2:pkg",
+        "//envoy/config/filter/http/csrf/v2:pkg",
+        "//envoy/config/filter/http/csrf/v3alpha:pkg",
+        "//envoy/config/filter/http/dynamic_forward_proxy/v2alpha:pkg",
+        "//envoy/config/filter/http/dynamic_forward_proxy/v3alpha:pkg",
+        "//envoy/config/filter/http/ext_authz/v2:pkg",
+        "//envoy/config/filter/http/ext_authz/v3alpha:pkg",
+        "//envoy/config/filter/http/fault/v2:pkg",
+        "//envoy/config/filter/http/fault/v3alpha:pkg",
+        "//envoy/config/filter/http/grpc_http1_reverse_bridge/v2alpha1:pkg",
+        "//envoy/config/filter/http/grpc_stats/v2alpha:pkg",
+        "//envoy/config/filter/http/gzip/v2:pkg",
+        "//envoy/config/filter/http/header_to_metadata/v2:pkg",
+        "//envoy/config/filter/http/health_check/v2:pkg",
+        "//envoy/config/filter/http/health_check/v3alpha:pkg",
+        "//envoy/config/filter/http/ip_tagging/v2:pkg",
+        "//envoy/config/filter/http/ip_tagging/v3alpha:pkg",
+        "//envoy/config/filter/http/jwt_authn/v2alpha:pkg",
+        "//envoy/config/filter/http/jwt_authn/v3alpha:pkg",
+        "//envoy/config/filter/http/lua/v2:pkg",
+        "//envoy/config/filter/http/original_src/v2alpha1:pkg",
+        "//envoy/config/filter/http/rate_limit/v2:pkg",
+        "//envoy/config/filter/http/rate_limit/v3alpha:pkg",
+        "//envoy/config/filter/http/rbac/v2:pkg",
+        "//envoy/config/filter/http/rbac/v3alpha:pkg",
+        "//envoy/config/filter/http/router/v2:pkg",
+        "//envoy/config/filter/http/router/v3alpha:pkg",
+        "//envoy/config/filter/http/squash/v2:pkg",
+        "//envoy/config/filter/http/tap/v2alpha:pkg",
+        "//envoy/config/filter/http/tap/v3alpha:pkg",
+        "//envoy/config/filter/http/transcoder/v2:pkg",
+        "//envoy/config/filter/listener/original_src/v2alpha1:pkg",
+        "//envoy/config/filter/network/client_ssl_auth/v2:pkg",
+        "//envoy/config/filter/network/client_ssl_auth/v3alpha:pkg",
+        "//envoy/config/filter/network/dubbo_proxy/v2alpha1:pkg",
+        "//envoy/config/filter/network/dubbo_proxy/v3alpha:pkg",
+        "//envoy/config/filter/network/ext_authz/v2:pkg",
+        "//envoy/config/filter/network/ext_authz/v3alpha:pkg",
+        "//envoy/config/filter/network/http_connection_manager/v2:pkg",
+        "//envoy/config/filter/network/http_connection_manager/v3alpha:pkg",
+        "//envoy/config/filter/network/mongo_proxy/v2:pkg",
+        "//envoy/config/filter/network/mongo_proxy/v3alpha:pkg",
+        "//envoy/config/filter/network/mysql_proxy/v1alpha1:pkg",
+        "//envoy/config/filter/network/rate_limit/v2:pkg",
+        "//envoy/config/filter/network/rate_limit/v3alpha:pkg",
+        "//envoy/config/filter/network/rbac/v2:pkg",
+        "//envoy/config/filter/network/rbac/v3alpha:pkg",
+        "//envoy/config/filter/network/redis_proxy/v2:pkg",
+        "//envoy/config/filter/network/redis_proxy/v3alpha:pkg",
+        "//envoy/config/filter/network/tcp_proxy/v2:pkg",
+        "//envoy/config/filter/network/tcp_proxy/v3alpha:pkg",
+        "//envoy/config/filter/network/thrift_proxy/v2alpha1:pkg",
+        "//envoy/config/filter/network/thrift_proxy/v3alpha:pkg",
+        "//envoy/config/filter/network/zookeeper_proxy/v1alpha1:pkg",
+        "//envoy/config/filter/thrift/rate_limit/v2alpha1:pkg",
+        "//envoy/config/filter/thrift/rate_limit/v3alpha:pkg",
+        "//envoy/config/filter/thrift/router/v2alpha1:pkg",
+        "//envoy/config/filter/udp/udp_proxy/v2alpha:pkg",
+        "//envoy/config/grpc_credential/v2alpha:pkg",
+        "//envoy/config/grpc_credential/v3alpha:pkg",
+        "//envoy/config/health_checker/redis/v2:pkg",
+        "//envoy/config/listener/v2:pkg",
+        "//envoy/config/metrics/v2:pkg",
+        "//envoy/config/metrics/v3alpha:pkg",
+        "//envoy/config/overload/v2alpha:pkg",
+        "//envoy/config/overload/v3alpha:pkg",
+        "//envoy/config/ratelimit/v2:pkg",
+        "//envoy/config/ratelimit/v3alpha:pkg",
+        "//envoy/config/rbac/v2:pkg",
+        "//envoy/config/rbac/v3alpha:pkg",
+        "//envoy/config/resource_monitor/fixed_heap/v2alpha:pkg",
+        "//envoy/config/resource_monitor/injected_resource/v2alpha:pkg",
+        "//envoy/config/retry/previous_priorities:pkg",
+        "//envoy/config/trace/v2:pkg",
+        "//envoy/config/trace/v2alpha:pkg",
+        "//envoy/config/trace/v3alpha:pkg",
+        "//envoy/config/transport_socket/alts/v2alpha:pkg",
+        "//envoy/config/transport_socket/tap/v2alpha:pkg",
+        "//envoy/config/transport_socket/tap/v3alpha:pkg",
+        "//envoy/data/accesslog/v2:pkg",
+        "//envoy/data/accesslog/v3alpha:pkg",
+        "//envoy/data/cluster/v2alpha:pkg",
+        "//envoy/data/core/v2alpha:pkg",
+        "//envoy/data/core/v3alpha:pkg",
+        "//envoy/data/tap/v2alpha:pkg",
+        "//envoy/data/tap/v3alpha:pkg",
+        "//envoy/service/accesslog/v2:pkg",
+        "//envoy/service/accesslog/v3alpha:pkg",
+        "//envoy/service/auth/v2:pkg",
+        "//envoy/service/auth/v3alpha:pkg",
+        "//envoy/service/discovery/v2:pkg",
+        "//envoy/service/discovery/v3alpha:pkg",
+        "//envoy/service/load_stats/v2:pkg",
+        "//envoy/service/load_stats/v3alpha:pkg",
+        "//envoy/service/metrics/v2:pkg",
+        "//envoy/service/metrics/v3alpha:pkg",
+        "//envoy/service/ratelimit/v2:pkg",
+        "//envoy/service/ratelimit/v3alpha:pkg",
+        "//envoy/service/route/v3alpha:pkg",
+        "//envoy/service/tap/v2alpha:pkg",
+        "//envoy/service/tap/v3alpha:pkg",
+        "//envoy/service/trace/v2:pkg",
+        "//envoy/service/trace/v3alpha:pkg",
+        "//envoy/type:pkg",
+        "//envoy/type/matcher:pkg",
+        "//envoy/type/matcher/v3alpha:pkg",
+        "//envoy/type/metadata/v2:pkg",
+        "//envoy/type/tracing/v2:pkg",
+        "//envoy/type/v3alpha:pkg",
+    ],
+)

generated_api_shadow/README.md

@@ -0,0 +1,6 @@
+This directory is for generated Envoy internal artifacts (via `proto_format`).
+
+Do not hand edit any file under `envoy/`. This shadow API may only be used be
+used in the Envoy source tree.
+
+The `bazel/` tree is an symlink back to the official API Bazel rules.

generated_api_shadow/bazel

@@ -0,0 +1 @@
+../api/bazel