api: manifest based edge default documentation.

This PR replaces envoyproxy#11058, taking a slightly different approach. We
utilize field options to annotate fields that should be set for
untrusted environments with [configure_for_untrusted_downstream,
configure_for_untrusted_downstream]. Defaults are provided out-of-band,
in a manifest files in docs/edge_defaults_manifest.yaml.

Protodoc glues the manifest and options together when generating field
documentation, providing an additional notice for sensitive fields.

This PR depends on envoyproxy#11108 first merging to provide the pip3 build
infrastructure.

Risk level: Low (docs only).
Testing: Inspection of generated docs.

Signed-off-by: Harvey Tuch <htuch@google.com>

api/bazel/repository_locations.bzl

@@ -13,8 +13,8 @@ GOOGLEAPIS_SHA = "a45019af4d3290f02eaeb1ce10990166978c807cb33a9692141a076ba46d14
 PROMETHEUS_GIT_SHA = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c"  # Nov 17, 2017
 PROMETHEUS_SHA = "783bdaf8ee0464b35ec0c8704871e1e72afa0005c3f3587f65d9d6694bf3911b"
 
-UDPA_GIT_SHA = "e8cd3a4bb307e2c810cffff99f93e96e6d7fee85"  # Mar 27, 2020
-UDPA_SHA256 = "1fd7857cb61daee7726fca8f4d55e4923774a8d00a53007a4093830dc0482685"
+UDPA_GIT_SHA = "9f54a527e3bf4d1f4a6527f93d329fb1cc4516ac"  # May 8, 2020
+UDPA_SHA256 = "7edae88586a84360203e5a4c724080c740b7b6002d5d56f5e806f27c912895cd"
 
 ZIPKINAPI_RELEASE = "0.2.2"  # Aug 23, 2019
 ZIPKINAPI_SHA256 = "688c4fe170821dd589f36ec45aaadc03a618a40283bc1f97da8fa11686fc816b"

api/envoy/config/bootstrap/v3/bootstrap.proto

@@ -19,6 +19,7 @@ import "google/protobuf/struct.proto";
 import "google/protobuf/wrappers.proto";
 
 import "envoy/annotations/deprecation.proto";
+import "udpa/annotations/security.proto";
 import "udpa/annotations/status.proto";
 import "udpa/annotations/versioning.proto";
 import "validate/validate.proto";
@@ -145,7 +146,10 @@ message Bootstrap {
   Admin admin = 12;
 
   // Optional overload manager configuration.
-  overload.v3.OverloadManager overload_manager = 15;
+  overload.v3.OverloadManager overload_manager = 15 [
+    (udpa.annotations.security).configure_for_untrusted_downstream = true,
+    (udpa.annotations.security).configure_for_untrusted_upstream = true
+  ];
 
   // Enable :ref:`stats for event dispatcher <operations_performance>`, defaults to false.
   // Note that this records a value for each iteration of the event loop on every thread. This

api/envoy/config/bootstrap/v4alpha/bootstrap.proto

@@ -18,6 +18,7 @@ import "google/protobuf/struct.proto";
 import "google/protobuf/wrappers.proto";
 
 import "envoy/annotations/deprecation.proto";
+import "udpa/annotations/security.proto";
 import "udpa/annotations/status.proto";
 import "udpa/annotations/versioning.proto";
 import "validate/validate.proto";
@@ -137,7 +138,10 @@ message Bootstrap {
   Admin admin = 12;
 
   // Optional overload manager configuration.
-  overload.v3.OverloadManager overload_manager = 15;
+  overload.v3.OverloadManager overload_manager = 15 [
+    (udpa.annotations.security).configure_for_untrusted_downstream = true,
+    (udpa.annotations.security).configure_for_untrusted_upstream = true
+  ];
 
   // Enable :ref:`stats for event dispatcher <operations_performance>`, defaults to false.
   // Note that this records a value for each iteration of the event loop on every thread. This

api/envoy/config/listener/v3/listener.proto

@@ -14,6 +14,7 @@ import "google/api/annotations.proto";
 import "google/protobuf/duration.proto";
 import "google/protobuf/wrappers.proto";
 
+import "udpa/annotations/security.proto";
 import "udpa/annotations/status.proto";
 import "udpa/annotations/versioning.proto";
 import "validate/validate.proto";
@@ -108,7 +109,8 @@ message Listener {
 
   // Soft limit on size of the listener’s new connection read and write buffers.
   // If unspecified, an implementation defined default is applied (1MiB).
-  google.protobuf.UInt32Value per_connection_buffer_limit_bytes = 5;
+  google.protobuf.UInt32Value per_connection_buffer_limit_bytes = 5
+      [(udpa.annotations.security).configure_for_untrusted_downstream = true];
 
   // Listener metadata.
   core.v3.Metadata metadata = 6;

api/envoy/config/listener/v4alpha/listener.proto

@@ -14,6 +14,7 @@ import "google/api/annotations.proto";
 import "google/protobuf/duration.proto";
 import "google/protobuf/wrappers.proto";
 
+import "udpa/annotations/security.proto";
 import "udpa/annotations/status.proto";
 import "udpa/annotations/versioning.proto";
 import "validate/validate.proto";
@@ -108,7 +109,8 @@ message Listener {
 
   // Soft limit on size of the listener’s new connection read and write buffers.
   // If unspecified, an implementation defined default is applied (1MiB).
-  google.protobuf.UInt32Value per_connection_buffer_limit_bytes = 5;
+  google.protobuf.UInt32Value per_connection_buffer_limit_bytes = 5
+      [(udpa.annotations.security).configure_for_untrusted_downstream = true];
 
   // Listener metadata.
   core.v4alpha.Metadata metadata = 6;

bazel/dependency_imports.bzl

@@ -4,7 +4,8 @@ load("@envoy_build_tools//toolchains:rbe_toolchains_config.bzl", "rbe_toolchains
 load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository")
 load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies")
 load("@upb//bazel:repository_defs.bzl", upb_bazel_version_repository = "bazel_version_repository")
-load("@config_validation//:requirements.bzl", config_validation_pip_install = "pip_install")
+load("@config_validation_pip3//:requirements.bzl", config_validation_pip_install = "pip_install")
+load("@protodoc_pip3//:requirements.bzl", protodoc_pip_install = "pip_install")
 
 # go version for rules_go
 GO_VERSION = "1.13.5"
@@ -41,3 +42,4 @@ def envoy_dependency_imports(go_version = GO_VERSION):
     )
 
     config_validation_pip_install()
+    protodoc_pip_install()

bazel/repositories_extra.bzl

@@ -7,9 +7,13 @@ def _python_deps():
     pip_repositories()
 
     pip3_import(
-        name = "config_validation",
+        name = "config_validation_pip3",
         requirements = "@envoy//tools/config_validation:requirements.txt",
     )
+    pip3_import(
+        name = "protodoc_pip3",
+        requirements = "//tools/protodoc:requirements.txt",
+    )
 
 # Envoy deps that rely on a first stage of dependency loading in envoy_dependencies().
 def envoy_dependencies_extra():

docs/BUILD

@@ -0,0 +1,3 @@
+licenses(["notice"])  # Apache 2
+
+exports_files(["edge_defaults_manifest.yaml"])

docs/edge_defaults_manifest.yaml

@@ -0,0 +1,21 @@
+envoy.config.bootstrap.v3.Bootstrap.overload_manager:
+  refresh_interval: 0.25s
+  resource_monitors:
+  - name: "envoy.resource_monitors.fixed_heap"
+    typed_config:
+      "@type": type.googleapis.com/envoy.config.resource_monitor.fixed_heap.v2alpha.FixedHeapConfig
+      # TODO: Tune for your system.
+      max_heap_size_bytes: 2147483648 # 2 GiB
+  actions:
+  - name: "envoy.overload_actions.shrink_heap"
+    triggers:
+    - name: "envoy.resource_monitors.fixed_heap"
+      threshold:
+        value: 0.95
+  - name: "envoy.overload_actions.stop_accepting_requests"
+    triggers:
+    - name: "envoy.resource_monitors.fixed_heap"
+      threshold:
+        value: 0.98
+
+envoy.config.listener.v3.Listener.per_connection_buffer_limit_bytes: 32768 # 32 KiB

tools/code_format/check_format.py

@@ -697,7 +697,8 @@ def checkSourceLine(line, file_path, reportError):
 
 
 def checkBuildLine(line, file_path, reportError):
-  if "@bazel_tools" in line and not (isSkylarkFile(file_path) or file_path.startswith("./bazel/")):
+  if "@bazel_tools" in line and not (isSkylarkFile(file_path) or file_path.startswith("./bazel/") or
+                                     "python/runfiles" in line):
     reportError("unexpected @bazel_tools reference, please indirect via a definition in //bazel")
   if not whitelistedForProtobufDeps(file_path) and '"protobuf"' in line:
     reportError("unexpected direct external dependency on protobuf, use "

tools/config_validation/BUILD

@@ -1,10 +1,15 @@
 licenses(["notice"])  # Apache 2
 
-load("@config_validation//:requirements.bzl", "requirement")
+load("@config_validation_pip3//:requirements.bzl", "requirement")
 
 py_binary(
-    name = "validate_yaml_fragment",
-    srcs = ["validate_yaml_fragment.py"],
+    name = "validate_fragment",
+    srcs = ["validate_fragment.py"],
+    data = ["//tools/type_whisperer:all_protos_with_ext_pb_text.pb_text"],
     visibility = ["//visibility:public"],
-    deps = [requirement("PyYAML")],
+    deps = [
+        requirement("PyYAML"),
+        "@bazel_tools//tools/python/runfiles",
+        "@com_google_protobuf//:protobuf_python",
+    ],
 )

tools/config_validation/validate_fragment.py

@@ -0,0 +1,56 @@
+# Validate a YAML fragment against an Envoy API proto3 type.
+#
+# Example usage:
+#
+# bazel run //tools/config_validation:validate_fragment -- \
+#   envoy.config.bootstrap.v3.Bootstrap $PWD/configs/google_com_proxy.v2.yaml
+
+import json
+import pathlib
+import sys
+
+import yaml
+
+from google.protobuf import descriptor_pb2
+from google.protobuf import descriptor_pool
+from google.protobuf import json_format
+from google.protobuf import message_factory
+from google.protobuf import text_format
+
+from bazel_tools.tools.python.runfiles import runfiles
+
+
+def ValidateFragment(type_name, fragment):
+  """Validate a dictionary representing a JSON/YAML fragment against an Envoy API proto3 type.
+
+  Throws Protobuf errors on parsing exceptions, successful validations produce
+  no result.
+
+  Args:
+    type_name: a string providing the type name, e.g.
+      envoy.config.bootstrap.v3.Bootstrap.
+    fragment: a dictionary representing the parsed JSON/YAML configuration
+      fragment.
+  """
+  json_fragment = json.dumps(fragment)
+
+  r = runfiles.Create()
+  all_protos_pb_text_path = r.Rlocation(
+      'envoy/tools/type_whisperer/all_protos_with_ext_pb_text.pb_text')
+  file_desc_set = descriptor_pb2.FileDescriptorSet()
+  text_format.Parse(pathlib.Path(all_protos_pb_text_path).read_text(),
+                    file_desc_set,
+                    allow_unknown_extension=True)
+
+  pool = descriptor_pool.DescriptorPool()
+  for f in file_desc_set.file:
+    pool.Add(f)
+  desc = pool.FindMessageTypeByName(type_name)
+  msg = message_factory.MessageFactory(pool=pool).GetPrototype(desc)()
+  json_format.Parse(json_fragment, msg, descriptor_pool=pool)
+
+
+if __name__ == '__main__':
+  type_name, yaml_path = sys.argv[1:]
+  ValidateFragment(type_name, yaml.load(pathlib.Path(yaml_path).read_text(),
+                                        Loader=yaml.FullLoader))

tools/protodoc/BUILD

@@ -1,5 +1,7 @@
 licenses(["notice"])  # Apache 2
 
+load("@protodoc_pip3//:requirements.bzl", "requirement")
+
 py_binary(
     name = "generate_empty",
     srcs = ["generate_empty.py"],
@@ -10,11 +12,14 @@ py_binary(
 py_binary(
     name = "protodoc",
     srcs = ["protodoc.py"],
+    data = ["//docs:edge_defaults_manifest.yaml"],
     visibility = ["//visibility:public"],
     deps = [
         "//tools/api_proto_plugin",
+        "//tools/config_validation:validate_fragment",
         "@com_envoyproxy_protoc_gen_validate//validate:validate_py",
         "@com_github_cncf_udpa//udpa/annotations:pkg_py_proto",
         "@com_google_protobuf//:protobuf_python",
+        requirement("PyYAML"),
     ],
 )