From 2109c04916865add62f9cf9e6b962170dbf772da Mon Sep 17 00:00:00 2001 From: Harvey Tuch Date: Thu, 7 May 2020 18:23:47 -0400 Subject: [PATCH] api: manifest based edge default documentation. This PR replaces #11058, taking a slightly different approach. We utilize field options to annotate fields that should be set for untrusted environments with [configure_for_untrusted_downstream, configure_for_untrusted_downstream]. Defaults are provided out-of-band, in a manifest files in docs/edge_defaults_manifest.yaml. Protodoc glues the manifest and options together when generating field documentation, providing an additional notice for sensitive fields. This PR depends on #11108 first merging to provide the pip3 build infrastructure. Risk level: Low (docs only). Testing: Inspection of generated docs. Signed-off-by: Harvey Tuch --- api/bazel/repository_locations.bzl | 4 +- api/envoy/config/bootstrap/v3/bootstrap.proto | 6 +- .../config/bootstrap/v4alpha/bootstrap.proto | 6 +- api/envoy/config/listener/v3/listener.proto | 4 +- .../config/listener/v4alpha/listener.proto | 4 +- bazel/dependency_imports.bzl | 4 +- bazel/repositories_extra.bzl | 6 +- docs/BUILD | 3 + docs/edge_defaults_manifest.yaml | 21 ++++++ .../bazel/repository_locations.bzl | 4 +- .../envoy/config/bootstrap/v3/bootstrap.proto | 6 +- .../config/bootstrap/v4alpha/bootstrap.proto | 6 +- .../envoy/config/listener/v3/listener.proto | 4 +- .../config/listener/v4alpha/listener.proto | 4 +- tools/code_format/check_format.py | 3 +- tools/config_validation/BUILD | 13 ++-- tools/config_validation/validate_fragment.py | 56 ++++++++++++++++ .../validate_yaml_fragment.py | 3 - tools/protodoc/BUILD | 5 ++ tools/protodoc/protodoc.py | 65 +++++++++++++++++-- tools/protodoc/requirements.txt | 1 + tools/protoxform/merge_active_shadow.py | 1 + tools/protoxform/migrate.py | 4 +- tools/protoxform/protoprint.py | 1 + tools/protoxform/protoxform.py | 5 +- tools/type_whisperer/BUILD | 6 ++ .../file_descriptor_set_text.bzl | 6 +- 27 files changed, 219 insertions(+), 32 deletions(-) create mode 100644 docs/BUILD create mode 100644 docs/edge_defaults_manifest.yaml create mode 100644 tools/config_validation/validate_fragment.py delete mode 100644 tools/config_validation/validate_yaml_fragment.py create mode 100644 tools/protodoc/requirements.txt diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index c275a8c65835..77539ee9b109 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -13,8 +13,8 @@ GOOGLEAPIS_SHA = "a45019af4d3290f02eaeb1ce10990166978c807cb33a9692141a076ba46d14 PROMETHEUS_GIT_SHA = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" # Nov 17, 2017 PROMETHEUS_SHA = "783bdaf8ee0464b35ec0c8704871e1e72afa0005c3f3587f65d9d6694bf3911b" -UDPA_GIT_SHA = "e8cd3a4bb307e2c810cffff99f93e96e6d7fee85" # Mar 27, 2020 -UDPA_SHA256 = "1fd7857cb61daee7726fca8f4d55e4923774a8d00a53007a4093830dc0482685" +UDPA_GIT_SHA = "9f54a527e3bf4d1f4a6527f93d329fb1cc4516ac" # May 8, 2020 +UDPA_SHA256 = "7edae88586a84360203e5a4c724080c740b7b6002d5d56f5e806f27c912895cd" ZIPKINAPI_RELEASE = "0.2.2" # Aug 23, 2019 ZIPKINAPI_SHA256 = "688c4fe170821dd589f36ec45aaadc03a618a40283bc1f97da8fa11686fc816b" diff --git a/api/envoy/config/bootstrap/v3/bootstrap.proto b/api/envoy/config/bootstrap/v3/bootstrap.proto index c20109884d90..8eba15a5ba72 100644 --- a/api/envoy/config/bootstrap/v3/bootstrap.proto +++ b/api/envoy/config/bootstrap/v3/bootstrap.proto @@ -19,6 +19,7 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "envoy/annotations/deprecation.proto"; +import "udpa/annotations/security.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; @@ -145,7 +146,10 @@ message Bootstrap { Admin admin = 12; // Optional overload manager configuration. - overload.v3.OverloadManager overload_manager = 15; + overload.v3.OverloadManager overload_manager = 15 [ + (udpa.annotations.security).configure_for_untrusted_downstream = true, + (udpa.annotations.security).configure_for_untrusted_upstream = true + ]; // Enable :ref:`stats for event dispatcher `, defaults to false. // Note that this records a value for each iteration of the event loop on every thread. This diff --git a/api/envoy/config/bootstrap/v4alpha/bootstrap.proto b/api/envoy/config/bootstrap/v4alpha/bootstrap.proto index ce6aa147fba2..bd4169356a4e 100644 --- a/api/envoy/config/bootstrap/v4alpha/bootstrap.proto +++ b/api/envoy/config/bootstrap/v4alpha/bootstrap.proto @@ -18,6 +18,7 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "envoy/annotations/deprecation.proto"; +import "udpa/annotations/security.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; @@ -137,7 +138,10 @@ message Bootstrap { Admin admin = 12; // Optional overload manager configuration. - overload.v3.OverloadManager overload_manager = 15; + overload.v3.OverloadManager overload_manager = 15 [ + (udpa.annotations.security).configure_for_untrusted_downstream = true, + (udpa.annotations.security).configure_for_untrusted_upstream = true + ]; // Enable :ref:`stats for event dispatcher `, defaults to false. // Note that this records a value for each iteration of the event loop on every thread. This diff --git a/api/envoy/config/listener/v3/listener.proto b/api/envoy/config/listener/v3/listener.proto index 473a5eb2b42b..03214150e773 100644 --- a/api/envoy/config/listener/v3/listener.proto +++ b/api/envoy/config/listener/v3/listener.proto @@ -14,6 +14,7 @@ import "google/api/annotations.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; +import "udpa/annotations/security.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; @@ -108,7 +109,8 @@ message Listener { // Soft limit on size of the listener’s new connection read and write buffers. // If unspecified, an implementation defined default is applied (1MiB). - google.protobuf.UInt32Value per_connection_buffer_limit_bytes = 5; + google.protobuf.UInt32Value per_connection_buffer_limit_bytes = 5 + [(udpa.annotations.security).configure_for_untrusted_downstream = true]; // Listener metadata. core.v3.Metadata metadata = 6; diff --git a/api/envoy/config/listener/v4alpha/listener.proto b/api/envoy/config/listener/v4alpha/listener.proto index 4438bd2974d4..b7f32a821443 100644 --- a/api/envoy/config/listener/v4alpha/listener.proto +++ b/api/envoy/config/listener/v4alpha/listener.proto @@ -14,6 +14,7 @@ import "google/api/annotations.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; +import "udpa/annotations/security.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; @@ -108,7 +109,8 @@ message Listener { // Soft limit on size of the listener’s new connection read and write buffers. // If unspecified, an implementation defined default is applied (1MiB). - google.protobuf.UInt32Value per_connection_buffer_limit_bytes = 5; + google.protobuf.UInt32Value per_connection_buffer_limit_bytes = 5 + [(udpa.annotations.security).configure_for_untrusted_downstream = true]; // Listener metadata. core.v4alpha.Metadata metadata = 6; diff --git a/bazel/dependency_imports.bzl b/bazel/dependency_imports.bzl index 2385a98ff25b..cc2ff635ede3 100644 --- a/bazel/dependency_imports.bzl +++ b/bazel/dependency_imports.bzl @@ -4,7 +4,8 @@ load("@envoy_build_tools//toolchains:rbe_toolchains_config.bzl", "rbe_toolchains load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository") load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies") load("@upb//bazel:repository_defs.bzl", upb_bazel_version_repository = "bazel_version_repository") -load("@config_validation//:requirements.bzl", config_validation_pip_install = "pip_install") +load("@config_validation_pip3//:requirements.bzl", config_validation_pip_install = "pip_install") +load("@protodoc_pip3//:requirements.bzl", protodoc_pip_install = "pip_install") # go version for rules_go GO_VERSION = "1.13.5" @@ -41,3 +42,4 @@ def envoy_dependency_imports(go_version = GO_VERSION): ) config_validation_pip_install() + protodoc_pip_install() diff --git a/bazel/repositories_extra.bzl b/bazel/repositories_extra.bzl index fe0e9adb6a29..e56caf2f02b8 100644 --- a/bazel/repositories_extra.bzl +++ b/bazel/repositories_extra.bzl @@ -7,9 +7,13 @@ def _python_deps(): pip_repositories() pip3_import( - name = "config_validation", + name = "config_validation_pip3", requirements = "@envoy//tools/config_validation:requirements.txt", ) + pip3_import( + name = "protodoc_pip3", + requirements = "//tools/protodoc:requirements.txt", + ) # Envoy deps that rely on a first stage of dependency loading in envoy_dependencies(). def envoy_dependencies_extra(): diff --git a/docs/BUILD b/docs/BUILD new file mode 100644 index 000000000000..d190c0a59a0a --- /dev/null +++ b/docs/BUILD @@ -0,0 +1,3 @@ +licenses(["notice"]) # Apache 2 + +exports_files(["edge_defaults_manifest.yaml"]) diff --git a/docs/edge_defaults_manifest.yaml b/docs/edge_defaults_manifest.yaml new file mode 100644 index 000000000000..b5072c26a32b --- /dev/null +++ b/docs/edge_defaults_manifest.yaml @@ -0,0 +1,21 @@ +envoy.config.bootstrap.v3.Bootstrap.overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.fixed_heap" + typed_config: + "@type": type.googleapis.com/envoy.config.resource_monitor.fixed_heap.v2alpha.FixedHeapConfig + # TODO: Tune for your system. + max_heap_size_bytes: 2147483648 # 2 GiB + actions: + - name: "envoy.overload_actions.shrink_heap" + triggers: + - name: "envoy.resource_monitors.fixed_heap" + threshold: + value: 0.95 + - name: "envoy.overload_actions.stop_accepting_requests" + triggers: + - name: "envoy.resource_monitors.fixed_heap" + threshold: + value: 0.98 + +envoy.config.listener.v3.Listener.per_connection_buffer_limit_bytes: 32768 # 32 KiB diff --git a/generated_api_shadow/bazel/repository_locations.bzl b/generated_api_shadow/bazel/repository_locations.bzl index c275a8c65835..77539ee9b109 100644 --- a/generated_api_shadow/bazel/repository_locations.bzl +++ b/generated_api_shadow/bazel/repository_locations.bzl @@ -13,8 +13,8 @@ GOOGLEAPIS_SHA = "a45019af4d3290f02eaeb1ce10990166978c807cb33a9692141a076ba46d14 PROMETHEUS_GIT_SHA = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" # Nov 17, 2017 PROMETHEUS_SHA = "783bdaf8ee0464b35ec0c8704871e1e72afa0005c3f3587f65d9d6694bf3911b" -UDPA_GIT_SHA = "e8cd3a4bb307e2c810cffff99f93e96e6d7fee85" # Mar 27, 2020 -UDPA_SHA256 = "1fd7857cb61daee7726fca8f4d55e4923774a8d00a53007a4093830dc0482685" +UDPA_GIT_SHA = "9f54a527e3bf4d1f4a6527f93d329fb1cc4516ac" # May 8, 2020 +UDPA_SHA256 = "7edae88586a84360203e5a4c724080c740b7b6002d5d56f5e806f27c912895cd" ZIPKINAPI_RELEASE = "0.2.2" # Aug 23, 2019 ZIPKINAPI_SHA256 = "688c4fe170821dd589f36ec45aaadc03a618a40283bc1f97da8fa11686fc816b" diff --git a/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto b/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto index 994af34c7ac2..de0bc8ffa443 100644 --- a/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto +++ b/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto @@ -19,6 +19,7 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "envoy/annotations/deprecation.proto"; +import "udpa/annotations/security.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; @@ -143,7 +144,10 @@ message Bootstrap { Admin admin = 12; // Optional overload manager configuration. - overload.v3.OverloadManager overload_manager = 15; + overload.v3.OverloadManager overload_manager = 15 [ + (udpa.annotations.security).configure_for_untrusted_downstream = true, + (udpa.annotations.security).configure_for_untrusted_upstream = true + ]; // Enable :ref:`stats for event dispatcher `, defaults to false. // Note that this records a value for each iteration of the event loop on every thread. This diff --git a/generated_api_shadow/envoy/config/bootstrap/v4alpha/bootstrap.proto b/generated_api_shadow/envoy/config/bootstrap/v4alpha/bootstrap.proto index cd05d6f4e46d..ec0a4b3d6a89 100644 --- a/generated_api_shadow/envoy/config/bootstrap/v4alpha/bootstrap.proto +++ b/generated_api_shadow/envoy/config/bootstrap/v4alpha/bootstrap.proto @@ -19,6 +19,7 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "envoy/annotations/deprecation.proto"; +import "udpa/annotations/security.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; @@ -145,7 +146,10 @@ message Bootstrap { Admin admin = 12; // Optional overload manager configuration. - overload.v3.OverloadManager overload_manager = 15; + overload.v3.OverloadManager overload_manager = 15 [ + (udpa.annotations.security).configure_for_untrusted_downstream = true, + (udpa.annotations.security).configure_for_untrusted_upstream = true + ]; // Enable :ref:`stats for event dispatcher `, defaults to false. // Note that this records a value for each iteration of the event loop on every thread. This diff --git a/generated_api_shadow/envoy/config/listener/v3/listener.proto b/generated_api_shadow/envoy/config/listener/v3/listener.proto index 2b4ecb826d86..b2892906a484 100644 --- a/generated_api_shadow/envoy/config/listener/v3/listener.proto +++ b/generated_api_shadow/envoy/config/listener/v3/listener.proto @@ -14,6 +14,7 @@ import "google/api/annotations.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; +import "udpa/annotations/security.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; @@ -106,7 +107,8 @@ message Listener { // Soft limit on size of the listener’s new connection read and write buffers. // If unspecified, an implementation defined default is applied (1MiB). - google.protobuf.UInt32Value per_connection_buffer_limit_bytes = 5; + google.protobuf.UInt32Value per_connection_buffer_limit_bytes = 5 + [(udpa.annotations.security).configure_for_untrusted_downstream = true]; // Listener metadata. core.v3.Metadata metadata = 6; diff --git a/generated_api_shadow/envoy/config/listener/v4alpha/listener.proto b/generated_api_shadow/envoy/config/listener/v4alpha/listener.proto index 4438bd2974d4..b7f32a821443 100644 --- a/generated_api_shadow/envoy/config/listener/v4alpha/listener.proto +++ b/generated_api_shadow/envoy/config/listener/v4alpha/listener.proto @@ -14,6 +14,7 @@ import "google/api/annotations.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; +import "udpa/annotations/security.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; @@ -108,7 +109,8 @@ message Listener { // Soft limit on size of the listener’s new connection read and write buffers. // If unspecified, an implementation defined default is applied (1MiB). - google.protobuf.UInt32Value per_connection_buffer_limit_bytes = 5; + google.protobuf.UInt32Value per_connection_buffer_limit_bytes = 5 + [(udpa.annotations.security).configure_for_untrusted_downstream = true]; // Listener metadata. core.v4alpha.Metadata metadata = 6; diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index d54d2a36abc4..da8ac0a6b02d 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -697,7 +697,8 @@ def checkSourceLine(line, file_path, reportError): def checkBuildLine(line, file_path, reportError): - if "@bazel_tools" in line and not (isSkylarkFile(file_path) or file_path.startswith("./bazel/")): + if "@bazel_tools" in line and not (isSkylarkFile(file_path) or file_path.startswith("./bazel/") or + "python/runfiles" in line): reportError("unexpected @bazel_tools reference, please indirect via a definition in //bazel") if not whitelistedForProtobufDeps(file_path) and '"protobuf"' in line: reportError("unexpected direct external dependency on protobuf, use " diff --git a/tools/config_validation/BUILD b/tools/config_validation/BUILD index a6e8b6db72b2..99d15311d6f0 100644 --- a/tools/config_validation/BUILD +++ b/tools/config_validation/BUILD @@ -1,10 +1,15 @@ licenses(["notice"]) # Apache 2 -load("@config_validation//:requirements.bzl", "requirement") +load("@config_validation_pip3//:requirements.bzl", "requirement") py_binary( - name = "validate_yaml_fragment", - srcs = ["validate_yaml_fragment.py"], + name = "validate_fragment", + srcs = ["validate_fragment.py"], + data = ["//tools/type_whisperer:all_protos_with_ext_pb_text.pb_text"], visibility = ["//visibility:public"], - deps = [requirement("PyYAML")], + deps = [ + requirement("PyYAML"), + "@bazel_tools//tools/python/runfiles", + "@com_google_protobuf//:protobuf_python", + ], ) diff --git a/tools/config_validation/validate_fragment.py b/tools/config_validation/validate_fragment.py new file mode 100644 index 000000000000..403b5540418f --- /dev/null +++ b/tools/config_validation/validate_fragment.py @@ -0,0 +1,56 @@ +# Validate a YAML fragment against an Envoy API proto3 type. +# +# Example usage: +# +# bazel run //tools/config_validation:validate_fragment -- \ +# envoy.config.bootstrap.v3.Bootstrap $PWD/configs/google_com_proxy.v2.yaml + +import json +import pathlib +import sys + +import yaml + +from google.protobuf import descriptor_pb2 +from google.protobuf import descriptor_pool +from google.protobuf import json_format +from google.protobuf import message_factory +from google.protobuf import text_format + +from bazel_tools.tools.python.runfiles import runfiles + + +def ValidateFragment(type_name, fragment): + """Validate a dictionary representing a JSON/YAML fragment against an Envoy API proto3 type. + + Throws Protobuf errors on parsing exceptions, successful validations produce + no result. + + Args: + type_name: a string providing the type name, e.g. + envoy.config.bootstrap.v3.Bootstrap. + fragment: a dictionary representing the parsed JSON/YAML configuration + fragment. + """ + json_fragment = json.dumps(fragment) + + r = runfiles.Create() + all_protos_pb_text_path = r.Rlocation( + 'envoy/tools/type_whisperer/all_protos_with_ext_pb_text.pb_text') + file_desc_set = descriptor_pb2.FileDescriptorSet() + text_format.Parse(pathlib.Path(all_protos_pb_text_path).read_text(), + file_desc_set, + allow_unknown_extension=True) + + pool = descriptor_pool.DescriptorPool() + for f in file_desc_set.file: + pool.Add(f) + desc = pool.FindMessageTypeByName(type_name) + msg = message_factory.MessageFactory(pool=pool).GetPrototype(desc)() + json_format.Parse(json_fragment, msg, descriptor_pool=pool) + + +if __name__ == '__main__': + type_name, yaml_path = sys.argv[1:] + ValidateFragment(type_name, yaml.load(pathlib.Path(yaml_path).read_text(), + Loader=yaml.FullLoader)) diff --git a/tools/config_validation/validate_yaml_fragment.py b/tools/config_validation/validate_yaml_fragment.py deleted file mode 100644 index 0cfac273b237..000000000000 --- a/tools/config_validation/validate_yaml_fragment.py +++ /dev/null @@ -1,3 +0,0 @@ -import yaml - -print('YAML version is %s' % yaml.__version__) diff --git a/tools/protodoc/BUILD b/tools/protodoc/BUILD index 812ceac0c66b..45480b086306 100644 --- a/tools/protodoc/BUILD +++ b/tools/protodoc/BUILD @@ -1,5 +1,7 @@ licenses(["notice"]) # Apache 2 +load("@protodoc_pip3//:requirements.bzl", "requirement") + py_binary( name = "generate_empty", srcs = ["generate_empty.py"], @@ -10,11 +12,14 @@ py_binary( py_binary( name = "protodoc", srcs = ["protodoc.py"], + data = ["//docs:edge_defaults_manifest.yaml"], visibility = ["//visibility:public"], deps = [ "//tools/api_proto_plugin", + "//tools/config_validation:validate_fragment", "@com_envoyproxy_protoc_gen_validate//validate:validate_py", "@com_github_cncf_udpa//udpa/annotations:pkg_py_proto", "@com_google_protobuf//:protobuf_python", + requirement("PyYAML"), ], ) diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index f360741db233..8911370b9409 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -10,11 +10,24 @@ import pathlib import re import string +import sys + +from bazel_tools.tools.python.runfiles import runfiles +import yaml + +# We have to do some evil things to sys.path due to the way that Python module +# resolution works; we have both tools/ trees in bazel_tools and envoy. By +# default, Bazel leaves us with a sys.path in which the @bazel_tools repository +# takes precedence. Now that we're done with importing runfiles above, we can +# just remove it from the sys.path. +sys.path = [p for p in sys.path if not p.endswith('bazel_tools')] from tools.api_proto_plugin import annotations from tools.api_proto_plugin import plugin from tools.api_proto_plugin import visitor +from tools.config_validation import validate_fragment +from udpa.annotations import security_pb2 from udpa.annotations import status_pb2 from validate import validate_pb2 @@ -388,7 +401,30 @@ def FormatAnchor(label): return '.. _%s:\n\n' % label -def FormatFieldAsDefinitionListItem(outer_type_context, type_context, field): +def FormatSecurityOptions(security_option, field, type_context, edge_default_yaml): + sections = [] + + if security_option.configure_for_untrusted_downstream: + sections.append( + Indent(4, 'This field should be configured in the presence of untrusted *downstreams*')) + if security_option.configure_for_untrusted_upstream: + sections.append( + Indent(4, 'This field should be configured in the presence of untrusted *upstreams*')) + + validate_fragment.ValidateFragment(field.type_name[1:], edge_default_yaml) + field_name = type_context.name.split('.')[-1] + example = {field_name: edge_default_yaml} + sections.append( + Indent(4, 'Example configuration for untrusted environments:\n\n') + + Indent(4, '.. code-block:: yaml\n\n') + + '\n'.join(IndentLines(6, + yaml.dump(example).split('\n')))) + + return '.. note::\n' + '\n\n'.join(sections) + + +def FormatFieldAsDefinitionListItem(outer_type_context, type_context, field, + edge_defaults_manifest): """Format a FieldDescriptorProto as RST definition list item. Args: @@ -441,13 +477,23 @@ def FormatFieldAsDefinitionListItem(outer_type_context, type_context, field): else: formatted_oneof_comment = '' + # If there is a udpa.annotations.security option, include it after the comment. + if field.options.HasExtension(security_pb2.security): + edge_default_yaml = edge_defaults_manifest.get(type_context.name) + if not edge_default_yaml: + raise ProtodocError('Missing edge default YAML example for %s' % type_context.name) + formatted_security_options = FormatSecurityOptions( + field.options.Extensions[security_pb2.security], field, type_context, edge_default_yaml) + else: + formatted_security_options = '' + comment = '(%s) ' % ', '.join([FormatFieldType(type_context, field)] + field_annotations) + formatted_leading_comment - return anchor + field.name + '\n' + MapLines(functools.partial(Indent, 2), - comment + formatted_oneof_comment) + return anchor + field.name + '\n' + MapLines(functools.partial( + Indent, 2), comment + formatted_oneof_comment) + formatted_security_options -def FormatMessageAsDefinitionList(type_context, msg): +def FormatMessageAsDefinitionList(type_context, msg, edge_defaults_manifest): """Format a DescriptorProto as RST definition list. Args: @@ -472,7 +518,8 @@ def FormatMessageAsDefinitionList(type_context, msg): type_context.oneof_names[index] = oneof_decl.name return '\n'.join( FormatFieldAsDefinitionListItem(type_context, type_context.ExtendField(index, field.name), - field) for index, field in enumerate(msg.field)) + '\n' + field, edge_defaults_manifest) + for index, field in enumerate(msg.field)) + '\n' def FormatEnumValueAsDefinitionListItem(type_context, enum_value): @@ -525,6 +572,11 @@ class RstFormatVisitor(visitor.Visitor): See visitor.Visitor for visitor method docs comments. """ + def __init__(self): + r = runfiles.Create() + with open(r.Rlocation('envoy/docs/edge_defaults_manifest.yaml'), 'r') as f: + self.edge_defaults_manifest = yaml.load(f.read()) + def VisitEnum(self, enum_proto, type_context): normal_enum_type = NormalizeTypeContextName(type_context.name) anchor = FormatAnchor(EnumCrossRefLabel(normal_enum_type)) @@ -553,7 +605,8 @@ def VisitMessage(self, msg_proto, type_context, nested_msgs, nested_enums): return '' return anchor + header + proto_link + formatted_leading_comment + FormatMessageAsJson( type_context, msg_proto) + FormatMessageAsDefinitionList( - type_context, msg_proto) + '\n'.join(nested_msgs) + '\n' + '\n'.join(nested_enums) + type_context, msg_proto, + self.edge_defaults_manifest) + '\n'.join(nested_msgs) + '\n' + '\n'.join(nested_enums) def VisitFile(self, file_proto, type_context, services, msgs, enums): has_messages = True diff --git a/tools/protodoc/requirements.txt b/tools/protodoc/requirements.txt new file mode 100644 index 000000000000..7a997b5e44bd --- /dev/null +++ b/tools/protodoc/requirements.txt @@ -0,0 +1 @@ +PyYAML==5.3.1 diff --git a/tools/protoxform/merge_active_shadow.py b/tools/protoxform/merge_active_shadow.py index 5d2cd029526c..cac6dbfe58e5 100644 --- a/tools/protoxform/merge_active_shadow.py +++ b/tools/protoxform/merge_active_shadow.py @@ -20,6 +20,7 @@ from envoy.annotations import deprecation_pb2 as _ from envoy.annotations import resource_pb2 as _ from udpa.annotations import migrate_pb2 as _ +from udpa.annotations import security_pb2 as _ from udpa.annotations import sensitive_pb2 as _ from udpa.annotations import status_pb2 as _ from udpa.annotations import versioning_pb2 as _ diff --git a/tools/protoxform/migrate.py b/tools/protoxform/migrate.py index 1be44af91acb..71f0c02da03a 100644 --- a/tools/protoxform/migrate.py +++ b/tools/protoxform/migrate.py @@ -8,7 +8,7 @@ from tools.protoxform import options from tools.protoxform import utils -from envoy.annotations import resource_pb2 +from envoy_api_canonical.envoy.annotations import resource_pb2 from udpa.annotations import migrate_pb2 from udpa.annotations import status_pb2 from google.api import annotations_pb2 @@ -249,6 +249,8 @@ def VersionUpgradeXform(n, envoy_internal_shadow, file_proto, params): v(N+1) FileDescriptorProto message. """ # Load type database. + if params['type_db_path']: + utils.LoadTypeDb(params['type_db_path']) typedb = utils.GetTypeDb() # If this isn't a proto in an upgraded package, return None. if file_proto.name not in typedb.next_version_protos or not typedb.next_version_protos[ diff --git a/tools/protoxform/protoprint.py b/tools/protoxform/protoprint.py index 804b93b86600..8108d45c57ea 100755 --- a/tools/protoxform/protoprint.py +++ b/tools/protoxform/protoprint.py @@ -36,6 +36,7 @@ from envoy.annotations import deprecation_pb2 as _ from envoy.annotations import resource_pb2 from udpa.annotations import migrate_pb2 +from udpa.annotations import security_pb2 as _ from udpa.annotations import sensitive_pb2 as _ from udpa.annotations import status_pb2 diff --git a/tools/protoxform/protoxform.py b/tools/protoxform/protoxform.py index 9331877aa17f..4bc9b55a2365 100755 --- a/tools/protoxform/protoxform.py +++ b/tools/protoxform/protoxform.py @@ -16,9 +16,10 @@ # during FileDescriptorProto printing. from google.api import annotations_pb2 as _ from validate import validate_pb2 as _ -from envoy.annotations import deprecation_pb2 as _ -from envoy.annotations import resource_pb2 +from envoy_api_canonical.envoy.annotations import deprecation_pb2 as _ +from envoy_api_canonical.envoy.annotations import resource_pb2 from udpa.annotations import migrate_pb2 +from udpa.annotations import security_pb2 as _ from udpa.annotations import sensitive_pb2 as _ from udpa.annotations import status_pb2 diff --git a/tools/type_whisperer/BUILD b/tools/type_whisperer/BUILD index 3acb95c8adbe..191e2b90d1ba 100644 --- a/tools/type_whisperer/BUILD +++ b/tools/type_whisperer/BUILD @@ -70,6 +70,12 @@ file_descriptor_set_text( deps = ["@envoy_api_canonical//:all_protos"], ) +file_descriptor_set_text( + name = "all_protos_with_ext_pb_text", + with_external_deps = True, + deps = ["@envoy_api_canonical//:all_protos"], +) + proto_cc_source( name = "embedded_all_protos", constant = "AllProtosPbText", diff --git a/tools/type_whisperer/file_descriptor_set_text.bzl b/tools/type_whisperer/file_descriptor_set_text.bzl index 2ed8c7c315fc..5146e1f82331 100644 --- a/tools/type_whisperer/file_descriptor_set_text.bzl +++ b/tools/type_whisperer/file_descriptor_set_text.bzl @@ -9,7 +9,7 @@ def _file_descriptor_set_text(ctx): args = [ctx.outputs.pb_text.path] for dep in file_descriptor_sets.to_list(): ws_name = dep.owner.workspace_name - if (not ws_name) or ws_name in ctx.attr.proto_repositories: + if (not ws_name) or ws_name in ctx.attr.proto_repositories or ctx.attr.with_external_deps: args.append(dep.path) ctx.actions.run( @@ -30,6 +30,10 @@ file_descriptor_set_text = rule( default = ["envoy_api_canonical"], allow_empty = False, ), + "with_external_deps": attr.bool( + doc = "Include file descriptors for external dependencies.", + default = False, + ), "_file_descriptor_set_text_gen": attr.label( default = Label("//tools/type_whisperer:file_descriptor_set_text_gen"), executable = True,