Merge remote-tracking branch 'origin/master' into grpc_proto

Signed-off-by: Tony Allen <tony@allen.gg>

.azure-pipelines/pipelines.yml

@@ -40,6 +40,7 @@ jobs:
     dependsOn: ["format"]
     # For master builds, continue even if format fails
     condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest')))
+    timeoutInMinutes: 360
     pool:
       vmImage: "ubuntu-16.04"
     steps:

api/envoy/api/v2/core/config_source.proto

@@ -135,6 +135,8 @@ message ConfigSource {
     option (validate.required) = true;
 
     // Path on the filesystem to source and watch for configuration updates.
+    // When sourcing configuration for :ref:`secret <envoy_api_msg_auth.Secret>`,
+    // the certificate and key files are also watched for updates.
     //
     // .. note::
     //

api/envoy/api/v2/listener/listener_components.proto

@@ -209,9 +209,32 @@ message FilterChain {
   string name = 7;
 }
 
-// [#not-implemented-hide:]
 // Listener filter chain match configuration. This is a recursive structure which allows complex
 // nested match configurations to be built using various logical operators.
+//
+// Examples:
+//
+// * Matches if the destination port is 3306.
+//
+// .. code-block:: yaml
+//
+//  destination_port_range:
+//   start: 3306
+//   end: 3307
+//
+// * Matches if the destination port is 3306 or 15000.
+//
+// .. code-block:: yaml
+//
+//  or_match:
+//    rules:
+//      - destination_port_range:
+//          start: 3306
+//          end: 3306
+//      - destination_port_range:
+//          start: 15000
+//          end: 15001
+//
 // [#next-free-field: 6]
 message ListenerFilterChainMatchPredicate {
   // A set of match configurations used for logical operations.
@@ -257,17 +280,8 @@ message ListenerFilter {
     google.protobuf.Any typed_config = 3;
   }
 
-  // [#not-implemented-hide:]
-  // Decide when to disable this listener filter on incoming traffic.
-  // Example:
-  // 0. always enable filter
-  //    don't set `filter_disabled`
-  // 1. disable when the destination port is 3306
-  //    rule.destination_port_range = Int32Range {start = 3306, end = 3307}
-  // 2. disable when the destination port is 3306 or 15000
-  //    rule.or_match = MatchSet.rules [
-  //       rule.destination_port_range = Int32Range {start = 3306, end = 3307},
-  //       rule.destination_port_range = Int32Range {start = 15000, end = 15001},
-  //    ]
+  // Optional match predicate used to disable the filter. The filter is enabled when this field is empty.
+  // See :ref:`ListenerFilterChainMatchPredicate <envoy_api_msg_listener.ListenerFilterChainMatchPredicate>`
+  // for further examples.
   ListenerFilterChainMatchPredicate filter_disabled = 4;
 }

api/envoy/config/core/v3/config_source.proto

@@ -144,6 +144,8 @@ message ConfigSource {
     option (validate.required) = true;
 
     // Path on the filesystem to source and watch for configuration updates.
+    // When sourcing configuration for :ref:`secret <envoy_api_msg_extensions.transport_sockets.tls.v3.Secret>`,
+    // the certificate and key files are also watched for updates.
     //
     // .. note::
     //

api/envoy/config/listener/v3/listener_components.proto

@@ -208,9 +208,32 @@ message FilterChain {
   string name = 7;
 }
 
-// [#not-implemented-hide:]
 // Listener filter chain match configuration. This is a recursive structure which allows complex
 // nested match configurations to be built using various logical operators.
+//
+// Examples:
+//
+// * Matches if the destination port is 3306.
+//
+// .. code-block:: yaml
+//
+//  destination_port_range:
+//   start: 3306
+//   end: 3307
+//
+// * Matches if the destination port is 3306 or 15000.
+//
+// .. code-block:: yaml
+//
+//  or_match:
+//    rules:
+//      - destination_port_range:
+//          start: 3306
+//          end: 3306
+//      - destination_port_range:
+//          start: 15000
+//          end: 15001
+//
 // [#next-free-field: 6]
 message ListenerFilterChainMatchPredicate {
   option (udpa.annotations.versioning).previous_message_type =
@@ -267,17 +290,8 @@ message ListenerFilter {
     google.protobuf.Any typed_config = 3;
   }
 
-  // [#not-implemented-hide:]
-  // Decide when to disable this listener filter on incoming traffic.
-  // Example:
-  // 0. always enable filter
-  //    don't set `filter_disabled`
-  // 1. disable when the destination port is 3306
-  //    rule.destination_port_range = Int32Range {start = 3306, end = 3307}
-  // 2. disable when the destination port is 3306 or 15000
-  //    rule.or_match = MatchSet.rules [
-  //       rule.destination_port_range = Int32Range {start = 3306, end = 3307},
-  //       rule.destination_port_range = Int32Range {start = 15000, end = 15001},
-  //    ]
+  // Optional match predicate used to disable the filter. The filter is enabled when this field is empty.
+  // See :ref:`ListenerFilterChainMatchPredicate <envoy_api_msg_config.listener.v3.ListenerFilterChainMatchPredicate>`
+  // for further examples.
   ListenerFilterChainMatchPredicate filter_disabled = 4;
 }

bazel/envoy_test.bzl

@@ -270,12 +270,14 @@ def envoy_cc_benchmark_binary(
 def envoy_benchmark_test(
         name,
         benchmark_binary,
-        data = []):
+        data = [],
+        **kargs):
     native.sh_test(
         name = name,
         srcs = ["//bazel:test_for_benchmark_wrapper.sh"],
         data = [":" + benchmark_binary] + data,
         args = ["%s/%s" % (native.package_name(), benchmark_binary)],
+        **kargs
     )
 
 # Envoy Python test binaries should be specified with this function.

bazel/external/compiler_rt.BUILD

@@ -2,8 +2,6 @@ licenses(["notice"])  # Apache 2
 
 cc_library(
     name = "fuzzed_data_provider",
-    hdrs = ["utils/FuzzedDataProvider.h"],
-    # This is moving from lib/fuzzer/utils to include/fuzzer after LLVM 9.0.
-    include_prefix = "compiler_rt/fuzzer",
+    hdrs = ["fuzzer/utils/FuzzedDataProvider.h"],
     visibility = ["//visibility:public"],
 )

bazel/repository_locations.bzl

@@ -302,7 +302,7 @@ REPOSITORY_LOCATIONS = dict(
     org_llvm_releases_compiler_rt = dict(
         sha256 = "56e4cd96dd1d8c346b07b4d6b255f976570c6f2389697347a6c3dcb9e820d10e",
         # Only allow peeking at fuzzer related files for now.
-        strip_prefix = "compiler-rt-9.0.0.src/lib/fuzzer",
+        strip_prefix = "compiler-rt-9.0.0.src/lib",
         urls = ["http://releases.llvm.org/9.0.0/compiler-rt-9.0.0.src.tar.xz"],
     ),
     fuzzit_linux = dict(

docs/root/configuration/security/secret.rst

@@ -33,7 +33,7 @@ SDS Configuration
 
 *SdsSecretConfig* is used in two fields in :ref:`CommonTlsContext <envoy_api_msg_auth.CommonTlsContext>`. The first field is *tls_certificate_sds_secret_configs* to use SDS to get :ref:`TlsCertificate <envoy_api_msg_auth.TlsCertificate>`. The second field is *validation_context_sds_secret_config* to use SDS to get :ref:`CertificateValidationContext <envoy_api_msg_auth.CertificateValidationContext>`.
 
-Examples one: static_resource
+Example one: static_resource
 -----------------------------
 
 This example show how to configure secrets in the static_resource:
@@ -88,7 +88,9 @@ This example show how to configure secrets in the static_resource:
 
 In this example, certificates are specified in the bootstrap static_resource, they are not fetched remotely. In the config, *secrets* static resource has 3 secrets: **client_cert**, **server_cert** and **validation_context**. In the cluster config, one of hosts uses **client_cert** in its *tls_certificate_sds_secret_configs*. In the listeners section, one of them uses **server_cert** in its *tls_certificate_sds_secret_configs* and **validation_context** for its *validation_context_sds_secret_config*.
 
-Examples two: SDS server
+.. _sds_server_example:
+
+Example two: SDS server
 ------------------------
 
 This example shows how to configure secrets fetched from remote SDS servers:
@@ -173,6 +175,68 @@ This example shows how to configure secrets fetched from remote SDS servers:
 
 For illustration, above example uses three methods to access the SDS server. A gRPC SDS server can be reached by Unix Domain Socket path **/tmp/uds_path** and **127.0.0.1:8234** by mTLS. It provides three secrets, **client_cert**, **server_cert** and **validation_context**. In the config, cluster **example_cluster** certificate **client_cert** is configured to use Google gRPC with UDS to talk to the SDS server. The Listener needs to fetch **server_cert** and **validation_context** from the SDS server. The **server_cert** is using Envoy gRPC with cluster **sds_server_mtls** configured with client certificate to use mTLS to talk to SDS server. The **validate_context** is using Envoy gRPC with cluster **sds_server_uds** configured with UDS path to talk to the SDS server.
 
+.. _xds_certificate_rotation:
+
+Example three: certificate rotation for xDS gRPC connection
+------------------------------------------------------------
+
+Managing certificates for xDS gRPC connection between Envoy and xDS server introduces a bootstrapping problem: SDS server cannot manage certificates that are required to connect to the server.
+
+This example shows how to set up xDS connection by sourcing SDS configuration from the filesystem.
+The certificate and key files are watched with inotify and reloaded automatically without restart.
+In contrast, :ref:`sds_server_example` requires a restart to reload xDS certificates and key after update.
+
+.. code-block:: yaml
+
+    clusters:
+    - name: control_plane
+      type: LOGICAL_DNS
+      connect_timeout: 1s
+      load_assignment:
+        cluster_name: control_plane
+        endpoints:
+        - lb_endpoints:
+          - endpoint:
+              address:
+                socket_address:
+                  address: controlplane
+                  port_value: 8443
+      http2_protocol_options: {}
+      transport_socket:
+        name: "envoy.transport_sockets.tls"
+        typed_config:
+          "@type": "type.googleapis.com/envoy.api.v2.auth.UpstreamTlsContext"
+          common_tls_context:
+            tls_certificate_sds_secret_configs:
+              sds_config:
+                path: /etc/envoy/tls_certificate_sds_secret.yaml
+            validation_context_sds_secret_config:
+              sds_config:
+                path: /etc/envoy/validation_context_sds_secret.yaml
+
+Paths to client certificate, including client's certificate chain and private key are given in SDS config file ``/etc/envoy/tls_certificate_sds_secret.yaml``:
+
+.. code-block:: yaml
+
+    resources:
+      - "@type": "type.googleapis.com/envoy.api.v2.auth.Secret"
+        tls_certificate:
+          certificate_chain:
+            filename: /certs/sds_cert.pem
+          private_key:
+            filename: /certs/sds_key.pem
+
+Path to CA certificate bundle for validating the xDS server certificate is given in SDS config file ``/etc/envoy/validation_context_sds_secret.yaml``:
+
+.. code-block:: yaml
+
+    resources:
+      - "@type": "type.googleapis.com/envoy.api.v2.auth.Secret"
+        validation_context:
+          trusted_ca:
+            filename: /certs/cacert.pem
+
+
 Statistics
 ----------
 SSL socket factory outputs following SDS related statistics. They are all counter type.
@@ -194,4 +258,3 @@ For upstream clusters, they are in the *cluster.<CLUSTER_NAME>.client_ssl_socket
 
      ssl_context_update_by_sds, Total number of ssl context has been updated.
      upstream_context_secrets_not_ready, Total number of upstream connections reset due to empty ssl certificate.
-

docs/root/intro/arch_overview/security/ssl.rst

@@ -163,7 +163,9 @@ Only a single TLS certificate is supported today for :ref:`UpstreamTlsContexts
 Secret discovery service (SDS)
 ------------------------------
 
-TLS certificates can be specified in the static resource or can be fetched remotely. Please see :ref:`SDS <config_secret_discovery_service>` for details.
+TLS certificates can be specified in the static resource or can be fetched remotely.
+Certificate rotation is supported for static resources by sourcing :ref:`SDS configuration from the filesystem <xds_certificate_rotation>` or by pushing updates from the SDS server.
+Please see :ref:`SDS <config_secret_discovery_service>` for details.
 
 .. _arch_overview_ssl_auth_filter:
 

docs/root/intro/version_history.rst

@@ -39,6 +39,7 @@ Version history
 * http: the runtime feature `http.connection_manager.log_flood_exception` is removed and replaced with a connection access log response code.
 * listener filters: listener filter extensions use the "envoy.filters.listener" name space. A
   mapping of extension names is available in the :ref:`deprecated <deprecated>` documentation.
+* listeners: added :ref:`listener filter matcher api <envoy_api_field_listener.ListenerFilter.filter_disabled>` to disable individual listener filter on matching downstream connections.
 * listeners: fixed issue where :ref:`TLS inspector listener filter <config_listener_filters_tls_inspector>` could have been bypassed by a client using only TLS 1.3.
 * loadbalancing: added support for using hostname for consistent hash loadbalancing via :ref:`consistent_hash_lb_config <envoy_api_field_Cluster.CommonLbConfig.consistent_hashing_lb_config>`.
 * lua: added a parameter to `httpCall` that makes it possible to have the call be asynchronous.
@@ -65,6 +66,7 @@ Version history
   disables the use of deprecated extension names.
 * runtime: integer values may now be parsed as booleans.
 * sds: added :ref:`GenericSecret <envoy_api_msg_auth.GenericSecret>` to support secret of generic type.
+* sds: added :ref:`certificate rotation <xds_certificate_rotation>` support for certificates in static resources.
 * sds: fix the SDS vulnerability that TLS validation context (e.g., subject alt name or hash) cannot be effectively validated in some cases.
 * stat sinks: stat sink extensions use the "envoy.stat_sinks" name space. A mapping of extension
   names is available in the :ref:`deprecated <deprecated>` documentation.