Full implementation of SDS

.circleci/config.yml

@@ -1,6 +1,6 @@
 references:
   envoy-build-image: &envoy-build-image
-    envoyproxy/envoy-build:1ef23d481a4701ad4a414d1ef98036bd2ed322e7
+    envoyproxy/envoy-build:e994c1c0b1cdc9a9470cff728311ff7c995685e6
 
 version: 2
 jobs:

api/envoy/config/filter/network/thrift_proxy/v2alpha1/thrift_proxy.proto

@@ -23,6 +23,9 @@ message ThriftProxy {
 
     // The Thrift proxy will assume the client is using the Thrift unframed transport.
     UNFRAMED = 2;
+
+    // The Thrift proxy will assume the client is using the Thrift header transport.
+    HEADER = 3;
   }
 
   // Supplies the type of transport that the Thrift proxy should use. Defaults to `AUTO_TRANSPORT`.

bazel/README.md

@@ -33,7 +33,7 @@ On Ubuntu, run the following commands:
  apt-get install libtool
  apt-get install cmake
  apt-get install realpath
- apt-get install clang-format-5.0
+ apt-get install clang-format-6.0
  apt-get install automake
  apt-get install ninja-build
  apt-get install curl
@@ -231,7 +231,7 @@ bazel test -c dbg --config=asan //test/...
 
 The ASAN failure stack traces include line numbers as a result of running ASAN with a `dbg` build above.
 
-If you have clang-5.0, additional checks are provided with:
+If you have clang-5.0 or newer, additional checks are provided with:
 
 ```
 bazel test -c dbg --config=clang-asan //test/...

ci/README.md

@@ -26,7 +26,7 @@ Currently there are three build images:
 * `envoyproxy/envoy-build-ubuntu` — based on Ubuntu 16.04 (Xenial) which uses the GCC 5.4 compiler.
 * `envoyproxy/envoy-build-centos` — based on CentOS 7 which uses the GCC 5.3.1 compiler (devtoolset-4).
 
-We also install and use the clang-5.0 compiler for some sanitizing runs.
+We also install and use the clang-6.0 compiler for some sanitizing runs.
 
 # Building and running tests as a developer
 
@@ -77,8 +77,8 @@ The build artifact can be found in `/tmp/envoy-docker-build/envoy/source/exe/env
 
 The `./ci/run_envoy_docker.sh './ci/do_ci.sh <TARGET>'` targets are:
 
-* `bazel.api` &mdash; build and run API tests under `-c fastbuild` with clang-5.0.
-* `bazel.asan` &mdash; build and run tests under `-c dbg --config=clang-asan` with clang-5.0.
+* `bazel.api` &mdash; build and run API tests under `-c fastbuild` with clang-6.0.
+* `bazel.asan` &mdash; build and run tests under `-c dbg --config=clang-asan` with clang-6.0.
 * `bazel.debug` &mdash; build Envoy static binary and run tests under `-c dbg`.
 * `bazel.debug.server_only` &mdash; build Envoy static binary under `-c dbg`.
 * `bazel.dev` &mdash; build Envoy static binary and run tests under `-c fastbuild` with gcc.
@@ -87,9 +87,9 @@ The `./ci/run_envoy_docker.sh './ci/do_ci.sh <TARGET>'` targets are:
 * `bazel.release.server_only` &mdash; build Envoy static binary under `-c opt` with gcc.
 * `bazel.coverage` &mdash; build and run tests under `-c dbg` with gcc, generating coverage information in `$ENVOY_DOCKER_BUILD_DIR/envoy/generated/coverage/coverage.html`.
 * `bazel.coverity` &mdash; build Envoy static binary and run Coverity Scan static analysis.
-* `bazel.tsan` &mdash; build and run tests under `-c dbg --config=clang-tsan` with clang-5.0.
-* `check_format`&mdash; run `clang-format` 5.0 and `buildifier` on entire source tree.
-* `fix_format`&mdash; run and enforce `clang-format` 5.0 and `buildifier` on entire source tree.
+* `bazel.tsan` &mdash; build and run tests under `-c dbg --config=clang-tsan` with clang-6.0.
+* `check_format`&mdash; run `clang-format-6.0` and `buildifier` on entire source tree.
+* `fix_format`&mdash; run and enforce `clang-format-6.0` and `buildifier` on entire source tree.
 * `docs`&mdash; build documentation tree in `generated/docs`.
 
 # Testing changes to the build image as a developer

ci/build_setup.sh

@@ -17,9 +17,9 @@ function setup_gcc_toolchain() {
 }
 
 function setup_clang_toolchain() {
-  export CC=clang-5.0
-  export CXX=clang++-5.0
-  export ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-5.0/bin/llvm-symbolizer
+  export CC=clang-6.0
+  export CXX=clang++-6.0
+  export ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-6.0/bin/llvm-symbolizer
   echo "$CC/$CXX toolchain configured"
 }
 

include/envoy/common/pure.h

@@ -5,4 +5,4 @@ namespace Envoy {
  * Friendly name for a pure virtual routine.
  */
 #define PURE = 0
-} // Envoy
+} // namespace Envoy

include/envoy/secret/BUILD

@@ -8,10 +8,16 @@ load(
 
 envoy_package()
 
+envoy_cc_library(
+    name = "secret_callbacks_interface",
+    hdrs = ["secret_callbacks.h"],
+)
+
 envoy_cc_library(
     name = "secret_provider_interface",
     hdrs = ["secret_provider.h"],
     deps = [
+        ":secret_callbacks_interface",
         "//include/envoy/ssl:tls_certificate_config_interface",
     ],
 )
@@ -22,5 +28,6 @@ envoy_cc_library(
     deps = [
         ":secret_provider_interface",
         "@envoy_api//envoy/api/v2/auth:cert_cc",
+        "@envoy_api//envoy/api/v2/core:config_source_cc",
     ],
 )

include/envoy/secret/secret_callbacks.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <string>
+
+#include "envoy/common/pure.h"
+
+namespace Envoy {
+namespace Secret {
+
+/**
+ * Callbacks invoked by a dynamic secret provider.
+ */
+class SecretCallbacks {
+public:
+  virtual ~SecretCallbacks() {}
+
+  virtual void onAddOrUpdateSecret() PURE;
+};
+
+} // namespace Secret
+} // namespace Envoy

include/envoy/secret/secret_manager.h

@@ -6,12 +6,17 @@
 #include "envoy/secret/secret_provider.h"
 
 namespace Envoy {
+
+namespace Server {
+namespace Configuration {
+class TransportSocketFactoryContext;
+} // namespace Configuration
+} // namespace Server
+
 namespace Secret {
 
 /**
- * A manager for static secrets.
- *
- * TODO(jaebong) Support dynamic secrets.
+ * A manager for static and dynamic secrets.
  */
 class SecretManager {
 public:
@@ -37,6 +42,20 @@ class SecretManager {
    */
   virtual TlsCertificateConfigProviderSharedPtr createInlineTlsCertificateProvider(
       const envoy::api::v2::auth::TlsCertificate& tls_certificate) PURE;
+
+  /**
+   * Finds and returns a dynamic secret provider associated to SDS config. Create
+   * a new one if such provider does not exist.
+   *
+   * @param config_source a protobuf message object contains SDS config source.
+   * @param config_name a name that uniquely refers to the SDS config source
+   * @param secret_provider_context context that provides components for creating and initializing
+   * secret provider.
+   * @return the dynamic TLS secret provider.
+   */
+  virtual TlsCertificateConfigProviderSharedPtr findOrCreateDynamicSecretProvider(
+      const envoy::api::v2::core::ConfigSource& config_source, const std::string& config_name,
+      Server::Configuration::TransportSocketFactoryContext& secret_provider_context) PURE;
 };
 
 } // namespace Secret

include/envoy/secret/secret_provider.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "envoy/common/pure.h"
+#include "envoy/secret/secret_callbacks.h"
 #include "envoy/ssl/tls_certificate_config.h"
 
 namespace Envoy {
@@ -18,7 +19,17 @@ template <class SecretType> class SecretProvider {
    */
   virtual const SecretType* secret() const PURE;
 
-  // TODO(lizan): Add more methods for dynamic secret provider.
+  /**
+   * Add secret callback into secret provider.
+   * @param callback callback that is executed by secret provider.
+   */
+  virtual void addUpdateCallback(SecretCallbacks& callback) PURE;
+
+  /**
+   * Remove secret callback from secret provider.
+   * @param callback callback that is executed by secret provider.
+   */
+  virtual void removeUpdateCallback(SecretCallbacks& callback) PURE;
 };
 
 typedef SecretProvider<Ssl::TlsCertificateConfig> TlsCertificateConfigProvider;

include/envoy/server/BUILD

@@ -177,6 +177,7 @@ envoy_cc_library(
     hdrs = ["transport_socket_config.h"],
     deps = [
         "//include/envoy/event:dispatcher_interface",
+        "//include/envoy/init:init_interface",
         "//include/envoy/local_info:local_info_interface",
         "//include/envoy/network:transport_socket_interface",
         "//include/envoy/runtime:runtime_interface",

include/envoy/server/transport_socket_config.h

@@ -3,6 +3,7 @@
 #include <string>
 
 #include "envoy/event/dispatcher.h"
+#include "envoy/init/init.h"
 #include "envoy/local_info/local_info.h"
 #include "envoy/network/transport_socket.h"
 #include "envoy/runtime/runtime.h"
@@ -63,6 +64,18 @@ class TransportSocketFactoryContext {
    * @return the server-wide stats store.
    */
   virtual Stats::Store& stats() PURE;
+
+  /**
+   * Pass an init manager to register dynamic secret provider.
+   * @param init_manager instance of init manager.
+   */
+  virtual void setInitManager(Init::Manager& init_manager) PURE;
+
+  /**
+   * @return a pointer pointing to the instance of an init manager, or nullptr
+   * if not set.
+   */
+  virtual Init::Manager* initManager() PURE;
 };
 
 class TransportSocketConfigFactory {

include/envoy/ssl/BUILD

@@ -22,7 +22,7 @@ envoy_cc_library(
     name = "context_config_interface",
     hdrs = ["context_config.h"],
     deps = [
-        ":tls_certificate_config_interface",
+        "//include/envoy/secret:secret_provider_interface",
     ],
 )
 

include/envoy/ssl/context_config.h

@@ -5,9 +5,12 @@
 #include <vector>
 
 #include "envoy/common/pure.h"
-#include "envoy/ssl/tls_certificate_config.h"
+#include "envoy/secret/secret_provider.h"
 
 namespace Envoy {
+namespace Secret {
+class SecretCallbacks;
+} // namespace Secret
 namespace Ssl {
 
 /**
@@ -95,6 +98,17 @@ class ContextConfig {
    * @return The maximum TLS protocol version to negotiate.
    */
   virtual unsigned maxProtocolVersion() const PURE;
+
+  /**
+   * @return true if the ssl config is ready.
+   */
+  virtual bool isReady() const PURE;
+
+  /**
+   * Add secret callback into context config.
+   * @param callback callback that is executed by context config.
+   */
+  virtual void setSecretUpdateCallback(Secret::SecretCallbacks& callback) PURE;
 };
 
 class ClientContextConfig : public virtual ContextConfig {

include/envoy/stats/stats_macros.h

@@ -40,4 +40,4 @@ namespace Envoy {
 #define POOL_COUNTER(POOL) POOL_COUNTER_PREFIX(POOL, "")
 #define POOL_GAUGE(POOL) POOL_GAUGE_PREFIX(POOL, "")
 #define POOL_HISTOGRAM(POOL) POOL_HISTOGRAM_PREFIX(POOL, "")
-} // Envoy
+} // namespace Envoy

include/envoy/tcp/conn_pool.h

@@ -57,6 +57,22 @@ class UpstreamCallbacks : public Network::ConnectionCallbacks {
   virtual void onUpstreamData(Buffer::Instance& data, bool end_stream) PURE;
 };
 
+/**
+ * ConnectionState is a base class for connection state maintained across requests. For example, a
+ * protocol may maintain a connection-specific request sequence number or negotiate options that
+ * affect the behavior of requests for the duration of the connection. A ConnectionState subclass
+ * is assigned to the ConnectionData to track this state when the connection is returned to the
+ * pool so that the state is available when the connection is re-used for a subsequent request.
+ * The ConnectionState assigned to a connection is automatically destroyed when the connection is
+ * closed.
+ */
+class ConnectionState {
+public:
+  virtual ~ConnectionState() {}
+};
+
+typedef std::unique_ptr<ConnectionState> ConnectionStatePtr;
+
 /*
  * ConnectionData wraps a ClientConnection allocated to a caller. Open ClientConnections are
  * released back to the pool for re-use when their containing ConnectionData is destroyed.
@@ -70,13 +86,31 @@ class ConnectionData {
    */
   virtual Network::ClientConnection& connection() PURE;
 
+  /**
+   * Sets the ConnectionState for this connection. Any existing ConnectionState is destroyed.
+   * @param ConnectionStatePtr&& new ConnectionState for this connection.
+   */
+  virtual void setConnectionState(ConnectionStatePtr&& state) PURE;
+
+  /**
+   * @return T* the current ConnectionState or nullptr if no state is set or if the state's type
+   *            is not T.
+   */
+  template <class T> T* connectionStateTyped() { return dynamic_cast<T*>(connectionState()); }
+
   /**
    * Sets the ConnectionPool::UpstreamCallbacks for the connection. If no callback is attached,
    * data from the upstream will cause the connection to be closed. Callbacks cease when the
    * connection is released.
    * @param callback the UpstreamCallbacks to invoke for upstream data
    */
   virtual void addUpstreamCallbacks(ConnectionPool::UpstreamCallbacks& callback) PURE;
+
+protected:
+  /**
+   * @return ConnectionState* pointer to the current ConnectionState or nullptr if not set
+   */
+  virtual ConnectionState* connectionState() PURE;
 };
 
 typedef std::unique_ptr<ConnectionData> ConnectionDataPtr;

include/envoy/upstream/host_description.h

@@ -112,5 +112,5 @@ class HostDescription {
 
 typedef std::shared_ptr<const HostDescription> HostDescriptionConstSharedPtr;
 
-} // Upstream
+} // namespace Upstream
 } // namespace Envoy

source/common/common/assert.h

@@ -66,4 +66,4 @@ namespace Envoy {
 // after a switch (some_enum) with all enum values included in the cases. The macro name includes
 // "GCOVR_EXCL_LINE" to exclude the macro's usage from code coverage reports.
 #define NOT_REACHED_GCOVR_EXCL_LINE PANIC("not reached")
-} // Envoy
+} // namespace Envoy

source/common/common/compiler_requirements.h

@@ -17,4 +17,4 @@ namespace Envoy {
   "ENVOY_IGNORE_GLIBCXX_USE_CXX11_ABI_ERROR=1 in your build."
 #endif
 
-} // Envoy
+} // namespace Envoy

source/common/common/logger.cc

@@ -88,5 +88,5 @@ void Registry::setLogFormat(const std::string& log_format) {
   }
 }
 
-} // Logger
+} // namespace Logger
 } // namespace Envoy

source/common/common/logger.h

@@ -44,6 +44,7 @@ namespace Logger {
   FUNCTION(router)               \
   FUNCTION(runtime)              \
   FUNCTION(stats)                \
+  FUNCTION(secret)               \
   FUNCTION(testing)              \
   FUNCTION(thrift)               \
   FUNCTION(tracing)              \
@@ -240,7 +241,7 @@ template <Id id> class Loggable {
   }
 };
 
-} // Logger
+} // namespace Logger
 
 // Convert the line macro to a string literal for concatenation in log macros.
 #define DO_STRINGIZE(x) STRINGIZE(x)

source/common/common/macros.h

@@ -44,4 +44,4 @@ namespace Envoy {
 #define FALLTHRU
 #endif
 
-} // Envoy
+} // namespace Envoy

source/common/config/BUILD

@@ -242,6 +242,7 @@ envoy_cc_library(
     hdrs = ["protobuf_link_hacks.h"],
     deps = [
         "@envoy_api//envoy/service/discovery/v2:ads_cc",
+        "@envoy_api//envoy/service/discovery/v2:sds_cc",
         "@envoy_api//envoy/service/ratelimit/v2:rls_cc",
     ],
 )