tools: api_booster tool for upgrading Envoy APIs (tool only) (envoypr…

…oxy#9329)

This is a beachhead PR for a Clang Libtooling based workflow that automagically updates Envoy's
source tree to the latest API version for every referenced package. So far, ths tool is only capable
of inferring types and performing header fixups, later PRs will expand this.

Risk level: Low
Testing: Manual cleanup of all headers in source/ test/ and include/, all tests pass.

Part of envoyproxy#8082

Signed-off-by: Harvey Tuch <htuch@google.com>
Signed-off-by: Prakhar <prakhar_au@yahoo.com>

.gitignore

@@ -19,7 +19,6 @@ cscope.*
 *.pyc
 **/pyformat
 SOURCE_VERSION
-source/common/config/api_type_db.generated.pb_text
 .settings/
 *.sw*
 tags

api/docs/BUILD

@@ -10,6 +10,7 @@ package_group(
 # This is where you add protos that will participate in docs RST generation.
 proto_library(
     name = "protos",
+    visibility = ["//visibility:public"],
     deps = [
         "//envoy/admin/v2alpha:pkg",
         "//envoy/api/v2:pkg",

bazel/envoy_test.bzl

@@ -231,11 +231,13 @@ def envoy_cc_test_library(
 # Envoy test binaries should be specified with this function.
 def envoy_cc_test_binary(
         name,
+        tags = [],
         **kargs):
     envoy_cc_binary(
         name,
         testonly = 1,
         linkopts = _envoy_test_linkopts(),
+        tags = tags + ["compilation_db_implied"],
         **kargs
     )
 

include/envoy/grpc/status.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <cstdint>
+
 namespace Envoy {
 namespace Grpc {
 

source/common/common/BUILD

@@ -91,6 +91,7 @@ envoy_basic_cc_library(
         "fmtlib",
     ],
     include_prefix = envoy_include_prefix(package_name()),
+    deps = ["//include/envoy/common:base_includes"],
 )
 
 envoy_cc_library(

source/common/config/BUILD

@@ -226,6 +226,7 @@ envoy_cc_library(
     hdrs = ["pausable_ack_queue.h"],
     deps = [
         "//source/common/common:assert_lib",
+        "@com_google_googleapis//google/rpc:status_cc_proto",
         "@envoy_api//envoy/api/v2:pkg_cc_proto",
     ],
 )

source/common/http/http3/BUILD

@@ -11,7 +11,10 @@ envoy_package()
 envoy_cc_library(
     name = "quic_codec_factory_lib",
     hdrs = ["quic_codec_factory.h"],
-    deps = ["//include/envoy/http:codec_interface"],
+    deps = [
+        "//include/envoy/http:codec_interface",
+        "//include/envoy/network:connection_interface",
+    ],
 )
 
 envoy_cc_library(

test/common/config/api_type_oracle_test.cc

@@ -12,6 +12,10 @@ namespace Config {
 namespace {
 
 TEST(ApiTypeOracleTest, All) {
+  // For proto descriptors only
+  static_cast<void>(envoy::config::filter::http::ip_tagging::v2::IPTagging::RequestType());
+  static_cast<void>(envoy::config::filter::http::ip_tagging::v3alpha::IPTagging::RequestType());
+
   EXPECT_EQ(nullptr, ApiTypeOracle::inferEarlierVersionDescriptor("foo", {}, ""));
   EXPECT_EQ(nullptr, ApiTypeOracle::inferEarlierVersionDescriptor("envoy.ip_tagging", {}, ""));
 

test/extensions/quic_listeners/quiche/BUILD

@@ -234,5 +234,8 @@ envoy_cc_test_library(
     name = "test_utils_lib",
     hdrs = ["test_utils.h"],
     tags = ["nofips"],
-    deps = ["@com_googlesource_quiche//:quic_core_http_spdy_session_lib"],
+    deps = [
+        "//source/extensions/quic_listeners/quiche:quic_filter_manager_connection_lib",
+        "@com_googlesource_quiche//:quic_core_http_spdy_session_lib",
+    ],
 )

test/tools/type_whisperer/BUILD

@@ -0,0 +1,11 @@
+licenses(["notice"])  # Apache 2
+
+load("//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_package")
+
+envoy_package()
+
+envoy_cc_test(
+    name = "api_type_db_test",
+    srcs = ["api_type_db_test.cc"],
+    deps = ["//tools/type_whisperer:api_type_db_lib"],
+)

test/tools/type_whisperer/api_type_db_test.cc

@@ -0,0 +1,22 @@
+#include "gtest/gtest.h"
+#include "tools/type_whisperer/api_type_db.h"
+
+namespace Envoy {
+namespace Tools {
+namespace TypeWhisperer {
+namespace {
+
+TEST(ApiTypeDb, GetProtoPathForTypeUnknown) {
+  const auto unknown_type_path = ApiTypeDb::getProtoPathForType("foo");
+  EXPECT_EQ(absl::nullopt, unknown_type_path);
+}
+
+TEST(ApiTypeDb, GetProtoPathForTypeKnown) {
+  const auto known_type_path = ApiTypeDb::getProtoPathForType("envoy.type.Int64Range");
+  EXPECT_EQ("envoy/type/range.proto", *known_type_path);
+}
+
+} // namespace
+} // namespace TypeWhisperer
+} // namespace Tools
+} // namespace Envoy

tools/api_boost/README.md

@@ -0,0 +1,28 @@
+# Envoy API upgrades
+
+This directory contains tooling to support the [Envoy API versioning
+guidelines](api/API_VERSIONING.md). Envoy internally tracks the latest API
+version for any given package. Since each package may have a different API
+version, and we have have > 15k of API protos, we require machine assistance to
+scale the upgrade process.
+
+We refer to the process of upgrading Envoy to the latest version of the API as
+*API boosting*. This is a manual process, where a developer wanting to bump
+major version at the API clock invokes:
+
+```console
+/tools/api_boost/api_boost.py --build_api_booster --generate_compilation_database
+```
+
+followed by `fix_format`. The full process is still WiP, but we expect that
+there will be some manual fixup required of test cases (e.g. YAML fragments) as
+well.
+
+You will need to configure `LLVM_CONFIG` as per the [Clang Libtooling setup
+guide](tools/clang_tools/README.md).
+
+## Status
+
+The API boosting tooling is still WiP. It is slated to land in the v3 release
+(EOY 2019), at which point it should be considered ready for general consumption
+by experienced developers who work on Envoy APIs.

tools/api_boost/api_boost.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+
+# Tool that assists in upgrading the Envoy source tree to the latest API.
+# Internally, Envoy uses the latest vN or vNalpha for a given package. Envoy
+# will perform a reflection based version upgrade on any older protos that are
+# presented to it in configuration at ingestion time.
+#
+# Usage (from a clean tree):
+#
+# api_boost.py --generate_compilation_database \
+#   --build_api_booster
+
+import argparse
+import functools
+import json
+import os
+import multiprocessing as mp
+import pathlib
+import re
+import subprocess as sp
+
+# Temporary location of modified files.
+TMP_SWP_SUFFIX = '.tmp.swp'
+
+# Detect API #includes.
+API_INCLUDE_REGEX = re.compile('#include "(envoy/.*)/[^/]+\.pb\.(validate\.)?h"')
+
+
+# Update a C++ file to the latest API.
+def ApiBoostFile(llvm_include_path, debug_log, path):
+  print('Processing %s' % path)
+  # Run the booster
+  try:
+    result = sp.run([
+        './bazel-bin/external/envoy_dev/clang_tools/api_booster/api_booster',
+        '--extra-arg-before=-xc++',
+        '--extra-arg=-isystem%s' % llvm_include_path, '--extra-arg=-Wno-undefined-internal', path
+    ],
+                    capture_output=True,
+                    check=True)
+  except sp.CalledProcessError as e:
+    print('api_booster failure for %s: %s %s' % (path, e, e.stderr.decode('utf-8')))
+    raise
+  if debug_log:
+    print(result.stderr.decode('utf-8'))
+
+  # Consume stdout containing the list of inferred API headers. We don't have
+  # rewrite capabilities yet in the API booster, so we rewrite here in Python
+  # below.
+  inferred_api_includes = sorted(set(result.stdout.decode('utf-8').splitlines()))
+
+  # We just dump the inferred API header includes at the start of the #includes
+  # in the file and remove all the present API header includes. This does not
+  # match Envoy style; we rely on later invocations of fix_format.sh to take
+  # care of this alignment.
+  output_lines = []
+  include_lines = ['#include "%s"' % f for f in inferred_api_includes]
+  input_text = pathlib.Path(path).read_text()
+  for line in input_text.splitlines():
+    if include_lines and line.startswith('#include'):
+      output_lines.extend(include_lines)
+      include_lines = None
+    # Exclude API includes, except for a special case related to v2alpha
+    # ext_authz; this is needed to include the service descriptor in the build
+    # and is a hack that will go away when we remove v2.
+    if re.match(API_INCLUDE_REGEX, line) and 'envoy/service/auth/v2alpha' not in line:
+      continue
+    output_lines.append(line)
+
+  # Write to temporary file. We can't overwrite in place as we're executing
+  # concurrently with other ApiBoostFile() invocations that might need the file
+  # we're writing to.
+  pathlib.Path(path + TMP_SWP_SUFFIX).write_text('\n'.join(output_lines) + '\n')
+
+
+# Replace the original file with the temporary file created by ApiBoostFile()
+# for a given path.
+def SwapTmpFile(path):
+  pathlib.Path(path + TMP_SWP_SUFFIX).rename(path)
+
+
+# Update the Envoy source tree the latest API.
+def ApiBoostTree(args):
+  # Optional setup of state. We need the compilation database and api_booster
+  # tool in place before we can start boosting.
+  if args.generate_compilation_database:
+    sp.run(['./tools/gen_compilation_database.py', '--run_bazel_build', '--include_headers'],
+           check=True)
+
+  if args.build_api_booster:
+    # Similar to gen_compilation_database.py, we only need the cc_library for
+    # setup. The long term fix for this is in
+    # https://github.com/bazelbuild/bazel/issues/9578.
+    dep_build_targets = [
+        '//source/...',
+        '//test/...',
+    ]
+    # Figure out some cc_libraries that cover most of our external deps. This is
+    # the same logic as in gen_compilation_database.py.
+    query = 'attr(include_prefix, ".+", kind(cc_library, deps({})))'.format(
+        ' union '.join(dep_build_targets))
+    dep_lib_build_targets = sp.check_output(['bazel', 'query', query]).decode().splitlines()
+    # We also need some misc. stuff such as test binaries for setup of benchmark
+    # dep.
+    query = 'attr("tags", "compilation_db_dep", {})'.format(' union '.join(dep_build_targets))
+    dep_lib_build_targets.extend(sp.check_output(['bazel', 'query', query]).decode().splitlines())
+    extra_api_booster_args = []
+    if args.debug_log:
+      extra_api_booster_args.append('--copt=-DENABLE_DEBUG_LOG')
+
+    # Slightly easier to debug when we build api_booster on its own.
+    sp.run([
+        'bazel',
+        'build',
+        '--strip=always',
+        '@envoy_dev//clang_tools/api_booster',
+    ] + extra_api_booster_args,
+           check=True)
+    sp.run([
+        'bazel',
+        'build',
+        '--strip=always',
+    ] + dep_lib_build_targets, check=True)
+
+  # Figure out where the LLVM include path is. We need to provide this
+  # explicitly as the api_booster is built inside the Bazel cache and doesn't
+  # know about this path.
+  # TODO(htuch): this is fragile and depends on Clang version, should figure out
+  # a cleaner approach.
+  llvm_include_path = os.path.join(
+      sp.check_output([os.getenv('LLVM_CONFIG'), '--libdir']).decode().rstrip(),
+      'clang/9.0.0/include')
+
+  # Determine the files in the target dirs eligible for API boosting, based on
+  # known files in the compilation database.
+  paths = set([])
+  for entry in json.loads(pathlib.Path('compile_commands.json').read_text()):
+    file_path = entry['file']
+    if any(file_path.startswith(prefix) for prefix in args.paths):
+      paths.add(file_path)
+
+  # The API boosting is file local, so this is trivially parallelizable, use
+  # multiprocessing pool with default worker pool sized to cpu_count(), since
+  # this is CPU bound.
+  with mp.Pool() as p:
+    # We need two phases, to ensure that any dependency on files being modified
+    # in one thread on consumed transitive headers on the other thread isn't an
+    # issue. This also ensures that we complete all analysis error free before
+    # any mutation takes place.
+    p.map(functools.partial(ApiBoostFile, llvm_include_path, args.debug_log), paths)
+    p.map(SwapTmpFile, paths)
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser(description='Update Envoy tree to the latest API')
+  parser.add_argument('--generate_compilation_database', action='store_true')
+  parser.add_argument('--build_api_booster', action='store_true')
+  parser.add_argument('--debug_log', action='store_true')
+  parser.add_argument('paths', nargs='*', default=['source', 'test', 'include'])
+  args = parser.parse_args()
+  ApiBoostTree(args)

tools/check_format.py

@@ -70,7 +70,8 @@
                        "./source/common/access_log/access_log_formatter.cc",
                        "./source/extensions/filters/http/squash/squash_filter.h",
                        "./source/extensions/filters/http/squash/squash_filter.cc",
-                       "./source/server/http/admin.h", "./source/server/http/admin.cc")
+                       "./source/server/http/admin.h", "./source/server/http/admin.cc",
+                       "./tools/clang_tools/api_booster/main.cc")
 
 # Only one C++ file should instantiate grpc_init
 GRPC_INIT_WHITELIST = ("./source/common/grpc/google_grpc_context.cc")
@@ -340,7 +341,8 @@ def isBuildFile(file_path):
 
 
 def isExternalBuildFile(file_path):
-  return isBuildFile(file_path) and file_path.startswith("./bazel/external/")
+  return isBuildFile(file_path) and (file_path.startswith("./bazel/external/") or
+                                     file_path.startswith("./tools/clang_tools"))
 
 
 def isSkylarkFile(file_path):

tools/clang_tools/README.md

@@ -1,4 +1,4 @@
-# Envoy Clang libtool developer tools
+# Envoy Clang Libtooling developer tools
 
 ## Overview
 

tools/clang_tools/api_booster/BUILD

@@ -0,0 +1,14 @@
+load("//clang_tools/support:clang_tools.bzl", "envoy_clang_tools_cc_binary")
+
+licenses(["notice"])  # Apache 2
+
+envoy_clang_tools_cc_binary(
+    name = "api_booster",
+    srcs = ["main.cc"],
+    deps = [
+        "@clang_tools//:clang_astmatchers",
+        "@clang_tools//:clang_basic",
+        "@clang_tools//:clang_tooling",
+        "@envoy//tools/type_whisperer:api_type_db_lib",
+    ],
+)