From 7394c12cebbb87bc3d11857eb90f8242ecac323f Mon Sep 17 00:00:00 2001 From: htuch Date: Fri, 13 Dec 2019 14:07:23 -0500 Subject: [PATCH] tools: api_booster tool for upgrading Envoy APIs (tool only) (#9329) This is a beachhead PR for a Clang Libtooling based workflow that automagically updates Envoy's source tree to the latest API version for every referenced package. So far, ths tool is only capable of inferring types and performing header fixups, later PRs will expand this. Risk level: Low Testing: Manual cleanup of all headers in source/ test/ and include/, all tests pass. Part of #8082 Signed-off-by: Harvey Tuch Signed-off-by: Prakhar --- .gitignore | 1 - api/docs/BUILD | 1 + bazel/envoy_test.bzl | 2 + include/envoy/grpc/status.h | 2 + source/common/common/BUILD | 1 + source/common/config/BUILD | 1 + source/common/http/http3/BUILD | 5 +- test/common/config/api_type_oracle_test.cc | 4 + test/extensions/quic_listeners/quiche/BUILD | 5 +- test/tools/type_whisperer/BUILD | 11 + test/tools/type_whisperer/api_type_db_test.cc | 22 ++ tools/api_boost/README.md | 28 +++ tools/api_boost/api_boost.py | 161 ++++++++++++++ tools/check_format.py | 6 +- tools/clang_tools/README.md | 2 +- tools/clang_tools/api_booster/BUILD | 14 ++ tools/clang_tools/api_booster/main.cc | 202 ++++++++++++++++++ tools/type_whisperer/BUILD | 39 +++- tools/type_whisperer/api_type_db.cc | 40 ++++ tools/type_whisperer/api_type_db.h | 20 ++ 20 files changed, 560 insertions(+), 7 deletions(-) create mode 100644 test/tools/type_whisperer/BUILD create mode 100644 test/tools/type_whisperer/api_type_db_test.cc create mode 100644 tools/api_boost/README.md create mode 100755 tools/api_boost/api_boost.py create mode 100644 tools/clang_tools/api_booster/BUILD create mode 100644 tools/clang_tools/api_booster/main.cc create mode 100644 tools/type_whisperer/api_type_db.cc create mode 100644 tools/type_whisperer/api_type_db.h diff --git a/.gitignore b/.gitignore index 5878246ab0a8..489bb1ab416d 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,6 @@ cscope.* *.pyc **/pyformat SOURCE_VERSION -source/common/config/api_type_db.generated.pb_text .settings/ *.sw* tags diff --git a/api/docs/BUILD b/api/docs/BUILD index 36e171bcd3fb..a2924cc5865a 100644 --- a/api/docs/BUILD +++ b/api/docs/BUILD @@ -10,6 +10,7 @@ package_group( # This is where you add protos that will participate in docs RST generation. proto_library( name = "protos", + visibility = ["//visibility:public"], deps = [ "//envoy/admin/v2alpha:pkg", "//envoy/api/v2:pkg", diff --git a/bazel/envoy_test.bzl b/bazel/envoy_test.bzl index 439913704b77..4ac47518ea17 100644 --- a/bazel/envoy_test.bzl +++ b/bazel/envoy_test.bzl @@ -231,11 +231,13 @@ def envoy_cc_test_library( # Envoy test binaries should be specified with this function. def envoy_cc_test_binary( name, + tags = [], **kargs): envoy_cc_binary( name, testonly = 1, linkopts = _envoy_test_linkopts(), + tags = tags + ["compilation_db_implied"], **kargs ) diff --git a/include/envoy/grpc/status.h b/include/envoy/grpc/status.h index 027ecd19f5db..b967d3e29164 100644 --- a/include/envoy/grpc/status.h +++ b/include/envoy/grpc/status.h @@ -1,5 +1,7 @@ #pragma once +#include + namespace Envoy { namespace Grpc { diff --git a/source/common/common/BUILD b/source/common/common/BUILD index f3616c60f9bf..189cf3560a70 100644 --- a/source/common/common/BUILD +++ b/source/common/common/BUILD @@ -91,6 +91,7 @@ envoy_basic_cc_library( "fmtlib", ], include_prefix = envoy_include_prefix(package_name()), + deps = ["//include/envoy/common:base_includes"], ) envoy_cc_library( diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 807ba1d47d3b..47fb3d42625d 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -226,6 +226,7 @@ envoy_cc_library( hdrs = ["pausable_ack_queue.h"], deps = [ "//source/common/common:assert_lib", + "@com_google_googleapis//google/rpc:status_cc_proto", "@envoy_api//envoy/api/v2:pkg_cc_proto", ], ) diff --git a/source/common/http/http3/BUILD b/source/common/http/http3/BUILD index 086a4f4b902d..67a18855fe7f 100644 --- a/source/common/http/http3/BUILD +++ b/source/common/http/http3/BUILD @@ -11,7 +11,10 @@ envoy_package() envoy_cc_library( name = "quic_codec_factory_lib", hdrs = ["quic_codec_factory.h"], - deps = ["//include/envoy/http:codec_interface"], + deps = [ + "//include/envoy/http:codec_interface", + "//include/envoy/network:connection_interface", + ], ) envoy_cc_library( diff --git a/test/common/config/api_type_oracle_test.cc b/test/common/config/api_type_oracle_test.cc index 4b5f436fab33..e4f9edca504c 100644 --- a/test/common/config/api_type_oracle_test.cc +++ b/test/common/config/api_type_oracle_test.cc @@ -12,6 +12,10 @@ namespace Config { namespace { TEST(ApiTypeOracleTest, All) { + // For proto descriptors only + static_cast(envoy::config::filter::http::ip_tagging::v2::IPTagging::RequestType()); + static_cast(envoy::config::filter::http::ip_tagging::v3alpha::IPTagging::RequestType()); + EXPECT_EQ(nullptr, ApiTypeOracle::inferEarlierVersionDescriptor("foo", {}, "")); EXPECT_EQ(nullptr, ApiTypeOracle::inferEarlierVersionDescriptor("envoy.ip_tagging", {}, "")); diff --git a/test/extensions/quic_listeners/quiche/BUILD b/test/extensions/quic_listeners/quiche/BUILD index 2b34c7904094..50f517b6f306 100644 --- a/test/extensions/quic_listeners/quiche/BUILD +++ b/test/extensions/quic_listeners/quiche/BUILD @@ -234,5 +234,8 @@ envoy_cc_test_library( name = "test_utils_lib", hdrs = ["test_utils.h"], tags = ["nofips"], - deps = ["@com_googlesource_quiche//:quic_core_http_spdy_session_lib"], + deps = [ + "//source/extensions/quic_listeners/quiche:quic_filter_manager_connection_lib", + "@com_googlesource_quiche//:quic_core_http_spdy_session_lib", + ], ) diff --git a/test/tools/type_whisperer/BUILD b/test/tools/type_whisperer/BUILD new file mode 100644 index 000000000000..281d75874102 --- /dev/null +++ b/test/tools/type_whisperer/BUILD @@ -0,0 +1,11 @@ +licenses(["notice"]) # Apache 2 + +load("//bazel:envoy_build_system.bzl", "envoy_cc_test", "envoy_package") + +envoy_package() + +envoy_cc_test( + name = "api_type_db_test", + srcs = ["api_type_db_test.cc"], + deps = ["//tools/type_whisperer:api_type_db_lib"], +) diff --git a/test/tools/type_whisperer/api_type_db_test.cc b/test/tools/type_whisperer/api_type_db_test.cc new file mode 100644 index 000000000000..e125b8d67806 --- /dev/null +++ b/test/tools/type_whisperer/api_type_db_test.cc @@ -0,0 +1,22 @@ +#include "gtest/gtest.h" +#include "tools/type_whisperer/api_type_db.h" + +namespace Envoy { +namespace Tools { +namespace TypeWhisperer { +namespace { + +TEST(ApiTypeDb, GetProtoPathForTypeUnknown) { + const auto unknown_type_path = ApiTypeDb::getProtoPathForType("foo"); + EXPECT_EQ(absl::nullopt, unknown_type_path); +} + +TEST(ApiTypeDb, GetProtoPathForTypeKnown) { + const auto known_type_path = ApiTypeDb::getProtoPathForType("envoy.type.Int64Range"); + EXPECT_EQ("envoy/type/range.proto", *known_type_path); +} + +} // namespace +} // namespace TypeWhisperer +} // namespace Tools +} // namespace Envoy diff --git a/tools/api_boost/README.md b/tools/api_boost/README.md new file mode 100644 index 000000000000..6a67e445c40b --- /dev/null +++ b/tools/api_boost/README.md @@ -0,0 +1,28 @@ +# Envoy API upgrades + +This directory contains tooling to support the [Envoy API versioning +guidelines](api/API_VERSIONING.md). Envoy internally tracks the latest API +version for any given package. Since each package may have a different API +version, and we have have > 15k of API protos, we require machine assistance to +scale the upgrade process. + +We refer to the process of upgrading Envoy to the latest version of the API as +*API boosting*. This is a manual process, where a developer wanting to bump +major version at the API clock invokes: + +```console +/tools/api_boost/api_boost.py --build_api_booster --generate_compilation_database +``` + +followed by `fix_format`. The full process is still WiP, but we expect that +there will be some manual fixup required of test cases (e.g. YAML fragments) as +well. + +You will need to configure `LLVM_CONFIG` as per the [Clang Libtooling setup +guide](tools/clang_tools/README.md). + +## Status + +The API boosting tooling is still WiP. It is slated to land in the v3 release +(EOY 2019), at which point it should be considered ready for general consumption +by experienced developers who work on Envoy APIs. diff --git a/tools/api_boost/api_boost.py b/tools/api_boost/api_boost.py new file mode 100755 index 000000000000..608f143f0c05 --- /dev/null +++ b/tools/api_boost/api_boost.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 + +# Tool that assists in upgrading the Envoy source tree to the latest API. +# Internally, Envoy uses the latest vN or vNalpha for a given package. Envoy +# will perform a reflection based version upgrade on any older protos that are +# presented to it in configuration at ingestion time. +# +# Usage (from a clean tree): +# +# api_boost.py --generate_compilation_database \ +# --build_api_booster + +import argparse +import functools +import json +import os +import multiprocessing as mp +import pathlib +import re +import subprocess as sp + +# Temporary location of modified files. +TMP_SWP_SUFFIX = '.tmp.swp' + +# Detect API #includes. +API_INCLUDE_REGEX = re.compile('#include "(envoy/.*)/[^/]+\.pb\.(validate\.)?h"') + + +# Update a C++ file to the latest API. +def ApiBoostFile(llvm_include_path, debug_log, path): + print('Processing %s' % path) + # Run the booster + try: + result = sp.run([ + './bazel-bin/external/envoy_dev/clang_tools/api_booster/api_booster', + '--extra-arg-before=-xc++', + '--extra-arg=-isystem%s' % llvm_include_path, '--extra-arg=-Wno-undefined-internal', path + ], + capture_output=True, + check=True) + except sp.CalledProcessError as e: + print('api_booster failure for %s: %s %s' % (path, e, e.stderr.decode('utf-8'))) + raise + if debug_log: + print(result.stderr.decode('utf-8')) + + # Consume stdout containing the list of inferred API headers. We don't have + # rewrite capabilities yet in the API booster, so we rewrite here in Python + # below. + inferred_api_includes = sorted(set(result.stdout.decode('utf-8').splitlines())) + + # We just dump the inferred API header includes at the start of the #includes + # in the file and remove all the present API header includes. This does not + # match Envoy style; we rely on later invocations of fix_format.sh to take + # care of this alignment. + output_lines = [] + include_lines = ['#include "%s"' % f for f in inferred_api_includes] + input_text = pathlib.Path(path).read_text() + for line in input_text.splitlines(): + if include_lines and line.startswith('#include'): + output_lines.extend(include_lines) + include_lines = None + # Exclude API includes, except for a special case related to v2alpha + # ext_authz; this is needed to include the service descriptor in the build + # and is a hack that will go away when we remove v2. + if re.match(API_INCLUDE_REGEX, line) and 'envoy/service/auth/v2alpha' not in line: + continue + output_lines.append(line) + + # Write to temporary file. We can't overwrite in place as we're executing + # concurrently with other ApiBoostFile() invocations that might need the file + # we're writing to. + pathlib.Path(path + TMP_SWP_SUFFIX).write_text('\n'.join(output_lines) + '\n') + + +# Replace the original file with the temporary file created by ApiBoostFile() +# for a given path. +def SwapTmpFile(path): + pathlib.Path(path + TMP_SWP_SUFFIX).rename(path) + + +# Update the Envoy source tree the latest API. +def ApiBoostTree(args): + # Optional setup of state. We need the compilation database and api_booster + # tool in place before we can start boosting. + if args.generate_compilation_database: + sp.run(['./tools/gen_compilation_database.py', '--run_bazel_build', '--include_headers'], + check=True) + + if args.build_api_booster: + # Similar to gen_compilation_database.py, we only need the cc_library for + # setup. The long term fix for this is in + # https://github.com/bazelbuild/bazel/issues/9578. + dep_build_targets = [ + '//source/...', + '//test/...', + ] + # Figure out some cc_libraries that cover most of our external deps. This is + # the same logic as in gen_compilation_database.py. + query = 'attr(include_prefix, ".+", kind(cc_library, deps({})))'.format( + ' union '.join(dep_build_targets)) + dep_lib_build_targets = sp.check_output(['bazel', 'query', query]).decode().splitlines() + # We also need some misc. stuff such as test binaries for setup of benchmark + # dep. + query = 'attr("tags", "compilation_db_dep", {})'.format(' union '.join(dep_build_targets)) + dep_lib_build_targets.extend(sp.check_output(['bazel', 'query', query]).decode().splitlines()) + extra_api_booster_args = [] + if args.debug_log: + extra_api_booster_args.append('--copt=-DENABLE_DEBUG_LOG') + + # Slightly easier to debug when we build api_booster on its own. + sp.run([ + 'bazel', + 'build', + '--strip=always', + '@envoy_dev//clang_tools/api_booster', + ] + extra_api_booster_args, + check=True) + sp.run([ + 'bazel', + 'build', + '--strip=always', + ] + dep_lib_build_targets, check=True) + + # Figure out where the LLVM include path is. We need to provide this + # explicitly as the api_booster is built inside the Bazel cache and doesn't + # know about this path. + # TODO(htuch): this is fragile and depends on Clang version, should figure out + # a cleaner approach. + llvm_include_path = os.path.join( + sp.check_output([os.getenv('LLVM_CONFIG'), '--libdir']).decode().rstrip(), + 'clang/9.0.0/include') + + # Determine the files in the target dirs eligible for API boosting, based on + # known files in the compilation database. + paths = set([]) + for entry in json.loads(pathlib.Path('compile_commands.json').read_text()): + file_path = entry['file'] + if any(file_path.startswith(prefix) for prefix in args.paths): + paths.add(file_path) + + # The API boosting is file local, so this is trivially parallelizable, use + # multiprocessing pool with default worker pool sized to cpu_count(), since + # this is CPU bound. + with mp.Pool() as p: + # We need two phases, to ensure that any dependency on files being modified + # in one thread on consumed transitive headers on the other thread isn't an + # issue. This also ensures that we complete all analysis error free before + # any mutation takes place. + p.map(functools.partial(ApiBoostFile, llvm_include_path, args.debug_log), paths) + p.map(SwapTmpFile, paths) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Update Envoy tree to the latest API') + parser.add_argument('--generate_compilation_database', action='store_true') + parser.add_argument('--build_api_booster', action='store_true') + parser.add_argument('--debug_log', action='store_true') + parser.add_argument('paths', nargs='*', default=['source', 'test', 'include']) + args = parser.parse_args() + ApiBoostTree(args) diff --git a/tools/check_format.py b/tools/check_format.py index d06cb169a27d..a7bb5daf7426 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -70,7 +70,8 @@ "./source/common/access_log/access_log_formatter.cc", "./source/extensions/filters/http/squash/squash_filter.h", "./source/extensions/filters/http/squash/squash_filter.cc", - "./source/server/http/admin.h", "./source/server/http/admin.cc") + "./source/server/http/admin.h", "./source/server/http/admin.cc", + "./tools/clang_tools/api_booster/main.cc") # Only one C++ file should instantiate grpc_init GRPC_INIT_WHITELIST = ("./source/common/grpc/google_grpc_context.cc") @@ -340,7 +341,8 @@ def isBuildFile(file_path): def isExternalBuildFile(file_path): - return isBuildFile(file_path) and file_path.startswith("./bazel/external/") + return isBuildFile(file_path) and (file_path.startswith("./bazel/external/") or + file_path.startswith("./tools/clang_tools")) def isSkylarkFile(file_path): diff --git a/tools/clang_tools/README.md b/tools/clang_tools/README.md index 2c4738d1eea5..a53ad6038af3 100644 --- a/tools/clang_tools/README.md +++ b/tools/clang_tools/README.md @@ -1,4 +1,4 @@ -# Envoy Clang libtool developer tools +# Envoy Clang Libtooling developer tools ## Overview diff --git a/tools/clang_tools/api_booster/BUILD b/tools/clang_tools/api_booster/BUILD new file mode 100644 index 000000000000..53872e22279b --- /dev/null +++ b/tools/clang_tools/api_booster/BUILD @@ -0,0 +1,14 @@ +load("//clang_tools/support:clang_tools.bzl", "envoy_clang_tools_cc_binary") + +licenses(["notice"]) # Apache 2 + +envoy_clang_tools_cc_binary( + name = "api_booster", + srcs = ["main.cc"], + deps = [ + "@clang_tools//:clang_astmatchers", + "@clang_tools//:clang_basic", + "@clang_tools//:clang_tooling", + "@envoy//tools/type_whisperer:api_type_db_lib", + ], +) diff --git a/tools/clang_tools/api_booster/main.cc b/tools/clang_tools/api_booster/main.cc new file mode 100644 index 000000000000..baaaf0d7adb0 --- /dev/null +++ b/tools/clang_tools/api_booster/main.cc @@ -0,0 +1,202 @@ +// Upgrade a single Envoy C++ file to the latest API version. +// +// Currently this tool is a WiP and only does inference of .pb[.validate].h +// #include locations. This already exercises some of the muscles we need, such +// as AST matching, rudimentary type inference and API type database lookup. +// +// NOLINT(namespace-envoy) + +#include +#include +#include + +// Declares clang::SyntaxOnlyAction. +#include "clang/ASTMatchers/ASTMatchers.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Tooling/CommonOptionsParser.h" +#include "clang/Tooling/Tooling.h" + +// Declares llvm::cl::extrahelp. +#include "llvm/Support/CommandLine.h" + +#include "tools/type_whisperer/api_type_db.h" + +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" + +// Enable to see debug log messages. +#ifdef ENABLE_DEBUG_LOG +#define DEBUG_LOG(s) \ + do { \ + std::cerr << (s) << std::endl; \ + } while (0) +#else +#define DEBUG_LOG(s) +#endif + +class ApiBooster : public clang::ast_matchers::MatchFinder::MatchCallback, + public clang::tooling::SourceFileCallbacks { +public: + // AST match callback. + void run(const clang::ast_matchers::MatchFinder::MatchResult& result) override { + // If we have a match on type, we should track the corresponding .pb.h. + if (const clang::TypeLoc* type = result.Nodes.getNodeAs("type")) { + const std::string type_name = + type->getType().getCanonicalType().getUnqualifiedType().getAsString(); + DEBUG_LOG(absl::StrCat("Matched type ", type_name)); + const auto result = getProtoPathFromCType(type_name); + if (result) { + source_api_proto_paths_.insert(*result + ".pb.h"); + } + return; + } + + // If we have a match on a call expression, check to see if it's something + // like loadFromYamlAndValidate; if so, we might need to look at the + // argument type to figure out any corresponding .pb.validate.h we require. + if (const clang::CallExpr* call_expr = result.Nodes.getNodeAs("call_expr")) { + auto* direct_callee = call_expr->getDirectCallee(); + if (direct_callee != nullptr) { + const std::unordered_map ValidateNameToArg = { + {"loadFromYamlAndValidate", 1}, + {"loadFromFileAndValidate", 1}, + {"downcastAndValidate", 0}, + {"validate", 0}, + }; + const std::string& callee_name = direct_callee->getNameInfo().getName().getAsString(); + const auto arg = ValidateNameToArg.find(callee_name); + // Sometimes we hit false positives because we aren't qualifying above. + // TODO(htuch): fix this before merging. + if (arg != ValidateNameToArg.end() && arg->second < call_expr->getNumArgs()) { + const std::string type_name = call_expr->getArg(arg->second) + ->getType() + .getCanonicalType() + .getUnqualifiedType() + .getAsString(); + const auto result = getProtoPathFromCType(type_name); + if (result) { + source_api_proto_paths_.insert(*result + ".pb.validate.h"); + } + } + } + return; + } + + // The last place we need to look for .pb.validate.h reference is + // instantiation of FactoryBase. + if (const clang::ClassTemplateSpecializationDecl* tmpl = + result.Nodes.getNodeAs("tmpl")) { + const std::string tmpl_type_name = tmpl->getSpecializedTemplate() + ->getInjectedClassNameSpecialization() + .getCanonicalType() + .getAsString(); + if (tmpl_type_name == "FactoryBase") { + const std::string type_name = tmpl->getTemplateArgs() + .get(0) + .getAsType() + .getCanonicalType() + .getUnqualifiedType() + .getAsString(); + const auto result = getProtoPathFromCType(type_name); + if (result) { + source_api_proto_paths_.insert(*result + ".pb.validate.h"); + } + } + } + } + + // Visitor callback for start of a compilation unit. + bool handleBeginSource(clang::CompilerInstance& CI) override { + source_api_proto_paths_.clear(); + return true; + } + + // Visitor callback for end of a compiation unit. + void handleEndSource() override { + // Dump known API header paths to stdout for api_boost.py to rewrite with + // (no rewriting support in this tool yet). + for (const std::string& proto_path : source_api_proto_paths_) { + std::cout << proto_path << std::endl; + } + } + +private: + // Convert from C++ type, e.g. envoy:config::v2::Cluster, to a proto path + // (minus the .proto suffix), e.g. envoy/config/v2/cluster. + absl::optional getProtoPathFromCType(const std::string& c_type_name) { + // Ignore compound or non-API types. + // TODO(htuch): without compound types, this is only an under-approximation + // of the types. Add proper logic to destructor compound types. + const std::string type_name = std::regex_replace(c_type_name, std::regex("^(class|enum) "), ""); + if (!absl::StartsWith(type_name, "envoy::") || absl::StrContains(type_name, " ")) { + return {}; + } + + // Convert from C++ to a qualified proto type. This is fairly hacky stuff, + // we're essentially reversing the conventions that the protobuf C++ + // compiler is using, e.g. replacing _ and :: with . as needed, guessing + // that a Case suffix implies some enum switching. + const std::string dotted_path = std::regex_replace(type_name, std::regex("::"), "."); + std::vector frags = absl::StrSplit(dotted_path, '.'); + for (std::string& frag : frags) { + if (!frag.empty() && isupper(frag[0])) { + frag = std::regex_replace(frag, std::regex("_"), "."); + } + } + if (absl::EndsWith(frags.back(), "Case")) { + frags.pop_back(); + } + const std::string proto_type_name = absl::StrJoin(frags, "."); + + // Use API type database to map from proto type to path. + auto result = Envoy::Tools::TypeWhisperer::ApiTypeDb::getProtoPathForType(proto_type_name); + if (result) { + // Remove the .proto extension. + return result->substr(0, result->size() - 6); + } else if (!absl::StartsWith(proto_type_name, "envoy.HotRestart") && + !absl::StartsWith(proto_type_name, "envoy.RouterCheckToolSchema") && + !absl::StartsWith(proto_type_name, "envoy.test")) { + // Die hard if we don't have a useful proto type for something that looks + // like an API type(modulo a short whitelist). + std::cerr << "Unknown API type: " << proto_type_name << std::endl; + // TODO(htuch): mabye there is a nicer way to terminate AST traversal? + ::exit(1); + } + + return {}; + } + + // Set of inferred .pb[.validate].h, updated as the AST matcher callbacks above fire. + std::set source_api_proto_paths_; +}; + +int main(int argc, const char** argv) { + // Apply a custom category to all command-line options so that they are the + // only ones displayed. + llvm::cl::OptionCategory api_booster_tool_category("api-booster options"); + + clang::tooling::CommonOptionsParser OptionsParser(argc, argv, api_booster_tool_category); + clang::tooling::ClangTool Tool(OptionsParser.getCompilations(), + OptionsParser.getSourcePathList()); + + ApiBooster api_booster; + clang::ast_matchers::MatchFinder finder; + + // Match on all mentions of types in the AST. + auto type_matcher = + clang::ast_matchers::typeLoc(clang::ast_matchers::isExpansionInMainFile()).bind("type"); + finder.addMatcher(type_matcher, &api_booster); + + // Match on all call expressions. We are interested in particular in calls + // where validation on protos is performed. + auto call_matcher = clang::ast_matchers::callExpr().bind("call_expr"); + finder.addMatcher(call_matcher, &api_booster); + + // Match on all template instantiations.We are interested in particular in + // instantiations of factories where validation on protos is performed. + auto tmpl_matcher = clang::ast_matchers::classTemplateSpecializationDecl().bind("tmpl"); + finder.addMatcher(tmpl_matcher, &api_booster); + + return Tool.run(newFrontendActionFactory(&finder, &api_booster).get()); +} diff --git a/tools/type_whisperer/BUILD b/tools/type_whisperer/BUILD index 23998b1a2015..ccc40e6a9f25 100644 --- a/tools/type_whisperer/BUILD +++ b/tools/type_whisperer/BUILD @@ -1,6 +1,7 @@ licenses(["notice"]) # Apache 2 -load("//bazel:envoy_build_system.bzl", "envoy_package", "envoy_proto_library") +load("//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package", "envoy_proto_library") +load("//tools/type_whisperer:type_database.bzl", "type_database") load("@com_google_protobuf//:protobuf.bzl", "py_proto_library") envoy_package() @@ -35,6 +36,42 @@ py_binary( ], ) +label_flag( + name = "api_type_db_target", + # TODO(htuch): break dependence of API type DB on docs target. + build_setting_default = "@envoy_api//docs:protos", + visibility = ["//visibility:public"], +) + +type_database( + name = "api_type_db", + targets = [":api_type_db_target"], + visibility = ["//visibility:public"], +) + +# Pack API type database text file into a char* string that can be referenced +# at the C++ level. +genrule( + name = "api_type_db_genrule", + srcs = [":api_type_db"], + outs = ["api_type_db_def.generated.cc"], + cmd = "(echo 'namespace Envoy { namespace Tools { namespace TypeWhisperer " + + "{ const char* ApiTypeDbPbText = R\"EOF('; cat $(SRCS); echo ')EOF\";}}}') > $@", +) + +envoy_cc_library( + name = "api_type_db_lib", + srcs = [ + "api_type_db.cc", + "api_type_db_def.generated.cc", + ], + hdrs = ["api_type_db.h"], + deps = [ + "//source/common/protobuf", + "//tools/type_whisperer:api_type_db_proto_cc_proto", + ], +) + envoy_proto_library( name = "api_type_db_proto", srcs = ["api_type_db.proto"], diff --git a/tools/type_whisperer/api_type_db.cc b/tools/type_whisperer/api_type_db.cc new file mode 100644 index 000000000000..dce28eadffd4 --- /dev/null +++ b/tools/type_whisperer/api_type_db.cc @@ -0,0 +1,40 @@ +#include "tools/type_whisperer/api_type_db.h" + +#include "common/protobuf/protobuf.h" + +#include "tools/type_whisperer/api_type_db.pb.h" + +namespace Envoy { +namespace Tools { +namespace TypeWhisperer { + +extern const char* ApiTypeDbPbText; + +namespace { + +tools::type_whisperer::TypeDb* loadApiTypeDb() { + tools::type_whisperer::TypeDb* api_type_db = new tools::type_whisperer::TypeDb; + if (Protobuf::TextFormat::ParseFromString(ApiTypeDbPbText, api_type_db)) { + return api_type_db; + } + return nullptr; +} + +const tools::type_whisperer::TypeDb& getApiTypeDb() { + static tools::type_whisperer::TypeDb* api_type_db = loadApiTypeDb(); + return *api_type_db; +} + +} // namespace + +absl::optional ApiTypeDb::getProtoPathForType(const std::string& type_name) { + auto it = getApiTypeDb().types().find(type_name); + if (it == getApiTypeDb().types().end()) { + return absl::nullopt; + } + return it->second.proto_path(); +} + +} // namespace TypeWhisperer +} // namespace Tools +} // namespace Envoy diff --git a/tools/type_whisperer/api_type_db.h b/tools/type_whisperer/api_type_db.h new file mode 100644 index 000000000000..34d4c2808e46 --- /dev/null +++ b/tools/type_whisperer/api_type_db.h @@ -0,0 +1,20 @@ +#pragma once + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" + +namespace Envoy { +namespace Tools { +namespace TypeWhisperer { + +// We don't expose the raw API type database to consumers, as this requires RTTI +// and this may be linked in environments where this is not available (e.g. +// libtooling binaries). +class ApiTypeDb { +public: + static absl::optional getProtoPathForType(const std::string& type_name); +}; + +} // namespace TypeWhisperer +} // namespace Tools +} // namespace Envoy