Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support protobuf 3.22 or upper #2163

Merged
merged 54 commits into from
Aug 11, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
54 commits
Select commit Hold shift + click to select a range
fbb68dd
Support protobuf 3.22 or upper
owent May 27, 2023
5c013c0
Test modern gRPC
owent May 27, 2023
7b5f955
Fix dependency
owent May 27, 2023
f95fd57
Using installed protobuf to build gRPC when using modern protobuf and…
owent May 27, 2023
f016a75
Make install_abseil.sh has the same options like it in setup_grpc.sh
owent May 27, 2023
c94e488
Fix abseil-cpp dependency of proto when using protobuf 3.22 or upper.
owent May 27, 2023
0dc7575
Fix protobuf installer with package absl.
owent May 27, 2023
8db4a61
Fix CI job for custom protobuf and abseil version
owent May 27, 2023
0f0085a
Allow to set `CXX_STANDARD` by environment variable in CI job.
owent May 27, 2023
1dcceaa
Update modern gRPC version
owent May 27, 2023
6fa4e95
Fix gcc version detection
owent May 27, 2023
93d347c
Fix abseil-cpp checking.
owent May 27, 2023
234ab40
Fix stringpiece including.
owent May 29, 2023
8956193
Fix link order of grpc example
owent May 29, 2023
5f3e6db
Patch dependencies.
owent May 29, 2023
8a36378
Allow to ignore more libraries's CONFIGs.
owent May 29, 2023
2ecaf75
Patch several third party libraries to allow they are built in a diff…
owent May 31, 2023
ace2e84
Support GTest cmake package
owent May 31, 2023
2e1af98
Optimize protoc execute.
owent May 31, 2023
c562424
Rename `proto-options-patch.cmake` to `tools.cmake`
owent May 31, 2023
77f4ec1
Support set PROTOBUF_PROTOC_EXECUTABLE from protobuf::protoc
owent May 31, 2023
1fa8631
Fix GTest package on Windows
owent May 31, 2023
d3db7c8
Merge remote-tracking branch 'opentelemetry/main' into support_protob…
owent Jun 1, 2023
68aa11c
Restore `WITH_OTLP_GRPC` in `cmake.maintainer.sync.test`
owent Jun 1, 2023
f63e145
Fix sepss problem.
owent Jun 1, 2023
0ba0fb2
Merge branch 'main' into support_protobuf_3.22_or_upper
ThomsonTan Jun 1, 2023
f2e3009
Fix undefined uint64_t in metrics headers.
owent Jun 1, 2023
53ef61b
Merge remote-tracking branch 'opentelemetry/main' into support_protob…
owent Jun 6, 2023
85bea45
Remove the legacy `protobuf_MODULE_COMPATIBLE` requirement.
owent Jun 6, 2023
b903710
Fix format
owent Jun 6, 2023
f783127
Merge branch 'main' into support_protobuf_3.22_or_upper
marcalff Jun 30, 2023
afed730
Try to use c++14 for modern protobuf and gRPC
owent Jul 4, 2023
22e767c
Merge remote-tracking branch 'github/main' into support_protobuf_3.22…
owent Jul 4, 2023
ecdc611
Merge remote-tracking branch 'github/main' into support_protobuf_3.22…
owent Jul 13, 2023
43f5996
Use environment to control c++ standardn in CI jobs.
owent Jul 13, 2023
95fb02d
Merge branch 'main' into support_protobuf_3.22_or_upper
owent Jul 14, 2023
a947146
Merge remote-tracking branch 'github/main' into support_protobuf_3.22…
owent Jul 26, 2023
b24f299
Restore the changelogs by mistake.
owent Jul 26, 2023
052dfea
Merge branch 'main' into support_protobuf_3.22_or_upper
owent Aug 1, 2023
c73f9b8
Ignore more warnings os MSVC for protobuf generated sources.
owent Aug 1, 2023
7a2c75b
Remove different package CONFIG supporting.
owent Aug 1, 2023
46caba6
Merge remote-tracking branch 'github/main' into support_protobuf_3.22…
owent Aug 1, 2023
5e08424
Merge branch 'main' into support_protobuf_3.22_or_upper
lalitb Aug 2, 2023
ef1f0be
Add CI job without abseil
owent Aug 3, 2023
bb90f44
Merge remote-tracking branch 'origin/support_protobuf_3.22_or_upper' …
owent Aug 3, 2023
edeed26
Remove dependency of absl for OTLP HTTP client.
owent Aug 3, 2023
9bfce35
Fix type
owent Aug 3, 2023
0d3b76f
Move base64 encoder and decoder into sdk
owent Aug 4, 2023
67dc6ff
Early exit if base64 source is empty
owent Aug 4, 2023
b8f2e5c
Fix including
owent Aug 4, 2023
c996738
Fix style
owent Aug 4, 2023
ec5f39e
Remove unsupport base64 format of absl
owent Aug 4, 2023
fd9dc99
Fix including
owent Aug 4, 2023
fdff03d
Restore protobuf and abseil checking
owent Aug 4, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 0 additions & 9 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -351,15 +351,6 @@ endif()

if(WITH_OTLP_GRPC OR WITH_OTLP_HTTP)
find_package(Protobuf)
if(Protobuf_VERSION AND Protobuf_VERSION VERSION_GREATER_EQUAL "3.22.0")
if(NOT WITH_ABSEIL)
message(
FATAL_ERROR
"Protobuf 3.22 or upper require abseil-cpp(Recommended version: 20230125 or upper)"
)
endif()
endif()

if(WITH_OTLP_GRPC)
find_package(gRPC)
endif()
Expand Down
169 changes: 143 additions & 26 deletions exporters/otlp/src/otlp_http_client.cc
Original file line number Diff line number Diff line change
Expand Up @@ -19,25 +19,10 @@
#include "google/protobuf/stubs/common.h"
#include "nlohmann/json.hpp"

#if defined(HAVE_ABSEIL)
# include "absl/strings/escaping.h"
#elif defined(GOOGLE_PROTOBUF_VERSION) && GOOGLE_PROTOBUF_VERSION >= 3007000
# include "google/protobuf/stubs/strutil.h"
#else
# include "google/protobuf/stubs/port.h"
# include "google/protobuf/stubs/stringpiece.h"
namespace google
{
namespace protobuf
{
LIBPROTOBUF_EXPORT void Base64Escape(StringPiece src, std::string *dest);
} // namespace protobuf
} // namespace google
#endif

#include "opentelemetry/exporters/otlp/protobuf_include_suffix.h"

#include "opentelemetry/common/timestamp.h"
#include "opentelemetry/nostd/string_view.h"
#include "opentelemetry/sdk/common/global_log_handler.h"
#include "opentelemetry/sdk_config.h"

Expand Down Expand Up @@ -65,6 +50,146 @@ namespace otlp
namespace
{

using Base64EscapeChars = const unsigned char[64];
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are a lot of codes here, do you think it's better to move it into sdk?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah good point, not required. We can move it later if need to share with other components. Thanks.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

base64 algorithm is moved into sdk now.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Really sorry if my comment above was not clear. I did realize that this will bloat the sdk, and we should move it only when we need to share it with other components.

static constexpr Base64EscapeChars kBase64CharsBasic = {
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'};

static int Base64EscapeInternal(unsigned char *dest,
size_t dlen,
size_t *olen,
const unsigned char *src,
size_t slen,
Base64EscapeChars &base64_enc_map,
unsigned char padding_char)
{
size_t i, n, nopadding;
int C1, C2, C3;
unsigned char *p;

if (slen == 0)
{
*olen = 0;
return (0);
}

n = (slen + 2) / 3;

if (n > (BASE64_SIZE_T_MAX - 1) / 4)
owent marked this conversation as resolved.
Show resolved Hide resolved
{
*olen = BASE64_SIZE_T_MAX;
return -1;
}

n *= 4;

// no padding
if (0 == padding_char)
{
nopadding = slen % 3;
if (0 != nopadding)
{
n -= 3 - nopadding;
}
}

if ((dlen < n + 1) || (nullptr == dest))
{
*olen = n + 1;
return -1;
}

n = (slen / 3) * 3;

for (i = 0, p = dest; i < n; i += 3)
{
C1 = *src++;
C2 = *src++;
C3 = *src++;

*p++ = base64_enc_map[(C1 >> 2) & 0x3F];
*p++ = base64_enc_map[(((C1 & 3) << 4) + (C2 >> 4)) & 0x3F];
*p++ = base64_enc_map[(((C2 & 15) << 2) + (C3 >> 6)) & 0x3F];
*p++ = base64_enc_map[C3 & 0x3F];
}

if (i < slen)
{
C1 = *src++;
C2 = ((i + 1) < slen) ? *src++ : 0;

*p++ = base64_enc_map[(C1 >> 2) & 0x3F];
*p++ = base64_enc_map[(((C1 & 3) << 4) + (C2 >> 4)) & 0x3F];

if ((i + 1) < slen)
{
*p++ = base64_enc_map[((C2 & 15) << 2) & 0x3F];
}
else if (padding_char)
{
*p++ = padding_char;
}

if (padding_char)
{
*p++ = padding_char;
}
}

*olen = static_cast<size_t>(p - dest);
*p = 0;

return (0);
}

static inline int Base64EscapeInternal(std::string &dest,
const unsigned char *src,
size_t slen,
Base64EscapeChars &base64_enc_map,
unsigned char padding_char)
{
size_t olen = 0;
Base64EscapeInternal(nullptr, 0, &olen, src, slen, base64_enc_map, padding_char);
dest.resize(olen);

if (nullptr == src || 0 == slen)
{
return 0;
}

int ret = Base64EscapeInternal(reinterpret_cast<unsigned char *>(&dest[0]), dest.size(), &olen,
src, slen, base64_enc_map, padding_char);
#if defined(HAVE_GSL)
Expects(0 != ret || dest.size() == olen + 1);
#else
assert(0 != ret || dest.size() == olen + 1);
#endif
// pop back last zero
if (!dest.empty() && *dest.rbegin() == 0)
{
dest.resize(dest.size() - 1);
}
return ret;
}

// Base64Escape()
//
// Encodes a `src` string into a base64-encoded 'dest' string with padding
// characters. This function conforms with RFC 4648 section 4 (base64) and RFC
// 2045. See also CalculateBase64EscapedLen().
static void Base64Escape(nostd::string_view src, std::string *dest)
owent marked this conversation as resolved.
Show resolved Hide resolved
{
if (nullptr == dest)
{
return;
}

owent marked this conversation as resolved.
Show resolved Hide resolved
Base64EscapeInternal(*dest, reinterpret_cast<const unsigned char *>(src.data()), src.size(),
kBase64CharsBasic, '=');
}

owent marked this conversation as resolved.
Show resolved Hide resolved
/**
* This class handles the response message from the HTTP request
*/
Expand Down Expand Up @@ -414,22 +539,14 @@ static std::string BytesMapping(const std::string &bytes,
else
{
std::string base64_value;
#if defined(HAVE_ABSEIL)
absl::Base64Escape(bytes, &base64_value);
#else
google::protobuf::Base64Escape(bytes, &base64_value);
#endif
Base64Escape(bytes, &base64_value);
return base64_value;
}
}
case JsonBytesMappingKind::kBase64: {
// Base64 is the default bytes mapping of protobuf
std::string base64_value;
#if defined(HAVE_ABSEIL)
absl::Base64Escape(bytes, &base64_value);
#else
google::protobuf::Base64Escape(bytes, &base64_value);
#endif
Base64Escape(bytes, &base64_value);
return base64_value;
}
case JsonBytesMappingKind::kHex:
Expand Down