Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support follow_redirects, tls #1896

Merged
merged 11 commits into from
Nov 26, 2024
Merged

feat: support follow_redirects, tls #1896

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
ef2f10f
feat: support follow_redirections
catdogpandas Nov 18, 2024
a1dd6ed
feat: support tls
catdogpandas Nov 18, 2024
0ee6881
add ut
catdogpandas Nov 19, 2024
47d08ba
chore: update
catdogpandas Nov 19, 2024
f70c6ad
chore: add ut
catdogpandas Nov 19, 2024
69094ca
chore: remove raw pointer when pass CurlTLS args
catdogpandas Nov 21, 2024
127cc90
chore: update code style
catdogpandas Nov 21, 2024
1149acb
update
catdogpandas Nov 22, 2024
8c27ef3
Merge branch 'main' into feat/prom-tls-curl
catdogpandas Nov 25, 2024
c57ff19
Merge branch 'main' into feat/prom-tls-curl
catdogpandas Nov 25, 2024
ec2c869
chore: fix ut
catdogpandas Nov 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions core/common/http/AsynCurlRunner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,10 @@ bool AsynCurlRunner::AddRequestToClient(unique_ptr<AsynHttpRequest>&& request) {
headers,
request->mTimeout,
AppConfig::GetInstance()->IsHostIPReplacePolicyEnabled(),
AppConfig::GetInstance()->GetBindInterface());
AppConfig::GetInstance()->GetBindInterface(),
request->mFollowRedirects,
request->mTls);

if (curl == nullptr) {
LOG_ERROR(sLogger, ("failed to send request", "failed to init curl handler")("request address", request.get()));
request->OnSendDone(request->mResponse);
Expand Down Expand Up @@ -135,9 +138,7 @@ void AsynCurlRunner::DoRun() {
}
}

struct timeval timeout {
1, 0
};
struct timeval timeout{1, 0};
long curlTimeout = -1;
if ((mc = curl_multi_timeout(mClient, &curlTimeout)) != CURLM_OK) {
LOG_WARNING(
Expand Down
25 changes: 23 additions & 2 deletions core/common/http/Curl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,9 @@ CURL* CreateCurlHandler(const std::string& method,
curl_slist*& headers,
uint32_t timeout,
bool replaceHostWithIp,
const std::string& intf) {
const std::string& intf,
bool followRedirects,
std::optional<CurlTLS> tls) {
static DnsCache* dnsCache = DnsCache::GetInstance();

CURL* curl = curl_easy_init();
Expand Down Expand Up @@ -102,11 +104,28 @@ CURL* CreateCurlHandler(const std::string& method,
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, body.size());
}

if (followRedirects) {
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
}

if (httpsFlag) {
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
}

if (tls.has_value()) {
if (!tls->mCaFile.empty()) {
curl_easy_setopt(curl, CURLOPT_CAINFO, tls->mCaFile.c_str());
catdogpandas marked this conversation as resolved.
Show resolved Hide resolved
}
if (!tls->mCertFile.empty()) {
curl_easy_setopt(curl, CURLOPT_SSLCERT, tls->mCertFile.c_str());
}
if (!tls->mKeyFile.empty()) {
curl_easy_setopt(curl, CURLOPT_SSLKEY, tls->mKeyFile.c_str());
}
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, tls->mInsecureSkipVerify ? 0 : 1);
}

curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
if (!intf.empty()) {
curl_easy_setopt(curl, CURLOPT_INTERFACE, intf.c_str());
Expand Down Expand Up @@ -139,7 +158,9 @@ bool SendHttpRequest(std::unique_ptr<HttpRequest>&& request, HttpResponse& respo
headers,
request->mTimeout,
AppConfig::GetInstance()->IsHostIPReplacePolicyEnabled(),
AppConfig::GetInstance()->GetBindInterface());
AppConfig::GetInstance()->GetBindInterface(),
request->mFollowRedirects,
request->mTls);
if (curl == NULL) {
LOG_ERROR(sLogger,
("failed to init curl handler", "failed to init curl client")("request address", request.get()));
Expand Down
6 changes: 4 additions & 2 deletions core/common/http/Curl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@

#include <cstdint>
#include <map>
#include <string>
#include <memory>
#include <string>

#include "common/http/HttpRequest.h"
#include "common/http/HttpResponse.h"
Expand All @@ -40,7 +40,9 @@ CURL* CreateCurlHandler(const std::string& method,
curl_slist*& headers,
uint32_t timeout,
bool replaceHostWithIp = true,
const std::string& intf = "");
const std::string& intf = "",
bool followRedirects = false,
std::optional<CurlTLS> tls = std::nullopt);

bool SendHttpRequest(std::unique_ptr<HttpRequest>&& request, HttpResponse& response);

Expand Down
25 changes: 21 additions & 4 deletions core/common/http/HttpRequest.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include <cstdint>
#include <map>
#include <string>
#include <utility>

#include "common/Flags.h"
#include "common/http/HttpResponse.h"
Expand All @@ -29,6 +30,13 @@ DECLARE_FLAG_INT32(default_http_request_max_try_cnt);

namespace logtail {

struct CurlTLS {
std::string mCaFile;
std::string mCertFile;
std::string mKeyFile;
catdogpandas marked this conversation as resolved.
Show resolved Hide resolved
bool mInsecureSkipVerify = true;
};

struct HttpRequest {
std::string mMethod;
// TODO: upgrade curl to 7.62, and replace the following 4 members
Expand All @@ -43,6 +51,8 @@ struct HttpRequest {
int32_t mPort;
uint32_t mTimeout = static_cast<uint32_t>(INT32_FLAG(default_http_request_timeout_secs));
uint32_t mMaxTryCnt = static_cast<uint32_t>(INT32_FLAG(default_http_request_max_try_cnt));
bool mFollowRedirects = false;
std::optional<CurlTLS> mTls = std::nullopt;

uint32_t mTryCnt = 1;
std::chrono::system_clock::time_point mLastSendTime;
Expand All @@ -56,7 +66,9 @@ struct HttpRequest {
const std::map<std::string, std::string>& header,
const std::string& body,
uint32_t timeout = static_cast<uint32_t>(INT32_FLAG(default_http_request_timeout_secs)),
uint32_t maxTryCnt = static_cast<uint32_t>(INT32_FLAG(default_http_request_max_try_cnt)))
uint32_t maxTryCnt = static_cast<uint32_t>(INT32_FLAG(default_http_request_max_try_cnt)),
bool followRedirects = false,
std::optional<CurlTLS> tls = std::nullopt)
: mMethod(method),
mHTTPSFlag(httpsFlag),
mUrl(url),
Expand All @@ -66,7 +78,9 @@ struct HttpRequest {
mHost(host),
mPort(port),
mTimeout(timeout),
mMaxTryCnt(maxTryCnt) {}
mMaxTryCnt(maxTryCnt),
mFollowRedirects(followRedirects),
mTls(std::move(tls)) {}
virtual ~HttpRequest() = default;
};

Expand All @@ -85,8 +99,11 @@ struct AsynHttpRequest : public HttpRequest {
const std::string& body,
HttpResponse&& response = HttpResponse(),
uint32_t timeout = static_cast<uint32_t>(INT32_FLAG(default_http_request_timeout_secs)),
uint32_t maxTryCnt = static_cast<uint32_t>(INT32_FLAG(default_http_request_max_try_cnt)))
: HttpRequest(method, httpsFlag, host, port, url, query, header, body, timeout, maxTryCnt),
uint32_t maxTryCnt = static_cast<uint32_t>(INT32_FLAG(default_http_request_max_try_cnt)),
bool followRedirects = false,
std::optional<CurlTLS> tls = std::nullopt)
: HttpRequest(
method, httpsFlag, host, port, url, query, header, body, timeout, maxTryCnt, followRedirects, std::move(tls)),
mResponse(std::move(response)) {}

virtual bool IsContextValid() const = 0;
Expand Down
6 changes: 5 additions & 1 deletion core/common/http/HttpResponse.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ class curl_slist;

namespace logtail {

struct CurlTLS;

bool caseInsensitiveComp(const char lhs, const char rhs);

bool compareHeader(const std::string& lhs, const std::string& rhs);
Expand All @@ -45,7 +47,9 @@ class HttpResponse {
curl_slist*& headers,
uint32_t timeout,
bool replaceHostWithIp,
const std::string& intf);
const std::string& intf,
bool followRedirects,
std::optional<CurlTLS> tls);

public:
HttpResponse()
Expand Down
8 changes: 8 additions & 0 deletions core/prometheus/Constants.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,14 @@ const char* const PASSWORD_FILE = "password_file";
const char* const BASIC_PREFIX = "Basic ";
const char* const HONOR_LABELS = "honor_labels";
const char* const HONOR_TIMESTAMPS = "honor_timestamps";
const char* const FOLLOW_REDIRECTS = "follow_redirects";
const char* const TLS_CONFIG = "tls_config";
const char* const CA_FILE = "ca_file";
const char* const CERT_FILE = "cert_file";
const char* const KEY_FILE = "key_file";
const char* const SERVER_NAME = "server_name";
const char* const HOST = "Host";
const char* const INSECURE_SKIP_VERIFY = "insecure_skip_verify";
catdogpandas marked this conversation as resolved.
Show resolved Hide resolved

// scrape protocols, from https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config
// text/plain, application/openmetrics-text will be used
Expand Down
9 changes: 7 additions & 2 deletions core/prometheus/async/PromHttpRequest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include <chrono>
#include <cstdint>
#include <string>
#include <utility>

#include "common/http/HttpRequest.h"

Expand All @@ -20,7 +21,9 @@ PromHttpRequest::PromHttpRequest(const std::string& method,
uint32_t timeout,
uint32_t maxTryCnt,
std::shared_ptr<PromFuture<HttpResponse&, uint64_t>> future,
std::shared_ptr<PromFuture<>> isContextValidFuture)
std::shared_ptr<PromFuture<>> isContextValidFuture,
bool followRedirects,
std::optional<CurlTLS> tls)
: AsynHttpRequest(method,
httpsFlag,
host,
Expand All @@ -31,7 +34,9 @@ PromHttpRequest::PromHttpRequest(const std::string& method,
body,
std::move(response),
timeout,
maxTryCnt),
maxTryCnt,
followRedirects,
std::move(tls)),
mFuture(std::move(future)),
mIsContextValidFuture(std::move(isContextValidFuture)) {
}
Expand Down
4 changes: 3 additions & 1 deletion core/prometheus/async/PromHttpRequest.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@ class PromHttpRequest : public AsynHttpRequest {
uint32_t timeout,
uint32_t maxTryCnt,
std::shared_ptr<PromFuture<HttpResponse&, uint64_t>> future,
std::shared_ptr<PromFuture<>> isContextValidFuture = nullptr);
std::shared_ptr<PromFuture<>> isContextValidFuture = nullptr,
bool followRedirects = false,
std::optional<CurlTLS> tls = std::nullopt);
PromHttpRequest(const PromHttpRequest&) = default;
~PromHttpRequest() override = default;

Expand Down
59 changes: 59 additions & 0 deletions core/prometheus/schedulers/ScrapeConfig.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,13 @@ ScrapeConfig::ScrapeConfig()
mHonorLabels(false),
mHonorTimestamps(true),
mScheme("http"),
mFollowRedirects(true),
mEnableTLS(false),
mMaxScrapeSizeBytes(0),
mSampleLimit(0),
mSeriesLimit(0) {
}

bool ScrapeConfig::Init(const Json::Value& scrapeConfig) {
if (!InitStaticConfig(scrapeConfig)) {
return false;
Expand All @@ -41,6 +44,17 @@ bool ScrapeConfig::Init(const Json::Value& scrapeConfig) {
InitScrapeProtocols(nullJson);
}

if (scrapeConfig.isMember(prometheus::FOLLOW_REDIRECTS) && scrapeConfig[prometheus::FOLLOW_REDIRECTS].isBool()) {
catdogpandas marked this conversation as resolved.
Show resolved Hide resolved
mFollowRedirects = scrapeConfig[prometheus::FOLLOW_REDIRECTS].asBool();
}

if (scrapeConfig.isMember(prometheus::TLS_CONFIG) && scrapeConfig[prometheus::TLS_CONFIG].isObject()) {
if (!InitTLSConfig(scrapeConfig[prometheus::TLS_CONFIG])) {
LOG_ERROR(sLogger, ("tls config error", ""));
return false;
}
}

if (scrapeConfig.isMember(prometheus::ENABLE_COMPRESSION)
&& scrapeConfig[prometheus::ENABLE_COMPRESSION].isBool()) {
// InitEnableCompression(scrapeConfig[prometheus::ENABLE_COMPRESSION].asBool());
Expand Down Expand Up @@ -338,4 +352,49 @@ void ScrapeConfig::InitEnableCompression(bool enableCompression) {
}
}

bool ScrapeConfig::InitTLSConfig(const Json::Value& tlsConfig) {
if (tlsConfig.isMember(prometheus::CA_FILE)) {
if (tlsConfig[prometheus::CA_FILE].isString()) {
mTLS.mCaFile = tlsConfig[prometheus::CA_FILE].asString();
} else {
catdogpandas marked this conversation as resolved.
Show resolved Hide resolved
LOG_ERROR(sLogger, ("tls config error", ""));
return false;
}
}
if (tlsConfig.isMember(prometheus::CERT_FILE)) {
if (tlsConfig[prometheus::CERT_FILE].isString()) {
mTLS.mCertFile = tlsConfig[prometheus::CERT_FILE].asString();
} else {
LOG_ERROR(sLogger, ("tls config error", ""));
return false;
}
}
if (tlsConfig.isMember(prometheus::KEY_FILE)) {
if (tlsConfig[prometheus::KEY_FILE].isString()) {
mTLS.mKeyFile = tlsConfig[prometheus::KEY_FILE].asString();
} else {
LOG_ERROR(sLogger, ("tls config error", ""));
return false;
}
}
if (tlsConfig.isMember(prometheus::SERVER_NAME)) {
if (tlsConfig[prometheus::SERVER_NAME].isString()) {
mRequestHeaders[prometheus::HOST] = tlsConfig[prometheus::SERVER_NAME].asString();
} else {
LOG_ERROR(sLogger, ("tls config error", ""));
return false;
}
}
if (tlsConfig.isMember(prometheus::INSECURE_SKIP_VERIFY)) {
if (tlsConfig[prometheus::INSECURE_SKIP_VERIFY].isBool()) {
mTLS.mInsecureSkipVerify = tlsConfig[prometheus::INSECURE_SKIP_VERIFY].asBool();
} else {
LOG_ERROR(sLogger, ("tls config error", ""));
return false;
}
}
mEnableTLS = true;
return true;
}

} // namespace logtail
6 changes: 6 additions & 0 deletions core/prometheus/schedulers/ScrapeConfig.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include <string>
#include <vector>

#include "common/http/HttpResponse.h"
#include "prometheus/labels/Relabel.h"


Expand All @@ -28,6 +29,10 @@ class ScrapeConfig {
// enable_compression Accept-Encoding header: gzip, identity
std::map<std::string, std::string> mRequestHeaders;

bool mFollowRedirects;
bool mEnableTLS;
CurlTLS mTLS;

uint64_t mMaxScrapeSizeBytes;
uint64_t mSampleLimit;
uint64_t mSeriesLimit;
Expand All @@ -47,6 +52,7 @@ class ScrapeConfig {
bool InitAuthorization(const Json::Value& authorization);
bool InitScrapeProtocols(const Json::Value& scrapeProtocols);
void InitEnableCompression(bool enableCompression);
bool InitTLSConfig(const Json::Value& tlsConfig);

#ifdef APSARA_UNIT_TEST_MAIN
friend class ScrapeConfigUnittest;
Expand Down
Loading