feat: add scrape state metrics

core/common/http/AsynCurlRunner.cpp

@@ -94,6 +94,7 @@ bool AsynCurlRunner::AddRequestToClient(unique_ptr<AsynHttpRequest>&& request) {
 
     if (curl == nullptr) {
         LOG_ERROR(sLogger, ("failed to send request", "failed to init curl handler")("request address", request.get()));
+        request->mResponse.SetNetworkStatus(CURLE_FAILED_INIT);
         request->OnSendDone(request->mResponse);
         return false;
     }
@@ -106,6 +107,7 @@ bool AsynCurlRunner::AddRequestToClient(unique_ptr<AsynHttpRequest>&& request) {
         LOG_ERROR(sLogger,
                   ("failed to send request", "failed to add the easy curl handle to multi_handle")(
                       "errMsg", curl_multi_strerror(res))("request address", request.get()));
+        request->mResponse.SetNetworkStatus(CURLE_FAILED_INIT);
         request->OnSendDone(request->mResponse);
         curl_easy_cleanup(curl);
         return false;
@@ -190,6 +192,7 @@ void AsynCurlRunner::HandleCompletedRequests(int& runningHandlers) {
                 case CURLE_OK: {
                     long statusCode = 0;
                     curl_easy_getinfo(handler, CURLINFO_RESPONSE_CODE, &statusCode);
+                    request->mResponse.SetNetworkStatus(CURLE_OK);
                     request->mResponse.SetStatusCode(statusCode);
                     request->OnSendDone(request->mResponse);
                     LOG_DEBUG(sLogger,
@@ -214,6 +217,7 @@ void AsynCurlRunner::HandleCompletedRequests(int& runningHandlers) {
                         ++runningHandlers;
                         requestReused = true;
                     } else {
+                        request->mResponse.SetNetworkStatus(msg->data.result);
                         request->OnSendDone(request->mResponse);
                         LOG_DEBUG(
                             sLogger,

core/common/http/HttpResponse.h

@@ -16,6 +16,8 @@
 
 #pragma once
 
+#include <curl/curl.h>
+
 #include <cstdint>
 #include <functional>
 #include <map>
@@ -28,6 +30,24 @@ namespace logtail {
 
 struct CurlTLS;
 
+enum NetworkCode {
+    Ok = 0,
+    ConnectionFailed,
+    RemoteAccessDenied,
+    SSLConnectError,
+    SSLCertError,
+    SSLOtherProblem,
+    SendDataFailed,
+    RecvDataFailed,
+    TIMEOUT, // 超时
+    Other // 回显请求收到
+};
+
+struct NetWorkStatus {
+    NetworkCode mCode = NetworkCode::Ok;
+    std::string mMessage;
+};
+
 bool caseInsensitiveComp(const char lhs, const char rhs);
 
 bool compareHeader(const std::string& lhs, const std::string& rhs);
@@ -76,8 +96,62 @@ class HttpResponse {
 
     void SetStatusCode(int32_t code) { mStatusCode = code; }
 
+    void SetNetworkStatus(CURLcode code) {
+        mNetworkStatus.mMessage = curl_easy_strerror(code);
+        // please refer to https://curl.se/libcurl/c/libcurl-errors.html
+        switch (code) {
+            case CURLE_OK:
+                mNetworkStatus.mCode = NetworkCode::Ok;
+                break;
+            case CURLE_COULDNT_CONNECT:
+                mNetworkStatus.mCode = NetworkCode::ConnectionFailed;
+                break;
+            case CURLE_LOGIN_DENIED:
+            case CURLE_REMOTE_ACCESS_DENIED:
+                mNetworkStatus.mCode = NetworkCode::RemoteAccessDenied;
+                break;
+            case CURLE_OPERATION_TIMEDOUT:
+                mNetworkStatus.mCode = NetworkCode::TIMEOUT;
+                break;
+            case CURLE_SSL_CONNECT_ERROR:
+                mNetworkStatus.mCode = NetworkCode::SSLConnectError;
+                break;
+            case CURLE_SSL_CERTPROBLEM:
+            case CURLE_SSL_CACERT:
+                mNetworkStatus.mCode = NetworkCode::SSLCertError;
+                break;
+            case CURLE_SEND_ERROR:
+            case CURLE_SEND_FAIL_REWIND:
+                mNetworkStatus.mCode = NetworkCode::SendDataFailed;
+                break;
+            case CURLE_RECV_ERROR:
+                mNetworkStatus.mCode = NetworkCode::RecvDataFailed;
+                break;
+            case CURLE_SSL_PINNEDPUBKEYNOTMATCH:
+            case CURLE_SSL_INVALIDCERTSTATUS:
+            case CURLE_SSL_CACERT_BADFILE:
+            case CURLE_SSL_CIPHER:
+            case CURLE_SSL_ENGINE_NOTFOUND:
+            case CURLE_SSL_ENGINE_SETFAILED:
+            case CURLE_USE_SSL_FAILED:
+            case CURLE_SSL_ENGINE_INITFAILED:
+            case CURLE_SSL_CRL_BADFILE:
+            case CURLE_SSL_ISSUER_ERROR:
+            case CURLE_SSL_SHUTDOWN_FAILED:
+                mNetworkStatus.mCode = NetworkCode::SSLOtherProblem;
+                break;
+            case CURLE_FAILED_INIT:
+            default:
+                mNetworkStatus.mCode = NetworkCode::Other;
+                break;
+        }
+    }
+
+    NetWorkStatus GetNetworkStatus() { return mNetworkStatus; }
+
 private:
     int32_t mStatusCode = 0; // 0 means no response from server
+    NetWorkStatus mNetworkStatus; // 0 means no error
     std::map<std::string, std::string, decltype(compareHeader)*> mHeader;
     std::unique_ptr<void, std::function<void(void*)>> mBody;
     size_t (*mWriteCallback)(char*, size_t, size_t, void*) = nullptr;

core/models/PipelineEventGroup.h

@@ -52,6 +52,7 @@ enum class EventGroupMetaKey {
     CONTAINER_IMAGE_NAME,
     CONTAINER_IMAGE_ID,
 
+    PROMETHEUS_SCRAPE_STATE,
     PROMETHEUS_SCRAPE_DURATION,
     PROMETHEUS_SCRAPE_RESPONSE_SIZE,
     PROMETHEUS_SAMPLES_SCRAPED,

core/plugin/processor/inner/ProcessorPromRelabelMetricNative.cpp

@@ -27,7 +27,10 @@
 #include "prometheus/Constants.h"
 
 using namespace std;
+
 DECLARE_FLAG_STRING(_pod_name_);
+DEFINE_FLAG_BOOL(enable_scrape_state, "enable scrape_state auto metric", true);
+
 namespace logtail {
 
 const string ProcessorPromRelabelMetricNative::sName = "processor_prom_relabel_metric_native";
@@ -192,6 +195,13 @@ void ProcessorPromRelabelMetricNative::AddAutoMetrics(PipelineEventGroup& metric
     // up metric must be the last one
     bool upState = StringTo<bool>(metricGroup.GetMetadata(EventGroupMetaKey::PROMETHEUS_UP_STATE).to_string());
 
+    if (BOOL_FLAG(enable_scrape_state) && metricGroup.HasMetadata(EventGroupMetaKey::PROMETHEUS_SCRAPE_STATE)) {
+        auto scrapeState = metricGroup.GetMetadata(EventGroupMetaKey::PROMETHEUS_SCRAPE_STATE);
+        AddMetric(metricGroup, prometheus::SCRAPE_STATE, 1.0 * upState, timestamp, nanoSec, targetTags);
+        auto& last = metricGroup.MutableEvents()[metricGroup.GetEvents().size() - 1];
+        last.Cast<MetricEvent>().SetTag(METRIC_LABEL_KEY_STATUS, scrapeState);
+    }
+
     AddMetric(metricGroup, prometheus::UP, 1.0 * upState, timestamp, nanoSec, targetTags);
 }
 

core/prometheus/Constants.h

@@ -99,6 +99,7 @@ const char* const PARAM_LABEL_NAME = "__param_";
 const char* const LABELS = "labels";
 
 // auto metrics
+const char* const SCRAPE_STATE = "scrape_state";
 const char* const SCRAPE_DURATION_SECONDS = "scrape_duration_seconds";
 const char* const SCRAPE_RESPONSE_SIZE_BYTES = "scrape_response_size_bytes";
 const char* const SCRAPE_SAMPLES_LIMIT = "scrape_samples_limit";

core/prometheus/Utils.cpp

@@ -5,6 +5,7 @@
 #include <iomanip>
 
 #include "common/StringTools.h"
+#include "http/HttpResponse.h"
 #include "models/StringView.h"
 
 using namespace std;
@@ -148,4 +149,26 @@ uint64_t GetRandSleepMilliSec(const std::string& key, uint64_t intervalSeconds,
     randSleep -= sleepOffset;
     return randSleep;
 }
+
+namespace prom {
+
+    std::string NetworkCodeToString(NetworkCode code) {
+        static map<uint64_t, string> sNetworkCodeMap = {{NetworkCode::Ok, "OK"},
+                                                        {NetworkCode::ConnectionFailed, "ERR_CONN_FAILED"},
+                                                        {NetworkCode::RemoteAccessDenied, "ERR_ACCESS_DENIED"},
+                                                        {NetworkCode::TIMEOUT, "ERR_TIMEOUT"},
+                                                        {NetworkCode::SSLConnectError, "ERR_SSL_CONN_ERR"},
+                                                        {NetworkCode::SSLCertError, "ERR_SSL_CERT_ERR"},
+                                                        {NetworkCode::SSLOtherProblem, "ERR_SSL_OTHER_PROBLEM"},
+                                                        {NetworkCode::SendDataFailed, "ERR_SEND_DATA_FAILED"},
+                                                        {NetworkCode::RecvDataFailed, "ERR_RECV_DATA_FAILED"},
+                                                        {NetworkCode::Other, "ERR_UNKNOWN"}};
+        static string sCurlOther = "ERR_UNKNOWN";
+        if (sNetworkCodeMap.count(code)) {
+            return sNetworkCodeMap[code];
+        }
+        return sCurlOther;
+    }
+
+} // namespace prom
 } // namespace logtail

core/prometheus/Utils.h

@@ -3,6 +3,7 @@
 
 #include <string>
 
+#include "common/http/HttpResponse.h"
 #include "models/StringView.h"
 
 namespace logtail {
@@ -19,4 +20,9 @@ void SplitStringView(const std::string& s, char delimiter, std::vector<StringVie
 bool IsNumber(const std::string& str);
 
 uint64_t GetRandSleepMilliSec(const std::string& key, uint64_t intervalSeconds, uint64_t currentMilliSeconds);
+
+namespace prom {
+    std::string NetworkCodeToString(NetworkCode code);
+}
+
 } // namespace logtail

core/prometheus/schedulers/ScrapeScheduler.cpp

@@ -95,6 +95,17 @@ void ScrapeScheduler::OnMetricResult(HttpResponse& response, uint64_t timestampM
                              response.GetStatusCode(),
                              GetCurrentTimeInMilliSeconds() - timestampMilliSec);
 
+    auto networkStatus = response.GetNetworkStatus();
+    if (networkStatus.mCode != NetworkCode::Ok) {
+        // not 0 means curl error
+        mScrapeState = prom::NetworkCodeToString(networkStatus.mCode);
+    } else if (response.GetStatusCode() != 200) {
+        mScrapeState = ToString(response.GetStatusCode());
+    } else {
+        // 0 means success
+        mScrapeState = prom::NetworkCodeToString(NetworkCode::Ok);
+    }
+
     mScrapeTimestampMilliSec = timestampMilliSec;
     mScrapeDurationSeconds = 1.0 * (GetCurrentTimeInMilliSeconds() - timestampMilliSec) / 1000;
     mScrapeResponseSizeBytes = responseBody.mRawSize;
@@ -118,6 +129,7 @@ void ScrapeScheduler::OnMetricResult(HttpResponse& response, uint64_t timestampM
 }
 
 void ScrapeScheduler::SetAutoMetricMeta(PipelineEventGroup& eGroup) {
+    eGroup.SetMetadata(EventGroupMetaKey::PROMETHEUS_SCRAPE_STATE, mScrapeState);
     eGroup.SetMetadata(EventGroupMetaKey::PROMETHEUS_SCRAPE_TIMESTAMP_MILLISEC, ToString(mScrapeTimestampMilliSec));
     eGroup.SetMetadata(EventGroupMetaKey::PROMETHEUS_SCRAPE_DURATION, ToString(mScrapeDurationSeconds));
     eGroup.SetMetadata(EventGroupMetaKey::PROMETHEUS_SCRAPE_RESPONSE_SIZE, ToString(mScrapeResponseSizeBytes));
@@ -241,11 +253,10 @@ void ScrapeScheduler::InitSelfMonitor(const MetricLabels& defaultLabels) {
     MetricLabels labels = defaultLabels;
     labels.emplace_back(METRIC_LABEL_KEY_INSTANCE, mInstance);
 
-    static const std::unordered_map<std::string, MetricType> sScrapeMetricKeys = {
-        {METRIC_PLUGIN_OUT_EVENTS_TOTAL, MetricType::METRIC_TYPE_COUNTER},
-        {METRIC_PLUGIN_OUT_SIZE_BYTES, MetricType::METRIC_TYPE_COUNTER},
-        {METRIC_PLUGIN_PROM_SCRAPE_TIME_MS, MetricType::METRIC_TYPE_COUNTER},
-    };
+    static const std::unordered_map<std::string, MetricType> sScrapeMetricKeys
+        = {{METRIC_PLUGIN_OUT_EVENTS_TOTAL, MetricType::METRIC_TYPE_COUNTER},
+           {METRIC_PLUGIN_OUT_SIZE_BYTES, MetricType::METRIC_TYPE_COUNTER},
+           {METRIC_PLUGIN_PROM_SCRAPE_TIME_MS, MetricType::METRIC_TYPE_COUNTER}};
 
     mSelfMonitor->InitMetricManager(sScrapeMetricKeys, labels);
 

core/prometheus/schedulers/ScrapeScheduler.h

@@ -96,6 +96,7 @@ class ScrapeScheduler : public BaseScheduler {
     size_t mInputIndex;
 
     // auto metrics
+    std::string mScrapeState;
     uint64_t mScrapeTimestampMilliSec = 0;
     double mScrapeDurationSeconds = 0;
     uint64_t mScrapeResponseSizeBytes = 0;