From 733f85345d64642600155694dde74940a46a3d75 Mon Sep 17 00:00:00 2001 From: linrunqi08 <90741255+linrunqi08@users.noreply.github.com> Date: Thu, 24 Aug 2023 13:55:54 +0800 Subject: [PATCH] [WIP]: add metric new (#1015) * add metric * refine code * refine code * refine code * refine code * refine code * add exportor * add exportor * refine code and test * refine code * fix comment * fix comment * fix raw ptr * fix comment * add change log * add group by region * fix comments * fix cooment * fix cooment * fix comments * refine code * refine code * refine code * refine code * refine code * refine code * refine code * refine code * refine code * refine code * refine code * refine code * refine coder * refine coder * refine code * refine code * refine code * refine code * refine code --- CHANGELOG.md | 2 + core/CMakeLists.txt | 5 +- core/aggregator/Aggregator.cpp | 2 +- core/aggregator/CMakeLists.txt | 2 +- core/app_config/AppConfigBase.cpp | 4 +- core/app_config/CMakeLists.txt | 2 +- core/checkpoint/CMakeLists.txt | 2 +- core/checkpoint/CheckPointManager.cpp | 2 +- core/checkpoint/CheckpointManagerV2.cpp | 2 +- core/common/LogFileCollectOffsetIndicator.cpp | 2 +- core/common/LogGroupContext.h | 2 +- core/common/LogstoreSenderQueue.cpp | 2 +- core/config_manager/CMakeLists.txt | 2 +- core/config_manager/ConfigManager.cpp | 8 +- core/config_manager/ConfigManagerBase.cpp | 8 +- core/controller/CMakeLists.txt | 2 +- core/controller/EventDispatcher.cpp | 8 +- core/controller/EventDispatcher.h | 2 +- core/controller/EventDispatcherBase.cpp | 10 +- core/controller/EventDispatcherBase.h | 2 +- core/event_handler/CMakeLists.txt | 2 +- core/event_handler/EventHandler.cpp | 2 +- core/event_handler/LogInput.cpp | 2 +- core/event_listener/CMakeLists.txt | 2 +- core/event_listener/EventListener_Linux.cpp | 2 +- core/logtail.cpp | 8 +- core/logtail_windows.cpp | 8 +- core/monitor/CMakeLists.txt | 1 - .../{profiler => monitor}/LogFileProfiler.cpp | 0 core/{profiler => monitor}/LogFileProfiler.h | 0 core/{profiler => monitor}/LogIntegrity.cpp | 0 core/{profiler => monitor}/LogIntegrity.h | 0 core/{profiler => monitor}/LogLineCount.cpp | 0 core/{profiler => monitor}/LogLineCount.h | 0 core/monitor/LogtaiMetric.cpp | 326 ++++++++++++++++++ core/{profiler => monitor}/LogtailAlarm.cpp | 0 core/{profiler => monitor}/LogtailAlarm.h | 0 core/monitor/LogtailMetric.h | 123 +++++++ core/monitor/MetricConstants.h | 36 ++ core/monitor/MetricExportor.cpp | 39 +++ core/monitor/MetricExportor.h | 22 ++ core/monitor/Monitor.cpp | 4 +- core/observer/CMakeLists.txt | 2 +- core/observer/interface/statistics.h | 2 +- core/observer/metas/ContainerProcessGroup.cpp | 2 +- core/parser/CMakeLists.txt | 2 +- core/parser/LogParser.cpp | 4 +- core/plugin/CMakeLists.txt | 2 +- core/plugin/LogtailPlugin.cpp | 4 +- core/polling/CMakeLists.txt | 2 +- core/polling/PollingDirFile.cpp | 2 +- core/polling/PollingModify.cpp | 2 +- core/processor/CMakeLists.txt | 2 +- core/processor/LogFilter.cpp | 2 +- core/processor/LogProcess.cpp | 8 +- core/processor/RegexFilterValueNode.h | 2 +- core/profiler/CMakeLists.txt | 26 -- core/reader/CMakeLists.txt | 2 +- core/reader/DelimiterLogFileReader.cpp | 2 +- core/reader/JsonLogFileReader.cpp | 2 +- core/reader/LogFileReader.cpp | 4 +- core/sender/CMakeLists.txt | 2 +- core/sender/Sender.cpp | 8 +- core/sls_control/SLSControl.cpp | 2 +- core/unittest/CMakeLists.txt | 2 +- .../{profiler => monitor}/CMakeLists.txt | 6 +- .../DataIntegrityUnittest.cpp | 0 .../monitor/LogtailMetricUnittest.cpp | 324 +++++++++++++++++ core/unittest/polling/PollingUnittest.cpp | 2 +- core/unittest/sender/SenderUnittest.cpp | 4 +- 70 files changed, 961 insertions(+), 111 deletions(-) rename core/{profiler => monitor}/LogFileProfiler.cpp (100%) rename core/{profiler => monitor}/LogFileProfiler.h (100%) rename core/{profiler => monitor}/LogIntegrity.cpp (100%) rename core/{profiler => monitor}/LogIntegrity.h (100%) rename core/{profiler => monitor}/LogLineCount.cpp (100%) rename core/{profiler => monitor}/LogLineCount.h (100%) create mode 100644 core/monitor/LogtaiMetric.cpp rename core/{profiler => monitor}/LogtailAlarm.cpp (100%) rename core/{profiler => monitor}/LogtailAlarm.h (100%) create mode 100644 core/monitor/LogtailMetric.h create mode 100644 core/monitor/MetricConstants.h create mode 100644 core/monitor/MetricExportor.cpp create mode 100644 core/monitor/MetricExportor.h delete mode 100644 core/profiler/CMakeLists.txt rename core/unittest/{profiler => monitor}/CMakeLists.txt (82%) rename core/unittest/{profiler => monitor}/DataIntegrityUnittest.cpp (100%) create mode 100644 core/unittest/monitor/LogtailMetricUnittest.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 69cbd0aaa3..75efdbf092 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,3 +36,5 @@ your changes, such as: - [public] [both] [updated] add a new feature ## [Unreleased] + +- [public] [both] [added] add new logtail metric module diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 7b10b217ad..34180643f1 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -81,8 +81,8 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/dependencies.cmake) # Subdirectories (modules). set(SUB_DIRECTORIES_LIST aggregator app_config checkpoint common config config_manager config_server_pb - controller event event_handler event_listener helper log_pb logger monitor - parser plugin polling processor profiler reader profile_sender sender shennong sdk + controller event event_handler event_listener helper log_pb logger monitor + parser plugin polling processor reader profile_sender sender shennong sdk fuse sls_control ) if (UNIX) @@ -120,7 +120,6 @@ target_link_libraries(${LOGTAIL_TARGET} plugin monitor sender - profiler profile_sender app_config sls_control diff --git a/core/aggregator/Aggregator.cpp b/core/aggregator/Aggregator.cpp index bb87e23d10..d7c896062f 100644 --- a/core/aggregator/Aggregator.cpp +++ b/core/aggregator/Aggregator.cpp @@ -13,7 +13,7 @@ // limitations under the License. #include -#include +#include #include #include #include diff --git a/core/aggregator/CMakeLists.txt b/core/aggregator/CMakeLists.txt index 07e5bae272..05195a326c 100644 --- a/core/aggregator/CMakeLists.txt +++ b/core/aggregator/CMakeLists.txt @@ -23,4 +23,4 @@ target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} config_manager) target_link_libraries(${PROJECT_NAME} sender) target_link_libraries(${PROJECT_NAME} shennong) -target_link_libraries(${PROJECT_NAME} profiler) \ No newline at end of file +target_link_libraries(${PROJECT_NAME} monitor) \ No newline at end of file diff --git a/core/app_config/AppConfigBase.cpp b/core/app_config/AppConfigBase.cpp index 999009d883..964af0d045 100644 --- a/core/app_config/AppConfigBase.cpp +++ b/core/app_config/AppConfigBase.cpp @@ -19,8 +19,8 @@ #endif #include "shennong/MetricSender.h" #include "sender/Sender.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogtailAlarm.h" #include "monitor/Monitor.h" #include "common/util.h" #include "common/LogtailCommonFlags.h" diff --git a/core/app_config/CMakeLists.txt b/core/app_config/CMakeLists.txt index 55428b7f13..1f3b68ffe4 100644 --- a/core/app_config/CMakeLists.txt +++ b/core/app_config/CMakeLists.txt @@ -23,4 +23,4 @@ target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} config_manager) target_link_libraries(${PROJECT_NAME} sender) target_link_libraries(${PROJECT_NAME} shennong) -target_link_libraries(${PROJECT_NAME} profiler) \ No newline at end of file +target_link_libraries(${PROJECT_NAME} monitor) \ No newline at end of file diff --git a/core/checkpoint/CMakeLists.txt b/core/checkpoint/CMakeLists.txt index b72b525a07..7b2782f292 100644 --- a/core/checkpoint/CMakeLists.txt +++ b/core/checkpoint/CMakeLists.txt @@ -20,7 +20,7 @@ append_source_files(LIB_SOURCE_FILES) add_library(${PROJECT_NAME} STATIC ${LIB_SOURCE_FILES}) target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} logger) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} config_manager) target_link_libraries(${PROJECT_NAME} app_config) link_leveldb(${PROJECT_NAME}) \ No newline at end of file diff --git a/core/checkpoint/CheckPointManager.cpp b/core/checkpoint/CheckPointManager.cpp index 54482a4683..f8c7e9936f 100644 --- a/core/checkpoint/CheckPointManager.cpp +++ b/core/checkpoint/CheckPointManager.cpp @@ -17,7 +17,7 @@ #include #include #include -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "app_config/AppConfig.h" #include "config_manager/ConfigManager.h" #include "common/Flags.h" diff --git a/core/checkpoint/CheckpointManagerV2.cpp b/core/checkpoint/CheckpointManagerV2.cpp index 58d65a7a92..56b38bbc69 100644 --- a/core/checkpoint/CheckpointManagerV2.cpp +++ b/core/checkpoint/CheckpointManagerV2.cpp @@ -19,7 +19,7 @@ #include "common/ScopeInvoker.h" #include "common/TimeUtil.h" #include "logger/Logger.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "app_config/AppConfig.h" #include "config/Config.h" #include "config_manager/ConfigManager.h" diff --git a/core/common/LogFileCollectOffsetIndicator.cpp b/core/common/LogFileCollectOffsetIndicator.cpp index 16ef620bd4..f3b32e18d3 100644 --- a/core/common/LogFileCollectOffsetIndicator.cpp +++ b/core/common/LogFileCollectOffsetIndicator.cpp @@ -15,7 +15,7 @@ #include "LogFileCollectOffsetIndicator.h" #include #include -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "common/Flags.h" #include "common/LogstoreSenderQueue.h" #include "sender/Sender.h" diff --git a/core/common/LogGroupContext.h b/core/common/LogGroupContext.h index c071d68b0a..7abffd9387 100644 --- a/core/common/LogGroupContext.h +++ b/core/common/LogGroupContext.h @@ -16,7 +16,7 @@ #pragma once #include "config/IntegrityConfig.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "FileInfo.h" #include "RangeCheckpoint.h" diff --git a/core/common/LogstoreSenderQueue.cpp b/core/common/LogstoreSenderQueue.cpp index 81c9e96779..73e5d8c3d1 100644 --- a/core/common/LogstoreSenderQueue.cpp +++ b/core/common/LogstoreSenderQueue.cpp @@ -15,7 +15,7 @@ #include "LogstoreSenderQueue.h" #include "common/Flags.h" #include "common/StringTools.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" DEFINE_FLAG_INT32(max_client_send_error_count, "set disabled if last send is all failed", 60); diff --git a/core/config_manager/CMakeLists.txt b/core/config_manager/CMakeLists.txt index 0664992881..81f42db89e 100644 --- a/core/config_manager/CMakeLists.txt +++ b/core/config_manager/CMakeLists.txt @@ -29,7 +29,7 @@ target_link_libraries(${PROJECT_NAME} config_server_pb) target_link_libraries(${PROJECT_NAME} controller) target_link_libraries(${PROJECT_NAME} event_handler) target_link_libraries(${PROJECT_NAME} sender) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} checkpoint) target_link_libraries(${PROJECT_NAME} app_config) target_link_libraries(${PROJECT_NAME} processor) diff --git a/core/config_manager/ConfigManager.cpp b/core/config_manager/ConfigManager.cpp index 8339337f53..55afb233c4 100644 --- a/core/config_manager/ConfigManager.cpp +++ b/core/config_manager/ConfigManager.cpp @@ -45,10 +45,10 @@ #include "common/GlobalPara.h" #include "common/version.h" #include "config/UserLogConfigParser.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogIntegrity.h" -#include "profiler/LogLineCount.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogIntegrity.h" +#include "monitor/LogLineCount.h" #include "sdk/Common.h" #include "sdk/CurlImp.h" #include "sdk/Exception.h" diff --git a/core/config_manager/ConfigManagerBase.cpp b/core/config_manager/ConfigManagerBase.cpp index 144f1b3099..17cd230fae 100644 --- a/core/config_manager/ConfigManagerBase.cpp +++ b/core/config_manager/ConfigManagerBase.cpp @@ -43,10 +43,10 @@ #include "common/GlobalPara.h" #include "common/version.h" #include "config/UserLogConfigParser.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogIntegrity.h" -#include "profiler/LogLineCount.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogIntegrity.h" +#include "monitor/LogLineCount.h" #include "app_config/AppConfig.h" #include "config_manager/ConfigYamlToJson.h" #include "checkpoint/CheckPointManager.h" diff --git a/core/controller/CMakeLists.txt b/core/controller/CMakeLists.txt index 9dddfdbdb4..243af2c031 100644 --- a/core/controller/CMakeLists.txt +++ b/core/controller/CMakeLists.txt @@ -28,7 +28,7 @@ target_link_libraries(${PROJECT_NAME} event) target_link_libraries(${PROJECT_NAME} event_handler) target_link_libraries(${PROJECT_NAME} event_listener) target_link_libraries(${PROJECT_NAME} sender) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} app_config) target_link_libraries(${PROJECT_NAME} log_pb) target_link_libraries(${PROJECT_NAME} shennong) diff --git a/core/controller/EventDispatcher.cpp b/core/controller/EventDispatcher.cpp index eb322dc777..d2c6e8ee99 100644 --- a/core/controller/EventDispatcher.cpp +++ b/core/controller/EventDispatcher.cpp @@ -50,10 +50,10 @@ #include "event/Event.h" #include "processor/LogProcess.h" #include "sender/Sender.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogIntegrity.h" -#include "profiler/LogLineCount.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogIntegrity.h" +#include "monitor/LogLineCount.h" #include "log_pb/metric.pb.h" #include "log_pb/sls_logs.pb.h" #include "checkpoint/CheckPointManager.h" diff --git a/core/controller/EventDispatcher.h b/core/controller/EventDispatcher.h index 8a3e9a680c..4ff662583c 100644 --- a/core/controller/EventDispatcher.h +++ b/core/controller/EventDispatcher.h @@ -29,7 +29,7 @@ #include #include #include -#include "profiler/LogFileProfiler.h" +#include "monitor/LogFileProfiler.h" #include "polling/PollingModify.h" #include "polling/PollingDirFile.h" #include "event_listener/EventListener.h" diff --git a/core/controller/EventDispatcherBase.cpp b/core/controller/EventDispatcherBase.cpp index 6a0e3d39cc..42d21e113b 100644 --- a/core/controller/EventDispatcherBase.cpp +++ b/core/controller/EventDispatcherBase.cpp @@ -51,10 +51,11 @@ #include "event/Event.h" #include "processor/LogProcess.h" #include "sender/Sender.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogIntegrity.h" -#include "profiler/LogLineCount.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogIntegrity.h" +#include "monitor/LogLineCount.h" +#include "monitor/MetricExportor.h" #include "log_pb/metric.pb.h" #include "log_pb/sls_logs.pb.h" #include "checkpoint/CheckPointManager.h" @@ -878,6 +879,7 @@ bool EventDispatcherBase::Dispatch() { DumpCheckPointPeriod(curTime); if (curTime - lastCheckDir >= INT32_FLAG(main_loop_check_interval)) { LogFileProfiler::GetInstance()->SendProfileData(); + MetricExportor::GetInstance()->PushMetrics(false); #if defined(__linux__) CheckShennong(); #endif diff --git a/core/controller/EventDispatcherBase.h b/core/controller/EventDispatcherBase.h index c737861391..1bcaabc691 100644 --- a/core/controller/EventDispatcherBase.h +++ b/core/controller/EventDispatcherBase.h @@ -29,7 +29,7 @@ #include #include #include -#include "profiler/LogFileProfiler.h" +#include "monitor/LogFileProfiler.h" #include "polling/PollingModify.h" #include "polling/PollingDirFile.h" #include "event_listener/EventListener.h" diff --git a/core/event_handler/CMakeLists.txt b/core/event_handler/CMakeLists.txt index cdbdc0dba4..19722fa0c8 100644 --- a/core/event_handler/CMakeLists.txt +++ b/core/event_handler/CMakeLists.txt @@ -25,6 +25,6 @@ target_link_libraries(${PROJECT_NAME} app_config) target_link_libraries(${PROJECT_NAME} event) target_link_libraries(${PROJECT_NAME} controller) target_link_libraries(${PROJECT_NAME} config_manager) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} processor) target_link_libraries(${PROJECT_NAME} fuse) diff --git a/core/event_handler/EventHandler.cpp b/core/event_handler/EventHandler.cpp index 9ebafeabf6..354e80a4b1 100644 --- a/core/event_handler/EventHandler.cpp +++ b/core/event_handler/EventHandler.cpp @@ -25,7 +25,7 @@ #include "event/BlockEventManager.h" #include "controller/EventDispatcher.h" #include "config_manager/ConfigManager.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "processor/LogProcess.h" #include "logger/Logger.h" #include "fuse/FuseFileBlacklist.h" diff --git a/core/event_handler/LogInput.cpp b/core/event_handler/LogInput.cpp index f4e2945b9a..1b6e62d1e7 100644 --- a/core/event_handler/LogInput.cpp +++ b/core/event_handler/LogInput.cpp @@ -24,7 +24,7 @@ #include "monitor/Monitor.h" #include "processor/LogProcess.h" #include "controller/EventDispatcher.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "checkpoint/CheckPointManager.h" #include "polling/PollingDirFile.h" #include "polling/PollingModify.h" diff --git a/core/event_listener/CMakeLists.txt b/core/event_listener/CMakeLists.txt index 5b95fd9dca..58a0f5c802 100644 --- a/core/event_listener/CMakeLists.txt +++ b/core/event_listener/CMakeLists.txt @@ -30,5 +30,5 @@ target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} logger) target_link_libraries(${PROJECT_NAME} polling) target_link_libraries(${PROJECT_NAME} event) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} controller) \ No newline at end of file diff --git a/core/event_listener/EventListener_Linux.cpp b/core/event_listener/EventListener_Linux.cpp index 6dfe176802..005824d9f1 100644 --- a/core/event_listener/EventListener_Linux.cpp +++ b/core/event_listener/EventListener_Linux.cpp @@ -17,7 +17,7 @@ #include #include #include "logger/Logger.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "common/ErrorUtil.h" #include "common/Flags.h" #include "controller/EventDispatcher.h" diff --git a/core/logtail.cpp b/core/logtail.cpp index c7383e8625..cf88bdaea0 100644 --- a/core/logtail.cpp +++ b/core/logtail.cpp @@ -40,10 +40,10 @@ #include "controller/EventDispatcher.h" #include "monitor/Monitor.h" #include "sender/Sender.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogIntegrity.h" -#include "profiler/LogLineCount.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogIntegrity.h" +#include "monitor/LogLineCount.h" #include "app_config/AppConfig.h" #include "ObserverManager.h" using namespace logtail; diff --git a/core/logtail_windows.cpp b/core/logtail_windows.cpp index 143fe2dc4d..d8f9a1fd7f 100644 --- a/core/logtail_windows.cpp +++ b/core/logtail_windows.cpp @@ -38,10 +38,10 @@ #include "controller/EventDispatcher.h" #include "monitor/Monitor.h" #include "sender/Sender.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogIntegrity.h" -#include "profiler/LogLineCount.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogIntegrity.h" +#include "monitor/LogLineCount.h" #include "app_config/AppConfig.h" using namespace logtail; diff --git a/core/monitor/CMakeLists.txt b/core/monitor/CMakeLists.txt index 5170cea98e..f24a185f85 100644 --- a/core/monitor/CMakeLists.txt +++ b/core/monitor/CMakeLists.txt @@ -27,7 +27,6 @@ target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} logger) target_link_libraries(${PROJECT_NAME} log_pb) target_link_libraries(${PROJECT_NAME} sender) -target_link_libraries(${PROJECT_NAME} profiler) target_link_libraries(${PROJECT_NAME} config_manager) target_link_libraries(${PROJECT_NAME} app_config) target_link_libraries(${PROJECT_NAME} event_handler) diff --git a/core/profiler/LogFileProfiler.cpp b/core/monitor/LogFileProfiler.cpp similarity index 100% rename from core/profiler/LogFileProfiler.cpp rename to core/monitor/LogFileProfiler.cpp diff --git a/core/profiler/LogFileProfiler.h b/core/monitor/LogFileProfiler.h similarity index 100% rename from core/profiler/LogFileProfiler.h rename to core/monitor/LogFileProfiler.h diff --git a/core/profiler/LogIntegrity.cpp b/core/monitor/LogIntegrity.cpp similarity index 100% rename from core/profiler/LogIntegrity.cpp rename to core/monitor/LogIntegrity.cpp diff --git a/core/profiler/LogIntegrity.h b/core/monitor/LogIntegrity.h similarity index 100% rename from core/profiler/LogIntegrity.h rename to core/monitor/LogIntegrity.h diff --git a/core/profiler/LogLineCount.cpp b/core/monitor/LogLineCount.cpp similarity index 100% rename from core/profiler/LogLineCount.cpp rename to core/monitor/LogLineCount.cpp diff --git a/core/profiler/LogLineCount.h b/core/monitor/LogLineCount.h similarity index 100% rename from core/profiler/LogLineCount.h rename to core/monitor/LogLineCount.h diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp new file mode 100644 index 0000000000..ab29423012 --- /dev/null +++ b/core/monitor/LogtaiMetric.cpp @@ -0,0 +1,326 @@ +#include "LogtailMetric.h" +#include "common/StringTools.h" +#include "MetricConstants.h" +#include "logger/Logger.h" +#include "common/TimeUtil.h" +#include "app_config/AppConfig.h" + + +using namespace sls_logs; + +namespace logtail { + +Counter::Counter(const std::string& name, uint64_t val = 0) : mName(name), mVal(val) { +} + +uint64_t Counter::GetValue() const { + return mVal; +} + +const std::string& Counter::GetName() const { + return mName; +} + +Counter* Counter::CopyAndReset() { + return new Counter(mName, mVal.exchange(0)); +} + +void Counter::Add(uint64_t value) { + mVal += value; +} + +Gauge::Gauge(const std::string& name, uint64_t val = 0) : mName(name), mVal(val) { +} + +uint64_t Gauge::GetValue() const { + return mVal; +} + +const std::string& Gauge::GetName() const { + return mName; +} + +Gauge* Gauge::CopyAndReset() { + return new Gauge(mName, mVal.exchange(0)); +} + +void Gauge::Set(uint64_t value) { + mVal = value; +} + +MetricsRecord::MetricsRecord(LabelsPtr labels) : mLabels(labels), mDeleted(false) { +} + +CounterPtr MetricsRecord::CreateCounter(const std::string& name) { + CounterPtr counterPtr = std::make_shared(name); + mCounters.emplace_back(counterPtr); + return counterPtr; +} + +GaugePtr MetricsRecord::CreateGauge(const std::string& name) { + GaugePtr gaugePtr = std::make_shared(name); + mGauges.emplace_back(gaugePtr); + return gaugePtr; +} + +void MetricsRecord::MarkDeleted() { + mDeleted = true; +} + +bool MetricsRecord::IsDeleted() const { + return mDeleted; +} + +const LabelsPtr& MetricsRecord::GetLabels() const { + return mLabels; +} + +const std::vector& MetricsRecord::GetCounters() const { + return mCounters; +} + +const std::vector& MetricsRecord::GetGauges() const { + return mGauges; +} + +MetricsRecord* MetricsRecord::CopyAndReset() { + MetricsRecord* metrics = new MetricsRecord(mLabels); + for (auto& item : mCounters) { + CounterPtr newPtr(item->CopyAndReset()); + metrics->mCounters.emplace_back(newPtr); + } + for (auto& item : mGauges) { + GaugePtr newPtr(item->CopyAndReset()); + metrics->mGauges.emplace_back(newPtr); + } + return metrics; +} + +MetricsRecord* MetricsRecord::GetNext() const { + return mNext; +} + +void MetricsRecord::SetNext(MetricsRecord* next) { + mNext = next; +} + +MetricsRecordRef::~MetricsRecordRef() { + if (mMetrics) { + mMetrics->MarkDeleted(); + } +} + + +void MetricsRecordRef::SetMetricsRecord(MetricsRecord* metricRecord) { + mMetrics = metricRecord; +} + +CounterPtr MetricsRecordRef::CreateCounter(const std::string& name) { + return mMetrics->CreateCounter(name); +} +GaugePtr MetricsRecordRef::CreateGauge(const std::string& name) { + return mMetrics->CreateGauge(name); +} + +const MetricsRecord* MetricsRecordRef::operator->() const { + return mMetrics; +} + +WriteMetrics::~WriteMetrics() { + Clear(); +} + +void WriteMetrics::PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels) { + MetricsRecord* cur = new MetricsRecord(std::make_shared(labels)); + ref.SetMetricsRecord(cur); + std::lock_guard lock(mMutex); + cur->SetNext(mHead); + mHead = cur; +} + +MetricsRecord* WriteMetrics::GetHead() { + std::lock_guard lock(mMutex); + return mHead; +} + +void WriteMetrics::Clear() { + std::lock_guard lock(mMutex); + while (mHead) { + MetricsRecord* toDeleted = mHead; + mHead = mHead->GetNext(); + delete toDeleted; + } +} + +MetricsRecord* WriteMetrics::DoSnapshot() { + // new read head + MetricsRecord* snapshot = nullptr; + MetricsRecord* toDeleteHead = nullptr; + MetricsRecord* tmp = nullptr; + + MetricsRecord emptyHead; + MetricsRecord* preTmp = nullptr; + + // find the first undeleted node and set as new mHead + { + std::lock_guard lock(mMutex); + emptyHead.SetNext(mHead); + preTmp = &emptyHead; + tmp = preTmp->GetNext(); + bool findHead = false; + while (tmp) { + if (tmp->IsDeleted()) { + preTmp->SetNext(tmp->GetNext()); + tmp->SetNext(toDeleteHead); + toDeleteHead = tmp; + tmp = preTmp->GetNext(); + } else { + // find head + mHead = tmp; + preTmp = mHead; + tmp = tmp->GetNext(); + findHead = true; + break; + } + } + // if no undeleted node, set null to mHead + if (!findHead) { + mHead = nullptr; + preTmp = mHead; + } + } + + // copy head + if (preTmp) { + MetricsRecord* newMetrics = preTmp->CopyAndReset(); + newMetrics->SetNext(snapshot); + snapshot = newMetrics; + } + + while (tmp) { + if (tmp->IsDeleted()) { + preTmp->SetNext(tmp->GetNext()); + tmp->SetNext(toDeleteHead); + toDeleteHead = tmp; + tmp = preTmp->GetNext(); + } else { + MetricsRecord* newMetrics = tmp->CopyAndReset(); + newMetrics->SetNext(snapshot); + snapshot = newMetrics; + preTmp = tmp; + tmp = tmp->GetNext(); + } + } + + while (toDeleteHead) { + MetricsRecord* toDelete = toDeleteHead; + toDeleteHead = toDeleteHead->GetNext(); + delete toDelete; + } + return snapshot; +} + +ReadMetrics::~ReadMetrics() { + Clear(); +} + +void ReadMetrics::ReadAsLogGroup(std::map& logGroupMap) const { + ReadLock lock(mReadWriteLock); + MetricsRecord* tmp = mHead; + while (tmp) { + Log* logPtr = nullptr; + // for (auto &item: tmp->GetLabels()) { + for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { + std::pair pair = *item; + if (METRIC_FIELD_REGION == pair.first) { + std::map::iterator iter; + std::string region = pair.second; + iter = logGroupMap.find(region); + if (iter != logGroupMap.end()) { + sls_logs::LogGroup* logGroup = iter->second; + logPtr = logGroup->add_logs(); + } else { + sls_logs::LogGroup* logGroup = new sls_logs::LogGroup(); + logPtr = logGroup->add_logs(); + logGroupMap.insert(std::pair(region, logGroup)); + } + } + } + if (!logPtr) { + std::map::iterator iter; + iter = logGroupMap.find(METRIC_REGION_DEFAULT); + if (iter != logGroupMap.end()) { + sls_logs::LogGroup* logGroup = iter->second; + logPtr = logGroup->add_logs(); + } else { + sls_logs::LogGroup* logGroup = new sls_logs::LogGroup(); + logPtr = logGroup->add_logs(); + logGroupMap.insert(std::pair(METRIC_REGION_DEFAULT, logGroup)); + } + } + auto now = GetCurrentLogtailTime(); + SetLogTime(logPtr, + AppConfig::GetInstance()->EnableLogTimeAutoAdjust() ? now.tv_sec + GetTimeDelta() : now.tv_sec, + now.tv_nsec); + for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { + std::pair pair = *item; + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(LABEL_PREFIX + pair.first); + contentPtr->set_value(pair.second); + } + + for (auto& item : tmp->GetCounters()) { + CounterPtr counter = item; + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(VALUE_PREFIX + counter->GetName()); + contentPtr->set_value(ToString(counter->GetValue())); + } + for (auto& item : tmp->GetGauges()) { + GaugePtr gauge = item; + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(VALUE_PREFIX + gauge->GetName()); + contentPtr->set_value(ToString(gauge->GetValue())); + } + // set default key + { + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(METRIC_TOPIC_FIELD_NAME); + contentPtr->set_value(METRIC_TOPIC_TYPE); + } + tmp = tmp->GetNext(); + } +} + + +void ReadMetrics::UpdateMetrics() { + MetricsRecord* snapshot = WriteMetrics::GetInstance()->DoSnapshot(); + MetricsRecord* toDelete; + { + // Only lock when change head + WriteLock lock(mReadWriteLock); + toDelete = mHead; + mHead = snapshot; + } + // delete old linklist + while (toDelete) { + MetricsRecord* obj = toDelete; + toDelete = toDelete->GetNext(); + delete obj; + } +} + +MetricsRecord* ReadMetrics::GetHead() { + WriteLock lock(mReadWriteLock); + return mHead; +} + +void ReadMetrics::Clear() { + WriteLock lock(mReadWriteLock); + while (mHead) { + MetricsRecord* toDelete = mHead; + mHead = mHead->GetNext(); + delete toDelete; + } +} + +} // namespace logtail diff --git a/core/profiler/LogtailAlarm.cpp b/core/monitor/LogtailAlarm.cpp similarity index 100% rename from core/profiler/LogtailAlarm.cpp rename to core/monitor/LogtailAlarm.cpp diff --git a/core/profiler/LogtailAlarm.h b/core/monitor/LogtailAlarm.h similarity index 100% rename from core/profiler/LogtailAlarm.h rename to core/monitor/LogtailAlarm.h diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h new file mode 100644 index 0000000000..c177b2e073 --- /dev/null +++ b/core/monitor/LogtailMetric.h @@ -0,0 +1,123 @@ +#pragma once +#include +#include +#include "common/Lock.h" +#include "log_pb/sls_logs.pb.h" + + +namespace logtail { + +class Counter { +private: + std::string mName; + std::atomic_long mVal; + +public: + Counter(const std::string& name, uint64_t val); + uint64_t GetValue() const; + const std::string& GetName() const; + void Add(uint64_t val); + Counter* CopyAndReset(); +}; + +using CounterPtr = std::shared_ptr; + +class Gauge { +private: + std::string mName; + std::atomic_long mVal; + +public: + Gauge(const std::string& name, uint64_t val); + uint64_t GetValue() const; + const std::string& GetName() const; + void Set(uint64_t val); + Gauge* CopyAndReset(); +}; + +using GaugePtr = std::shared_ptr; + +using MetricLabels = std::vector>; +using LabelsPtr = std::shared_ptr; + +class MetricsRecord { +private: + LabelsPtr mLabels; + std::atomic_bool mDeleted; + std::vector mCounters; + std::vector mGauges; + MetricsRecord* mNext = nullptr; + +public: + MetricsRecord(LabelsPtr labels); + MetricsRecord() = default; + void MarkDeleted(); + bool IsDeleted() const; + const LabelsPtr& GetLabels() const; + const std::vector& GetCounters() const; + const std::vector& GetGauges() const; + CounterPtr CreateCounter(const std::string& name); + GaugePtr CreateGauge(const std::string& name); + MetricsRecord* CopyAndReset(); + void SetNext(MetricsRecord* next); + MetricsRecord* GetNext() const; +}; + +class MetricsRecordRef { +private: + MetricsRecord* mMetrics = nullptr; + +public: + ~MetricsRecordRef(); + void SetMetricsRecord(MetricsRecord* metricRecord); + CounterPtr CreateCounter(const std::string& name); + GaugePtr CreateGauge(const std::string& name); + const MetricsRecord* operator->() const; +}; + +class WriteMetrics { +private: + WriteMetrics() = default; + std::mutex mMutex; + MetricsRecord* mHead = nullptr; + + void Clear(); + MetricsRecord* GetHead(); + +public: + ~WriteMetrics(); + static WriteMetrics* GetInstance() { + static WriteMetrics* ptr = new WriteMetrics(); + return ptr; + } + void PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels); + MetricsRecord* DoSnapshot(); + + +#ifdef APSARA_UNIT_TEST_MAIN + friend class ILogtailMetricUnittest; +#endif +}; + +class ReadMetrics { +private: + ReadMetrics() = default; + mutable ReadWriteLock mReadWriteLock; + MetricsRecord* mHead = nullptr; + void Clear(); + MetricsRecord* GetHead(); + +public: + ~ReadMetrics(); + static ReadMetrics* GetInstance() { + static ReadMetrics* ptr = new ReadMetrics(); + return ptr; + } + void ReadAsLogGroup(std::map& logGroupMap) const; + void UpdateMetrics(); + +#ifdef APSARA_UNIT_TEST_MAIN + friend class ILogtailMetricUnittest; +#endif +}; +} // namespace logtail diff --git a/core/monitor/MetricConstants.h b/core/monitor/MetricConstants.h new file mode 100644 index 0000000000..803830e89c --- /dev/null +++ b/core/monitor/MetricConstants.h @@ -0,0 +1,36 @@ +/* + * Copyright 2022 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace logtail { + +const std::string METRIC_FIELD_REGION = "region"; +const std::string METRIC_REGION_DEFAULT = "default"; +const std::string METRIC_SLS_LOGSTORE_NAME = "shennong_log_profile"; +const std::string METRIC_TOPIC_TYPE = "logtail_metric"; +const std::string METRIC_TOPIC_FIELD_NAME = "__topic__"; + +const std::string LABEL_PREFIX = "label."; +const std::string VALUE_PREFIX = "value."; + + + +const std::string METRIC_FILE_READ_COUNT = "file_read_count"; +const std::string METRIC_FILE_READ_BYTES = "file_read_bytes"; + +} // namespace logtail \ No newline at end of file diff --git a/core/monitor/MetricExportor.cpp b/core/monitor/MetricExportor.cpp new file mode 100644 index 0000000000..89d4711709 --- /dev/null +++ b/core/monitor/MetricExportor.cpp @@ -0,0 +1,39 @@ +#include "MetricExportor.h" +#include "sender/Sender.h" +#include "log_pb/sls_logs.pb.h" +#include "LogtailMetric.h" +#include "config_manager/ConfigManager.h" +#include "LogFileProfiler.h" +#include "MetricConstants.h" + +using namespace sls_logs; +using namespace std; + +namespace logtail { + +MetricExportor::MetricExportor() : mSendInterval(60), mLastSendTime(time(NULL) - (rand() % (mSendInterval / 10)) * 10) {} + +void MetricExportor::PushMetrics(bool forceSend) { + int32_t curTime = time(NULL); + if (!forceSend && (curTime - mLastSendTime < mSendInterval)) { + return; + } + ReadMetrics::GetInstance()->UpdateMetrics(); + + std::map logGroupMap; + ReadMetrics::GetInstance()->ReadAsLogGroup(logGroupMap); + + std::map::iterator iter; + for (iter = logGroupMap.begin(); iter != logGroupMap.end(); iter ++) { + sls_logs::LogGroup* logGroup = iter->second; + logGroup->set_category(METRIC_SLS_LOGSTORE_NAME); + logGroup->set_source(LogFileProfiler::mIpAddr); + if (METRIC_REGION_DEFAULT == iter->first) { + mProfileSender.SendToProfileProject(ConfigManager::GetInstance()->GetDefaultProfileRegion(), *logGroup); + } else { + mProfileSender.SendToProfileProject(iter->first, *logGroup); + } + delete logGroup; + } +} +} \ No newline at end of file diff --git a/core/monitor/MetricExportor.h b/core/monitor/MetricExportor.h new file mode 100644 index 0000000000..cbcf956d55 --- /dev/null +++ b/core/monitor/MetricExportor.h @@ -0,0 +1,22 @@ +#pragma once + +#include "profile_sender/ProfileSender.h" + + +namespace logtail { + +class MetricExportor { +public: + static MetricExportor* GetInstance() { + static MetricExportor* ptr = new MetricExportor(); + return ptr; + } + void PushMetrics(bool forceSend); + +private: + MetricExportor(); + ProfileSender mProfileSender; + int32_t mSendInterval; + int32_t mLastSendTime; +}; +} \ No newline at end of file diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index d02258b02d..0c33f174e7 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -34,8 +34,8 @@ #include "log_pb/sls_logs.pb.h" #include "logger/Logger.h" #include "sender/Sender.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogtailAlarm.h" #include "config_manager/ConfigManager.h" #include "app_config/AppConfig.h" #include "event_handler/LogInput.h" diff --git a/core/observer/CMakeLists.txt b/core/observer/CMakeLists.txt index 71ed8b47f8..4f53e1e134 100644 --- a/core/observer/CMakeLists.txt +++ b/core/observer/CMakeLists.txt @@ -9,7 +9,7 @@ target_link_libraries(${PROJECT_NAME} logger) target_link_libraries(${PROJECT_NAME} config) target_link_libraries(${PROJECT_NAME} config_manager) target_link_libraries(${PROJECT_NAME} monitor) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} event) target_link_libraries(${PROJECT_NAME} log_pb) diff --git a/core/observer/interface/statistics.h b/core/observer/interface/statistics.h index ea623e1f84..9c15efa252 100644 --- a/core/observer/interface/statistics.h +++ b/core/observer/interface/statistics.h @@ -18,7 +18,7 @@ #include "Monitor.h" #include "iostream" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include #include diff --git a/core/observer/metas/ContainerProcessGroup.cpp b/core/observer/metas/ContainerProcessGroup.cpp index 4e8bc3abaa..c72a0301ec 100644 --- a/core/observer/metas/ContainerProcessGroup.cpp +++ b/core/observer/metas/ContainerProcessGroup.cpp @@ -16,7 +16,7 @@ #include "common/TimeUtil.h" #include "logger/Logger.h" #include "CGroupPathResolver.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "FileSystemUtil.h" #include "StringTools.h" #include "plugin/LogtailPlugin.h" diff --git a/core/parser/CMakeLists.txt b/core/parser/CMakeLists.txt index 035da778a1..f8751f4550 100644 --- a/core/parser/CMakeLists.txt +++ b/core/parser/CMakeLists.txt @@ -21,5 +21,5 @@ add_library(${PROJECT_NAME} STATIC ${LIB_SOURCE_FILES}) target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} logger) target_link_libraries(${PROJECT_NAME} log_pb) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} app_config) \ No newline at end of file diff --git a/core/parser/LogParser.cpp b/core/parser/LogParser.cpp index f1c993f3bd..c36d305fc2 100644 --- a/core/parser/LogParser.cpp +++ b/core/parser/LogParser.cpp @@ -25,8 +25,8 @@ #include "log_pb/sls_logs.pb.h" #include "logger/Logger.h" #include "config/LogType.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogtailAlarm.h" #include "app_config/AppConfig.h" #include "config_manager/ConfigManager.h" diff --git a/core/plugin/CMakeLists.txt b/core/plugin/CMakeLists.txt index 54dc0e7c10..49e60934b3 100644 --- a/core/plugin/CMakeLists.txt +++ b/core/plugin/CMakeLists.txt @@ -24,7 +24,7 @@ target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} logger) target_link_libraries(${PROJECT_NAME} config_manager) target_link_libraries(${PROJECT_NAME} sender) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} app_config) target_link_libraries(${PROJECT_NAME} config) if (UNIX) diff --git a/core/plugin/LogtailPlugin.cpp b/core/plugin/LogtailPlugin.cpp index effe89ed15..9deb69e72c 100644 --- a/core/plugin/LogtailPlugin.cpp +++ b/core/plugin/LogtailPlugin.cpp @@ -19,8 +19,8 @@ #include "logger/Logger.h" #include "config_manager/ConfigManager.h" #include "sender/Sender.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogFileProfiler.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" #include "app_config/AppConfig.h" #include "common/DynamicLibHelper.h" #include "common/LogtailCommonFlags.h" diff --git a/core/polling/CMakeLists.txt b/core/polling/CMakeLists.txt index 4411e7d0fc..f44d2df2e9 100644 --- a/core/polling/CMakeLists.txt +++ b/core/polling/CMakeLists.txt @@ -22,5 +22,5 @@ target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} logger) target_link_libraries(${PROJECT_NAME} config_manager) target_link_libraries(${PROJECT_NAME} monitor) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} event) \ No newline at end of file diff --git a/core/polling/PollingDirFile.cpp b/core/polling/PollingDirFile.cpp index 03a8a48de3..7a876c47c3 100644 --- a/core/polling/PollingDirFile.cpp +++ b/core/polling/PollingDirFile.cpp @@ -29,7 +29,7 @@ #include "event/Event.h" #include "logger/Logger.h" #include "config_manager/ConfigManager.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "monitor/Monitor.h" #include "PollingModify.h" #include "PollingEventQueue.h" diff --git a/core/polling/PollingModify.cpp b/core/polling/PollingModify.cpp index 2e864f7a6b..1be1851638 100644 --- a/core/polling/PollingModify.cpp +++ b/core/polling/PollingModify.cpp @@ -24,7 +24,7 @@ #include "common/FileSystemUtil.h" #include "event/Event.h" #include "logger/Logger.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "monitor/Monitor.h" using namespace std; diff --git a/core/processor/CMakeLists.txt b/core/processor/CMakeLists.txt index a4ba6e1fed..f20ed34d24 100644 --- a/core/processor/CMakeLists.txt +++ b/core/processor/CMakeLists.txt @@ -26,7 +26,7 @@ target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} logger) target_link_libraries(${PROJECT_NAME} parser) target_link_libraries(${PROJECT_NAME} log_pb) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} app_config) target_link_libraries(${PROJECT_NAME} sdk) target_link_libraries(${PROJECT_NAME} sender) diff --git a/core/processor/LogFilter.cpp b/core/processor/LogFilter.cpp index 4c3302a4b6..64724009aa 100644 --- a/core/processor/LogFilter.cpp +++ b/core/processor/LogFilter.cpp @@ -14,7 +14,7 @@ #include "LogFilter.h" #include -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "app_config/AppConfig.h" #include "common/util.h" #include "sdk/Common.h" diff --git a/core/processor/LogProcess.cpp b/core/processor/LogProcess.cpp index 51d4d67df7..8f1553587d 100644 --- a/core/processor/LogProcess.cpp +++ b/core/processor/LogProcess.cpp @@ -32,12 +32,12 @@ #include "sender/Sender.h" #include "log_pb/sls_logs.pb.h" #include "common/StringTools.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogIntegrity.h" -#include "profiler/LogLineCount.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogIntegrity.h" +#include "monitor/LogLineCount.h" #include "config/IntegrityConfig.h" #include "app_config/AppConfig.h" -#include "profiler/LogFileProfiler.h" +#include "monitor/LogFileProfiler.h" #include "config_manager/ConfigManager.h" #include "logger/Logger.h" #include "aggregator/Aggregator.h" diff --git a/core/processor/RegexFilterValueNode.h b/core/processor/RegexFilterValueNode.h index c9addfb52c..d56cd67478 100644 --- a/core/processor/RegexFilterValueNode.h +++ b/core/processor/RegexFilterValueNode.h @@ -21,7 +21,7 @@ #include "BaseFilterNode.h" #include "common/util.h" #include "app_config/AppConfig.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "logger/Logger.h" namespace logtail { diff --git a/core/profiler/CMakeLists.txt b/core/profiler/CMakeLists.txt deleted file mode 100644 index 1f95152065..0000000000 --- a/core/profiler/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2022 iLogtail Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cmake_minimum_required(VERSION 2.9) -project(profiler) - -file(GLOB LIB_SOURCE_FILES *.cpp *.h) -append_source_files(LIB_SOURCE_FILES) -add_library(${PROJECT_NAME} STATIC ${LIB_SOURCE_FILES}) -target_link_libraries(${PROJECT_NAME} common) -target_link_libraries(${PROJECT_NAME} log_pb) -target_link_libraries(${PROJECT_NAME} sender) -target_link_libraries(${PROJECT_NAME} config_manager) -target_link_libraries(${PROJECT_NAME} fuse) -link_jsoncpp(${PROJECT_NAME}) \ No newline at end of file diff --git a/core/reader/CMakeLists.txt b/core/reader/CMakeLists.txt index c9c9bcb398..d21a53fe54 100644 --- a/core/reader/CMakeLists.txt +++ b/core/reader/CMakeLists.txt @@ -26,7 +26,7 @@ target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} logger) target_link_libraries(${PROJECT_NAME} parser) target_link_libraries(${PROJECT_NAME} checkpoint) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} event_handler) target_link_libraries(${PROJECT_NAME} app_config) target_link_libraries(${PROJECT_NAME} config_manager) diff --git a/core/reader/DelimiterLogFileReader.cpp b/core/reader/DelimiterLogFileReader.cpp index 85c4907160..3d41cc5725 100644 --- a/core/reader/DelimiterLogFileReader.cpp +++ b/core/reader/DelimiterLogFileReader.cpp @@ -16,7 +16,7 @@ #include #include #include "common/LogtailCommonFlags.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "parser/LogParser.h" #include "parser/DelimiterModeFsmParser.h" #include "log_pb/sls_logs.pb.h" diff --git a/core/reader/JsonLogFileReader.cpp b/core/reader/JsonLogFileReader.cpp index ce67636f81..a3adff06f2 100644 --- a/core/reader/JsonLogFileReader.cpp +++ b/core/reader/JsonLogFileReader.cpp @@ -17,7 +17,7 @@ #include #include #include "common/LogtailCommonFlags.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "parser/LogParser.h" #include "log_pb/sls_logs.pb.h" #include "logger/Logger.h" diff --git a/core/reader/LogFileReader.cpp b/core/reader/LogFileReader.cpp index a8a5030594..7204019c59 100644 --- a/core/reader/LogFileReader.cpp +++ b/core/reader/LogFileReader.cpp @@ -35,8 +35,8 @@ #include "logger/Logger.h" #include "checkpoint/CheckPointManager.h" #include "checkpoint/CheckpointManagerV2.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogFileProfiler.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" #include "event_handler/LogInput.h" #include "app_config/AppConfig.h" #include "config_manager/ConfigManager.h" diff --git a/core/sender/CMakeLists.txt b/core/sender/CMakeLists.txt index d38ab90c63..31adc53577 100644 --- a/core/sender/CMakeLists.txt +++ b/core/sender/CMakeLists.txt @@ -25,7 +25,7 @@ target_link_libraries(${PROJECT_NAME} app_config) target_link_libraries(${PROJECT_NAME} aggregator) target_link_libraries(${PROJECT_NAME} sdk) target_link_libraries(${PROJECT_NAME} monitor) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} log_pb) target_link_libraries(${PROJECT_NAME} config_manager) target_link_libraries(${PROJECT_NAME} fuse) diff --git a/core/sender/Sender.cpp b/core/sender/Sender.cpp index 8c4bc44240..8c5b069822 100644 --- a/core/sender/Sender.cpp +++ b/core/sender/Sender.cpp @@ -41,10 +41,10 @@ #include "config/Config.h" #include "processor/LogProcess.h" #include "processor/LogFilter.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogIntegrity.h" -#include "profiler/LogLineCount.h" -#include "profiler/LogFileProfiler.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogIntegrity.h" +#include "monitor/LogLineCount.h" +#include "monitor/LogFileProfiler.h" #include "app_config/AppConfig.h" #include "monitor/Monitor.h" #include "config_manager/ConfigManager.h" diff --git a/core/sls_control/SLSControl.cpp b/core/sls_control/SLSControl.cpp index a94f973f48..e42ca4e760 100644 --- a/core/sls_control/SLSControl.cpp +++ b/core/sls_control/SLSControl.cpp @@ -22,7 +22,7 @@ #include "common/LogtailCommonFlags.h" #include "common/version.h" #include "logger/Logger.h" -#include "profiler/LogFileProfiler.h" +#include "monitor/LogFileProfiler.h" // for windows compatability, to avoid conflict with the same function defined in windows.h #ifdef SetPort #undef SetPort diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 54ffa3ef2a..d412dbfbba 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -57,7 +57,7 @@ add_subdirectory(polling) add_subdirectory(processor) add_subdirectory(reader) add_subdirectory(sender) -add_subdirectory(profiler) +add_subdirectory(monitor) add_subdirectory(sdk) if (UNIX) add_subdirectory(observer) diff --git a/core/unittest/profiler/CMakeLists.txt b/core/unittest/monitor/CMakeLists.txt similarity index 82% rename from core/unittest/profiler/CMakeLists.txt rename to core/unittest/monitor/CMakeLists.txt index c2cf855388..cf3f1ac22e 100644 --- a/core/unittest/profiler/CMakeLists.txt +++ b/core/unittest/monitor/CMakeLists.txt @@ -13,7 +13,11 @@ # limitations under the License. cmake_minimum_required(VERSION 2.9) -project(profiler_unittest) +project(logtail_metric_unittest) + +add_executable(logtail_metric_unittest LogtailMetricUnittest.cpp) +target_link_libraries(logtail_metric_unittest unittest_base) + add_executable(profiler_data_integrity_unittest DataIntegrityUnittest.cpp) target_link_libraries(profiler_data_integrity_unittest unittest_base) \ No newline at end of file diff --git a/core/unittest/profiler/DataIntegrityUnittest.cpp b/core/unittest/monitor/DataIntegrityUnittest.cpp similarity index 100% rename from core/unittest/profiler/DataIntegrityUnittest.cpp rename to core/unittest/monitor/DataIntegrityUnittest.cpp diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp new file mode 100644 index 0000000000..3f9e36368e --- /dev/null +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -0,0 +1,324 @@ +#include "unittest/Unittest.h" +#include +#include +#include +#include +#include +#include "util.h" +#include "LogtailMetric.h" +#include "MetricExportor.h" +#include "MetricConstants.h" + +namespace logtail { + + +static std::atomic_bool running(true); + + +class ILogtailMetricUnittest : public ::testing::Test { +public: + void SetUp() {} + + void TearDown() { + ReadMetrics::GetInstance()->Clear(); + WriteMetrics::GetInstance()->Clear(); + } + + void TestCreateMetricAutoDelete(); + void TestCreateMetricAutoDeleteMultiThread(); + void TestCreateAndDeleteMetric(); +}; + +APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateMetricAutoDelete, 0); +APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateMetricAutoDeleteMultiThread, 1); +APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateAndDeleteMetric, 2); + + +void ILogtailMetricUnittest::TestCreateMetricAutoDelete() { + std::vector> labels; + labels.emplace_back(std::make_pair("project", "project1")); + labels.emplace_back(std::make_pair("logstore", "logstore1")); + labels.emplace_back(std::make_pair("region", "cn-hangzhou")); + + MetricsRecordRef fileMetric; + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric, std::move(labels)); + APSARA_TEST_EQUAL(fileMetric->GetLabels()->size(), 3); + + + CounterPtr fileCounter = fileMetric.CreateCounter("filed1"); + fileCounter->Add(111UL); + fileCounter->Add(111UL); + APSARA_TEST_EQUAL(fileCounter->GetValue(), 222); + + MetricExportor::GetInstance()->PushMetrics(true); + + // assert WriteMetrics count + MetricsRecord* tmp = WriteMetrics::GetInstance()->GetHead(); + int count = 0; + while (tmp) { + tmp = tmp->GetNext(); + count++; + } + APSARA_TEST_EQUAL(count, 1); + + + // assert ReadMetrics count + tmp = ReadMetrics::GetInstance()->GetHead(); + count = 0; + while (tmp) { + tmp = tmp->GetNext(); + count++; + } + APSARA_TEST_EQUAL(count, 1); + + // mock create in other class, should be delete after + { + std::vector> labels; + labels.emplace_back(std::make_pair("project", "project1")); + labels.emplace_back(std::make_pair("logstore", "logstore1")); + labels.emplace_back(std::make_pair("region", "cn-hangzhou")); + + MetricsRecordRef fileMetric2; + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric2, std::move(labels)); + CounterPtr fileCounter2 = fileMetric2.CreateCounter("filed2"); + fileCounter2->Add(222UL); + } + + { + std::vector> labels; + labels.emplace_back(std::make_pair("project", "project1")); + labels.emplace_back(std::make_pair("logstore", "logstore1")); + labels.emplace_back(std::make_pair("region", "cn-hangzhou")); + MetricsRecordRef fileMetric3; + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric3, std::move(labels)); + CounterPtr fileCounter3 = fileMetric3.CreateCounter("filed3"); + fileCounter3->Add(333UL); + } + + MetricExportor::GetInstance()->PushMetrics(true); + + // assert WriteMetrics count + tmp = WriteMetrics::GetInstance()->GetHead(); + count = 0; + while (tmp) { + tmp = tmp->GetNext(); + count++; + } + APSARA_TEST_EQUAL(count, 1); + + + // assert ReadMetrics count + tmp = ReadMetrics::GetInstance()->GetHead(); + count = 0; + while (tmp) { + tmp = tmp->GetNext(); + count++; + } + APSARA_TEST_EQUAL(count, 1); +} + +void PushMetrics() { + for (int i = 0; i < 10; i++) { + LOG_INFO(sLogger, ("PushMetricsCount", i)); + MetricExportor::GetInstance()->PushMetrics(true); + } +} + +void createMetrics(int count) { + for (int i = 0; i < count; i++) { + std::vector> labels; + labels.emplace_back(std::make_pair("num", std::to_string(i))); + labels.emplace_back(std::make_pair("count", std::to_string(count))); + labels.emplace_back(std::make_pair("region", "cn-beijing")); + MetricsRecordRef fileMetric; + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric, std::move(labels)); + CounterPtr fileCounter = fileMetric.CreateCounter("filed1"); + fileCounter->Add(111UL); + } +} + +void ILogtailMetricUnittest::TestCreateMetricAutoDeleteMultiThread() { + std::thread t1(createMetrics, 1); + std::thread t2(createMetrics, 2); + std::thread t3(createMetrics, 3); + std::thread t4(createMetrics, 4); + + t1.join(); + t2.join(); + t3.join(); + t4.join(); + + // assert WriteMetrics count + MetricsRecord* tmp = WriteMetrics::GetInstance()->GetHead(); + int count = 0; + while (tmp) { + tmp = tmp->GetNext(); + count++; + } + // 1 + 2 + 3 + 4 = 10 + APSARA_TEST_EQUAL(count, 10); + + for (int i = 0; i < 10; i++) { + MetricExportor::GetInstance()->PushMetrics(true); + } + + // assert WriteMetrics count + tmp = WriteMetrics::GetInstance()->GetHead(); + count = 0; + while (tmp) { + for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { + std::pair pair = *item; + LOG_INFO(sLogger, ("key", pair.first)("value", pair.second)); + } + tmp = tmp->GetNext(); + count++; + } + APSARA_TEST_EQUAL(count, 0); + + // assert ReadMetrics count + tmp = ReadMetrics::GetInstance()->GetHead(); + count = 0; + while (tmp) { + tmp = tmp->GetNext(); + count++; + } + APSARA_TEST_EQUAL(count, 0); +} + + +void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { + std::thread t1(createMetrics, 1); + std::thread t2(createMetrics, 2); + + MetricsRecordRef* fileMetric1 = new MetricsRecordRef(); + MetricsRecordRef* fileMetric2 = new MetricsRecordRef(); + MetricsRecordRef* fileMetric3 = new MetricsRecordRef(); + + + std::vector> labels; + labels.emplace_back(std::make_pair("project", "test1")); + labels.emplace_back(std::make_pair("logstore", "test1")); + labels.emplace_back(std::make_pair("region", "cn-beijing")); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric1, std::move(labels)); + CounterPtr fileCounter = fileMetric1->CreateCounter("filed1"); + fileCounter->Add(111UL); + + { + std::vector> labels; + labels.emplace_back(std::make_pair("project", "test2")); + labels.emplace_back(std::make_pair("logstore", "test2")); + labels.emplace_back(std::make_pair("region", "cn-beijing")); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric2, std::move(labels)); + CounterPtr fileCounter = fileMetric2->CreateCounter("filed1"); + fileCounter->Add(111UL); + } + + { + std::vector> labels; + labels.emplace_back(std::make_pair("project", "test3")); + labels.emplace_back(std::make_pair("logstore", "test3")); + labels.emplace_back(std::make_pair("region", "cn-beijing")); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric3, std::move(labels)); + CounterPtr fileCounter = fileMetric3->CreateCounter("filed1"); + fileCounter->Add(111UL); + } + std::thread t3(createMetrics, 3); + std::thread t4(createMetrics, 4); + + + t1.join(); + t2.join(); + t3.join(); + t4.join(); + // assert WriteMetrics count + MetricsRecord* tmp = WriteMetrics::GetInstance()->GetHead(); + int count = 0; + while (tmp) { + tmp = tmp->GetNext(); + count++; + } + // 10 + 3 + APSARA_TEST_EQUAL(count, 13); + + delete fileMetric2; + delete fileMetric3; + + MetricExportor::GetInstance()->PushMetrics(true); + + // assert WriteMetrics count + tmp = WriteMetrics::GetInstance()->GetHead(); + count = 0; + while (tmp) { + for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { + std::pair pair = *item; + LOG_INFO(sLogger, ("key", pair.first)("value", pair.second)); + } + tmp = tmp->GetNext(); + count++; + } + APSARA_TEST_EQUAL(count, 1); + // assert writeMetric value + if (count == 1) { + tmp = WriteMetrics::GetInstance()->GetHead(); + std::vector values = tmp->GetCounters(); + APSARA_TEST_EQUAL(values.size(), 1); + if (values.size() == 1) { + APSARA_TEST_EQUAL(values.at(0)->GetValue(), 0); + } + } + + // assert ReadMetrics count + tmp = ReadMetrics::GetInstance()->GetHead(); + count = 0; + while (tmp) { + tmp = tmp->GetNext(); + count++; + } + APSARA_TEST_EQUAL(count, 1); + + // assert readMetric value + if (count == 1) { + tmp = ReadMetrics::GetInstance()->GetHead(); + std::vector values = tmp->GetCounters(); + APSARA_TEST_EQUAL(values.size(), 1); + if (values.size() == 1) { + APSARA_TEST_EQUAL(values.at(0)->GetValue(), 111); + } + } + + // after dosnapshot, add value again + fileCounter->Add(111UL); + fileCounter->Add(111UL); + fileCounter->Add(111UL); + + APSARA_TEST_EQUAL(fileCounter->GetValue(), 333); + + MetricExportor::GetInstance()->PushMetrics(true); + // assert ReadMetrics count + tmp = ReadMetrics::GetInstance()->GetHead(); + count = 0; + while (tmp) { + tmp = tmp->GetNext(); + count++; + } + APSARA_TEST_EQUAL(count, 1); + + // assert readMetric value + if (count == 1) { + tmp = ReadMetrics::GetInstance()->GetHead(); + std::vector values = tmp->GetCounters(); + APSARA_TEST_EQUAL(values.size(), 1); + if (values.size() == 1) { + APSARA_TEST_EQUAL(values.at(0)->GetValue(), 333); + } + } + delete fileMetric1; +} + +} // namespace logtail + +int main(int argc, char** argv) { + logtail::Logger::Instance().InitGlobalLoggers(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/core/unittest/polling/PollingUnittest.cpp b/core/unittest/polling/PollingUnittest.cpp index d6fafe6cd4..48c37d7da0 100644 --- a/core/unittest/polling/PollingUnittest.cpp +++ b/core/unittest/polling/PollingUnittest.cpp @@ -48,7 +48,7 @@ #include #include "log_pb/metric.pb.h" #include "log_pb/sls_logs.pb.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "event_handler/LogInput.h" #include "common/FileEncryption.h" #include "common/FileSystemUtil.h" diff --git a/core/unittest/sender/SenderUnittest.cpp b/core/unittest/sender/SenderUnittest.cpp index af5f6f967d..03486e6eb6 100644 --- a/core/unittest/sender/SenderUnittest.cpp +++ b/core/unittest/sender/SenderUnittest.cpp @@ -43,8 +43,8 @@ #include #include "log_pb/metric.pb.h" #include "log_pb/sls_logs.pb.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogIntegrity.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogIntegrity.h" #include "event_handler/LogInput.h" #include "common/FileEncryption.h" #include "processor/LogProcess.h"