[WIP]: add metric new (#1015)

* add metric * refine code * refine code * refine code * refine code * refine code * add exportor * add exportor * refine code and test * refine code * fix comment * fix comment * fix raw ptr * fix comment * add change log * add group by region * fix comments * fix cooment * fix cooment * fix comments * refine code * refine code * refine code * refine code * refine code * refine code * refine code * refine code * refine code * refine code * refine code * refine code * refine coder * refine coder * refine code * refine code * refine code * refine code * refine code
alibaba · Aug 24, 2023 · 733f853 · 733f853
1 parent 06ca877
commit 733f853
Show file tree

Hide file tree

Showing 70 changed files with 961 additions and 111 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -36,3 +36,5 @@ your changes, such as:
 - [public] [both] [updated] add a new feature
 
 ## [Unreleased]
+
+- [public] [both] [added] add new logtail metric module
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
@@ -81,8 +81,8 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/dependencies.cmake)
 # Subdirectories (modules).
 set(SUB_DIRECTORIES_LIST
         aggregator app_config checkpoint common config config_manager config_server_pb
-        controller event event_handler event_listener helper log_pb logger monitor
-        parser plugin polling processor profiler reader profile_sender sender shennong sdk
+        controller event event_handler event_listener helper log_pb logger monitor 
+        parser plugin polling processor reader profile_sender sender shennong sdk
         fuse sls_control
         )
 if (UNIX)
@@ -120,7 +120,6 @@ target_link_libraries(${LOGTAIL_TARGET}
         plugin
         monitor
         sender
-        profiler
         profile_sender
         app_config
         sls_control

diff --git a/core/aggregator/Aggregator.cpp b/core/aggregator/Aggregator.cpp
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 #include <processor/LogFilter.h>
-#include <profiler/LogFileProfiler.h>
+#include <monitor/LogFileProfiler.h>
 #include <common/Constants.h>
 #include <config_manager/ConfigManager.h>
 #include <common/StringTools.h>

diff --git a/core/aggregator/CMakeLists.txt b/core/aggregator/CMakeLists.txt
@@ -23,4 +23,4 @@ target_link_libraries(${PROJECT_NAME} common)
 target_link_libraries(${PROJECT_NAME} config_manager)
 target_link_libraries(${PROJECT_NAME} sender)
 target_link_libraries(${PROJECT_NAME} shennong)
-target_link_libraries(${PROJECT_NAME} profiler)
+target_link_libraries(${PROJECT_NAME} monitor)
diff --git a/core/app_config/AppConfigBase.cpp b/core/app_config/AppConfigBase.cpp
@@ -19,8 +19,8 @@
 #endif
 #include "shennong/MetricSender.h"
 #include "sender/Sender.h"
-#include "profiler/LogFileProfiler.h"
-#include "profiler/LogtailAlarm.h"
+#include "monitor/LogFileProfiler.h"
+#include "monitor/LogtailAlarm.h"
 #include "monitor/Monitor.h"
 #include "common/util.h"
 #include "common/LogtailCommonFlags.h"

diff --git a/core/app_config/CMakeLists.txt b/core/app_config/CMakeLists.txt
@@ -23,4 +23,4 @@ target_link_libraries(${PROJECT_NAME} common)
 target_link_libraries(${PROJECT_NAME} config_manager)
 target_link_libraries(${PROJECT_NAME} sender)
 target_link_libraries(${PROJECT_NAME} shennong)
-target_link_libraries(${PROJECT_NAME} profiler)
+target_link_libraries(${PROJECT_NAME} monitor)
diff --git a/core/checkpoint/CMakeLists.txt b/core/checkpoint/CMakeLists.txt
@@ -20,7 +20,7 @@ append_source_files(LIB_SOURCE_FILES)
 add_library(${PROJECT_NAME} STATIC ${LIB_SOURCE_FILES})
 target_link_libraries(${PROJECT_NAME} common)
 target_link_libraries(${PROJECT_NAME} logger)
-target_link_libraries(${PROJECT_NAME} profiler)
+target_link_libraries(${PROJECT_NAME} monitor)
 target_link_libraries(${PROJECT_NAME} config_manager)
 target_link_libraries(${PROJECT_NAME} app_config)
 link_leveldb(${PROJECT_NAME})
diff --git a/core/checkpoint/CheckPointManager.cpp b/core/checkpoint/CheckPointManager.cpp
@@ -17,7 +17,7 @@
 #include <fstream>
 #include <thread>
 #include <fcntl.h>
-#include "profiler/LogtailAlarm.h"
+#include "monitor/LogtailAlarm.h"
 #include "app_config/AppConfig.h"
 #include "config_manager/ConfigManager.h"
 #include "common/Flags.h"

diff --git a/core/checkpoint/CheckpointManagerV2.cpp b/core/checkpoint/CheckpointManagerV2.cpp
@@ -19,7 +19,7 @@
 #include "common/ScopeInvoker.h"
 #include "common/TimeUtil.h"
 #include "logger/Logger.h"
-#include "profiler/LogtailAlarm.h"
+#include "monitor/LogtailAlarm.h"
 #include "app_config/AppConfig.h"
 #include "config/Config.h"
 #include "config_manager/ConfigManager.h"

diff --git a/core/common/LogFileCollectOffsetIndicator.cpp b/core/common/LogFileCollectOffsetIndicator.cpp
@@ -15,7 +15,7 @@
 #include "LogFileCollectOffsetIndicator.h"
 #include <vector>
 #include <algorithm>
-#include "profiler/LogtailAlarm.h"
+#include "monitor/LogtailAlarm.h"
 #include "common/Flags.h"
 #include "common/LogstoreSenderQueue.h"
 #include "sender/Sender.h"

diff --git a/core/common/LogGroupContext.h b/core/common/LogGroupContext.h
@@ -16,7 +16,7 @@
 
 #pragma once
 #include "config/IntegrityConfig.h"
-#include "profiler/LogtailAlarm.h"
+#include "monitor/LogtailAlarm.h"
 #include "FileInfo.h"
 #include "RangeCheckpoint.h"
 

diff --git a/core/common/LogstoreSenderQueue.cpp b/core/common/LogstoreSenderQueue.cpp
@@ -15,7 +15,7 @@
 #include "LogstoreSenderQueue.h"
 #include "common/Flags.h"
 #include "common/StringTools.h"
-#include "profiler/LogtailAlarm.h"
+#include "monitor/LogtailAlarm.h"
 
 
 DEFINE_FLAG_INT32(max_client_send_error_count, "set disabled if last send is all failed", 60);

diff --git a/core/config_manager/CMakeLists.txt b/core/config_manager/CMakeLists.txt
@@ -29,7 +29,7 @@ target_link_libraries(${PROJECT_NAME} config_server_pb)
 target_link_libraries(${PROJECT_NAME} controller)
 target_link_libraries(${PROJECT_NAME} event_handler)
 target_link_libraries(${PROJECT_NAME} sender)
-target_link_libraries(${PROJECT_NAME} profiler)
+target_link_libraries(${PROJECT_NAME} monitor)
 target_link_libraries(${PROJECT_NAME} checkpoint)
 target_link_libraries(${PROJECT_NAME} app_config)
 target_link_libraries(${PROJECT_NAME} processor)

diff --git a/core/config_manager/ConfigManager.cpp b/core/config_manager/ConfigManager.cpp
@@ -45,10 +45,10 @@
 #include "common/GlobalPara.h"
 #include "common/version.h"
 #include "config/UserLogConfigParser.h"
-#include "profiler/LogtailAlarm.h"
-#include "profiler/LogFileProfiler.h"
-#include "profiler/LogIntegrity.h"
-#include "profiler/LogLineCount.h"
+#include "monitor/LogtailAlarm.h"
+#include "monitor/LogFileProfiler.h"
+#include "monitor/LogIntegrity.h"
+#include "monitor/LogLineCount.h"
 #include "sdk/Common.h"
 #include "sdk/CurlImp.h"
 #include "sdk/Exception.h"

diff --git a/core/config_manager/ConfigManagerBase.cpp b/core/config_manager/ConfigManagerBase.cpp
@@ -43,10 +43,10 @@
 #include "common/GlobalPara.h"
 #include "common/version.h"
 #include "config/UserLogConfigParser.h"
-#include "profiler/LogtailAlarm.h"
-#include "profiler/LogFileProfiler.h"
-#include "profiler/LogIntegrity.h"
-#include "profiler/LogLineCount.h"
+#include "monitor/LogtailAlarm.h"
+#include "monitor/LogFileProfiler.h"
+#include "monitor/LogIntegrity.h"
+#include "monitor/LogLineCount.h"
 #include "app_config/AppConfig.h"
 #include "config_manager/ConfigYamlToJson.h"
 #include "checkpoint/CheckPointManager.h"

diff --git a/core/controller/CMakeLists.txt b/core/controller/CMakeLists.txt
@@ -28,7 +28,7 @@ target_link_libraries(${PROJECT_NAME} event)
 target_link_libraries(${PROJECT_NAME} event_handler)
 target_link_libraries(${PROJECT_NAME} event_listener)
 target_link_libraries(${PROJECT_NAME} sender)
-target_link_libraries(${PROJECT_NAME} profiler)
+target_link_libraries(${PROJECT_NAME} monitor)
 target_link_libraries(${PROJECT_NAME} app_config)
 target_link_libraries(${PROJECT_NAME} log_pb)
 target_link_libraries(${PROJECT_NAME} shennong)

diff --git a/core/controller/EventDispatcher.cpp b/core/controller/EventDispatcher.cpp
@@ -50,10 +50,10 @@
 #include "event/Event.h"
 #include "processor/LogProcess.h"
 #include "sender/Sender.h"
-#include "profiler/LogFileProfiler.h"
-#include "profiler/LogtailAlarm.h"
-#include "profiler/LogIntegrity.h"
-#include "profiler/LogLineCount.h"
+#include "monitor/LogFileProfiler.h"
+#include "monitor/LogtailAlarm.h"
+#include "monitor/LogIntegrity.h"
+#include "monitor/LogLineCount.h"
 #include "log_pb/metric.pb.h"
 #include "log_pb/sls_logs.pb.h"
 #include "checkpoint/CheckPointManager.h"

diff --git a/core/controller/EventDispatcher.h b/core/controller/EventDispatcher.h
@@ -29,7 +29,7 @@
 #include <unordered_map>
 #include <unordered_set>
 #include <set>
-#include "profiler/LogFileProfiler.h"
+#include "monitor/LogFileProfiler.h"
 #include "polling/PollingModify.h"
 #include "polling/PollingDirFile.h"
 #include "event_listener/EventListener.h"

diff --git a/core/controller/EventDispatcherBase.cpp b/core/controller/EventDispatcherBase.cpp
@@ -51,10 +51,11 @@
 #include "event/Event.h"
 #include "processor/LogProcess.h"
 #include "sender/Sender.h"
-#include "profiler/LogFileProfiler.h"
-#include "profiler/LogtailAlarm.h"
-#include "profiler/LogIntegrity.h"
-#include "profiler/LogLineCount.h"
+#include "monitor/LogFileProfiler.h"
+#include "monitor/LogtailAlarm.h"
+#include "monitor/LogIntegrity.h"
+#include "monitor/LogLineCount.h"
+#include "monitor/MetricExportor.h"
 #include "log_pb/metric.pb.h"
 #include "log_pb/sls_logs.pb.h"
 #include "checkpoint/CheckPointManager.h"
@@ -878,6 +879,7 @@ bool EventDispatcherBase::Dispatch() {
         DumpCheckPointPeriod(curTime);
         if (curTime - lastCheckDir >= INT32_FLAG(main_loop_check_interval)) {
             LogFileProfiler::GetInstance()->SendProfileData();
+            MetricExportor::GetInstance()->PushMetrics(false);
 #if defined(__linux__)
             CheckShennong();
 #endif

diff --git a/core/controller/EventDispatcherBase.h b/core/controller/EventDispatcherBase.h
@@ -29,7 +29,7 @@
 #include <unordered_map>
 #include <unordered_set>
 #include <set>
-#include "profiler/LogFileProfiler.h"
+#include "monitor/LogFileProfiler.h"
 #include "polling/PollingModify.h"
 #include "polling/PollingDirFile.h"
 #include "event_listener/EventListener.h"

diff --git a/core/event_handler/CMakeLists.txt b/core/event_handler/CMakeLists.txt
@@ -25,6 +25,6 @@ target_link_libraries(${PROJECT_NAME} app_config)
 target_link_libraries(${PROJECT_NAME} event)
 target_link_libraries(${PROJECT_NAME} controller)
 target_link_libraries(${PROJECT_NAME} config_manager)
-target_link_libraries(${PROJECT_NAME} profiler)
+target_link_libraries(${PROJECT_NAME} monitor)
 target_link_libraries(${PROJECT_NAME} processor)
 target_link_libraries(${PROJECT_NAME} fuse)
diff --git a/core/event_handler/EventHandler.cpp b/core/event_handler/EventHandler.cpp
@@ -25,7 +25,7 @@
 #include "event/BlockEventManager.h"
 #include "controller/EventDispatcher.h"
 #include "config_manager/ConfigManager.h"
-#include "profiler/LogtailAlarm.h"
+#include "monitor/LogtailAlarm.h"
 #include "processor/LogProcess.h"
 #include "logger/Logger.h"
 #include "fuse/FuseFileBlacklist.h"

diff --git a/core/event_handler/LogInput.cpp b/core/event_handler/LogInput.cpp
@@ -24,7 +24,7 @@
 #include "monitor/Monitor.h"
 #include "processor/LogProcess.h"
 #include "controller/EventDispatcher.h"
-#include "profiler/LogtailAlarm.h"
+#include "monitor/LogtailAlarm.h"
 #include "checkpoint/CheckPointManager.h"
 #include "polling/PollingDirFile.h"
 #include "polling/PollingModify.h"

diff --git a/core/event_listener/CMakeLists.txt b/core/event_listener/CMakeLists.txt
@@ -30,5 +30,5 @@ target_link_libraries(${PROJECT_NAME} common)
 target_link_libraries(${PROJECT_NAME} logger)
 target_link_libraries(${PROJECT_NAME} polling)
 target_link_libraries(${PROJECT_NAME} event)
-target_link_libraries(${PROJECT_NAME} profiler)
+target_link_libraries(${PROJECT_NAME} monitor)
 target_link_libraries(${PROJECT_NAME} controller)
diff --git a/core/event_listener/EventListener_Linux.cpp b/core/event_listener/EventListener_Linux.cpp
@@ -17,7 +17,7 @@
 #include <unistd.h>
 #include <sys/ioctl.h>
 #include "logger/Logger.h"
-#include "profiler/LogtailAlarm.h"
+#include "monitor/LogtailAlarm.h"
 #include "common/ErrorUtil.h"
 #include "common/Flags.h"
 #include "controller/EventDispatcher.h"

diff --git a/core/logtail.cpp b/core/logtail.cpp
@@ -40,10 +40,10 @@
 #include "controller/EventDispatcher.h"
 #include "monitor/Monitor.h"
 #include "sender/Sender.h"
-#include "profiler/LogtailAlarm.h"
-#include "profiler/LogFileProfiler.h"
-#include "profiler/LogIntegrity.h"
-#include "profiler/LogLineCount.h"
+#include "monitor/LogtailAlarm.h"
+#include "monitor/LogFileProfiler.h"
+#include "monitor/LogIntegrity.h"
+#include "monitor/LogLineCount.h"
 #include "app_config/AppConfig.h"
 #include "ObserverManager.h"
 using namespace logtail;

diff --git a/core/logtail_windows.cpp b/core/logtail_windows.cpp
@@ -38,10 +38,10 @@
 #include "controller/EventDispatcher.h"
 #include "monitor/Monitor.h"
 #include "sender/Sender.h"
-#include "profiler/LogtailAlarm.h"
-#include "profiler/LogFileProfiler.h"
-#include "profiler/LogIntegrity.h"
-#include "profiler/LogLineCount.h"
+#include "monitor/LogtailAlarm.h"
+#include "monitor/LogFileProfiler.h"
+#include "monitor/LogIntegrity.h"
+#include "monitor/LogLineCount.h"
 #include "app_config/AppConfig.h"
 using namespace logtail;
 

diff --git a/core/monitor/CMakeLists.txt b/core/monitor/CMakeLists.txt
@@ -27,7 +27,6 @@ target_link_libraries(${PROJECT_NAME} common)
 target_link_libraries(${PROJECT_NAME} logger)
 target_link_libraries(${PROJECT_NAME} log_pb)
 target_link_libraries(${PROJECT_NAME} sender)
-target_link_libraries(${PROJECT_NAME} profiler)
 target_link_libraries(${PROJECT_NAME} config_manager)
 target_link_libraries(${PROJECT_NAME} app_config)
 target_link_libraries(${PROJECT_NAME} event_handler)

diff --git a/core/profiler/LogFileProfiler.cpp → core/monitor/LogFileProfiler.cpp b/core/profiler/LogFileProfiler.cpp → core/monitor/LogFileProfiler.cpp
diff --git a/core/profiler/LogFileProfiler.h → core/monitor/LogFileProfiler.h b/core/profiler/LogFileProfiler.h → core/monitor/LogFileProfiler.h
diff --git a/core/profiler/LogIntegrity.cpp → core/monitor/LogIntegrity.cpp b/core/profiler/LogIntegrity.cpp → core/monitor/LogIntegrity.cpp
diff --git a/core/profiler/LogIntegrity.h → core/monitor/LogIntegrity.h b/core/profiler/LogIntegrity.h → core/monitor/LogIntegrity.h
diff --git a/core/profiler/LogLineCount.cpp → core/monitor/LogLineCount.cpp b/core/profiler/LogLineCount.cpp → core/monitor/LogLineCount.cpp
diff --git a/core/profiler/LogLineCount.h → core/monitor/LogLineCount.h b/core/profiler/LogLineCount.h → core/monitor/LogLineCount.h