From 8be811cb7d1622f322be832c339994d4e6c058fb Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Fri, 14 Jul 2023 06:43:54 +0000 Subject: [PATCH 01/39] add metric --- core/CMakeLists.txt | 3 +- core/ilogtail_metric/CMakeLists.txt | 30 ++++ core/ilogtail_metric/ILogtaiMetric.cpp | 157 ++++++++++++++++++ core/ilogtail_metric/ILogtailMetric.h | 77 +++++++++ core/ilogtail_metric/MetricConstants.h | 25 +++ core/ilogtail_metric/MetricExportor.cpp | 26 +++ core/ilogtail_metric/MetricExportor.h | 45 +++++ core/unittest/CMakeLists.txt | 1 + core/unittest/ilogtail_metric/CMakeLists.txt | 19 +++ .../ILogtailMetricUnittest.cpp | 71 ++++++++ 10 files changed, 453 insertions(+), 1 deletion(-) create mode 100644 core/ilogtail_metric/CMakeLists.txt create mode 100644 core/ilogtail_metric/ILogtaiMetric.cpp create mode 100644 core/ilogtail_metric/ILogtailMetric.h create mode 100644 core/ilogtail_metric/MetricConstants.h create mode 100644 core/ilogtail_metric/MetricExportor.cpp create mode 100644 core/ilogtail_metric/MetricExportor.h create mode 100644 core/unittest/ilogtail_metric/CMakeLists.txt create mode 100644 core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index ecb3703144..7d0d04b352 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -81,7 +81,7 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/dependencies.cmake) # Subdirectories (modules). set(SUB_DIRECTORIES_LIST aggregator app_config checkpoint common config config_manager config_server_pb - controller event event_handler event_listener helper log_pb logger monitor + controller event event_handler event_listener helper log_pb logger monitor ilogtail_metric parser plugin polling processor profiler reader profile_sender sender shennong sdk fuse sls_control ) @@ -119,6 +119,7 @@ target_link_libraries(${LOGTAIL_TARGET} controller plugin monitor + ilogtail_metric sender profiler profile_sender diff --git a/core/ilogtail_metric/CMakeLists.txt b/core/ilogtail_metric/CMakeLists.txt new file mode 100644 index 0000000000..225313a0cb --- /dev/null +++ b/core/ilogtail_metric/CMakeLists.txt @@ -0,0 +1,30 @@ +# Copyright 2022 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 2.9) +project(ilogtail_metric) + +if (MSVC) + add_definitions(-DTARGETLIBS=Psapi.lib) + add_definitions(-DPSAPI_VERSION=1) +endif () + +file(GLOB LIB_SOURCE_FILES *.cpp *.h) +append_source_files(LIB_SOURCE_FILES) +add_library(${PROJECT_NAME} STATIC ${LIB_SOURCE_FILES}) +target_link_libraries(${PROJECT_NAME} log_pb) + +if (MSVC) + target_link_libraries(${PROJECT_NAME} "Psapi.lib") +endif () \ No newline at end of file diff --git a/core/ilogtail_metric/ILogtaiMetric.cpp b/core/ilogtail_metric/ILogtaiMetric.cpp new file mode 100644 index 0000000000..2135c21aeb --- /dev/null +++ b/core/ilogtail_metric/ILogtaiMetric.cpp @@ -0,0 +1,157 @@ +#include "ILogtailMetric.h" + +namespace logtail { + + +Counter::Counter(std::string name) { + mName = name; + mVal = (uint64_t)0; + mTimestamp = time(NULL); +} + +Counter::Counter() { + mVal = (uint64_t)0; + mTimestamp = time(NULL); +} + +Counter::~Counter() { +} + +Metrics::Metrics() { + mDeleted.store(false); +} + +Metrics::Metrics(std::vector> labels) { + mLabels = labels; + mDeleted.store(false); +} + +Metrics::~Metrics() { +} + +WriteMetrics::WriteMetrics() { +} + +WriteMetrics::~WriteMetrics() { +} + +ReadMetrics::ReadMetrics() { + mWriteMetrics = WriteMetrics::GetInstance(); +} + +ReadMetrics::~ReadMetrics() { +} + +void Counter::Add(uint64_t value) { + mVal += value; + mTimestamp = time(NULL); +} + +void Counter::Set(uint64_t value) { + mVal = value; + mTimestamp = time(NULL); +} + +Counter* Counter::Copy() { + Counter* counter = new Counter(mName); + counter->mVal = mVal.exchange(0); + counter->mTimestamp = mTimestamp.exchange(0); + return counter; +} + +Counter* Metrics::CreateCounter(std::string name) { + Counter* counter = new Counter(name); + mValues.push_back(counter); + return counter; +} + + +Metrics* Metrics::Copy() { + Metrics* metrics = new Metrics(); + for (std::vector>::iterator it = mLabels.begin(); it != mLabels.end(); ++it) { + std::pair pair = *it; + metrics->mLabels.push_back(pair); + } + for (std::vector::iterator it = mValues.begin(); it != mValues.end(); ++it) { + Counter* cur = *it; + metrics->mValues.push_back(cur->Copy()); + } + return metrics; +} + +Metrics* WriteMetrics::CreateMetrics(std::vector> labels) { + std::lock_guard lock(mMutex); + Metrics* cur = new Metrics(labels); + mTail->next = cur; + mTail = cur; + return cur; +} + + +void WriteMetrics::DestroyMetrics(Metrics* metrics) { + std::lock_guard lock(mMutex); + // mark as delete + metrics->mDeleted = true; + mDeletedMetrics.push_back(metrics); + +} + +void WriteMetrics::ClearDeleted() { + std::lock_guard lock(mMutex); + Metrics* cur = mHead; + while(cur) { + // unlink metrics from mHead and add to mDeletedMetrics; + // do not modify metrics's next pointer to let who is reading this Metrics can read next Metrics normally + Metrics* next = cur->next; + if (next != NULL && next->mDeleted) { + cur->next = next->next; + } + cur = cur->next; + } + std::vector tmp; + tmp.swap(mDeletedMetrics); +} + +Metrics* WriteMetrics::DoSnapshot() { + Metrics* snapshot; + Metrics* tmp = mHead; + Metrics* cTmp; + if (tmp) { + Metrics* newMetrics = tmp->Copy(); + tmp = tmp -> next; + snapshot = newMetrics; + cTmp = snapshot; + while(tmp) { + while(tmp->mDeleted) { + tmp = tmp -> next; + } + Metrics* newMetrics = tmp->Copy(); + tmp = tmp -> next; + cTmp -> next = newMetrics; + cTmp = newMetrics; + } + } + ClearDeleted(); + return snapshot; +} + +void ReadMetrics::Read() { + std::lock_guard lock(mMutex); + // Do some read +} + +void ReadMetrics::UpdateMetrics() { + Metrics* snapshot = mWriteMetrics->DoSnapshot(); + Metrics* toDelete = mHead; + { + std::lock_guard lock(mMutex); + mHead = snapshot->next; + } + // do deletion + while(toDelete) { + Metrics* obj = toDelete; + toDelete = toDelete->next; + delete obj; + } +} +} \ No newline at end of file diff --git a/core/ilogtail_metric/ILogtailMetric.h b/core/ilogtail_metric/ILogtailMetric.h new file mode 100644 index 0000000000..9825ea4f2d --- /dev/null +++ b/core/ilogtail_metric/ILogtailMetric.h @@ -0,0 +1,77 @@ +#pragma once +#include +#include +#include +#include +#include "logger/Logger.h" +#include + + +namespace logtail { + + +class Counter{ + + public: + Counter(); + Counter(std::string name); + ~Counter(); + + std::string mName; + std::atomic_long mVal; + std::atomic_long mTimestamp; + void Add(uint64_t val); + void Set(uint64_t val); + Counter* Copy(); +}; + +class Metrics { + public: + Metrics(std::vector > labels); + Metrics(); + ~Metrics(); + + std::atomic_bool mDeleted; + std::vector> mLabels; + std::vector mValues; + Counter* CreateCounter(std::string Name); + Metrics* Copy(); + Metrics* next = NULL; +}; + +class WriteMetrics { + private: + WriteMetrics(); + ~WriteMetrics(); + public: + static WriteMetrics* GetInstance() { + static WriteMetrics* ptr = new WriteMetrics(); + return ptr; + } + Metrics* CreateMetrics(std::vector> Labels); + void DestroyMetrics(Metrics* metrics); + void ClearDeleted(); + Metrics* DoSnapshot(); + // empty head node + Metrics* mHead = new Metrics(); + Metrics* mTail = mHead; + std::vector mDeletedMetrics; + std::mutex mMutex; +}; + +class ReadMetrics { + private: + ReadMetrics(); + ~ReadMetrics(); + public: + static ReadMetrics* GetInstance() { + static ReadMetrics* ptr = new ReadMetrics(); + return ptr; + } + void Read(); + void UpdateMetrics(); + Metrics* mHead = NULL; + std::mutex mMutex; + WriteMetrics* mWriteMetrics; +}; +} \ No newline at end of file diff --git a/core/ilogtail_metric/MetricConstants.h b/core/ilogtail_metric/MetricConstants.h new file mode 100644 index 0000000000..794039fc9d --- /dev/null +++ b/core/ilogtail_metric/MetricConstants.h @@ -0,0 +1,25 @@ +/* + * Copyright 2022 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace logtail { + +const std::string METRIC_FILE_READ_COUNT = "file_read_count"; +const std::string METRIC_FILE_READ_BYTES = "file_read_bytes"; + +} // namespace logtail \ No newline at end of file diff --git a/core/ilogtail_metric/MetricExportor.cpp b/core/ilogtail_metric/MetricExportor.cpp new file mode 100644 index 0000000000..d4287f5322 --- /dev/null +++ b/core/ilogtail_metric/MetricExportor.cpp @@ -0,0 +1,26 @@ +#include "MetricExportor.h" +#include "sender/Sender.h" +#include "log_pb/sls_logs.pb.h" +#include "common/LogtailCommonFlags.h" + +using namespace sls_logs; +using namespace std; + +namespace logtail { + +MetricExportor::MetricExportor() { + mSendInterval = 10; + mSnapshotInterval = 10; + mLastSendTime = 0; + mLastSnapshotTime = 0; +} + +void MetricExportor::pushMetrics() { + LOG_INFO(sLogger, ("pushMetrics", "")); + //pushInstanceMetric(false); + //pushPluginMetric(false); + //pushSubPluginMetric(false); +} + + +} \ No newline at end of file diff --git a/core/ilogtail_metric/MetricExportor.h b/core/ilogtail_metric/MetricExportor.h new file mode 100644 index 0000000000..64d615421e --- /dev/null +++ b/core/ilogtail_metric/MetricExportor.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include "profile_sender/ProfileSender.h" +#include "ILogtailMetric.h" +#include "logger/Logger.h" +#include "plugin/LogtailPlugin.h" +#include "ProfileSender.h" +#include "log_pb/sls_logs.pb.h" + +namespace logtail { + +class MetricExportor { +public: + static MetricExportor* GetInstance() { + static MetricExportor* ptr = new MetricExportor(); + return ptr; + } + void pushMetrics(); + +private: + MetricExportor(); + ~MetricExportor() {} + + + /* + void snapshotMetrics(bool force); + void pushInstanceMetric(bool forceSend); + void snapshotPluginMetrics(); + void BuildLogFromMetric(sls_logs::LogGroup& logGroup, PipelineMetricPtr pipelineMetric); + std::string BuildMetricLabel(std::unordered_map labels); + std::list mSnapshotPipelineMetrics; + */ + ProfileSender mProfileSender; + int32_t mSendInterval; + int32_t mLastSendTime; + int32_t mSnapshotInterval; + int32_t mLastSnapshotTime; +}; +} \ No newline at end of file diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 54ffa3ef2a..9245d93b1d 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -57,6 +57,7 @@ add_subdirectory(polling) add_subdirectory(processor) add_subdirectory(reader) add_subdirectory(sender) +add_subdirectory(ilogtail_metric) add_subdirectory(profiler) add_subdirectory(sdk) if (UNIX) diff --git a/core/unittest/ilogtail_metric/CMakeLists.txt b/core/unittest/ilogtail_metric/CMakeLists.txt new file mode 100644 index 0000000000..ab1a53d73f --- /dev/null +++ b/core/unittest/ilogtail_metric/CMakeLists.txt @@ -0,0 +1,19 @@ +# Copyright 2022 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 2.9) +project(ilogtail_metric_unittest) + +add_executable(ilogtail_metric_unittest ILogtailMetricUnittest.cpp) +target_link_libraries(ilogtail_metric_unittest unittest_base) \ No newline at end of file diff --git a/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp b/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp new file mode 100644 index 0000000000..0e359dbf3b --- /dev/null +++ b/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp @@ -0,0 +1,71 @@ +#include "unittest/Unittest.h" +#include +#include +#include +#include +#include "util.h" +#include "ILogtailMetric.h" +#include "MetricConstants.h" + +namespace logtail { + +class ILogtailMetricUnittest : public ::testing::Test { +public: + void TestCreateMetric(); + void TestDeleteMetric(); +}; + +APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateMetric, 0); +APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestDeleteMetric, 0); + +void ILogtailMetricUnittest::TestCreateMetric() { + // create + //PluginMetric* pluginMetric = new PluginMetric(); + std::vector> labels; + labels.push_back(std::make_pair("project","project1")); + labels.push_back(std::make_pair("logstore","logstore1")); + + Metrics* fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); + LOG_INFO(sLogger, ("labelSize", fileMetric->mLabels.size())); + APSARA_TEST_EQUAL(fileMetric->mLabels.size(), 2); + + Metrics* fileMetric2 = WriteMetrics::GetInstance()->CreateMetrics(labels); + LOG_INFO(sLogger, ("labelSize", fileMetric2->mLabels.size())); + APSARA_TEST_EQUAL(fileMetric2->mLabels.size(), 2); + + Metrics* fileMetric3 = WriteMetrics::GetInstance()->CreateMetrics(labels); + LOG_INFO(sLogger, ("labelSize", fileMetric3->mLabels.size())); + APSARA_TEST_EQUAL(fileMetric3->mLabels.size(), 2); + + + Counter* fileCounter = fileMetric->CreateCounter("filed1"); + fileCounter->Add((uint64_t)111); + LOG_INFO(sLogger, ("value", fileCounter->mVal)("time", fileCounter->mTimestamp)); + + Counter* fileCounter2 = fileMetric->CreateCounter("filed2"); + fileCounter2->Add((uint64_t)222); + LOG_INFO(sLogger, ("value", fileCounter2->mVal)("time", fileCounter2->mTimestamp)); + + + WriteMetrics::GetInstance()->DestroyMetrics(fileMetric2); + // delete first element + WriteMetrics::GetInstance()->DestroyMetrics(fileMetric); + ReadMetrics::GetInstance()->UpdateMetrics(); + + Metrics* head = ReadMetrics::GetInstance()->mHead; + while(head) { + LOG_INFO(sLogger, ("ReadMetrics", head->mLabels.size())); + head = head->next; + } +} + +void ILogtailMetricUnittest::TestDeleteMetric() { + +} +}// namespace logtail + +int main(int argc, char** argv) { + logtail::Logger::Instance().InitGlobalLoggers(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file From 91321641b2ad46156b662ca91d84dde115d523e0 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Mon, 17 Jul 2023 10:12:06 +0800 Subject: [PATCH 02/39] refine code --- core/ilogtail_metric/ILogtaiMetric.cpp | 115 ++++++++++-------- core/ilogtail_metric/ILogtailMetric.h | 34 +++--- core/logtail.cpp | 2 +- .../ILogtailMetricUnittest.cpp | 18 +-- 4 files changed, 93 insertions(+), 76 deletions(-) diff --git a/core/ilogtail_metric/ILogtaiMetric.cpp b/core/ilogtail_metric/ILogtaiMetric.cpp index 2135c21aeb..3afb9eefe5 100644 --- a/core/ilogtail_metric/ILogtaiMetric.cpp +++ b/core/ilogtail_metric/ILogtaiMetric.cpp @@ -9,18 +9,9 @@ Counter::Counter(std::string name) { mTimestamp = time(NULL); } -Counter::Counter() { - mVal = (uint64_t)0; - mTimestamp = time(NULL); -} - Counter::~Counter() { } -Metrics::Metrics() { - mDeleted.store(false); -} - Metrics::Metrics(std::vector> labels) { mLabels = labels; mDeleted.store(false); @@ -52,7 +43,16 @@ void Counter::Set(uint64_t value) { mTimestamp = time(NULL); } -Counter* Counter::Copy() { +uint64_t Counter::GetValue() { + return mVal; +} + + uint64_t Counter::GetTimestamp() { + return mTimestamp; + } + + +Counter* Counter::CopyAndReset() { Counter* counter = new Counter(mName); counter->mVal = mVal.exchange(0); counter->mTimestamp = mTimestamp.exchange(0); @@ -66,15 +66,29 @@ Counter* Metrics::CreateCounter(std::string name) { } +void Metrics::MarkDeleted() { + mDeleted = true; +} + +bool Metrics::IsDeleted() { + return mDeleted; +} + +std::vector> Metrics::GetLabels() { + return mLabels; +} + + Metrics* Metrics::Copy() { - Metrics* metrics = new Metrics(); + std::vector> newLabels; for (std::vector>::iterator it = mLabels.begin(); it != mLabels.end(); ++it) { std::pair pair = *it; - metrics->mLabels.push_back(pair); + newLabels.push_back(pair); } + Metrics* metrics = new Metrics(newLabels); for (std::vector::iterator it = mValues.begin(); it != mValues.end(); ++it) { Counter* cur = *it; - metrics->mValues.push_back(cur->Copy()); + metrics->mValues.push_back(cur->CopyAndReset()); } return metrics; } @@ -82,61 +96,59 @@ Metrics* Metrics::Copy() { Metrics* WriteMetrics::CreateMetrics(std::vector> labels) { std::lock_guard lock(mMutex); Metrics* cur = new Metrics(labels); - mTail->next = cur; - mTail = cur; + + Metrics* oldHead = mHead; + mHead = cur; + mHead -> next = oldHead; return cur; } void WriteMetrics::DestroyMetrics(Metrics* metrics) { - std::lock_guard lock(mMutex); // mark as delete - metrics->mDeleted = true; - mDeletedMetrics.push_back(metrics); - + metrics->MarkDeleted(); } -void WriteMetrics::ClearDeleted() { - std::lock_guard lock(mMutex); - Metrics* cur = mHead; - while(cur) { - // unlink metrics from mHead and add to mDeletedMetrics; - // do not modify metrics's next pointer to let who is reading this Metrics can read next Metrics normally - Metrics* next = cur->next; - if (next != NULL && next->mDeleted) { - cur->next = next->next; - } - cur = cur->next; - } - std::vector tmp; - tmp.swap(mDeletedMetrics); -} Metrics* WriteMetrics::DoSnapshot() { - Metrics* snapshot; + Metrics* snapshot = NULL; + Metrics* wTmp = NULL; + Metrics* wTmpHead = NULL; + Metrics* tmp = mHead; - Metrics* cTmp; - if (tmp) { + Metrics* rTmp ; + while(tmp) { + if (tmp->IsDeleted()) { + Metrics* toDeleted = tmp; + tmp = tmp -> next; + delete toDeleted; + continue; + } Metrics* newMetrics = tmp->Copy(); - tmp = tmp -> next; - snapshot = newMetrics; - cTmp = snapshot; - while(tmp) { - while(tmp->mDeleted) { - tmp = tmp -> next; - } - Metrics* newMetrics = tmp->Copy(); + // Get Head + if (!snapshot) { + wTmpHead = tmp; + wTmp = wTmpHead; tmp = tmp -> next; - cTmp -> next = newMetrics; - cTmp = newMetrics; + snapshot = newMetrics; + rTmp = snapshot; + continue; } + wTmp -> next = tmp; + tmp = tmp -> next; + rTmp -> next = newMetrics; + rTmp = newMetrics; + } + { + std::lock_guard lock(mMutex); + mHead = wTmpHead; } - ClearDeleted(); return snapshot; } void ReadMetrics::Read() { - std::lock_guard lock(mMutex); + ReadLock lock(mReadWriteLock); + // Do some read } @@ -144,8 +156,9 @@ void ReadMetrics::UpdateMetrics() { Metrics* snapshot = mWriteMetrics->DoSnapshot(); Metrics* toDelete = mHead; { - std::lock_guard lock(mMutex); - mHead = snapshot->next; + mReadWriteLock.lock(); + mHead = snapshot; + mReadWriteLock.unlock(); } // do deletion while(toDelete) { diff --git a/core/ilogtail_metric/ILogtailMetric.h b/core/ilogtail_metric/ILogtailMetric.h index 9825ea4f2d..7c26af49b5 100644 --- a/core/ilogtail_metric/ILogtailMetric.h +++ b/core/ilogtail_metric/ILogtailMetric.h @@ -5,35 +5,41 @@ #include #include "logger/Logger.h" #include +#include "common/Lock.h" namespace logtail { class Counter{ + private: + std::string mName; + std::atomic_long mVal; + std::atomic_long mTimestamp; public: - Counter(); Counter(std::string name); ~Counter(); - - std::string mName; - std::atomic_long mVal; - std::atomic_long mTimestamp; + uint64_t GetValue(); + uint64_t GetTimestamp(); + void Add(uint64_t val); void Set(uint64_t val); - Counter* Copy(); + Counter* CopyAndReset(); }; class Metrics { + private: + std::vector> mLabels; + std::vector mValues; + std::atomic_bool mDeleted; + public: Metrics(std::vector > labels); - Metrics(); ~Metrics(); - - std::atomic_bool mDeleted; - std::vector> mLabels; - std::vector mValues; + void MarkDeleted(); + bool IsDeleted(); + std::vector> GetLabels(); Counter* CreateCounter(std::string Name); Metrics* Copy(); Metrics* next = NULL; @@ -53,9 +59,7 @@ class WriteMetrics { void ClearDeleted(); Metrics* DoSnapshot(); // empty head node - Metrics* mHead = new Metrics(); - Metrics* mTail = mHead; - std::vector mDeletedMetrics; + Metrics* mHead = NULL; std::mutex mMutex; }; @@ -71,7 +75,7 @@ class ReadMetrics { void Read(); void UpdateMetrics(); Metrics* mHead = NULL; - std::mutex mMutex; WriteMetrics* mWriteMetrics; + ReadWriteLock mReadWriteLock; }; } \ No newline at end of file diff --git a/core/logtail.cpp b/core/logtail.cpp index ba2b2f32f9..caa174b6f6 100644 --- a/core/logtail.cpp +++ b/core/logtail.cpp @@ -58,7 +58,7 @@ void* __wrap_memcpy(void* dest, const void* src, size_t n) { } #endif -DEFINE_FLAG_BOOL(ilogtail_disable_core, "disable core in worker process", true); +DEFINE_FLAG_BOOL(ilogtail_disable_core, "disable core in worker process", false); DEFINE_FLAG_INT32(fork_interval, "fork dispatcher process interval", 10); DECLARE_FLAG_INT32(max_open_files_limit); DECLARE_FLAG_INT32(max_reader_open_files); diff --git a/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp b/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp index 0e359dbf3b..0b6cedd0d0 100644 --- a/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp +++ b/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp @@ -26,25 +26,25 @@ void ILogtailMetricUnittest::TestCreateMetric() { labels.push_back(std::make_pair("logstore","logstore1")); Metrics* fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); - LOG_INFO(sLogger, ("labelSize", fileMetric->mLabels.size())); - APSARA_TEST_EQUAL(fileMetric->mLabels.size(), 2); + LOG_INFO(sLogger, ("labelSize", fileMetric->GetLabels().size())); + APSARA_TEST_EQUAL(fileMetric->GetLabels().size(), 2); Metrics* fileMetric2 = WriteMetrics::GetInstance()->CreateMetrics(labels); - LOG_INFO(sLogger, ("labelSize", fileMetric2->mLabels.size())); - APSARA_TEST_EQUAL(fileMetric2->mLabels.size(), 2); + LOG_INFO(sLogger, ("labelSize", fileMetric2->GetLabels().size())); + APSARA_TEST_EQUAL(fileMetric2->GetLabels().size(), 2); Metrics* fileMetric3 = WriteMetrics::GetInstance()->CreateMetrics(labels); - LOG_INFO(sLogger, ("labelSize", fileMetric3->mLabels.size())); - APSARA_TEST_EQUAL(fileMetric3->mLabels.size(), 2); + LOG_INFO(sLogger, ("labelSize", fileMetric3->GetLabels().size())); + APSARA_TEST_EQUAL(fileMetric3->GetLabels().size(), 2); Counter* fileCounter = fileMetric->CreateCounter("filed1"); fileCounter->Add((uint64_t)111); - LOG_INFO(sLogger, ("value", fileCounter->mVal)("time", fileCounter->mTimestamp)); + LOG_INFO(sLogger, ("value", fileCounter->GetValue())("time", fileCounter->GetTimestamp())); Counter* fileCounter2 = fileMetric->CreateCounter("filed2"); fileCounter2->Add((uint64_t)222); - LOG_INFO(sLogger, ("value", fileCounter2->mVal)("time", fileCounter2->mTimestamp)); + LOG_INFO(sLogger, ("value", fileCounter2->GetValue())("time", fileCounter2->GetTimestamp())); WriteMetrics::GetInstance()->DestroyMetrics(fileMetric2); @@ -54,7 +54,7 @@ void ILogtailMetricUnittest::TestCreateMetric() { Metrics* head = ReadMetrics::GetInstance()->mHead; while(head) { - LOG_INFO(sLogger, ("ReadMetrics", head->mLabels.size())); + LOG_INFO(sLogger, ("ReadMetrics", head->GetLabels().size())); head = head->next; } } From 7ee4a51e82911d56caa41f7871a343c68af15103 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Mon, 17 Jul 2023 10:25:24 +0800 Subject: [PATCH 03/39] refine code --- core/ilogtail_metric/ILogtaiMetric.cpp | 37 +++++++++++++++---------- core/ilogtail_metric/ILogtailMetric.h | 1 - core/ilogtail_metric/MetricExportor.cpp | 3 -- core/ilogtail_metric/MetricExportor.h | 9 ------ 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/core/ilogtail_metric/ILogtaiMetric.cpp b/core/ilogtail_metric/ILogtaiMetric.cpp index 3afb9eefe5..34ef05df61 100644 --- a/core/ilogtail_metric/ILogtaiMetric.cpp +++ b/core/ilogtail_metric/ILogtaiMetric.cpp @@ -78,7 +78,6 @@ std::vector> Metrics::GetLabels() { return mLabels; } - Metrics* Metrics::Copy() { std::vector> newLabels; for (std::vector>::iterator it = mLabels.begin(); it != mLabels.end(); ++it) { @@ -99,28 +98,34 @@ Metrics* WriteMetrics::CreateMetrics(std::vector next = oldHead; + mHead->next = oldHead; return cur; } +// mark as deleted void WriteMetrics::DestroyMetrics(Metrics* metrics) { - // mark as delete + // deleted is atomic_bool, no need to lock metrics->MarkDeleted(); } Metrics* WriteMetrics::DoSnapshot() { + // new read head Metrics* snapshot = NULL; - Metrics* wTmp = NULL; + // new read head iter + Metrics* rTmp ; + // new write head Metrics* wTmpHead = NULL; - + // new write head iter + Metrics* wTmp = NULL; + // old head iter Metrics* tmp = mHead; - Metrics* rTmp ; + while(tmp) { if (tmp->IsDeleted()) { Metrics* toDeleted = tmp; - tmp = tmp -> next; + tmp = tmp->next; delete toDeleted; continue; } @@ -129,14 +134,14 @@ Metrics* WriteMetrics::DoSnapshot() { if (!snapshot) { wTmpHead = tmp; wTmp = wTmpHead; - tmp = tmp -> next; + tmp = tmp->next; snapshot = newMetrics; rTmp = snapshot; continue; } - wTmp -> next = tmp; - tmp = tmp -> next; - rTmp -> next = newMetrics; + wTmp->next = tmp; + tmp = tmp->next; + rTmp->next = newMetrics; rTmp = newMetrics; } { @@ -146,10 +151,14 @@ Metrics* WriteMetrics::DoSnapshot() { return snapshot; } -void ReadMetrics::Read() { +void ReadMetrics::ReadAsLogGroup() { ReadLock lock(mReadWriteLock); + // Do some read +} - // Do some read +void ReadMetrics::ReadAsPrometheus() { + ReadLock lock(mReadWriteLock); + // Do some read } void ReadMetrics::UpdateMetrics() { @@ -160,7 +169,7 @@ void ReadMetrics::UpdateMetrics() { mHead = snapshot; mReadWriteLock.unlock(); } - // do deletion + // delete old linklist while(toDelete) { Metrics* obj = toDelete; toDelete = toDelete->next; diff --git a/core/ilogtail_metric/ILogtailMetric.h b/core/ilogtail_metric/ILogtailMetric.h index 7c26af49b5..9237e9b881 100644 --- a/core/ilogtail_metric/ILogtailMetric.h +++ b/core/ilogtail_metric/ILogtailMetric.h @@ -56,7 +56,6 @@ class WriteMetrics { } Metrics* CreateMetrics(std::vector> Labels); void DestroyMetrics(Metrics* metrics); - void ClearDeleted(); Metrics* DoSnapshot(); // empty head node Metrics* mHead = NULL; diff --git a/core/ilogtail_metric/MetricExportor.cpp b/core/ilogtail_metric/MetricExportor.cpp index d4287f5322..4d7347bbc7 100644 --- a/core/ilogtail_metric/MetricExportor.cpp +++ b/core/ilogtail_metric/MetricExportor.cpp @@ -17,9 +17,6 @@ MetricExportor::MetricExportor() { void MetricExportor::pushMetrics() { LOG_INFO(sLogger, ("pushMetrics", "")); - //pushInstanceMetric(false); - //pushPluginMetric(false); - //pushSubPluginMetric(false); } diff --git a/core/ilogtail_metric/MetricExportor.h b/core/ilogtail_metric/MetricExportor.h index 64d615421e..f3ede878f5 100644 --- a/core/ilogtail_metric/MetricExportor.h +++ b/core/ilogtail_metric/MetricExportor.h @@ -27,15 +27,6 @@ class MetricExportor { MetricExportor(); ~MetricExportor() {} - - /* - void snapshotMetrics(bool force); - void pushInstanceMetric(bool forceSend); - void snapshotPluginMetrics(); - void BuildLogFromMetric(sls_logs::LogGroup& logGroup, PipelineMetricPtr pipelineMetric); - std::string BuildMetricLabel(std::unordered_map labels); - std::list mSnapshotPipelineMetrics; - */ ProfileSender mProfileSender; int32_t mSendInterval; int32_t mLastSendTime; From f3c5677b93a68accad457fd93f324ef32b918b38 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Mon, 17 Jul 2023 10:35:02 +0800 Subject: [PATCH 04/39] refine code --- core/ilogtail_metric/ILogtaiMetric.cpp | 16 ++++++++++------ core/ilogtail_metric/ILogtailMetric.h | 3 ++- .../ilogtail_metric/ILogtailMetricUnittest.cpp | 5 ----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/ilogtail_metric/ILogtaiMetric.cpp b/core/ilogtail_metric/ILogtaiMetric.cpp index 34ef05df61..854ae6e2b2 100644 --- a/core/ilogtail_metric/ILogtaiMetric.cpp +++ b/core/ilogtail_metric/ILogtaiMetric.cpp @@ -67,7 +67,7 @@ Counter* Metrics::CreateCounter(std::string name) { void Metrics::MarkDeleted() { - mDeleted = true; + mDeleted.store(true); } bool Metrics::IsDeleted() { @@ -93,12 +93,14 @@ Metrics* Metrics::Copy() { } Metrics* WriteMetrics::CreateMetrics(std::vector> labels) { - std::lock_guard lock(mMutex); - Metrics* cur = new Metrics(labels); - + Metrics* cur = new Metrics(labels); + // add metric to head Metrics* oldHead = mHead; - mHead = cur; - mHead->next = oldHead; + { + std::lock_guard lock(mMutex); + mHead = cur; + mHead->next = oldHead; + } return cur; } @@ -144,6 +146,7 @@ Metrics* WriteMetrics::DoSnapshot() { rTmp->next = newMetrics; rTmp = newMetrics; } + // Only lock when change head { std::lock_guard lock(mMutex); mHead = wTmpHead; @@ -165,6 +168,7 @@ void ReadMetrics::UpdateMetrics() { Metrics* snapshot = mWriteMetrics->DoSnapshot(); Metrics* toDelete = mHead; { + // Only lock when change head mReadWriteLock.lock(); mHead = snapshot; mReadWriteLock.unlock(); diff --git a/core/ilogtail_metric/ILogtailMetric.h b/core/ilogtail_metric/ILogtailMetric.h index 9237e9b881..479c99ac0f 100644 --- a/core/ilogtail_metric/ILogtailMetric.h +++ b/core/ilogtail_metric/ILogtailMetric.h @@ -71,7 +71,8 @@ class ReadMetrics { static ReadMetrics* ptr = new ReadMetrics(); return ptr; } - void Read(); + void ReadAsLogGroup(); + void ReadAsPrometheus(); void UpdateMetrics(); Metrics* mHead = NULL; WriteMetrics* mWriteMetrics; diff --git a/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp b/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp index 0b6cedd0d0..bcf1480aae 100644 --- a/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp +++ b/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp @@ -12,11 +12,9 @@ namespace logtail { class ILogtailMetricUnittest : public ::testing::Test { public: void TestCreateMetric(); - void TestDeleteMetric(); }; APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateMetric, 0); -APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestDeleteMetric, 0); void ILogtailMetricUnittest::TestCreateMetric() { // create @@ -59,9 +57,6 @@ void ILogtailMetricUnittest::TestCreateMetric() { } } -void ILogtailMetricUnittest::TestDeleteMetric() { - -} }// namespace logtail int main(int argc, char** argv) { From 356047e9af338d2d4d9b1cbcd90975ce7ddafd9e Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Mon, 17 Jul 2023 18:50:57 +0800 Subject: [PATCH 05/39] refine code --- core/ilogtail_metric/ILogtaiMetric.cpp | 2 +- core/ilogtail_metric/ILogtailMetric.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/core/ilogtail_metric/ILogtaiMetric.cpp b/core/ilogtail_metric/ILogtaiMetric.cpp index 854ae6e2b2..006bacd513 100644 --- a/core/ilogtail_metric/ILogtaiMetric.cpp +++ b/core/ilogtail_metric/ILogtaiMetric.cpp @@ -82,7 +82,7 @@ Metrics* Metrics::Copy() { std::vector> newLabels; for (std::vector>::iterator it = mLabels.begin(); it != mLabels.end(); ++it) { std::pair pair = *it; - newLabels.push_back(pair); + newLabels.push_back(std::make_pair(pair.first, pair.second)); } Metrics* metrics = new Metrics(newLabels); for (std::vector::iterator it = mValues.begin(); it != mValues.end(); ++it) { diff --git a/core/ilogtail_metric/ILogtailMetric.h b/core/ilogtail_metric/ILogtailMetric.h index 479c99ac0f..f56d8301e6 100644 --- a/core/ilogtail_metric/ILogtailMetric.h +++ b/core/ilogtail_metric/ILogtailMetric.h @@ -57,7 +57,6 @@ class WriteMetrics { Metrics* CreateMetrics(std::vector> Labels); void DestroyMetrics(Metrics* metrics); Metrics* DoSnapshot(); - // empty head node Metrics* mHead = NULL; std::mutex mMutex; }; From 4e1e3293aca46d432499c0823e9a3c9f36d1e610 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Mon, 17 Jul 2023 19:05:23 +0800 Subject: [PATCH 06/39] refine code --- core/ilogtail_metric/ILogtaiMetric.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/ilogtail_metric/ILogtaiMetric.cpp b/core/ilogtail_metric/ILogtaiMetric.cpp index 006bacd513..f88f413b3b 100644 --- a/core/ilogtail_metric/ILogtaiMetric.cpp +++ b/core/ilogtail_metric/ILogtaiMetric.cpp @@ -169,9 +169,8 @@ void ReadMetrics::UpdateMetrics() { Metrics* toDelete = mHead; { // Only lock when change head - mReadWriteLock.lock(); + WriteLock lock(mReadWriteLock); mHead = snapshot; - mReadWriteLock.unlock(); } // delete old linklist while(toDelete) { From 0a60735d644940e115a4cb88d984b1f0b741f16e Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Mon, 24 Jul 2023 10:57:18 +0800 Subject: [PATCH 07/39] add exportor --- core/ilogtail_metric/ILogtaiMetric.cpp | 76 +++++++++++++++---- core/ilogtail_metric/ILogtailMetric.h | 8 +- core/ilogtail_metric/MetricExportor.cpp | 15 +++- core/ilogtail_metric/MetricExportor.h | 2 +- .../ILogtailMetricUnittest.cpp | 2 + 5 files changed, 83 insertions(+), 20 deletions(-) diff --git a/core/ilogtail_metric/ILogtaiMetric.cpp b/core/ilogtail_metric/ILogtaiMetric.cpp index f88f413b3b..7c2d4f72d3 100644 --- a/core/ilogtail_metric/ILogtaiMetric.cpp +++ b/core/ilogtail_metric/ILogtaiMetric.cpp @@ -1,5 +1,7 @@ #include "ILogtailMetric.h" +using namespace sls_logs; + namespace logtail { @@ -17,6 +19,10 @@ Metrics::Metrics(std::vector> labels) { mDeleted.store(false); } +Metrics::Metrics() { + mDeleted.store(false); +} + Metrics::~Metrics() { } @@ -47,9 +53,13 @@ uint64_t Counter::GetValue() { return mVal; } - uint64_t Counter::GetTimestamp() { +uint64_t Counter::GetTimestamp() { return mTimestamp; - } +} + +std::string Counter::GetName() { + return mName; +} Counter* Counter::CopyAndReset() { @@ -78,6 +88,10 @@ std::vector> Metrics::GetLabels() { return mLabels; } +std::vector Metrics::GetValues() { + return mValues; +} + Metrics* Metrics::Copy() { std::vector> newLabels; for (std::vector>::iterator it = mLabels.begin(); it != mLabels.end(); ++it) { @@ -116,47 +130,81 @@ Metrics* WriteMetrics::DoSnapshot() { // new read head Metrics* snapshot = NULL; // new read head iter - Metrics* rTmp ; + Metrics* rTmp; // new write head Metrics* wTmpHead = NULL; // new write head iter Metrics* wTmp = NULL; // old head iter - Metrics* tmp = mHead; + //Metrics* tmp = mHead; + Metrics* emptyHead = new Metrics(); + emptyHead->next = mHead; - while(tmp) { + Metrics* preTmp = emptyHead; + + while(preTmp) { + Metrics* tmp = preTmp->next; + if (!tmp) { + break; + } if (tmp->IsDeleted()) { Metrics* toDeleted = tmp; - tmp = tmp->next; + //tmp = tmp->next; + preTmp->next = tmp->next; + //preTmp = preTmp->next; delete toDeleted; continue; } Metrics* newMetrics = tmp->Copy(); // Get Head if (!snapshot) { - wTmpHead = tmp; - wTmp = wTmpHead; - tmp = tmp->next; + //wTmpHead = tmp; + //wTmp = wTmpHead; + preTmp = preTmp->next; snapshot = newMetrics; rTmp = snapshot; continue; } - wTmp->next = tmp; - tmp = tmp->next; + //wTmp->next = tmp; + preTmp = preTmp->next; rTmp->next = newMetrics; rTmp = newMetrics; } // Only lock when change head { std::lock_guard lock(mMutex); - mHead = wTmpHead; + mHead = emptyHead->next; + delete emptyHead; } return snapshot; } -void ReadMetrics::ReadAsLogGroup() { +void ReadMetrics::ReadAsLogGroup(sls_logs::LogGroup& logGroup) { + logGroup.set_category("metric-test"); + ReadLock lock(mReadWriteLock); - // Do some read + Metrics* tmp = mHead; + while(tmp) { + Log* logPtr = logGroup.add_logs(); + logPtr->set_time(time(NULL)); + std::vector> labels = tmp->GetLabels(); + for (std::vector>::iterator it = labels.begin(); it != labels.end(); ++it) { + std::pair pair = *it; + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key("label." + pair.first); + contentPtr->set_value(pair.second); + } + + std::vector values = tmp->GetValues(); + + for (std::vector::iterator it = values.begin(); it != values.end(); ++it) { + Counter* counter = *it; + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key("value." + counter->GetName()); + contentPtr->set_value(ToString(counter->GetValue())); + } + tmp = tmp->next; + } } void ReadMetrics::ReadAsPrometheus() { diff --git a/core/ilogtail_metric/ILogtailMetric.h b/core/ilogtail_metric/ILogtailMetric.h index f56d8301e6..4d18bbba01 100644 --- a/core/ilogtail_metric/ILogtailMetric.h +++ b/core/ilogtail_metric/ILogtailMetric.h @@ -6,11 +6,12 @@ #include "logger/Logger.h" #include #include "common/Lock.h" +#include "log_pb/sls_logs.pb.h" +#include "common/StringTools.h" namespace logtail { - class Counter{ private: std::string mName; @@ -22,6 +23,7 @@ class Counter{ ~Counter(); uint64_t GetValue(); uint64_t GetTimestamp(); + std::string GetName(); void Add(uint64_t val); void Set(uint64_t val); @@ -36,10 +38,12 @@ class Metrics { public: Metrics(std::vector > labels); + Metrics(); ~Metrics(); void MarkDeleted(); bool IsDeleted(); std::vector> GetLabels(); + std::vector GetValues(); Counter* CreateCounter(std::string Name); Metrics* Copy(); Metrics* next = NULL; @@ -70,7 +74,7 @@ class ReadMetrics { static ReadMetrics* ptr = new ReadMetrics(); return ptr; } - void ReadAsLogGroup(); + void ReadAsLogGroup(sls_logs::LogGroup& logGroup); void ReadAsPrometheus(); void UpdateMetrics(); Metrics* mHead = NULL; diff --git a/core/ilogtail_metric/MetricExportor.cpp b/core/ilogtail_metric/MetricExportor.cpp index 4d7347bbc7..051c9aeab0 100644 --- a/core/ilogtail_metric/MetricExportor.cpp +++ b/core/ilogtail_metric/MetricExportor.cpp @@ -15,9 +15,18 @@ MetricExportor::MetricExportor() { mLastSnapshotTime = 0; } -void MetricExportor::pushMetrics() { - LOG_INFO(sLogger, ("pushMetrics", "")); +void MetricExportor::PushMetrics() { + LOG_INFO(sLogger, ("pushMetrics", "")); + ReadMetrics::GetInstance()->UpdateMetrics(); + sls_logs::LogGroup logGroup; + ReadMetrics::GetInstance()->ReadAsLogGroup(logGroup); + for (int i = 0; i < logGroup.logs_size(); ++i) { + const sls_logs::Log& log = logGroup.logs(i); + for (int j = 0; j < log.contents_size(); ++j) { + const sls_logs::Log_Content& content = log.contents(j); + LOG_INFO(sLogger, ("contentKey", content.key())("contentValue", content.value())); + } + } } - } \ No newline at end of file diff --git a/core/ilogtail_metric/MetricExportor.h b/core/ilogtail_metric/MetricExportor.h index f3ede878f5..5ffa7c571a 100644 --- a/core/ilogtail_metric/MetricExportor.h +++ b/core/ilogtail_metric/MetricExportor.h @@ -21,7 +21,7 @@ class MetricExportor { static MetricExportor* ptr = new MetricExportor(); return ptr; } - void pushMetrics(); + void PushMetrics(); private: MetricExportor(); diff --git a/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp b/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp index bcf1480aae..20090faf27 100644 --- a/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp +++ b/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp @@ -5,6 +5,7 @@ #include #include "util.h" #include "ILogtailMetric.h" +#include "MetricExportor.h" #include "MetricConstants.h" namespace logtail { @@ -55,6 +56,7 @@ void ILogtailMetricUnittest::TestCreateMetric() { LOG_INFO(sLogger, ("ReadMetrics", head->GetLabels().size())); head = head->next; } + MetricExportor::GetInstance()->PushMetrics(); } }// namespace logtail From 0dabb6d5f34bce30aeca42652a429b6a9e32f8fa Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Mon, 24 Jul 2023 11:00:49 +0800 Subject: [PATCH 08/39] add exportor --- core/ilogtail_metric/ILogtaiMetric.cpp | 5 ++--- core/ilogtail_metric/ILogtailMetric.h | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/core/ilogtail_metric/ILogtaiMetric.cpp b/core/ilogtail_metric/ILogtaiMetric.cpp index 7c2d4f72d3..766a1faa2d 100644 --- a/core/ilogtail_metric/ILogtaiMetric.cpp +++ b/core/ilogtail_metric/ILogtaiMetric.cpp @@ -84,11 +84,11 @@ bool Metrics::IsDeleted() { return mDeleted; } -std::vector> Metrics::GetLabels() { +const std::vector>& Metrics::GetLabels() { return mLabels; } -std::vector Metrics::GetValues() { +const std::vector& Metrics::GetValues() { return mValues; } @@ -196,7 +196,6 @@ void ReadMetrics::ReadAsLogGroup(sls_logs::LogGroup& logGroup) { } std::vector values = tmp->GetValues(); - for (std::vector::iterator it = values.begin(); it != values.end(); ++it) { Counter* counter = *it; Log_Content* contentPtr = logPtr->add_contents(); diff --git a/core/ilogtail_metric/ILogtailMetric.h b/core/ilogtail_metric/ILogtailMetric.h index 4d18bbba01..e799695bf0 100644 --- a/core/ilogtail_metric/ILogtailMetric.h +++ b/core/ilogtail_metric/ILogtailMetric.h @@ -42,8 +42,8 @@ class Metrics { ~Metrics(); void MarkDeleted(); bool IsDeleted(); - std::vector> GetLabels(); - std::vector GetValues(); + const std::vector>& GetLabels(); + const std::vector& GetValues(); Counter* CreateCounter(std::string Name); Metrics* Copy(); Metrics* next = NULL; From c712ff2ea18ab14e9b64c029203d0bc51e9131d6 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Mon, 24 Jul 2023 07:21:57 +0000 Subject: [PATCH 09/39] refine code and test --- core/ilogtail_metric/ILogtaiMetric.cpp | 49 ++++-- core/ilogtail_metric/ILogtailMetric.h | 4 + .../ILogtailMetricUnittest.cpp | 141 ++++++++++++++++++ 3 files changed, 180 insertions(+), 14 deletions(-) diff --git a/core/ilogtail_metric/ILogtaiMetric.cpp b/core/ilogtail_metric/ILogtaiMetric.cpp index 766a1faa2d..63834392b2 100644 --- a/core/ilogtail_metric/ILogtaiMetric.cpp +++ b/core/ilogtail_metric/ILogtaiMetric.cpp @@ -27,6 +27,7 @@ Metrics::~Metrics() { } WriteMetrics::WriteMetrics() { + mSnapshotting.store(false); } WriteMetrics::~WriteMetrics() { @@ -107,13 +108,21 @@ Metrics* Metrics::Copy() { } Metrics* WriteMetrics::CreateMetrics(std::vector> labels) { - Metrics* cur = new Metrics(labels); + Metrics* cur = new Metrics(labels); + std::lock_guard lock(mMutex); // add metric to head - Metrics* oldHead = mHead; - { - std::lock_guard lock(mMutex); + // 根据mSnapshotting的状态，决定节点加到哪里 + if (mSnapshotting) { + Metrics* oldHead = mSnapshottingHead; + mSnapshottingHead = cur; + mSnapshottingHead->next = oldHead; + LOG_INFO(sLogger, ("CreateMetrics", "mSnapshottingHead")); + + } else { + Metrics* oldHead = mHead; mHead = cur; mHead->next = oldHead; + LOG_INFO(sLogger, ("CreateMetrics", "mHead")); } return cur; } @@ -127,21 +136,19 @@ void WriteMetrics::DestroyMetrics(Metrics* metrics) { Metrics* WriteMetrics::DoSnapshot() { + mSnapshotting.store(true); // new read head Metrics* snapshot = NULL; // new read head iter Metrics* rTmp; - // new write head - Metrics* wTmpHead = NULL; - // new write head iter - Metrics* wTmp = NULL; // old head iter //Metrics* tmp = mHead; Metrics* emptyHead = new Metrics(); - emptyHead->next = mHead; - + { + std::lock_guard lock(mMutex); + emptyHead->next = mHead; + } Metrics* preTmp = emptyHead; - while(preTmp) { Metrics* tmp = preTmp->next; if (!tmp) { @@ -158,14 +165,12 @@ Metrics* WriteMetrics::DoSnapshot() { Metrics* newMetrics = tmp->Copy(); // Get Head if (!snapshot) { - //wTmpHead = tmp; - //wTmp = wTmpHead; + preTmp = preTmp->next; snapshot = newMetrics; rTmp = snapshot; continue; } - //wTmp->next = tmp; preTmp = preTmp->next; rTmp->next = newMetrics; rTmp = newMetrics; @@ -173,7 +178,23 @@ Metrics* WriteMetrics::DoSnapshot() { // Only lock when change head { std::lock_guard lock(mMutex); + // 把临时的节点加到链表里 + if (mSnapshottingHead) { + Metrics* tmpSnapHead = mSnapshottingHead; + Metrics* tmpMHead = emptyHead->next; + emptyHead->next = mSnapshottingHead; + while(tmpSnapHead) { + if (tmpSnapHead->next == NULL) { + tmpSnapHead->next = tmpMHead; + break; + } else { + tmpSnapHead = tmpSnapHead->next; + } + } + mSnapshottingHead = NULL; + } mHead = emptyHead->next; + mSnapshotting.store(false); delete emptyHead; } return snapshot; diff --git a/core/ilogtail_metric/ILogtailMetric.h b/core/ilogtail_metric/ILogtailMetric.h index e799695bf0..0bec280be2 100644 --- a/core/ilogtail_metric/ILogtailMetric.h +++ b/core/ilogtail_metric/ILogtailMetric.h @@ -53,6 +53,7 @@ class WriteMetrics { private: WriteMetrics(); ~WriteMetrics(); + std::atomic_bool mSnapshotting; public: static WriteMetrics* GetInstance() { static WriteMetrics* ptr = new WriteMetrics(); @@ -62,7 +63,10 @@ class WriteMetrics { void DestroyMetrics(Metrics* metrics); Metrics* DoSnapshot(); Metrics* mHead = NULL; + Metrics* mSnapshottingHead = NULL; std::mutex mMutex; + std::mutex mSnapshottingMutex; + }; class ReadMetrics { diff --git a/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp b/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp index 20090faf27..49aa7a17fc 100644 --- a/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp +++ b/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "util.h" #include "ILogtailMetric.h" #include "MetricExportor.h" @@ -13,9 +14,14 @@ namespace logtail { class ILogtailMetricUnittest : public ::testing::Test { public: void TestCreateMetric(); + void TestCreateMetricMultiThread(); + void TestCreateAndDeleteMetricMultiThread(); }; APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateMetric, 0); +APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateMetricMultiThread, 0); +APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateAndDeleteMetricMultiThread, 0); + void ILogtailMetricUnittest::TestCreateMetric() { // create @@ -45,6 +51,10 @@ void ILogtailMetricUnittest::TestCreateMetric() { fileCounter2->Add((uint64_t)222); LOG_INFO(sLogger, ("value", fileCounter2->GetValue())("time", fileCounter2->GetTimestamp())); + Counter* fileCounter3 = fileMetric3->CreateCounter("filed2"); + fileCounter3->Add((uint64_t)222); + LOG_INFO(sLogger, ("value", fileCounter3->GetValue())("time", fileCounter3->GetTimestamp())); + WriteMetrics::GetInstance()->DestroyMetrics(fileMetric2); // delete first element @@ -59,6 +69,137 @@ void ILogtailMetricUnittest::TestCreateMetric() { MetricExportor::GetInstance()->PushMetrics(); } +void createMetrics(int count) { + for (int i = 0; i < count; i ++) { + std::vector> labels; + labels.push_back(std::make_pair("num", std::to_string(i))); + labels.push_back(std::make_pair("count", std::to_string(count))); + Metrics* fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); + Counter* fileCounter = fileMetric->CreateCounter("filed1"); + fileCounter->Add((uint64_t)111); + //LOG_INFO(sLogger, ("num", i )("count", count)); + } +} + +void createAndDeleteMetrics(int count) { + for (int i = 0; i < count; i ++) { + std::vector> labels; + labels.push_back(std::make_pair("num", std::to_string(i))); + labels.push_back(std::make_pair("count", std::to_string(count))); + Metrics* fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); + Counter* fileCounter = fileMetric->CreateCounter("filed1"); + fileCounter->Add((uint64_t)111); + //LOG_INFO(sLogger, ("num", i )("count", count)); + WriteMetrics::GetInstance()->DestroyMetrics(fileMetric); + } +} + +void UpdateMetrics() { + ReadMetrics::GetInstance()->UpdateMetrics(); + Metrics* head = ReadMetrics::GetInstance()->mHead; + while(head) { + std::vector> labels = head->GetLabels(); + for (std::vector>::iterator it = labels.begin(); it != labels.end(); ++it) { + std::pair pair = *it; + //LOG_INFO(sLogger, ("key", pair.first)("value", pair.second)); + } + head = head->next; + } +} + +void ILogtailMetricUnittest::TestCreateMetricMultiThread() { + + std::thread t1(createMetrics, 1); + std::thread t2(createMetrics, 2); + std::thread t3(createMetrics, 3); + std::thread t4(createMetrics, 4); + + t1.join(); + t2.join(); + t3.join(); + t4.join(); + + Metrics* head = WriteMetrics::GetInstance()->mHead; + int count = 0; + while(head) { + std::vector> labels = head->GetLabels(); + for (std::vector>::iterator it = labels.begin(); it != labels.end(); ++it) { + std::pair pair = *it; + //LOG_INFO(sLogger, ("key", pair.first)("value", pair.second)); + } + head = head->next; + count ++; + } + LOG_INFO(sLogger, ("Count", count)); +} + + + +void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { + + std::thread t1(createAndDeleteMetrics, 5); + std::thread t2(createAndDeleteMetrics, 6); + + std::thread tUpdate(UpdateMetrics); + + std::thread t3(createAndDeleteMetrics, 7); + std::thread t4(createAndDeleteMetrics, 8); + + t1.join(); + t2.join(); + tUpdate.join(); + t3.join(); + t4.join(); + + { + Metrics* head = WriteMetrics::GetInstance()->mHead; + int count = 0; + while(head) { + if (!head-> IsDeleted()) { + count ++; + } + head = head->next; + } + LOG_INFO(sLogger, ("WriteCount", count)); + + } + { + Metrics* head = ReadMetrics::GetInstance()->mHead; + int count = 0; + while(head) { + head = head->next; + count ++; + } + LOG_INFO(sLogger, ("ReadCount", count)); + + } + + UpdateMetrics(); + { + Metrics* head = WriteMetrics::GetInstance()->mHead; + int count = 0; + while(head) { + if (!head-> IsDeleted()) { + count ++; + } + head = head->next; + } + LOG_INFO(sLogger, ("FinalWriteCount", count)); + + } + { + Metrics* head = ReadMetrics::GetInstance()->mHead; + int count = 0; + while(head) { + head = head->next; + count ++; + } + LOG_INFO(sLogger, ("FinalReadCount", count)); + + } +} + + }// namespace logtail int main(int argc, char** argv) { From a9e5c2fa88fa86d08f43e506bd9956d71e51af81 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Mon, 24 Jul 2023 08:45:11 +0000 Subject: [PATCH 10/39] refine code --- core/ilogtail_metric/ILogtaiMetric.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/core/ilogtail_metric/ILogtaiMetric.cpp b/core/ilogtail_metric/ILogtaiMetric.cpp index 63834392b2..b3d2d588a8 100644 --- a/core/ilogtail_metric/ILogtaiMetric.cpp +++ b/core/ilogtail_metric/ILogtaiMetric.cpp @@ -141,8 +141,7 @@ Metrics* WriteMetrics::DoSnapshot() { Metrics* snapshot = NULL; // new read head iter Metrics* rTmp; - // old head iter - //Metrics* tmp = mHead; + Metrics* emptyHead = new Metrics(); { std::lock_guard lock(mMutex); @@ -156,9 +155,7 @@ Metrics* WriteMetrics::DoSnapshot() { } if (tmp->IsDeleted()) { Metrics* toDeleted = tmp; - //tmp = tmp->next; preTmp->next = tmp->next; - //preTmp = preTmp->next; delete toDeleted; continue; } @@ -202,7 +199,6 @@ Metrics* WriteMetrics::DoSnapshot() { void ReadMetrics::ReadAsLogGroup(sls_logs::LogGroup& logGroup) { logGroup.set_category("metric-test"); - ReadLock lock(mReadWriteLock); Metrics* tmp = mHead; while(tmp) { @@ -234,10 +230,11 @@ void ReadMetrics::ReadAsPrometheus() { void ReadMetrics::UpdateMetrics() { Metrics* snapshot = mWriteMetrics->DoSnapshot(); - Metrics* toDelete = mHead; + Metrics* toDelete; { // Only lock when change head WriteLock lock(mReadWriteLock); + toDelete = mHead; mHead = snapshot; } // delete old linklist From a9fd2563354bd976b65ba93003dae0a9f67e46d2 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Mon, 31 Jul 2023 15:22:27 +0800 Subject: [PATCH 11/39] fix comment --- core/CMakeLists.txt | 6 ++-- core/aggregator/Aggregator.cpp | 2 +- core/aggregator/CMakeLists.txt | 2 +- core/app_config/AppConfigBase.cpp | 4 +-- core/app_config/CMakeLists.txt | 2 +- core/checkpoint/CMakeLists.txt | 2 +- core/checkpoint/CheckPointManager.cpp | 2 +- core/checkpoint/CheckpointManagerV2.cpp | 2 +- core/common/LogFileCollectOffsetIndicator.cpp | 2 +- core/common/LogGroupContext.h | 2 +- core/common/LogstoreSenderQueue.cpp | 2 +- core/config_manager/CMakeLists.txt | 2 +- core/config_manager/ConfigManager.cpp | 8 ++--- core/config_manager/ConfigManagerBase.cpp | 8 ++--- core/controller/CMakeLists.txt | 2 +- core/controller/EventDispatcher.cpp | 8 ++--- core/controller/EventDispatcher.h | 2 +- core/controller/EventDispatcherBase.cpp | 8 ++--- core/controller/EventDispatcherBase.h | 2 +- core/event_handler/CMakeLists.txt | 2 +- core/event_handler/EventHandler.cpp | 2 +- core/event_handler/LogInput.cpp | 2 +- core/event_listener/CMakeLists.txt | 2 +- core/event_listener/EventListener_Linux.cpp | 2 +- core/ilogtail_metric/CMakeLists.txt | 30 ------------------- core/logtail.cpp | 8 ++--- core/logtail_windows.cpp | 8 ++--- core/monitor/CMakeLists.txt | 1 - .../{profiler => monitor}/LogFileProfiler.cpp | 0 core/{profiler => monitor}/LogFileProfiler.h | 0 core/{profiler => monitor}/LogIntegrity.cpp | 0 core/{profiler => monitor}/LogIntegrity.h | 0 core/{profiler => monitor}/LogLineCount.cpp | 0 core/{profiler => monitor}/LogLineCount.h | 0 .../LogtaiMetric.cpp} | 4 +-- core/{profiler => monitor}/LogtailAlarm.cpp | 0 core/{profiler => monitor}/LogtailAlarm.h | 0 .../LogtailMetric.h} | 1 + .../MetricConstants.h | 0 .../MetricExportor.cpp | 1 + .../MetricExportor.h | 2 +- core/monitor/Monitor.cpp | 4 +-- core/observer/CMakeLists.txt | 2 +- core/observer/interface/statistics.h | 2 +- core/observer/metas/ContainerProcessGroup.cpp | 2 +- core/parser/CMakeLists.txt | 2 +- core/parser/LogParser.cpp | 4 +-- core/plugin/CMakeLists.txt | 2 +- core/plugin/LogtailPlugin.cpp | 4 +-- core/polling/CMakeLists.txt | 2 +- core/polling/PollingDirFile.cpp | 2 +- core/polling/PollingModify.cpp | 2 +- core/processor/CMakeLists.txt | 2 +- core/processor/LogFilter.cpp | 2 +- core/processor/LogProcess.cpp | 8 ++--- core/processor/RegexFilterValueNode.h | 2 +- core/profiler/CMakeLists.txt | 26 ---------------- core/reader/CMakeLists.txt | 2 +- core/reader/DelimiterLogFileReader.cpp | 2 +- core/reader/JsonLogFileReader.cpp | 2 +- core/reader/LogFileReader.cpp | 4 +-- core/sender/CMakeLists.txt | 2 +- core/sender/Sender.cpp | 8 ++--- core/sls_control/SLSControl.cpp | 2 +- core/unittest/CMakeLists.txt | 3 +- .../CMakeLists.txt | 6 ++-- .../LogtailMetricUnittest.cpp} | 2 +- core/unittest/polling/PollingUnittest.cpp | 2 +- core/unittest/sender/SenderUnittest.cpp | 4 +-- 69 files changed, 91 insertions(+), 149 deletions(-) delete mode 100644 core/ilogtail_metric/CMakeLists.txt rename core/{profiler => monitor}/LogFileProfiler.cpp (100%) rename core/{profiler => monitor}/LogFileProfiler.h (100%) rename core/{profiler => monitor}/LogIntegrity.cpp (100%) rename core/{profiler => monitor}/LogIntegrity.h (100%) rename core/{profiler => monitor}/LogLineCount.cpp (100%) rename core/{profiler => monitor}/LogLineCount.h (100%) rename core/{ilogtail_metric/ILogtaiMetric.cpp => monitor/LogtaiMetric.cpp} (98%) rename core/{profiler => monitor}/LogtailAlarm.cpp (100%) rename core/{profiler => monitor}/LogtailAlarm.h (100%) rename core/{ilogtail_metric/ILogtailMetric.h => monitor/LogtailMetric.h} (97%) rename core/{ilogtail_metric => monitor}/MetricConstants.h (100%) rename core/{ilogtail_metric => monitor}/MetricExportor.cpp (95%) rename core/{ilogtail_metric => monitor}/MetricExportor.h (96%) delete mode 100644 core/profiler/CMakeLists.txt rename core/unittest/{ilogtail_metric => monitor}/CMakeLists.txt (79%) rename core/unittest/{ilogtail_metric/ILogtailMetricUnittest.cpp => monitor/LogtailMetricUnittest.cpp} (99%) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 7d0d04b352..c07b1652bf 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -81,8 +81,8 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/dependencies.cmake) # Subdirectories (modules). set(SUB_DIRECTORIES_LIST aggregator app_config checkpoint common config config_manager config_server_pb - controller event event_handler event_listener helper log_pb logger monitor ilogtail_metric - parser plugin polling processor profiler reader profile_sender sender shennong sdk + controller event event_handler event_listener helper log_pb logger monitor + parser plugin polling processor reader profile_sender sender shennong sdk fuse sls_control ) if (UNIX) @@ -119,9 +119,7 @@ target_link_libraries(${LOGTAIL_TARGET} controller plugin monitor - ilogtail_metric sender - profiler profile_sender app_config sls_control diff --git a/core/aggregator/Aggregator.cpp b/core/aggregator/Aggregator.cpp index bb87e23d10..d7c896062f 100644 --- a/core/aggregator/Aggregator.cpp +++ b/core/aggregator/Aggregator.cpp @@ -13,7 +13,7 @@ // limitations under the License. #include -#include +#include #include #include #include diff --git a/core/aggregator/CMakeLists.txt b/core/aggregator/CMakeLists.txt index 07e5bae272..05195a326c 100644 --- a/core/aggregator/CMakeLists.txt +++ b/core/aggregator/CMakeLists.txt @@ -23,4 +23,4 @@ target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} config_manager) target_link_libraries(${PROJECT_NAME} sender) target_link_libraries(${PROJECT_NAME} shennong) -target_link_libraries(${PROJECT_NAME} profiler) \ No newline at end of file +target_link_libraries(${PROJECT_NAME} monitor) \ No newline at end of file diff --git a/core/app_config/AppConfigBase.cpp b/core/app_config/AppConfigBase.cpp index c103f26bef..624f66d42b 100644 --- a/core/app_config/AppConfigBase.cpp +++ b/core/app_config/AppConfigBase.cpp @@ -19,8 +19,8 @@ #endif #include "shennong/MetricSender.h" #include "sender/Sender.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogtailAlarm.h" #include "monitor/Monitor.h" #include "common/util.h" #include "common/LogtailCommonFlags.h" diff --git a/core/app_config/CMakeLists.txt b/core/app_config/CMakeLists.txt index 55428b7f13..1f3b68ffe4 100644 --- a/core/app_config/CMakeLists.txt +++ b/core/app_config/CMakeLists.txt @@ -23,4 +23,4 @@ target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} config_manager) target_link_libraries(${PROJECT_NAME} sender) target_link_libraries(${PROJECT_NAME} shennong) -target_link_libraries(${PROJECT_NAME} profiler) \ No newline at end of file +target_link_libraries(${PROJECT_NAME} monitor) \ No newline at end of file diff --git a/core/checkpoint/CMakeLists.txt b/core/checkpoint/CMakeLists.txt index b72b525a07..7b2782f292 100644 --- a/core/checkpoint/CMakeLists.txt +++ b/core/checkpoint/CMakeLists.txt @@ -20,7 +20,7 @@ append_source_files(LIB_SOURCE_FILES) add_library(${PROJECT_NAME} STATIC ${LIB_SOURCE_FILES}) target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} logger) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} config_manager) target_link_libraries(${PROJECT_NAME} app_config) link_leveldb(${PROJECT_NAME}) \ No newline at end of file diff --git a/core/checkpoint/CheckPointManager.cpp b/core/checkpoint/CheckPointManager.cpp index 54482a4683..f8c7e9936f 100644 --- a/core/checkpoint/CheckPointManager.cpp +++ b/core/checkpoint/CheckPointManager.cpp @@ -17,7 +17,7 @@ #include #include #include -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "app_config/AppConfig.h" #include "config_manager/ConfigManager.h" #include "common/Flags.h" diff --git a/core/checkpoint/CheckpointManagerV2.cpp b/core/checkpoint/CheckpointManagerV2.cpp index 58d65a7a92..56b38bbc69 100644 --- a/core/checkpoint/CheckpointManagerV2.cpp +++ b/core/checkpoint/CheckpointManagerV2.cpp @@ -19,7 +19,7 @@ #include "common/ScopeInvoker.h" #include "common/TimeUtil.h" #include "logger/Logger.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "app_config/AppConfig.h" #include "config/Config.h" #include "config_manager/ConfigManager.h" diff --git a/core/common/LogFileCollectOffsetIndicator.cpp b/core/common/LogFileCollectOffsetIndicator.cpp index 16ef620bd4..f3b32e18d3 100644 --- a/core/common/LogFileCollectOffsetIndicator.cpp +++ b/core/common/LogFileCollectOffsetIndicator.cpp @@ -15,7 +15,7 @@ #include "LogFileCollectOffsetIndicator.h" #include #include -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "common/Flags.h" #include "common/LogstoreSenderQueue.h" #include "sender/Sender.h" diff --git a/core/common/LogGroupContext.h b/core/common/LogGroupContext.h index c071d68b0a..7abffd9387 100644 --- a/core/common/LogGroupContext.h +++ b/core/common/LogGroupContext.h @@ -16,7 +16,7 @@ #pragma once #include "config/IntegrityConfig.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "FileInfo.h" #include "RangeCheckpoint.h" diff --git a/core/common/LogstoreSenderQueue.cpp b/core/common/LogstoreSenderQueue.cpp index 81c9e96779..73e5d8c3d1 100644 --- a/core/common/LogstoreSenderQueue.cpp +++ b/core/common/LogstoreSenderQueue.cpp @@ -15,7 +15,7 @@ #include "LogstoreSenderQueue.h" #include "common/Flags.h" #include "common/StringTools.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" DEFINE_FLAG_INT32(max_client_send_error_count, "set disabled if last send is all failed", 60); diff --git a/core/config_manager/CMakeLists.txt b/core/config_manager/CMakeLists.txt index 0664992881..81f42db89e 100644 --- a/core/config_manager/CMakeLists.txt +++ b/core/config_manager/CMakeLists.txt @@ -29,7 +29,7 @@ target_link_libraries(${PROJECT_NAME} config_server_pb) target_link_libraries(${PROJECT_NAME} controller) target_link_libraries(${PROJECT_NAME} event_handler) target_link_libraries(${PROJECT_NAME} sender) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} checkpoint) target_link_libraries(${PROJECT_NAME} app_config) target_link_libraries(${PROJECT_NAME} processor) diff --git a/core/config_manager/ConfigManager.cpp b/core/config_manager/ConfigManager.cpp index a5cccb220e..83ddb913af 100644 --- a/core/config_manager/ConfigManager.cpp +++ b/core/config_manager/ConfigManager.cpp @@ -45,10 +45,10 @@ #include "common/GlobalPara.h" #include "common/version.h" #include "config/UserLogConfigParser.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogIntegrity.h" -#include "profiler/LogLineCount.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogIntegrity.h" +#include "monitor/LogLineCount.h" #include "sdk/Common.h" #include "sdk/CurlImp.h" #include "sdk/Exception.h" diff --git a/core/config_manager/ConfigManagerBase.cpp b/core/config_manager/ConfigManagerBase.cpp index 22f0d1448b..41d1dad14e 100644 --- a/core/config_manager/ConfigManagerBase.cpp +++ b/core/config_manager/ConfigManagerBase.cpp @@ -43,10 +43,10 @@ #include "common/GlobalPara.h" #include "common/version.h" #include "config/UserLogConfigParser.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogIntegrity.h" -#include "profiler/LogLineCount.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogIntegrity.h" +#include "monitor/LogLineCount.h" #include "app_config/AppConfig.h" #include "config_manager/ConfigYamlToJson.h" #include "checkpoint/CheckPointManager.h" diff --git a/core/controller/CMakeLists.txt b/core/controller/CMakeLists.txt index 9dddfdbdb4..243af2c031 100644 --- a/core/controller/CMakeLists.txt +++ b/core/controller/CMakeLists.txt @@ -28,7 +28,7 @@ target_link_libraries(${PROJECT_NAME} event) target_link_libraries(${PROJECT_NAME} event_handler) target_link_libraries(${PROJECT_NAME} event_listener) target_link_libraries(${PROJECT_NAME} sender) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} app_config) target_link_libraries(${PROJECT_NAME} log_pb) target_link_libraries(${PROJECT_NAME} shennong) diff --git a/core/controller/EventDispatcher.cpp b/core/controller/EventDispatcher.cpp index eb322dc777..d2c6e8ee99 100644 --- a/core/controller/EventDispatcher.cpp +++ b/core/controller/EventDispatcher.cpp @@ -50,10 +50,10 @@ #include "event/Event.h" #include "processor/LogProcess.h" #include "sender/Sender.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogIntegrity.h" -#include "profiler/LogLineCount.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogIntegrity.h" +#include "monitor/LogLineCount.h" #include "log_pb/metric.pb.h" #include "log_pb/sls_logs.pb.h" #include "checkpoint/CheckPointManager.h" diff --git a/core/controller/EventDispatcher.h b/core/controller/EventDispatcher.h index 8a3e9a680c..4ff662583c 100644 --- a/core/controller/EventDispatcher.h +++ b/core/controller/EventDispatcher.h @@ -29,7 +29,7 @@ #include #include #include -#include "profiler/LogFileProfiler.h" +#include "monitor/LogFileProfiler.h" #include "polling/PollingModify.h" #include "polling/PollingDirFile.h" #include "event_listener/EventListener.h" diff --git a/core/controller/EventDispatcherBase.cpp b/core/controller/EventDispatcherBase.cpp index 80ba10a480..57380629fe 100644 --- a/core/controller/EventDispatcherBase.cpp +++ b/core/controller/EventDispatcherBase.cpp @@ -51,10 +51,10 @@ #include "event/Event.h" #include "processor/LogProcess.h" #include "sender/Sender.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogIntegrity.h" -#include "profiler/LogLineCount.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogIntegrity.h" +#include "monitor/LogLineCount.h" #include "log_pb/metric.pb.h" #include "log_pb/sls_logs.pb.h" #include "checkpoint/CheckPointManager.h" diff --git a/core/controller/EventDispatcherBase.h b/core/controller/EventDispatcherBase.h index c737861391..1bcaabc691 100644 --- a/core/controller/EventDispatcherBase.h +++ b/core/controller/EventDispatcherBase.h @@ -29,7 +29,7 @@ #include #include #include -#include "profiler/LogFileProfiler.h" +#include "monitor/LogFileProfiler.h" #include "polling/PollingModify.h" #include "polling/PollingDirFile.h" #include "event_listener/EventListener.h" diff --git a/core/event_handler/CMakeLists.txt b/core/event_handler/CMakeLists.txt index cdbdc0dba4..19722fa0c8 100644 --- a/core/event_handler/CMakeLists.txt +++ b/core/event_handler/CMakeLists.txt @@ -25,6 +25,6 @@ target_link_libraries(${PROJECT_NAME} app_config) target_link_libraries(${PROJECT_NAME} event) target_link_libraries(${PROJECT_NAME} controller) target_link_libraries(${PROJECT_NAME} config_manager) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} processor) target_link_libraries(${PROJECT_NAME} fuse) diff --git a/core/event_handler/EventHandler.cpp b/core/event_handler/EventHandler.cpp index 95dc2e1e30..4e42e771e3 100644 --- a/core/event_handler/EventHandler.cpp +++ b/core/event_handler/EventHandler.cpp @@ -25,7 +25,7 @@ #include "event/BlockEventManager.h" #include "controller/EventDispatcher.h" #include "config_manager/ConfigManager.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "processor/LogProcess.h" #include "logger/Logger.h" #include "fuse/FuseFileBlacklist.h" diff --git a/core/event_handler/LogInput.cpp b/core/event_handler/LogInput.cpp index f4e2945b9a..1b6e62d1e7 100644 --- a/core/event_handler/LogInput.cpp +++ b/core/event_handler/LogInput.cpp @@ -24,7 +24,7 @@ #include "monitor/Monitor.h" #include "processor/LogProcess.h" #include "controller/EventDispatcher.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "checkpoint/CheckPointManager.h" #include "polling/PollingDirFile.h" #include "polling/PollingModify.h" diff --git a/core/event_listener/CMakeLists.txt b/core/event_listener/CMakeLists.txt index 5b95fd9dca..58a0f5c802 100644 --- a/core/event_listener/CMakeLists.txt +++ b/core/event_listener/CMakeLists.txt @@ -30,5 +30,5 @@ target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} logger) target_link_libraries(${PROJECT_NAME} polling) target_link_libraries(${PROJECT_NAME} event) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} controller) \ No newline at end of file diff --git a/core/event_listener/EventListener_Linux.cpp b/core/event_listener/EventListener_Linux.cpp index 6dfe176802..005824d9f1 100644 --- a/core/event_listener/EventListener_Linux.cpp +++ b/core/event_listener/EventListener_Linux.cpp @@ -17,7 +17,7 @@ #include #include #include "logger/Logger.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "common/ErrorUtil.h" #include "common/Flags.h" #include "controller/EventDispatcher.h" diff --git a/core/ilogtail_metric/CMakeLists.txt b/core/ilogtail_metric/CMakeLists.txt deleted file mode 100644 index 225313a0cb..0000000000 --- a/core/ilogtail_metric/CMakeLists.txt +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2022 iLogtail Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cmake_minimum_required(VERSION 2.9) -project(ilogtail_metric) - -if (MSVC) - add_definitions(-DTARGETLIBS=Psapi.lib) - add_definitions(-DPSAPI_VERSION=1) -endif () - -file(GLOB LIB_SOURCE_FILES *.cpp *.h) -append_source_files(LIB_SOURCE_FILES) -add_library(${PROJECT_NAME} STATIC ${LIB_SOURCE_FILES}) -target_link_libraries(${PROJECT_NAME} log_pb) - -if (MSVC) - target_link_libraries(${PROJECT_NAME} "Psapi.lib") -endif () \ No newline at end of file diff --git a/core/logtail.cpp b/core/logtail.cpp index caa174b6f6..0c7d623037 100644 --- a/core/logtail.cpp +++ b/core/logtail.cpp @@ -40,10 +40,10 @@ #include "controller/EventDispatcher.h" #include "monitor/Monitor.h" #include "sender/Sender.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogIntegrity.h" -#include "profiler/LogLineCount.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogIntegrity.h" +#include "monitor/LogLineCount.h" #include "app_config/AppConfig.h" #include "ObserverManager.h" using namespace logtail; diff --git a/core/logtail_windows.cpp b/core/logtail_windows.cpp index f9a8a3c785..a3631aa150 100644 --- a/core/logtail_windows.cpp +++ b/core/logtail_windows.cpp @@ -38,10 +38,10 @@ #include "controller/EventDispatcher.h" #include "monitor/Monitor.h" #include "sender/Sender.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogIntegrity.h" -#include "profiler/LogLineCount.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogIntegrity.h" +#include "monitor/LogLineCount.h" #include "app_config/AppConfig.h" using namespace logtail; diff --git a/core/monitor/CMakeLists.txt b/core/monitor/CMakeLists.txt index 5170cea98e..f24a185f85 100644 --- a/core/monitor/CMakeLists.txt +++ b/core/monitor/CMakeLists.txt @@ -27,7 +27,6 @@ target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} logger) target_link_libraries(${PROJECT_NAME} log_pb) target_link_libraries(${PROJECT_NAME} sender) -target_link_libraries(${PROJECT_NAME} profiler) target_link_libraries(${PROJECT_NAME} config_manager) target_link_libraries(${PROJECT_NAME} app_config) target_link_libraries(${PROJECT_NAME} event_handler) diff --git a/core/profiler/LogFileProfiler.cpp b/core/monitor/LogFileProfiler.cpp similarity index 100% rename from core/profiler/LogFileProfiler.cpp rename to core/monitor/LogFileProfiler.cpp diff --git a/core/profiler/LogFileProfiler.h b/core/monitor/LogFileProfiler.h similarity index 100% rename from core/profiler/LogFileProfiler.h rename to core/monitor/LogFileProfiler.h diff --git a/core/profiler/LogIntegrity.cpp b/core/monitor/LogIntegrity.cpp similarity index 100% rename from core/profiler/LogIntegrity.cpp rename to core/monitor/LogIntegrity.cpp diff --git a/core/profiler/LogIntegrity.h b/core/monitor/LogIntegrity.h similarity index 100% rename from core/profiler/LogIntegrity.h rename to core/monitor/LogIntegrity.h diff --git a/core/profiler/LogLineCount.cpp b/core/monitor/LogLineCount.cpp similarity index 100% rename from core/profiler/LogLineCount.cpp rename to core/monitor/LogLineCount.cpp diff --git a/core/profiler/LogLineCount.h b/core/monitor/LogLineCount.h similarity index 100% rename from core/profiler/LogLineCount.h rename to core/monitor/LogLineCount.h diff --git a/core/ilogtail_metric/ILogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp similarity index 98% rename from core/ilogtail_metric/ILogtaiMetric.cpp rename to core/monitor/LogtaiMetric.cpp index b3d2d588a8..cdf4314ce4 100644 --- a/core/ilogtail_metric/ILogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -1,4 +1,4 @@ -#include "ILogtailMetric.h" +#include "LogtailMetric.h" using namespace sls_logs; @@ -198,7 +198,6 @@ Metrics* WriteMetrics::DoSnapshot() { } void ReadMetrics::ReadAsLogGroup(sls_logs::LogGroup& logGroup) { - logGroup.set_category("metric-test"); ReadLock lock(mReadWriteLock); Metrics* tmp = mHead; while(tmp) { @@ -223,6 +222,7 @@ void ReadMetrics::ReadAsLogGroup(sls_logs::LogGroup& logGroup) { } } + void ReadMetrics::ReadAsPrometheus() { ReadLock lock(mReadWriteLock); // Do some read diff --git a/core/profiler/LogtailAlarm.cpp b/core/monitor/LogtailAlarm.cpp similarity index 100% rename from core/profiler/LogtailAlarm.cpp rename to core/monitor/LogtailAlarm.cpp diff --git a/core/profiler/LogtailAlarm.h b/core/monitor/LogtailAlarm.h similarity index 100% rename from core/profiler/LogtailAlarm.h rename to core/monitor/LogtailAlarm.h diff --git a/core/ilogtail_metric/ILogtailMetric.h b/core/monitor/LogtailMetric.h similarity index 97% rename from core/ilogtail_metric/ILogtailMetric.h rename to core/monitor/LogtailMetric.h index 0bec280be2..3f6a5d7add 100644 --- a/core/ilogtail_metric/ILogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -79,6 +79,7 @@ class ReadMetrics { return ptr; } void ReadAsLogGroup(sls_logs::LogGroup& logGroup); + void ReadAsMap(std::map map); void ReadAsPrometheus(); void UpdateMetrics(); Metrics* mHead = NULL; diff --git a/core/ilogtail_metric/MetricConstants.h b/core/monitor/MetricConstants.h similarity index 100% rename from core/ilogtail_metric/MetricConstants.h rename to core/monitor/MetricConstants.h diff --git a/core/ilogtail_metric/MetricExportor.cpp b/core/monitor/MetricExportor.cpp similarity index 95% rename from core/ilogtail_metric/MetricExportor.cpp rename to core/monitor/MetricExportor.cpp index 051c9aeab0..46e6f32d42 100644 --- a/core/ilogtail_metric/MetricExportor.cpp +++ b/core/monitor/MetricExportor.cpp @@ -19,6 +19,7 @@ void MetricExportor::PushMetrics() { LOG_INFO(sLogger, ("pushMetrics", "")); ReadMetrics::GetInstance()->UpdateMetrics(); sls_logs::LogGroup logGroup; + logGroup.set_category("metric-test"); ReadMetrics::GetInstance()->ReadAsLogGroup(logGroup); for (int i = 0; i < logGroup.logs_size(); ++i) { const sls_logs::Log& log = logGroup.logs(i); diff --git a/core/ilogtail_metric/MetricExportor.h b/core/monitor/MetricExportor.h similarity index 96% rename from core/ilogtail_metric/MetricExportor.h rename to core/monitor/MetricExportor.h index 5ffa7c571a..62d254c8e2 100644 --- a/core/ilogtail_metric/MetricExportor.h +++ b/core/monitor/MetricExportor.h @@ -7,7 +7,7 @@ #include #include #include "profile_sender/ProfileSender.h" -#include "ILogtailMetric.h" +#include "LogtailMetric.h" #include "logger/Logger.h" #include "plugin/LogtailPlugin.h" #include "ProfileSender.h" diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index 25bc57dd53..1b939b1bcc 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -34,8 +34,8 @@ #include "log_pb/sls_logs.pb.h" #include "logger/Logger.h" #include "sender/Sender.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogtailAlarm.h" #include "config_manager/ConfigManager.h" #include "app_config/AppConfig.h" #include "event_handler/LogInput.h" diff --git a/core/observer/CMakeLists.txt b/core/observer/CMakeLists.txt index 71ed8b47f8..4f53e1e134 100644 --- a/core/observer/CMakeLists.txt +++ b/core/observer/CMakeLists.txt @@ -9,7 +9,7 @@ target_link_libraries(${PROJECT_NAME} logger) target_link_libraries(${PROJECT_NAME} config) target_link_libraries(${PROJECT_NAME} config_manager) target_link_libraries(${PROJECT_NAME} monitor) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} event) target_link_libraries(${PROJECT_NAME} log_pb) diff --git a/core/observer/interface/statistics.h b/core/observer/interface/statistics.h index ea623e1f84..9c15efa252 100644 --- a/core/observer/interface/statistics.h +++ b/core/observer/interface/statistics.h @@ -18,7 +18,7 @@ #include "Monitor.h" #include "iostream" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include #include diff --git a/core/observer/metas/ContainerProcessGroup.cpp b/core/observer/metas/ContainerProcessGroup.cpp index 4e8bc3abaa..c72a0301ec 100644 --- a/core/observer/metas/ContainerProcessGroup.cpp +++ b/core/observer/metas/ContainerProcessGroup.cpp @@ -16,7 +16,7 @@ #include "common/TimeUtil.h" #include "logger/Logger.h" #include "CGroupPathResolver.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "FileSystemUtil.h" #include "StringTools.h" #include "plugin/LogtailPlugin.h" diff --git a/core/parser/CMakeLists.txt b/core/parser/CMakeLists.txt index 035da778a1..f8751f4550 100644 --- a/core/parser/CMakeLists.txt +++ b/core/parser/CMakeLists.txt @@ -21,5 +21,5 @@ add_library(${PROJECT_NAME} STATIC ${LIB_SOURCE_FILES}) target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} logger) target_link_libraries(${PROJECT_NAME} log_pb) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} app_config) \ No newline at end of file diff --git a/core/parser/LogParser.cpp b/core/parser/LogParser.cpp index c5eeaf9a4c..271c189a4e 100644 --- a/core/parser/LogParser.cpp +++ b/core/parser/LogParser.cpp @@ -24,8 +24,8 @@ #include "log_pb/sls_logs.pb.h" #include "logger/Logger.h" #include "config/LogType.h" -#include "profiler/LogFileProfiler.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" +#include "monitor/LogtailAlarm.h" #include "app_config/AppConfig.h" #include "config_manager/ConfigManager.h" diff --git a/core/plugin/CMakeLists.txt b/core/plugin/CMakeLists.txt index 54dc0e7c10..49e60934b3 100644 --- a/core/plugin/CMakeLists.txt +++ b/core/plugin/CMakeLists.txt @@ -24,7 +24,7 @@ target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} logger) target_link_libraries(${PROJECT_NAME} config_manager) target_link_libraries(${PROJECT_NAME} sender) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} app_config) target_link_libraries(${PROJECT_NAME} config) if (UNIX) diff --git a/core/plugin/LogtailPlugin.cpp b/core/plugin/LogtailPlugin.cpp index 5e7dc4b795..cc65ae5090 100644 --- a/core/plugin/LogtailPlugin.cpp +++ b/core/plugin/LogtailPlugin.cpp @@ -19,8 +19,8 @@ #include "logger/Logger.h" #include "config_manager/ConfigManager.h" #include "sender/Sender.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogFileProfiler.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" #include "app_config/AppConfig.h" #include "common/DynamicLibHelper.h" #include "common/LogtailCommonFlags.h" diff --git a/core/polling/CMakeLists.txt b/core/polling/CMakeLists.txt index 4411e7d0fc..f44d2df2e9 100644 --- a/core/polling/CMakeLists.txt +++ b/core/polling/CMakeLists.txt @@ -22,5 +22,5 @@ target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} logger) target_link_libraries(${PROJECT_NAME} config_manager) target_link_libraries(${PROJECT_NAME} monitor) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} event) \ No newline at end of file diff --git a/core/polling/PollingDirFile.cpp b/core/polling/PollingDirFile.cpp index 37edf266fd..0a01dec37d 100644 --- a/core/polling/PollingDirFile.cpp +++ b/core/polling/PollingDirFile.cpp @@ -28,7 +28,7 @@ #include "event/Event.h" #include "logger/Logger.h" #include "config_manager/ConfigManager.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "monitor/Monitor.h" #include "PollingModify.h" #include "PollingEventQueue.h" diff --git a/core/polling/PollingModify.cpp b/core/polling/PollingModify.cpp index 2e864f7a6b..1be1851638 100644 --- a/core/polling/PollingModify.cpp +++ b/core/polling/PollingModify.cpp @@ -24,7 +24,7 @@ #include "common/FileSystemUtil.h" #include "event/Event.h" #include "logger/Logger.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "monitor/Monitor.h" using namespace std; diff --git a/core/processor/CMakeLists.txt b/core/processor/CMakeLists.txt index a4ba6e1fed..f20ed34d24 100644 --- a/core/processor/CMakeLists.txt +++ b/core/processor/CMakeLists.txt @@ -26,7 +26,7 @@ target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} logger) target_link_libraries(${PROJECT_NAME} parser) target_link_libraries(${PROJECT_NAME} log_pb) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} app_config) target_link_libraries(${PROJECT_NAME} sdk) target_link_libraries(${PROJECT_NAME} sender) diff --git a/core/processor/LogFilter.cpp b/core/processor/LogFilter.cpp index 4c3302a4b6..64724009aa 100644 --- a/core/processor/LogFilter.cpp +++ b/core/processor/LogFilter.cpp @@ -14,7 +14,7 @@ #include "LogFilter.h" #include -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "app_config/AppConfig.h" #include "common/util.h" #include "sdk/Common.h" diff --git a/core/processor/LogProcess.cpp b/core/processor/LogProcess.cpp index 3eb939f6a7..6b5dacf253 100644 --- a/core/processor/LogProcess.cpp +++ b/core/processor/LogProcess.cpp @@ -32,12 +32,12 @@ #include "sender/Sender.h" #include "log_pb/sls_logs.pb.h" #include "common/StringTools.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogIntegrity.h" -#include "profiler/LogLineCount.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogIntegrity.h" +#include "monitor/LogLineCount.h" #include "config/IntegrityConfig.h" #include "app_config/AppConfig.h" -#include "profiler/LogFileProfiler.h" +#include "monitor/LogFileProfiler.h" #include "config_manager/ConfigManager.h" #include "logger/Logger.h" #include "aggregator/Aggregator.h" diff --git a/core/processor/RegexFilterValueNode.h b/core/processor/RegexFilterValueNode.h index c9addfb52c..d56cd67478 100644 --- a/core/processor/RegexFilterValueNode.h +++ b/core/processor/RegexFilterValueNode.h @@ -21,7 +21,7 @@ #include "BaseFilterNode.h" #include "common/util.h" #include "app_config/AppConfig.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "logger/Logger.h" namespace logtail { diff --git a/core/profiler/CMakeLists.txt b/core/profiler/CMakeLists.txt deleted file mode 100644 index 1f95152065..0000000000 --- a/core/profiler/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2022 iLogtail Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cmake_minimum_required(VERSION 2.9) -project(profiler) - -file(GLOB LIB_SOURCE_FILES *.cpp *.h) -append_source_files(LIB_SOURCE_FILES) -add_library(${PROJECT_NAME} STATIC ${LIB_SOURCE_FILES}) -target_link_libraries(${PROJECT_NAME} common) -target_link_libraries(${PROJECT_NAME} log_pb) -target_link_libraries(${PROJECT_NAME} sender) -target_link_libraries(${PROJECT_NAME} config_manager) -target_link_libraries(${PROJECT_NAME} fuse) -link_jsoncpp(${PROJECT_NAME}) \ No newline at end of file diff --git a/core/reader/CMakeLists.txt b/core/reader/CMakeLists.txt index c9c9bcb398..d21a53fe54 100644 --- a/core/reader/CMakeLists.txt +++ b/core/reader/CMakeLists.txt @@ -26,7 +26,7 @@ target_link_libraries(${PROJECT_NAME} common) target_link_libraries(${PROJECT_NAME} logger) target_link_libraries(${PROJECT_NAME} parser) target_link_libraries(${PROJECT_NAME} checkpoint) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} event_handler) target_link_libraries(${PROJECT_NAME} app_config) target_link_libraries(${PROJECT_NAME} config_manager) diff --git a/core/reader/DelimiterLogFileReader.cpp b/core/reader/DelimiterLogFileReader.cpp index adca8cc003..7e01a294fa 100644 --- a/core/reader/DelimiterLogFileReader.cpp +++ b/core/reader/DelimiterLogFileReader.cpp @@ -16,7 +16,7 @@ #include #include #include "common/LogtailCommonFlags.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "parser/LogParser.h" #include "parser/DelimiterModeFsmParser.h" #include "log_pb/sls_logs.pb.h" diff --git a/core/reader/JsonLogFileReader.cpp b/core/reader/JsonLogFileReader.cpp index 3ca8f1dc7e..8bb1ae4ae9 100644 --- a/core/reader/JsonLogFileReader.cpp +++ b/core/reader/JsonLogFileReader.cpp @@ -17,7 +17,7 @@ #include #include #include "common/LogtailCommonFlags.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "parser/LogParser.h" #include "log_pb/sls_logs.pb.h" #include "logger/Logger.h" diff --git a/core/reader/LogFileReader.cpp b/core/reader/LogFileReader.cpp index f90a644553..b89e0b272e 100644 --- a/core/reader/LogFileReader.cpp +++ b/core/reader/LogFileReader.cpp @@ -35,8 +35,8 @@ #include "logger/Logger.h" #include "checkpoint/CheckPointManager.h" #include "checkpoint/CheckpointManagerV2.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogFileProfiler.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogFileProfiler.h" #include "event_handler/LogInput.h" #include "app_config/AppConfig.h" #include "config_manager/ConfigManager.h" diff --git a/core/sender/CMakeLists.txt b/core/sender/CMakeLists.txt index d38ab90c63..31adc53577 100644 --- a/core/sender/CMakeLists.txt +++ b/core/sender/CMakeLists.txt @@ -25,7 +25,7 @@ target_link_libraries(${PROJECT_NAME} app_config) target_link_libraries(${PROJECT_NAME} aggregator) target_link_libraries(${PROJECT_NAME} sdk) target_link_libraries(${PROJECT_NAME} monitor) -target_link_libraries(${PROJECT_NAME} profiler) +target_link_libraries(${PROJECT_NAME} monitor) target_link_libraries(${PROJECT_NAME} log_pb) target_link_libraries(${PROJECT_NAME} config_manager) target_link_libraries(${PROJECT_NAME} fuse) diff --git a/core/sender/Sender.cpp b/core/sender/Sender.cpp index 8c4bc44240..8c5b069822 100644 --- a/core/sender/Sender.cpp +++ b/core/sender/Sender.cpp @@ -41,10 +41,10 @@ #include "config/Config.h" #include "processor/LogProcess.h" #include "processor/LogFilter.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogIntegrity.h" -#include "profiler/LogLineCount.h" -#include "profiler/LogFileProfiler.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogIntegrity.h" +#include "monitor/LogLineCount.h" +#include "monitor/LogFileProfiler.h" #include "app_config/AppConfig.h" #include "monitor/Monitor.h" #include "config_manager/ConfigManager.h" diff --git a/core/sls_control/SLSControl.cpp b/core/sls_control/SLSControl.cpp index a94f973f48..e42ca4e760 100644 --- a/core/sls_control/SLSControl.cpp +++ b/core/sls_control/SLSControl.cpp @@ -22,7 +22,7 @@ #include "common/LogtailCommonFlags.h" #include "common/version.h" #include "logger/Logger.h" -#include "profiler/LogFileProfiler.h" +#include "monitor/LogFileProfiler.h" // for windows compatability, to avoid conflict with the same function defined in windows.h #ifdef SetPort #undef SetPort diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 9245d93b1d..d412dbfbba 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -57,8 +57,7 @@ add_subdirectory(polling) add_subdirectory(processor) add_subdirectory(reader) add_subdirectory(sender) -add_subdirectory(ilogtail_metric) -add_subdirectory(profiler) +add_subdirectory(monitor) add_subdirectory(sdk) if (UNIX) add_subdirectory(observer) diff --git a/core/unittest/ilogtail_metric/CMakeLists.txt b/core/unittest/monitor/CMakeLists.txt similarity index 79% rename from core/unittest/ilogtail_metric/CMakeLists.txt rename to core/unittest/monitor/CMakeLists.txt index ab1a53d73f..0fa0009221 100644 --- a/core/unittest/ilogtail_metric/CMakeLists.txt +++ b/core/unittest/monitor/CMakeLists.txt @@ -13,7 +13,7 @@ # limitations under the License. cmake_minimum_required(VERSION 2.9) -project(ilogtail_metric_unittest) +project(logtail_metric_unittest) -add_executable(ilogtail_metric_unittest ILogtailMetricUnittest.cpp) -target_link_libraries(ilogtail_metric_unittest unittest_base) \ No newline at end of file +add_executable(logtail_metric_unittest LogtailMetricUnittest.cpp) +target_link_libraries(logtail_metric_unittest unittest_base) \ No newline at end of file diff --git a/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp similarity index 99% rename from core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp rename to core/unittest/monitor/LogtailMetricUnittest.cpp index 49aa7a17fc..6a10159139 100644 --- a/core/unittest/ilogtail_metric/ILogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -5,7 +5,7 @@ #include #include #include "util.h" -#include "ILogtailMetric.h" +#include "LogtailMetric.h" #include "MetricExportor.h" #include "MetricConstants.h" diff --git a/core/unittest/polling/PollingUnittest.cpp b/core/unittest/polling/PollingUnittest.cpp index d6fafe6cd4..48c37d7da0 100644 --- a/core/unittest/polling/PollingUnittest.cpp +++ b/core/unittest/polling/PollingUnittest.cpp @@ -48,7 +48,7 @@ #include #include "log_pb/metric.pb.h" #include "log_pb/sls_logs.pb.h" -#include "profiler/LogtailAlarm.h" +#include "monitor/LogtailAlarm.h" #include "event_handler/LogInput.h" #include "common/FileEncryption.h" #include "common/FileSystemUtil.h" diff --git a/core/unittest/sender/SenderUnittest.cpp b/core/unittest/sender/SenderUnittest.cpp index bdadc4041a..96017beaf8 100644 --- a/core/unittest/sender/SenderUnittest.cpp +++ b/core/unittest/sender/SenderUnittest.cpp @@ -43,8 +43,8 @@ #include #include "log_pb/metric.pb.h" #include "log_pb/sls_logs.pb.h" -#include "profiler/LogtailAlarm.h" -#include "profiler/LogIntegrity.h" +#include "monitor/LogtailAlarm.h" +#include "monitor/LogIntegrity.h" #include "event_handler/LogInput.h" #include "common/FileEncryption.h" #include "processor/LogProcess.h" From 378b8dfd7f14a165a2337e77ff1d8630cefd56a2 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Mon, 31 Jul 2023 15:27:00 +0800 Subject: [PATCH 12/39] fix comment --- core/logtail.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/logtail.cpp b/core/logtail.cpp index 0c7d623037..e797b34a89 100644 --- a/core/logtail.cpp +++ b/core/logtail.cpp @@ -58,7 +58,7 @@ void* __wrap_memcpy(void* dest, const void* src, size_t n) { } #endif -DEFINE_FLAG_BOOL(ilogtail_disable_core, "disable core in worker process", false); +DEFINE_FLAG_BOOL(ilogtail_disable_core, "disable core in worker process", true); DEFINE_FLAG_INT32(fork_interval, "fork dispatcher process interval", 10); DECLARE_FLAG_INT32(max_open_files_limit); DECLARE_FLAG_INT32(max_reader_open_files); From c5b4c60da5de5d925ea8ecd0fc8dc5b36aeade35 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Mon, 31 Jul 2023 17:04:26 +0800 Subject: [PATCH 13/39] fix raw ptr --- core/monitor/LogtaiMetric.cpp | 23 ++++++++++--------- core/monitor/LogtailMetric.h | 8 ++++--- core/unittest/monitor/CMakeLists.txt | 6 ++++- .../DataIntegrityUnittest.cpp | 0 .../monitor/LogtailMetricUnittest.cpp | 10 ++++---- core/unittest/profiler/CMakeLists.txt | 19 --------------- 6 files changed, 27 insertions(+), 39 deletions(-) rename core/unittest/{profiler => monitor}/DataIntegrityUnittest.cpp (100%) delete mode 100644 core/unittest/profiler/CMakeLists.txt diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index cdf4314ce4..f90829ec9a 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -70,10 +70,10 @@ Counter* Counter::CopyAndReset() { return counter; } -Counter* Metrics::CreateCounter(std::string name) { - Counter* counter = new Counter(name); - mValues.push_back(counter); - return counter; +CounterPtr Metrics::CreateCounter(std::string name) { + CounterPtr counterPtr(new Counter(name)); + mValues.push_back(counterPtr); + return counterPtr; } @@ -89,7 +89,7 @@ const std::vector>& Metrics::GetLabels() { return mLabels; } -const std::vector& Metrics::GetValues() { +const std::vector& Metrics::GetValues() { return mValues; } @@ -100,9 +100,10 @@ Metrics* Metrics::Copy() { newLabels.push_back(std::make_pair(pair.first, pair.second)); } Metrics* metrics = new Metrics(newLabels); - for (std::vector::iterator it = mValues.begin(); it != mValues.end(); ++it) { - Counter* cur = *it; - metrics->mValues.push_back(cur->CopyAndReset()); + for (std::vector::iterator it = mValues.begin(); it != mValues.end(); ++it) { + CounterPtr cur = *it; + CounterPtr newCounterPtr(cur->CopyAndReset()); + metrics->mValues.push_back(newCounterPtr); } return metrics; } @@ -211,9 +212,9 @@ void ReadMetrics::ReadAsLogGroup(sls_logs::LogGroup& logGroup) { contentPtr->set_value(pair.second); } - std::vector values = tmp->GetValues(); - for (std::vector::iterator it = values.begin(); it != values.end(); ++it) { - Counter* counter = *it; + std::vector values = tmp->GetValues(); + for (std::vector::iterator it = values.begin(); it != values.end(); ++it) { + CounterPtr counter = *it; Log_Content* contentPtr = logPtr->add_contents(); contentPtr->set_key("value." + counter->GetName()); contentPtr->set_value(ToString(counter->GetValue())); diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 3f6a5d7add..1e19470e0f 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -30,10 +30,12 @@ class Counter{ Counter* CopyAndReset(); }; +typedef std::shared_ptr CounterPtr; + class Metrics { private: std::vector> mLabels; - std::vector mValues; + std::vector mValues; std::atomic_bool mDeleted; public: @@ -43,8 +45,8 @@ class Metrics { void MarkDeleted(); bool IsDeleted(); const std::vector>& GetLabels(); - const std::vector& GetValues(); - Counter* CreateCounter(std::string Name); + const std::vector& GetValues(); + CounterPtr CreateCounter(std::string Name); Metrics* Copy(); Metrics* next = NULL; }; diff --git a/core/unittest/monitor/CMakeLists.txt b/core/unittest/monitor/CMakeLists.txt index 0fa0009221..cf3f1ac22e 100644 --- a/core/unittest/monitor/CMakeLists.txt +++ b/core/unittest/monitor/CMakeLists.txt @@ -16,4 +16,8 @@ cmake_minimum_required(VERSION 2.9) project(logtail_metric_unittest) add_executable(logtail_metric_unittest LogtailMetricUnittest.cpp) -target_link_libraries(logtail_metric_unittest unittest_base) \ No newline at end of file +target_link_libraries(logtail_metric_unittest unittest_base) + + +add_executable(profiler_data_integrity_unittest DataIntegrityUnittest.cpp) +target_link_libraries(profiler_data_integrity_unittest unittest_base) \ No newline at end of file diff --git a/core/unittest/profiler/DataIntegrityUnittest.cpp b/core/unittest/monitor/DataIntegrityUnittest.cpp similarity index 100% rename from core/unittest/profiler/DataIntegrityUnittest.cpp rename to core/unittest/monitor/DataIntegrityUnittest.cpp diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp index 6a10159139..f618da9c5a 100644 --- a/core/unittest/monitor/LogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -43,15 +43,15 @@ void ILogtailMetricUnittest::TestCreateMetric() { APSARA_TEST_EQUAL(fileMetric3->GetLabels().size(), 2); - Counter* fileCounter = fileMetric->CreateCounter("filed1"); + CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); fileCounter->Add((uint64_t)111); LOG_INFO(sLogger, ("value", fileCounter->GetValue())("time", fileCounter->GetTimestamp())); - Counter* fileCounter2 = fileMetric->CreateCounter("filed2"); + CounterPtr fileCounter2 = fileMetric->CreateCounter("filed2"); fileCounter2->Add((uint64_t)222); LOG_INFO(sLogger, ("value", fileCounter2->GetValue())("time", fileCounter2->GetTimestamp())); - Counter* fileCounter3 = fileMetric3->CreateCounter("filed2"); + CounterPtr fileCounter3 = fileMetric3->CreateCounter("filed2"); fileCounter3->Add((uint64_t)222); LOG_INFO(sLogger, ("value", fileCounter3->GetValue())("time", fileCounter3->GetTimestamp())); @@ -75,7 +75,7 @@ void createMetrics(int count) { labels.push_back(std::make_pair("num", std::to_string(i))); labels.push_back(std::make_pair("count", std::to_string(count))); Metrics* fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); - Counter* fileCounter = fileMetric->CreateCounter("filed1"); + CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); fileCounter->Add((uint64_t)111); //LOG_INFO(sLogger, ("num", i )("count", count)); } @@ -87,7 +87,7 @@ void createAndDeleteMetrics(int count) { labels.push_back(std::make_pair("num", std::to_string(i))); labels.push_back(std::make_pair("count", std::to_string(count))); Metrics* fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); - Counter* fileCounter = fileMetric->CreateCounter("filed1"); + CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); fileCounter->Add((uint64_t)111); //LOG_INFO(sLogger, ("num", i )("count", count)); WriteMetrics::GetInstance()->DestroyMetrics(fileMetric); diff --git a/core/unittest/profiler/CMakeLists.txt b/core/unittest/profiler/CMakeLists.txt deleted file mode 100644 index c2cf855388..0000000000 --- a/core/unittest/profiler/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2022 iLogtail Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cmake_minimum_required(VERSION 2.9) -project(profiler_unittest) - -add_executable(profiler_data_integrity_unittest DataIntegrityUnittest.cpp) -target_link_libraries(profiler_data_integrity_unittest unittest_base) \ No newline at end of file From 7b934c3297f6e88dce5a26b9c7b0817920fe99d5 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Wed, 2 Aug 2023 15:27:34 +0800 Subject: [PATCH 14/39] fix comment --- core/monitor/LogtaiMetric.cpp | 2 -- core/monitor/MetricExportor.cpp | 11 +------ .../monitor/LogtailMetricUnittest.cpp | 32 ++++++++----------- 3 files changed, 14 insertions(+), 31 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index f90829ec9a..8e7a82e4b3 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -117,13 +117,11 @@ Metrics* WriteMetrics::CreateMetrics(std::vectornext = oldHead; - LOG_INFO(sLogger, ("CreateMetrics", "mSnapshottingHead")); } else { Metrics* oldHead = mHead; mHead = cur; mHead->next = oldHead; - LOG_INFO(sLogger, ("CreateMetrics", "mHead")); } return cur; } diff --git a/core/monitor/MetricExportor.cpp b/core/monitor/MetricExportor.cpp index 46e6f32d42..a5d6e280db 100644 --- a/core/monitor/MetricExportor.cpp +++ b/core/monitor/MetricExportor.cpp @@ -16,18 +16,9 @@ MetricExportor::MetricExportor() { } void MetricExportor::PushMetrics() { - LOG_INFO(sLogger, ("pushMetrics", "")); ReadMetrics::GetInstance()->UpdateMetrics(); sls_logs::LogGroup logGroup; - logGroup.set_category("metric-test"); ReadMetrics::GetInstance()->ReadAsLogGroup(logGroup); - for (int i = 0; i < logGroup.logs_size(); ++i) { - const sls_logs::Log& log = logGroup.logs(i); - for (int j = 0; j < log.contents_size(); ++j) { - const sls_logs::Log_Content& content = log.contents(j); - LOG_INFO(sLogger, ("contentKey", content.key())("contentValue", content.value())); - } - } + // TODO: send } - } \ No newline at end of file diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp index f618da9c5a..c584730c19 100644 --- a/core/unittest/monitor/LogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -31,41 +31,36 @@ void ILogtailMetricUnittest::TestCreateMetric() { labels.push_back(std::make_pair("logstore","logstore1")); Metrics* fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); - LOG_INFO(sLogger, ("labelSize", fileMetric->GetLabels().size())); APSARA_TEST_EQUAL(fileMetric->GetLabels().size(), 2); Metrics* fileMetric2 = WriteMetrics::GetInstance()->CreateMetrics(labels); - LOG_INFO(sLogger, ("labelSize", fileMetric2->GetLabels().size())); - APSARA_TEST_EQUAL(fileMetric2->GetLabels().size(), 2); Metrics* fileMetric3 = WriteMetrics::GetInstance()->CreateMetrics(labels); - LOG_INFO(sLogger, ("labelSize", fileMetric3->GetLabels().size())); - APSARA_TEST_EQUAL(fileMetric3->GetLabels().size(), 2); CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); fileCounter->Add((uint64_t)111); - LOG_INFO(sLogger, ("value", fileCounter->GetValue())("time", fileCounter->GetTimestamp())); + APSARA_TEST_EQUAL(fileCounter->GetValue(), 111); CounterPtr fileCounter2 = fileMetric->CreateCounter("filed2"); fileCounter2->Add((uint64_t)222); - LOG_INFO(sLogger, ("value", fileCounter2->GetValue())("time", fileCounter2->GetTimestamp())); - - CounterPtr fileCounter3 = fileMetric3->CreateCounter("filed2"); - fileCounter3->Add((uint64_t)222); - LOG_INFO(sLogger, ("value", fileCounter3->GetValue())("time", fileCounter3->GetTimestamp())); + + CounterPtr fileCounter3 = fileMetric3->CreateCounter("filed3"); + fileCounter3->Add((uint64_t)333); WriteMetrics::GetInstance()->DestroyMetrics(fileMetric2); // delete first element - WriteMetrics::GetInstance()->DestroyMetrics(fileMetric); + WriteMetrics::GetInstance()->DestroyMetrics(fileMetric3); ReadMetrics::GetInstance()->UpdateMetrics(); Metrics* head = ReadMetrics::GetInstance()->mHead; + int count = 0; while(head) { - LOG_INFO(sLogger, ("ReadMetrics", head->GetLabels().size())); head = head->next; - } + count ++; + } + APSARA_TEST_EQUAL(count, 1); MetricExportor::GetInstance()->PushMetrics(); } @@ -77,7 +72,6 @@ void createMetrics(int count) { Metrics* fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); fileCounter->Add((uint64_t)111); - //LOG_INFO(sLogger, ("num", i )("count", count)); } } @@ -89,7 +83,6 @@ void createAndDeleteMetrics(int count) { Metrics* fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); fileCounter->Add((uint64_t)111); - //LOG_INFO(sLogger, ("num", i )("count", count)); WriteMetrics::GetInstance()->DestroyMetrics(fileMetric); } } @@ -101,7 +94,6 @@ void UpdateMetrics() { std::vector> labels = head->GetLabels(); for (std::vector>::iterator it = labels.begin(); it != labels.end(); ++it) { std::pair pair = *it; - //LOG_INFO(sLogger, ("key", pair.first)("value", pair.second)); } head = head->next; } @@ -130,6 +122,8 @@ void ILogtailMetricUnittest::TestCreateMetricMultiThread() { head = head->next; count ++; } + // first test left 1, 1 + (1 + 2 + 3 + 4) + APSARA_TEST_EQUAL(count, 11); LOG_INFO(sLogger, ("Count", count)); } @@ -161,7 +155,6 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { head = head->next; } LOG_INFO(sLogger, ("WriteCount", count)); - } { Metrics* head = ReadMetrics::GetInstance()->mHead; @@ -171,7 +164,6 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { count ++; } LOG_INFO(sLogger, ("ReadCount", count)); - } UpdateMetrics(); @@ -184,6 +176,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { } head = head->next; } + APSARA_TEST_EQUAL(count, 11); LOG_INFO(sLogger, ("FinalWriteCount", count)); } @@ -194,6 +187,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { head = head->next; count ++; } + APSARA_TEST_EQUAL(count, 11); LOG_INFO(sLogger, ("FinalReadCount", count)); } From e6c8a8e85c8d9083d7ca66af6a540dbb63bf7cef Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Wed, 2 Aug 2023 15:37:58 +0800 Subject: [PATCH 15/39] add change log --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0d4e5f2ad..fda5701192 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,4 +42,5 @@ your changes, such as: - [public] [both] [doc] elasticsearch flusher config examples - [public] [both] [added] add new plugin: input_command - [public] [both] [added] add new plugin: processor_log_to_sls_metric -- [public] [both] [fixed] fix service_go_profile nil panic \ No newline at end of file +- [public] [both] [fixed] fix service_go_profile nil panic +- [public] [both] [added] add new logtail metric module \ No newline at end of file From 2d21c4e073e321ba8c36075a489c881fc81f10e8 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Mon, 7 Aug 2023 09:44:09 +0800 Subject: [PATCH 16/39] add group by region --- core/monitor/LogtaiMetric.cpp | 34 +++++++++++++++++-- core/monitor/LogtailMetric.h | 2 +- core/monitor/MetricExportor.cpp | 17 ++++++++-- .../monitor/LogtailMetricUnittest.cpp | 8 +++-- 4 files changed, 53 insertions(+), 8 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index 8e7a82e4b3..a82970995c 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -196,13 +196,41 @@ Metrics* WriteMetrics::DoSnapshot() { return snapshot; } -void ReadMetrics::ReadAsLogGroup(sls_logs::LogGroup& logGroup) { +void ReadMetrics::ReadAsLogGroup(std::map& logGroupMap) { ReadLock lock(mReadWriteLock); Metrics* tmp = mHead; while(tmp) { - Log* logPtr = logGroup.add_logs(); - logPtr->set_time(time(NULL)); + Log* logPtr = NULL; std::vector> labels = tmp->GetLabels(); + for (std::vector>::iterator it = labels.begin(); it != labels.end(); ++it) { + std::pair pair = *it; + if ("region" == pair.first) { + std::map::iterator iter; + std::string region = pair.second; + iter = logGroupMap.find(region); + if (iter != logGroupMap.end()) { + sls_logs::LogGroup logGroup = iter->second; + logPtr = logGroup.add_logs(); + } else { + sls_logs::LogGroup logGroup; + logPtr = logGroup.add_logs(); + logGroupMap.insert(std::pair(region, logGroup)); + } + } + } + if (!logPtr) { + std::map::iterator iter; + iter = logGroupMap.find("default"); + if (iter != logGroupMap.end()) { + sls_logs::LogGroup logGroup = iter->second; + logPtr = logGroup.add_logs(); + } else { + sls_logs::LogGroup logGroup; + logPtr = logGroup.add_logs(); + logGroupMap.insert(std::pair("default", logGroup)); + } + } + logPtr->set_time(time(NULL)); for (std::vector>::iterator it = labels.begin(); it != labels.end(); ++it) { std::pair pair = *it; Log_Content* contentPtr = logPtr->add_contents(); diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 1e19470e0f..0f2d1a244e 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -80,7 +80,7 @@ class ReadMetrics { static ReadMetrics* ptr = new ReadMetrics(); return ptr; } - void ReadAsLogGroup(sls_logs::LogGroup& logGroup); + void ReadAsLogGroup(std::map& logGroupMap); void ReadAsMap(std::map map); void ReadAsPrometheus(); void UpdateMetrics(); diff --git a/core/monitor/MetricExportor.cpp b/core/monitor/MetricExportor.cpp index a5d6e280db..cc6debdc0b 100644 --- a/core/monitor/MetricExportor.cpp +++ b/core/monitor/MetricExportor.cpp @@ -17,8 +17,21 @@ MetricExportor::MetricExportor() { void MetricExportor::PushMetrics() { ReadMetrics::GetInstance()->UpdateMetrics(); - sls_logs::LogGroup logGroup; - ReadMetrics::GetInstance()->ReadAsLogGroup(logGroup); + std::map logGroupMap; + ReadMetrics::GetInstance()->ReadAsLogGroup(logGroupMap); // TODO: send + std::map::iterator iter; + for (iter = logGroupMap.begin(); iter != logGroupMap.end(); iter ++) { + LOG_INFO(sLogger, ("Region", iter->first)); + sls_logs::LogGroup logGroup = iter->second; + logGroup.set_category("shennong_log_profile"); + for (int i = 0; i < logGroup.logs_size(); ++i) { + const sls_logs::Log& log = logGroup.logs(i); + for (int j = 0; j < log.contents_size(); ++j) { + const sls_logs::Log_Content& content = log.contents(j); + LOG_INFO(sLogger, ("contentKey", content.key())("contentValue", content.value())); + } + } + } } } \ No newline at end of file diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp index c584730c19..9b9242ea2b 100644 --- a/core/unittest/monitor/LogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -29,9 +29,10 @@ void ILogtailMetricUnittest::TestCreateMetric() { std::vector> labels; labels.push_back(std::make_pair("project","project1")); labels.push_back(std::make_pair("logstore","logstore1")); + labels.push_back(std::make_pair("region","cn-hangzhou")); Metrics* fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); - APSARA_TEST_EQUAL(fileMetric->GetLabels().size(), 2); + APSARA_TEST_EQUAL(fileMetric->GetLabels().size(), 3); Metrics* fileMetric2 = WriteMetrics::GetInstance()->CreateMetrics(labels); @@ -69,6 +70,7 @@ void createMetrics(int count) { std::vector> labels; labels.push_back(std::make_pair("num", std::to_string(i))); labels.push_back(std::make_pair("count", std::to_string(count))); + labels.push_back(std::make_pair("region","cn-beijing")); Metrics* fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); fileCounter->Add((uint64_t)111); @@ -80,6 +82,7 @@ void createAndDeleteMetrics(int count) { std::vector> labels; labels.push_back(std::make_pair("num", std::to_string(i))); labels.push_back(std::make_pair("count", std::to_string(count))); + labels.push_back(std::make_pair("region","cn-shanghai")); Metrics* fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); fileCounter->Add((uint64_t)111); @@ -125,6 +128,7 @@ void ILogtailMetricUnittest::TestCreateMetricMultiThread() { // first test left 1, 1 + (1 + 2 + 3 + 4) APSARA_TEST_EQUAL(count, 11); LOG_INFO(sLogger, ("Count", count)); + MetricExportor::GetInstance()->PushMetrics(); } @@ -189,8 +193,8 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { } APSARA_TEST_EQUAL(count, 11); LOG_INFO(sLogger, ("FinalReadCount", count)); - } + MetricExportor::GetInstance()->PushMetrics(); } From 54d1fae48f7672ed4744f2d0e734b21da0d3ae8b Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Tue, 15 Aug 2023 15:41:02 +0800 Subject: [PATCH 17/39] fix comments --- core/monitor/LogtaiMetric.cpp | 121 +++++++++--------- core/monitor/LogtailMetric.h | 58 ++++++--- core/monitor/MetricConstants.h | 7 + .../monitor/LogtailMetricUnittest.cpp | 2 +- 4 files changed, 107 insertions(+), 81 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index a82970995c..321a1076a0 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -5,77 +5,84 @@ using namespace sls_logs; namespace logtail { -Counter::Counter(std::string name) { - mName = name; - mVal = (uint64_t)0; - mTimestamp = time(NULL); -} - -Counter::~Counter() { -} - -Metrics::Metrics(std::vector> labels) { - mLabels = labels; - mDeleted.store(false); +const uint64_t BaseMetric::GetValue() const { + return mVal; } -Metrics::Metrics() { - mDeleted.store(false); +const uint64_t BaseMetric::GetTimestamp() const { + return mTimestamp; } -Metrics::~Metrics() { +const std::string BaseMetric::GetName() const { + return mName; } -WriteMetrics::WriteMetrics() { - mSnapshotting.store(false); +Counter::Counter(std::string name) { + mName = name; + mVal = (uint64_t)0; + mTimestamp = time(NULL); } -WriteMetrics::~WriteMetrics() { +Counter* Counter::CopyAndReset() { + Counter* counter = new Counter(mName); + counter->mVal = mVal.exchange(0); + counter->mTimestamp = mTimestamp.exchange(0); + return counter; } -ReadMetrics::ReadMetrics() { - mWriteMetrics = WriteMetrics::GetInstance(); +void Counter::Add(uint64_t value) { + mVal += value; + mTimestamp = time(NULL); } -ReadMetrics::~ReadMetrics() { +Gauge::Gauge(std::string name) { + mName = name; + mVal = (uint64_t)0; + mTimestamp = time(NULL); } -void Counter::Add(uint64_t value) { - mVal += value; - mTimestamp = time(NULL); +Gauge* Gauge::CopyAndReset() { + Gauge* gauge = new Gauge(mName); + gauge->mVal = mVal.exchange(0); + gauge->mTimestamp = mTimestamp.exchange(0); + return gauge; } -void Counter::Set(uint64_t value) { +void Gauge::Set(uint64_t value) { mVal = value; mTimestamp = time(NULL); } -uint64_t Counter::GetValue() { - return mVal; -} -uint64_t Counter::GetTimestamp() { - return mTimestamp; +Metrics::Metrics(std::vector> labels) { + mLabels = labels; + mDeleted.store(false); } -std::string Counter::GetName() { - return mName; +Metrics::Metrics() { + mDeleted.store(false); } +WriteMetrics::WriteMetrics() { + mSnapshotting.store(false); +} -Counter* Counter::CopyAndReset() { - Counter* counter = new Counter(mName); - counter->mVal = mVal.exchange(0); - counter->mTimestamp = mTimestamp.exchange(0); - return counter; +ReadMetrics::ReadMetrics() { + mWriteMetrics = WriteMetrics::GetInstance(); } -CounterPtr Metrics::CreateCounter(std::string name) { - CounterPtr counterPtr(new Counter(name)); + +CounterPtr Metrics::CreateCounter(const std::string name) { + CounterPtr counterPtr = std::make_shared(name); mValues.push_back(counterPtr); return counterPtr; } +GaugePtr Metrics::CreateGauge(const std::string name) { + GaugePtr gaugePtr = std::make_shared(name); + mValues.push_back(gaugePtr); + return gaugePtr; +} void Metrics::MarkDeleted() { mDeleted.store(true); @@ -85,11 +92,11 @@ bool Metrics::IsDeleted() { return mDeleted; } -const std::vector>& Metrics::GetLabels() { +const std::vector>& Metrics::GetLabels() const { return mLabels; } -const std::vector& Metrics::GetValues() { +const std::vector& Metrics::GetValues() const { return mValues; } @@ -99,11 +106,11 @@ Metrics* Metrics::Copy() { std::pair pair = *it; newLabels.push_back(std::make_pair(pair.first, pair.second)); } - Metrics* metrics = new Metrics(newLabels); - for (std::vector::iterator it = mValues.begin(); it != mValues.end(); ++it) { - CounterPtr cur = *it; - CounterPtr newCounterPtr(cur->CopyAndReset()); - metrics->mValues.push_back(newCounterPtr); + Metrics* metrics = new Metrics(newLabels); + for (std::vector::iterator it = mValues.begin(); it != mValues.end(); ++it) { + MetricPtr cur = *it; + MetricPtr newPtr(cur->CopyAndReset()); + metrics->mValues.push_back(newPtr); } return metrics; } @@ -158,7 +165,7 @@ Metrics* WriteMetrics::DoSnapshot() { delete toDeleted; continue; } - Metrics* newMetrics = tmp->Copy(); + Metrics* newMetrics(tmp->Copy()); // Get Head if (!snapshot) { @@ -204,7 +211,7 @@ void ReadMetrics::ReadAsLogGroup(std::map& logG std::vector> labels = tmp->GetLabels(); for (std::vector>::iterator it = labels.begin(); it != labels.end(); ++it) { std::pair pair = *it; - if ("region" == pair.first) { + if (METRIC_FIELD_REGION == pair.first) { std::map::iterator iter; std::string region = pair.second; iter = logGroupMap.find(region); @@ -220,29 +227,29 @@ void ReadMetrics::ReadAsLogGroup(std::map& logG } if (!logPtr) { std::map::iterator iter; - iter = logGroupMap.find("default"); + iter = logGroupMap.find(METRIC_REGION_DEFAULT); if (iter != logGroupMap.end()) { sls_logs::LogGroup logGroup = iter->second; logPtr = logGroup.add_logs(); } else { sls_logs::LogGroup logGroup; logPtr = logGroup.add_logs(); - logGroupMap.insert(std::pair("default", logGroup)); + logGroupMap.insert(std::pair(METRIC_REGION_DEFAULT, logGroup)); } } logPtr->set_time(time(NULL)); for (std::vector>::iterator it = labels.begin(); it != labels.end(); ++it) { std::pair pair = *it; Log_Content* contentPtr = logPtr->add_contents(); - contentPtr->set_key("label." + pair.first); + contentPtr->set_key(LABEL_PREFIX + pair.first); contentPtr->set_value(pair.second); } - std::vector values = tmp->GetValues(); - for (std::vector::iterator it = values.begin(); it != values.end(); ++it) { - CounterPtr counter = *it; + std::vector values = tmp->GetValues(); + for (std::vector::iterator it = values.begin(); it != values.end(); ++it) { + MetricPtr counter = *it; Log_Content* contentPtr = logPtr->add_contents(); - contentPtr->set_key("value." + counter->GetName()); + contentPtr->set_key(VALUE_PREFIX + counter->GetName()); contentPtr->set_value(ToString(counter->GetValue())); } tmp = tmp->next; @@ -250,10 +257,6 @@ void ReadMetrics::ReadAsLogGroup(std::map& logG } -void ReadMetrics::ReadAsPrometheus() { - ReadLock lock(mReadWriteLock); - // Do some read -} void ReadMetrics::UpdateMetrics() { Metrics* snapshot = mWriteMetrics->DoSnapshot(); diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 0f2d1a244e..fb4bc57c87 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -12,49 +12,66 @@ namespace logtail { -class Counter{ - private: + +class BaseMetric { + protected: std::string mName; std::atomic_long mVal; std::atomic_long mTimestamp; + public: + const uint64_t GetValue() const; + const uint64_t GetTimestamp() const; + const std::string GetName() const; + virtual BaseMetric* CopyAndReset() = 0; +}; + +class Gauge : public BaseMetric { public: - Counter(std::string name); - ~Counter(); - uint64_t GetValue(); - uint64_t GetTimestamp(); - std::string GetName(); - - void Add(uint64_t val); + Gauge(const std::string name); void Set(uint64_t val); + Gauge* CopyAndReset(); +}; + + +class Counter : public BaseMetric { + public: + Counter(const std::string name); + void Add(uint64_t val); Counter* CopyAndReset(); }; -typedef std::shared_ptr CounterPtr; +using MetricPtr = std::shared_ptr; +using CounterPtr = std::shared_ptr; +using GaugePtr = std::shared_ptr; + class Metrics { private: std::vector> mLabels; - std::vector mValues; + std::vector mValues; std::atomic_bool mDeleted; + public: - Metrics(std::vector > labels); + Metrics(std::vector> labels); Metrics(); - ~Metrics(); void MarkDeleted(); bool IsDeleted(); - const std::vector>& GetLabels(); - const std::vector& GetValues(); - CounterPtr CreateCounter(std::string Name); + const std::vector>& GetLabels() const; + const std::vector& GetValues() const; + CounterPtr CreateCounter(const std::string Name); + GaugePtr CreateGauge(const std::string Name); + + //using MetricsUniq = std::unique_ptr; Metrics* Copy(); - Metrics* next = NULL; + Metrics* next; }; + class WriteMetrics { private: WriteMetrics(); - ~WriteMetrics(); std::atomic_bool mSnapshotting; public: static WriteMetrics* GetInstance() { @@ -74,15 +91,14 @@ class WriteMetrics { class ReadMetrics { private: ReadMetrics(); - ~ReadMetrics(); public: static ReadMetrics* GetInstance() { static ReadMetrics* ptr = new ReadMetrics(); return ptr; } void ReadAsLogGroup(std::map& logGroupMap); - void ReadAsMap(std::map map); - void ReadAsPrometheus(); + // void ReadAsMap(std::map map); + // void ReadAsPrometheus(); void UpdateMetrics(); Metrics* mHead = NULL; WriteMetrics* mWriteMetrics; diff --git a/core/monitor/MetricConstants.h b/core/monitor/MetricConstants.h index 794039fc9d..b565c71613 100644 --- a/core/monitor/MetricConstants.h +++ b/core/monitor/MetricConstants.h @@ -19,6 +19,13 @@ namespace logtail { +const std::string METRIC_FIELD_REGION = "region"; +const std::string METRIC_REGION_DEFAULT = "default"; +const std::string LABEL_PREFIX = "label."; +const std::string VALUE_PREFIX = "value."; + + + const std::string METRIC_FILE_READ_COUNT = "file_read_count"; const std::string METRIC_FILE_READ_BYTES = "file_read_bytes"; diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp index 9b9242ea2b..305fc6414a 100644 --- a/core/unittest/monitor/LogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -86,7 +86,7 @@ void createAndDeleteMetrics(int count) { Metrics* fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); fileCounter->Add((uint64_t)111); - WriteMetrics::GetInstance()->DestroyMetrics(fileMetric); + //WriteMetrics::GetInstance()->DestroyMetrics(fileMetric); } } From baf998900fa16aabf48dc4a6a243b96e16f05859 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Thu, 17 Aug 2023 12:00:01 +0800 Subject: [PATCH 18/39] fix cooment --- core/monitor/LogtaiMetric.cpp | 204 ++++++++---------- core/monitor/LogtailMetric.h | 43 ++-- core/monitor/MetricExportor.cpp | 23 +- core/monitor/MetricExportor.h | 19 -- .../monitor/LogtailMetricUnittest.cpp | 201 +++++++++-------- 5 files changed, 225 insertions(+), 265 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index 321a1076a0..c3e78c3a9f 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -1,4 +1,8 @@ #include "LogtailMetric.h" +#include "common/StringTools.h" +#include "MetricConstants.h" +#include "logger/Logger.h" + using namespace sls_logs; @@ -9,10 +13,6 @@ const uint64_t BaseMetric::GetValue() const { return mVal; } -const uint64_t BaseMetric::GetTimestamp() const { - return mTimestamp; -} - const std::string BaseMetric::GetName() const { return mName; } @@ -20,56 +20,41 @@ const std::string BaseMetric::GetName() const { Counter::Counter(std::string name) { mName = name; mVal = (uint64_t)0; - mTimestamp = time(NULL); } Counter* Counter::CopyAndReset() { Counter* counter = new Counter(mName); counter->mVal = mVal.exchange(0); - counter->mTimestamp = mTimestamp.exchange(0); return counter; } void Counter::Add(uint64_t value) { mVal += value; - mTimestamp = time(NULL); } Gauge::Gauge(std::string name) { mName = name; mVal = (uint64_t)0; - mTimestamp = time(NULL); } Gauge* Gauge::CopyAndReset() { Gauge* gauge = new Gauge(mName); gauge->mVal = mVal.exchange(0); - gauge->mTimestamp = mTimestamp.exchange(0); return gauge; } void Gauge::Set(uint64_t value) { mVal = value; - mTimestamp = time(NULL); } -Metrics::Metrics(std::vector> labels) { - mLabels = labels; - mDeleted.store(false); -} +Metrics::Metrics(const std::vector>& labels) : mLabels(std::move(labels)), mDeleted(false) {} -Metrics::Metrics() { - mDeleted.store(false); -} +Metrics::Metrics() : mDeleted(false) {} -WriteMetrics::WriteMetrics() { - mSnapshotting.store(false); -} +WriteMetrics::WriteMetrics() {} -ReadMetrics::ReadMetrics() { - mWriteMetrics = WriteMetrics::GetInstance(); -} +ReadMetrics::ReadMetrics() {} CounterPtr Metrics::CreateCounter(const std::string name) { @@ -85,7 +70,7 @@ GaugePtr Metrics::CreateGauge(const std::string name) { } void Metrics::MarkDeleted() { - mDeleted.store(true); + mDeleted = true; } bool Metrics::IsDeleted() { @@ -100,106 +85,98 @@ const std::vector& Metrics::GetValues() const { return mValues; } -Metrics* Metrics::Copy() { - std::vector> newLabels; - for (std::vector>::iterator it = mLabels.begin(); it != mLabels.end(); ++it) { - std::pair pair = *it; - newLabels.push_back(std::make_pair(pair.first, pair.second)); - } +Metrics* Metrics::CopyAndReset() { + std::vector> newLabels(mLabels); Metrics* metrics = new Metrics(newLabels); - for (std::vector::iterator it = mValues.begin(); it != mValues.end(); ++it) { - MetricPtr cur = *it; - MetricPtr newPtr(cur->CopyAndReset()); + for (auto &item: mValues) { + MetricPtr newPtr(item->CopyAndReset()); metrics->mValues.push_back(newPtr); } return metrics; } -Metrics* WriteMetrics::CreateMetrics(std::vector> labels) { - Metrics* cur = new Metrics(labels); - std::lock_guard lock(mMutex); - // add metric to head - // 根据mSnapshotting的状态，决定节点加到哪里 - if (mSnapshotting) { - Metrics* oldHead = mSnapshottingHead; - mSnapshottingHead = cur; - mSnapshottingHead->next = oldHead; - - } else { - Metrics* oldHead = mHead; - mHead = cur; - mHead->next = oldHead; - } - return cur; +Metrics* Metrics::GetNext() { + return mNext; } +void Metrics::SetNext(Metrics* next) { + mNext = next; +} // mark as deleted -void WriteMetrics::DestroyMetrics(Metrics* metrics) { +void DestroyMetrics(Metrics* metrics) { // deleted is atomic_bool, no need to lock - metrics->MarkDeleted(); + LOG_INFO(sLogger, ("DestroyMetrics", "true")); + metrics->MarkDeleted(); +} + +MetricsPtr WriteMetrics::CreateMetrics(const std::vector>& labels) { + Metrics* cur = new Metrics(std::move(labels)); + std::lock_guard lock(mMutex); + + Metrics* oldHead = mHead; + mHead = cur; + mHead->SetNext(oldHead); + + MetricsPtr curUnique(cur, DestroyMetrics); + return curUnique; } +Metrics* WriteMetrics::GetHead() { + return mHead; +} Metrics* WriteMetrics::DoSnapshot() { - mSnapshotting.store(true); // new read head - Metrics* snapshot = NULL; - // new read head iter - Metrics* rTmp; - + Metrics* snapshot = nullptr; + Metrics* toDeleteHead = nullptr; + Metrics* tmp = nullptr; + Metrics* emptyHead = new Metrics(); + Metrics* preTmp = nullptr; + // find the first not deleted node and set as new mHead { - std::lock_guard lock(mMutex); - emptyHead->next = mHead; - } - Metrics* preTmp = emptyHead; - while(preTmp) { - Metrics* tmp = preTmp->next; + std::lock_guard lock(mMutex); + emptyHead->SetNext(mHead); + preTmp = emptyHead; + tmp = preTmp->GetNext(); + while (tmp) { + if (tmp->IsDeleted()) { + preTmp->SetNext(tmp->GetNext()); + tmp->SetNext(toDeleteHead); + toDeleteHead = tmp; + tmp = preTmp->GetNext(); + } else { + mHead = tmp; + break; + } + } if (!tmp) { - break; + mHead = nullptr; } + } + + while (tmp) { if (tmp->IsDeleted()) { - Metrics* toDeleted = tmp; - preTmp->next = tmp->next; - delete toDeleted; - continue; - } - Metrics* newMetrics(tmp->Copy()); - // Get Head - if (!snapshot) { - - preTmp = preTmp->next; + preTmp->SetNext(tmp->GetNext()); + tmp->SetNext(toDeleteHead); + toDeleteHead = tmp; + tmp = preTmp->GetNext(); + } else { + Metrics* newMetrics = tmp->CopyAndReset(); + newMetrics->SetNext(snapshot); snapshot = newMetrics; - rTmp = snapshot; - continue; + preTmp = tmp; + tmp = tmp->GetNext(); } - preTmp = preTmp->next; - rTmp->next = newMetrics; - rTmp = newMetrics; } - // Only lock when change head - { - std::lock_guard lock(mMutex); - // 把临时的节点加到链表里 - if (mSnapshottingHead) { - Metrics* tmpSnapHead = mSnapshottingHead; - Metrics* tmpMHead = emptyHead->next; - emptyHead->next = mSnapshottingHead; - while(tmpSnapHead) { - if (tmpSnapHead->next == NULL) { - tmpSnapHead->next = tmpMHead; - break; - } else { - tmpSnapHead = tmpSnapHead->next; - } - } - mSnapshottingHead = NULL; - } - mHead = emptyHead->next; - mSnapshotting.store(false); - delete emptyHead; + + while(toDeleteHead) { + Metrics* toDeleted = toDeleteHead; + toDeleteHead = toDeleteHead->GetNext(); + delete toDeleted; } + delete emptyHead; return snapshot; } @@ -207,10 +184,9 @@ void ReadMetrics::ReadAsLogGroup(std::map& logG ReadLock lock(mReadWriteLock); Metrics* tmp = mHead; while(tmp) { - Log* logPtr = NULL; - std::vector> labels = tmp->GetLabels(); - for (std::vector>::iterator it = labels.begin(); it != labels.end(); ++it) { - std::pair pair = *it; + Log* logPtr = nullptr; + for (auto &item: tmp->GetLabels()) { + std::pair pair = item; if (METRIC_FIELD_REGION == pair.first) { std::map::iterator iter; std::string region = pair.second; @@ -237,29 +213,31 @@ void ReadMetrics::ReadAsLogGroup(std::map& logG logGroupMap.insert(std::pair(METRIC_REGION_DEFAULT, logGroup)); } } - logPtr->set_time(time(NULL)); - for (std::vector>::iterator it = labels.begin(); it != labels.end(); ++it) { - std::pair pair = *it; + logPtr->set_time(time(nullptr)); + for (auto &item: tmp->GetLabels()) { + std::pair pair = item; Log_Content* contentPtr = logPtr->add_contents(); contentPtr->set_key(LABEL_PREFIX + pair.first); contentPtr->set_value(pair.second); } - std::vector values = tmp->GetValues(); - for (std::vector::iterator it = values.begin(); it != values.end(); ++it) { - MetricPtr counter = *it; + //std::vector values = tmp->GetValues(); + for (auto &item: tmp->GetValues()) { + MetricPtr counter = item; Log_Content* contentPtr = logPtr->add_contents(); contentPtr->set_key(VALUE_PREFIX + counter->GetName()); contentPtr->set_value(ToString(counter->GetValue())); } - tmp = tmp->next; + tmp = tmp->GetNext(); } } - +Metrics* ReadMetrics::GetHead() { + return mHead; +} void ReadMetrics::UpdateMetrics() { - Metrics* snapshot = mWriteMetrics->DoSnapshot(); + Metrics* snapshot = WriteMetrics::GetInstance()->DoSnapshot(); Metrics* toDelete; { // Only lock when change head @@ -270,7 +248,7 @@ void ReadMetrics::UpdateMetrics() { // delete old linklist while(toDelete) { Metrics* obj = toDelete; - toDelete = toDelete->next; + toDelete = toDelete->GetNext(); delete obj; } } diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index fb4bc57c87..70a63ed88c 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -1,13 +1,8 @@ #pragma once #include -#include -#include #include -#include "logger/Logger.h" -#include #include "common/Lock.h" #include "log_pb/sls_logs.pb.h" -#include "common/StringTools.h" namespace logtail { @@ -17,10 +12,8 @@ class BaseMetric { protected: std::string mName; std::atomic_long mVal; - std::atomic_long mTimestamp; public: const uint64_t GetValue() const; - const uint64_t GetTimestamp() const; const std::string GetName() const; virtual BaseMetric* CopyAndReset() = 0; }; @@ -49,12 +42,11 @@ using GaugePtr = std::shared_ptr; class Metrics { private: std::vector> mLabels; - std::vector mValues; std::atomic_bool mDeleted; - - + std::vector mValues; + Metrics* mNext = nullptr; public: - Metrics(std::vector> labels); + Metrics(const std::vector>& labels); Metrics(); void MarkDeleted(); bool IsDeleted(); @@ -62,46 +54,41 @@ class Metrics { const std::vector& GetValues() const; CounterPtr CreateCounter(const std::string Name); GaugePtr CreateGauge(const std::string Name); - - //using MetricsUniq = std::unique_ptr; - Metrics* Copy(); - Metrics* next; + Metrics* CopyAndReset(); + void SetNext(Metrics* next); + Metrics* GetNext(); }; +typedef void (*functionPointer)(Metrics*); +using MetricsPtr = std::unique_ptr; class WriteMetrics { private: WriteMetrics(); - std::atomic_bool mSnapshotting; + std::mutex mMutex; + Metrics* mHead = nullptr; public: static WriteMetrics* GetInstance() { static WriteMetrics* ptr = new WriteMetrics(); return ptr; } - Metrics* CreateMetrics(std::vector> Labels); - void DestroyMetrics(Metrics* metrics); + MetricsPtr CreateMetrics(const std::vector>& Labels); Metrics* DoSnapshot(); - Metrics* mHead = NULL; - Metrics* mSnapshottingHead = NULL; - std::mutex mMutex; - std::mutex mSnapshottingMutex; - + Metrics* GetHead(); }; class ReadMetrics { private: ReadMetrics(); + ReadWriteLock mReadWriteLock; + Metrics* mHead = nullptr; public: static ReadMetrics* GetInstance() { static ReadMetrics* ptr = new ReadMetrics(); return ptr; } void ReadAsLogGroup(std::map& logGroupMap); - // void ReadAsMap(std::map map); - // void ReadAsPrometheus(); void UpdateMetrics(); - Metrics* mHead = NULL; - WriteMetrics* mWriteMetrics; - ReadWriteLock mReadWriteLock; + Metrics* GetHead(); }; } \ No newline at end of file diff --git a/core/monitor/MetricExportor.cpp b/core/monitor/MetricExportor.cpp index cc6debdc0b..61604a3931 100644 --- a/core/monitor/MetricExportor.cpp +++ b/core/monitor/MetricExportor.cpp @@ -1,37 +1,20 @@ #include "MetricExportor.h" #include "sender/Sender.h" #include "log_pb/sls_logs.pb.h" -#include "common/LogtailCommonFlags.h" +#include "LogtailMetric.h" +#include "log_pb/sls_logs.pb.h" using namespace sls_logs; using namespace std; namespace logtail { -MetricExportor::MetricExportor() { - mSendInterval = 10; - mSnapshotInterval = 10; - mLastSendTime = 0; - mLastSnapshotTime = 0; -} +MetricExportor::MetricExportor() {} void MetricExportor::PushMetrics() { ReadMetrics::GetInstance()->UpdateMetrics(); std::map logGroupMap; ReadMetrics::GetInstance()->ReadAsLogGroup(logGroupMap); // TODO: send - std::map::iterator iter; - for (iter = logGroupMap.begin(); iter != logGroupMap.end(); iter ++) { - LOG_INFO(sLogger, ("Region", iter->first)); - sls_logs::LogGroup logGroup = iter->second; - logGroup.set_category("shennong_log_profile"); - for (int i = 0; i < logGroup.logs_size(); ++i) { - const sls_logs::Log& log = logGroup.logs(i); - for (int j = 0; j < log.contents_size(); ++j) { - const sls_logs::Log_Content& content = log.contents(j); - LOG_INFO(sLogger, ("contentKey", content.key())("contentValue", content.value())); - } - } - } } } \ No newline at end of file diff --git a/core/monitor/MetricExportor.h b/core/monitor/MetricExportor.h index 62d254c8e2..42b1202419 100644 --- a/core/monitor/MetricExportor.h +++ b/core/monitor/MetricExportor.h @@ -1,17 +1,5 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include "profile_sender/ProfileSender.h" -#include "LogtailMetric.h" -#include "logger/Logger.h" -#include "plugin/LogtailPlugin.h" -#include "ProfileSender.h" -#include "log_pb/sls_logs.pb.h" namespace logtail { @@ -25,12 +13,5 @@ class MetricExportor { private: MetricExportor(); - ~MetricExportor() {} - - ProfileSender mProfileSender; - int32_t mSendInterval; - int32_t mLastSendTime; - int32_t mSnapshotInterval; - int32_t mLastSnapshotTime; }; } \ No newline at end of file diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp index 305fc6414a..8125fe1cce 100644 --- a/core/unittest/monitor/LogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -26,80 +26,93 @@ APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateAndDeleteMetricMultiThre void ILogtailMetricUnittest::TestCreateMetric() { // create //PluginMetric* pluginMetric = new PluginMetric(); + std::vector> labels; - labels.push_back(std::make_pair("project","project1")); - labels.push_back(std::make_pair("logstore","logstore1")); - labels.push_back(std::make_pair("region","cn-hangzhou")); + labels.emplace_back(std::make_pair("project","project1")); + labels.emplace_back(std::make_pair("logstore","logstore1")); + labels.emplace_back(std::make_pair("region","cn-hangzhou")); - Metrics* fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); + auto fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); APSARA_TEST_EQUAL(fileMetric->GetLabels().size(), 3); - Metrics* fileMetric2 = WriteMetrics::GetInstance()->CreateMetrics(labels); - - Metrics* fileMetric3 = WriteMetrics::GetInstance()->CreateMetrics(labels); - CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); fileCounter->Add((uint64_t)111); APSARA_TEST_EQUAL(fileCounter->GetValue(), 111); + - CounterPtr fileCounter2 = fileMetric->CreateCounter("filed2"); - fileCounter2->Add((uint64_t)222); + { + std::vector> labels; + labels.emplace_back(std::make_pair("project","project1")); + labels.emplace_back(std::make_pair("logstore","logstore1")); + labels.emplace_back(std::make_pair("region","cn-hangzhou")); + + auto fileMetric2 = WriteMetrics::GetInstance()->CreateMetrics(labels); + CounterPtr fileCounter2 = fileMetric2->CreateCounter("filed2"); + fileCounter2->Add((uint64_t)222); + } - CounterPtr fileCounter3 = fileMetric3->CreateCounter("filed3"); - fileCounter3->Add((uint64_t)333); + { + std::vector> labels; + labels.emplace_back(std::make_pair("project","project1")); + labels.emplace_back(std::make_pair("logstore","logstore1")); + labels.emplace_back(std::make_pair("region","cn-hangzhou")); + auto fileMetric3 = WriteMetrics::GetInstance()->CreateMetrics(labels); + CounterPtr fileCounter3 = fileMetric3->CreateCounter("filed3"); + fileCounter3->Add((uint64_t)333); + } - WriteMetrics::GetInstance()->DestroyMetrics(fileMetric2); - // delete first element - WriteMetrics::GetInstance()->DestroyMetrics(fileMetric3); ReadMetrics::GetInstance()->UpdateMetrics(); - Metrics* head = ReadMetrics::GetInstance()->mHead; + Metrics* head = ReadMetrics::GetInstance()->GetHead(); int count = 0; + while(head) { - head = head->next; + head = head->GetNext(); count ++; } APSARA_TEST_EQUAL(count, 1); + MetricExportor::GetInstance()->PushMetrics(); } void createMetrics(int count) { for (int i = 0; i < count; i ++) { std::vector> labels; - labels.push_back(std::make_pair("num", std::to_string(i))); - labels.push_back(std::make_pair("count", std::to_string(count))); - labels.push_back(std::make_pair("region","cn-beijing")); - Metrics* fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); + labels.emplace_back(std::make_pair("num", std::to_string(i))); + labels.emplace_back(std::make_pair("count", std::to_string(count))); + labels.emplace_back(std::make_pair("region","cn-beijing")); + auto fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); fileCounter->Add((uint64_t)111); } } -void createAndDeleteMetrics(int count) { - for (int i = 0; i < count; i ++) { - std::vector> labels; - labels.push_back(std::make_pair("num", std::to_string(i))); - labels.push_back(std::make_pair("count", std::to_string(count))); - labels.push_back(std::make_pair("region","cn-shanghai")); - Metrics* fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); - CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); - fileCounter->Add((uint64_t)111); - //WriteMetrics::GetInstance()->DestroyMetrics(fileMetric); - } -} - void UpdateMetrics() { - ReadMetrics::GetInstance()->UpdateMetrics(); - Metrics* head = ReadMetrics::GetInstance()->mHead; - while(head) { - std::vector> labels = head->GetLabels(); - for (std::vector>::iterator it = labels.begin(); it != labels.end(); ++it) { - std::pair pair = *it; + for (int i = 0; i < 10; i ++) { + ReadMetrics::GetInstance()->UpdateMetrics(); + { + Metrics* head = WriteMetrics::GetInstance()->GetHead(); + int count = 0; + while(head) { + if (!head->IsDeleted()) { + count ++; + } + head = head->GetNext(); + } + LOG_INFO(sLogger, ("WriteCount", count)); } - head = head->next; - } + { + Metrics* head = ReadMetrics::GetInstance()->GetHead(); + int count = 0; + while(head) { + head = head->GetNext(); + count ++; + } + LOG_INFO(sLogger, ("ReadCount", count)); + } + } } void ILogtailMetricUnittest::TestCreateMetricMultiThread() { @@ -113,85 +126,103 @@ void ILogtailMetricUnittest::TestCreateMetricMultiThread() { t2.join(); t3.join(); t4.join(); - - Metrics* head = WriteMetrics::GetInstance()->mHead; + + Metrics* head = WriteMetrics::GetInstance()->GetHead(); + Metrics* tmp = head; int count = 0; - while(head) { - std::vector> labels = head->GetLabels(); - for (std::vector>::iterator it = labels.begin(); it != labels.end(); ++it) { - std::pair pair = *it; - //LOG_INFO(sLogger, ("key", pair.first)("value", pair.second)); + while(tmp != nullptr) { + for (auto& it: tmp->GetLabels()) { + LOG_INFO(sLogger, ("key", it.first)("value", it.second)); } - head = head->next; + tmp = tmp->GetNext(); count ++; } // first test left 1, 1 + (1 + 2 + 3 + 4) APSARA_TEST_EQUAL(count, 11); LOG_INFO(sLogger, ("Count", count)); - MetricExportor::GetInstance()->PushMetrics(); + + // UpdateMetrics multi time + for (int i = 0; i < 10; i ++) { + ReadMetrics::GetInstance()->UpdateMetrics(); + } + + { + Metrics* head = WriteMetrics::GetInstance()->GetHead(); + Metrics* tmp = head; + int count = 0; + while(tmp != nullptr) { + for (auto& it: tmp->GetLabels()) { + LOG_INFO(sLogger, ("key", it.first)("value", it.second)); + } + tmp = tmp->GetNext(); + count ++; + } + APSARA_TEST_EQUAL(count, 0); + } + } void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { - std::thread t1(createAndDeleteMetrics, 5); - std::thread t2(createAndDeleteMetrics, 6); + std::thread t1(createMetrics, 5); + std::thread t2(createMetrics, 6); + // create one in main thread + std::vector> labels; + labels.emplace_back(std::make_pair("project","project1")); + labels.emplace_back(std::make_pair("logstore","logstore1")); + labels.emplace_back(std::make_pair("region","cn-hangzhou")); + + auto fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); + CounterPtr fileCounter = fileMetric->CreateCounter("filed2"); + fileCounter->Add((uint64_t)1); + + // UpdateMetrics multi time while createMetrics std::thread tUpdate(UpdateMetrics); - std::thread t3(createAndDeleteMetrics, 7); - std::thread t4(createAndDeleteMetrics, 8); + std::thread t3(createMetrics, 7); + std::thread t4(createMetrics, 8); t1.join(); t2.join(); - tUpdate.join(); t3.join(); t4.join(); + tUpdate.join(); + + // final UpdateMetrics after createMetrics + ReadMetrics::GetInstance()->UpdateMetrics(); { - Metrics* head = WriteMetrics::GetInstance()->mHead; - int count = 0; - while(head) { - if (!head-> IsDeleted()) { - count ++; - } - head = head->next; - } - LOG_INFO(sLogger, ("WriteCount", count)); - } - { - Metrics* head = ReadMetrics::GetInstance()->mHead; - int count = 0; - while(head) { - head = head->next; - count ++; - } - LOG_INFO(sLogger, ("ReadCount", count)); - } - - UpdateMetrics(); - { - Metrics* head = WriteMetrics::GetInstance()->mHead; + Metrics* head = WriteMetrics::GetInstance()->GetHead(); int count = 0; while(head) { - if (!head-> IsDeleted()) { + if (!head->IsDeleted()) { count ++; } - head = head->next; + head = head->GetNext(); } - APSARA_TEST_EQUAL(count, 11); + APSARA_TEST_EQUAL(count, 1); LOG_INFO(sLogger, ("FinalWriteCount", count)); } { - Metrics* head = ReadMetrics::GetInstance()->mHead; + Metrics* head = ReadMetrics::GetInstance()->GetHead(); int count = 0; while(head) { - head = head->next; + head = head->GetNext(); count ++; } - APSARA_TEST_EQUAL(count, 11); + // only one left + APSARA_TEST_EQUAL(count, 1); + + head = ReadMetrics::GetInstance()->GetHead(); + fileCounter->Add((uint64_t)2); + // after UpdateMetrics, couter will be reset + APSARA_TEST_EQUAL(fileCounter->GetValue(), 2); + APSARA_TEST_EQUAL(head->GetLabels().size(), 3); + LOG_INFO(sLogger, ("FinalReadCount", count)); } MetricExportor::GetInstance()->PushMetrics(); From 3d551845fe86f3e22d6580cb2aa8cf9897af13a2 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Thu, 17 Aug 2023 12:03:13 +0800 Subject: [PATCH 19/39] fix cooment --- core/monitor/LogtaiMetric.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index c3e78c3a9f..e1e979ad3d 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -52,11 +52,6 @@ Metrics::Metrics(const std::vector>& labels) Metrics::Metrics() : mDeleted(false) {} -WriteMetrics::WriteMetrics() {} - -ReadMetrics::ReadMetrics() {} - - CounterPtr Metrics::CreateCounter(const std::string name) { CounterPtr counterPtr = std::make_shared(name); mValues.push_back(counterPtr); @@ -110,6 +105,9 @@ void DestroyMetrics(Metrics* metrics) { metrics->MarkDeleted(); } + +WriteMetrics::WriteMetrics() {} + MetricsPtr WriteMetrics::CreateMetrics(const std::vector>& labels) { Metrics* cur = new Metrics(std::move(labels)); std::lock_guard lock(mMutex); @@ -180,6 +178,8 @@ Metrics* WriteMetrics::DoSnapshot() { return snapshot; } +ReadMetrics::ReadMetrics() {} + void ReadMetrics::ReadAsLogGroup(std::map& logGroupMap) { ReadLock lock(mReadWriteLock); Metrics* tmp = mHead; @@ -232,9 +232,6 @@ void ReadMetrics::ReadAsLogGroup(std::map& logG } } -Metrics* ReadMetrics::GetHead() { - return mHead; -} void ReadMetrics::UpdateMetrics() { Metrics* snapshot = WriteMetrics::GetInstance()->DoSnapshot(); @@ -252,4 +249,9 @@ void ReadMetrics::UpdateMetrics() { delete obj; } } + +Metrics* ReadMetrics::GetHead() { + return mHead; +} + } \ No newline at end of file From ff73c037846992bcfdb8dd4437f2fa04e2e0ca4c Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Thu, 17 Aug 2023 13:31:43 +0800 Subject: [PATCH 20/39] fix comments --- core/monitor/LogtaiMetric.cpp | 25 ++++++++++--------- core/monitor/LogtailMetric.h | 13 +++++++--- .../monitor/LogtailMetricUnittest.cpp | 22 ++++++++-------- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index e1e979ad3d..7d92fa0072 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -98,17 +98,19 @@ void Metrics::SetNext(Metrics* next) { mNext = next; } -// mark as deleted -void DestroyMetrics(Metrics* metrics) { - // deleted is atomic_bool, no need to lock - LOG_INFO(sLogger, ("DestroyMetrics", "true")); - metrics->MarkDeleted(); +MetricsRef::MetricsRef(Metrics* metrics) : mMetrics(metrics) {} + +MetricsRef::~MetricsRef() { + mMetrics->MarkDeleted(); } +Metrics* MetricsRef::Get() { + return mMetrics; +} WriteMetrics::WriteMetrics() {} -MetricsPtr WriteMetrics::CreateMetrics(const std::vector>& labels) { +MetricsRef WriteMetrics::CreateMetrics(const std::vector>& labels) { Metrics* cur = new Metrics(std::move(labels)); std::lock_guard lock(mMutex); @@ -116,8 +118,8 @@ MetricsPtr WriteMetrics::CreateMetrics(const std::vectorSetNext(oldHead); - MetricsPtr curUnique(cur, DestroyMetrics); - return curUnique; + MetricsRef curRef(cur); + return curRef; } Metrics* WriteMetrics::GetHead() { @@ -130,13 +132,13 @@ Metrics* WriteMetrics::DoSnapshot() { Metrics* toDeleteHead = nullptr; Metrics* tmp = nullptr; - Metrics* emptyHead = new Metrics(); + Metrics emptyHead; Metrics* preTmp = nullptr; // find the first not deleted node and set as new mHead { std::lock_guard lock(mMutex); - emptyHead->SetNext(mHead); - preTmp = emptyHead; + emptyHead.SetNext(mHead); + preTmp = &emptyHead; tmp = preTmp->GetNext(); while (tmp) { if (tmp->IsDeleted()) { @@ -174,7 +176,6 @@ Metrics* WriteMetrics::DoSnapshot() { toDeleteHead = toDeleteHead->GetNext(); delete toDeleted; } - delete emptyHead; return snapshot; } diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 70a63ed88c..0df6ecdf79 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -59,8 +59,14 @@ class Metrics { Metrics* GetNext(); }; -typedef void (*functionPointer)(Metrics*); -using MetricsPtr = std::unique_ptr; +class MetricsRef { + private: + Metrics* mMetrics; + public: + MetricsRef(Metrics* metrics); + ~MetricsRef(); + Metrics* Get(); +}; class WriteMetrics { private: @@ -72,7 +78,8 @@ class WriteMetrics { static WriteMetrics* ptr = new WriteMetrics(); return ptr; } - MetricsPtr CreateMetrics(const std::vector>& Labels); + MetricsRef CreateMetrics(const std::vector>& Labels); + Metrics* DoSnapshot(); Metrics* GetHead(); }; diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp index 8125fe1cce..24202860ba 100644 --- a/core/unittest/monitor/LogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -32,11 +32,11 @@ void ILogtailMetricUnittest::TestCreateMetric() { labels.emplace_back(std::make_pair("logstore","logstore1")); labels.emplace_back(std::make_pair("region","cn-hangzhou")); - auto fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); - APSARA_TEST_EQUAL(fileMetric->GetLabels().size(), 3); + MetricsRef fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); + APSARA_TEST_EQUAL(fileMetric.Get()->GetLabels().size(), 3); - CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); + CounterPtr fileCounter = fileMetric.Get()->CreateCounter("filed1"); fileCounter->Add((uint64_t)111); APSARA_TEST_EQUAL(fileCounter->GetValue(), 111); @@ -47,8 +47,8 @@ void ILogtailMetricUnittest::TestCreateMetric() { labels.emplace_back(std::make_pair("logstore","logstore1")); labels.emplace_back(std::make_pair("region","cn-hangzhou")); - auto fileMetric2 = WriteMetrics::GetInstance()->CreateMetrics(labels); - CounterPtr fileCounter2 = fileMetric2->CreateCounter("filed2"); + MetricsRef fileMetric2 = WriteMetrics::GetInstance()->CreateMetrics(labels); + CounterPtr fileCounter2 = fileMetric2.Get()->CreateCounter("filed2"); fileCounter2->Add((uint64_t)222); } @@ -58,8 +58,8 @@ void ILogtailMetricUnittest::TestCreateMetric() { labels.emplace_back(std::make_pair("logstore","logstore1")); labels.emplace_back(std::make_pair("region","cn-hangzhou")); - auto fileMetric3 = WriteMetrics::GetInstance()->CreateMetrics(labels); - CounterPtr fileCounter3 = fileMetric3->CreateCounter("filed3"); + MetricsRef fileMetric3 = WriteMetrics::GetInstance()->CreateMetrics(labels); + CounterPtr fileCounter3 = fileMetric3.Get()->CreateCounter("filed3"); fileCounter3->Add((uint64_t)333); } @@ -83,8 +83,8 @@ void createMetrics(int count) { labels.emplace_back(std::make_pair("num", std::to_string(i))); labels.emplace_back(std::make_pair("count", std::to_string(count))); labels.emplace_back(std::make_pair("region","cn-beijing")); - auto fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); - CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); + MetricsRef fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); + CounterPtr fileCounter = fileMetric.Get()->CreateCounter("filed1"); fileCounter->Add((uint64_t)111); } } @@ -175,8 +175,8 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { labels.emplace_back(std::make_pair("logstore","logstore1")); labels.emplace_back(std::make_pair("region","cn-hangzhou")); - auto fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); - CounterPtr fileCounter = fileMetric->CreateCounter("filed2"); + MetricsRef fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); + CounterPtr fileCounter = fileMetric.Get()->CreateCounter("filed2"); fileCounter->Add((uint64_t)1); // UpdateMetrics multi time while createMetrics From e22d5aa1d6f5c8b47b9c2ab1eb876cd687cf7150 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Mon, 21 Aug 2023 09:21:12 +0800 Subject: [PATCH 21/39] refine code --- core/monitor/LogtaiMetric.cpp | 49 +++++++++++-------- core/monitor/LogtailMetric.h | 9 ++-- core/monitor/MetricExportor.cpp | 3 +- .../monitor/LogtailMetricUnittest.cpp | 26 +++++++--- 4 files changed, 53 insertions(+), 34 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index 7d92fa0072..91da163e49 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -2,6 +2,8 @@ #include "common/StringTools.h" #include "MetricConstants.h" #include "logger/Logger.h" +#include "common/TimeUtil.h" +#include "app_config/AppConfig.h" using namespace sls_logs; @@ -98,10 +100,18 @@ void Metrics::SetNext(Metrics* next) { mNext = next; } -MetricsRef::MetricsRef(Metrics* metrics) : mMetrics(metrics) {} +MetricsRef::MetricsRef() {} MetricsRef::~MetricsRef() { - mMetrics->MarkDeleted(); + if (mMetrics) { + mMetrics->MarkDeleted(); + } +} + +void MetricsRef::Init(const std::vector>& Labels) { + if (!mMetrics) { + mMetrics = WriteMetrics::GetInstance()->CreateMetrics(Labels); + } } Metrics* MetricsRef::Get() { @@ -110,7 +120,7 @@ Metrics* MetricsRef::Get() { WriteMetrics::WriteMetrics() {} -MetricsRef WriteMetrics::CreateMetrics(const std::vector>& labels) { +Metrics* WriteMetrics::CreateMetrics(const std::vector>& labels) { Metrics* cur = new Metrics(std::move(labels)); std::lock_guard lock(mMutex); @@ -118,8 +128,7 @@ MetricsRef WriteMetrics::CreateMetrics(const std::vectorSetNext(oldHead); - MetricsRef curRef(cur); - return curRef; + return cur; } Metrics* WriteMetrics::GetHead() { @@ -181,7 +190,7 @@ Metrics* WriteMetrics::DoSnapshot() { ReadMetrics::ReadMetrics() {} -void ReadMetrics::ReadAsLogGroup(std::map& logGroupMap) { +void ReadMetrics::ReadAsLogGroup(std::map& logGroupMap) { ReadLock lock(mReadWriteLock); Metrics* tmp = mHead; while(tmp) { @@ -189,32 +198,33 @@ void ReadMetrics::ReadAsLogGroup(std::map& logG for (auto &item: tmp->GetLabels()) { std::pair pair = item; if (METRIC_FIELD_REGION == pair.first) { - std::map::iterator iter; + std::map::iterator iter; std::string region = pair.second; iter = logGroupMap.find(region); if (iter != logGroupMap.end()) { - sls_logs::LogGroup logGroup = iter->second; - logPtr = logGroup.add_logs(); + sls_logs::LogGroup* logGroup = iter->second; + logPtr = logGroup->add_logs(); } else { - sls_logs::LogGroup logGroup; - logPtr = logGroup.add_logs(); - logGroupMap.insert(std::pair(region, logGroup)); + sls_logs::LogGroup* logGroup = new sls_logs::LogGroup(); + logPtr = logGroup->add_logs(); + logGroupMap.insert(std::pair(region, logGroup)); } } } if (!logPtr) { - std::map::iterator iter; + std::map::iterator iter; iter = logGroupMap.find(METRIC_REGION_DEFAULT); if (iter != logGroupMap.end()) { - sls_logs::LogGroup logGroup = iter->second; - logPtr = logGroup.add_logs(); + sls_logs::LogGroup* logGroup = iter->second; + logPtr = logGroup->add_logs(); } else { - sls_logs::LogGroup logGroup; - logPtr = logGroup.add_logs(); - logGroupMap.insert(std::pair(METRIC_REGION_DEFAULT, logGroup)); + sls_logs::LogGroup* logGroup = new sls_logs::LogGroup(); + logPtr = logGroup->add_logs(); + logGroupMap.insert(std::pair(METRIC_REGION_DEFAULT, logGroup)); } } - logPtr->set_time(time(nullptr)); + auto now = GetCurrentLogtailTime(); + SetLogTime(logPtr, AppConfig::GetInstance()->EnableLogTimeAutoAdjust() ? now.tv_sec + GetTimeDelta() : now.tv_sec, now.tv_nsec); for (auto &item: tmp->GetLabels()) { std::pair pair = item; Log_Content* contentPtr = logPtr->add_contents(); @@ -222,7 +232,6 @@ void ReadMetrics::ReadAsLogGroup(std::map& logG contentPtr->set_value(pair.second); } - //std::vector values = tmp->GetValues(); for (auto &item: tmp->GetValues()) { MetricPtr counter = item; Log_Content* contentPtr = logPtr->add_contents(); diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 0df6ecdf79..dbb49f261c 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -61,10 +61,11 @@ class Metrics { class MetricsRef { private: - Metrics* mMetrics; + Metrics* mMetrics = nullptr; public: - MetricsRef(Metrics* metrics); + MetricsRef(); ~MetricsRef(); + void Init(const std::vector>& Labels); Metrics* Get(); }; @@ -78,7 +79,7 @@ class WriteMetrics { static WriteMetrics* ptr = new WriteMetrics(); return ptr; } - MetricsRef CreateMetrics(const std::vector>& Labels); + Metrics* CreateMetrics(const std::vector>& Labels); Metrics* DoSnapshot(); Metrics* GetHead(); @@ -94,7 +95,7 @@ class ReadMetrics { static ReadMetrics* ptr = new ReadMetrics(); return ptr; } - void ReadAsLogGroup(std::map& logGroupMap); + void ReadAsLogGroup(std::map& logGroupMap); void UpdateMetrics(); Metrics* GetHead(); }; diff --git a/core/monitor/MetricExportor.cpp b/core/monitor/MetricExportor.cpp index 61604a3931..1573631277 100644 --- a/core/monitor/MetricExportor.cpp +++ b/core/monitor/MetricExportor.cpp @@ -13,8 +13,7 @@ MetricExportor::MetricExportor() {} void MetricExportor::PushMetrics() { ReadMetrics::GetInstance()->UpdateMetrics(); - std::map logGroupMap; - ReadMetrics::GetInstance()->ReadAsLogGroup(logGroupMap); + // TODO: send } } \ No newline at end of file diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp index 24202860ba..d5e7c99d07 100644 --- a/core/unittest/monitor/LogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -32,7 +32,8 @@ void ILogtailMetricUnittest::TestCreateMetric() { labels.emplace_back(std::make_pair("logstore","logstore1")); labels.emplace_back(std::make_pair("region","cn-hangzhou")); - MetricsRef fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); + MetricsRef fileMetric; + fileMetric.Init(labels); APSARA_TEST_EQUAL(fileMetric.Get()->GetLabels().size(), 3); @@ -47,7 +48,8 @@ void ILogtailMetricUnittest::TestCreateMetric() { labels.emplace_back(std::make_pair("logstore","logstore1")); labels.emplace_back(std::make_pair("region","cn-hangzhou")); - MetricsRef fileMetric2 = WriteMetrics::GetInstance()->CreateMetrics(labels); + MetricsRef fileMetric2; + fileMetric2.Init(labels); CounterPtr fileCounter2 = fileMetric2.Get()->CreateCounter("filed2"); fileCounter2->Add((uint64_t)222); } @@ -57,8 +59,8 @@ void ILogtailMetricUnittest::TestCreateMetric() { labels.emplace_back(std::make_pair("project","project1")); labels.emplace_back(std::make_pair("logstore","logstore1")); labels.emplace_back(std::make_pair("region","cn-hangzhou")); - - MetricsRef fileMetric3 = WriteMetrics::GetInstance()->CreateMetrics(labels); + MetricsRef fileMetric3; + fileMetric3.Init(labels); CounterPtr fileCounter3 = fileMetric3.Get()->CreateCounter("filed3"); fileCounter3->Add((uint64_t)333); } @@ -77,13 +79,20 @@ void ILogtailMetricUnittest::TestCreateMetric() { MetricExportor::GetInstance()->PushMetrics(); } +void PushMetrics() { + for (int i = 0; i < 10; i ++) { + MetricExportor::GetInstance()->PushMetrics(); + } +} + void createMetrics(int count) { for (int i = 0; i < count; i ++) { std::vector> labels; labels.emplace_back(std::make_pair("num", std::to_string(i))); labels.emplace_back(std::make_pair("count", std::to_string(count))); labels.emplace_back(std::make_pair("region","cn-beijing")); - MetricsRef fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); + MetricsRef fileMetric; + fileMetric.Init(labels); CounterPtr fileCounter = fileMetric.Get()->CreateCounter("filed1"); fileCounter->Add((uint64_t)111); } @@ -175,21 +184,22 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { labels.emplace_back(std::make_pair("logstore","logstore1")); labels.emplace_back(std::make_pair("region","cn-hangzhou")); - MetricsRef fileMetric = WriteMetrics::GetInstance()->CreateMetrics(labels); + MetricsRef fileMetric; + fileMetric.Init(labels); CounterPtr fileCounter = fileMetric.Get()->CreateCounter("filed2"); fileCounter->Add((uint64_t)1); // UpdateMetrics multi time while createMetrics - std::thread tUpdate(UpdateMetrics); + std::thread tUpdate(PushMetrics); std::thread t3(createMetrics, 7); std::thread t4(createMetrics, 8); t1.join(); t2.join(); + tUpdate.join(); t3.join(); t4.join(); - tUpdate.join(); // final UpdateMetrics after createMetrics From 7a725d8028efbded1e6bede132dc30463d0bc040 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Mon, 21 Aug 2023 17:44:52 +0800 Subject: [PATCH 22/39] refine code --- core/monitor/LogtaiMetric.cpp | 66 +++++----- core/monitor/LogtailMetric.h | 29 +++-- .../monitor/LogtailMetricUnittest.cpp | 122 +++++++++++++----- 3 files changed, 143 insertions(+), 74 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index 91da163e49..bdaf241b6b 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -15,53 +15,43 @@ const uint64_t BaseMetric::GetValue() const { return mVal; } -const std::string BaseMetric::GetName() const { +const std::string& BaseMetric::GetName() const { return mName; } -Counter::Counter(std::string name) { - mName = name; - mVal = (uint64_t)0; -} +Counter::Counter(const std::string &name, uint64_t val = 0) : BaseMetric(name, val) {} Counter* Counter::CopyAndReset() { - Counter* counter = new Counter(mName); - counter->mVal = mVal.exchange(0); - return counter; + return new Counter(mName, mVal.exchange(0)); } -void Counter::Add(uint64_t value) { +void Counter::SetValue(uint64_t value) { mVal += value; } -Gauge::Gauge(std::string name) { - mName = name; - mVal = (uint64_t)0; -} +Gauge::Gauge(const std::string &name, uint64_t val = 0) : BaseMetric(name, val) {} Gauge* Gauge::CopyAndReset() { - Gauge* gauge = new Gauge(mName); - gauge->mVal = mVal.exchange(0); - return gauge; + return new Gauge(mName, mVal.exchange(0)); } -void Gauge::Set(uint64_t value) { +void Gauge::SetValue(uint64_t value) { mVal = value; } -Metrics::Metrics(const std::vector>& labels) : mLabels(std::move(labels)), mDeleted(false) {} +Metrics::Metrics(const std::vector>&& labels) : mLabels(labels), mDeleted(false) {} Metrics::Metrics() : mDeleted(false) {} -CounterPtr Metrics::CreateCounter(const std::string name) { - CounterPtr counterPtr = std::make_shared(name); +MetricPtr Metrics::CreateCounter(const std::string& name) { + MetricPtr counterPtr = std::make_shared(name); mValues.push_back(counterPtr); return counterPtr; } -GaugePtr Metrics::CreateGauge(const std::string name) { - GaugePtr gaugePtr = std::make_shared(name); +MetricPtr Metrics::CreateGauge(const std::string& name) { + MetricPtr gaugePtr = std::make_shared(name); mValues.push_back(gaugePtr); return gaugePtr; } @@ -84,7 +74,7 @@ const std::vector& Metrics::GetValues() const { Metrics* Metrics::CopyAndReset() { std::vector> newLabels(mLabels); - Metrics* metrics = new Metrics(newLabels); + Metrics* metrics = new Metrics(std::move(newLabels)); for (auto &item: mValues) { MetricPtr newPtr(item->CopyAndReset()); metrics->mValues.push_back(newPtr); @@ -110,7 +100,7 @@ MetricsRef::~MetricsRef() { void MetricsRef::Init(const std::vector>& Labels) { if (!mMetrics) { - mMetrics = WriteMetrics::GetInstance()->CreateMetrics(Labels); + mMetrics = WriteMetrics::GetInstance()->CreateMetrics(std::move(Labels)); } } @@ -149,6 +139,7 @@ Metrics* WriteMetrics::DoSnapshot() { emptyHead.SetNext(mHead); preTmp = &emptyHead; tmp = preTmp->GetNext(); + bool findHead = false; while (tmp) { if (tmp->IsDeleted()) { preTmp->SetNext(tmp->GetNext()); @@ -156,15 +147,32 @@ Metrics* WriteMetrics::DoSnapshot() { toDeleteHead = tmp; tmp = preTmp->GetNext(); } else { - mHead = tmp; - break; + Metrics* newMetrics = tmp->CopyAndReset(); + newMetrics->SetNext(snapshot); + snapshot = newMetrics; + // find head + if (!findHead) { + mHead = tmp; + preTmp = tmp; + tmp = tmp->GetNext(); + + mHead->SetNext(nullptr); + findHead = true; + // find head next + } else { + mHead->SetNext(tmp); + preTmp = tmp; + tmp = tmp->GetNext(); + break; + } } } - if (!tmp) { + // if no undeleted node, set null to mHead + if (!findHead) { mHead = nullptr; } - } - + } + while (tmp) { if (tmp->IsDeleted()) { preTmp->SetNext(tmp->GetNext()); diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index dbb49f261c..b815620688 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -13,31 +13,34 @@ class BaseMetric { std::string mName; std::atomic_long mVal; public: + BaseMetric(const std::string &name, uint64_t val) { + mName = name; + mVal = val; + }; + virtual ~BaseMetric() {}; const uint64_t GetValue() const; - const std::string GetName() const; + const std::string& GetName() const; + virtual void SetValue(uint64_t val) = 0; virtual BaseMetric* CopyAndReset() = 0; }; class Gauge : public BaseMetric { public: - Gauge(const std::string name); - void Set(uint64_t val); - Gauge* CopyAndReset(); + Gauge(const std::string &name, uint64_t val); + void SetValue(uint64_t val) override; + Gauge* CopyAndReset() override; }; class Counter : public BaseMetric { public: - Counter(const std::string name); - void Add(uint64_t val); - Counter* CopyAndReset(); + Counter(const std::string &name, uint64_t val); + void SetValue(uint64_t val) override; + Counter* CopyAndReset() override; }; using MetricPtr = std::shared_ptr; -using CounterPtr = std::shared_ptr; -using GaugePtr = std::shared_ptr; - class Metrics { private: @@ -46,14 +49,14 @@ class Metrics { std::vector mValues; Metrics* mNext = nullptr; public: - Metrics(const std::vector>& labels); + Metrics(const std::vector>&& labels); Metrics(); void MarkDeleted(); bool IsDeleted(); const std::vector>& GetLabels() const; const std::vector& GetValues() const; - CounterPtr CreateCounter(const std::string Name); - GaugePtr CreateGauge(const std::string Name); + MetricPtr CreateCounter(const std::string& Name); + MetricPtr CreateGauge(const std::string& Name); Metrics* CopyAndReset(); void SetNext(Metrics* next); Metrics* GetNext(); diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp index d5e7c99d07..92e2f15fbc 100644 --- a/core/unittest/monitor/LogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -11,6 +11,10 @@ namespace logtail { + +static std::atomic_bool running(true); + + class ILogtailMetricUnittest : public ::testing::Test { public: void TestCreateMetric(); @@ -19,8 +23,8 @@ class ILogtailMetricUnittest : public ::testing::Test { }; APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateMetric, 0); -APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateMetricMultiThread, 0); -APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateAndDeleteMetricMultiThread, 0); +APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateMetricMultiThread, 1); +APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateAndDeleteMetricMultiThread, 2); void ILogtailMetricUnittest::TestCreateMetric() { @@ -33,12 +37,12 @@ void ILogtailMetricUnittest::TestCreateMetric() { labels.emplace_back(std::make_pair("region","cn-hangzhou")); MetricsRef fileMetric; - fileMetric.Init(labels); + fileMetric.Init(std::move(labels)); APSARA_TEST_EQUAL(fileMetric.Get()->GetLabels().size(), 3); - CounterPtr fileCounter = fileMetric.Get()->CreateCounter("filed1"); - fileCounter->Add((uint64_t)111); + MetricPtr fileCounter = fileMetric.Get()->CreateCounter("filed1"); + fileCounter->SetValue((uint64_t)111); APSARA_TEST_EQUAL(fileCounter->GetValue(), 111); @@ -49,9 +53,9 @@ void ILogtailMetricUnittest::TestCreateMetric() { labels.emplace_back(std::make_pair("region","cn-hangzhou")); MetricsRef fileMetric2; - fileMetric2.Init(labels); - CounterPtr fileCounter2 = fileMetric2.Get()->CreateCounter("filed2"); - fileCounter2->Add((uint64_t)222); + fileMetric2.Init(std::move(labels)); + MetricPtr fileCounter2 = fileMetric2.Get()->CreateCounter("filed2"); + fileCounter2->SetValue((uint64_t)222); } { @@ -61,8 +65,8 @@ void ILogtailMetricUnittest::TestCreateMetric() { labels.emplace_back(std::make_pair("region","cn-hangzhou")); MetricsRef fileMetric3; fileMetric3.Init(labels); - CounterPtr fileCounter3 = fileMetric3.Get()->CreateCounter("filed3"); - fileCounter3->Add((uint64_t)333); + MetricPtr fileCounter3 = fileMetric3.Get()->CreateCounter("filed3"); + fileCounter3->SetValue((uint64_t)333); } ReadMetrics::GetInstance()->UpdateMetrics(); @@ -81,6 +85,7 @@ void ILogtailMetricUnittest::TestCreateMetric() { void PushMetrics() { for (int i = 0; i < 10; i ++) { + LOG_INFO(sLogger, ("PushMetricsCount", i)); MetricExportor::GetInstance()->PushMetrics(); } } @@ -92,9 +97,9 @@ void createMetrics(int count) { labels.emplace_back(std::make_pair("count", std::to_string(count))); labels.emplace_back(std::make_pair("region","cn-beijing")); MetricsRef fileMetric; - fileMetric.Init(labels); - CounterPtr fileCounter = fileMetric.Get()->CreateCounter("filed1"); - fileCounter->Add((uint64_t)111); + fileMetric.Init(std::move(labels)); + MetricPtr fileCounter = fileMetric.Get()->CreateCounter("filed1"); + fileCounter->SetValue((uint64_t)111); } } @@ -124,6 +129,29 @@ void UpdateMetrics() { } } + + +void createRunningMetrics() { + std::vector> labels; + labels.emplace_back(std::make_pair("region","cn-beijing")); + MetricsRef fileMetric; + + + fileMetric.Init(std::move(labels)); + MetricPtr fileCounter = fileMetric.Get()->CreateCounter("filed1"); + + LOG_INFO(sLogger, ("createRunningMetrics", fileMetric.Get())); + + int count = 0; + while(running) { + fileCounter->SetValue((uint64_t)1); + count++; + sleep(1); + } +} + + + void ILogtailMetricUnittest::TestCreateMetricMultiThread() { std::thread t1(createMetrics, 1); @@ -174,20 +202,12 @@ void ILogtailMetricUnittest::TestCreateMetricMultiThread() { void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { - + std::thread t1(createMetrics, 5); std::thread t2(createMetrics, 6); - // create one in main thread - std::vector> labels; - labels.emplace_back(std::make_pair("project","project1")); - labels.emplace_back(std::make_pair("logstore","logstore1")); - labels.emplace_back(std::make_pair("region","cn-hangzhou")); - - MetricsRef fileMetric; - fileMetric.Init(labels); - CounterPtr fileCounter = fileMetric.Get()->CreateCounter("filed2"); - fileCounter->Add((uint64_t)1); + std::thread tRunning1(createRunningMetrics); + std::thread tRunning2(createRunningMetrics); // UpdateMetrics multi time while createMetrics std::thread tUpdate(PushMetrics); @@ -201,7 +221,6 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { t3.join(); t4.join(); - // final UpdateMetrics after createMetrics ReadMetrics::GetInstance()->UpdateMetrics(); { @@ -213,10 +232,11 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { } head = head->GetNext(); } - APSARA_TEST_EQUAL(count, 1); + APSARA_TEST_EQUAL(count, 2); LOG_INFO(sLogger, ("FinalWriteCount", count)); } + { Metrics* head = ReadMetrics::GetInstance()->GetHead(); int count = 0; @@ -224,18 +244,56 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { head = head->GetNext(); count ++; } - // only one left - APSARA_TEST_EQUAL(count, 1); + APSARA_TEST_EQUAL(count, 2); head = ReadMetrics::GetInstance()->GetHead(); - fileCounter->Add((uint64_t)2); - // after UpdateMetrics, couter will be reset - APSARA_TEST_EQUAL(fileCounter->GetValue(), 2); - APSARA_TEST_EQUAL(head->GetLabels().size(), 3); + APSARA_TEST_EQUAL(head->GetLabels().size(), 1); + + LOG_INFO(sLogger, ("FinalReadCount", count)); + } + + std::thread tRunning3(createRunningMetrics); + std::thread tRunning4(createRunningMetrics); + + usleep(100); + + // final UpdateMetrics after createMetrics + ReadMetrics::GetInstance()->UpdateMetrics(); + { + Metrics* head = WriteMetrics::GetInstance()->GetHead(); + int count = 0; + while(head) { + if (!head->IsDeleted()) { + count ++; + } + head = head->GetNext(); + } + APSARA_TEST_EQUAL(count, 4); + LOG_INFO(sLogger, ("FinalWriteCount", count)); + + } + + { + Metrics* head = ReadMetrics::GetInstance()->GetHead(); + int count = 0; + while(head) { + head = head->GetNext(); + count ++; + } + APSARA_TEST_EQUAL(count, 4); LOG_INFO(sLogger, ("FinalReadCount", count)); } + + MetricExportor::GetInstance()->PushMetrics(); + running.store(false); + tRunning1.join(); + tRunning2.join(); + tRunning3.join(); + tRunning4.join(); + + LOG_INFO(sLogger, ("end", "test")); } From b0892d867be96bb3064ce86e1216b8a190cc539a Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Mon, 21 Aug 2023 17:57:17 +0800 Subject: [PATCH 23/39] refine code --- core/monitor/LogtaiMetric.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index bdaf241b6b..5a6c446991 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -46,13 +46,13 @@ Metrics::Metrics() : mDeleted(false) {} MetricPtr Metrics::CreateCounter(const std::string& name) { MetricPtr counterPtr = std::make_shared(name); - mValues.push_back(counterPtr); + mValues.emplace_back(counterPtr); return counterPtr; } MetricPtr Metrics::CreateGauge(const std::string& name) { MetricPtr gaugePtr = std::make_shared(name); - mValues.push_back(gaugePtr); + mValues.emplace_back(gaugePtr); return gaugePtr; } @@ -77,7 +77,7 @@ Metrics* Metrics::CopyAndReset() { Metrics* metrics = new Metrics(std::move(newLabels)); for (auto &item: mValues) { MetricPtr newPtr(item->CopyAndReset()); - metrics->mValues.push_back(newPtr); + metrics->mValues.emplace_back(newPtr); } return metrics; } From 2fb349f65dbf0d7da8c01f8e780e8777de39271a Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Tue, 22 Aug 2023 09:28:21 +0800 Subject: [PATCH 24/39] refine code --- core/controller/EventDispatcherBase.cpp | 2 ++ core/monitor/LogtaiMetric.cpp | 6 ++++ core/monitor/MetricConstants.h | 4 +++ core/monitor/MetricExportor.cpp | 28 ++++++++++++++++--- core/monitor/MetricExportor.h | 7 ++++- .../monitor/LogtailMetricUnittest.cpp | 6 ++-- 6 files changed, 45 insertions(+), 8 deletions(-) diff --git a/core/controller/EventDispatcherBase.cpp b/core/controller/EventDispatcherBase.cpp index 073fee285f..5722a0ee50 100644 --- a/core/controller/EventDispatcherBase.cpp +++ b/core/controller/EventDispatcherBase.cpp @@ -55,6 +55,7 @@ #include "monitor/LogtailAlarm.h" #include "monitor/LogIntegrity.h" #include "monitor/LogLineCount.h" +#include "monitor/MetricExportor.h" #include "log_pb/metric.pb.h" #include "log_pb/sls_logs.pb.h" #include "checkpoint/CheckPointManager.h" @@ -874,6 +875,7 @@ bool EventDispatcherBase::Dispatch() { DumpCheckPointPeriod(curTime); if (curTime - lastCheckDir >= INT32_FLAG(main_loop_check_interval)) { LogFileProfiler::GetInstance()->SendProfileData(); + MetricExportor::GetInstance()->PushMetrics(false); #if defined(__linux__) CheckShennong(); #endif diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index 5a6c446991..c39e4acb94 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -246,6 +246,12 @@ void ReadMetrics::ReadAsLogGroup(std::map& log contentPtr->set_key(VALUE_PREFIX + counter->GetName()); contentPtr->set_value(ToString(counter->GetValue())); } + // set default key + { + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(METRIC_TOPIC_FIELD_NAME); + contentPtr->set_value(METRIC_TOPIC_TYPE); + } tmp = tmp->GetNext(); } } diff --git a/core/monitor/MetricConstants.h b/core/monitor/MetricConstants.h index b565c71613..803830e89c 100644 --- a/core/monitor/MetricConstants.h +++ b/core/monitor/MetricConstants.h @@ -21,6 +21,10 @@ namespace logtail { const std::string METRIC_FIELD_REGION = "region"; const std::string METRIC_REGION_DEFAULT = "default"; +const std::string METRIC_SLS_LOGSTORE_NAME = "shennong_log_profile"; +const std::string METRIC_TOPIC_TYPE = "logtail_metric"; +const std::string METRIC_TOPIC_FIELD_NAME = "__topic__"; + const std::string LABEL_PREFIX = "label."; const std::string VALUE_PREFIX = "value."; diff --git a/core/monitor/MetricExportor.cpp b/core/monitor/MetricExportor.cpp index 1573631277..89d4711709 100644 --- a/core/monitor/MetricExportor.cpp +++ b/core/monitor/MetricExportor.cpp @@ -2,18 +2,38 @@ #include "sender/Sender.h" #include "log_pb/sls_logs.pb.h" #include "LogtailMetric.h" -#include "log_pb/sls_logs.pb.h" +#include "config_manager/ConfigManager.h" +#include "LogFileProfiler.h" +#include "MetricConstants.h" using namespace sls_logs; using namespace std; namespace logtail { -MetricExportor::MetricExportor() {} +MetricExportor::MetricExportor() : mSendInterval(60), mLastSendTime(time(NULL) - (rand() % (mSendInterval / 10)) * 10) {} -void MetricExportor::PushMetrics() { +void MetricExportor::PushMetrics(bool forceSend) { + int32_t curTime = time(NULL); + if (!forceSend && (curTime - mLastSendTime < mSendInterval)) { + return; + } ReadMetrics::GetInstance()->UpdateMetrics(); - // TODO: send + std::map logGroupMap; + ReadMetrics::GetInstance()->ReadAsLogGroup(logGroupMap); + + std::map::iterator iter; + for (iter = logGroupMap.begin(); iter != logGroupMap.end(); iter ++) { + sls_logs::LogGroup* logGroup = iter->second; + logGroup->set_category(METRIC_SLS_LOGSTORE_NAME); + logGroup->set_source(LogFileProfiler::mIpAddr); + if (METRIC_REGION_DEFAULT == iter->first) { + mProfileSender.SendToProfileProject(ConfigManager::GetInstance()->GetDefaultProfileRegion(), *logGroup); + } else { + mProfileSender.SendToProfileProject(iter->first, *logGroup); + } + delete logGroup; + } } } \ No newline at end of file diff --git a/core/monitor/MetricExportor.h b/core/monitor/MetricExportor.h index 42b1202419..cbcf956d55 100644 --- a/core/monitor/MetricExportor.h +++ b/core/monitor/MetricExportor.h @@ -1,5 +1,7 @@ #pragma once +#include "profile_sender/ProfileSender.h" + namespace logtail { @@ -9,9 +11,12 @@ class MetricExportor { static MetricExportor* ptr = new MetricExportor(); return ptr; } - void PushMetrics(); + void PushMetrics(bool forceSend); private: MetricExportor(); + ProfileSender mProfileSender; + int32_t mSendInterval; + int32_t mLastSendTime; }; } \ No newline at end of file diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp index 92e2f15fbc..c30cbdbc54 100644 --- a/core/unittest/monitor/LogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -80,13 +80,13 @@ void ILogtailMetricUnittest::TestCreateMetric() { } APSARA_TEST_EQUAL(count, 1); - MetricExportor::GetInstance()->PushMetrics(); + MetricExportor::GetInstance()->PushMetrics(true); } void PushMetrics() { for (int i = 0; i < 10; i ++) { LOG_INFO(sLogger, ("PushMetricsCount", i)); - MetricExportor::GetInstance()->PushMetrics(); + MetricExportor::GetInstance()->PushMetrics(true); } } @@ -286,7 +286,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { } - MetricExportor::GetInstance()->PushMetrics(); + MetricExportor::GetInstance()->PushMetrics(true); running.store(false); tRunning1.join(); tRunning2.join(); From ad2c7ee2f4bed306fe3994a3d782f4ef27244a3a Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Tue, 22 Aug 2023 10:52:39 +0800 Subject: [PATCH 25/39] refine code --- core/monitor/LogtaiMetric.cpp | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index c39e4acb94..9145d280b9 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -133,6 +133,9 @@ Metrics* WriteMetrics::DoSnapshot() { Metrics emptyHead; Metrics* preTmp = nullptr; + + Metrics* tmpHead = nullptr; + Metrics* tmpHeadNext = nullptr; // find the first not deleted node and set as new mHead { std::lock_guard lock(mMutex); @@ -147,9 +150,6 @@ Metrics* WriteMetrics::DoSnapshot() { toDeleteHead = tmp; tmp = preTmp->GetNext(); } else { - Metrics* newMetrics = tmp->CopyAndReset(); - newMetrics->SetNext(snapshot); - snapshot = newMetrics; // find head if (!findHead) { mHead = tmp; @@ -170,8 +170,24 @@ Metrics* WriteMetrics::DoSnapshot() { // if no undeleted node, set null to mHead if (!findHead) { mHead = nullptr; + } else { + tmpHead = mHead; + tmpHeadNext = mHead->GetNext(); } - } + } + + // copy head + if (tmpHead) { + Metrics* newMetrics = tmpHead->CopyAndReset(); + newMetrics->SetNext(snapshot); + snapshot = newMetrics; + } + // copy head next + if (tmpHeadNext) { + Metrics* newMetrics = tmpHead->CopyAndReset(); + newMetrics->SetNext(snapshot); + snapshot = newMetrics; + } while (tmp) { if (tmp->IsDeleted()) { From 6944fce4a893dabae099b5375bdf0b12903e1143 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Wed, 23 Aug 2023 10:57:35 +0800 Subject: [PATCH 26/39] refine code --- core/monitor/LogtaiMetric.cpp | 186 +++++++-------- core/monitor/LogtailMetric.h | 163 ++++++------- .../monitor/LogtailMetricUnittest.cpp | 214 +++++++++--------- 3 files changed, 284 insertions(+), 279 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index 9145d280b9..c7d94dcdaa 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -11,15 +11,16 @@ using namespace sls_logs; namespace logtail { -const uint64_t BaseMetric::GetValue() const { +const uint64_t MetricNameValue::GetValue() const { return mVal; } -const std::string& BaseMetric::GetName() const { +const std::string& MetricNameValue::GetName() const { return mName; } -Counter::Counter(const std::string &name, uint64_t val = 0) : BaseMetric(name, val) {} +Counter::Counter(const std::string& name, uint64_t val = 0) : MetricNameValue(name, val) { +} Counter* Counter::CopyAndReset() { return new Counter(mName, mVal.exchange(0)); @@ -29,7 +30,8 @@ void Counter::SetValue(uint64_t value) { mVal += value; } -Gauge::Gauge(const std::string &name, uint64_t val = 0) : BaseMetric(name, val) {} +Gauge::Gauge(const std::string& name, uint64_t val = 0) : MetricNameValue(name, val) { +} Gauge* Gauge::CopyAndReset() { return new Gauge(mName, mVal.exchange(0)); @@ -39,106 +41,113 @@ void Gauge::SetValue(uint64_t value) { mVal = value; } +MetricsRecord::MetricsRecord(LabelsPtr labels) : mLabels(labels), mDeleted(false) { +} -Metrics::Metrics(const std::vector>&& labels) : mLabels(labels), mDeleted(false) {} - -Metrics::Metrics() : mDeleted(false) {} +MetricsRecord::MetricsRecord() : mDeleted(false) { +} -MetricPtr Metrics::CreateCounter(const std::string& name) { - MetricPtr counterPtr = std::make_shared(name); +MetricNameValuePtr MetricsRecord::CreateCounter(const std::string& name) { + MetricNameValuePtr counterPtr = std::make_shared(name); mValues.emplace_back(counterPtr); return counterPtr; } -MetricPtr Metrics::CreateGauge(const std::string& name) { - MetricPtr gaugePtr = std::make_shared(name); +MetricNameValuePtr MetricsRecord::CreateGauge(const std::string& name) { + MetricNameValuePtr gaugePtr = std::make_shared(name); mValues.emplace_back(gaugePtr); return gaugePtr; } -void Metrics::MarkDeleted() { +void MetricsRecord::MarkDeleted() { mDeleted = true; } -bool Metrics::IsDeleted() { +bool MetricsRecord::IsDeleted() { return mDeleted; } -const std::vector>& Metrics::GetLabels() const { +LabelsPtr MetricsRecord::GetLabels() { return mLabels; } -const std::vector& Metrics::GetValues() const { +const std::vector& MetricsRecord::GetValues() const { return mValues; } -Metrics* Metrics::CopyAndReset() { - std::vector> newLabels(mLabels); - Metrics* metrics = new Metrics(std::move(newLabels)); - for (auto &item: mValues) { - MetricPtr newPtr(item->CopyAndReset()); +MetricsRecord* MetricsRecord::CopyAndReset() { + MetricsRecord* metrics = new MetricsRecord(mLabels); + for (auto& item : mValues) { + MetricNameValuePtr newPtr(item->CopyAndReset()); metrics->mValues.emplace_back(newPtr); } return metrics; } -Metrics* Metrics::GetNext() { +MetricsRecord* MetricsRecord::GetNext() { return mNext; } -void Metrics::SetNext(Metrics* next) { - mNext = next; +void MetricsRecord::SetNext(MetricsRecord* next) { + mNext = next; } -MetricsRef::MetricsRef() {} +MetricsRecordRef::MetricsRecordRef() { +} -MetricsRef::~MetricsRef() { +MetricsRecordRef::~MetricsRecordRef() { if (mMetrics) { mMetrics->MarkDeleted(); } } -void MetricsRef::Init(const std::vector>& Labels) { +void MetricsRecordRef::Init(const std::vector>& labels) { if (!mMetrics) { - mMetrics = WriteMetrics::GetInstance()->CreateMetrics(std::move(Labels)); + mMetrics = WriteMetrics::GetInstance()->CreateMetrics( + std::make_shared>>(labels)); } } -Metrics* MetricsRef::Get() { +MetricsRecord* MetricsRecordRef::operator->() const { return mMetrics; } -WriteMetrics::WriteMetrics() {} +WriteMetrics::WriteMetrics() { +} -Metrics* WriteMetrics::CreateMetrics(const std::vector>& labels) { - Metrics* cur = new Metrics(std::move(labels)); - std::lock_guard lock(mMutex); +WriteMetrics::~WriteMetrics() { + while (mHead) { + MetricsRecord* toDeleted = mHead; + mHead = mHead->GetNext(); + delete toDeleted; + } +} - Metrics* oldHead = mHead; +MetricsRecord* WriteMetrics::CreateMetrics(LabelsPtr labels) { + MetricsRecord* cur = new MetricsRecord(labels); + std::lock_guard lock(mMutex); + cur->SetNext(mHead); mHead = cur; - mHead->SetNext(oldHead); - return cur; } -Metrics* WriteMetrics::GetHead() { +MetricsRecord* WriteMetrics::GetHead() { return mHead; } -Metrics* WriteMetrics::DoSnapshot() { +MetricsRecord* WriteMetrics::DoSnapshot() { // new read head - Metrics* snapshot = nullptr; - Metrics* toDeleteHead = nullptr; - Metrics* tmp = nullptr; - - Metrics emptyHead; - Metrics* preTmp = nullptr; - - Metrics* tmpHead = nullptr; - Metrics* tmpHeadNext = nullptr; - // find the first not deleted node and set as new mHead + MetricsRecord* snapshot = nullptr; + MetricsRecord* toDeleteHead = nullptr; + MetricsRecord* tmp = nullptr; + + MetricsRecord emptyHead; + MetricsRecord* preTmp = nullptr; + + MetricsRecord* tmpHead = nullptr; + // find the first undeleted node and set as new mHead { - std::lock_guard lock(mMutex); + std::lock_guard lock(mMutex); emptyHead.SetNext(mHead); preTmp = &emptyHead; tmp = preTmp->GetNext(); @@ -151,40 +160,23 @@ Metrics* WriteMetrics::DoSnapshot() { tmp = preTmp->GetNext(); } else { // find head - if (!findHead) { - mHead = tmp; - preTmp = tmp; - tmp = tmp->GetNext(); - - mHead->SetNext(nullptr); - findHead = true; - // find head next - } else { - mHead->SetNext(tmp); - preTmp = tmp; - tmp = tmp->GetNext(); - break; - } + mHead = tmp; + preTmp = tmp; + tmp = tmp->GetNext(); + findHead = true; + break; } } // if no undeleted node, set null to mHead if (!findHead) { mHead = nullptr; - } else { - tmpHead = mHead; - tmpHeadNext = mHead->GetNext(); } + tmpHead = mHead; } // copy head if (tmpHead) { - Metrics* newMetrics = tmpHead->CopyAndReset(); - newMetrics->SetNext(snapshot); - snapshot = newMetrics; - } - // copy head next - if (tmpHeadNext) { - Metrics* newMetrics = tmpHead->CopyAndReset(); + MetricsRecord* newMetrics = tmpHead->CopyAndReset(); newMetrics->SetNext(snapshot); snapshot = newMetrics; } @@ -196,7 +188,7 @@ Metrics* WriteMetrics::DoSnapshot() { toDeleteHead = tmp; tmp = preTmp->GetNext(); } else { - Metrics* newMetrics = tmp->CopyAndReset(); + MetricsRecord* newMetrics = tmp->CopyAndReset(); newMetrics->SetNext(snapshot); snapshot = newMetrics; preTmp = tmp; @@ -204,23 +196,33 @@ Metrics* WriteMetrics::DoSnapshot() { } } - while(toDeleteHead) { - Metrics* toDeleted = toDeleteHead; + while (toDeleteHead) { + MetricsRecord* toDeleted = toDeleteHead; toDeleteHead = toDeleteHead->GetNext(); delete toDeleted; } return snapshot; } -ReadMetrics::ReadMetrics() {} +ReadMetrics::ReadMetrics() { +} + +ReadMetrics::~ReadMetrics() { + while (mHead) { + MetricsRecord* toDeleted = mHead; + mHead = mHead->GetNext(); + delete toDeleted; + } +} void ReadMetrics::ReadAsLogGroup(std::map& logGroupMap) { ReadLock lock(mReadWriteLock); - Metrics* tmp = mHead; - while(tmp) { + MetricsRecord* tmp = mHead; + while (tmp) { Log* logPtr = nullptr; - for (auto &item: tmp->GetLabels()) { - std::pair pair = item; + // for (auto &item: tmp->GetLabels()) { + for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { + std::pair pair = *item; if (METRIC_FIELD_REGION == pair.first) { std::map::iterator iter; std::string region = pair.second; @@ -248,16 +250,18 @@ void ReadMetrics::ReadAsLogGroup(std::map& log } } auto now = GetCurrentLogtailTime(); - SetLogTime(logPtr, AppConfig::GetInstance()->EnableLogTimeAutoAdjust() ? now.tv_sec + GetTimeDelta() : now.tv_sec, now.tv_nsec); - for (auto &item: tmp->GetLabels()) { - std::pair pair = item; + SetLogTime(logPtr, + AppConfig::GetInstance()->EnableLogTimeAutoAdjust() ? now.tv_sec + GetTimeDelta() : now.tv_sec, + now.tv_nsec); + for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { + std::pair pair = *item; Log_Content* contentPtr = logPtr->add_contents(); contentPtr->set_key(LABEL_PREFIX + pair.first); contentPtr->set_value(pair.second); } - for (auto &item: tmp->GetValues()) { - MetricPtr counter = item; + for (auto& item : tmp->GetValues()) { + MetricNameValuePtr counter = item; Log_Content* contentPtr = logPtr->add_contents(); contentPtr->set_key(VALUE_PREFIX + counter->GetName()); contentPtr->set_value(ToString(counter->GetValue())); @@ -274,8 +278,8 @@ void ReadMetrics::ReadAsLogGroup(std::map& log void ReadMetrics::UpdateMetrics() { - Metrics* snapshot = WriteMetrics::GetInstance()->DoSnapshot(); - Metrics* toDelete; + MetricsRecord* snapshot = WriteMetrics::GetInstance()->DoSnapshot(); + MetricsRecord* toDelete; { // Only lock when change head WriteLock lock(mReadWriteLock); @@ -283,15 +287,15 @@ void ReadMetrics::UpdateMetrics() { mHead = snapshot; } // delete old linklist - while(toDelete) { - Metrics* obj = toDelete; + while (toDelete) { + MetricsRecord* obj = toDelete; toDelete = toDelete->GetNext(); delete obj; } } -Metrics* ReadMetrics::GetHead() { +MetricsRecord* ReadMetrics::GetHead() { return mHead; } -} \ No newline at end of file +} // namespace logtail \ No newline at end of file diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index b815620688..05c4c02101 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -8,98 +8,105 @@ namespace logtail { -class BaseMetric { - protected: - std::string mName; - std::atomic_long mVal; - public: - BaseMetric(const std::string &name, uint64_t val) { - mName = name; - mVal = val; - }; - virtual ~BaseMetric() {}; - const uint64_t GetValue() const; - const std::string& GetName() const; - virtual void SetValue(uint64_t val) = 0; - virtual BaseMetric* CopyAndReset() = 0; +class MetricNameValue { +protected: + std::string mName; + std::atomic_long mVal; + +public: + MetricNameValue(const std::string& name, uint64_t val) { + mName = name; + mVal = val; + }; + virtual ~MetricNameValue(){}; + const uint64_t GetValue() const; + const std::string& GetName() const; + virtual void SetValue(uint64_t val) = 0; + virtual MetricNameValue* CopyAndReset() = 0; }; -class Gauge : public BaseMetric { - public: - Gauge(const std::string &name, uint64_t val); - void SetValue(uint64_t val) override; - Gauge* CopyAndReset() override; +class Gauge : public MetricNameValue { +public: + Gauge(const std::string& name, uint64_t val); + void SetValue(uint64_t val) override; + Gauge* CopyAndReset() override; }; -class Counter : public BaseMetric { - public: - Counter(const std::string &name, uint64_t val); - void SetValue(uint64_t val) override; - Counter* CopyAndReset() override; +class Counter : public MetricNameValue { +public: + Counter(const std::string& name, uint64_t val); + void SetValue(uint64_t val) override; + Counter* CopyAndReset() override; }; -using MetricPtr = std::shared_ptr; - -class Metrics { - private: - std::vector> mLabels; - std::atomic_bool mDeleted; - std::vector mValues; - Metrics* mNext = nullptr; - public: - Metrics(const std::vector>&& labels); - Metrics(); - void MarkDeleted(); - bool IsDeleted(); - const std::vector>& GetLabels() const; - const std::vector& GetValues() const; - MetricPtr CreateCounter(const std::string& Name); - MetricPtr CreateGauge(const std::string& Name); - Metrics* CopyAndReset(); - void SetNext(Metrics* next); - Metrics* GetNext(); +using MetricNameValuePtr = std::shared_ptr; +using LabelsPtr = std::shared_ptr>>; + +class MetricsRecord { +private: + LabelsPtr mLabels; + std::atomic_bool mDeleted; + std::vector mValues; + MetricsRecord* mNext = nullptr; + +public: + MetricsRecord(LabelsPtr labels); + MetricsRecord(); + void MarkDeleted(); + bool IsDeleted(); + LabelsPtr GetLabels(); + const std::vector& GetValues() const; + MetricNameValuePtr CreateCounter(const std::string& Name); + MetricNameValuePtr CreateGauge(const std::string& Name); + MetricsRecord* CopyAndReset(); + void SetNext(MetricsRecord* next); + MetricsRecord* GetNext(); }; -class MetricsRef { - private: - Metrics* mMetrics = nullptr; - public: - MetricsRef(); - ~MetricsRef(); - void Init(const std::vector>& Labels); - Metrics* Get(); +class MetricsRecordRef { +private: + MetricsRecord* mMetrics = nullptr; + +public: + MetricsRecordRef(); + ~MetricsRecordRef(); + void Init(const std::vector>& labels); + MetricsRecord* operator->() const; }; class WriteMetrics { - private: - WriteMetrics(); - std::mutex mMutex; - Metrics* mHead = nullptr; - public: - static WriteMetrics* GetInstance() { - static WriteMetrics* ptr = new WriteMetrics(); - return ptr; - } - Metrics* CreateMetrics(const std::vector>& Labels); - - Metrics* DoSnapshot(); - Metrics* GetHead(); +private: + WriteMetrics(); + std::mutex mMutex; + MetricsRecord* mHead = nullptr; + +public: + ~WriteMetrics(); + static WriteMetrics* GetInstance() { + static WriteMetrics* ptr = new WriteMetrics(); + return ptr; + } + MetricsRecord* CreateMetrics(LabelsPtr Labels); + MetricsRecord* DoSnapshot(); + MetricsRecord* GetHead(); }; class ReadMetrics { - private: - ReadMetrics(); - ReadWriteLock mReadWriteLock; - Metrics* mHead = nullptr; - public: - static ReadMetrics* GetInstance() { - static ReadMetrics* ptr = new ReadMetrics(); - return ptr; - } - void ReadAsLogGroup(std::map& logGroupMap); - void UpdateMetrics(); - Metrics* GetHead(); +private: + ReadMetrics(); + ReadWriteLock mReadWriteLock; + MetricsRecord* mHead = nullptr; + +public: + ~ReadMetrics(); + static ReadMetrics* GetInstance() { + static ReadMetrics* ptr = new ReadMetrics(); + return ptr; + } + void ReadAsLogGroup(std::map& logGroupMap); + void UpdateMetrics(); + MetricsRecord* GetHead(); }; -} \ No newline at end of file +} // namespace logtail \ No newline at end of file diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp index c30cbdbc54..af4cc79d61 100644 --- a/core/unittest/monitor/LogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -29,180 +29,176 @@ APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateAndDeleteMetricMultiThre void ILogtailMetricUnittest::TestCreateMetric() { // create - //PluginMetric* pluginMetric = new PluginMetric(); - + // PluginMetric* pluginMetric = new PluginMetric(); + std::vector> labels; - labels.emplace_back(std::make_pair("project","project1")); - labels.emplace_back(std::make_pair("logstore","logstore1")); - labels.emplace_back(std::make_pair("region","cn-hangzhou")); + labels.emplace_back(std::make_pair("project", "project1")); + labels.emplace_back(std::make_pair("logstore", "logstore1")); + labels.emplace_back(std::make_pair("region", "cn-hangzhou")); + + MetricsRecordRef fileMetric; + fileMetric.Init(labels); + APSARA_TEST_EQUAL(fileMetric->GetLabels()->size(), 3); - MetricsRef fileMetric; - fileMetric.Init(std::move(labels)); - APSARA_TEST_EQUAL(fileMetric.Get()->GetLabels().size(), 3); + MetricNameValuePtr fileCounter = fileMetric->CreateCounter("filed1"); + fileCounter->SetValue(111UL); + APSARA_TEST_EQUAL(fileCounter->GetValue(), 111); - MetricPtr fileCounter = fileMetric.Get()->CreateCounter("filed1"); - fileCounter->SetValue((uint64_t)111); - APSARA_TEST_EQUAL(fileCounter->GetValue(), 111); - { std::vector> labels; - labels.emplace_back(std::make_pair("project","project1")); - labels.emplace_back(std::make_pair("logstore","logstore1")); - labels.emplace_back(std::make_pair("region","cn-hangzhou")); - - MetricsRef fileMetric2; - fileMetric2.Init(std::move(labels)); - MetricPtr fileCounter2 = fileMetric2.Get()->CreateCounter("filed2"); - fileCounter2->SetValue((uint64_t)222); + labels.emplace_back(std::make_pair("project", "project1")); + labels.emplace_back(std::make_pair("logstore", "logstore1")); + labels.emplace_back(std::make_pair("region", "cn-hangzhou")); + + MetricsRecordRef fileMetric2; + fileMetric2.Init(labels); + MetricNameValuePtr fileCounter2 = fileMetric2->CreateCounter("filed2"); + fileCounter2->SetValue(222UL); } - + { std::vector> labels; - labels.emplace_back(std::make_pair("project","project1")); - labels.emplace_back(std::make_pair("logstore","logstore1")); - labels.emplace_back(std::make_pair("region","cn-hangzhou")); - MetricsRef fileMetric3; + labels.emplace_back(std::make_pair("project", "project1")); + labels.emplace_back(std::make_pair("logstore", "logstore1")); + labels.emplace_back(std::make_pair("region", "cn-hangzhou")); + MetricsRecordRef fileMetric3; fileMetric3.Init(labels); - MetricPtr fileCounter3 = fileMetric3.Get()->CreateCounter("filed3"); - fileCounter3->SetValue((uint64_t)333); + MetricNameValuePtr fileCounter3 = fileMetric3->CreateCounter("filed3"); + fileCounter3->SetValue(333UL); } ReadMetrics::GetInstance()->UpdateMetrics(); - - Metrics* head = ReadMetrics::GetInstance()->GetHead(); + + MetricsRecord* head = ReadMetrics::GetInstance()->GetHead(); int count = 0; - while(head) { + while (head) { head = head->GetNext(); - count ++; - } - APSARA_TEST_EQUAL(count, 1); + count++; + } + APSARA_TEST_EQUAL(count, 1); MetricExportor::GetInstance()->PushMetrics(true); } void PushMetrics() { - for (int i = 0; i < 10; i ++) { + for (int i = 0; i < 10; i++) { LOG_INFO(sLogger, ("PushMetricsCount", i)); MetricExportor::GetInstance()->PushMetrics(true); } } void createMetrics(int count) { - for (int i = 0; i < count; i ++) { + for (int i = 0; i < count; i++) { std::vector> labels; labels.emplace_back(std::make_pair("num", std::to_string(i))); labels.emplace_back(std::make_pair("count", std::to_string(count))); - labels.emplace_back(std::make_pair("region","cn-beijing")); - MetricsRef fileMetric; - fileMetric.Init(std::move(labels)); - MetricPtr fileCounter = fileMetric.Get()->CreateCounter("filed1"); - fileCounter->SetValue((uint64_t)111); + labels.emplace_back(std::make_pair("region", "cn-beijing")); + MetricsRecordRef fileMetric; + fileMetric.Init(labels); + MetricNameValuePtr fileCounter = fileMetric->CreateCounter("filed1"); + fileCounter->SetValue(111UL); } } void UpdateMetrics() { - for (int i = 0; i < 10; i ++) { + for (int i = 0; i < 10; i++) { ReadMetrics::GetInstance()->UpdateMetrics(); { - Metrics* head = WriteMetrics::GetInstance()->GetHead(); + MetricsRecord* head = WriteMetrics::GetInstance()->GetHead(); int count = 0; - while(head) { + while (head) { if (!head->IsDeleted()) { - count ++; + count++; } head = head->GetNext(); - } + } LOG_INFO(sLogger, ("WriteCount", count)); } { - Metrics* head = ReadMetrics::GetInstance()->GetHead(); + MetricsRecord* head = ReadMetrics::GetInstance()->GetHead(); int count = 0; - while(head) { + while (head) { head = head->GetNext(); - count ++; - } + count++; + } LOG_INFO(sLogger, ("ReadCount", count)); } } } - void createRunningMetrics() { std::vector> labels; - labels.emplace_back(std::make_pair("region","cn-beijing")); - MetricsRef fileMetric; - + labels.emplace_back(std::make_pair("region", "cn-beijing")); + MetricsRecordRef fileMetric; - fileMetric.Init(std::move(labels)); - MetricPtr fileCounter = fileMetric.Get()->CreateCounter("filed1"); - LOG_INFO(sLogger, ("createRunningMetrics", fileMetric.Get())); + fileMetric.Init(labels); + MetricNameValuePtr fileCounter = fileMetric->CreateCounter("filed1"); + + // LOG_INFO(sLogger, ("createRunningMetrics", fileMetric)); int count = 0; - while(running) { - fileCounter->SetValue((uint64_t)1); + while (running) { + fileCounter->SetValue(1); count++; sleep(1); } } - void ILogtailMetricUnittest::TestCreateMetricMultiThread() { - std::thread t1(createMetrics, 1); std::thread t2(createMetrics, 2); std::thread t3(createMetrics, 3); std::thread t4(createMetrics, 4); - + t1.join(); t2.join(); t3.join(); t4.join(); - - Metrics* head = WriteMetrics::GetInstance()->GetHead(); - Metrics* tmp = head; + + MetricsRecord* head = WriteMetrics::GetInstance()->GetHead(); + MetricsRecord* tmp = head; int count = 0; - while(tmp != nullptr) { - for (auto& it: tmp->GetLabels()) { - LOG_INFO(sLogger, ("key", it.first)("value", it.second)); + while (tmp != nullptr) { + for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { + std::pair pair = *item; + LOG_INFO(sLogger, ("key", pair.first)("value", pair.second)); } tmp = tmp->GetNext(); - count ++; - } + count++; + } // first test left 1, 1 + (1 + 2 + 3 + 4) - APSARA_TEST_EQUAL(count, 11); + APSARA_TEST_EQUAL(count, 11); LOG_INFO(sLogger, ("Count", count)); - + // UpdateMetrics multi time - for (int i = 0; i < 10; i ++) { + for (int i = 0; i < 10; i++) { ReadMetrics::GetInstance()->UpdateMetrics(); } { - Metrics* head = WriteMetrics::GetInstance()->GetHead(); - Metrics* tmp = head; + MetricsRecord* head = WriteMetrics::GetInstance()->GetHead(); + MetricsRecord* tmp = head; int count = 0; - while(tmp != nullptr) { - for (auto& it: tmp->GetLabels()) { - LOG_INFO(sLogger, ("key", it.first)("value", it.second)); + while (tmp != nullptr) { + for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { + std::pair pair = *item; + LOG_INFO(sLogger, ("key", pair.first)("value", pair.second)); } tmp = tmp->GetNext(); - count ++; - } - APSARA_TEST_EQUAL(count, 0); + count++; + } + APSARA_TEST_EQUAL(count, 0); } - } - void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { - std::thread t1(createMetrics, 5); std::thread t2(createMetrics, 6); @@ -214,41 +210,40 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { std::thread t3(createMetrics, 7); std::thread t4(createMetrics, 8); - + t1.join(); t2.join(); tUpdate.join(); t3.join(); t4.join(); - + // final UpdateMetrics after createMetrics ReadMetrics::GetInstance()->UpdateMetrics(); { - Metrics* head = WriteMetrics::GetInstance()->GetHead(); + MetricsRecord* head = WriteMetrics::GetInstance()->GetHead(); int count = 0; - while(head) { + while (head) { if (!head->IsDeleted()) { - count ++; + count++; } head = head->GetNext(); - } - APSARA_TEST_EQUAL(count, 2); + } + APSARA_TEST_EQUAL(count, 2); LOG_INFO(sLogger, ("FinalWriteCount", count)); - } { - Metrics* head = ReadMetrics::GetInstance()->GetHead(); + MetricsRecord* head = ReadMetrics::GetInstance()->GetHead(); int count = 0; - while(head) { + while (head) { head = head->GetNext(); - count ++; - } - APSARA_TEST_EQUAL(count, 2); + count++; + } + APSARA_TEST_EQUAL(count, 2); head = ReadMetrics::GetInstance()->GetHead(); - APSARA_TEST_EQUAL(head->GetLabels().size(), 1); + APSARA_TEST_EQUAL(head->GetLabels()->size(), 1); LOG_INFO(sLogger, ("FinalReadCount", count)); } @@ -261,31 +256,30 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { // final UpdateMetrics after createMetrics ReadMetrics::GetInstance()->UpdateMetrics(); { - Metrics* head = WriteMetrics::GetInstance()->GetHead(); + MetricsRecord* head = WriteMetrics::GetInstance()->GetHead(); int count = 0; - while(head) { + while (head) { if (!head->IsDeleted()) { - count ++; + count++; } head = head->GetNext(); - } - APSARA_TEST_EQUAL(count, 4); + } + APSARA_TEST_EQUAL(count, 4); LOG_INFO(sLogger, ("FinalWriteCount", count)); - } { - Metrics* head = ReadMetrics::GetInstance()->GetHead(); + MetricsRecord* head = ReadMetrics::GetInstance()->GetHead(); int count = 0; - while(head) { + while (head) { head = head->GetNext(); - count ++; - } - APSARA_TEST_EQUAL(count, 4); + count++; + } + APSARA_TEST_EQUAL(count, 4); LOG_INFO(sLogger, ("FinalReadCount", count)); } - + MetricExportor::GetInstance()->PushMetrics(true); running.store(false); tRunning1.join(); @@ -297,7 +291,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { } -}// namespace logtail +} // namespace logtail int main(int argc, char** argv) { logtail::Logger::Instance().InitGlobalLoggers(); From ea21800ba05f9bddf669710284b68dda9f908e9c Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Wed, 23 Aug 2023 11:47:10 +0800 Subject: [PATCH 27/39] refine code --- core/monitor/LogtaiMetric.cpp | 10 +++++----- core/monitor/LogtailMetric.h | 6 +++--- core/unittest/monitor/LogtailMetricUnittest.cpp | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index c7d94dcdaa..33bfe17561 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -67,11 +67,11 @@ bool MetricsRecord::IsDeleted() { return mDeleted; } -LabelsPtr MetricsRecord::GetLabels() { +const LabelsPtr& MetricsRecord::GetLabels() const { return mLabels; } -const std::vector& MetricsRecord::GetValues() const { +const std::vector& MetricsRecord::GetMetricNameValues() const { return mValues; } @@ -103,7 +103,7 @@ MetricsRecordRef::~MetricsRecordRef() { void MetricsRecordRef::Init(const std::vector>& labels) { if (!mMetrics) { - mMetrics = WriteMetrics::GetInstance()->CreateMetrics( + mMetrics = WriteMetrics::GetInstance()->CreateMetricsRecords( std::make_shared>>(labels)); } } @@ -123,7 +123,7 @@ WriteMetrics::~WriteMetrics() { } } -MetricsRecord* WriteMetrics::CreateMetrics(LabelsPtr labels) { +MetricsRecord* WriteMetrics::CreateMetricsRecords(LabelsPtr labels) { MetricsRecord* cur = new MetricsRecord(labels); std::lock_guard lock(mMutex); cur->SetNext(mHead); @@ -260,7 +260,7 @@ void ReadMetrics::ReadAsLogGroup(std::map& log contentPtr->set_value(pair.second); } - for (auto& item : tmp->GetValues()) { + for (auto& item : tmp->GetMetricNameValues()) { MetricNameValuePtr counter = item; Log_Content* contentPtr = logPtr->add_contents(); contentPtr->set_key(VALUE_PREFIX + counter->GetName()); diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 05c4c02101..87939621a3 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -56,8 +56,8 @@ class MetricsRecord { MetricsRecord(); void MarkDeleted(); bool IsDeleted(); - LabelsPtr GetLabels(); - const std::vector& GetValues() const; + const LabelsPtr& GetLabels() const; + const std::vector& GetMetricNameValues() const; MetricNameValuePtr CreateCounter(const std::string& Name); MetricNameValuePtr CreateGauge(const std::string& Name); MetricsRecord* CopyAndReset(); @@ -88,7 +88,7 @@ class WriteMetrics { static WriteMetrics* ptr = new WriteMetrics(); return ptr; } - MetricsRecord* CreateMetrics(LabelsPtr Labels); + MetricsRecord* CreateMetricsRecords(LabelsPtr Labels); MetricsRecord* DoSnapshot(); MetricsRecord* GetHead(); }; diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp index af4cc79d61..94e1b3b851 100644 --- a/core/unittest/monitor/LogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -138,14 +138,10 @@ void createRunningMetrics() { fileMetric.Init(labels); MetricNameValuePtr fileCounter = fileMetric->CreateCounter("filed1"); - - // LOG_INFO(sLogger, ("createRunningMetrics", fileMetric)); - int count = 0; while (running) { - fileCounter->SetValue(1); - count++; - sleep(1); + fileCounter->SetValue(111); + usleep(100000); } } @@ -287,6 +283,10 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { tRunning3.join(); tRunning4.join(); + MetricsRecord* head = ReadMetrics::GetInstance()->GetHead(); + std::vector nameValues = head->GetMetricNameValues(); + + APSARA_TEST_EQUAL(nameValues.size(), 1); LOG_INFO(sLogger, ("end", "test")); } From 6ca91dc114b132cdc3be99c877f4687d160f2add Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Wed, 23 Aug 2023 14:17:47 +0800 Subject: [PATCH 28/39] refine code --- core/monitor/LogtaiMetric.cpp | 38 +- core/monitor/LogtailMetric.h | 14 +- .../monitor/LogtailMetricUnittest.cpp | 344 ++++++++++-------- 3 files changed, 217 insertions(+), 179 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index 33bfe17561..a6a6c03ac6 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -63,7 +63,7 @@ void MetricsRecord::MarkDeleted() { mDeleted = true; } -bool MetricsRecord::IsDeleted() { +const bool MetricsRecord::IsDeleted() const { return mDeleted; } @@ -84,7 +84,7 @@ MetricsRecord* MetricsRecord::CopyAndReset() { return metrics; } -MetricsRecord* MetricsRecord::GetNext() { +MetricsRecord* MetricsRecord::GetNext() const { return mNext; } @@ -116,11 +116,7 @@ WriteMetrics::WriteMetrics() { } WriteMetrics::~WriteMetrics() { - while (mHead) { - MetricsRecord* toDeleted = mHead; - mHead = mHead->GetNext(); - delete toDeleted; - } + Clear(); } MetricsRecord* WriteMetrics::CreateMetricsRecords(LabelsPtr labels) { @@ -131,10 +127,18 @@ MetricsRecord* WriteMetrics::CreateMetricsRecords(LabelsPtr labels) { return cur; } -MetricsRecord* WriteMetrics::GetHead() { +MetricsRecord* WriteMetrics::GetHead() const { return mHead; } +void WriteMetrics::Clear() { + while (mHead) { + MetricsRecord* toDeleted = mHead; + mHead = mHead->GetNext(); + delete toDeleted; + } +} + MetricsRecord* WriteMetrics::DoSnapshot() { // new read head MetricsRecord* snapshot = nullptr; @@ -208,14 +212,10 @@ ReadMetrics::ReadMetrics() { } ReadMetrics::~ReadMetrics() { - while (mHead) { - MetricsRecord* toDeleted = mHead; - mHead = mHead->GetNext(); - delete toDeleted; - } + Clear(); } -void ReadMetrics::ReadAsLogGroup(std::map& logGroupMap) { +void ReadMetrics::ReadAsLogGroup(std::map& logGroupMap) const { ReadLock lock(mReadWriteLock); MetricsRecord* tmp = mHead; while (tmp) { @@ -294,8 +294,16 @@ void ReadMetrics::UpdateMetrics() { } } -MetricsRecord* ReadMetrics::GetHead() { +MetricsRecord* ReadMetrics::GetHead() const { return mHead; } +void ReadMetrics::Clear() { + while (mHead) { + MetricsRecord* toDeleted = mHead; + mHead = mHead->GetNext(); + delete toDeleted; + } +} + } // namespace logtail \ No newline at end of file diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 87939621a3..4f52709f91 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -55,14 +55,14 @@ class MetricsRecord { MetricsRecord(LabelsPtr labels); MetricsRecord(); void MarkDeleted(); - bool IsDeleted(); + const bool IsDeleted() const; const LabelsPtr& GetLabels() const; const std::vector& GetMetricNameValues() const; MetricNameValuePtr CreateCounter(const std::string& Name); MetricNameValuePtr CreateGauge(const std::string& Name); MetricsRecord* CopyAndReset(); void SetNext(MetricsRecord* next); - MetricsRecord* GetNext(); + MetricsRecord* GetNext() const; }; class MetricsRecordRef { @@ -90,13 +90,14 @@ class WriteMetrics { } MetricsRecord* CreateMetricsRecords(LabelsPtr Labels); MetricsRecord* DoSnapshot(); - MetricsRecord* GetHead(); + MetricsRecord* GetHead() const; + void Clear(); }; class ReadMetrics { private: ReadMetrics(); - ReadWriteLock mReadWriteLock; + mutable ReadWriteLock mReadWriteLock; MetricsRecord* mHead = nullptr; public: @@ -105,8 +106,9 @@ class ReadMetrics { static ReadMetrics* ptr = new ReadMetrics(); return ptr; } - void ReadAsLogGroup(std::map& logGroupMap); + void ReadAsLogGroup(std::map& logGroupMap) const; void UpdateMetrics(); - MetricsRecord* GetHead(); + MetricsRecord* GetHead() const; + void Clear(); }; } // namespace logtail \ No newline at end of file diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp index 94e1b3b851..3535d353af 100644 --- a/core/unittest/monitor/LogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -17,20 +17,26 @@ static std::atomic_bool running(true); class ILogtailMetricUnittest : public ::testing::Test { public: - void TestCreateMetric(); - void TestCreateMetricMultiThread(); - void TestCreateAndDeleteMetricMultiThread(); + void SetUp() {} + + void TearDown() { + ReadMetrics::GetInstance()->Clear(); + WriteMetrics::GetInstance()->Clear(); + } + + void TestCreateMetricAutoDelete(); + void TestCreateMetricAutoDeleteMultiThread(); + void TestCreateAndDeleteMetric(); + }; -APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateMetric, 0); -APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateMetricMultiThread, 1); -APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateAndDeleteMetricMultiThread, 2); +APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateMetricAutoDelete, 0); +APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateMetricAutoDeleteMultiThread, 1); +APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateAndDeleteMetric, 2); -void ILogtailMetricUnittest::TestCreateMetric() { - // create - // PluginMetric* pluginMetric = new PluginMetric(); +void ILogtailMetricUnittest::TestCreateMetricAutoDelete() { std::vector> labels; labels.emplace_back(std::make_pair("project", "project1")); labels.emplace_back(std::make_pair("logstore", "logstore1")); @@ -43,9 +49,31 @@ void ILogtailMetricUnittest::TestCreateMetric() { MetricNameValuePtr fileCounter = fileMetric->CreateCounter("filed1"); fileCounter->SetValue(111UL); - APSARA_TEST_EQUAL(fileCounter->GetValue(), 111); + fileCounter->SetValue(111UL); + APSARA_TEST_EQUAL(fileCounter->GetValue(), 222); + + MetricExportor::GetInstance()->PushMetrics(true); + // assert WriteMetrics count + MetricsRecord* tmp = WriteMetrics::GetInstance()->GetHead(); + int count = 0; + while (tmp) { + tmp = tmp->GetNext(); + count++; + } + APSARA_TEST_EQUAL(count, 1); + + + // assert ReadMetrics count + tmp = ReadMetrics::GetInstance()->GetHead(); + count = 0; + while (tmp) { + tmp = tmp->GetNext(); + count++; + } + APSARA_TEST_EQUAL(count, 1); + // mock create in other class, should be delete after { std::vector> labels; labels.emplace_back(std::make_pair("project", "project1")); @@ -69,18 +97,26 @@ void ILogtailMetricUnittest::TestCreateMetric() { fileCounter3->SetValue(333UL); } - ReadMetrics::GetInstance()->UpdateMetrics(); - - MetricsRecord* head = ReadMetrics::GetInstance()->GetHead(); - int count = 0; + MetricExportor::GetInstance()->PushMetrics(true); - while (head) { - head = head->GetNext(); + // assert WriteMetrics count + tmp = WriteMetrics::GetInstance()->GetHead(); + count = 0; + while (tmp) { + tmp = tmp->GetNext(); count++; } APSARA_TEST_EQUAL(count, 1); - MetricExportor::GetInstance()->PushMetrics(true); + + // assert ReadMetrics count + tmp = ReadMetrics::GetInstance()->GetHead(); + count = 0; + while (tmp) { + tmp = tmp->GetNext(); + count++; + } + APSARA_TEST_EQUAL(count, 1); } void PushMetrics() { @@ -103,50 +139,7 @@ void createMetrics(int count) { } } -void UpdateMetrics() { - for (int i = 0; i < 10; i++) { - ReadMetrics::GetInstance()->UpdateMetrics(); - { - MetricsRecord* head = WriteMetrics::GetInstance()->GetHead(); - int count = 0; - while (head) { - if (!head->IsDeleted()) { - count++; - } - head = head->GetNext(); - } - LOG_INFO(sLogger, ("WriteCount", count)); - } - { - MetricsRecord* head = ReadMetrics::GetInstance()->GetHead(); - int count = 0; - while (head) { - head = head->GetNext(); - count++; - } - LOG_INFO(sLogger, ("ReadCount", count)); - } - } -} - - -void createRunningMetrics() { - std::vector> labels; - labels.emplace_back(std::make_pair("region", "cn-beijing")); - MetricsRecordRef fileMetric; - - - fileMetric.Init(labels); - MetricNameValuePtr fileCounter = fileMetric->CreateCounter("filed1"); - int count = 0; - while (running) { - fileCounter->SetValue(111); - usleep(100000); - } -} - - -void ILogtailMetricUnittest::TestCreateMetricMultiThread() { +void ILogtailMetricUnittest::TestCreateMetricAutoDeleteMultiThread() { std::thread t1(createMetrics, 1); std::thread t2(createMetrics, 2); std::thread t3(createMetrics, 3); @@ -157,139 +150,174 @@ void ILogtailMetricUnittest::TestCreateMetricMultiThread() { t3.join(); t4.join(); - MetricsRecord* head = WriteMetrics::GetInstance()->GetHead(); - MetricsRecord* tmp = head; + // assert WriteMetrics count + MetricsRecord* tmp = WriteMetrics::GetInstance()->GetHead(); int count = 0; - while (tmp != nullptr) { - for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { - std::pair pair = *item; - LOG_INFO(sLogger, ("key", pair.first)("value", pair.second)); - } + while (tmp) { tmp = tmp->GetNext(); count++; } - // first test left 1, 1 + (1 + 2 + 3 + 4) - APSARA_TEST_EQUAL(count, 11); - LOG_INFO(sLogger, ("Count", count)); + // 1 + 2 + 3 + 4 = 10 + APSARA_TEST_EQUAL(count, 10); - // UpdateMetrics multi time for (int i = 0; i < 10; i++) { - ReadMetrics::GetInstance()->UpdateMetrics(); + MetricExportor::GetInstance()->PushMetrics(true); } - { - MetricsRecord* head = WriteMetrics::GetInstance()->GetHead(); - MetricsRecord* tmp = head; - int count = 0; - while (tmp != nullptr) { - for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { - std::pair pair = *item; - LOG_INFO(sLogger, ("key", pair.first)("value", pair.second)); - } - tmp = tmp->GetNext(); - count++; + // assert WriteMetrics count + tmp = WriteMetrics::GetInstance()->GetHead(); + count = 0; + while (tmp) { + for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { + std::pair pair = *item; + LOG_INFO(sLogger, ("key", pair.first)("value", pair.second)); } - APSARA_TEST_EQUAL(count, 0); + tmp = tmp->GetNext(); + count++; } -} + APSARA_TEST_EQUAL(count, 0); + // assert ReadMetrics count + tmp = ReadMetrics::GetInstance()->GetHead(); + count = 0; + while (tmp) { + tmp = tmp->GetNext(); + count++; + } + APSARA_TEST_EQUAL(count, 0); +} -void ILogtailMetricUnittest::TestCreateAndDeleteMetricMultiThread() { - std::thread t1(createMetrics, 5); - std::thread t2(createMetrics, 6); - std::thread tRunning1(createRunningMetrics); - std::thread tRunning2(createRunningMetrics); - // UpdateMetrics multi time while createMetrics - std::thread tUpdate(PushMetrics); - std::thread t3(createMetrics, 7); - std::thread t4(createMetrics, 8); +void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { + std::thread t1(createMetrics, 1); + std::thread t2(createMetrics, 2); - t1.join(); - t2.join(); - tUpdate.join(); - t3.join(); - t4.join(); + MetricsRecordRef* fileMetric1 = new MetricsRecordRef(); + MetricsRecordRef* fileMetric2 = new MetricsRecordRef(); + MetricsRecordRef* fileMetric3 = new MetricsRecordRef(); - // final UpdateMetrics after createMetrics - ReadMetrics::GetInstance()->UpdateMetrics(); + + std::vector> labels; + labels.emplace_back(std::make_pair("project", "test1")); + labels.emplace_back(std::make_pair("logstore", "test1")); + labels.emplace_back(std::make_pair("region", "cn-beijing")); + fileMetric1 -> Init(labels); + MetricNameValuePtr fileCounter = fileMetric1->operator->()->CreateCounter("filed1"); + fileCounter->SetValue(111UL); + { - MetricsRecord* head = WriteMetrics::GetInstance()->GetHead(); - int count = 0; - while (head) { - if (!head->IsDeleted()) { - count++; - } - head = head->GetNext(); - } - APSARA_TEST_EQUAL(count, 2); - LOG_INFO(sLogger, ("FinalWriteCount", count)); + std::vector> labels; + labels.emplace_back(std::make_pair("project", "test2")); + labels.emplace_back(std::make_pair("logstore", "test2")); + labels.emplace_back(std::make_pair("region", "cn-beijing")); + fileMetric2 -> Init(labels); + MetricNameValuePtr fileCounter = fileMetric2->operator->()->CreateCounter("filed1"); + fileCounter->SetValue(111UL); } { - MetricsRecord* head = ReadMetrics::GetInstance()->GetHead(); - int count = 0; - while (head) { - head = head->GetNext(); - count++; - } - APSARA_TEST_EQUAL(count, 2); - - head = ReadMetrics::GetInstance()->GetHead(); + std::vector> labels; + labels.emplace_back(std::make_pair("project", "test3")); + labels.emplace_back(std::make_pair("logstore", "test3")); + labels.emplace_back(std::make_pair("region", "cn-beijing")); + fileMetric3 -> Init(labels); + MetricNameValuePtr fileCounter = fileMetric3->operator->()->CreateCounter("filed1"); + fileCounter->SetValue(111UL); + } + std::thread t3(createMetrics, 3); + std::thread t4(createMetrics, 4); - APSARA_TEST_EQUAL(head->GetLabels()->size(), 1); - LOG_INFO(sLogger, ("FinalReadCount", count)); + t1.join(); + t2.join(); + t3.join(); + t4.join(); + // assert WriteMetrics count + MetricsRecord* tmp = WriteMetrics::GetInstance()->GetHead(); + int count = 0; + while (tmp) { + tmp = tmp->GetNext(); + count++; } + // 10 + 3 + APSARA_TEST_EQUAL(count, 13); - std::thread tRunning3(createRunningMetrics); - std::thread tRunning4(createRunningMetrics); + delete fileMetric2; + delete fileMetric3; - usleep(100); + MetricExportor::GetInstance()->PushMetrics(true); - // final UpdateMetrics after createMetrics - ReadMetrics::GetInstance()->UpdateMetrics(); - { - MetricsRecord* head = WriteMetrics::GetInstance()->GetHead(); - int count = 0; - while (head) { - if (!head->IsDeleted()) { - count++; - } - head = head->GetNext(); + // assert WriteMetrics count + tmp = WriteMetrics::GetInstance()->GetHead(); + count = 0; + while (tmp) { + for (auto item = tmp->GetLabels()->begin(); item != tmp->GetLabels()->end(); ++item) { + std::pair pair = *item; + LOG_INFO(sLogger, ("key", pair.first)("value", pair.second)); } - APSARA_TEST_EQUAL(count, 4); - LOG_INFO(sLogger, ("FinalWriteCount", count)); + tmp = tmp->GetNext(); + count++; + } + APSARA_TEST_EQUAL(count, 1); + // assert writeMetric value + if (count == 1) { + tmp = WriteMetrics::GetInstance()->GetHead(); + std::vector values = tmp->GetMetricNameValues(); + APSARA_TEST_EQUAL(values.size(), 1); + if (values.size() == 1) { + APSARA_TEST_EQUAL(values.at(0)->GetValue(), 0); + } } - { - MetricsRecord* head = ReadMetrics::GetInstance()->GetHead(); - int count = 0; - while (head) { - head = head->GetNext(); - count++; - } - APSARA_TEST_EQUAL(count, 4); - LOG_INFO(sLogger, ("FinalReadCount", count)); + // assert ReadMetrics count + tmp = ReadMetrics::GetInstance()->GetHead(); + count = 0; + while (tmp) { + tmp = tmp->GetNext(); + count++; } + APSARA_TEST_EQUAL(count, 1); + // assert readMetric value + if (count == 1) { + tmp = ReadMetrics::GetInstance()->GetHead(); + std::vector values = tmp->GetMetricNameValues(); + APSARA_TEST_EQUAL(values.size(), 1); + if (values.size() == 1) { + APSARA_TEST_EQUAL(values.at(0)->GetValue(), 111); + } + } - MetricExportor::GetInstance()->PushMetrics(true); - running.store(false); - tRunning1.join(); - tRunning2.join(); - tRunning3.join(); - tRunning4.join(); + // after dosnapshot, add value again + fileCounter->SetValue(111UL); + fileCounter->SetValue(111UL); + fileCounter->SetValue(111UL); - MetricsRecord* head = ReadMetrics::GetInstance()->GetHead(); - std::vector nameValues = head->GetMetricNameValues(); + APSARA_TEST_EQUAL(fileCounter->GetValue(), 333); - APSARA_TEST_EQUAL(nameValues.size(), 1); - LOG_INFO(sLogger, ("end", "test")); -} + MetricExportor::GetInstance()->PushMetrics(true); + // assert ReadMetrics count + tmp = ReadMetrics::GetInstance()->GetHead(); + count = 0; + while (tmp) { + tmp = tmp->GetNext(); + count++; + } + APSARA_TEST_EQUAL(count, 1); + // assert readMetric value + if (count == 1) { + tmp = ReadMetrics::GetInstance()->GetHead(); + std::vector values = tmp->GetMetricNameValues(); + APSARA_TEST_EQUAL(values.size(), 1); + if (values.size() == 1) { + APSARA_TEST_EQUAL(values.at(0)->GetValue(), 333); + } + } + delete fileMetric1; +} } // namespace logtail From 17b5be6b48efea05d8a9aa4cf5080e1f8aeff4be Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Wed, 23 Aug 2023 14:18:48 +0800 Subject: [PATCH 29/39] refine code --- .../monitor/LogtailMetricUnittest.cpp | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp index 3535d353af..0238b684fa 100644 --- a/core/unittest/monitor/LogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -27,7 +27,6 @@ class ILogtailMetricUnittest : public ::testing::Test { void TestCreateMetricAutoDelete(); void TestCreateMetricAutoDeleteMultiThread(); void TestCreateAndDeleteMetric(); - }; APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateMetricAutoDelete, 0); @@ -35,7 +34,6 @@ APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateMetricAutoDeleteMultiThr APSARA_UNIT_TEST_CASE(ILogtailMetricUnittest, TestCreateAndDeleteMetric, 2); - void ILogtailMetricUnittest::TestCreateMetricAutoDelete() { std::vector> labels; labels.emplace_back(std::make_pair("project", "project1")); @@ -188,8 +186,6 @@ void ILogtailMetricUnittest::TestCreateMetricAutoDeleteMultiThread() { } - - void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { std::thread t1(createMetrics, 1); std::thread t2(createMetrics, 2); @@ -198,21 +194,21 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { MetricsRecordRef* fileMetric2 = new MetricsRecordRef(); MetricsRecordRef* fileMetric3 = new MetricsRecordRef(); - + std::vector> labels; labels.emplace_back(std::make_pair("project", "test1")); labels.emplace_back(std::make_pair("logstore", "test1")); labels.emplace_back(std::make_pair("region", "cn-beijing")); - fileMetric1 -> Init(labels); + fileMetric1->Init(labels); MetricNameValuePtr fileCounter = fileMetric1->operator->()->CreateCounter("filed1"); fileCounter->SetValue(111UL); - + { std::vector> labels; labels.emplace_back(std::make_pair("project", "test2")); labels.emplace_back(std::make_pair("logstore", "test2")); labels.emplace_back(std::make_pair("region", "cn-beijing")); - fileMetric2 -> Init(labels); + fileMetric2->Init(labels); MetricNameValuePtr fileCounter = fileMetric2->operator->()->CreateCounter("filed1"); fileCounter->SetValue(111UL); } @@ -222,7 +218,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { labels.emplace_back(std::make_pair("project", "test3")); labels.emplace_back(std::make_pair("logstore", "test3")); labels.emplace_back(std::make_pair("region", "cn-beijing")); - fileMetric3 -> Init(labels); + fileMetric3->Init(labels); MetricNameValuePtr fileCounter = fileMetric3->operator->()->CreateCounter("filed1"); fileCounter->SetValue(111UL); } @@ -268,7 +264,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { APSARA_TEST_EQUAL(values.size(), 1); if (values.size() == 1) { APSARA_TEST_EQUAL(values.at(0)->GetValue(), 0); - } + } } // assert ReadMetrics count @@ -287,7 +283,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { APSARA_TEST_EQUAL(values.size(), 1); if (values.size() == 1) { APSARA_TEST_EQUAL(values.at(0)->GetValue(), 111); - } + } } // after dosnapshot, add value again @@ -314,7 +310,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { APSARA_TEST_EQUAL(values.size(), 1); if (values.size() == 1) { APSARA_TEST_EQUAL(values.at(0)->GetValue(), 333); - } + } } delete fileMetric1; } From b3f437e109725209239a414f4308448bbb111175 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Wed, 23 Aug 2023 14:26:17 +0800 Subject: [PATCH 30/39] refine code --- core/monitor/LogtaiMetric.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index a6a6c03ac6..d90e8c65f7 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -201,9 +201,9 @@ MetricsRecord* WriteMetrics::DoSnapshot() { } while (toDeleteHead) { - MetricsRecord* toDeleted = toDeleteHead; + MetricsRecord* toDelete = toDeleteHead; toDeleteHead = toDeleteHead->GetNext(); - delete toDeleted; + delete toDelete; } return snapshot; } @@ -300,9 +300,9 @@ MetricsRecord* ReadMetrics::GetHead() const { void ReadMetrics::Clear() { while (mHead) { - MetricsRecord* toDeleted = mHead; + MetricsRecord* toDelete = mHead; mHead = mHead->GetNext(); - delete toDeleted; + delete toDelete; } } From 9db41933e1307b3de64c8e70f7f75c04fac5ccbb Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Wed, 23 Aug 2023 16:59:20 +0800 Subject: [PATCH 31/39] refine code --- core/monitor/LogtaiMetric.cpp | 19 ++++++++++--------- core/monitor/LogtailMetric.h | 12 +++++------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index d90e8c65f7..680fe0ea83 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -10,6 +10,9 @@ using namespace sls_logs; namespace logtail { +MetricNameValue::MetricNameValue(const std::string& name, uint64_t val) : mName(name), mVal(val) { +} + const uint64_t MetricNameValue::GetValue() const { return mVal; @@ -63,7 +66,7 @@ void MetricsRecord::MarkDeleted() { mDeleted = true; } -const bool MetricsRecord::IsDeleted() const { +bool MetricsRecord::IsDeleted() const { return mDeleted; } @@ -101,10 +104,9 @@ MetricsRecordRef::~MetricsRecordRef() { } } -void MetricsRecordRef::Init(const std::vector>& labels) { +void MetricsRecordRef::Init(const MetricLabels& labels) { if (!mMetrics) { - mMetrics = WriteMetrics::GetInstance()->CreateMetricsRecords( - std::make_shared>>(labels)); + mMetrics = WriteMetrics::GetInstance()->CreateMetricsRecords(std::make_shared(labels)); } } @@ -148,7 +150,6 @@ MetricsRecord* WriteMetrics::DoSnapshot() { MetricsRecord emptyHead; MetricsRecord* preTmp = nullptr; - MetricsRecord* tmpHead = nullptr; // find the first undeleted node and set as new mHead { std::lock_guard lock(mMutex); @@ -165,7 +166,7 @@ MetricsRecord* WriteMetrics::DoSnapshot() { } else { // find head mHead = tmp; - preTmp = tmp; + preTmp = mHead; tmp = tmp->GetNext(); findHead = true; break; @@ -174,13 +175,13 @@ MetricsRecord* WriteMetrics::DoSnapshot() { // if no undeleted node, set null to mHead if (!findHead) { mHead = nullptr; + preTmp = mHead; } - tmpHead = mHead; } // copy head - if (tmpHead) { - MetricsRecord* newMetrics = tmpHead->CopyAndReset(); + if (preTmp) { + MetricsRecord* newMetrics = preTmp->CopyAndReset(); newMetrics->SetNext(snapshot); snapshot = newMetrics; } diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 4f52709f91..0ed051dbee 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -14,10 +14,7 @@ class MetricNameValue { std::atomic_long mVal; public: - MetricNameValue(const std::string& name, uint64_t val) { - mName = name; - mVal = val; - }; + MetricNameValue(const std::string& name, uint64_t val); virtual ~MetricNameValue(){}; const uint64_t GetValue() const; const std::string& GetName() const; @@ -42,7 +39,8 @@ class Counter : public MetricNameValue { }; using MetricNameValuePtr = std::shared_ptr; -using LabelsPtr = std::shared_ptr>>; +using MetricLabels = std::vector>; +using LabelsPtr = std::shared_ptr; class MetricsRecord { private: @@ -55,7 +53,7 @@ class MetricsRecord { MetricsRecord(LabelsPtr labels); MetricsRecord(); void MarkDeleted(); - const bool IsDeleted() const; + bool IsDeleted() const; const LabelsPtr& GetLabels() const; const std::vector& GetMetricNameValues() const; MetricNameValuePtr CreateCounter(const std::string& Name); @@ -72,7 +70,7 @@ class MetricsRecordRef { public: MetricsRecordRef(); ~MetricsRecordRef(); - void Init(const std::vector>& labels); + void Init(const MetricLabels& labels); MetricsRecord* operator->() const; }; From f34a75f2604a2cdc1458bba4c2e52bc7d1994914 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Wed, 23 Aug 2023 17:07:25 +0800 Subject: [PATCH 32/39] refine code --- core/monitor/LogtaiMetric.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index 680fe0ea83..1df835fbab 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -134,6 +134,7 @@ MetricsRecord* WriteMetrics::GetHead() const { } void WriteMetrics::Clear() { + std::lock_guard lock(mMutex); while (mHead) { MetricsRecord* toDeleted = mHead; mHead = mHead->GetNext(); @@ -300,6 +301,7 @@ MetricsRecord* ReadMetrics::GetHead() const { } void ReadMetrics::Clear() { + WriteLock lock(mReadWriteLock); while (mHead) { MetricsRecord* toDelete = mHead; mHead = mHead->GetNext(); From 2485093576b47dac1e5ce42e0be2577d8a25e31c Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Wed, 23 Aug 2023 20:26:55 +0800 Subject: [PATCH 33/39] refine coder --- core/monitor/LogtaiMetric.cpp | 88 +++++++++++-------- core/monitor/LogtailMetric.h | 51 ++++++----- .../monitor/LogtailMetricUnittest.cpp | 56 ++++++------ 3 files changed, 102 insertions(+), 93 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index 1df835fbab..599643dd6c 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -10,37 +10,41 @@ using namespace sls_logs; namespace logtail { -MetricNameValue::MetricNameValue(const std::string& name, uint64_t val) : mName(name), mVal(val) { +Counter::Counter(const std::string& name, uint64_t val = 0) : mName(name), mVal(val) { } - -const uint64_t MetricNameValue::GetValue() const { +uint64_t Counter::GetValue() const { return mVal; } -const std::string& MetricNameValue::GetName() const { +const std::string& Counter::GetName() const { return mName; } -Counter::Counter(const std::string& name, uint64_t val = 0) : MetricNameValue(name, val) { -} - Counter* Counter::CopyAndReset() { return new Counter(mName, mVal.exchange(0)); } -void Counter::SetValue(uint64_t value) { +void Counter::Add(uint64_t value) { mVal += value; } -Gauge::Gauge(const std::string& name, uint64_t val = 0) : MetricNameValue(name, val) { +Gauge::Gauge(const std::string& name, uint64_t val = 0) : mName(name), mVal(val) { +} + +uint64_t Gauge::GetValue() const { + return mVal; +} + +const std::string& Gauge::GetName() const { + return mName; } Gauge* Gauge::CopyAndReset() { return new Gauge(mName, mVal.exchange(0)); } -void Gauge::SetValue(uint64_t value) { +void Gauge::Set(uint64_t value) { mVal = value; } @@ -50,15 +54,15 @@ MetricsRecord::MetricsRecord(LabelsPtr labels) : mLabels(labels), mDeleted(false MetricsRecord::MetricsRecord() : mDeleted(false) { } -MetricNameValuePtr MetricsRecord::CreateCounter(const std::string& name) { - MetricNameValuePtr counterPtr = std::make_shared(name); - mValues.emplace_back(counterPtr); +CounterPtr MetricsRecord::CreateCounter(const std::string& name) { + CounterPtr counterPtr = std::make_shared(name); + mCounters.emplace_back(counterPtr); return counterPtr; } -MetricNameValuePtr MetricsRecord::CreateGauge(const std::string& name) { - MetricNameValuePtr gaugePtr = std::make_shared(name); - mValues.emplace_back(gaugePtr); +GaugePtr MetricsRecord::CreateGauge(const std::string& name) { + GaugePtr gaugePtr = std::make_shared(name); + mGauges.emplace_back(gaugePtr); return gaugePtr; } @@ -74,15 +78,23 @@ const LabelsPtr& MetricsRecord::GetLabels() const { return mLabels; } -const std::vector& MetricsRecord::GetMetricNameValues() const { - return mValues; +const std::vector& MetricsRecord::GetCounters() const { + return mCounters; +} + +const std::vector& MetricsRecord::GetGauges() const { + return mGauges; } MetricsRecord* MetricsRecord::CopyAndReset() { MetricsRecord* metrics = new MetricsRecord(mLabels); - for (auto& item : mValues) { - MetricNameValuePtr newPtr(item->CopyAndReset()); - metrics->mValues.emplace_back(newPtr); + for (auto& item : mCounters) { + CounterPtr newPtr(item->CopyAndReset()); + metrics->mCounters.emplace_back(newPtr); + } + for (auto& item : mGauges) { + GaugePtr newPtr(item->CopyAndReset()); + metrics->mGauges.emplace_back(newPtr); } return metrics; } @@ -95,38 +107,32 @@ void MetricsRecord::SetNext(MetricsRecord* next) { mNext = next; } -MetricsRecordRef::MetricsRecordRef() { -} - MetricsRecordRef::~MetricsRecordRef() { if (mMetrics) { mMetrics->MarkDeleted(); } } -void MetricsRecordRef::Init(const MetricLabels& labels) { - if (!mMetrics) { - mMetrics = WriteMetrics::GetInstance()->CreateMetricsRecords(std::make_shared(labels)); - } + +void MetricsRecordRef::SetMetricsRecord(MetricsRecord* metricRecord) { + mMetrics = metricRecord; } + MetricsRecord* MetricsRecordRef::operator->() const { return mMetrics; } -WriteMetrics::WriteMetrics() { -} - WriteMetrics::~WriteMetrics() { Clear(); } -MetricsRecord* WriteMetrics::CreateMetricsRecords(LabelsPtr labels) { - MetricsRecord* cur = new MetricsRecord(labels); +void WriteMetrics::PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels) { + MetricsRecord* cur = new MetricsRecord(std::make_shared>>(labels)); + ref.SetMetricsRecord(cur); std::lock_guard lock(mMutex); cur->SetNext(mHead); mHead = cur; - return cur; } MetricsRecord* WriteMetrics::GetHead() const { @@ -210,8 +216,6 @@ MetricsRecord* WriteMetrics::DoSnapshot() { return snapshot; } -ReadMetrics::ReadMetrics() { -} ReadMetrics::~ReadMetrics() { Clear(); @@ -262,12 +266,18 @@ void ReadMetrics::ReadAsLogGroup(std::map& log contentPtr->set_value(pair.second); } - for (auto& item : tmp->GetMetricNameValues()) { - MetricNameValuePtr counter = item; + for (auto& item : tmp->GetCounters()) { + CounterPtr counter = item; Log_Content* contentPtr = logPtr->add_contents(); contentPtr->set_key(VALUE_PREFIX + counter->GetName()); contentPtr->set_value(ToString(counter->GetValue())); } + for (auto& item : tmp->GetGauges()) { + GaugePtr gauge = item; + Log_Content* contentPtr = logPtr->add_contents(); + contentPtr->set_key(VALUE_PREFIX + gauge->GetName()); + contentPtr->set_value(ToString(gauge->GetValue())); + } // set default key { Log_Content* contentPtr = logPtr->add_contents(); @@ -309,4 +319,4 @@ void ReadMetrics::Clear() { } } -} // namespace logtail \ No newline at end of file +} // namespace logtail diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 0ed051dbee..fc219efaf4 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -7,38 +7,36 @@ namespace logtail { - -class MetricNameValue { -protected: +class Counter { +private: std::string mName; std::atomic_long mVal; public: - MetricNameValue(const std::string& name, uint64_t val); - virtual ~MetricNameValue(){}; - const uint64_t GetValue() const; + Counter(const std::string& name, uint64_t val); + uint64_t GetValue() const; const std::string& GetName() const; - virtual void SetValue(uint64_t val) = 0; - virtual MetricNameValue* CopyAndReset() = 0; + void Add(uint64_t val); + Counter* CopyAndReset(); }; +class Gauge { +private: + std::string mName; + std::atomic_long mVal; -class Gauge : public MetricNameValue { public: Gauge(const std::string& name, uint64_t val); - void SetValue(uint64_t val) override; - Gauge* CopyAndReset() override; + uint64_t GetValue() const; + const std::string& GetName() const; + void Set(uint64_t val); + Gauge* CopyAndReset(); }; -class Counter : public MetricNameValue { -public: - Counter(const std::string& name, uint64_t val); - void SetValue(uint64_t val) override; - Counter* CopyAndReset() override; -}; +using CounterPtr = std::shared_ptr; +using GaugePtr = std::shared_ptr; -using MetricNameValuePtr = std::shared_ptr; using MetricLabels = std::vector>; using LabelsPtr = std::shared_ptr; @@ -46,7 +44,8 @@ class MetricsRecord { private: LabelsPtr mLabels; std::atomic_bool mDeleted; - std::vector mValues; + std::vector mCounters; + std::vector mGauges; MetricsRecord* mNext = nullptr; public: @@ -55,9 +54,10 @@ class MetricsRecord { void MarkDeleted(); bool IsDeleted() const; const LabelsPtr& GetLabels() const; - const std::vector& GetMetricNameValues() const; - MetricNameValuePtr CreateCounter(const std::string& Name); - MetricNameValuePtr CreateGauge(const std::string& Name); + const std::vector& GetCounters() const; + const std::vector& GetGauges() const; + CounterPtr CreateCounter(const std::string& Name); + GaugePtr CreateGauge(const std::string& Name); MetricsRecord* CopyAndReset(); void SetNext(MetricsRecord* next); MetricsRecord* GetNext() const; @@ -68,9 +68,8 @@ class MetricsRecordRef { MetricsRecord* mMetrics = nullptr; public: - MetricsRecordRef(); ~MetricsRecordRef(); - void Init(const MetricLabels& labels); + void SetMetricsRecord(MetricsRecord* metricRecord); MetricsRecord* operator->() const; }; @@ -86,7 +85,7 @@ class WriteMetrics { static WriteMetrics* ptr = new WriteMetrics(); return ptr; } - MetricsRecord* CreateMetricsRecords(LabelsPtr Labels); + void PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& Labels); MetricsRecord* DoSnapshot(); MetricsRecord* GetHead() const; void Clear(); @@ -109,4 +108,4 @@ class ReadMetrics { MetricsRecord* GetHead() const; void Clear(); }; -} // namespace logtail \ No newline at end of file +} // namespace logtail diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp index 0238b684fa..af44eacf8d 100644 --- a/core/unittest/monitor/LogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -41,13 +41,13 @@ void ILogtailMetricUnittest::TestCreateMetricAutoDelete() { labels.emplace_back(std::make_pair("region", "cn-hangzhou")); MetricsRecordRef fileMetric; - fileMetric.Init(labels); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric, std::move(labels)); APSARA_TEST_EQUAL(fileMetric->GetLabels()->size(), 3); - MetricNameValuePtr fileCounter = fileMetric->CreateCounter("filed1"); - fileCounter->SetValue(111UL); - fileCounter->SetValue(111UL); + CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); + fileCounter->Add(111UL); + fileCounter->Add(111UL); APSARA_TEST_EQUAL(fileCounter->GetValue(), 222); MetricExportor::GetInstance()->PushMetrics(true); @@ -79,9 +79,9 @@ void ILogtailMetricUnittest::TestCreateMetricAutoDelete() { labels.emplace_back(std::make_pair("region", "cn-hangzhou")); MetricsRecordRef fileMetric2; - fileMetric2.Init(labels); - MetricNameValuePtr fileCounter2 = fileMetric2->CreateCounter("filed2"); - fileCounter2->SetValue(222UL); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric2, std::move(labels)); + CounterPtr fileCounter2 = fileMetric2->CreateCounter("filed2"); + fileCounter2->Add(222UL); } { @@ -90,9 +90,9 @@ void ILogtailMetricUnittest::TestCreateMetricAutoDelete() { labels.emplace_back(std::make_pair("logstore", "logstore1")); labels.emplace_back(std::make_pair("region", "cn-hangzhou")); MetricsRecordRef fileMetric3; - fileMetric3.Init(labels); - MetricNameValuePtr fileCounter3 = fileMetric3->CreateCounter("filed3"); - fileCounter3->SetValue(333UL); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric3, std::move(labels)); + CounterPtr fileCounter3 = fileMetric3->CreateCounter("filed3"); + fileCounter3->Add(333UL); } MetricExportor::GetInstance()->PushMetrics(true); @@ -131,9 +131,9 @@ void createMetrics(int count) { labels.emplace_back(std::make_pair("count", std::to_string(count))); labels.emplace_back(std::make_pair("region", "cn-beijing")); MetricsRecordRef fileMetric; - fileMetric.Init(labels); - MetricNameValuePtr fileCounter = fileMetric->CreateCounter("filed1"); - fileCounter->SetValue(111UL); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric, std::move(labels)); + CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); + fileCounter->Add(111UL); } } @@ -199,18 +199,18 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { labels.emplace_back(std::make_pair("project", "test1")); labels.emplace_back(std::make_pair("logstore", "test1")); labels.emplace_back(std::make_pair("region", "cn-beijing")); - fileMetric1->Init(labels); - MetricNameValuePtr fileCounter = fileMetric1->operator->()->CreateCounter("filed1"); - fileCounter->SetValue(111UL); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric1, std::move(labels)); + CounterPtr fileCounter = fileMetric1->operator->()->CreateCounter("filed1"); + fileCounter->Add(111UL); { std::vector> labels; labels.emplace_back(std::make_pair("project", "test2")); labels.emplace_back(std::make_pair("logstore", "test2")); labels.emplace_back(std::make_pair("region", "cn-beijing")); - fileMetric2->Init(labels); - MetricNameValuePtr fileCounter = fileMetric2->operator->()->CreateCounter("filed1"); - fileCounter->SetValue(111UL); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric2, std::move(labels)); + CounterPtr fileCounter = fileMetric2->operator->()->CreateCounter("filed1"); + fileCounter->Add(111UL); } { @@ -218,9 +218,9 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { labels.emplace_back(std::make_pair("project", "test3")); labels.emplace_back(std::make_pair("logstore", "test3")); labels.emplace_back(std::make_pair("region", "cn-beijing")); - fileMetric3->Init(labels); - MetricNameValuePtr fileCounter = fileMetric3->operator->()->CreateCounter("filed1"); - fileCounter->SetValue(111UL); + WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric3, std::move(labels)); + CounterPtr fileCounter = fileMetric3->operator->()->CreateCounter("filed1"); + fileCounter->Add(111UL); } std::thread t3(createMetrics, 3); std::thread t4(createMetrics, 4); @@ -260,7 +260,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { // assert writeMetric value if (count == 1) { tmp = WriteMetrics::GetInstance()->GetHead(); - std::vector values = tmp->GetMetricNameValues(); + std::vector values = tmp->GetCounters(); APSARA_TEST_EQUAL(values.size(), 1); if (values.size() == 1) { APSARA_TEST_EQUAL(values.at(0)->GetValue(), 0); @@ -279,7 +279,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { // assert readMetric value if (count == 1) { tmp = ReadMetrics::GetInstance()->GetHead(); - std::vector values = tmp->GetMetricNameValues(); + std::vector values = tmp->GetCounters(); APSARA_TEST_EQUAL(values.size(), 1); if (values.size() == 1) { APSARA_TEST_EQUAL(values.at(0)->GetValue(), 111); @@ -287,9 +287,9 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { } // after dosnapshot, add value again - fileCounter->SetValue(111UL); - fileCounter->SetValue(111UL); - fileCounter->SetValue(111UL); + fileCounter->Add(111UL); + fileCounter->Add(111UL); + fileCounter->Add(111UL); APSARA_TEST_EQUAL(fileCounter->GetValue(), 333); @@ -306,7 +306,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { // assert readMetric value if (count == 1) { tmp = ReadMetrics::GetInstance()->GetHead(); - std::vector values = tmp->GetMetricNameValues(); + std::vector values = tmp->GetCounters(); APSARA_TEST_EQUAL(values.size(), 1); if (values.size() == 1) { APSARA_TEST_EQUAL(values.at(0)->GetValue(), 333); From bc44e7a81b9cdf5954925257a994cb5bb98837b2 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Wed, 23 Aug 2023 20:39:34 +0800 Subject: [PATCH 34/39] refine coder --- core/monitor/LogtaiMetric.cpp | 3 +++ core/monitor/LogtailMetric.h | 14 ++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index 599643dd6c..ab172ce496 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -123,6 +123,8 @@ MetricsRecord* MetricsRecordRef::operator->() const { return mMetrics; } +WriteMetrics::WriteMetrics() {} + WriteMetrics::~WriteMetrics() { Clear(); } @@ -216,6 +218,7 @@ MetricsRecord* WriteMetrics::DoSnapshot() { return snapshot; } +ReadMetrics::ReadMetrics() {} ReadMetrics::~ReadMetrics() { Clear(); diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index fc219efaf4..79aa726be3 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -78,7 +78,7 @@ class WriteMetrics { WriteMetrics(); std::mutex mMutex; MetricsRecord* mHead = nullptr; - + void Clear(); public: ~WriteMetrics(); static WriteMetrics* GetInstance() { @@ -88,7 +88,10 @@ class WriteMetrics { void PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& Labels); MetricsRecord* DoSnapshot(); MetricsRecord* GetHead() const; - void Clear(); + +#ifdef APSARA_UNIT_TEST_MAIN + friend class ILogtailMetricUnittest; +#endif }; class ReadMetrics { @@ -96,7 +99,7 @@ class ReadMetrics { ReadMetrics(); mutable ReadWriteLock mReadWriteLock; MetricsRecord* mHead = nullptr; - + void Clear(); public: ~ReadMetrics(); static ReadMetrics* GetInstance() { @@ -106,6 +109,9 @@ class ReadMetrics { void ReadAsLogGroup(std::map& logGroupMap) const; void UpdateMetrics(); MetricsRecord* GetHead() const; - void Clear(); + +#ifdef APSARA_UNIT_TEST_MAIN + friend class ILogtailMetricUnittest; +#endif }; } // namespace logtail From de2f991c97d0a96e24566ca565dec58e8f4a562e Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Wed, 23 Aug 2023 21:37:42 +0800 Subject: [PATCH 35/39] refine code --- core/monitor/LogtaiMetric.cpp | 13 ++++++++++--- core/monitor/LogtailMetric.h | 12 ++++++++---- core/unittest/monitor/LogtailMetricUnittest.cpp | 14 +++++++------- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index ab172ce496..fb3798ada6 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -118,11 +118,18 @@ void MetricsRecordRef::SetMetricsRecord(MetricsRecord* metricRecord) { mMetrics = metricRecord; } +CounterPtr MetricsRecordRef::CreateCounter(const std::string& Name) { + return mMetrics->CreateCounter(Name); +} +GaugePtr MetricsRecordRef::CreateGauge(const std::string& Name) { + return mMetrics->CreateGauge(Name); +} -MetricsRecord* MetricsRecordRef::operator->() const { +const MetricsRecord* MetricsRecordRef::operator->() const { return mMetrics; } + WriteMetrics::WriteMetrics() {} WriteMetrics::~WriteMetrics() { @@ -137,7 +144,7 @@ void WriteMetrics::PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& mHead = cur; } -MetricsRecord* WriteMetrics::GetHead() const { +MetricsRecord* WriteMetrics::GetHead() { return mHead; } @@ -309,7 +316,7 @@ void ReadMetrics::UpdateMetrics() { } } -MetricsRecord* ReadMetrics::GetHead() const { +MetricsRecord* ReadMetrics::GetHead() { return mHead; } diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 79aa726be3..dad8d2329e 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -70,7 +70,9 @@ class MetricsRecordRef { public: ~MetricsRecordRef(); void SetMetricsRecord(MetricsRecord* metricRecord); - MetricsRecord* operator->() const; + CounterPtr CreateCounter(const std::string& Name); + GaugePtr CreateGauge(const std::string& Name); + const MetricsRecord* operator->() const; }; class WriteMetrics { @@ -78,7 +80,9 @@ class WriteMetrics { WriteMetrics(); std::mutex mMutex; MetricsRecord* mHead = nullptr; + void Clear(); + MetricsRecord* GetHead(); public: ~WriteMetrics(); static WriteMetrics* GetInstance() { @@ -87,7 +91,7 @@ class WriteMetrics { } void PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& Labels); MetricsRecord* DoSnapshot(); - MetricsRecord* GetHead() const; + #ifdef APSARA_UNIT_TEST_MAIN friend class ILogtailMetricUnittest; @@ -100,6 +104,7 @@ class ReadMetrics { mutable ReadWriteLock mReadWriteLock; MetricsRecord* mHead = nullptr; void Clear(); + MetricsRecord* GetHead(); public: ~ReadMetrics(); static ReadMetrics* GetInstance() { @@ -107,8 +112,7 @@ class ReadMetrics { return ptr; } void ReadAsLogGroup(std::map& logGroupMap) const; - void UpdateMetrics(); - MetricsRecord* GetHead() const; + void UpdateMetrics(); #ifdef APSARA_UNIT_TEST_MAIN friend class ILogtailMetricUnittest; diff --git a/core/unittest/monitor/LogtailMetricUnittest.cpp b/core/unittest/monitor/LogtailMetricUnittest.cpp index af44eacf8d..3f9e36368e 100644 --- a/core/unittest/monitor/LogtailMetricUnittest.cpp +++ b/core/unittest/monitor/LogtailMetricUnittest.cpp @@ -45,7 +45,7 @@ void ILogtailMetricUnittest::TestCreateMetricAutoDelete() { APSARA_TEST_EQUAL(fileMetric->GetLabels()->size(), 3); - CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); + CounterPtr fileCounter = fileMetric.CreateCounter("filed1"); fileCounter->Add(111UL); fileCounter->Add(111UL); APSARA_TEST_EQUAL(fileCounter->GetValue(), 222); @@ -80,7 +80,7 @@ void ILogtailMetricUnittest::TestCreateMetricAutoDelete() { MetricsRecordRef fileMetric2; WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric2, std::move(labels)); - CounterPtr fileCounter2 = fileMetric2->CreateCounter("filed2"); + CounterPtr fileCounter2 = fileMetric2.CreateCounter("filed2"); fileCounter2->Add(222UL); } @@ -91,7 +91,7 @@ void ILogtailMetricUnittest::TestCreateMetricAutoDelete() { labels.emplace_back(std::make_pair("region", "cn-hangzhou")); MetricsRecordRef fileMetric3; WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric3, std::move(labels)); - CounterPtr fileCounter3 = fileMetric3->CreateCounter("filed3"); + CounterPtr fileCounter3 = fileMetric3.CreateCounter("filed3"); fileCounter3->Add(333UL); } @@ -132,7 +132,7 @@ void createMetrics(int count) { labels.emplace_back(std::make_pair("region", "cn-beijing")); MetricsRecordRef fileMetric; WriteMetrics::GetInstance()->PrepareMetricsRecordRef(fileMetric, std::move(labels)); - CounterPtr fileCounter = fileMetric->CreateCounter("filed1"); + CounterPtr fileCounter = fileMetric.CreateCounter("filed1"); fileCounter->Add(111UL); } } @@ -200,7 +200,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { labels.emplace_back(std::make_pair("logstore", "test1")); labels.emplace_back(std::make_pair("region", "cn-beijing")); WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric1, std::move(labels)); - CounterPtr fileCounter = fileMetric1->operator->()->CreateCounter("filed1"); + CounterPtr fileCounter = fileMetric1->CreateCounter("filed1"); fileCounter->Add(111UL); { @@ -209,7 +209,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { labels.emplace_back(std::make_pair("logstore", "test2")); labels.emplace_back(std::make_pair("region", "cn-beijing")); WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric2, std::move(labels)); - CounterPtr fileCounter = fileMetric2->operator->()->CreateCounter("filed1"); + CounterPtr fileCounter = fileMetric2->CreateCounter("filed1"); fileCounter->Add(111UL); } @@ -219,7 +219,7 @@ void ILogtailMetricUnittest::TestCreateAndDeleteMetric() { labels.emplace_back(std::make_pair("logstore", "test3")); labels.emplace_back(std::make_pair("region", "cn-beijing")); WriteMetrics::GetInstance()->PrepareMetricsRecordRef(*fileMetric3, std::move(labels)); - CounterPtr fileCounter = fileMetric3->operator->()->CreateCounter("filed1"); + CounterPtr fileCounter = fileMetric3->CreateCounter("filed1"); fileCounter->Add(111UL); } std::thread t3(createMetrics, 3); From 6316ad5db94413ccb0bff1e0b31d699908ab6867 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Wed, 23 Aug 2023 21:40:56 +0800 Subject: [PATCH 36/39] refine code --- core/monitor/LogtaiMetric.cpp | 7 ++++--- core/monitor/LogtailMetric.h | 8 +++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index fb3798ada6..15a506db8d 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -129,8 +129,8 @@ const MetricsRecord* MetricsRecordRef::operator->() const { return mMetrics; } - -WriteMetrics::WriteMetrics() {} +WriteMetrics::WriteMetrics() { +} WriteMetrics::~WriteMetrics() { Clear(); @@ -225,7 +225,8 @@ MetricsRecord* WriteMetrics::DoSnapshot() { return snapshot; } -ReadMetrics::ReadMetrics() {} +ReadMetrics::ReadMetrics() { +} ReadMetrics::~ReadMetrics() { Clear(); diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index dad8d2329e..91a4ec4e73 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -83,6 +83,7 @@ class WriteMetrics { void Clear(); MetricsRecord* GetHead(); + public: ~WriteMetrics(); static WriteMetrics* GetInstance() { @@ -91,7 +92,7 @@ class WriteMetrics { } void PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& Labels); MetricsRecord* DoSnapshot(); - + #ifdef APSARA_UNIT_TEST_MAIN friend class ILogtailMetricUnittest; @@ -105,6 +106,7 @@ class ReadMetrics { MetricsRecord* mHead = nullptr; void Clear(); MetricsRecord* GetHead(); + public: ~ReadMetrics(); static ReadMetrics* GetInstance() { @@ -112,8 +114,8 @@ class ReadMetrics { return ptr; } void ReadAsLogGroup(std::map& logGroupMap) const; - void UpdateMetrics(); - + void UpdateMetrics(); + #ifdef APSARA_UNIT_TEST_MAIN friend class ILogtailMetricUnittest; #endif From 4ae61bb3db051d8ee5419042c1015b894a7fbe42 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Wed, 23 Aug 2023 22:04:48 +0800 Subject: [PATCH 37/39] refine code --- core/monitor/LogtaiMetric.cpp | 11 +---------- core/monitor/LogtailMetric.h | 6 +++--- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index 15a506db8d..673a062120 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -51,9 +51,6 @@ void Gauge::Set(uint64_t value) { MetricsRecord::MetricsRecord(LabelsPtr labels) : mLabels(labels), mDeleted(false) { } -MetricsRecord::MetricsRecord() : mDeleted(false) { -} - CounterPtr MetricsRecord::CreateCounter(const std::string& name) { CounterPtr counterPtr = std::make_shared(name); mCounters.emplace_back(counterPtr); @@ -129,15 +126,12 @@ const MetricsRecord* MetricsRecordRef::operator->() const { return mMetrics; } -WriteMetrics::WriteMetrics() { -} - WriteMetrics::~WriteMetrics() { Clear(); } void WriteMetrics::PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels) { - MetricsRecord* cur = new MetricsRecord(std::make_shared>>(labels)); + MetricsRecord* cur = new MetricsRecord(std::make_shared(labels)); ref.SetMetricsRecord(cur); std::lock_guard lock(mMutex); cur->SetNext(mHead); @@ -225,9 +219,6 @@ MetricsRecord* WriteMetrics::DoSnapshot() { return snapshot; } -ReadMetrics::ReadMetrics() { -} - ReadMetrics::~ReadMetrics() { Clear(); } diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 91a4ec4e73..1eb2b31ebe 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -50,7 +50,7 @@ class MetricsRecord { public: MetricsRecord(LabelsPtr labels); - MetricsRecord(); + MetricsRecord() = default; void MarkDeleted(); bool IsDeleted() const; const LabelsPtr& GetLabels() const; @@ -77,7 +77,7 @@ class MetricsRecordRef { class WriteMetrics { private: - WriteMetrics(); + WriteMetrics() = default; std::mutex mMutex; MetricsRecord* mHead = nullptr; @@ -101,7 +101,7 @@ class WriteMetrics { class ReadMetrics { private: - ReadMetrics(); + ReadMetrics() = default; mutable ReadWriteLock mReadWriteLock; MetricsRecord* mHead = nullptr; void Clear(); From e7eaad0c629ccfd2b8f8bc3ab639d50da4f21862 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Wed, 23 Aug 2023 23:04:42 +0800 Subject: [PATCH 38/39] refine code --- core/monitor/LogtaiMetric.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index 673a062120..9d8f792906 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -139,6 +139,7 @@ void WriteMetrics::PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& } MetricsRecord* WriteMetrics::GetHead() { + std::lock_guard lock(mMutex); return mHead; } @@ -309,6 +310,7 @@ void ReadMetrics::UpdateMetrics() { } MetricsRecord* ReadMetrics::GetHead() { + WriteLock lock(mReadWriteLock); return mHead; } From dee1a1e022385234a8536b1b532aa83767c95c99 Mon Sep 17 00:00:00 2001 From: linrunqi08 Date: Thu, 24 Aug 2023 13:19:28 +0800 Subject: [PATCH 39/39] refine code --- core/monitor/LogtaiMetric.cpp | 8 ++++---- core/monitor/LogtailMetric.h | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/core/monitor/LogtaiMetric.cpp b/core/monitor/LogtaiMetric.cpp index 9d8f792906..ab29423012 100644 --- a/core/monitor/LogtaiMetric.cpp +++ b/core/monitor/LogtaiMetric.cpp @@ -115,11 +115,11 @@ void MetricsRecordRef::SetMetricsRecord(MetricsRecord* metricRecord) { mMetrics = metricRecord; } -CounterPtr MetricsRecordRef::CreateCounter(const std::string& Name) { - return mMetrics->CreateCounter(Name); +CounterPtr MetricsRecordRef::CreateCounter(const std::string& name) { + return mMetrics->CreateCounter(name); } -GaugePtr MetricsRecordRef::CreateGauge(const std::string& Name) { - return mMetrics->CreateGauge(Name); +GaugePtr MetricsRecordRef::CreateGauge(const std::string& name) { + return mMetrics->CreateGauge(name); } const MetricsRecord* MetricsRecordRef::operator->() const { diff --git a/core/monitor/LogtailMetric.h b/core/monitor/LogtailMetric.h index 1eb2b31ebe..c177b2e073 100644 --- a/core/monitor/LogtailMetric.h +++ b/core/monitor/LogtailMetric.h @@ -20,6 +20,8 @@ class Counter { Counter* CopyAndReset(); }; +using CounterPtr = std::shared_ptr; + class Gauge { private: std::string mName; @@ -33,8 +35,6 @@ class Gauge { Gauge* CopyAndReset(); }; - -using CounterPtr = std::shared_ptr; using GaugePtr = std::shared_ptr; using MetricLabels = std::vector>; @@ -56,8 +56,8 @@ class MetricsRecord { const LabelsPtr& GetLabels() const; const std::vector& GetCounters() const; const std::vector& GetGauges() const; - CounterPtr CreateCounter(const std::string& Name); - GaugePtr CreateGauge(const std::string& Name); + CounterPtr CreateCounter(const std::string& name); + GaugePtr CreateGauge(const std::string& name); MetricsRecord* CopyAndReset(); void SetNext(MetricsRecord* next); MetricsRecord* GetNext() const; @@ -70,8 +70,8 @@ class MetricsRecordRef { public: ~MetricsRecordRef(); void SetMetricsRecord(MetricsRecord* metricRecord); - CounterPtr CreateCounter(const std::string& Name); - GaugePtr CreateGauge(const std::string& Name); + CounterPtr CreateCounter(const std::string& name); + GaugePtr CreateGauge(const std::string& name); const MetricsRecord* operator->() const; }; @@ -90,7 +90,7 @@ class WriteMetrics { static WriteMetrics* ptr = new WriteMetrics(); return ptr; } - void PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& Labels); + void PrepareMetricsRecordRef(MetricsRecordRef& ref, MetricLabels&& labels); MetricsRecord* DoSnapshot();