From a9b322ab2a467827d0ce0574336fb214468362d8 Mon Sep 17 00:00:00 2001 From: Lloyd-Pottiger <60744015+Lloyd-Pottiger@users.noreply.github.com> Date: Fri, 10 Jun 2022 10:54:31 +0800 Subject: [PATCH] feat: add page_stress_testing as a subcommand of tiflash (#5038) close pingcap/tiflash#5037 --- CMakeLists.txt | 3 - dbms/CMakeLists.txt | 13 ++-- dbms/src/Server/CMakeLists.txt | 4 ++ dbms/src/Server/config_tools.h.in | 1 + dbms/src/Server/main.cpp | 8 ++- dbms/src/Storages/CMakeLists.txt | 9 ++- .../DeltaMerge/tools/workload/Main.cpp | 22 ------ .../{tools => }/workload/CMakeLists.txt | 3 - .../{tools => }/workload/DTWorkload.cpp | 20 +++--- .../{tools => }/workload/DTWorkload.h | 2 +- .../{tools => }/workload/DataGenerator.cpp | 16 +++-- .../{tools => }/workload/DataGenerator.h | 2 +- .../DeltaMerge/{tools => }/workload/Handle.h | 18 ++--- .../{tools => }/workload/KeyGenerator.cpp | 10 +-- .../{tools => }/workload/KeyGenerator.h | 4 +- .../{tools => }/workload/Limiter.cpp | 10 +-- .../DeltaMerge/{tools => }/workload/Limiter.h | 2 +- .../{tools => }/workload/MainEntry.cpp | 8 +-- .../{tools => }/workload/Options.cpp | 4 +- .../DeltaMerge/{tools => }/workload/Options.h | 0 .../workload/ReadColumnsGenerator.h | 4 +- .../{tools => }/workload/TableGenerator.cpp | 8 +-- .../{tools => }/workload/TableGenerator.h | 2 +- .../{tools => }/workload/TimestampGenerator.h | 0 .../DeltaMerge/{tools => }/workload/Utils.cpp | 8 +-- .../DeltaMerge/{tools => }/workload/Utils.h | 0 dbms/src/Storages/Page/CMakeLists.txt | 10 --- .../Page/stress/stress_page_storage.cpp | 46 ------------ .../tools => Page/workload}/CMakeLists.txt | 7 +- .../workload/HeavyMemoryCostInGC.cpp | 5 +- .../Page/{stress => }/workload/HeavyRead.cpp | 7 +- .../workload/HeavySkewWriteRead.cpp | 7 +- .../Page/{stress => }/workload/HeavyWrite.cpp | 7 +- .../workload/HighValidBigFileGC.cpp | 5 +- .../workload/HoldSnapshotsLongTime.cpp | 7 +- dbms/src/Storages/Page/workload/MainEntry.cpp | 70 +++++++++++++++++++ .../Page/{stress => }/workload/Normal.cpp | 5 +- .../{stress => workload}/PSBackground.cpp | 6 +- .../Page/{stress => workload}/PSBackground.h | 5 +- .../Page/{stress => workload}/PSRunnable.cpp | 13 ++-- .../Page/{stress => workload}/PSRunnable.h | 37 +++++----- .../Page/{stress => workload}/PSStressEnv.cpp | 7 +- .../Page/{stress => workload}/PSStressEnv.h | 3 + .../Page/{stress => workload}/PSWorkload.cpp | 5 +- .../Page/{stress => workload}/PSWorkload.h | 30 ++++---- .../workload/PageStorageInMemoryCapacity.cpp | 12 ++-- .../workload/ThousandsOfOffset.cpp | 7 +- 47 files changed, 270 insertions(+), 212 deletions(-) delete mode 100644 dbms/src/Storages/DeltaMerge/tools/workload/Main.cpp rename dbms/src/Storages/DeltaMerge/{tools => }/workload/CMakeLists.txt (86%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/DTWorkload.cpp (94%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/DTWorkload.h (97%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/DataGenerator.cpp (95%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/DataGenerator.h (96%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/Handle.h (90%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/KeyGenerator.cpp (92%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/KeyGenerator.h (92%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/Limiter.cpp (77%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/Limiter.h (96%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/MainEntry.cpp (97%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/Options.cpp (98%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/Options.h (100%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/ReadColumnsGenerator.h (93%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/TableGenerator.cpp (97%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/TableGenerator.h (96%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/TimestampGenerator.h (100%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/Utils.cpp (94%) rename dbms/src/Storages/DeltaMerge/{tools => }/workload/Utils.h (100%) delete mode 100644 dbms/src/Storages/Page/stress/stress_page_storage.cpp rename dbms/src/Storages/{DeltaMerge/tools => Page/workload}/CMakeLists.txt (53%) rename dbms/src/Storages/Page/{stress => }/workload/HeavyMemoryCostInGC.cpp (96%) rename dbms/src/Storages/Page/{stress => }/workload/HeavyRead.cpp (94%) rename dbms/src/Storages/Page/{stress => }/workload/HeavySkewWriteRead.cpp (95%) rename dbms/src/Storages/Page/{stress => }/workload/HeavyWrite.cpp (94%) rename dbms/src/Storages/Page/{stress => }/workload/HighValidBigFileGC.cpp (97%) rename dbms/src/Storages/Page/{stress => }/workload/HoldSnapshotsLongTime.cpp (95%) create mode 100644 dbms/src/Storages/Page/workload/MainEntry.cpp rename dbms/src/Storages/Page/{stress => }/workload/Normal.cpp (95%) rename dbms/src/Storages/Page/{stress => workload}/PSBackground.cpp (96%) rename dbms/src/Storages/Page/{stress => workload}/PSBackground.h (97%) rename dbms/src/Storages/Page/{stress => workload}/PSRunnable.cpp (97%) rename dbms/src/Storages/Page/{stress => workload}/PSRunnable.h (90%) rename dbms/src/Storages/Page/{stress => workload}/PSStressEnv.cpp (97%) rename dbms/src/Storages/Page/{stress => workload}/PSStressEnv.h (98%) rename dbms/src/Storages/Page/{stress => workload}/PSWorkload.cpp (98%) rename dbms/src/Storages/Page/{stress => workload}/PSWorkload.h (92%) rename dbms/src/Storages/Page/{stress => }/workload/PageStorageInMemoryCapacity.cpp (96%) rename dbms/src/Storages/Page/{stress => }/workload/ThousandsOfOffset.cpp (97%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e14c205f18..f2ec9f3316b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -432,9 +432,6 @@ else (ENABLE_FAILPOINTS) message (STATUS "Failpoints are disabled") endif (ENABLE_FAILPOINTS) -# Enable PageStorage V3 test. -option (ENABLE_V3_PAGESTORAGE "Enables V3 PageStorage" ON) - # Flags for test coverage option (TEST_COVERAGE "Enables flags for test coverage" OFF) option (TEST_COVERAGE_XML "Output XML report for test coverage" OFF) diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index cce11bd6997..e1e52fab73b 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -91,12 +91,10 @@ add_headers_and_sources(dbms src/Storages/Page/V2/VersionSet) add_headers_and_sources(dbms src/Storages/Page/V2/gc) add_headers_and_sources(dbms src/WindowFunctions) add_headers_and_sources(dbms src/TiDB/Schema) -if (ENABLE_V3_PAGESTORAGE) - add_headers_and_sources(dbms src/Storages/Page/V3) - add_headers_and_sources(dbms src/Storages/Page/V3/LogFile) - add_headers_and_sources(dbms src/Storages/Page/V3/WAL) - add_headers_and_sources(dbms src/Storages/Page/V3/spacemap) -endif() +add_headers_and_sources(dbms src/Storages/Page/V3) +add_headers_and_sources(dbms src/Storages/Page/V3/LogFile) +add_headers_and_sources(dbms src/Storages/Page/V3/WAL) +add_headers_and_sources(dbms src/Storages/Page/V3/spacemap) add_headers_and_sources(dbms src/Storages/Page/) add_headers_and_sources(dbms src/TiDB) add_headers_and_sources(dbms src/Client) @@ -323,6 +321,9 @@ if (ENABLE_TESTS) if (ENABLE_TIFLASH_DTWORKLOAD) target_link_libraries(bench_dbms dt-workload-lib) endif () + if (ENABLE_TIFLASH_PAGEWORKLOAD) + target_link_libraries(bench_dbms page-workload-lib) + endif () add_check(bench_dbms) endif () diff --git a/dbms/src/Server/CMakeLists.txt b/dbms/src/Server/CMakeLists.txt index 63cf6d0e1f9..104b4f34e4a 100644 --- a/dbms/src/Server/CMakeLists.txt +++ b/dbms/src/Server/CMakeLists.txt @@ -22,6 +22,7 @@ option(ENABLE_CLICKHOUSE_SERVER "Enable server" ${ENABLE_CLICKHOUSE_ALL}) option(ENABLE_CLICKHOUSE_CLIENT "Enable client" ${ENABLE_CLICKHOUSE_ALL}) option(ENABLE_TIFLASH_DTTOOL "Enable dttool: tools to manage dmfile" ${ENABLE_CLICKHOUSE_ALL}) option(ENABLE_TIFLASH_DTWORKLOAD "Enable dtworkload: tools to test and stress DeltaTree" ${ENABLE_CLICKHOUSE_ALL}) +option(ENABLE_TIFLASH_PAGEWORKLOAD "Enable pageworkload: tools to test and stress PageStorage" ${ENABLE_CLICKHOUSE_ALL}) option(ENABLE_TIFLASH_PAGECTL "Enable pagectl: tools to debug page storage" ${ENABLE_CLICKHOUSE_ALL}) configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h) @@ -136,6 +137,9 @@ endif () if (ENABLE_TIFLASH_DTWORKLOAD) target_link_libraries(tiflash dt-workload-lib) endif () +if (ENABLE_TIFLASH_PAGEWORKLOAD) + target_link_libraries(tiflash page-workload-lib) +endif() if (ENABLE_TIFLASH_PAGECTL) target_link_libraries(tiflash page-ctl-lib) endif () diff --git a/dbms/src/Server/config_tools.h.in b/dbms/src/Server/config_tools.h.in index 61aa3f41591..03a478a6473 100644 --- a/dbms/src/Server/config_tools.h.in +++ b/dbms/src/Server/config_tools.h.in @@ -6,4 +6,5 @@ #cmakedefine01 ENABLE_CLICKHOUSE_CLIENT #cmakedefine01 ENABLE_TIFLASH_DTTOOL #cmakedefine01 ENABLE_TIFLASH_DTWORKLOAD +#cmakedefine01 ENABLE_TIFLASH_PAGEWORKLOAD #cmakedefine01 ENABLE_TIFLASH_PAGECTL diff --git a/dbms/src/Server/main.cpp b/dbms/src/Server/main.cpp index 11cccf84729..dbcaa4f38fc 100644 --- a/dbms/src/Server/main.cpp +++ b/dbms/src/Server/main.cpp @@ -36,7 +36,10 @@ #include #endif #if ENABLE_TIFLASH_DTWORKLOAD -#include +#include +#endif +#if ENABLE_TIFLASH_PAGEWORKLOAD +#include #endif #if ENABLE_TIFLASH_PAGECTL #include @@ -107,6 +110,9 @@ std::pair clickhouse_applications[] = { #if ENABLE_TIFLASH_DTWORKLOAD {"dtworkload", DB::DM::tests::DTWorkload::mainEntry}, #endif +#if ENABLE_TIFLASH_PAGEWORKLOAD + {"pageworkload", DB::PS::tests::StressWorkload::mainEntry}, +#endif #if ENABLE_TIFLASH_PAGECTL {"pagectl", DB::PageStorageCtl::mainEntry}, #endif diff --git a/dbms/src/Storages/CMakeLists.txt b/dbms/src/Storages/CMakeLists.txt index 90cc7a01d5b..68a2e6c9a74 100644 --- a/dbms/src/Storages/CMakeLists.txt +++ b/dbms/src/Storages/CMakeLists.txt @@ -15,16 +15,15 @@ add_subdirectory (System) add_subdirectory (Page) add_subdirectory (DeltaMerge/File/dtpb) -add_subdirectory (DeltaMerge/tools) +add_subdirectory (DeltaMerge/workload) +add_subdirectory (Page/workload) if (ENABLE_TESTS) add_subdirectory (tests EXCLUDE_FROM_ALL) add_subdirectory (Transaction/tests EXCLUDE_FROM_ALL) add_subdirectory (Page/V2/tests EXCLUDE_FROM_ALL) - if (ENABLE_V3_PAGESTORAGE) - add_subdirectory (Page/V3 EXCLUDE_FROM_ALL) - add_subdirectory (Page/V3/tests EXCLUDE_FROM_ALL) - endif () + add_subdirectory (Page/V3 EXCLUDE_FROM_ALL) + add_subdirectory (Page/V3/tests EXCLUDE_FROM_ALL) add_subdirectory (DeltaMerge/tests EXCLUDE_FROM_ALL) endif () diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/Main.cpp b/dbms/src/Storages/DeltaMerge/tools/workload/Main.cpp deleted file mode 100644 index 092c8a89a42..00000000000 --- a/dbms/src/Storages/DeltaMerge/tools/workload/Main.cpp +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -using namespace DB::DM::tests; - -int main(int argc, char ** argv) -{ - return DTWorkload::mainEntry(argc, argv); -} diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/CMakeLists.txt b/dbms/src/Storages/DeltaMerge/workload/CMakeLists.txt similarity index 86% rename from dbms/src/Storages/DeltaMerge/tools/workload/CMakeLists.txt rename to dbms/src/Storages/DeltaMerge/workload/CMakeLists.txt index 7227f1cf563..7a83cbec57c 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/CMakeLists.txt +++ b/dbms/src/Storages/DeltaMerge/workload/CMakeLists.txt @@ -18,6 +18,3 @@ set(dt-workload-src MainEntry.cpp DTWorkload.cpp KeyGenerator.cpp TableGenerator add_library(dt-workload-lib ${dt-workload-src}) target_link_libraries(dt-workload-lib dbms clickhouse_functions clickhouse-server-lib) - -add_executable(dt-workload Main.cpp ${dt-workload-src}) -target_link_libraries(dt-workload dbms gtest clickhouse_functions clickhouse-server-lib) diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/DTWorkload.cpp b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp similarity index 94% rename from dbms/src/Storages/DeltaMerge/tools/workload/DTWorkload.cpp rename to dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp index a6113f91d91..a53a1b9ebbd 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/DTWorkload.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp @@ -19,16 +19,16 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/DTWorkload.h b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.h similarity index 97% rename from dbms/src/Storages/DeltaMerge/tools/workload/DTWorkload.h rename to dbms/src/Storages/DeltaMerge/workload/DTWorkload.h index 26cc5b6e07c..1ee5ba6b871 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/DTWorkload.h +++ b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.h @@ -73,7 +73,7 @@ class ThreadStat class Statistics { public: - Statistics(int write_thread_count = 0, int read_thread_count = 0) + explicit Statistics(int write_thread_count = 0, int read_thread_count = 0) : init_ms(0) , write_stats(write_thread_count) , read_stats(read_thread_count) diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/DataGenerator.cpp b/dbms/src/Storages/DeltaMerge/workload/DataGenerator.cpp similarity index 95% rename from dbms/src/Storages/DeltaMerge/tools/workload/DataGenerator.cpp rename to dbms/src/Storages/DeltaMerge/workload/DataGenerator.cpp index be6ff1dcbbe..479977d46d1 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/DataGenerator.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/DataGenerator.cpp @@ -13,11 +13,11 @@ // limitations under the License. #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include @@ -33,7 +33,7 @@ class RandomDataGenerator : public DataGenerator , rand_gen(std::random_device()()) {} - virtual std::tuple get(uint64_t key) override + std::tuple get(uint64_t key) override { Block block; // Generate 'rowkeys'. @@ -227,7 +227,9 @@ class RandomDataGenerator : public DataGenerator struct tm randomLocalTime() { time_t t = randomUTCTimestamp(); - struct tm res; + struct tm res + { + }; if (localtime_r(&t, &res) == nullptr) { throw std::invalid_argument(fmt::format("localtime_r({}) ret {}", t, strerror(errno))); diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/DataGenerator.h b/dbms/src/Storages/DeltaMerge/workload/DataGenerator.h similarity index 96% rename from dbms/src/Storages/DeltaMerge/tools/workload/DataGenerator.h rename to dbms/src/Storages/DeltaMerge/workload/DataGenerator.h index e32de4591e6..cd29f1a3a80 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/DataGenerator.h +++ b/dbms/src/Storages/DeltaMerge/workload/DataGenerator.h @@ -27,7 +27,7 @@ class DataGenerator public: static std::unique_ptr create(const WorkloadOptions & opts, const TableInfo & table_info, TimestampGenerator & ts_gen); virtual std::tuple get(uint64_t key) = 0; - virtual ~DataGenerator() {} + virtual ~DataGenerator() = default; }; std::string blockToString(const Block & block); diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/Handle.h b/dbms/src/Storages/DeltaMerge/workload/Handle.h similarity index 90% rename from dbms/src/Storages/DeltaMerge/tools/workload/Handle.h rename to dbms/src/Storages/DeltaMerge/workload/Handle.h index eb117a4fddd..c4949c15a1f 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/Handle.h +++ b/dbms/src/Storages/DeltaMerge/workload/Handle.h @@ -16,7 +16,7 @@ #include #include -#include +#include #include #include @@ -40,7 +40,7 @@ class HandleLock static constexpr uint64_t default_lock_count = 4096; static std::unique_ptr create(const TableInfo & table_info); - HandleLock(uint64_t lock_count = default_lock_count) + explicit HandleLock(uint64_t lock_count = default_lock_count) : rmtxs(lock_count) {} @@ -51,14 +51,14 @@ class HandleLock std::vector> getLocks(const std::vector & handles) { - std::vector indexes; + std::vector indexes(handles.size()); for (const auto & h : handles) { indexes.push_back(index(h)); } // Sort mutex indexes to avoid dead lock. sort(indexes.begin(), indexes.end()); - std::vector> locks; + std::vector> locks(indexes.size()); for (auto i : indexes) { locks.push_back(getLockByIndex(i)); @@ -105,7 +105,7 @@ class HandleTable std::lock_guard lock(mtx); handle_to_ts[handle] = ts; Record r{handle, ts}; - if (wal != nullptr && wal->write((char *)&r, sizeof(r)) != sizeof(r)) + if (wal != nullptr && wal->write(reinterpret_cast(&r), sizeof(r)) != sizeof(r)) { throw std::runtime_error(fmt::format("write ret {}", strerror(errno))); } @@ -134,8 +134,8 @@ class HandleTable try { PosixRandomAccessFile f(fname, -1); - Record r; - while (f.read((char *)&r, sizeof(r)) == sizeof(r)) + Record r{}; + while (f.read(reinterpret_cast(&r), sizeof(r)) == sizeof(r)) { handle_to_ts[r.handle] = r.ts; } @@ -156,7 +156,7 @@ class HandleTable for (const auto & pa : handle_to_ts) { Record r{pa.first, pa.second}; - if (f.write((char *)&r, sizeof(r)) != sizeof(r)) + if (f.write(reinterpret_cast(&r), sizeof(r)) != sizeof(r)) { throw std::runtime_error(fmt::format("write ret {}", strerror(errno))); } @@ -191,7 +191,7 @@ class SharedHandleTable public: static constexpr uint64_t default_shared_count = 4096; - SharedHandleTable(uint64_t max_key_count, const std::string & waldir = "", uint64_t shared_cnt = default_shared_count) + explicit SharedHandleTable(uint64_t max_key_count, const std::string & waldir = "", uint64_t shared_cnt = default_shared_count) : tables(shared_cnt) { uint64_t max_key_count_per_shared = max_key_count / default_shared_count + 1; diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/KeyGenerator.cpp b/dbms/src/Storages/DeltaMerge/workload/KeyGenerator.cpp similarity index 92% rename from dbms/src/Storages/DeltaMerge/tools/workload/KeyGenerator.cpp rename to dbms/src/Storages/DeltaMerge/workload/KeyGenerator.cpp index bb2f2253279..f899ec71b4b 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/KeyGenerator.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/KeyGenerator.cpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include -#include +#include +#include #include #include @@ -31,7 +31,7 @@ class IncrementalKeyGenerator : public KeyGenerator , key(0) {} - virtual uint64_t get64() override + uint64_t get64() override { return key.fetch_add(1, std::memory_order_relaxed) % key_count + start_key; } @@ -54,7 +54,7 @@ class UniformDistributionKeyGenerator : public KeyGenerator , uniform_dist(0, key_count) {} - virtual uint64_t get64() override + uint64_t get64() override { std::lock_guard lock(mtx); return uniform_dist(rand_gen); @@ -78,7 +78,7 @@ class NormalDistributionKeyGenerator : public KeyGenerator , normal_dist(key_count / 2.0, key_count / 20.0) {} - virtual uint64_t get64() override + uint64_t get64() override { std::lock_guard lock(mtx); return normal_dist(rand_gen); diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/KeyGenerator.h b/dbms/src/Storages/DeltaMerge/workload/KeyGenerator.h similarity index 92% rename from dbms/src/Storages/DeltaMerge/tools/workload/KeyGenerator.h rename to dbms/src/Storages/DeltaMerge/workload/KeyGenerator.h index 447f3ffc27a..7c8b8fd0080 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/KeyGenerator.h +++ b/dbms/src/Storages/DeltaMerge/workload/KeyGenerator.h @@ -23,8 +23,8 @@ class KeyGenerator public: static std::unique_ptr create(const WorkloadOptions & opts); - KeyGenerator() {} - virtual ~KeyGenerator() {} + KeyGenerator() = default; + virtual ~KeyGenerator() = default; virtual uint64_t get64() = 0; }; diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/Limiter.cpp b/dbms/src/Storages/DeltaMerge/workload/Limiter.cpp similarity index 77% rename from dbms/src/Storages/DeltaMerge/tools/workload/Limiter.cpp rename to dbms/src/Storages/DeltaMerge/workload/Limiter.cpp index 73764d27bc5..65f9e3ce72c 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/Limiter.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/Limiter.cpp @@ -13,8 +13,8 @@ // limitations under the License. #include -#include -#include +#include +#include #include #include @@ -24,10 +24,10 @@ namespace DB::DM::tests class ConstantLimiter : public Limiter { public: - ConstantLimiter(uint64_t rate_per_sec) + explicit ConstantLimiter(uint64_t rate_per_sec) : limiter(rate_per_sec, LimiterType::UNKNOW) {} - virtual void request() override + void request() override { limiter.request(1); } @@ -38,7 +38,7 @@ class ConstantLimiter : public Limiter std::unique_ptr Limiter::create(const WorkloadOptions & opts) { - uint64_t per_sec = std::ceil(static_cast(opts.max_write_per_sec / opts.write_thread_count)); + uint64_t per_sec = std::ceil(opts.max_write_per_sec * 1.0 / opts.write_thread_count); return std::make_unique(per_sec); } diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/Limiter.h b/dbms/src/Storages/DeltaMerge/workload/Limiter.h similarity index 96% rename from dbms/src/Storages/DeltaMerge/tools/workload/Limiter.h rename to dbms/src/Storages/DeltaMerge/workload/Limiter.h index e2892b178a2..da2d31c7915 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/Limiter.h +++ b/dbms/src/Storages/DeltaMerge/workload/Limiter.h @@ -23,6 +23,6 @@ class Limiter public: static std::unique_ptr create(const WorkloadOptions & opts); virtual void request() = 0; - virtual ~Limiter() {} + virtual ~Limiter() = default; }; } // namespace DB::DM::tests \ No newline at end of file diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/MainEntry.cpp b/dbms/src/Storages/DeltaMerge/workload/MainEntry.cpp similarity index 97% rename from dbms/src/Storages/DeltaMerge/tools/workload/MainEntry.cpp rename to dbms/src/Storages/DeltaMerge/workload/MainEntry.cpp index f79d414f20b..88cf0b6322f 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/MainEntry.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/MainEntry.cpp @@ -14,10 +14,10 @@ #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include #include diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/Options.cpp b/dbms/src/Storages/DeltaMerge/workload/Options.cpp similarity index 98% rename from dbms/src/Storages/DeltaMerge/tools/workload/Options.cpp rename to dbms/src/Storages/DeltaMerge/workload/Options.cpp index 1c6409f3c53..8545d22ca8d 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/Options.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/Options.cpp @@ -13,8 +13,8 @@ // limitations under the License. #include -#include -#include +#include +#include #include #include diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/Options.h b/dbms/src/Storages/DeltaMerge/workload/Options.h similarity index 100% rename from dbms/src/Storages/DeltaMerge/tools/workload/Options.h rename to dbms/src/Storages/DeltaMerge/workload/Options.h diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/ReadColumnsGenerator.h b/dbms/src/Storages/DeltaMerge/workload/ReadColumnsGenerator.h similarity index 93% rename from dbms/src/Storages/DeltaMerge/tools/workload/ReadColumnsGenerator.h rename to dbms/src/Storages/DeltaMerge/workload/ReadColumnsGenerator.h index 180409f89e1..c881bb148a2 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/ReadColumnsGenerator.h +++ b/dbms/src/Storages/DeltaMerge/workload/ReadColumnsGenerator.h @@ -14,7 +14,7 @@ #pragma once -#include +#include #include @@ -28,7 +28,7 @@ class ReadColumnsGenerator return std::make_unique(table_info); } - ReadColumnsGenerator(const TableInfo & table_info_) + explicit ReadColumnsGenerator(const TableInfo & table_info_) : table_info(table_info_) , rand_gen(std::random_device()()) , uniform_dist(0, table_info_.columns->size() - 1) diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/TableGenerator.cpp b/dbms/src/Storages/DeltaMerge/workload/TableGenerator.cpp similarity index 97% rename from dbms/src/Storages/DeltaMerge/tools/workload/TableGenerator.cpp rename to dbms/src/Storages/DeltaMerge/workload/TableGenerator.cpp index cf52e808ab1..ec29a476d6a 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/TableGenerator.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/TableGenerator.cpp @@ -15,8 +15,8 @@ #include #include #include -#include -#include +#include +#include #include #include @@ -237,7 +237,7 @@ class RandomTableGenerator : public TableGenerator , rand_gen(std::random_device()()) {} - virtual TableInfo get(int64_t table_id, std::string table_name) override + TableInfo get(int64_t table_id, std::string table_name) override { TableInfo table_info; @@ -293,7 +293,7 @@ class RandomTableGenerator : public TableGenerator class ConstantTableGenerator : public TableGenerator { - virtual TableInfo get(int64_t table_id, std::string table_name) override + TableInfo get(int64_t table_id, std::string table_name) override { TableInfo table_info; diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/TableGenerator.h b/dbms/src/Storages/DeltaMerge/workload/TableGenerator.h similarity index 96% rename from dbms/src/Storages/DeltaMerge/tools/workload/TableGenerator.h rename to dbms/src/Storages/DeltaMerge/workload/TableGenerator.h index aba5c1590b7..b88bf2b72e2 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/TableGenerator.h +++ b/dbms/src/Storages/DeltaMerge/workload/TableGenerator.h @@ -38,6 +38,6 @@ class TableGenerator virtual TableInfo get(int64_t table_id, std::string table_name) = 0; - virtual ~TableGenerator() {} + virtual ~TableGenerator() = default; }; } // namespace DB::DM::tests \ No newline at end of file diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/TimestampGenerator.h b/dbms/src/Storages/DeltaMerge/workload/TimestampGenerator.h similarity index 100% rename from dbms/src/Storages/DeltaMerge/tools/workload/TimestampGenerator.h rename to dbms/src/Storages/DeltaMerge/workload/TimestampGenerator.h diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/Utils.cpp b/dbms/src/Storages/DeltaMerge/workload/Utils.cpp similarity index 94% rename from dbms/src/Storages/DeltaMerge/tools/workload/Utils.cpp rename to dbms/src/Storages/DeltaMerge/workload/Utils.cpp index 1cefae724c6..80d9f788016 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/Utils.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/Utils.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include #include @@ -83,7 +83,7 @@ std::string fieldToString(const DataTypePtr & data_type, const Field & f) } else if (t == Field::Types::Which::Decimal256) { - auto i = f.get(); + const auto & i = f.get(); auto scale = dynamic_cast(data_type.get())->getScale(); return i.toString(scale); } @@ -105,8 +105,8 @@ std::vector colToVec(const DataTypePtr & data_type, const ColumnPtr std::string blockToString(const Block & block) { std::string s = "id name type values\n"; - auto & cols = block.getColumnsWithTypeAndName(); - for (auto & col : cols) + const auto & cols = block.getColumnsWithTypeAndName(); + for (const auto & col : cols) { s += fmt::format("{} {} {} {}\n", col.column_id, col.name, col.type->getFamilyName(), colToVec(col.type, col.column)); } diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/Utils.h b/dbms/src/Storages/DeltaMerge/workload/Utils.h similarity index 100% rename from dbms/src/Storages/DeltaMerge/tools/workload/Utils.h rename to dbms/src/Storages/DeltaMerge/workload/Utils.h diff --git a/dbms/src/Storages/Page/CMakeLists.txt b/dbms/src/Storages/Page/CMakeLists.txt index cead83fa126..f208dc84be2 100644 --- a/dbms/src/Storages/Page/CMakeLists.txt +++ b/dbms/src/Storages/Page/CMakeLists.txt @@ -14,13 +14,3 @@ add_subdirectory(V2) add_subdirectory(tools) - -# PageStorage Stress test -if (ENABLE_V3_PAGESTORAGE) - add_headers_and_sources(page_stress_testing stress) - add_headers_and_sources(page_stress_testing stress/workload) - add_executable(page_stress_testing EXCLUDE_FROM_ALL ${page_stress_testing_sources}) - target_link_libraries(page_stress_testing dbms page_storage_v3) - target_include_directories(page_stress_testing PRIVATE stress) - target_compile_options(page_stress_testing PRIVATE -Wno-format -lc++) # turn off printf format check -endif() \ No newline at end of file diff --git a/dbms/src/Storages/Page/stress/stress_page_storage.cpp b/dbms/src/Storages/Page/stress/stress_page_storage.cpp deleted file mode 100644 index 818be710363..00000000000 --- a/dbms/src/Storages/Page/stress/stress_page_storage.cpp +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -namespace DB -{ -// Define is_background_thread for this binary -// It is required for `RateLimiter` but we do not link with `BackgroundProcessingPool`. -#if __APPLE__ && __clang__ -__thread bool is_background_thread = false; -#else -thread_local bool is_background_thread = false; -#endif -} // namespace DB - -int main(int argc, char ** argv) -try -{ - StressEnv::initGlobalLogger(); - auto env = StressEnv::parse(argc, argv); - env.setup(); - - auto & mamager = StressWorkloadManger::getInstance(); - mamager.setEnv(env); - mamager.runWorkload(); - - return StressEnvStatus::getInstance().isSuccess(); -} -catch (...) -{ - DB::tryLogCurrentException(""); - exit(-1); -} diff --git a/dbms/src/Storages/DeltaMerge/tools/CMakeLists.txt b/dbms/src/Storages/Page/workload/CMakeLists.txt similarity index 53% rename from dbms/src/Storages/DeltaMerge/tools/CMakeLists.txt rename to dbms/src/Storages/Page/workload/CMakeLists.txt index 36270a0c8e4..5c8ecb34d97 100644 --- a/dbms/src/Storages/DeltaMerge/tools/CMakeLists.txt +++ b/dbms/src/Storages/Page/workload/CMakeLists.txt @@ -14,4 +14,9 @@ include_directories (${CMAKE_CURRENT_BINARY_DIR}) -add_subdirectory (workload EXCLUDE_FROM_ALL) +set (page-workload-src HeavyMemoryCostInGC.cpp HeavyRead.cpp HeavySkewWriteRead.cpp HeavyWrite.cpp HighValidBigFileGC.cpp HoldSnapshotsLongTime.cpp Normal.cpp + PageStorageInMemoryCapacity.cpp ThousandsOfOffset.cpp MainEntry.cpp Normal.cpp PageStorageInMemoryCapacity.cpp PSBackground.cpp PSRunnable.cpp PSStressEnv.cpp PSWorkload.cpp) + +add_library (page-workload-lib ${page-workload-src}) +target_link_libraries (page-workload-lib dbms clickhouse_functions clickhouse-server-lib) +target_compile_options (page-workload-lib PRIVATE -Wno-format -lc++) \ No newline at end of file diff --git a/dbms/src/Storages/Page/stress/workload/HeavyMemoryCostInGC.cpp b/dbms/src/Storages/Page/workload/HeavyMemoryCostInGC.cpp similarity index 96% rename from dbms/src/Storages/Page/stress/workload/HeavyMemoryCostInGC.cpp rename to dbms/src/Storages/Page/workload/HeavyMemoryCostInGC.cpp index 40595f0cb59..7e745e29fc2 100644 --- a/dbms/src/Storages/Page/stress/workload/HeavyMemoryCostInGC.cpp +++ b/dbms/src/Storages/Page/workload/HeavyMemoryCostInGC.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +namespace DB::PS::tests +{ class HeavyMemoryCostInGC : public StressWorkload , public StressWorkloadFunc @@ -81,3 +83,4 @@ class HeavyMemoryCostInGC }; REGISTER_WORKLOAD(HeavyMemoryCostInGC) +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/workload/HeavyRead.cpp b/dbms/src/Storages/Page/workload/HeavyRead.cpp similarity index 94% rename from dbms/src/Storages/Page/stress/workload/HeavyRead.cpp rename to dbms/src/Storages/Page/workload/HeavyRead.cpp index 15aeb1320cf..a67c435e84c 100644 --- a/dbms/src/Storages/Page/stress/workload/HeavyRead.cpp +++ b/dbms/src/Storages/Page/workload/HeavyRead.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +namespace DB::PS::tests +{ class HeavyRead : public StressWorkload , public StressWorkloadFunc { @@ -69,4 +71,5 @@ class HeavyRead : public StressWorkload } }; -REGISTER_WORKLOAD(HeavyRead) \ No newline at end of file +REGISTER_WORKLOAD(HeavyRead) +} // namespace DB::PS::tests \ No newline at end of file diff --git a/dbms/src/Storages/Page/stress/workload/HeavySkewWriteRead.cpp b/dbms/src/Storages/Page/workload/HeavySkewWriteRead.cpp similarity index 95% rename from dbms/src/Storages/Page/stress/workload/HeavySkewWriteRead.cpp rename to dbms/src/Storages/Page/workload/HeavySkewWriteRead.cpp index 78ffa5b60e0..805bf105358 100644 --- a/dbms/src/Storages/Page/stress/workload/HeavySkewWriteRead.cpp +++ b/dbms/src/Storages/Page/workload/HeavySkewWriteRead.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +namespace DB::PS::tests +{ class HeavySkewWriteRead : public StressWorkload , public StressWorkloadFunc { @@ -84,4 +86,5 @@ class HeavySkewWriteRead : public StressWorkload } }; -REGISTER_WORKLOAD(HeavySkewWriteRead) \ No newline at end of file +REGISTER_WORKLOAD(HeavySkewWriteRead) +} // namespace DB::PS::tests \ No newline at end of file diff --git a/dbms/src/Storages/Page/stress/workload/HeavyWrite.cpp b/dbms/src/Storages/Page/workload/HeavyWrite.cpp similarity index 94% rename from dbms/src/Storages/Page/stress/workload/HeavyWrite.cpp rename to dbms/src/Storages/Page/workload/HeavyWrite.cpp index 265b289db56..8dfd7f810f7 100644 --- a/dbms/src/Storages/Page/stress/workload/HeavyWrite.cpp +++ b/dbms/src/Storages/Page/workload/HeavyWrite.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +namespace DB::PS::tests +{ class HeavyWrite : public StressWorkload , public StressWorkloadFunc { @@ -71,4 +73,5 @@ class HeavyWrite : public StressWorkload } }; -REGISTER_WORKLOAD(HeavyWrite) \ No newline at end of file +REGISTER_WORKLOAD(HeavyWrite) +} // namespace DB::PS::tests \ No newline at end of file diff --git a/dbms/src/Storages/Page/stress/workload/HighValidBigFileGC.cpp b/dbms/src/Storages/Page/workload/HighValidBigFileGC.cpp similarity index 97% rename from dbms/src/Storages/Page/stress/workload/HighValidBigFileGC.cpp rename to dbms/src/Storages/Page/workload/HighValidBigFileGC.cpp index 866782c9578..a9af6aebb76 100644 --- a/dbms/src/Storages/Page/stress/workload/HighValidBigFileGC.cpp +++ b/dbms/src/Storages/Page/workload/HighValidBigFileGC.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +namespace DB::PS::tests +{ class HighValidBigFileGCWorkload : public StressWorkload , public StressWorkloadFunc @@ -129,3 +131,4 @@ class HighValidBigFileGCWorkload }; REGISTER_WORKLOAD(HighValidBigFileGCWorkload) +} // namespace DB::PS::tests \ No newline at end of file diff --git a/dbms/src/Storages/Page/stress/workload/HoldSnapshotsLongTime.cpp b/dbms/src/Storages/Page/workload/HoldSnapshotsLongTime.cpp similarity index 95% rename from dbms/src/Storages/Page/stress/workload/HoldSnapshotsLongTime.cpp rename to dbms/src/Storages/Page/workload/HoldSnapshotsLongTime.cpp index b49347fc858..f02fbf65bcd 100644 --- a/dbms/src/Storages/Page/stress/workload/HoldSnapshotsLongTime.cpp +++ b/dbms/src/Storages/Page/workload/HoldSnapshotsLongTime.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +namespace DB::PS::tests +{ class HoldSnapshotsLongTime : public StressWorkload , public StressWorkloadFunc { @@ -93,4 +95,5 @@ class HoldSnapshotsLongTime : public StressWorkload } }; -REGISTER_WORKLOAD(HoldSnapshotsLongTime) \ No newline at end of file +REGISTER_WORKLOAD(HoldSnapshotsLongTime) +} // namespace DB::PS::tests \ No newline at end of file diff --git a/dbms/src/Storages/Page/workload/MainEntry.cpp b/dbms/src/Storages/Page/workload/MainEntry.cpp new file mode 100644 index 00000000000..ac82e1ea4bc --- /dev/null +++ b/dbms/src/Storages/Page/workload/MainEntry.cpp @@ -0,0 +1,70 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include + +using namespace DB::PS::tests; + +int StressWorkload::mainEntry(int argc, char ** argv) +{ + { + // maybe due to sequence of linking, REGISTER_WORKLOAD is not visible to main function in dbms/src/Server/main.cpp + // cause that REGISTER_WORKLOAD will not be triggered before mainEntry + // we do this to trigger REGISTER_WORKLOAD explicitly. + void _work_load_register_named_HeavyMemoryCostInGC(); + void (*f)() = _work_load_register_named_HeavyMemoryCostInGC; + (void)f; + void _work_load_register_named_HeavyRead(); + f = _work_load_register_named_HeavyRead; + (void)f; + void _work_load_register_named_HeavySkewWriteRead(); + f = _work_load_register_named_HeavySkewWriteRead; + (void)f; + void _work_load_register_named_HeavyWrite(); + f = _work_load_register_named_HeavyWrite; + (void)f; + void _work_load_register_named_HighValidBigFileGCWorkload(); + f = _work_load_register_named_HighValidBigFileGCWorkload; + (void)f; + void _work_load_register_named_HoldSnapshotsLongTime(); + f = _work_load_register_named_HoldSnapshotsLongTime; + (void)f; + void _work_load_register_named_PageStorageInMemoryCapacity(); + f = _work_load_register_named_PageStorageInMemoryCapacity; + (void)f; + void _work_load_register_named_NormalWorkload(); + f = _work_load_register_named_NormalWorkload; + (void)f; + void _work_load_register_named_ThousandsOfOffset(); + f = _work_load_register_named_ThousandsOfOffset; + (void)f; + } + try + { + StressEnv::initGlobalLogger(); + auto env = StressEnv::parse(argc, argv); + env.setup(); + + auto & mamager = StressWorkloadManger::getInstance(); + mamager.setEnv(env); + mamager.runWorkload(); + + return StressEnvStatus::getInstance().isSuccess(); + } + catch (...) + { + DB::tryLogCurrentException(""); + exit(-1); + } +} \ No newline at end of file diff --git a/dbms/src/Storages/Page/stress/workload/Normal.cpp b/dbms/src/Storages/Page/workload/Normal.cpp similarity index 95% rename from dbms/src/Storages/Page/stress/workload/Normal.cpp rename to dbms/src/Storages/Page/workload/Normal.cpp index 0323b857613..57229395809 100644 --- a/dbms/src/Storages/Page/stress/workload/Normal.cpp +++ b/dbms/src/Storages/Page/workload/Normal.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +namespace DB::PS::tests +{ class NormalWorkload : public StressWorkload , public StressWorkloadFunc @@ -77,3 +79,4 @@ class NormalWorkload }; REGISTER_WORKLOAD(NormalWorkload) +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/PSBackground.cpp b/dbms/src/Storages/Page/workload/PSBackground.cpp similarity index 96% rename from dbms/src/Storages/Page/stress/PSBackground.cpp rename to dbms/src/Storages/Page/workload/PSBackground.cpp index af7329e8348..247bea23dcc 100644 --- a/dbms/src/Storages/Page/stress/PSBackground.cpp +++ b/dbms/src/Storages/Page/workload/PSBackground.cpp @@ -13,11 +13,14 @@ // limitations under the License. #include -#include #include #include +#include #include + +namespace DB::PS::tests +{ void PSMetricsDumper::onTime(Poco::Timer & /*timer*/) { for (auto & metric : metrics) @@ -107,3 +110,4 @@ void StressTimeout::start() { timeout_timer.start(Poco::TimerCallback(*this, &StressTimeout::onTime)); } +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/PSBackground.h b/dbms/src/Storages/Page/workload/PSBackground.h similarity index 97% rename from dbms/src/Storages/Page/stress/PSBackground.h rename to dbms/src/Storages/Page/workload/PSBackground.h index 8c22458c5e8..c91dad1361f 100644 --- a/dbms/src/Storages/Page/stress/PSBackground.h +++ b/dbms/src/Storages/Page/workload/PSBackground.h @@ -15,14 +15,16 @@ #pragma once #include #include -#include #include +#include namespace CurrentMetrics { extern const Metric PSMVCCSnapshotsList; } +namespace DB::PS::tests +{ class PSMetricsDumper { public: @@ -162,3 +164,4 @@ class StressTimeout Poco::Timer timeout_timer; }; using StressTimeoutPtr = std::shared_ptr; +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/PSRunnable.cpp b/dbms/src/Storages/Page/workload/PSRunnable.cpp similarity index 97% rename from dbms/src/Storages/Page/stress/PSRunnable.cpp rename to dbms/src/Storages/Page/workload/PSRunnable.cpp index 5d6c8ecc5c6..5e9774ccc99 100644 --- a/dbms/src/Storages/Page/stress/PSRunnable.cpp +++ b/dbms/src/Storages/Page/workload/PSRunnable.cpp @@ -16,14 +16,16 @@ #include #include #include -#include #include #include +#include #include #include #include +namespace DB::PS::tests +{ void PSRunnable::run() try { @@ -69,7 +71,7 @@ DB::ReadBufferPtr PSWriter::genRandomData(const DB::PageId pageId, DB::MemHolder std::uniform_int_distribution<> dist(0, 3000); const size_t buff_sz = approx_page_mb * DB::MB + dist(size_gen); - char * buff = static_cast(malloc(buff_sz)); + char * buff = static_cast(malloc(buff_sz)); // NOLINT if (buff == nullptr) { throw DB::Exception("Alloc fix memory failed.", DB::ErrorCodes::LOGICAL_ERROR); @@ -78,7 +80,7 @@ DB::ReadBufferPtr PSWriter::genRandomData(const DB::PageId pageId, DB::MemHolder const char buff_ch = pageId % 0xFF; memset(buff, buff_ch, buff_sz); - holder = DB::createMemHolder(buff, [&](char * p) { free(p); }); + holder = DB::createMemHolder(buff, [&](char * p) { free(p); }); // NOLINT return std::make_shared(const_cast(buff), buff_sz); } @@ -88,7 +90,7 @@ void PSWriter::updatedRandomData() size_t memory_size = approx_page_mb * DB::MB * 2; if (memory == nullptr) { - memory = static_cast(malloc(memory_size)); + memory = static_cast(malloc(memory_size)); // NOLINT if (memory == nullptr) { throw DB::Exception("Alloc fix memory failed.", DB::ErrorCodes::LOGICAL_ERROR); @@ -147,7 +149,7 @@ void PSCommonWriter::updatedRandomData() if (memory == nullptr) { - memory = static_cast(malloc(memory_size)); + memory = static_cast(malloc(memory_size)); // NOLINT if (memory == nullptr) { throw DB::Exception("Alloc fix memory failed.", DB::ErrorCodes::LOGICAL_ERROR); @@ -415,3 +417,4 @@ DB::PageId PSIncreaseWriter::genRandomPageId() { return static_cast(begin_page_id++); } +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/PSRunnable.h b/dbms/src/Storages/Page/workload/PSRunnable.h similarity index 90% rename from dbms/src/Storages/Page/stress/PSRunnable.h rename to dbms/src/Storages/Page/workload/PSRunnable.h index 3ddcd73c093..b723236391d 100644 --- a/dbms/src/Storages/Page/stress/PSRunnable.h +++ b/dbms/src/Storages/Page/workload/PSRunnable.h @@ -13,12 +13,14 @@ // limitations under the License. #pragma once -#include #include #include +#include const DB::PageId MAX_PAGE_ID_DEFAULT = 1000; +namespace DB::PS::tests +{ class PSRunnable : public Poco::Runnable { public: @@ -46,7 +48,7 @@ class PSWriter : public PSRunnable gen.seed(time(nullptr)); } - virtual ~PSWriter() + ~PSWriter() override { if (memory != nullptr) { @@ -54,7 +56,7 @@ class PSWriter : public PSRunnable } } - virtual String description() override + String description() override { return fmt::format("(Stress Test Writer {})", index); } @@ -67,7 +69,7 @@ class PSWriter : public PSRunnable static void fillAllPages(const PSPtr & ps); - virtual bool runImpl() override; + bool runImpl() override; protected: virtual DB::PageId genRandomPageId(); @@ -91,11 +93,11 @@ class PSCommonWriter : public PSWriter : PSWriter(ps_, index_) {} - virtual void updatedRandomData() override; + void updatedRandomData() override; - virtual String description() override { return fmt::format("(Stress Test Common Writer {})", index); } + String description() override { return fmt::format("(Stress Test Common Writer {})", index); } - virtual bool runImpl() override; + bool runImpl() override; void setBatchBufferNums(size_t numbers); @@ -120,7 +122,7 @@ class PSCommonWriter : public PSWriter DB::PageFieldSizes data_sizes = {}; - virtual DB::PageId genRandomPageId() override; + DB::PageId genRandomPageId() override; virtual size_t genBufferSize(); }; @@ -154,7 +156,7 @@ class PSWindowWriter : public PSCommonWriter void setNormalDistributionSigma(size_t sigma); protected: - virtual DB::PageId genRandomPageId() override; + DB::PageId genRandomPageId() override; protected: size_t window_size = 100; @@ -170,12 +172,12 @@ class PSIncreaseWriter : public PSCommonWriter String description() override { return fmt::format("(Stress Test Increase Writer {})", index); } - virtual bool runImpl() override; + bool runImpl() override; void setPageRange(size_t page_range); protected: - virtual DB::PageId genRandomPageId() override; + DB::PageId genRandomPageId() override; protected: size_t begin_page_id = 1; @@ -192,9 +194,9 @@ class PSReader : public PSRunnable gen.seed(time(nullptr)); } - virtual String description() override { return fmt::format("(Stress Test PSReader {})", index); } + String description() override { return fmt::format("(Stress Test PSReader {})", index); } - virtual bool runImpl() override; + bool runImpl() override; void setPageReadOnce(size_t page_read_once); @@ -242,7 +244,7 @@ class PSWindowReader : public PSReader void setWriterNums(size_t writer_nums); protected: - virtual DB::PageIds genRandomPageIds() override; + DB::PageIds genRandomPageIds() override; protected: size_t window_size = 100; @@ -261,12 +263,13 @@ class PSSnapshotReader : public PSReader : PSReader(ps_, index_) {} - virtual bool runImpl() override; + bool runImpl() override; void setSnapshotGetIntervalMs(size_t snapshot_get_interval_ms_); protected: - size_t snapshots_hold_num; + size_t snapshots_hold_num = 0; size_t snapshot_get_interval_ms = 0; std::list snapshots; -}; \ No newline at end of file +}; +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/PSStressEnv.cpp b/dbms/src/Storages/Page/workload/PSStressEnv.cpp similarity index 97% rename from dbms/src/Storages/Page/stress/PSStressEnv.cpp rename to dbms/src/Storages/Page/workload/PSStressEnv.cpp index 7d680cd43c0..f5cead0a158 100644 --- a/dbms/src/Storages/Page/stress/PSStressEnv.cpp +++ b/dbms/src/Storages/Page/workload/PSStressEnv.cpp @@ -16,18 +16,20 @@ #include #include #include -#include -#include #include #include #include #include #include #include +#include +#include #include #include +namespace DB::PS::tests +{ Poco::Logger * StressEnv::logger; void StressEnv::initGlobalLogger() { @@ -146,3 +148,4 @@ void StressEnv::setup() init_pages = true; setupSignal(); } +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/PSStressEnv.h b/dbms/src/Storages/Page/workload/PSStressEnv.h similarity index 98% rename from dbms/src/Storages/Page/stress/PSStressEnv.h rename to dbms/src/Storages/Page/workload/PSStressEnv.h index 1c7d8ee761f..e67cb325430 100644 --- a/dbms/src/Storages/Page/stress/PSStressEnv.h +++ b/dbms/src/Storages/Page/workload/PSStressEnv.h @@ -25,6 +25,8 @@ namespace Poco class Logger; } +namespace DB::PS::tests +{ using PSPtr = std::shared_ptr; enum StressEnvStat @@ -124,3 +126,4 @@ struct StressEnv void setup(); }; +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/PSWorkload.cpp b/dbms/src/Storages/Page/workload/PSWorkload.cpp similarity index 98% rename from dbms/src/Storages/Page/stress/PSWorkload.cpp rename to dbms/src/Storages/Page/workload/PSWorkload.cpp index ce1f8d92ce0..81f13527f48 100644 --- a/dbms/src/Storages/Page/stress/PSWorkload.cpp +++ b/dbms/src/Storages/Page/workload/PSWorkload.cpp @@ -14,12 +14,14 @@ #include #include -#include #include #include #include +#include #include +namespace DB::PS::tests +{ void StressWorkload::onDumpResult() { UInt64 time_interval = stop_watch.elapsedMilliseconds(); @@ -177,3 +179,4 @@ void StressWorkloadManger::runWorkload() } } } +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/PSWorkload.h b/dbms/src/Storages/Page/workload/PSWorkload.h similarity index 92% rename from dbms/src/Storages/Page/stress/PSWorkload.h rename to dbms/src/Storages/Page/workload/PSWorkload.h index cb099b4203a..eaaaf4eba5b 100644 --- a/dbms/src/Storages/Page/stress/PSWorkload.h +++ b/dbms/src/Storages/Page/workload/PSWorkload.h @@ -16,15 +16,17 @@ #include #include -#include -#include -#include #include #include #include +#include +#include +#include #include #define NORMAL_WORKLOAD 0 +namespace DB::PS::tests +{ template class StressWorkloadFunc { @@ -45,6 +47,8 @@ class StressWorkloadFunc class StressWorkload { public: + static int mainEntry(int argc, char ** argv); + explicit StressWorkload(StressEnv options_) : options(options_) {} @@ -189,13 +193,15 @@ class StressWorkloadManger StressEnv options; }; -#define REGISTER_WORKLOAD(WORKLOAD) \ - static void __attribute__((constructor)) _work_load_register_named_##WORKLOAD(void) \ - { \ - StressWorkloadManger::getInstance().reg( \ - WORKLOAD::nameFunc(), \ - WORKLOAD::maskFunc(), \ - [](const StressEnv & opts) -> std::shared_ptr { \ - return std::make_shared(opts); \ - }); \ +#define REGISTER_WORKLOAD(WORKLOAD) \ + void __attribute__((constructor)) _work_load_register_named_##WORKLOAD(void) \ + { \ + StressWorkloadManger::getInstance().reg( \ + WORKLOAD::nameFunc(), \ + WORKLOAD::maskFunc(), \ + [](const StressEnv & opts) -> std::shared_ptr { \ + return std::make_shared(opts); \ + }); \ } + +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/workload/PageStorageInMemoryCapacity.cpp b/dbms/src/Storages/Page/workload/PageStorageInMemoryCapacity.cpp similarity index 96% rename from dbms/src/Storages/Page/stress/workload/PageStorageInMemoryCapacity.cpp rename to dbms/src/Storages/Page/workload/PageStorageInMemoryCapacity.cpp index 190cbf6b323..6ab321d1a10 100644 --- a/dbms/src/Storages/Page/stress/workload/PageStorageInMemoryCapacity.cpp +++ b/dbms/src/Storages/Page/workload/PageStorageInMemoryCapacity.cpp @@ -12,13 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include #include #include #include - #ifdef __APPLE__ #include @@ -27,6 +26,8 @@ #include #endif +namespace DB::PS::tests +{ class PageStorageInMemoryCapacity : public StressWorkload , public StressWorkloadFunc { @@ -89,14 +90,14 @@ class PageStorageInMemoryCapacity : public StressWorkload } FILE * file = fopen("/proc/meminfo", "r"); - if (file != NULL) + if (file != nullptr) { char buffer[128]; #define MEMORY_TOTAL_LABEL "MemTotal:" while (fgets(buffer, 128, file)) { if ((strncmp((buffer), (MEMORY_TOTAL_LABEL), strlen(MEMORY_TOTAL_LABEL)) == 0) - && sscanf(buffer + strlen(MEMORY_TOTAL_LABEL), " %32llu kB", &total_mem)) + && sscanf(buffer + strlen(MEMORY_TOTAL_LABEL), " %32llu kB", &total_mem)) // NOLINT { break; } @@ -174,4 +175,5 @@ class PageStorageInMemoryCapacity : public StressWorkload } }; -REGISTER_WORKLOAD(PageStorageInMemoryCapacity) \ No newline at end of file +REGISTER_WORKLOAD(PageStorageInMemoryCapacity) +} // namespace DB::PS::tests \ No newline at end of file diff --git a/dbms/src/Storages/Page/stress/workload/ThousandsOfOffset.cpp b/dbms/src/Storages/Page/workload/ThousandsOfOffset.cpp similarity index 97% rename from dbms/src/Storages/Page/stress/workload/ThousandsOfOffset.cpp rename to dbms/src/Storages/Page/workload/ThousandsOfOffset.cpp index 3a215f76769..5a02ef48d68 100644 --- a/dbms/src/Storages/Page/stress/workload/ThousandsOfOffset.cpp +++ b/dbms/src/Storages/Page/workload/ThousandsOfOffset.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +namespace DB::PS::tests +{ class ThousandsOfOffset : public StressWorkload , public StressWorkloadFunc { @@ -168,4 +170,5 @@ class ThousandsOfOffset : public StressWorkload } }; -REGISTER_WORKLOAD(ThousandsOfOffset) \ No newline at end of file +REGISTER_WORKLOAD(ThousandsOfOffset) +} // namespace DB::PS::tests \ No newline at end of file