feat: add page_stress_testing as a subcommand of tiflash

CMakeLists.txt

@@ -424,9 +424,6 @@ else (ENABLE_FAILPOINTS)
     message (STATUS "Failpoints are disabled")
 endif (ENABLE_FAILPOINTS)
 
-# Enable PageStorage V3 test.
-option (ENABLE_V3_PAGESTORAGE "Enables V3 PageStorage" ON)
-
 # Flags for test coverage
 option (TEST_COVERAGE "Enables flags for test coverage" OFF)
 option (TEST_COVERAGE_XML "Output XML report for test coverage" OFF)

dbms/CMakeLists.txt

@@ -91,12 +91,10 @@ add_headers_and_sources(dbms src/Storages/Page/V2/VersionSet)
 add_headers_and_sources(dbms src/Storages/Page/V2/gc)
 add_headers_and_sources(dbms src/WindowFunctions)
 add_headers_and_sources(dbms src/TiDB/Schema)
-if (ENABLE_V3_PAGESTORAGE)
-    add_headers_and_sources(dbms src/Storages/Page/V3)
-    add_headers_and_sources(dbms src/Storages/Page/V3/LogFile)
-    add_headers_and_sources(dbms src/Storages/Page/V3/WAL)
-    add_headers_and_sources(dbms src/Storages/Page/V3/spacemap)
-endif()
+add_headers_and_sources(dbms src/Storages/Page/V3)
+add_headers_and_sources(dbms src/Storages/Page/V3/LogFile)
+add_headers_and_sources(dbms src/Storages/Page/V3/WAL)
+add_headers_and_sources(dbms src/Storages/Page/V3/spacemap)
 add_headers_and_sources(dbms src/Storages/Page/)
 add_headers_and_sources(dbms src/TiDB)
 add_headers_and_sources(dbms src/Client)
@@ -311,6 +309,9 @@ if (ENABLE_TESTS)
     if (ENABLE_TIFLASH_DTWORKLOAD)
         target_link_libraries(bench_dbms dt-workload-lib)
     endif ()
+    if (ENABLE_TIFLASH_PAGEWORKLOAD)
+        target_link_libraries(bench_dbms page-workload-lib)
+    endif ()
 
     add_check(bench_dbms)
 endif ()

dbms/src/Server/CMakeLists.txt

@@ -22,6 +22,7 @@ option(ENABLE_CLICKHOUSE_SERVER "Enable server" ${ENABLE_CLICKHOUSE_ALL})
 option(ENABLE_CLICKHOUSE_CLIENT "Enable client" ${ENABLE_CLICKHOUSE_ALL})
 option(ENABLE_TIFLASH_DTTOOL "Enable dttool: tools to manage dmfile" ${ENABLE_CLICKHOUSE_ALL})
 option(ENABLE_TIFLASH_DTWORKLOAD "Enable dtworkload: tools to test and stress DeltaTree" ${ENABLE_CLICKHOUSE_ALL})
+option(ENABLE_TIFLASH_PAGEWORKLOAD "Enable pageworkload: tools to test and stress PageStorage" ${ENABLE_CLICKHOUSE_ALL})
 option(ENABLE_TIFLASH_PAGECTL "Enable pagectl: tools to debug page storage" ${ENABLE_CLICKHOUSE_ALL})
 
 configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h)
@@ -136,6 +137,9 @@ endif ()
 if (ENABLE_TIFLASH_DTWORKLOAD)
     target_link_libraries(tiflash dt-workload-lib)
 endif ()
+if (ENABLE_TIFLASH_PAGEWORKLOAD)
+    target_link_libraries(tiflash page-workload-lib)
+endif()
 if (ENABLE_TIFLASH_PAGECTL)
     target_link_libraries(tiflash page-ctl-lib)
 endif ()

dbms/src/Server/config_tools.h.in

@@ -6,4 +6,5 @@
 #cmakedefine01 ENABLE_CLICKHOUSE_CLIENT
 #cmakedefine01 ENABLE_TIFLASH_DTTOOL
 #cmakedefine01 ENABLE_TIFLASH_DTWORKLOAD
+#cmakedefine01 ENABLE_TIFLASH_PAGEWORKLOAD
 #cmakedefine01 ENABLE_TIFLASH_PAGECTL

dbms/src/Server/main.cpp

@@ -38,6 +38,9 @@
 #if ENABLE_TIFLASH_DTWORKLOAD
 #include <Storages/DeltaMerge/tools/workload/DTWorkload.h>
 #endif
+#if ENABLE_TIFLASH_PAGEWORKLOAD
+#include <Storages/Page/stress/workload/PSWorkload.h>
+#endif
 #if ENABLE_TIFLASH_PAGECTL
 #include <Storages/Page/tools/PageCtl/PageStorageCtl.h>
 #endif
@@ -107,6 +110,9 @@ std::pair<const char *, MainFunc> clickhouse_applications[] = {
 #if ENABLE_TIFLASH_DTWORKLOAD
     {"dtworkload", DB::DM::tests::DTWorkload::mainEntry},
 #endif
+#if ENABLE_TIFLASH_PAGEWORKLOAD
+    {"pageworkload", DB::PS::tests::StressWorkload::mainEntry},
+#endif
 #if ENABLE_TIFLASH_PAGECTL
     {"pagectl", DB::PageStorageCtl::mainEntry},
 #endif

dbms/src/Storages/CMakeLists.txt

@@ -16,15 +16,14 @@ add_subdirectory (System)
 add_subdirectory (Page)
 add_subdirectory (DeltaMerge/File/dtpb)
 add_subdirectory (DeltaMerge/tools)
+add_subdirectory (Page/stress)
 
 if (ENABLE_TESTS)
     add_subdirectory (tests EXCLUDE_FROM_ALL)
     add_subdirectory (Transaction/tests EXCLUDE_FROM_ALL)
     add_subdirectory (Page/V2/tests EXCLUDE_FROM_ALL)
-    if (ENABLE_V3_PAGESTORAGE)
-        add_subdirectory (Page/V3 EXCLUDE_FROM_ALL)
-        add_subdirectory (Page/V3/tests EXCLUDE_FROM_ALL)
-    endif ()
+    add_subdirectory (Page/V3 EXCLUDE_FROM_ALL)
+    add_subdirectory (Page/V3/tests EXCLUDE_FROM_ALL)
     add_subdirectory (DeltaMerge/tests EXCLUDE_FROM_ALL)
 endif ()
 

dbms/src/Storages/DeltaMerge/tools/workload/CMakeLists.txt

@@ -18,6 +18,3 @@ set(dt-workload-src MainEntry.cpp DTWorkload.cpp KeyGenerator.cpp TableGenerator
 
 add_library(dt-workload-lib ${dt-workload-src})
 target_link_libraries(dt-workload-lib dbms clickhouse_functions clickhouse-server-lib)
-
-add_executable(dt-workload Main.cpp ${dt-workload-src})
-target_link_libraries(dt-workload dbms gtest clickhouse_functions clickhouse-server-lib)

dbms/src/Storages/Page/CMakeLists.txt

@@ -14,13 +14,3 @@
 
 add_subdirectory(V2)
 add_subdirectory(tools)
-
-# PageStorage Stress test
-if (ENABLE_V3_PAGESTORAGE)
-    add_headers_and_sources(page_stress_testing stress)
-    add_headers_and_sources(page_stress_testing stress/workload)
-    add_executable(page_stress_testing EXCLUDE_FROM_ALL ${page_stress_testing_sources})
-    target_link_libraries(page_stress_testing dbms page_storage_v3)
-    target_include_directories(page_stress_testing PRIVATE stress)
-    target_compile_options(page_stress_testing PRIVATE -Wno-format -lc++) # turn off printf format check
-endif()

dbms/src/Storages/Page/stress/CMakeLists.txt

@@ -0,0 +1,17 @@
+# Copyright 2022 PingCAP, Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include_directories (${CMAKE_CURRENT_BINARY_DIR})
+
+add_subdirectory (workload EXCLUDE_FROM_ALL)

dbms/src/Storages/Page/stress/workload/CMakeLists.txt

@@ -0,0 +1,22 @@
+# Copyright 2022 PingCAP, Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include_directories (${CMAKE_CURRENT_BINARY_DIR})
+
+set (page-workload-src HeavyMemoryCostInGC.cpp HeavyRead.cpp HeavySkewWriteRead.cpp HeavyWrite.cpp HighValidBigFileGC.cpp HoldSnapshotsLongTime.cpp Normal.cpp 
+     PageStorageInMemoryCapacity.cpp ThousandsOfOffset.cpp MainEntry.cpp Normal.cpp PageStorageInMemoryCapacity.cpp PSBackground.cpp PSRunnable.cpp PSStressEnv.cpp PSWorkload.cpp)
+
+add_library (page-workload-lib ${page-workload-src})
+target_link_libraries (page-workload-lib dbms clickhouse_functions clickhouse-server-lib)
+target_compile_options (page-workload-lib PRIVATE -Wno-format -lc++)

dbms/src/Storages/Page/stress/workload/HeavyMemoryCostInGC.cpp

@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <PSWorkload.h>
+#include <Storages/Page/stress/workload/PSWorkload.h>
 
+namespace DB::PS::tests
+{
 class HeavyMemoryCostInGC
     : public StressWorkload
     , public StressWorkloadFunc<HeavyMemoryCostInGC>
@@ -81,3 +83,4 @@ class HeavyMemoryCostInGC
 };
 
 REGISTER_WORKLOAD(HeavyMemoryCostInGC)
+} // namespace DB::PS::tests

dbms/src/Storages/Page/stress/workload/HeavyRead.cpp

@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <PSWorkload.h>
+#include <Storages/Page/stress/workload/PSWorkload.h>
 
+namespace DB::PS::tests
+{
 class HeavyRead : public StressWorkload
     , public StressWorkloadFunc<HeavyRead>
 {
@@ -69,4 +71,5 @@ class HeavyRead : public StressWorkload
     }
 };
 
-REGISTER_WORKLOAD(HeavyRead)
+REGISTER_WORKLOAD(HeavyRead)
+} // namespace DB::PS::tests

dbms/src/Storages/Page/stress/workload/HeavySkewWriteRead.cpp

@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <PSWorkload.h>
+#include <Storages/Page/stress/workload/PSWorkload.h>
 
+namespace DB::PS::tests
+{
 class HeavySkewWriteRead : public StressWorkload
     , public StressWorkloadFunc<HeavySkewWriteRead>
 {
@@ -84,4 +86,5 @@ class HeavySkewWriteRead : public StressWorkload
     }
 };
 
-REGISTER_WORKLOAD(HeavySkewWriteRead)
+REGISTER_WORKLOAD(HeavySkewWriteRead)
+} // namespace DB::PS::tests

dbms/src/Storages/Page/stress/workload/HeavyWrite.cpp

@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <PSWorkload.h>
+#include <Storages/Page/stress/workload/PSWorkload.h>
 
+namespace DB::PS::tests
+{
 class HeavyWrite : public StressWorkload
     , public StressWorkloadFunc<HeavyWrite>
 {
@@ -71,4 +73,5 @@ class HeavyWrite : public StressWorkload
     }
 };
 
-REGISTER_WORKLOAD(HeavyWrite)
+REGISTER_WORKLOAD(HeavyWrite)
+} // namespace DB::PS::tests

dbms/src/Storages/Page/stress/workload/HighValidBigFileGC.cpp

@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <PSWorkload.h>
+#include <Storages/Page/stress/workload/PSWorkload.h>
 
+namespace DB::PS::tests
+{
 class HighValidBigFileGCWorkload
     : public StressWorkload
     , public StressWorkloadFunc<HighValidBigFileGCWorkload>
@@ -129,3 +131,4 @@ class HighValidBigFileGCWorkload
 };
 
 REGISTER_WORKLOAD(HighValidBigFileGCWorkload)
+} // namespace DB::PS::tests

dbms/src/Storages/Page/stress/workload/HoldSnapshotsLongTime.cpp

@@ -12,8 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <PSWorkload.h>
+#include <Storages/Page/stress/workload/PSWorkload.h>
 
+namespace DB::PS::tests
+{
 class HoldSnapshotsLongTime : public StressWorkload
     , public StressWorkloadFunc<HoldSnapshotsLongTime>
 {
@@ -93,4 +95,5 @@ class HoldSnapshotsLongTime : public StressWorkload
     }
 };
 
-REGISTER_WORKLOAD(HoldSnapshotsLongTime)
+REGISTER_WORKLOAD(HoldSnapshotsLongTime)
+} // namespace DB::PS::tests