From 70fb05e9e111b4a9e08c916bfaf4f971cd5f4ece Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Thu, 26 May 2022 12:19:32 +0800 Subject: [PATCH 01/15] Optimize modules by PCH and PImpl --- CMakeLists.txt | 11 ++++ dbms/CMakeLists.txt | 30 +++++++++++ dbms/pch-common.h | 21 ++++++++ dbms/pch-dbms.h | 18 +++++++ dbms/pch-kvpb.h | 23 +++++++++ dbms/pch-stl.h | 28 ++++++++++ dbms/src/Common/TiFlashException.cpp | 77 +++++++++++++++++++++++++++- dbms/src/Common/TiFlashException.h | 58 +++++---------------- 8 files changed, 219 insertions(+), 47 deletions(-) create mode 100644 dbms/pch-common.h create mode 100644 dbms/pch-dbms.h create mode 100644 dbms/pch-kvpb.h create mode 100644 dbms/pch-stl.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f1dd740b0b4..1539474ddd1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,6 +50,17 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS 1) option (USE_CCACHE "Set to OFF to disable ccache" ON) include (cmake/find_ccache.cmake) +if (USE_CCACHE) + set(NOT_USE_CCACHE 0) +else() + set(NOT_USE_CCACHE 1) +endif() + +option(ENABLE_PCH "Enable `Precompiled header`" ${NOT_USE_CCACHE}) +if (USE_CCACHE AND ENABLE_PCH) + message(STATUS "`Precompiled header` will be unavailable with ccache") +endif () + if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None") set (CMAKE_BUILD_TYPE "RelWithDebInfo") message (STATUS "CMAKE_BUILD_TYPE is not set, set to default = ${CMAKE_BUILD_TYPE}") diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 718d18c4954..8797edd22ea 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -258,6 +258,16 @@ target_include_directories (clickhouse_common_io BEFORE PRIVATE ${COMMON_INCLUDE # https://cmake.org/pipermail/cmake/2016-May/063400.html target_link_libraries (clickhouse_common_io PUBLIC ${TIFLASH_XXHASH_LIBRARY}) +function(add_target_pch context target) + if (ENABLE_PCH) + message(STATUS "Add PCH `${context}` for library `${target}`") + target_precompile_headers(${target} PRIVATE ${context}) + endif () + if(${ARGC} GREATER 2) + add_target_pch(${context} ${ARGN}) + endif() +endfunction() + if (ENABLE_TESTS) include (${TiFlash_SOURCE_DIR}/cmake/find_gtest.cmake) @@ -295,6 +305,7 @@ if (ENABLE_TESTS) target_compile_options(gtests_dbms PRIVATE -Wno-unknown-pragmas -Wno-deprecated-copy) add_check(gtests_dbms) + add_target_pch("pch-dbms.h" gtests_dbms) grep_bench_sources(${TiFlash_SOURCE_DIR}/dbms dbms_bench_sources) add_executable(bench_dbms EXCLUDE_FROM_ALL @@ -341,3 +352,22 @@ if (TEST_COVERAGE AND CMAKE_BUILD_TYPE STREQUAL "Debug") ) endif () endif () + +# dbms +add_target_pch("pch-dbms.h" dbms flash_service) +add_target_pch("pch-common.h" clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions) +add_target_pch("pch-common.h" clickhouse_parsers clickhouse_storages_system dt-workload-lib clickhouse-server-lib) + +# common +add_target_pch("pch-stl.h" ${Boost_SYSTEM_LIBRARY} cctz ${RE2_LIBRARY} ${RE2_ST_LIBRARY}) +add_target_pch("pch-kvpb.h" kv_client) + +# grpc +add_target_pch("$<$:${CMAKE_CURRENT_SOURCE_DIR}/pch-stl.h>" grpc grpc++) + +# pb +add_target_pch("pch-stl.h" libprotobuf kvproto tipb libprotoc) + +# poco +add_target_pch("pch-stl.h" Net Crypto Util Data NetSSL) +add_target_pch("$<$:${CMAKE_CURRENT_SOURCE_DIR}/pch-stl.h>" XML Foundation JSON) diff --git a/dbms/pch-common.h b/dbms/pch-common.h new file mode 100644 index 00000000000..5212269719e --- /dev/null +++ b/dbms/pch-common.h @@ -0,0 +1,21 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#include "pch-stl.h" diff --git a/dbms/pch-dbms.h b/dbms/pch-dbms.h new file mode 100644 index 00000000000..60687073bf8 --- /dev/null +++ b/dbms/pch-dbms.h @@ -0,0 +1,18 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "pch-common.h" +#include "pch-kvpb.h" diff --git a/dbms/pch-kvpb.h b/dbms/pch-kvpb.h new file mode 100644 index 00000000000..85b61846d39 --- /dev/null +++ b/dbms/pch-kvpb.h @@ -0,0 +1,23 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#pragma once + +#include +#include +#include + +#include "pch-stl.h" diff --git a/dbms/pch-stl.h b/dbms/pch-stl.h new file mode 100644 index 00000000000..fff3845dd3e --- /dev/null +++ b/dbms/pch-stl.h @@ -0,0 +1,28 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/dbms/src/Common/TiFlashException.cpp b/dbms/src/Common/TiFlashException.cpp index bd28a57c093..11ee229ded0 100644 --- a/dbms/src/Common/TiFlashException.cpp +++ b/dbms/src/Common/TiFlashException.cpp @@ -18,6 +18,32 @@ namespace DB { +struct TiFlashErrorRegistry::Errors : std::map, TiFlashError> +{ +}; + +TiFlashErrorRegistry::Errors & TiFlashErrorRegistry::errors() +{ + return *inner_data; +} + +TiFlashErrorRegistry::Errors & TiFlashErrorRegistry::errors() const +{ + return *inner_data; +} + +TiFlashErrorRegistry::TiFlashErrorRegistry() + : inner_data(new Errors{}) +{ + initialize(); +} + +TiFlashErrorRegistry::~TiFlashErrorRegistry() +{ + delete inner_data; + inner_data = nullptr; +} + void TiFlashErrorRegistry::initialize() { // Used to check uniqueness of classes @@ -46,9 +72,9 @@ void TiFlashErrorRegistry::initialize() void TiFlashErrorRegistry::registerError(const std::string & error_class, const std::string & error_code, const std::string & description, const std::string & workaround, const std::string & message_template) { TiFlashError error{error_class, error_code, description, workaround, message_template}; - if (all_errors.find({error_class, error_code}) == all_errors.end()) + if (errors().find({error_class, error_code}) == errors().end()) { - all_errors.emplace(std::make_pair(error_class, error_code), std::move(error)); + errors().emplace(std::make_pair(error_class, error_code), std::move(error)); } else { @@ -77,4 +103,51 @@ std::string TiFlashException::standardText() const return text; } +std::optional TiFlashErrorRegistry::get(const std::string & error_class, const std::string & error_code) const +{ + auto error = errors().find({error_class, error_code}); + if (error != errors().end()) + { + return error->second; + } + else + { + return {}; + } +} +std::optional TiFlashErrorRegistry::get(const std::string & error_class, int error_code) const +{ + return get(error_class, std::to_string(error_code)); +} + +std::vector TiFlashErrorRegistry::allErrors() const +{ + std::vector res; + res.reserve(errors().size()); + for (const auto & error : errors()) + { + res.push_back(error.second); + } + return res; +} + +TiFlashError TiFlashErrorRegistry::simpleGet(const std::string & error_class, const std::string & error_code) +{ + auto & m_instance = instance(); + auto error = m_instance.get(error_class, error_code); + if (error.has_value()) + { + return error.value(); + } + else + { + throw Exception("Unregistered TiFlashError: FLASH:" + error_class + ":" + error_code); + } +} +TiFlashError TiFlashErrorRegistry::simpleGet(const std::string & error_class, int error_code) +{ + return simpleGet(error_class, std::to_string(error_code)); +} + + } // namespace DB diff --git a/dbms/src/Common/TiFlashException.h b/dbms/src/Common/TiFlashException.h index 2026571859e..739170edbd3 100644 --- a/dbms/src/Common/TiFlashException.h +++ b/dbms/src/Common/TiFlashException.h @@ -194,56 +194,19 @@ class TiFlashErrorRegistry : public ext::Singleton public: friend ext::Singleton; - static TiFlashError simpleGet(const std::string & error_class, const std::string & error_code) - { - auto & m_instance = instance(); - auto error = m_instance.get(error_class, error_code); - if (error.has_value()) - { - return error.value(); - } - else - { - throw Exception("Unregistered TiFlashError: FLASH:" + error_class + ":" + error_code); - } - } + static TiFlashError simpleGet(const std::string & error_class, const std::string & error_code); - static TiFlashError simpleGet(const std::string & error_class, int error_code) - { - return simpleGet(error_class, std::to_string(error_code)); - } + static TiFlashError simpleGet(const std::string & error_class, int error_code); - std::optional get(const std::string & error_class, const std::string & error_code) const - { - auto error = all_errors.find({error_class, error_code}); - if (error != all_errors.end()) - { - return error->second; - } - else - { - return {}; - } - } + std::optional get(const std::string & error_class, const std::string & error_code) const; - std::optional get(const std::string & error_class, int error_code) const - { - return get(error_class, std::to_string(error_code)); - } + std::optional get(const std::string & error_class, int error_code) const; - std::vector allErrors() const - { - std::vector res; - res.reserve(all_errors.size()); - for (const auto & error : all_errors) - { - res.push_back(error.second); - } - return res; - } + std::vector allErrors() const; protected: - TiFlashErrorRegistry() { initialize(); } + TiFlashErrorRegistry(); + ~TiFlashErrorRegistry(); private: void registerError(const std::string & error_class, const std::string & error_code, const std::string & description, const std::string & workaround, const std::string & message_template = ""); @@ -252,8 +215,13 @@ class TiFlashErrorRegistry : public ext::Singleton void initialize(); + struct Errors; + + Errors & errors(); + Errors & errors() const; + private: - std::map, TiFlashError> all_errors; + Errors * inner_data; }; /// TiFlashException implements TiDB's standardized error. From 6858eed2554c403ef791bed82e3b6fbf192cc626 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Thu, 26 May 2022 12:23:37 +0800 Subject: [PATCH 02/15] wip --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1539474ddd1..fdd6e525db0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,7 +58,7 @@ endif() option(ENABLE_PCH "Enable `Precompiled header`" ${NOT_USE_CCACHE}) if (USE_CCACHE AND ENABLE_PCH) - message(STATUS "`Precompiled header` will be unavailable with ccache") + message(WARNING "`Precompiled header` will be unavailable with ccache") endif () if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None") From 139a3df4aff24e32695831115577d8866298bf82 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Fri, 27 May 2022 13:52:45 +0800 Subject: [PATCH 03/15] optimize modules --- dbms/src/Common/Decimal.cpp | 66 ++++++++++++++++++- dbms/src/Common/Decimal.h | 65 ++++-------------- .../DataStreams/TiRemoteBlockInputStream.h | 1 - .../src/Flash/Coprocessor/CoprocessorReader.h | 1 - .../Coprocessor/DAGStorageInterpreter.cpp | 1 + .../Flash/Coprocessor/DAGStorageInterpreter.h | 2 +- .../Flash/Coprocessor/GenSchemaAndColumn.cpp | 1 + dbms/src/Flash/Coprocessor/RemoteRequest.cpp | 2 + .../Flash/Coprocessor/TablesRegionsInfo.cpp | 1 + .../src/Flash/Coprocessor/TablesRegionsInfo.h | 3 +- dbms/src/Flash/Mpp/MinTSOScheduler.h | 11 +++- dbms/src/Storages/Transaction/TMTContext.h | 6 -- .../Storages/Transaction/TiDBSchemaSyncer.h | 1 - 13 files changed, 93 insertions(+), 68 deletions(-) diff --git a/dbms/src/Common/Decimal.cpp b/dbms/src/Common/Decimal.cpp index fc34f617e3b..79a7109a185 100644 --- a/dbms/src/Common/Decimal.cpp +++ b/dbms/src/Common/Decimal.cpp @@ -157,5 +157,69 @@ template struct Decimal; template struct Decimal; template struct Decimal; -// end namespace + +std::tuple PlusDecimalInferer::infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale) +{ + ScaleType result_scale = std::max(left_scale, right_scale); + PrecType result_int = std::max(left_prec - left_scale, right_prec - right_scale); + PrecType result_prec = std::min(result_scale + result_int + 1, decimal_max_prec); + return {result_prec, result_scale}; +} +std::tuple MulDecimalInferer::infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale) +{ + return {std::min(left_prec + right_prec, decimal_max_prec), std::min(left_scale + right_scale, decimal_max_scale)}; +} + +std::tuple DivDecimalInferer::infer(PrecType left_prec, ScaleType left_scale, PrecType /* right_prec is not used */, ScaleType right_scale) +{ + return { + std::min(left_prec + right_scale + div_precincrement, decimal_max_prec), + std::min(left_scale + div_precincrement, decimal_max_scale)}; +} + +std::tuple SumDecimalInferer::infer(PrecType prec, ScaleType scale) +{ + return {std::min(prec + decimal_longlong_digits, decimal_max_prec), scale}; +} + +std::tuple AvgDecimalInferer::infer(PrecType left_prec, ScaleType left_scale) +{ + return {std::min(left_prec + div_precincrement, decimal_max_prec), std::min(left_scale + div_precincrement, decimal_max_scale)}; +} + +std::tuple ModDecimalInferer::infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale) +{ + return {std::max(left_prec, right_prec), std::max(left_scale, right_scale)}; +} + +std::tuple OtherInferer::infer(PrecType, ScaleType, PrecType, ScaleType) +{ + return {}; +} + +Int256 DecimalMaxValue::get(PrecType idx) +{ + // In case DecimalMaxValue::get(IntPrec::prec), where IntPrec::prec > 65. + assert(idx <= decimal_max_prec); + return instance().getInternal(idx); +} + +Int256 DecimalMaxValue::maxValue() +{ + return get(maxDecimalPrecision()); +} + +DecimalMaxValue::DecimalMaxValue() +{ + for (PrecType i = 1; i <= decimal_max_prec; i++) + { + number[i] = number[i - 1] * 10 + 9; + } +} + +Int256 DecimalMaxValue::getInternal(PrecType idx) const +{ + return number[idx]; +} + } // namespace DB diff --git a/dbms/src/Common/Decimal.h b/dbms/src/Common/Decimal.h index 71dcf82ea39..b465a27f938 100644 --- a/dbms/src/Common/Decimal.h +++ b/dbms/src/Common/Decimal.h @@ -108,66 +108,40 @@ struct IntPrec struct PlusDecimalInferer { - static std::tuple infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale) - { - ScaleType result_scale = std::max(left_scale, right_scale); - PrecType result_int = std::max(left_prec - left_scale, right_prec - right_scale); - PrecType result_prec = std::min(result_scale + result_int + 1, decimal_max_prec); - return {result_prec, result_scale}; - } + static std::tuple infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale); }; struct MulDecimalInferer { - static std::tuple infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale) - { - return {std::min(left_prec + right_prec, decimal_max_prec), std::min(left_scale + right_scale, decimal_max_scale)}; - } + static std::tuple infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale); }; struct DivDecimalInferer { static const ScaleType div_precincrement = 4; - static std::tuple infer(PrecType left_prec, ScaleType left_scale, PrecType /* right_prec is not used */, ScaleType right_scale) - { - return { - std::min(left_prec + right_scale + div_precincrement, decimal_max_prec), - std::min(left_scale + div_precincrement, decimal_max_scale)}; - } + static std::tuple infer(PrecType left_prec, ScaleType left_scale, PrecType /* right_prec is not used */, ScaleType right_scale); }; struct SumDecimalInferer { static constexpr PrecType decimal_longlong_digits = 22; - static std::tuple infer(PrecType prec, ScaleType scale) - { - return {std::min(prec + decimal_longlong_digits, decimal_max_prec), scale}; - } + static std::tuple infer(PrecType prec, ScaleType scale); }; struct AvgDecimalInferer { static const ScaleType div_precincrement = 4; - static std::tuple infer(PrecType left_prec, ScaleType left_scale) - { - return {std::min(left_prec + div_precincrement, decimal_max_prec), std::min(left_scale + div_precincrement, decimal_max_scale)}; - } + static std::tuple infer(PrecType left_prec, ScaleType left_scale); }; struct ModDecimalInferer { - static std::tuple infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale) - { - return {std::max(left_prec, right_prec), std::max(left_scale, right_scale)}; - } + static std::tuple infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale); }; struct OtherInferer { - static std::tuple infer(PrecType, ScaleType, PrecType, ScaleType) - { - return {}; - } + static std::tuple infer(PrecType, ScaleType, PrecType, ScaleType); }; template @@ -389,31 +363,14 @@ class DecimalMaxValue final : public ext::Singleton Int256 number[decimal_max_prec + 1]; public: - static Int256 get(PrecType idx) - { - // In case DecimalMaxValue::get(IntPrec::prec), where IntPrec::prec > 65. - assert(idx <= decimal_max_prec); - return instance().getInternal(idx); - } + static Int256 get(PrecType idx); - static Int256 maxValue() - { - return get(maxDecimalPrecision()); - } + static Int256 maxValue(); private: - DecimalMaxValue() - { - for (PrecType i = 1; i <= decimal_max_prec; i++) - { - number[i] = number[i - 1] * 10 + 9; - } - } + DecimalMaxValue(); - Int256 getInternal(PrecType idx) const - { - return number[idx]; - } + Int256 getInternal(PrecType idx) const; }; // In some case, getScaleMultiplier and its callee may not be auto inline by the compiler. diff --git a/dbms/src/DataStreams/TiRemoteBlockInputStream.h b/dbms/src/DataStreams/TiRemoteBlockInputStream.h index c1e29617586..ec442ba6a29 100644 --- a/dbms/src/DataStreams/TiRemoteBlockInputStream.h +++ b/dbms/src/DataStreams/TiRemoteBlockInputStream.h @@ -21,7 +21,6 @@ #include #include #include -#include #include #include diff --git a/dbms/src/Flash/Coprocessor/CoprocessorReader.h b/dbms/src/Flash/Coprocessor/CoprocessorReader.h index 8a3eb471e54..e3a40a9662b 100644 --- a/dbms/src/Flash/Coprocessor/CoprocessorReader.h +++ b/dbms/src/Flash/Coprocessor/CoprocessorReader.h @@ -20,7 +20,6 @@ #include #include #include -#include #include #include diff --git a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp index f5354994b44..f9353b26a41 100644 --- a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" diff --git a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.h b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.h index d86274a1e22..0425abe04db 100644 --- a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.h +++ b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.h @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include @@ -33,6 +32,7 @@ namespace DB { +class TMTContext; using TablesRegionInfoMap = std::unordered_map>; /// DAGStorageInterpreter encapsulates operations around storage during interprete stage. /// It's only intended to be used by DAGQueryBlockInterpreter. diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp index e7964021709..be3475f714f 100644 --- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include #include +#include namespace DB { diff --git a/dbms/src/Flash/Coprocessor/RemoteRequest.cpp b/dbms/src/Flash/Coprocessor/RemoteRequest.cpp index 086cdb43d20..d3b5c202136 100644 --- a/dbms/src/Flash/Coprocessor/RemoteRequest.cpp +++ b/dbms/src/Flash/Coprocessor/RemoteRequest.cpp @@ -13,8 +13,10 @@ // limitations under the License. #include +#include #include #include +#include namespace DB { diff --git a/dbms/src/Flash/Coprocessor/TablesRegionsInfo.cpp b/dbms/src/Flash/Coprocessor/TablesRegionsInfo.cpp index 7c4ddedabf4..ab4a0f82e95 100644 --- a/dbms/src/Flash/Coprocessor/TablesRegionsInfo.cpp +++ b/dbms/src/Flash/Coprocessor/TablesRegionsInfo.cpp @@ -16,6 +16,7 @@ #include #include #include +#include namespace DB { diff --git a/dbms/src/Flash/Coprocessor/TablesRegionsInfo.h b/dbms/src/Flash/Coprocessor/TablesRegionsInfo.h index dccb8b95466..f80a44b92a3 100644 --- a/dbms/src/Flash/Coprocessor/TablesRegionsInfo.h +++ b/dbms/src/Flash/Coprocessor/TablesRegionsInfo.h @@ -15,7 +15,7 @@ #pragma once #include #include -#include + #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" @@ -25,6 +25,7 @@ namespace DB { +class TMTContext; struct SingleTableRegions { RegionInfoMap local_regions; diff --git a/dbms/src/Flash/Mpp/MinTSOScheduler.h b/dbms/src/Flash/Mpp/MinTSOScheduler.h index 17ab1f4dfa3..dbc0cb84cc3 100644 --- a/dbms/src/Flash/Mpp/MinTSOScheduler.h +++ b/dbms/src/Flash/Mpp/MinTSOScheduler.h @@ -15,12 +15,19 @@ #pragma once #include -#include -#include #include namespace DB { +class MinTSOScheduler; +using MPPTaskSchedulerPtr = std::unique_ptr; + +class MPPTaskManager; +using MPPTaskManagerPtr = std::shared_ptr; + +struct MPPQueryTaskSet; +using MPPQueryTaskSetPtr = std::shared_ptr; + /// scheduling tasks in the set according to the tso order under the soft limit of threads, but allow the min_tso query to preempt threads under the hard limit of threads. /// The min_tso query avoids the deadlock resulted from threads competition among nodes. /// schedule tasks under the lock protection of the task manager. diff --git a/dbms/src/Storages/Transaction/TMTContext.h b/dbms/src/Storages/Transaction/TMTContext.h index 27e0482b787..c84ada2eec0 100644 --- a/dbms/src/Storages/Transaction/TMTContext.h +++ b/dbms/src/Storages/Transaction/TMTContext.h @@ -34,15 +34,9 @@ using SchemaSyncerPtr = std::shared_ptr; class BackgroundService; using BackGroundServicePtr = std::unique_ptr; -class MinTSOScheduler; -using MPPTaskSchedulerPtr = std::unique_ptr; - class MPPTaskManager; using MPPTaskManagerPtr = std::shared_ptr; -struct MPPQueryTaskSet; -using MPPQueryTaskSetPtr = std::shared_ptr; - class GCManager; using GCManagerPtr = std::shared_ptr; diff --git a/dbms/src/Storages/Transaction/TiDBSchemaSyncer.h b/dbms/src/Storages/Transaction/TiDBSchemaSyncer.h index c9b2f2aa567..9b017dae0dd 100644 --- a/dbms/src/Storages/Transaction/TiDBSchemaSyncer.h +++ b/dbms/src/Storages/Transaction/TiDBSchemaSyncer.h @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include From f39e8b18a2c0b8dcfbea6607b7aff53e2f6ffd10 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Mon, 30 May 2022 16:05:47 +0800 Subject: [PATCH 04/15] add comment --- dbms/src/Common/TiFlashException.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Common/TiFlashException.h b/dbms/src/Common/TiFlashException.h index 739170edbd3..3b4e3d75813 100644 --- a/dbms/src/Common/TiFlashException.h +++ b/dbms/src/Common/TiFlashException.h @@ -221,7 +221,7 @@ class TiFlashErrorRegistry : public ext::Singleton Errors & errors() const; private: - Errors * inner_data; + Errors * inner_data; // PImpl }; /// TiFlashException implements TiDB's standardized error. From 217c140946eaa670b25c3f8181ff92385159bf9e Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Tue, 31 May 2022 09:37:53 +0800 Subject: [PATCH 05/15] fix pch options --- CMakeLists.txt | 7 ++----- cmake/find_ccache.cmake | 12 ++++++++++++ dbms/pch-kvpb.h | 5 +++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fdd6e525db0..17658ef1b94 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,8 +48,6 @@ endif () set(CMAKE_EXPORT_COMPILE_COMMANDS 1) option (USE_CCACHE "Set to OFF to disable ccache" ON) -include (cmake/find_ccache.cmake) - if (USE_CCACHE) set(NOT_USE_CCACHE 0) else() @@ -57,9 +55,8 @@ else() endif() option(ENABLE_PCH "Enable `Precompiled header`" ${NOT_USE_CCACHE}) -if (USE_CCACHE AND ENABLE_PCH) - message(WARNING "`Precompiled header` will be unavailable with ccache") -endif () + +include (cmake/find_ccache.cmake) if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None") set (CMAKE_BUILD_TYPE "RelWithDebInfo") diff --git a/cmake/find_ccache.cmake b/cmake/find_ccache.cmake index 910caebb046..5d3eb207095 100644 --- a/cmake/find_ccache.cmake +++ b/cmake/find_ccache.cmake @@ -24,6 +24,18 @@ if (USE_CCACHE AND CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "cca message ("${CCACHE_CONFIG}") set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND}) set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND}) + + if (ENABLE_PCH) + message (WARNING "`Precompiled header` can not be used with `ccache` because of unexpected behaviors") + set (ENABLE_PCH FALSE CACHE BOOL "" FORCE) + + # execute_process (COMMAND ${CCACHE_FOUND} --get-config sloppiness OUTPUT_VARIABLE _CCACHE_SLOPPINESS OUTPUT_STRIP_TRAILING_WHITESPACE) + # set (CCACHE_SLOPPINESS "${_CCACHE_SLOPPINESS}" CACHE STRING "ccache --get-config sloppiness" FORCE) + # if (NOT CCACHE_SLOPPINESS STREQUAL "time_macros, pch_defines") + # message(WARNING "`Precompiled header` won't be cached by ccache, sloppiness = `${CCACHE_SLOPPINESS}`,please execute `ccache -o sloppiness=pch_defines,time_macros`") + # endif () + endif () + else () message (STATUS "Not using ccache ${CCACHE_FOUND}, USE_CCACHE=${USE_CCACHE}") endif () diff --git a/dbms/pch-kvpb.h b/dbms/pch-kvpb.h index 85b61846d39..0df92fcb424 100644 --- a/dbms/pch-kvpb.h +++ b/dbms/pch-kvpb.h @@ -16,6 +16,11 @@ #pragma once +/** There is unexpected behaviors for `ccache` to deal with PCH which includes those header files generated by tools: + 'xxx' has been modified since the precompiled header 'xxx' was built: mtime changed + `Precompiled header includes xxx.h, which has a new mtime` + It's not recommended to use PCH and ccache together +*/ #include #include #include From 3da1c11cebd0e8e7200793ff595061677d1aa909 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Tue, 31 May 2022 13:54:08 +0800 Subject: [PATCH 06/15] wip --- dbms/pch-kvpb.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbms/pch-kvpb.h b/dbms/pch-kvpb.h index 0df92fcb424..3ee578404a2 100644 --- a/dbms/pch-kvpb.h +++ b/dbms/pch-kvpb.h @@ -14,8 +14,6 @@ #pragma once -#pragma once - /** There is unexpected behaviors for `ccache` to deal with PCH which includes those header files generated by tools: 'xxx' has been modified since the precompiled header 'xxx' was built: mtime changed `Precompiled header includes xxx.h, which has a new mtime` From f5324f782a93eeaf4e528f0d6dc73aa60a932b86 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Thu, 2 Jun 2022 13:03:33 +0800 Subject: [PATCH 07/15] Fix compile --- dbms/src/Flash/Coprocessor/CoprocessorReader.h | 1 + dbms/src/Flash/Coprocessor/RemoteRequest.h | 3 ++- dbms/src/Flash/Management/tests/gtest_manual_compact.cpp | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dbms/src/Flash/Coprocessor/CoprocessorReader.h b/dbms/src/Flash/Coprocessor/CoprocessorReader.h index e3a40a9662b..25c07cff49c 100644 --- a/dbms/src/Flash/Coprocessor/CoprocessorReader.h +++ b/dbms/src/Flash/Coprocessor/CoprocessorReader.h @@ -28,6 +28,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" #include #include #include diff --git a/dbms/src/Flash/Coprocessor/RemoteRequest.h b/dbms/src/Flash/Coprocessor/RemoteRequest.h index 1e42e18a7bd..5af3f66298c 100644 --- a/dbms/src/Flash/Coprocessor/RemoteRequest.h +++ b/dbms/src/Flash/Coprocessor/RemoteRequest.h @@ -17,11 +17,12 @@ #include #include #include -#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" #include +#include #include #pragma GCC diagnostic pop diff --git a/dbms/src/Flash/Management/tests/gtest_manual_compact.cpp b/dbms/src/Flash/Management/tests/gtest_manual_compact.cpp index 8ec3eb54406..e457db6eb57 100644 --- a/dbms/src/Flash/Management/tests/gtest_manual_compact.cpp +++ b/dbms/src/Flash/Management/tests/gtest_manual_compact.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include From d532a7f25e31b6b7581a87acf0f59db5f43b17cb Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Thu, 2 Jun 2022 13:38:06 +0800 Subject: [PATCH 08/15] Fix compile2 --- dbms/src/TestUtils/ColumnsToTiPBExpr.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/TestUtils/ColumnsToTiPBExpr.cpp b/dbms/src/TestUtils/ColumnsToTiPBExpr.cpp index 2c3bf243176..dcf727614b1 100644 --- a/dbms/src/TestUtils/ColumnsToTiPBExpr.cpp +++ b/dbms/src/TestUtils/ColumnsToTiPBExpr.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include namespace DB From 28451ca8584ec694c7fdc546615b4c72f40e699c Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Thu, 2 Jun 2022 16:18:19 +0800 Subject: [PATCH 09/15] only enable `ENABLE_PCH` in release build Signed-off-by: Zhigao Tong --- CMakeLists.txt | 2 +- release-centos7-llvm/scripts/build-tiflash-release.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 17658ef1b94..4e14c205f18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,7 +54,7 @@ else() set(NOT_USE_CCACHE 1) endif() -option(ENABLE_PCH "Enable `Precompiled header`" ${NOT_USE_CCACHE}) +option(ENABLE_PCH "Enable `Precompiled header`" OFF) include (cmake/find_ccache.cmake) diff --git a/release-centos7-llvm/scripts/build-tiflash-release.sh b/release-centos7-llvm/scripts/build-tiflash-release.sh index bb62e4743f6..42993b51afe 100755 --- a/release-centos7-llvm/scripts/build-tiflash-release.sh +++ b/release-centos7-llvm/scripts/build-tiflash-release.sh @@ -42,6 +42,7 @@ SRCPATH=$( ) NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} ENABLE_THINLTO=${ENABLE_THINLTO:-ON} +ENABLE_PCH=${ENABLE_PCH:-ON} INSTALL_DIR="${SRCPATH}/release-centos7-llvm/tiflash" rm -rf ${INSTALL_DIR} && mkdir -p ${INSTALL_DIR} @@ -59,6 +60,7 @@ cmake -S "${SRCPATH}" \ -DRUN_HAVE_STD_REGEX=0 \ -DENABLE_THINLTO=${ENABLE_THINLTO} \ -DTHINLTO_JOBS=${NPROC} \ + -DENABLE_PCH=${ENABLE_PCH} \ -GNinja cmake --build . --target tiflash --parallel ${NPROC} From 620594876683b1604baa02bc3d7c326ec5901ab2 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Sun, 5 Jun 2022 14:25:17 +0800 Subject: [PATCH 10/15] optimize header files include --- dbms/src/Common/FmtUtils.cpp | 43 ++++++++++++++++++++++++++++++++++++ dbms/src/Common/FmtUtils.h | 23 ++++++------------- dbms/src/Server/CLIService.h | 1 - 3 files changed, 50 insertions(+), 17 deletions(-) create mode 100644 dbms/src/Common/FmtUtils.cpp diff --git a/dbms/src/Common/FmtUtils.cpp b/dbms/src/Common/FmtUtils.cpp new file mode 100644 index 00000000000..6646ef5dc10 --- /dev/null +++ b/dbms/src/Common/FmtUtils.cpp @@ -0,0 +1,43 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +namespace DB +{ +FmtBuffer & FmtBuffer::append(std::string_view s) +{ + buffer.append(s.data(), s.data() + s.size()); + return *this; +} + +std::string FmtBuffer::toString() const +{ + return fmt::to_string(buffer); +} + +void FmtBuffer::resize(size_t count) +{ + buffer.resize(count); +} +void FmtBuffer::reserve(size_t capacity) +{ + buffer.reserve(capacity); +} +void FmtBuffer::clear() +{ + buffer.clear(); +} + +} // namespace DB diff --git a/dbms/src/Common/FmtUtils.h b/dbms/src/Common/FmtUtils.h index 71f93a34078..34bcad20630 100644 --- a/dbms/src/Common/FmtUtils.h +++ b/dbms/src/Common/FmtUtils.h @@ -14,8 +14,6 @@ #pragma once -#include -#include #include namespace DB @@ -32,16 +30,9 @@ class FmtBuffer return *this; } - FmtBuffer & append(StringRef s) - { - buffer.append(s.data, s.data + s.size); - return *this; - } + FmtBuffer & append(std::string_view s); - std::string toString() const - { - return fmt::to_string(buffer); - } + std::string toString() const; template FmtBuffer & joinStr( @@ -55,7 +46,7 @@ class FmtBuffer FmtBuffer & joinStr( Iter first, Iter end, - StringRef delimiter) + std::string_view delimiter) { auto func = [](const auto & s, FmtBuffer & fb) { fb.append(s); @@ -68,7 +59,7 @@ class FmtBuffer Iter first, Iter end, FF && toStringFunc, // void (const auto &, FmtBuffer &) - StringRef delimiter) + std::string_view delimiter) { if (first == end) return *this; @@ -82,9 +73,9 @@ class FmtBuffer return *this; } - void resize(size_t count) { buffer.resize(count); } - void reserve(size_t capacity) { buffer.reserve(capacity); } - void clear() { buffer.clear(); } + void resize(size_t count); + void reserve(size_t capacity); + void clear(); private: fmt::memory_buffer buffer; diff --git a/dbms/src/Server/CLIService.h b/dbms/src/Server/CLIService.h index 18c9d61260f..9078fa991f3 100644 --- a/dbms/src/Server/CLIService.h +++ b/dbms/src/Server/CLIService.h @@ -25,7 +25,6 @@ #include #include #include -#include #include #include From f2a406aa93f06e98e1868597f0ff25ae87ed40c5 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Sun, 5 Jun 2022 14:26:01 +0800 Subject: [PATCH 11/15] refine pch with ccache --- cmake/find_ccache.cmake | 16 +++++++++------- dbms/CMakeLists.txt | 6 ++++-- dbms/pch-common.h | 2 ++ dbms/pch-kvpb.h | 8 ++++---- dbms/pch-stl.h | 3 +++ 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/cmake/find_ccache.cmake b/cmake/find_ccache.cmake index 5d3eb207095..71fa337a922 100644 --- a/cmake/find_ccache.cmake +++ b/cmake/find_ccache.cmake @@ -26,14 +26,16 @@ if (USE_CCACHE AND CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "cca set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND}) if (ENABLE_PCH) - message (WARNING "`Precompiled header` can not be used with `ccache` because of unexpected behaviors") - set (ENABLE_PCH FALSE CACHE BOOL "" FORCE) + execute_process (COMMAND ${CCACHE_FOUND} --get-config sloppiness OUTPUT_VARIABLE _CCACHE_SLOPPINESS OUTPUT_STRIP_TRAILING_WHITESPACE) + string (FIND "${_CCACHE_SLOPPINESS}" "pch_defines" _CCACHE_SLOPPINESS_RES) + if (NOT _CCACHE_SLOPPINESS_RES STREQUAL "-1") + string (FIND "${_CCACHE_SLOPPINESS}" "time_macros" _CCACHE_SLOPPINESS_RES) + endif () - # execute_process (COMMAND ${CCACHE_FOUND} --get-config sloppiness OUTPUT_VARIABLE _CCACHE_SLOPPINESS OUTPUT_STRIP_TRAILING_WHITESPACE) - # set (CCACHE_SLOPPINESS "${_CCACHE_SLOPPINESS}" CACHE STRING "ccache --get-config sloppiness" FORCE) - # if (NOT CCACHE_SLOPPINESS STREQUAL "time_macros, pch_defines") - # message(WARNING "`Precompiled header` won't be cached by ccache, sloppiness = `${CCACHE_SLOPPINESS}`,please execute `ccache -o sloppiness=pch_defines,time_macros`") - # endif () + if (_CCACHE_SLOPPINESS_RES STREQUAL "-1") + message(WARNING "`Precompiled header` won't be cached by ccache, sloppiness = `${CCACHE_SLOPPINESS}`,please execute `ccache -o sloppiness=pch_defines,time_macros`") + set (ENABLE_PCH FALSE CACHE BOOL "" FORCE) + endif () endif () else () diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 0ffcfa9261a..cce11bd6997 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -261,7 +261,7 @@ target_link_libraries (clickhouse_common_io PUBLIC ${TIFLASH_XXHASH_LIBRARY}) function(add_target_pch context target) if (ENABLE_PCH) - message(STATUS "Add PCH `${context}` for library `${target}`") + message(STATUS "Add PCH `${context}` for target `${target}`") target_precompile_headers(${target} PRIVATE ${context}) endif () if(${ARGC} GREATER 2) @@ -306,6 +306,7 @@ if (ENABLE_TESTS) target_compile_options(gtests_dbms PRIVATE -Wno-unknown-pragmas -Wno-deprecated-copy) add_check(gtests_dbms) + add_target_pch("pch-dbms.h" gtests_dbms) grep_bench_sources(${TiFlash_SOURCE_DIR}/dbms dbms_bench_sources) @@ -360,9 +361,10 @@ add_target_pch("pch-common.h" clickhouse_common_io clickhouse_functions clickhou add_target_pch("pch-common.h" clickhouse_parsers clickhouse_storages_system dt-workload-lib clickhouse-server-lib) # common -add_target_pch("pch-stl.h" ${Boost_SYSTEM_LIBRARY} cctz ${RE2_LIBRARY} ${RE2_ST_LIBRARY}) add_target_pch("pch-kvpb.h" kv_client) +add_target_pch("pch-stl.h" ${Boost_SYSTEM_LIBRARY} cctz ${RE2_LIBRARY} ${RE2_ST_LIBRARY}) + # grpc add_target_pch("$<$:${CMAKE_CURRENT_SOURCE_DIR}/pch-stl.h>" grpc grpc++) diff --git a/dbms/pch-common.h b/dbms/pch-common.h index 5212269719e..878254a3529 100644 --- a/dbms/pch-common.h +++ b/dbms/pch-common.h @@ -17,5 +17,7 @@ #include #include #include +#include +#include #include "pch-stl.h" diff --git a/dbms/pch-kvpb.h b/dbms/pch-kvpb.h index 3ee578404a2..d74bfc6bb89 100644 --- a/dbms/pch-kvpb.h +++ b/dbms/pch-kvpb.h @@ -14,10 +14,10 @@ #pragma once -/** There is unexpected behaviors for `ccache` to deal with PCH which includes those header files generated by tools: - 'xxx' has been modified since the precompiled header 'xxx' was built: mtime changed - `Precompiled header includes xxx.h, which has a new mtime` - It's not recommended to use PCH and ccache together +/** +There may be unexpected behaviors for `ccache` to deal with PCH which includes those header files generated by tools: + 'xxx' has been modified since the precompiled header 'xxx' was built: mtime changed + `Precompiled header includes xxx.h, which has a new mtime` */ #include #include diff --git a/dbms/pch-stl.h b/dbms/pch-stl.h index fff3845dd3e..6037372b333 100644 --- a/dbms/pch-stl.h +++ b/dbms/pch-stl.h @@ -19,9 +19,12 @@ #include #include #include +#include #include #include +#include #include +#include #include #include #include From dae201bfae3ea998ec3db992b61b7f12048fcdf1 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Tue, 7 Jun 2022 12:43:02 +0800 Subject: [PATCH 12/15] rollback modification --- dbms/src/Common/Decimal.cpp | 66 +----------------------------------- dbms/src/Common/Decimal.h | 65 +++++++++++++++++++++++++++++------ dbms/src/Common/FmtUtils.cpp | 43 ----------------------- dbms/src/Common/FmtUtils.h | 17 +++++++--- 4 files changed, 67 insertions(+), 124 deletions(-) delete mode 100644 dbms/src/Common/FmtUtils.cpp diff --git a/dbms/src/Common/Decimal.cpp b/dbms/src/Common/Decimal.cpp index 79a7109a185..fc34f617e3b 100644 --- a/dbms/src/Common/Decimal.cpp +++ b/dbms/src/Common/Decimal.cpp @@ -157,69 +157,5 @@ template struct Decimal; template struct Decimal; template struct Decimal; - -std::tuple PlusDecimalInferer::infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale) -{ - ScaleType result_scale = std::max(left_scale, right_scale); - PrecType result_int = std::max(left_prec - left_scale, right_prec - right_scale); - PrecType result_prec = std::min(result_scale + result_int + 1, decimal_max_prec); - return {result_prec, result_scale}; -} -std::tuple MulDecimalInferer::infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale) -{ - return {std::min(left_prec + right_prec, decimal_max_prec), std::min(left_scale + right_scale, decimal_max_scale)}; -} - -std::tuple DivDecimalInferer::infer(PrecType left_prec, ScaleType left_scale, PrecType /* right_prec is not used */, ScaleType right_scale) -{ - return { - std::min(left_prec + right_scale + div_precincrement, decimal_max_prec), - std::min(left_scale + div_precincrement, decimal_max_scale)}; -} - -std::tuple SumDecimalInferer::infer(PrecType prec, ScaleType scale) -{ - return {std::min(prec + decimal_longlong_digits, decimal_max_prec), scale}; -} - -std::tuple AvgDecimalInferer::infer(PrecType left_prec, ScaleType left_scale) -{ - return {std::min(left_prec + div_precincrement, decimal_max_prec), std::min(left_scale + div_precincrement, decimal_max_scale)}; -} - -std::tuple ModDecimalInferer::infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale) -{ - return {std::max(left_prec, right_prec), std::max(left_scale, right_scale)}; -} - -std::tuple OtherInferer::infer(PrecType, ScaleType, PrecType, ScaleType) -{ - return {}; -} - -Int256 DecimalMaxValue::get(PrecType idx) -{ - // In case DecimalMaxValue::get(IntPrec::prec), where IntPrec::prec > 65. - assert(idx <= decimal_max_prec); - return instance().getInternal(idx); -} - -Int256 DecimalMaxValue::maxValue() -{ - return get(maxDecimalPrecision()); -} - -DecimalMaxValue::DecimalMaxValue() -{ - for (PrecType i = 1; i <= decimal_max_prec; i++) - { - number[i] = number[i - 1] * 10 + 9; - } -} - -Int256 DecimalMaxValue::getInternal(PrecType idx) const -{ - return number[idx]; -} - +// end namespace } // namespace DB diff --git a/dbms/src/Common/Decimal.h b/dbms/src/Common/Decimal.h index 4839e9be608..d4a3f382208 100644 --- a/dbms/src/Common/Decimal.h +++ b/dbms/src/Common/Decimal.h @@ -108,40 +108,66 @@ struct IntPrec struct PlusDecimalInferer { - static std::tuple infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale); + static std::tuple infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale) + { + ScaleType result_scale = std::max(left_scale, right_scale); + PrecType result_int = std::max(left_prec - left_scale, right_prec - right_scale); + PrecType result_prec = std::min(result_scale + result_int + 1, decimal_max_prec); + return {result_prec, result_scale}; + } }; struct MulDecimalInferer { - static std::tuple infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale); + static std::tuple infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale) + { + return {std::min(left_prec + right_prec, decimal_max_prec), std::min(left_scale + right_scale, decimal_max_scale)}; + } }; struct DivDecimalInferer { static const ScaleType div_precincrement = 4; - static std::tuple infer(PrecType left_prec, ScaleType left_scale, PrecType /* right_prec is not used */, ScaleType right_scale); + static std::tuple infer(PrecType left_prec, ScaleType left_scale, PrecType /* right_prec is not used */, ScaleType right_scale) + { + return { + std::min(left_prec + right_scale + div_precincrement, decimal_max_prec), + std::min(left_scale + div_precincrement, decimal_max_scale)}; + } }; struct SumDecimalInferer { static constexpr PrecType decimal_longlong_digits = 22; - static std::tuple infer(PrecType prec, ScaleType scale); + static std::tuple infer(PrecType prec, ScaleType scale) + { + return {std::min(prec + decimal_longlong_digits, decimal_max_prec), scale}; + } }; struct AvgDecimalInferer { static const ScaleType div_precincrement = 4; - static std::tuple infer(PrecType left_prec, ScaleType left_scale); + static std::tuple infer(PrecType left_prec, ScaleType left_scale) + { + return {std::min(left_prec + div_precincrement, decimal_max_prec), std::min(left_scale + div_precincrement, decimal_max_scale)}; + } }; struct ModDecimalInferer { - static std::tuple infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale); + static std::tuple infer(PrecType left_prec, ScaleType left_scale, PrecType right_prec, ScaleType right_scale) + { + return {std::max(left_prec, right_prec), std::max(left_scale, right_scale)}; + } }; struct OtherInferer { - static std::tuple infer(PrecType, ScaleType, PrecType, ScaleType); + static std::tuple infer(PrecType, ScaleType, PrecType, ScaleType) + { + return {}; + } }; template @@ -386,14 +412,31 @@ class DecimalMaxValue final : public ext::Singleton Int256 number[decimal_max_prec + 1]; public: - static Int256 get(PrecType idx); + static Int256 get(PrecType idx) + { + // In case DecimalMaxValue::get(IntPrec::prec), where IntPrec::prec > 65. + assert(idx <= decimal_max_prec); + return instance().getInternal(idx); + } - static Int256 maxValue(); + static Int256 maxValue() + { + return get(maxDecimalPrecision()); + } private: - DecimalMaxValue(); + DecimalMaxValue() + { + for (PrecType i = 1; i <= decimal_max_prec; i++) + { + number[i] = number[i - 1] * 10 + 9; + } + } - Int256 getInternal(PrecType idx) const; + Int256 getInternal(PrecType idx) const + { + return number[idx]; + } }; // In some case, getScaleMultiplier and its callee may not be auto inline by the compiler. diff --git a/dbms/src/Common/FmtUtils.cpp b/dbms/src/Common/FmtUtils.cpp deleted file mode 100644 index 6646ef5dc10..00000000000 --- a/dbms/src/Common/FmtUtils.cpp +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -namespace DB -{ -FmtBuffer & FmtBuffer::append(std::string_view s) -{ - buffer.append(s.data(), s.data() + s.size()); - return *this; -} - -std::string FmtBuffer::toString() const -{ - return fmt::to_string(buffer); -} - -void FmtBuffer::resize(size_t count) -{ - buffer.resize(count); -} -void FmtBuffer::reserve(size_t capacity) -{ - buffer.reserve(capacity); -} -void FmtBuffer::clear() -{ - buffer.clear(); -} - -} // namespace DB diff --git a/dbms/src/Common/FmtUtils.h b/dbms/src/Common/FmtUtils.h index 34bcad20630..ab37194ed10 100644 --- a/dbms/src/Common/FmtUtils.h +++ b/dbms/src/Common/FmtUtils.h @@ -30,9 +30,16 @@ class FmtBuffer return *this; } - FmtBuffer & append(std::string_view s); + FmtBuffer & append(std::string_view s) + { + buffer.append(s.data(), s.data() + s.size()); + return *this; + } - std::string toString() const; + std::string toString() const + { + return fmt::to_string(buffer); + } template FmtBuffer & joinStr( @@ -73,9 +80,9 @@ class FmtBuffer return *this; } - void resize(size_t count); - void reserve(size_t capacity); - void clear(); + void resize(size_t count) { buffer.resize(count); } + void reserve(size_t capacity) { buffer.reserve(capacity); } + void clear() { buffer.clear(); } private: fmt::memory_buffer buffer; From 5384155ca843253ab950df9ce369106707a8ff66 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Tue, 7 Jun 2022 23:21:24 +0800 Subject: [PATCH 13/15] Fix tidy --- dbms/pch-stl.h | 5 ++- dbms/src/Interpreters/Context.h | 1 - dbms/src/Storages/Transaction/KVStore.cpp | 36 +++++++++++++--------- dbms/src/Storages/Transaction/KVStore.h | 21 ++++++------- dbms/src/Storages/Transaction/ProxyFFI.cpp | 2 ++ 5 files changed, 36 insertions(+), 29 deletions(-) diff --git a/dbms/pch-stl.h b/dbms/pch-stl.h index 6037372b333..01b3123650d 100644 --- a/dbms/pch-stl.h +++ b/dbms/pch-stl.h @@ -18,14 +18,17 @@ #include #include #include +#include #include #include #include #include #include #include +#include +#include #include #include #include #include -#include +#include \ No newline at end of file diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 63aefcbece9..5d5c39263c6 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -14,7 +14,6 @@ #pragma once -#include #include #include #include diff --git a/dbms/src/Storages/Transaction/KVStore.cpp b/dbms/src/Storages/Transaction/KVStore.cpp index 2b82220cbc4..318a04c6ed9 100644 --- a/dbms/src/Storages/Transaction/KVStore.cpp +++ b/dbms/src/Storages/Transaction/KVStore.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -38,15 +39,15 @@ extern const int TABLE_IS_DROPPED; } // namespace ErrorCodes KVStore::KVStore(Context & context, TiDB::SnapshotApplyMethod snapshot_apply_method_) - : region_persister(context, region_manager) + : region_persister(std::make_unique(context, region_manager)) , raft_cmd_res(std::make_unique()) , snapshot_apply_method(snapshot_apply_method_) , log(&Poco::Logger::get("KVStore")) + , region_compact_log_period(120) + , region_compact_log_min_rows(40 * 1024) + , region_compact_log_min_bytes(32 * 1024 * 1024) { // default config about compact-log: period 120s, rows 40k, bytes 32MB. - REGION_COMPACT_LOG_PERIOD = 120; - REGION_COMPACT_LOG_MIN_ROWS = 40 * 1024; - REGION_COMPACT_LOG_MIN_BYTES = 32 * 1024 * 1024; } void KVStore::restore(const TiFlashRaftProxyHelper * proxy_helper) @@ -55,7 +56,7 @@ void KVStore::restore(const TiFlashRaftProxyHelper * proxy_helper) auto manage_lock = genRegionWriteLock(task_lock); this->proxy_helper = proxy_helper; - manage_lock.regions = region_persister.restore(proxy_helper); + manage_lock.regions = region_persister->restore(proxy_helper); LOG_FMT_INFO(log, "Restored {} regions", manage_lock.regions.size()); @@ -166,7 +167,7 @@ void KVStore::tryPersist(RegionID region_id) if (region) { LOG_FMT_INFO(log, "Try to persist {}", region->toString(false)); - region_persister.persist(*region); + region_persister->persist(*region); LOG_FMT_INFO(log, "After persisted {}, cache {} bytes", region->toString(false), region->dataSize()); } } @@ -182,7 +183,7 @@ void KVStore::gcRegionPersistedCache(Seconds gc_persist_period) if (now < (last_gc_time.load() + gc_persist_period)) return; last_gc_time = now; - region_persister.gc(); + region_persister->gc(); } void KVStore::removeRegion(RegionID region_id, bool remove_data, RegionTable & region_table, const KVStoreTaskLock & task_lock, const RegionTaskLock & region_lock) @@ -203,7 +204,7 @@ void KVStore::removeRegion(RegionID region_id, bool remove_data, RegionTable & r } } - region_persister.drop(region_id, region_lock); + region_persister->drop(region_id, region_lock); LOG_FMT_INFO(log, "Persisted [region {}] deleted", region_id); region_table.removeRegion(region_id, remove_data, region_lock); @@ -306,9 +307,9 @@ void KVStore::handleDestroy(UInt64 region_id, TMTContext & tmt, const KVStoreTas void KVStore::setRegionCompactLogConfig(UInt64 sec, UInt64 rows, UInt64 bytes) { - REGION_COMPACT_LOG_PERIOD = sec; - REGION_COMPACT_LOG_MIN_ROWS = rows; - REGION_COMPACT_LOG_MIN_BYTES = bytes; + region_compact_log_period = sec; + region_compact_log_min_rows = rows; + region_compact_log_min_bytes = bytes; LOG_FMT_INFO( log, @@ -321,7 +322,7 @@ void KVStore::setRegionCompactLogConfig(UInt64 sec, UInt64 rows, UInt64 bytes) void KVStore::persistRegion(const Region & region, const RegionTaskLock & region_task_lock, const char * caller) { LOG_FMT_INFO(log, "Start to persist {}, cache size: {} bytes for `{}`", region.toString(true), region.dataSize(), caller); - region_persister.persist(region, region_task_lock); + region_persister->persist(region, region_task_lock); LOG_FMT_DEBUG(log, "Persist {} done", region.toString(false)); } @@ -362,8 +363,8 @@ EngineStoreApplyRes KVStore::handleUselessAdminRaftCmd( LOG_FMT_DEBUG(log, "{} approx mem cache info: rows {}, bytes {}", curr_region.toString(false), rows, size_bytes); - if (rows >= REGION_COMPACT_LOG_MIN_ROWS.load(std::memory_order_relaxed) - || size_bytes >= REGION_COMPACT_LOG_MIN_BYTES.load(std::memory_order_relaxed)) + if (rows >= region_compact_log_min_rows.load(std::memory_order_relaxed) + || size_bytes >= region_compact_log_min_bytes.load(std::memory_order_relaxed)) { // if rows or bytes more than threshold, flush cache and perist mem data. return true; @@ -372,7 +373,7 @@ EngineStoreApplyRes KVStore::handleUselessAdminRaftCmd( { // if thhere is little data in mem, wait until time interval reached threshold. // use random period so that lots of regions will not be persisted at same time. - auto compact_log_period = std::rand() % REGION_COMPACT_LOG_PERIOD.load(std::memory_order_relaxed); // NOLINT + auto compact_log_period = std::rand() % region_compact_log_period.load(std::memory_order_relaxed); // NOLINT return !(curr_region.lastCompactLogTime() + Seconds{compact_log_period} > Clock::now()); } } @@ -765,4 +766,9 @@ KVStore::~KVStore() releaseReadIndexWorkers(); } +FileUsageStatistics KVStore::getFileUsageStatistics() const +{ + return region_persister->getFileUsageStatistics(); +} + } // namespace DB diff --git a/dbms/src/Storages/Transaction/KVStore.h b/dbms/src/Storages/Transaction/KVStore.h index 8673cae3ff3..bb45e65d18b 100644 --- a/dbms/src/Storages/Transaction/KVStore.h +++ b/dbms/src/Storages/Transaction/KVStore.h @@ -16,20 +16,19 @@ #include #include -#include #include - namespace TiDB { struct TableInfo; } namespace DB { +class Context; namespace RegionBench { extern void concurrentBatchInsert(const TiDB::TableInfo &, Int64, Int64, Int64, UInt64, UInt64, Context &); -} +} // namespace RegionBench namespace DM { enum class FileConvertJobType; @@ -40,7 +39,6 @@ namespace tests class RegionKVStoreTest; } -class Context; class IAST; using ASTPtr = std::shared_ptr; using ASTs = std::vector; @@ -71,6 +69,8 @@ using RegionPreDecodeBlockDataPtr = std::unique_ptr; class ReadIndexWorkerManager; using BatchReadIndexRes = std::vector>; class ReadIndexStressTest; +struct FileUsageStatistics; +class RegionPersister; /// TODO: brief design document. class KVStore final : private boost::noncopyable @@ -157,10 +157,7 @@ class KVStore final : private boost::noncopyable ~KVStore(); - FileUsageStatistics getFileUsageStatistics() const - { - return region_persister.getFileUsageStatistics(); - } + FileUsageStatistics getFileUsageStatistics() const; private: friend class MockTiDB; @@ -229,7 +226,7 @@ class KVStore final : private boost::noncopyable private: RegionManager region_manager; - RegionPersister region_persister; + std::unique_ptr region_persister; std::atomic last_gc_time = Timepoint::min(); @@ -242,9 +239,9 @@ class KVStore final : private boost::noncopyable Poco::Logger * log; - std::atomic REGION_COMPACT_LOG_PERIOD; - std::atomic REGION_COMPACT_LOG_MIN_ROWS; - std::atomic REGION_COMPACT_LOG_MIN_BYTES; + std::atomic region_compact_log_period; + std::atomic region_compact_log_min_rows; + std::atomic region_compact_log_min_bytes; mutable std::mutex bg_gc_region_data_mutex; std::list bg_gc_region_data; diff --git a/dbms/src/Storages/Transaction/ProxyFFI.cpp b/dbms/src/Storages/Transaction/ProxyFFI.cpp index cc7d1e10a49..8a40ca9b15e 100644 --- a/dbms/src/Storages/Transaction/ProxyFFI.cpp +++ b/dbms/src/Storages/Transaction/ProxyFFI.cpp @@ -24,6 +24,8 @@ #include #include +#include + #define CHECK_PARSE_PB_BUFF_IMPL(n, a, b, c) \ do \ { \ From 5662fdd0ab54a10c5ea9e828b4ebd47953360ad9 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Tue, 7 Jun 2022 23:50:47 +0800 Subject: [PATCH 14/15] Fix compile --- dbms/src/Storages/Transaction/ReadIndexWorker.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/Storages/Transaction/ReadIndexWorker.cpp b/dbms/src/Storages/Transaction/ReadIndexWorker.cpp index 97a8f4b3e0b..a761ed8c831 100644 --- a/dbms/src/Storages/Transaction/ReadIndexWorker.cpp +++ b/dbms/src/Storages/Transaction/ReadIndexWorker.cpp @@ -20,6 +20,7 @@ #include #include +#include namespace DB { From 96567cdf7ba67ea6af2db2e9a3736e8449bd683f Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 8 Jun 2022 00:08:47 +0800 Subject: [PATCH 15/15] Fix --- dbms/src/Storages/Transaction/ReadIndexWorker.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/src/Storages/Transaction/ReadIndexWorker.cpp b/dbms/src/Storages/Transaction/ReadIndexWorker.cpp index a761ed8c831..3223c815989 100644 --- a/dbms/src/Storages/Transaction/ReadIndexWorker.cpp +++ b/dbms/src/Storages/Transaction/ReadIndexWorker.cpp @@ -175,7 +175,7 @@ struct BlockedReadIndexHelper : BlockedReadIndexHelperTrait return waker.waitFor(tm); } - virtual ~BlockedReadIndexHelper() = default; + ~BlockedReadIndexHelper() override = default; private: AsyncWaker & waker; @@ -194,7 +194,7 @@ struct BlockedReadIndexHelperV3 : BlockedReadIndexHelperTrait return notifier.blockedWaitFor(tm); } - virtual ~BlockedReadIndexHelperV3() = default; + ~BlockedReadIndexHelperV3() override = default; private: AsyncWaker::Notifier & notifier; @@ -343,7 +343,7 @@ struct RegionReadIndexNotifier : AsyncNotifier notify->wake(); } - virtual ~RegionReadIndexNotifier() = default; + ~RegionReadIndexNotifier() override = default; RegionReadIndexNotifier( RegionID region_id_,