Merge branch 'master' into refine_interpreter_join

.clang-tidy

@@ -18,9 +18,11 @@ Checks: '-*,
     modernize-redundant-void-arg,
     modernize-replace-auto-ptr,
     modernize-replace-random-shuffle,
+    modernize-use-auto,
     modernize-use-bool-literals,
     modernize-use-nullptr,
     modernize-use-using,
+    modernize-use-override,
     modernize-use-equals-default,
     modernize-use-equals-delete,
 
@@ -160,6 +162,11 @@ Checks: '-*,
     clang-analyzer-unix.cstring.NullArg,
 
     boost-use-to-string,
+    
+    cppcoreguidelines-pro-type-cstyle-cast,
+    cppcoreguidelines-pro-type-member-init,
+    cppcoreguidelines-no-malloc,
+    cppcoreguidelines-virtual-class-destructor,
 '
 WarningsAsErrors: '*'
 

README.md

@@ -1,29 +1,44 @@
 # TiFlash
+
 ![tiflash-architecture](tiflash-architecture.png)
 
-[TiFlash](https://docs.pingcap.com/tidb/stable/tiflash-overview) is a columnar storage component of [TiDB](https://docs.pingcap.com/tidb/stable). It mainly plays the role of Analytical Processing (AP) in the Hybrid Transactional/Analytical Processing (HTAP) architecture of TiDB.
+[TiFlash](https://docs.pingcap.com/tidb/stable/tiflash-overview) is a columnar storage component of [TiDB](https://docs.pingcap.com/tidb/stable) and [TiDB Cloud](https://en.pingcap.com/tidb-cloud/), the fully-managed service of TiDB. It mainly plays the role of Analytical Processing (AP) in the Hybrid Transactional/Analytical Processing (HTAP) architecture of TiDB.
 
 TiFlash stores data in columnar format and synchronizes data updates in real-time from [TiKV](https://github.com/tikv/tikv) by Raft logs with sub-second latency. Reads in TiFlash are guaranteed transactionally consistent with Snapshot Isolation level. TiFlash utilizes Massively Parallel Processing (MPP) computing architecture to accelerate the analytical workloads.
 
-TiFlash repository is based on [ClickHouse](https://github.com/ClickHouse/ClickHouse). We appreciate the excellent work of ClickHouse team.
+TiFlash repository is based on [ClickHouse](https://github.com/ClickHouse/ClickHouse). We appreciate the excellent work of the ClickHouse team.
+
+## Quick Start
+
+### Start with TiDB Cloud
+
+Quickly explore TiFlash with [a free trial of TiDB Cloud](https://tidbcloud.com/signup).
+
+See [TiDB Cloud Quick Start Guide](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart).
+
+### Start with TiDB
+
+See [Quick Start with HTAP](https://docs.pingcap.com/tidb/stable/quick-start-with-htap) and [Use TiFlash](https://docs.pingcap.com/tidb/stable/use-tiflash).
 
 ## Build TiFlash
 
 TiFlash supports building on the following hardware architectures:
-* x86-64/amd64
-* aarch64
+
+- x86-64/amd64
+- aarch64
 
 And the following operating systems:
-* Linux
-* MacOS
+
+- Linux
+- MacOS
 
 ### 1. Checkout Source Code
 
 Assume `$WORKSPACE` to be the directory under which the TiFlash repo is placed.
 
 ```shell
 cd $WORKSPACE
-git clone --recursive https://github.com/pingcap/tiflash.git
+git clone https://github.com/pingcap/tiflash.git --recursive -j 20
 ```
 
 ### 2. Prepare Prerequisites
@@ -44,19 +59,20 @@ The following packages are needed for all platforms:
 - Ninja or GNU Make
 
 The following are platform-specific prerequisites. Click to expand details:
+
 <details>
 <summary><b>Linux specific prerequisites</b></summary>
 
-  TiFlash can be built using either LLVM or GCC toolchain on Linux. LLVM toolchain is our official one for releasing.
-  > But for GCC, only GCC 7.x is supported as far, and is not planned to be a long term support. So it may get broken some day, silently.
-
+TiFlash can be built using either LLVM or GCC toolchain on Linux. LLVM toolchain is our official one for releasing.
+
+> But for GCC, only GCC 7.x is supported as far, and is not planned to be a long term support. So it may get broken some day, silently.
+
 - LLVM 13.0.0+
 
   TiFlash compiles using full LLVM toolchain (`clang/compiler-rt/libc++/libc++abi`) by default. You can use a system-wise toolchain if `clang/compiler-rt/libc++/libc++abi` can be installed in your environment.
 
   Click sections below to see detailed instructions:
 
-
   <details>
   <summary><b>Set up LLVM via package managers in Debian/Ubuntu</b></summary>
 
@@ -123,6 +139,8 @@ cmake $WORKSPACE/tiflash
 make tiflash -j
 ```
 
+> **NOTE**: Option `-j` (defaults to your system CPU core count, otherwise you can optionally specify a number) is used to control the build parallelism. Higher parallelism consumes more memory. If you encounter compiler OOM or hang, try to lower the parallelism by specifying a reasonable number, e.g., half of your system CPU core count or even smaller, after `-j`, depending on the available memory in your system.
+
 After building, you can get TiFlash binary under `$BUILD/dbms/src/Server/tiflash`.
 
 ### Build Options
@@ -195,11 +213,12 @@ To run unit tests, you need to build with `-DCMAKE_BUILD_TYPE=DEBUG`:
 ```shell
 cd $BUILD
 cmake $WORKSPACE/tiflash -GNinja -DCMAKE_BUILD_TYPE=DEBUG
-ninja gtests_dbms
+ninja gtests_dbms       # Most TiFlash unit tests
+ninja gtests_libdaemon  # Settings related tests
 ninja gtests_libcommon
-ninja gtests_libdaemon
 ```
-And the unit-test executables are at `$BUILD/dbms/gtests_dbms`, `$BUILD/libs/libcommon/src/tests/gtests_libcommon` and `$BUILD/libs/libdaemon/src/tests/gtests_libdaemon`.
+
+And the unit-test executables are at `$BUILD/dbms/gtests_dbms`, `$BUILD/libs/libdaemon/src/tests/gtests_libdaemon` and `$BUILD/libs/libcommon/src/tests/gtests_libcommon`.
 
 ## Run Sanitizer Tests
 
@@ -211,13 +230,13 @@ To generate unit test executables with sanitizer enabled:
 cd $BUILD
 cmake $WORKSPACE/tiflash -GNinja -DENABLE_TESTS=ON -DCMAKE_BUILD_TYPE=ASan # or TSan
 ninja gtests_dbms
-ninja gtests_libcommon
 ninja gtests_libdaemon
+ninja gtests_libcommon
 ```
 
 There are known false positives reported from leak sanitizer (which is included in address sanitizer). To suppress these errors, set the following environment variables before running the executables:
 
-```
+```shell
 LSAN_OPTIONS=suppressions=$WORKSPACE/tiflash/test/sanitize/asan.suppression
 ```
 

dbms/src/Common/FailPoint.cpp

@@ -68,6 +68,7 @@ std::unordered_map<String, std::shared_ptr<FailPointChannel>> FailPointHelper::f
     M(exception_after_drop_segment)
 
 #define APPLY_FOR_FAILPOINTS(M)                              \
+    M(skip_check_segment_update)                             \
     M(force_set_page_file_write_errno)                       \
     M(force_split_io_size_4k)                                \
     M(minimum_block_size_for_cross_join)                     \
@@ -82,7 +83,8 @@ std::unordered_map<String, std::shared_ptr<FailPointChannel>> FailPointHelper::f
     M(force_no_local_region_for_mpp_task)                    \
     M(force_remote_read_for_batch_cop)                       \
     M(force_context_path)                                    \
-    M(force_slow_page_storage_snapshot_release)
+    M(force_slow_page_storage_snapshot_release)              \
+    M(force_change_all_blobs_to_read_only)
 
 #define APPLY_FOR_FAILPOINTS_ONCE_WITH_CHANNEL(M) \
     M(pause_with_alter_locks_acquired)            \
@@ -99,7 +101,8 @@ std::unordered_map<String, std::shared_ptr<FailPointChannel>> FailPointHelper::f
     M(pause_when_writing_to_dt_store)        \
     M(pause_when_ingesting_to_dt_store)      \
     M(pause_when_altering_dt_store)          \
-    M(pause_after_copr_streams_acquired)
+    M(pause_after_copr_streams_acquired)     \
+    M(pause_before_server_merge_one_delta)
 
 namespace FailPoints
 {

dbms/src/Common/MyTime.cpp

@@ -429,7 +429,7 @@ std::tuple<int, int> MyTimeBase::calcWeek(UInt32 mode) const
 
     if (week_year && days >= 52 * 7)
     {
-        week_day = (week_day + calcDaysInYear(year)) % 7;
+        week_day = (week_day + calcDaysInYear(ret_year)) % 7;
         if ((!first_week_day && week_day < 4) || (first_week_day && week_day == 0))
         {
             ret_year++;
@@ -876,7 +876,7 @@ String MyDateTime::toString(int fsp) const
 //TODO: we can use modern c++ api instead.
 MyDateTime MyDateTime::getSystemDateTimeByTimezone(const TimezoneInfo & timezoneInfo, UInt8 fsp)
 {
-    struct timespec ts;
+    struct timespec ts; // NOLINT(cppcoreguidelines-pro-type-member-init)
     clock_gettime(CLOCK_REALTIME, &ts);
 
     time_t second = ts.tv_sec;
@@ -1202,6 +1202,11 @@ void fromDayNum(MyDateTime & t, int day_num)
         // the day number of the last 100 years should be DAY_NUM_PER_100_YEARS + 1
         // so can not use day_num % DAY_NUM_PER_100_YEARS
         day_num = day_num - (num_of_100_years * DAY_NUM_PER_100_YEARS);
+        if (num_of_100_years == 4)
+        {
+            num_of_100_years = 3;
+            day_num = DAY_NUM_PER_100_YEARS;
+        }
 
         int num_of_4_years = day_num / DAY_NUM_PER_4_YEARS;
         // can not use day_num % DAY_NUM_PER_4_YEARS
@@ -1211,6 +1216,12 @@ void fromDayNum(MyDateTime & t, int day_num)
         // can not use day_num % DAY_NUM_PER_YEARS
         day_num = day_num - (num_of_years * DAY_NUM_PER_YEARS);
 
+        if (num_of_years == 4)
+        {
+            num_of_years = 3;
+            day_num = DAY_NUM_PER_YEARS;
+        }
+
         year = 1 + num_of_400_years * 400 + num_of_100_years * 100 + num_of_4_years * 4 + num_of_years;
     }
     static const int ACCUMULATED_DAYS_PER_MONTH[] = {30, 58, 89, 119, 150, 180, 211, 242, 272, 303, 333, 364};
@@ -1246,7 +1257,7 @@ void addMonths(MyDateTime & t, Int64 months)
     Int64 current_month = t.month - 1;
     current_month += months;
     Int64 current_year = 0;
-    Int64 year = static_cast<Int64>(t.year);
+    auto year = static_cast<Int64>(t.year);
     if (current_month >= 0)
     {
         current_year = current_month / 12;

dbms/src/Common/MyTime.h

@@ -195,6 +195,7 @@ MyDateTime convertUTC2TimeZoneByOffset(time_t utc_ts, UInt32 micro_second, Int64
 std::pair<time_t, UInt32> roundTimeByFsp(time_t second, UInt64 nano_second, UInt8 fsp);
 
 int calcDayNum(int year, int month, int day);
+void fromDayNum(MyDateTime & t, int day_num);
 
 // returns seconds since '0000-00-00'
 UInt64 calcSeconds(int year, int month, int day, int hour, int minute, int second);

dbms/src/Common/TiFlashMetrics.h

@@ -115,13 +115,16 @@ namespace DB
     M(tiflash_storage_command_count, "Total number of storage's command, such as delete range / shutdown /startup", Counter,              \
         F(type_delete_range, {"type", "delete_range"}), F(type_ingest, {"type", "ingest"}))                                               \
     M(tiflash_storage_subtask_count, "Total number of storage's sub task", Counter, F(type_delta_merge, {"type", "delta_merge"}),         \
-        F(type_delta_merge_fg, {"type", "delta_merge_fg"}), F(type_delta_merge_bg_gc, {"type", "delta_merge_bg_gc"}),                     \
+        F(type_delta_merge_fg, {"type", "delta_merge_fg"}),                                                                               \
+        F(type_delta_merge_fg_rpc, {"type", "delta_merge_fg_rpc"}),                                                                       \
+        F(type_delta_merge_bg_gc, {"type", "delta_merge_bg_gc"}),                                                                         \
         F(type_delta_compact, {"type", "delta_compact"}), F(type_delta_flush, {"type", "delta_flush"}),                                   \
         F(type_seg_split, {"type", "seg_split"}), F(type_seg_split_fg, {"type", "seg_split_fg"}),                                         \
         F(type_seg_merge, {"type", "seg_merge"}), F(type_place_index_update, {"type", "place_index_update"}))                             \
     M(tiflash_storage_subtask_duration_seconds, "Bucketed histogram of storage's sub task duration", Histogram,                           \
         F(type_delta_merge, {{"type", "delta_merge"}}, ExpBuckets{0.0005, 2, 20}),                                                        \
         F(type_delta_merge_fg, {{"type", "delta_merge_fg"}}, ExpBuckets{0.0005, 2, 20}),                                                  \
+        F(type_delta_merge_fg_rpc, {{"type", "delta_merge_fg_rpc"}}, ExpBuckets{0.0005, 2, 20}),                                          \
         F(type_delta_merge_bg_gc, {{"type", "delta_merge_bg_gc"}}, ExpBuckets{0.0005, 2, 20}),                                            \
         F(type_delta_compact, {{"type", "delta_compact"}}, ExpBuckets{0.0005, 2, 20}),                                                    \
         F(type_delta_flush, {{"type", "delta_flush"}}, ExpBuckets{0.0005, 2, 20}),                                                        \

dbms/src/Flash/BatchCoprocessorHandler.cpp

@@ -16,6 +16,7 @@
 #include <Flash/BatchCoprocessorHandler.h>
 #include <Flash/Coprocessor/DAGDriver.h>
 #include <Flash/Coprocessor/InterpreterDAG.h>
+#include <Flash/ServiceUtils.h>
 #include <Storages/IStorage.h>
 #include <Storages/Transaction/SchemaSyncer.h>
 #include <Storages/Transaction/TMTContext.h>

dbms/src/Flash/CMakeLists.txt

@@ -18,11 +18,12 @@ add_headers_and_sources(flash_service .)
 add_headers_and_sources(flash_service ./Coprocessor)
 add_headers_and_sources(flash_service ./Mpp)
 add_headers_and_sources(flash_service ./Statistics)
+add_headers_and_sources(flash_service ./Management)
 
 add_library(flash_service ${flash_service_headers} ${flash_service_sources})
 target_link_libraries(flash_service dbms)
 
 if (ENABLE_TESTS)
-    add_subdirectory (Coprocessor/tests)
-    add_subdirectory (tests)
+    add_subdirectory(Coprocessor/tests)
+    add_subdirectory(tests)
 endif ()

dbms/src/Flash/Coprocessor/DAGUtils.cpp

@@ -29,14 +29,6 @@
 #include <unordered_map>
 namespace DB
 {
-namespace ErrorCodes
-{
-extern const int NOT_IMPLEMENTED;
-extern const int UNKNOWN_USER;
-extern const int WRONG_PASSWORD;
-extern const int REQUIRED_PASSWORD;
-extern const int IP_ADDRESS_NOT_ALLOWED;
-} // namespace ErrorCodes
 
 const Int8 VAR_SIZE = 0;
 
@@ -572,7 +564,7 @@ const std::unordered_map<tipb::ScalarFuncSig, String> scalar_func_map({
     //{tipb::ScalarFuncSig::SecToTime, "cast"},
     //{tipb::ScalarFuncSig::TimeToSec, "cast"},
     //{tipb::ScalarFuncSig::TimestampAdd, "cast"},
-    //{tipb::ScalarFuncSig::ToDays, "cast"},
+    {tipb::ScalarFuncSig::ToDays, "tidbToDays"},
     {tipb::ScalarFuncSig::ToSeconds, "tidbToSeconds"},
     //{tipb::ScalarFuncSig::UTCTimeWithArg, "cast"},
     //{tipb::ScalarFuncSig::UTCTimestampWithoutArg, "cast"},
@@ -607,7 +599,7 @@ const std::unordered_map<tipb::ScalarFuncSig, String> scalar_func_map({
     //{tipb::ScalarFuncSig::SubDateDatetimeString, "cast"},
     {tipb::ScalarFuncSig::SubDateDatetimeInt, "date_sub"},
 
-    //{tipb::ScalarFuncSig::FromDays, "cast"},
+    {tipb::ScalarFuncSig::FromDays, "tidbFromDays"},
     //{tipb::ScalarFuncSig::TimeFormat, "cast"},
     {tipb::ScalarFuncSig::TimestampDiff, "tidbTimestampDiff"},
 
@@ -1360,17 +1352,6 @@ bool hasUnsignedFlag(const tipb::FieldType & tp)
     return tp.flag() & TiDB::ColumnFlagUnsigned;
 }
 
-grpc::StatusCode tiflashErrorCodeToGrpcStatusCode(int error_code)
-{
-    /// do not use switch statement because ErrorCodes::XXXX is not a compile time constant
-    if (error_code == ErrorCodes::NOT_IMPLEMENTED)
-        return grpc::StatusCode::UNIMPLEMENTED;
-    if (error_code == ErrorCodes::UNKNOWN_USER || error_code == ErrorCodes::WRONG_PASSWORD || error_code == ErrorCodes::REQUIRED_PASSWORD
-        || error_code == ErrorCodes::IP_ADDRESS_NOT_ALLOWED)
-        return grpc::StatusCode::UNAUTHENTICATED;
-    return grpc::StatusCode::INTERNAL;
-}
-
 void assertBlockSchema(
     const DataTypes & expected_types,
     const Block & block,

dbms/src/Flash/Coprocessor/DAGUtils.h

@@ -70,7 +70,6 @@ bool isUnsupportedEncodeType(const std::vector<tipb::FieldType> & types, tipb::E
 TiDB::TiDBCollatorPtr getCollatorFromExpr(const tipb::Expr & expr);
 TiDB::TiDBCollatorPtr getCollatorFromFieldType(const tipb::FieldType & field_type);
 bool hasUnsignedFlag(const tipb::FieldType & tp);
-grpc::StatusCode tiflashErrorCodeToGrpcStatusCode(int error_code);
 
 void assertBlockSchema(
     const DataTypes & expected_types,

dbms/src/Flash/CoprocessorHandler.cpp

@@ -18,6 +18,7 @@
 #include <Flash/Coprocessor/DAGDriver.h>
 #include <Flash/Coprocessor/InterpreterDAG.h>
 #include <Flash/CoprocessorHandler.h>
+#include <Flash/ServiceUtils.h>
 #include <Storages/IStorage.h>
 #include <Storages/Transaction/LockException.h>
 #include <Storages/Transaction/RegionException.h>