Merge branch 'master' into pagestorage_test

CMakeLists.txt

@@ -102,6 +102,28 @@ if (COMPILER_CLANG)
     # Clang doesn't have int128 predefined macros, workaround by manually defining them
     # Reference: https://stackoverflow.com/questions/41198673/uint128-t-not-working-with-clang-and-libstdc
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__GLIBCXX_BITSIZE_INT_N_0=128 -D__GLIBCXX_TYPE_INT_N_0=__int128")
+
+    option(ENABLE_TIME_TRACES "Enable clang feature time traces" OFF)
+    if (ENABLE_TIME_TRACES)
+        set (CLANG_TIME_TRACES_FLAGS "-ftime-trace")
+        message (STATUS "Using clang time traces flag `${CLANG_TIME_TRACES_FLAGS}`. Generates JSON file based on output filename. Results can be analyzed with chrome://tracing or https://www.speedscope.app for flamegraph visualization.")
+        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CLANG_TIME_TRACES_FLAGS}")
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CLANG_TIME_TRACES_FLAGS}")
+    endif ()
+
+    # https://clang.llvm.org/docs/ThinLTO.html
+    # Applies to clang only.
+    option(ENABLE_THINLTO "Clang-specific link time optimization" OFF)
+
+    if (ENABLE_THINLTO AND NOT ENABLE_TESTS)
+        # Link time optimization
+        set (THINLTO_JOBS "0" CACHE STRING "ThinLTO compilation parallelism")
+        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -flto=thin -fvisibility=hidden -fvisibility-inlines-hidden -fsplit-lto-unit")
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto=thin -fvisibility=hidden -fvisibility-inlines-hidden -fwhole-program-vtables -fsplit-lto-unit")
+        set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -flto=thin -flto-jobs=${THINLTO_JOBS} -fvisibility=hidden -fvisibility-inlines-hidden -fwhole-program-vtables -fsplit-lto-unit")
+    elseif (ENABLE_THINLTO)
+        message (WARNING "Cannot enable ThinLTO")
+    endif ()
 endif ()
 
 if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")

dbms/src/Common/CurrentMetrics.cpp

@@ -78,7 +78,12 @@
     M(IOLimiterPendingBgWriteReq)               \
     M(IOLimiterPendingFgWriteReq)               \
     M(IOLimiterPendingBgReadReq)                \
-    M(IOLimiterPendingFgReadReq)
+    M(IOLimiterPendingFgReadReq)                \
+    M(StoragePoolV2Only)                        \
+    M(StoragePoolV3Only)                        \
+    M(StoragePoolMixMode)                       \
+    M(RegionPersisterRunMode)                   \
+    M(GlobalStorageRunMode)
 
 namespace CurrentMetrics
 {

dbms/src/Common/ErrorCodes.cpp

@@ -424,6 +424,7 @@ extern const int DEADLOCK_AVOIDED = 10013;
 extern const int PTHREAD_ERROR = 10014;
 extern const int PS_ENTRY_NOT_EXISTS = 10015;
 extern const int PS_ENTRY_NO_VALID_VERSION = 10016;
+extern const int PS_DIR_APPLY_INVALID_STATUS = 10017;
 } // namespace ErrorCodes
 
 } // namespace DB

dbms/src/Common/MyTime.cpp

@@ -992,6 +992,15 @@ int calcDayNum(int year, int month, int day)
     return delsum + year / 4 - temp;
 }
 
+UInt64 calcSeconds(int year, int month, int day, int hour, int minute, int second)
+{
+    if (year == 0 && month == 0)
+        return 0;
+    Int32 current_days = calcDayNum(year, month, day);
+    return current_days * MyTimeBase::SECOND_IN_ONE_DAY + hour * MyTimeBase::SECOND_IN_ONE_HOUR
+        + minute * MyTimeBase::SECOND_IN_ONE_MINUTE + second;
+}
+
 size_t maxFormattedDateTimeStringLength(const String & format)
 {
     size_t result = 0;
@@ -1142,7 +1151,7 @@ UInt64 addSeconds(UInt64 t, Int64 delta)
         return t;
     }
     MyDateTime my_time(t);
-    Int64 current_second = my_time.hour * 3600 + my_time.minute * 60 + my_time.second;
+    Int64 current_second = my_time.hour * MyTimeBase::SECOND_IN_ONE_HOUR + my_time.minute * MyTimeBase::SECOND_IN_ONE_MINUTE + my_time.second;
     current_second += delta;
     if (current_second >= 0)
     {
@@ -1161,9 +1170,9 @@ UInt64 addSeconds(UInt64 t, Int64 delta)
         current_second += days * MyTimeBase::SECOND_IN_ONE_DAY;
         addDays(my_time, -days);
     }
-    my_time.hour = current_second / 3600;
-    my_time.minute = (current_second % 3600) / 60;
-    my_time.second = current_second % 60;
+    my_time.hour = current_second / MyTimeBase::SECOND_IN_ONE_HOUR;
+    my_time.minute = (current_second % MyTimeBase::SECOND_IN_ONE_HOUR) / MyTimeBase::SECOND_IN_ONE_MINUTE;
+    my_time.second = current_second % MyTimeBase::SECOND_IN_ONE_MINUTE;
     return my_time.toPackedUInt();
 }
 

dbms/src/Common/MyTime.h

@@ -24,6 +24,9 @@ namespace DB
 struct MyTimeBase
 {
     static constexpr Int64 SECOND_IN_ONE_DAY = 86400;
+    static constexpr Int64 SECOND_IN_ONE_HOUR = 3600;
+    static constexpr Int64 SECOND_IN_ONE_MINUTE = 60;
+
 
     // copied from https://github.com/pingcap/tidb/blob/master/types/time.go
     // Core time bit fields.
@@ -193,6 +196,9 @@ std::pair<time_t, UInt32> roundTimeByFsp(time_t second, UInt64 nano_second, UInt
 
 int calcDayNum(int year, int month, int day);
 
+// returns seconds since '0000-00-00'
+UInt64 calcSeconds(int year, int month, int day, int hour, int minute, int second);
+
 size_t maxFormattedDateTimeStringLength(const String & format);
 
 inline time_t getEpochSecond(const MyDateTime & my_time, const DateLUTImpl & time_zone)

dbms/src/DataStreams/MockExchangeReceiverInputStream.cpp

@@ -0,0 +1,57 @@
+// Copyright 2022 PingCAP, Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <DataStreams/MockExchangeReceiverInputStream.h>
+#include <Flash/Coprocessor/ChunkCodec.h>
+
+namespace DB
+{
+MockExchangeReceiverInputStream::MockExchangeReceiverInputStream(const tipb::ExchangeReceiver & receiver, size_t max_block_size, size_t rows_)
+    : output_index(0)
+    , max_block_size(max_block_size)
+    , rows(rows_)
+{
+    for (int i = 0; i < receiver.field_types_size(); ++i)
+    {
+        columns.emplace_back(
+            getDataTypeByColumnInfoForComputingLayer(TiDB::fieldTypeToColumnInfo(receiver.field_types(i))),
+            fmt::format("exchange_receiver_{}", i));
+    }
+}
+
+ColumnPtr MockExchangeReceiverInputStream::makeColumn(ColumnWithTypeAndName elem) const
+{
+    auto column = elem.type->createColumn();
+    size_t row_count = 0;
+    for (size_t i = output_index; (i < rows) & (row_count < max_block_size); ++i)
+    {
+        column->insert((*elem.column)[i]);
+        ++row_count;
+    }
+    return column;
+}
+
+Block MockExchangeReceiverInputStream::readImpl()
+{
+    if (output_index >= rows)
+        return {};
+    ColumnsWithTypeAndName output_columns;
+    for (const auto & elem : columns)
+    {
+        output_columns.push_back({makeColumn(elem), elem.type, elem.name, elem.column_id});
+    }
+    output_index += max_block_size;
+    return Block(output_columns);
+}
+} // namespace DB

dbms/src/DataStreams/MockExchangeReceiverInputStream.h

@@ -0,0 +1,41 @@
+// Copyright 2022 PingCAP, Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <DataStreams/IProfilingBlockInputStream.h>
+#include <Storages/Transaction/TypeMapping.h>
+#include <tipb/executor.pb.h>
+
+namespace DB
+{
+/// Mock the receiver like table scan, can mock blocks according to the receiver's schema.
+/// TODO: Mock the receiver process
+class MockExchangeReceiverInputStream : public IProfilingBlockInputStream
+{
+public:
+    MockExchangeReceiverInputStream(const tipb::ExchangeReceiver & receiver, size_t max_block_size, size_t rows_);
+    Block getHeader() const override { return Block(columns); }
+    String getName() const override { return "MockExchangeReceiver"; }
+    ColumnsWithTypeAndName columns;
+    size_t output_index;
+    size_t max_block_size;
+    size_t rows;
+
+protected:
+    Block readImpl() override;
+    ColumnPtr makeColumn(ColumnWithTypeAndName elem) const;
+};
+
+} // namespace DB

dbms/src/DataStreams/MockExchangeSenderInputStream.cpp

@@ -0,0 +1,46 @@
+// Copyright 2022 PingCAP, Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <DataStreams/MockExchangeSenderInputStream.h>
+namespace DB
+{
+MockExchangeSenderInputStream::MockExchangeSenderInputStream(
+    const BlockInputStreamPtr & input,
+    const String & req_id)
+    : log(Logger::get(NAME, req_id))
+{
+    children.push_back(input);
+}
+
+Block MockExchangeSenderInputStream::getTotals()
+{
+    if (IProfilingBlockInputStream * child = dynamic_cast<IProfilingBlockInputStream *>(&*children.back()))
+    {
+        totals = child->getTotals();
+    }
+
+    return totals;
+}
+
+Block MockExchangeSenderInputStream::getHeader() const
+{
+    return children.back()->getHeader();
+}
+
+Block MockExchangeSenderInputStream::readImpl()
+{
+    return children.back()->read();
+}
+
+} // namespace DB

dbms/src/DataStreams/MockExchangeSenderInputStream.h

@@ -0,0 +1,46 @@
+// Copyright 2022 PingCAP, Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <DataStreams/IProfilingBlockInputStream.h>
+#include <Storages/Transaction/TypeMapping.h>
+#include <tipb/executor.pb.h>
+
+namespace DB
+{
+/// Currently, this operator do nothing with the block.
+/// TODO: Mock the sender process
+class MockExchangeSenderInputStream : public IProfilingBlockInputStream
+{
+private:
+    static constexpr auto NAME = "MockExchangeSender";
+
+public:
+    MockExchangeSenderInputStream(
+        const BlockInputStreamPtr & input,
+        const String & req_id);
+
+    String getName() const override { return NAME; }
+    Block getTotals() override;
+    Block getHeader() const override;
+
+protected:
+    Block readImpl() override;
+
+private:
+    const LoggerPtr log;
+};
+
+} // namespace DB

dbms/src/DataStreams/MockTableScanBlockInputStream.cpp

@@ -36,7 +36,7 @@ ColumnPtr MockTableScanBlockInputStream::makeColumn(ColumnWithTypeAndName elem)
 {
     auto column = elem.type->createColumn();
     size_t row_count = 0;
-    for (size_t i = output_index; i < rows & row_count < max_block_size; ++i)
+    for (size_t i = output_index; (i < rows) & (row_count < max_block_size); ++i)
     {
         column->insert((*elem.column)[i]);
         row_count++;

dbms/src/DataTypes/IDataType.h

@@ -471,6 +471,7 @@ class IDataType : private boost::noncopyable
     virtual bool isEnum() const { return false; };
 
     virtual bool isNullable() const { return false; }
+
     /** Is this type can represent only NULL value? (It also implies isNullable)
       */
     virtual bool onlyNull() const { return false; }