Merge branch 'master' into func-unhex

pingcap · Dec 28, 2022 · 496855d · 496855d
2 parents d3c4ac5 + d989239
commit 496855d
Show file tree

Hide file tree

Showing 334 changed files with 7,575 additions and 4,323 deletions.
diff --git a/.clang-format b/.clang-format
@@ -89,4 +89,8 @@ SpacesInContainerLiterals: true
 SpacesInCStyleCastParentheses: false
 SpacesInParentheses: false
 SpacesInSquareBrackets: false
+---
+Language: Proto
+# Added to avoid annoying error message
+DisableFormat: true
 ...
diff --git a/contrib/client-c b/contrib/client-c
diff --git a/contrib/kvproto b/contrib/kvproto
diff --git a/contrib/tiflash-proxy b/contrib/tiflash-proxy
diff --git a/dbms/src/Columns/ColumnAggregateFunction.cpp b/dbms/src/Columns/ColumnAggregateFunction.cpp
@@ -372,21 +372,24 @@ void ColumnAggregateFunction::popBack(size_t n)
     data.resize_assume_reserved(new_size);
 }
 
-ColumnPtr ColumnAggregateFunction::replicate(const IColumn::Offsets & offsets) const
+ColumnPtr ColumnAggregateFunction::replicateRange(size_t start_row, size_t end_row, const IColumn::Offsets & offsets) const
 {
     size_t size = data.size();
     if (size != offsets.size())
         throw Exception("Size of offsets doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
 
+    assert(start_row < end_row);
+    assert(end_row <= size);
+
     if (size == 0)
         return cloneEmpty();
 
     auto res = createView();
     auto & res_data = res->getData();
-    res_data.reserve(offsets.back());
+    res_data.reserve(offsets[end_row - 1]);
 
     IColumn::Offset prev_offset = 0;
-    for (size_t i = 0; i < size; ++i)
+    for (size_t i = start_row; i < end_row; ++i)
     {
         size_t size_to_replicate = offsets[i] - prev_offset;
         prev_offset = offsets[i];

diff --git a/dbms/src/Columns/ColumnAggregateFunction.h b/dbms/src/Columns/ColumnAggregateFunction.h
@@ -173,7 +173,7 @@ class ColumnAggregateFunction final : public COWPtrHelper<IColumn, ColumnAggrega
 
     ColumnPtr permute(const Permutation & perm, size_t limit) const override;
 
-    ColumnPtr replicate(const Offsets & offsets) const override;
+    ColumnPtr replicateRange(size_t start_row, size_t end_row, const IColumn::Offsets & offsets) const override;
 
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
 

diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp
@@ -734,38 +734,9 @@ void ColumnArray::getPermutation(bool reverse, size_t limit, int nan_direction_h
     }
 }
 
-
-ColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
+ColumnPtr ColumnArray::replicateRange(size_t /*start_row*/, size_t /*end_row*/, const IColumn::Offsets & /*offsets*/) const
 {
-    if (typeid_cast<const ColumnUInt8 *>(data.get()))
-        return replicateNumber<UInt8>(replicate_offsets);
-    if (typeid_cast<const ColumnUInt16 *>(data.get()))
-        return replicateNumber<UInt16>(replicate_offsets);
-    if (typeid_cast<const ColumnUInt32 *>(data.get()))
-        return replicateNumber<UInt32>(replicate_offsets);
-    if (typeid_cast<const ColumnUInt64 *>(data.get()))
-        return replicateNumber<UInt64>(replicate_offsets);
-    if (typeid_cast<const ColumnInt8 *>(data.get()))
-        return replicateNumber<Int8>(replicate_offsets);
-    if (typeid_cast<const ColumnInt16 *>(data.get()))
-        return replicateNumber<Int16>(replicate_offsets);
-    if (typeid_cast<const ColumnInt32 *>(data.get()))
-        return replicateNumber<Int32>(replicate_offsets);
-    if (typeid_cast<const ColumnInt64 *>(data.get()))
-        return replicateNumber<Int64>(replicate_offsets);
-    if (typeid_cast<const ColumnFloat32 *>(data.get()))
-        return replicateNumber<Float32>(replicate_offsets);
-    if (typeid_cast<const ColumnFloat64 *>(data.get()))
-        return replicateNumber<Float64>(replicate_offsets);
-    if (typeid_cast<const ColumnString *>(data.get()))
-        return replicateString(replicate_offsets);
-    if (typeid_cast<const ColumnConst *>(data.get()))
-        return replicateConst(replicate_offsets);
-    if (typeid_cast<const ColumnNullable *>(data.get()))
-        return replicateNullable(replicate_offsets);
-    if (typeid_cast<const ColumnTuple *>(data.get()))
-        return replicateTuple(replicate_offsets);
-    return replicateGeneric(replicate_offsets);
+    throw Exception("not implement.", ErrorCodes::NOT_IMPLEMENTED);
 }
 
 

diff --git a/dbms/src/Columns/ColumnArray.h b/dbms/src/Columns/ColumnArray.h
@@ -95,7 +95,7 @@ class ColumnArray final : public COWPtrHelper<IColumn, ColumnArray>
     size_t byteSize() const override;
     size_t byteSize(size_t offset, size_t limit) const override;
     size_t allocatedBytes() const override;
-    ColumnPtr replicate(const Offsets & replicate_offsets) const override;
+    ColumnPtr replicateRange(size_t start_row, size_t end_row, const IColumn::Offsets & replicate_offsets) const override;
     ColumnPtr convertToFullColumnIfConst() const override;
     void getExtremes(Field & min, Field & max) const override;
 

diff --git a/dbms/src/Columns/ColumnConst.cpp b/dbms/src/Columns/ColumnConst.cpp
@@ -55,17 +55,19 @@ ColumnPtr ColumnConst::filter(const Filter & filt, ssize_t /*result_size_hint*/)
     return ColumnConst::create(data, countBytesInFilter(filt));
 }
 
-ColumnPtr ColumnConst::replicate(const Offsets & offsets) const
+ColumnPtr ColumnConst::replicateRange(size_t /*start_row*/, size_t end_row, const IColumn::Offsets & offsets) const
 {
     if (s != offsets.size())
         throw Exception(
             fmt::format("Size of offsets ({}) doesn't match size of column ({})", offsets.size(), s),
             ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
 
-    size_t replicated_size = 0 == s ? 0 : offsets.back();
+    assert(end_row <= s);
+    size_t replicated_size = 0 == s ? 0 : (offsets[end_row - 1]);
     return ColumnConst::create(data, replicated_size);
 }
 
+
 ColumnPtr ColumnConst::permute(const Permutation & perm, size_t limit) const
 {
     if (limit == 0)

diff --git a/dbms/src/Columns/ColumnConst.h b/dbms/src/Columns/ColumnConst.h
@@ -171,7 +171,7 @@ class ColumnConst final : public COWPtrHelper<IColumn, ColumnConst>
     void updateWeakHash32(WeakHash32 & hash, const TiDB::TiDBCollatorPtr &, String &) const override;
 
     ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
-    ColumnPtr replicate(const Offsets & offsets) const override;
+    ColumnPtr replicateRange(size_t start_row, size_t end_row, const IColumn::Offsets & offsets) const override;
     ColumnPtr permute(const Permutation & perm, size_t limit) const override;
     void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
 

diff --git a/dbms/src/Columns/ColumnDecimal.cpp b/dbms/src/Columns/ColumnDecimal.cpp
@@ -325,21 +325,24 @@ ColumnPtr ColumnDecimal<T>::filter(const IColumn::Filter & filt, ssize_t result_
 }
 
 template <typename T>
-ColumnPtr ColumnDecimal<T>::replicate(const IColumn::Offsets & offsets) const
+ColumnPtr ColumnDecimal<T>::replicateRange(size_t start_row, size_t end_row, const IColumn::Offsets & offsets) const
 {
     size_t size = data.size();
     if (size != offsets.size())
         throw Exception("Size of offsets doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
 
+    assert(start_row < end_row);
+    assert(end_row <= size);
+
     auto res = this->create(0, scale);
     if (0 == size)
         return res;
 
     typename Self::Container & res_data = res->getData();
-    res_data.reserve(offsets.back());
+    res_data.reserve(offsets[end_row - 1]);
 
     IColumn::Offset prev_offset = 0;
-    for (size_t i = 0; i < size; ++i)
+    for (size_t i = start_row; i < end_row; ++i)
     {
         size_t size_to_replicate = offsets[i] - prev_offset;
         prev_offset = offsets[i];
@@ -351,6 +354,7 @@ ColumnPtr ColumnDecimal<T>::replicate(const IColumn::Offsets & offsets) const
     return res;
 }
 
+
 template <typename T>
 void ColumnDecimal<T>::gather(ColumnGathererStream & gatherer)
 {

diff --git a/dbms/src/Columns/ColumnDecimal.h b/dbms/src/Columns/ColumnDecimal.h
@@ -167,7 +167,8 @@ class ColumnDecimal final : public COWPtrHelper<ColumnVectorHelper, ColumnDecima
     template <typename Type>
     ColumnPtr indexImpl(const PaddedPODArray<Type> & indexes, size_t limit) const;
 
-    ColumnPtr replicate(const IColumn::Offsets & offsets) const override;
+    ColumnPtr replicateRange(size_t start_row, size_t end_row, const IColumn::Offsets & offsets) const override;
+
     void getExtremes(Field & min, Field & max) const override;
 
     MutableColumns scatter(IColumn::ColumnIndex num_columns, const IColumn::Selector & selector) const override

diff --git a/dbms/src/Columns/ColumnFixedString.cpp b/dbms/src/Columns/ColumnFixedString.cpp
@@ -301,22 +301,25 @@ ColumnPtr ColumnFixedString::permute(const Permutation & perm, size_t limit) con
     return res;
 }
 
-ColumnPtr ColumnFixedString::replicate(const Offsets & offsets) const
+ColumnPtr ColumnFixedString::replicateRange(size_t start_row, size_t end_row, const IColumn::Offsets & offsets) const
 {
-    size_t col_size = size();
-    if (col_size != offsets.size())
+    size_t col_rows = size();
+    if (col_rows != offsets.size())
         throw Exception("Size of offsets doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
 
+    assert(start_row < end_row);
+    assert(end_row <= col_rows);
+
     auto res = ColumnFixedString::create(n);
 
-    if (0 == col_size)
+    if (0 == col_rows)
         return res;
 
     Chars_t & res_chars = res->chars;
-    res_chars.resize(n * offsets.back());
+    res_chars.resize(n * (offsets[end_row - 1]));
 
     Offset curr_offset = 0;
-    for (size_t i = 0; i < col_size; ++i)
+    for (size_t i = start_row; i < end_row; ++i)
         for (size_t next_offset = offsets[i]; curr_offset < next_offset; ++curr_offset)
             memcpySmallAllowReadWriteOverflow15(&res->chars[curr_offset * n], &chars[i * n], n);
 

diff --git a/dbms/src/Columns/ColumnFixedString.h b/dbms/src/Columns/ColumnFixedString.h
@@ -134,7 +134,7 @@ class ColumnFixedString final : public COWPtrHelper<IColumn, ColumnFixedString>
 
     ColumnPtr permute(const Permutation & perm, size_t limit) const override;
 
-    ColumnPtr replicate(const Offsets & offsets) const override;
+    ColumnPtr replicateRange(size_t start_row, size_t end_row, const IColumn::Offsets & offsets) const override;
 
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override
     {

diff --git a/dbms/src/Columns/ColumnFunction.cpp b/dbms/src/Columns/ColumnFunction.cpp
@@ -44,18 +44,21 @@ MutableColumnPtr ColumnFunction::cloneResized(size_t size) const
     return ColumnFunction::create(size, function, capture);
 }
 
-ColumnPtr ColumnFunction::replicate(const Offsets & offsets) const
+ColumnPtr ColumnFunction::replicateRange(size_t start_row, size_t end_row, const IColumn::Offsets & offsets) const
 {
     if (column_size != offsets.size())
         throw Exception(
             fmt::format("Size of offsets ({}) doesn't match size of column ({})", offsets.size(), column_size),
             ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
 
+    assert(start_row < end_row);
+    assert(end_row <= column_size);
+
     ColumnsWithTypeAndName capture = captured_columns;
     for (auto & column : capture)
-        column.column = column.column->replicate(offsets);
+        column.column = column.column->replicateRange(start_row, end_row, offsets);
 
-    size_t replicated_size = 0 == column_size ? 0 : offsets.back();
+    size_t replicated_size = 0 == column_size ? 0 : (offsets[end_row - 1]);
     return ColumnFunction::create(replicated_size, function, capture);
 }
 
@@ -68,20 +71,20 @@ ColumnPtr ColumnFunction::cut(size_t start, size_t length) const
     return ColumnFunction::create(length, function, capture);
 }
 
-ColumnPtr ColumnFunction::filter(const Filter & filt, ssize_t result_size_hint) const
+ColumnPtr ColumnFunction::filter(const Filter & filter, ssize_t result_size_hint) const
 {
-    if (column_size != filt.size())
+    if (column_size != filter.size())
         throw Exception(
-            fmt::format("Size of filter ({}) doesn't match size of column ({})", filt.size(), column_size),
+            fmt::format("Size of filter ({}) doesn't match size of column ({})", filter.size(), column_size),
             ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
 
     ColumnsWithTypeAndName capture = captured_columns;
     for (auto & column : capture)
-        column.column = column.column->filter(filt, result_size_hint);
+        column.column = column.column->filter(filter, result_size_hint);
 
     size_t filtered_size = 0;
     if (capture.empty())
-        filtered_size = countBytesInFilter(filt);
+        filtered_size = countBytesInFilter(filter);
     else
         filtered_size = capture.front().column->size();
 

diff --git a/dbms/src/Columns/ColumnFunction.h b/dbms/src/Columns/ColumnFunction.h
@@ -41,8 +41,8 @@ class ColumnFunction final : public COWPtrHelper<IColumn, ColumnFunction>
     size_t size() const override { return column_size; }
 
     ColumnPtr cut(size_t start, size_t length) const override;
-    ColumnPtr replicate(const Offsets & offsets) const override;
-    ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
+    ColumnPtr replicateRange(size_t start_row, size_t end_row, const IColumn::Offsets & offsets) const override;
+    ColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override;
     ColumnPtr permute(const Permutation & perm, size_t limit) const override;
     void insertDefault() override;
     void popBack(size_t n) override;

diff --git a/dbms/src/Columns/ColumnNullable.cpp b/dbms/src/Columns/ColumnNullable.cpp
@@ -524,14 +524,13 @@ void ColumnNullable::getExtremes(Field & min, Field & max) const
     });
 }
 
-ColumnPtr ColumnNullable::replicate(const Offsets & offsets) const
+ColumnPtr ColumnNullable::replicateRange(size_t start_row, size_t end_row, const IColumn::Offsets & offsets) const
 {
-    ColumnPtr replicated_data = getNestedColumn().replicate(offsets);
-    ColumnPtr replicated_null_map = getNullMapColumn().replicate(offsets);
+    ColumnPtr replicated_data = getNestedColumn().replicateRange(start_row, end_row, offsets);
+    ColumnPtr replicated_null_map = getNullMapColumn().replicateRange(start_row, end_row, offsets);
     return ColumnNullable::create(replicated_data, replicated_null_map);
 }
 
-
 template <bool negative>
 void ColumnNullable::applyNullMapImpl(const ColumnUInt8 & map)
 {

diff --git a/dbms/src/Columns/ColumnNullable.h b/dbms/src/Columns/ColumnNullable.h
@@ -92,7 +92,7 @@ class ColumnNullable final : public COWPtrHelper<IColumn, ColumnNullable>
     size_t byteSize() const override;
     size_t byteSize(size_t offset, size_t limit) const override;
     size_t allocatedBytes() const override;
-    ColumnPtr replicate(const Offsets & replicate_offsets) const override;
+    ColumnPtr replicateRange(size_t start_row, size_t end_row, const IColumn::Offsets & replicate_offsets) const override;
     void updateHashWithValue(size_t n, SipHash & hash, const TiDB::TiDBCollatorPtr &, String &) const override;
     void updateHashWithValues(IColumn::HashValues & hash_values, const TiDB::TiDBCollatorPtr &, String &) const override;
     void updateWeakHash32(WeakHash32 & hash, const TiDB::TiDBCollatorPtr &, String &) const override;

diff --git a/dbms/src/Columns/ColumnString.cpp b/dbms/src/Columns/ColumnString.cpp
@@ -230,28 +230,30 @@ void ColumnString::getPermutation(bool reverse, size_t limit, int /*nan_directio
     }
 }
 
-
-ColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const
+ColumnPtr ColumnString::replicateRange(size_t start_row, size_t end_row, const IColumn::Offsets & replicate_offsets) const
 {
-    size_t col_size = size();
-    if (col_size != replicate_offsets.size())
+    size_t col_rows = size();
+    if (col_rows != replicate_offsets.size())
         throw Exception("Size of offsets doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
 
+    assert(start_row < end_row);
+    assert(end_row <= col_rows);
+
     auto res = ColumnString::create();
 
-    if (0 == col_size)
+    if (0 == col_rows)
         return res;
 
     Chars_t & res_chars = res->chars;
     Offsets & res_offsets = res->offsets;
-    res_chars.reserve(chars.size() / col_size * replicate_offsets.back());
-    res_offsets.reserve(replicate_offsets.back());
+    res_chars.reserve(chars.size() / col_rows * (replicate_offsets[end_row - 1]));
+    res_offsets.reserve(replicate_offsets[end_row - 1]);
 
     Offset prev_replicate_offset = 0;
-    Offset prev_string_offset = 0;
+    Offset prev_string_offset = start_row == 0 ? 0 : offsets[start_row - 1];
     Offset current_new_offset = 0;
 
-    for (size_t i = 0; i < col_size; ++i)
+    for (size_t i = start_row; i < end_row; ++i)
     {
         size_t size_to_replicate = replicate_offsets[i] - prev_replicate_offset;
         size_t string_size = offsets[i] - prev_string_offset;

diff --git a/dbms/src/Columns/ColumnString.h b/dbms/src/Columns/ColumnString.h
@@ -301,7 +301,7 @@ class ColumnString final : public COWPtrHelper<IColumn, ColumnString>
     /// Sorting with respect of collation.
     void getPermutationWithCollationImpl(const ICollator & collator, bool reverse, size_t limit, Permutation & res) const;
 
-    ColumnPtr replicate(const Offsets & replicate_offsets) const override;
+    ColumnPtr replicateRange(size_t start_row, size_t end_row, const IColumn::Offsets & replicate_offsets) const override;
 
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override
     {
@@ -320,14 +320,29 @@ class ColumnString final : public COWPtrHelper<IColumn, ColumnString>
     void getExtremes(Field & min, Field & max) const override;
 
 
-    bool canBeInsideNullable() const override { return true; }
+    bool canBeInsideNullable() const override
+    {
+        return true;
+    }
 
 
-    Chars_t & getChars() { return chars; }
-    const Chars_t & getChars() const { return chars; }
+    Chars_t & getChars()
+    {
+        return chars;
+    }
+    const Chars_t & getChars() const
+    {
+        return chars;
+    }
 
-    Offsets & getOffsets() { return offsets; }
-    const Offsets & getOffsets() const { return offsets; }
+    Offsets & getOffsets()
+    {
+        return offsets;
+    }
+    const Offsets & getOffsets() const
+    {
+        return offsets;
+    }
 };