Optimize agg with collation & Optimize mem utils (#5834)

ref #5294

CMakeLists.txt

@@ -511,6 +511,11 @@ if (ARCH_AMD64)
     else()
         add_definitions(-DTIFLASH_COMPILER_VPCLMULQDQ_SUPPORT=0)
     endif()
+
+    check_cxx_compiler_flag("-mmovbe" TIFLASH_COMPILER_MOVBE_SUPPORT)
+    if (TIFLASH_COMPILER_MOVBE_SUPPORT)
+        set(COMPILER_MOVBE_FLAG "-mmovbe")
+    endif()
 else()
     add_definitions(-DTIFLASH_COMPILER_VPCLMULQDQ_SUPPORT=0)
 endif()

dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h

@@ -193,36 +193,36 @@ struct SingleValueDataString
 
     Int32 size = -1; /// -1 indicates that there is no value.
     Int32 capacity = 0; /// power of two or zero
-    char * large_data;
-    TiDB::TiDBCollatorPtr collator = nullptr;
+    char * large_data{};
+    TiDB::TiDBCollatorPtr collator{};
 
     bool less(const StringRef & a, const StringRef & b) const
     {
-        if (collator == nullptr)
+        if (unlikely(collator == nullptr))
             return a < b;
-        return collator->compare(a.data, a.size, b.data, b.size) < 0;
+        return collator->compareFastPath(a.data, a.size, b.data, b.size) < 0;
     }
 
     bool greater(const StringRef & a, const StringRef & b) const
     {
-        if (collator == nullptr)
+        if (unlikely(collator == nullptr))
             return a > b;
-        return collator->compare(a.data, a.size, b.data, b.size) > 0;
+        return collator->compareFastPath(a.data, a.size, b.data, b.size) > 0;
     }
 
     bool equalTo(const StringRef & a, const StringRef & b) const
     {
-        if (collator == nullptr)
+        if (unlikely(collator == nullptr))
             return a == b;
-        return collator->compare(a.data, a.size, b.data, b.size) == 0;
+        return collator->compareFastPath(a.data, a.size, b.data, b.size) == 0;
     }
 
 public:
     static constexpr Int32 AUTOMATIC_STORAGE_SIZE = 64;
     static constexpr Int32 MAX_SMALL_STRING_SIZE = AUTOMATIC_STORAGE_SIZE - sizeof(size) - sizeof(capacity) - sizeof(large_data) - sizeof(collator);
 
 private:
-    char small_data[MAX_SMALL_STRING_SIZE]; /// Including the terminating zero.
+    char small_data[MAX_SMALL_STRING_SIZE]{}; /// Including the terminating zero.
 
 public:
     bool has() const

dbms/src/AggregateFunctions/IAggregateFunction.h

@@ -439,7 +439,7 @@ struct AggregationCollatorsWrapper<true>
         if (likely(collators.size() > column_index))
         {
             if (collators[column_index] != nullptr)
-                return collators[column_index]->sortKey(in.data, in.size, sort_key_containers[column_index]);
+                return collators[column_index]->sortKeyFastPath(in.data, in.size, sort_key_containers[column_index]);
             return in;
         }
         else if (collators.empty())

dbms/src/Columns/ColumnString.cpp

@@ -18,6 +18,7 @@
 #include <Common/HashTable/Hash.h>
 #include <DataStreams/ColumnGathererStream.h>
 #include <Storages/Transaction/CollatorUtils.h>
+#include <common/memcpy.h>
 #include <fmt/core.h>
 
 
@@ -94,7 +95,7 @@ void ColumnString::insertRangeFrom(const IColumn & src, size_t start, size_t len
 
     size_t old_chars_size = chars.size();
     chars.resize(old_chars_size + nested_length);
-    memcpy(&chars[old_chars_size], &src_concrete.chars[nested_offset], nested_length);
+    inline_memcpy(&chars[old_chars_size], &src_concrete.chars[nested_offset], nested_length);
 
     if (start == 0 && offsets.empty())
     {
@@ -315,12 +316,10 @@ int ColumnString::compareAtWithCollationImpl(size_t n, size_t m, const IColumn &
 {
     const auto & rhs = static_cast<const ColumnString &>(rhs_);
 
-    return collator.compare(
-        reinterpret_cast<const char *>(&chars[offsetAt(n)]),
-        sizeAt(n) - 1, // Skip last zero byte.
-        reinterpret_cast<const char *>(&rhs.chars[rhs.offsetAt(m)]),
-        rhs.sizeAt(m) - 1 // Skip last zero byte.
-    );
+    auto a = getDataAt(n);
+    auto b = rhs.getDataAt(m);
+
+    return collator.compare(a.data, a.size, b.data, b.size);
 }
 
 // Derived must implement function `int compare(const char *, size_t, const char *, size_t)`.

dbms/src/Columns/ColumnString.h

@@ -211,10 +211,10 @@ class ColumnString final : public COWPtrHelper<IColumn, ColumnString>
 
         StringRef res;
 
-        if (collator != nullptr)
+        if (likely(collator != nullptr))
         {
             // Skip last zero byte.
-            auto sort_key = collator->sortKey(reinterpret_cast<const char *>(src), string_size - 1, sort_key_container);
+            auto sort_key = collator->sortKeyFastPath(reinterpret_cast<const char *>(src), string_size - 1, sort_key_container);
             string_size = sort_key.size;
             src = sort_key.data;
         }
@@ -244,10 +244,10 @@ class ColumnString final : public COWPtrHelper<IColumn, ColumnString>
     {
         size_t string_size = sizeAt(n);
         size_t offset = offsetAt(n);
-        if (collator != nullptr)
+        if (likely(collator != nullptr))
         {
             // Skip last zero byte.
-            auto sort_key = collator->sortKey(reinterpret_cast<const char *>(&chars[offset]), string_size - 1, sort_key_container);
+            auto sort_key = collator->sortKeyFastPath(reinterpret_cast<const char *>(&chars[offset]), string_size - 1, sort_key_container);
             string_size = sort_key.size;
             hash.update(reinterpret_cast<const char *>(&string_size), sizeof(string_size));
             hash.update(sort_key.data, sort_key.size);
@@ -278,16 +278,7 @@ class ColumnString final : public COWPtrHelper<IColumn, ColumnString>
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int /*nan_direction_hint*/) const override
     {
         const auto & rhs = static_cast<const ColumnString &>(rhs_);
-
-        const size_t size = sizeAt(n);
-        const size_t rhs_size = rhs.sizeAt(m);
-
-        int cmp = memcmp(&chars[offsetAt(n)], &rhs.chars[rhs.offsetAt(m)], std::min(size, rhs_size));
-
-        if (cmp != 0)
-            return cmp;
-        else
-            return size > rhs_size ? 1 : (size < rhs_size ? -1 : 0);
+        return getDataAtWithTerminatingZero(n).compare(rhs.getDataAtWithTerminatingZero(m));
     }
 
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int, const ICollator & collator) const override

dbms/src/Columns/ColumnsCommon.cpp

@@ -12,13 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#if __SSE2__
-#include <emmintrin.h>
-#endif
-
 #include <Columns/ColumnsCommon.h>
 #include <Columns/IColumn.h>
-
+#include <common/memcpy.h>
 
 namespace DB
 {
@@ -146,7 +142,7 @@ struct ResultOffsetsBuilder
     {
         const auto offsets_size_old = res_offsets.size();
         res_offsets.resize(offsets_size_old + SIMD_BYTES);
-        memcpy(&res_offsets[offsets_size_old], src_offsets_pos, SIMD_BYTES * sizeof(IColumn::Offset));
+        inline_memcpy(&res_offsets[offsets_size_old], src_offsets_pos, SIMD_BYTES * sizeof(IColumn::Offset));
 
         if (!first)
         {
@@ -194,7 +190,7 @@ void filterArraysImplGeneric(
 {
     const size_t size = src_offsets.size();
     if (size != filt.size())
-        throw Exception("Size of filter doesn't match size of column.", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
+        throw Exception(fmt::format("size of filter {} doesn't match size of column {}", filt.size(), size), ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
 
     ResultOffsetsBuilder result_offsets_builder(res_offsets);
 
@@ -223,7 +219,7 @@ void filterArraysImplGeneric(
 
         const auto elems_size_old = res_elems.size();
         res_elems.resize(elems_size_old + size);
-        memcpy(&res_elems[elems_size_old], &src_elems[offset], size * sizeof(T));
+        inline_memcpy(&res_elems[elems_size_old], &src_elems[offset], size * sizeof(T));
     };
 
 #if __SSE2__
@@ -233,7 +229,7 @@ void filterArraysImplGeneric(
 
     while (filt_pos < filt_end_aligned)
     {
-        const auto mask = _mm_movemask_epi8(_mm_cmpgt_epi8(
+        uint32_t mask = _mm_movemask_epi8(_mm_cmpgt_epi8(
             _mm_loadu_si128(reinterpret_cast<const __m128i *>(filt_pos)),
             zero_vec));
 
@@ -254,13 +250,16 @@ void filterArraysImplGeneric(
             /// copy elements for SIMD_BYTES arrays at once
             const auto elems_size_old = res_elems.size();
             res_elems.resize(elems_size_old + chunk_size);
-            memcpy(&res_elems[elems_size_old], &src_elems[chunk_offset], chunk_size * sizeof(T));
+            inline_memcpy(&res_elems[elems_size_old], &src_elems[chunk_offset], chunk_size * sizeof(T));
         }
         else
         {
-            for (size_t i = 0; i < SIMD_BYTES; ++i)
-                if (filt_pos[i])
-                    copy_array(offsets_pos + i);
+            while (mask)
+            {
+                size_t index = __builtin_ctz(mask);
+                copy_array(offsets_pos + index);
+                mask = mask & (mask - 1);
+            }
         }
 
         filt_pos += SIMD_BYTES;