Skip to content

Commit

Permalink
Merge branch 'master' into fix-mac-llvm
Browse files Browse the repository at this point in the history
  • Loading branch information
solotzg authored Jul 22, 2022
2 parents ff2706d + 6a8c26c commit 94c4cb4
Show file tree
Hide file tree
Showing 9 changed files with 404 additions and 7 deletions.
2 changes: 1 addition & 1 deletion contrib/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ SET (BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE BOOL "Disable google-benchmark testi
add_subdirectory(benchmark)

set (BUILD_TESTING OFF CACHE BOOL "Disable cpu-features testing" FORCE)
if ((NOT APPLE) AND (NOT ARCH_AARCH64))
if (NOT (APPLE AND ARCH_AARCH64))
add_subdirectory(cpu_features)
endif()

Expand Down
4 changes: 2 additions & 2 deletions dbms/src/Flash/Coprocessor/DAGUtils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -622,8 +622,8 @@ const std::unordered_map<tipb::ScalarFuncSig, String> scalar_func_map({
//{tipb::ScalarFuncSig::Format, "cast"},
//{tipb::ScalarFuncSig::FormatWithLocale, "cast"},
//{tipb::ScalarFuncSig::FromBase64, "cast"},
//{tipb::ScalarFuncSig::HexIntArg, "cast"},
//{tipb::ScalarFuncSig::HexStrArg, "cast"},
{tipb::ScalarFuncSig::HexIntArg, "hexInt"},
{tipb::ScalarFuncSig::HexStrArg, "hexStr"},
//{tipb::ScalarFuncSig::InsertUTF8, "cast"},
//{tipb::ScalarFuncSig::Insert, "cast"},
//{tipb::ScalarFuncSig::InstrUTF8, "cast"},
Expand Down
183 changes: 182 additions & 1 deletion dbms/src/Functions/FunctionsString.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4083,7 +4083,7 @@ class PadUTF8Impl : public IFunction
break;
default:
throw Exception(fmt::format("the second argument type of {} is invalid, expect integer, got {}", getName(), type_index), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
};
}
}
};

Expand Down Expand Up @@ -5120,6 +5120,185 @@ class FunctionFormatWithLocale : public IFunction
}
};

class FunctionHexStr : public IFunction
{
public:
static constexpr auto name = "hexStr";
FunctionHexStr() = default;

static FunctionPtr create(const Context & /*context*/)
{
return std::make_shared<FunctionHexStr>();
}

std::string getName() const override { return name; }
size_t getNumberOfArguments() const override { return 1; }
bool useDefaultImplementationForConstants() const override { return true; }

DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (!arguments[0]->isStringOrFixedString())
throw Exception(
fmt::format("Illegal type {} of first argument of function {}", arguments[0]->getName(), getName()),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return std::make_shared<DataTypeString>();
}

void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override
{
const ColumnPtr & column = block.getByPosition(arguments[0]).column;
if (const auto * col = checkAndGetColumn<ColumnString>(column.get()))
{
auto col_res = ColumnString::create();
vector(col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets());
block.getByPosition(result).column = std::move(col_res);
}
else if (const auto * col = checkAndGetColumn<ColumnFixedString>(column.get()))
{
auto col_res = ColumnFixedString::create(col->getN() * 2);
vectorFixed(col->getChars(), col->getN(), col_res->getChars());
block.getByPosition(result).column = std::move(col_res);
}
else
throw Exception(
fmt::format("Illegal column {} of argument of function {}", block.getByPosition(arguments[0]).column->getName(), getName()),
ErrorCodes::ILLEGAL_COLUMN);
}

private:
static constexpr UInt8 hexTable[17] = "0123456789ABCDEF";

static void vector(const ColumnString::Chars_t & data,
const ColumnString::Offsets & offsets,
ColumnString::Chars_t & res_data,
ColumnString::Offsets & res_offsets)
{
size_t size = offsets.size();
// every string contains a tailing zero, which will not be hexed, so minus size to remove these doubled zeros
res_data.resize(data.size() * 2 - size);
res_offsets.resize(size);

ColumnString::Offset prev_offset = 0;
for (size_t i = 0; i < size; ++i)
{
for (size_t j = prev_offset; j < offsets[i] - 1; ++j)
{
ColumnString::Offset pos = j * 2 - i;
UInt8 byte = data[j];
res_data[pos] = hexTable[byte >> 4];
res_data[pos + 1] = hexTable[byte & 0x0f];
}
// the last element written by the previous loop is:
// `(offsets[i] - 2) * 2 - i + 1 = offsets[i] * 2 - i - 3`
// then the zero should be written to `offsets[i] * 2 - i - 2`
res_data[offsets[i] * 2 - i - 2] = 0;
res_offsets[i] = offsets[i] * 2 - i - 1;

prev_offset = offsets[i];
}
}

static void vectorFixed(const ColumnString::Chars_t & data, size_t length, ColumnString::Chars_t & res_data)
{
size_t size = data.size() / length;
res_data.resize(data.size() * 2);

for (size_t i = 0; i < size; ++i)
for (size_t j = i * length; j < (i + 1) * length; ++j)
{
ColumnString::Offset pos = j * 2;
UInt8 byte = data[j];
res_data[pos] = hexTable[byte >> 4];
res_data[pos + 1] = hexTable[byte & 0x0f];
}
}
};

class FunctionHexInt : public IFunction
{
public:
static constexpr auto name = "hexInt";
FunctionHexInt() = default;

static FunctionPtr create(const Context & /*context*/)
{
return std::make_shared<FunctionHexInt>();
}

std::string getName() const override { return name; }
size_t getNumberOfArguments() const override { return 1; }
bool useDefaultImplementationForConstants() const override { return true; }

DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (!arguments[0]->isNumber())
throw Exception(
fmt::format("Illegal type {} of first argument of function {}", arguments[0]->getName(), getName()),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return std::make_shared<DataTypeString>();
}

void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override
{
if (executeHexInt<UInt8>(block, arguments, result)
|| executeHexInt<UInt16>(block, arguments, result)
|| executeHexInt<UInt32>(block, arguments, result)
|| executeHexInt<UInt64>(block, arguments, result)
|| executeHexInt<Int8>(block, arguments, result)
|| executeHexInt<Int16>(block, arguments, result)
|| executeHexInt<Int32>(block, arguments, result)
|| executeHexInt<Int64>(block, arguments, result))
{
return;
}
else
{
throw Exception(fmt::format("Illegal argument of function {}", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
}

private:
template <typename IntType>
bool executeHexInt(
Block & block,
const ColumnNumbers & arguments,
const size_t result) const
{
ColumnPtr & column = block.getByPosition(arguments[0]).column;
const auto col = checkAndGetColumn<ColumnVector<IntType>>(column.get());
if (col == nullptr)
{
return false;
}
size_t size = col->size();

auto col_res = ColumnString::create();

ColumnString::Chars_t & res_chars = col_res->getChars();
// Convert a UInt64 to hex, will cost 17 bytes at most
res_chars.reserve(size * 17);
ColumnString::Offsets & res_offsets = col_res->getOffsets();
res_offsets.resize(size);

size_t prev_res_offset = 0;
for (size_t i = 0; i < size; ++i)
{
UInt64 number = col->getUInt(i);

int print_size = sprintf(reinterpret_cast<char *>(&res_chars[prev_res_offset]), "%lX", number);
res_chars[prev_res_offset + print_size] = 0;
// Add the size of printed string and a tailing zero
prev_res_offset += print_size + 1;
res_offsets[i] = prev_res_offset;
}
res_chars.resize(prev_res_offset);

block.getByPosition(result).column = std::move(col_res);

return true;
}
};

// clang-format off
struct NameEmpty { static constexpr auto name = "empty"; };
struct NameNotEmpty { static constexpr auto name = "notEmpty"; };
Expand Down Expand Up @@ -5204,6 +5383,8 @@ void registerFunctionsString(FunctionFactory & factory)
factory.registerFunction<FunctionSubStringIndex>();
factory.registerFunction<FunctionFormat>();
factory.registerFunction<FunctionFormatWithLocale>();
factory.registerFunction<FunctionHexStr>();
factory.registerFunction<FunctionHexInt>();
factory.registerFunction<FunctionRepeat>();
}
} // namespace DB
86 changes: 86 additions & 0 deletions dbms/src/Functions/tests/gtest_strings_hexint.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
// Copyright 2022 PingCAP, Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <DataTypes/DataTypeNullable.h>
#include <TestUtils/FunctionTestUtils.h>
#include <TestUtils/TiFlashTestBasic.h>

#include <string>
#include <vector>


namespace DB
{
namespace tests
{
class HexIntTest : public DB::tests::FunctionTest
{
};

TEST_F(HexIntTest, hexint_all_unit_Test)
try
{
const String & func_name = "hexInt";

ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"1348B21", std::nullopt, "0", "FFFFFFFFFECB74DF", "8000000000000000", "7FFFFFFFFFFFFFFF"}),
executeFunction(
func_name,
createColumn<Nullable<Int64>>({20220705, std::nullopt, 0, -20220705, std::numeric_limits<Int64>::min(), std::numeric_limits<Int64>::max()})));

ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"1348B21", std::nullopt, "0", "0", "FFFFFFFFFFFFFFFF"}),
executeFunction(
func_name,
createColumn<Nullable<UInt64>>({20220705, std::nullopt, 0, std::numeric_limits<UInt64>::min(), std::numeric_limits<UInt64>::max()})));

ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"13414DA", std::nullopt, "0", "FFFFFFFFFECBEB26", "FFFFFFFF80000000", "7FFFFFFF"}),
executeFunction(
func_name,
createColumn<Nullable<Int32>>({20190426, std::nullopt, 0, -20190426, std::numeric_limits<Int32>::min(), std::numeric_limits<Int32>::max()})));

ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"13414DA", std::nullopt, "0", "0", "FFFFFFFF"}),
executeFunction(
func_name,
createColumn<Nullable<UInt32>>({20190426, std::nullopt, 0, std::numeric_limits<UInt32>::min(), std::numeric_limits<UInt32>::max()})));

ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"3039", std::nullopt, "0", "FFFFFFFFFFFFCFC7", "FFFFFFFFFFFF8000", "7FFF"}),
executeFunction(
func_name,
createColumn<Nullable<Int16>>({12345, std::nullopt, 0, -12345, std::numeric_limits<Int16>::min(), std::numeric_limits<Int16>::max()})));

ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"3039", std::nullopt, "0", "0", "FFFF"}),
executeFunction(
func_name,
createColumn<Nullable<UInt16>>({12345, std::nullopt, 0, std::numeric_limits<UInt16>::min(), std::numeric_limits<UInt16>::max()})));

ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"78", std::nullopt, "0", "FFFFFFFFFFFFFF88", "FFFFFFFFFFFFFF80", "7F"}),
executeFunction(
func_name,
createColumn<Nullable<Int8>>({120, std::nullopt, 0, -120, std::numeric_limits<Int8>::min(), std::numeric_limits<Int8>::max()})));

ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"8F", std::nullopt, "0", "0", "FF"}),
executeFunction(
func_name,
createColumn<Nullable<UInt8>>({143, std::nullopt, 0, std::numeric_limits<UInt8>::min(), std::numeric_limits<UInt8>::max()})));
}
CATCH
} // namespace tests
} // namespace DB
58 changes: 58 additions & 0 deletions dbms/src/Functions/tests/gtest_strings_hexstr.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// Copyright 2022 PingCAP, Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <DataTypes/DataTypeNullable.h>
#include <TestUtils/FunctionTestUtils.h>
#include <TestUtils/TiFlashTestBasic.h>

#include <string>
#include <vector>


namespace DB
{
namespace tests
{
class HexStrTest : public DB::tests::FunctionTest
{
};

TEST_F(HexStrTest, hexstr_all_unit_Test)
try
{
const String & func_name = "hexStr";

ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"7777772E70696E676361702E636F6D", "61626364", std::nullopt, ""}),
executeFunction(
func_name,
createColumn<Nullable<String>>({"www.pingcap.com", "abcd", std::nullopt, ""})));

// CJK and emoji
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"E38195E38289E381ABE585A5", "E6B58BE8AF95E6B58BE8AF95E6B58BE8AF95E6B58BE8AF9561626364E6B58BE8AF95", "F09F8DBB", "F09F8FB4E2808DE298A0EFB88F"}),
executeFunction(
func_name,
createColumn<Nullable<String>>({"さらに入", "测试测试测试测试abcd测试", "🍻", "🏴‍☠️"})));

// Special Empty Character
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"09", "0A", "20"}),
executeFunction(
func_name,
createColumn<Nullable<String>>({"\t", "\n", " "})));
}
CATCH
} // namespace tests
} // namespace DB
4 changes: 2 additions & 2 deletions dbms/src/Storages/Transaction/KVStore.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -369,10 +369,10 @@ bool KVStore::canFlushRegionDataImpl(const RegionPtr & curr_region_ptr, UInt8 fl
}
if (can_flush && flush_if_possible)
{
LOG_FMT_DEBUG(log, "{} flush region due to can_flush_data", curr_region.toString(false));
LOG_FMT_DEBUG(log, "{} flush region due to canFlushRegionData", curr_region.toString(false));
if (tryFlushRegionCacheInStorage(tmt, curr_region, log, try_until_succeed))
{
persistRegion(curr_region, region_task_lock, "compact raft log");
persistRegion(curr_region, region_task_lock, "canFlushRegionData before compact raft log");
curr_region.markCompactLog();
curr_region.cleanApproxMemCacheInfo();
return true;
Expand Down
Loading

0 comments on commit 94c4cb4

Please sign in to comment.