diff --git a/dbms/src/Flash/Coprocessor/DAGUtils.cpp b/dbms/src/Flash/Coprocessor/DAGUtils.cpp index 1c4a8e521e9..8b6a700bce9 100755 --- a/dbms/src/Flash/Coprocessor/DAGUtils.cpp +++ b/dbms/src/Flash/Coprocessor/DAGUtils.cpp @@ -648,8 +648,8 @@ const std::unordered_map scalar_func_map({ {tipb::ScalarFuncSig::LpadUTF8, "lpadUTF8"}, {tipb::ScalarFuncSig::Lpad, "lpad"}, //{tipb::ScalarFuncSig::MakeSet, "cast"}, - //{tipb::ScalarFuncSig::OctInt, "cast"}, - //{tipb::ScalarFuncSig::OctString, "cast"}, + {tipb::ScalarFuncSig::OctInt, "octInt"}, + {tipb::ScalarFuncSig::OctString, "octString"}, //{tipb::ScalarFuncSig::Ord, "cast"}, //{tipb::ScalarFuncSig::Quote, "cast"}, {tipb::ScalarFuncSig::Repeat, "repeat"}, diff --git a/dbms/src/Functions/FunctionsString.cpp b/dbms/src/Functions/FunctionsString.cpp index e0c41857c66..f5a3a9b7342 100644 --- a/dbms/src/Functions/FunctionsString.cpp +++ b/dbms/src/Functions/FunctionsString.cpp @@ -32,6 +32,9 @@ #include #include +#if __has_include() +#include +#endif #include #include @@ -5782,6 +5785,219 @@ class FunctionElt : public IFunction } }; +class FunctionOctInt : public IFunction +{ +public: + static constexpr auto name = "octInt"; + FunctionOctInt() = default; + + static FunctionPtr create(const Context & /*context*/) + { + return std::make_shared(); + } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 1; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (arguments.size() != 1) + throw Exception( + fmt::format("Number of arguments for function {} doesn't match: passed {}, should be 1.", getName(), arguments.size()), + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + const auto & first_argument = removeNullable(arguments[0]); + if (!first_argument->isInteger()) + throw Exception( + fmt::format("Illegal type {} of first argument of function {}", first_argument->getName(), getName()), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override + { + if (executeOctInt(block, arguments, result) + || executeOctInt(block, arguments, result) + || executeOctInt(block, arguments, result) + || executeOctInt(block, arguments, result) + || executeOctInt(block, arguments, result) + || executeOctInt(block, arguments, result) + || executeOctInt(block, arguments, result) + || executeOctInt(block, arguments, result)) + { + return; + } + else + { + throw Exception(fmt::format("Illegal argument of function {}", getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + } + +private: + template + bool executeOctInt( + Block & block, + const ColumnNumbers & arguments, + const size_t result) const + { + ColumnPtr & column = block.getByPosition(arguments[0]).column; + const auto col = checkAndGetColumn>(column.get()); + if (col == nullptr) + { + return false; + } + size_t size = col->size(); + + auto col_res = ColumnString::create(); + + ColumnString::Chars_t & res_chars = col_res->getChars(); + // Convert a UInt64 to oct, will cost 23 bytes at most + res_chars.reserve(size * 23); + ColumnString::Offsets & res_offsets = col_res->getOffsets(); + res_offsets.resize(size); + + auto res_chars_iter = res_chars.begin(); + for (size_t i = 0; i < size; ++i) + { + UInt64 number = col->getUInt(i); + + res_chars_iter = fmt::format_to(res_chars_iter, "{:o}", number); + *(++res_chars_iter) = 0; + // Add the size of printed string and a tailing zero + res_offsets[i] = res_chars_iter - res_chars.begin(); + } + res_chars.resize(res_chars_iter - res_chars.begin()); + + block.getByPosition(result).column = std::move(col_res); + + return true; + } +}; + + +struct OctStringImpl +{ + static String execute(const String & arg) + { + bool is_negative = false, is_overflow = false; + // Removing the leading space + auto begin_pos_iter = std::find_if_not(arg.begin(), arg.end(), isWhitespaceASCII); + if (begin_pos_iter == arg.end()) + { + return "0"; + } + if (*begin_pos_iter == '-') + { + is_negative = true; + ++begin_pos_iter; + } + else if (*begin_pos_iter == '+') + { + ++begin_pos_iter; + } + + // Special treatment for case ` pingcap16` + if (!std::isdigit(*begin_pos_iter)) + { + return "0"; + } + auto begin_pos = begin_pos_iter - arg.begin(); + +#if __has_include() + UInt64 value = std::numeric_limits::max(); + auto from_chars_res = std::from_chars(arg.data() + begin_pos, arg.data() + arg.size(), value); + if (from_chars_res.ec != std::errc{}) + { + if (from_chars_res.ec != std::errc::result_out_of_range) + { + return ""; + } + is_overflow = true; + } +#else + UInt64 value = strtoull(arg.c_str() + begin_pos, nullptr, 10); + if (errno) + { + if (errno != ERANGE) + { + errno = 0; + return ""; + } + errno = 0; + is_overflow = true; + } +#endif + + if (is_negative && !is_overflow) + { + value = -value; + } + + return fmt::format("{:o}", value); + } + + template + static void execute(const Column * arg_col0, ColumnString & res_col) + { + for (size_t i = 0; i < arg_col0->size(); ++i) + { + // we want some String operation in OctStringImpl so we call toString here + String result = execute(arg_col0->getDataAt(i).toString()); + res_col.insertData(result.c_str(), result.size()); + } + } +}; +class FunctionOctString : public IFunction +{ +public: + static constexpr auto name = "octString"; + FunctionOctString() = default; + + static FunctionPtr create(const Context & /*context*/) + { + return std::make_shared(); + } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 1; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (arguments.size() != 1) + throw Exception( + fmt::format("Number of arguments for function {} doesn't match: passed {}, should be 1.", getName(), arguments.size()), + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + const auto & first_argument = removeNullable(arguments[0]); + if (!first_argument->isString()) + throw Exception( + fmt::format("Illegal type {} of first argument of function {}", first_argument->getName(), getName()), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override + { + const ColumnPtr & column = block.getByPosition(arguments[0]).column; + if (const auto * col = checkAndGetColumn(column.get())) + { + auto col_res = ColumnString::create(); + OctStringImpl::execute(col, *col_res); + block.getByPosition(result).column = std::move(col_res); + } + else + { + throw Exception( + fmt::format("Illegal column {} of argument of function {}", block.getByPosition(arguments[0]).column->getName(), getName()), + ErrorCodes::ILLEGAL_COLUMN); + } + } +}; + // clang-format off struct NameEmpty { static constexpr auto name = "empty"; }; struct NameNotEmpty { static constexpr auto name = "notEmpty"; }; @@ -5872,5 +6088,7 @@ void registerFunctionsString(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); } } // namespace DB diff --git a/dbms/src/Functions/tests/gtest_strings_octint.cpp b/dbms/src/Functions/tests/gtest_strings_octint.cpp new file mode 100644 index 00000000000..038548405b4 --- /dev/null +++ b/dbms/src/Functions/tests/gtest_strings_octint.cpp @@ -0,0 +1,196 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include + +namespace DB +{ +namespace tests +{ +class OctIntTest : public DB::tests::FunctionTest +{ +protected: + static constexpr auto func_name = "octInt"; + + static ColumnWithTypeAndName toConst(const String & s) + { + return createConstColumn(1, s); + } +}; + +TEST_F(OctIntTest, Simple) +try +{ + ASSERT_COLUMN_EQ( + toNullableVec({"0", + "1", + "2", + "4", + "10"}), + executeFunction( + func_name, + toNullableVec({0, + 1, + 2, + 4, + 8}))); +} +CATCH + +TEST_F(OctIntTest, AllType) +try +{ + ASSERT_COLUMN_EQ( + toNullableVec({{}, + "10", + "0", + "377"}), + executeFunction( + func_name, + toNullableVec({{}, + 8, + 0, + 255}))); + ASSERT_COLUMN_EQ( + toConst("10"), + executeFunction( + func_name, + createConstColumn(1, 8))); + + ASSERT_COLUMN_EQ( + toNullableVec({{}, + "10", + "0", + "177777"}), + executeFunction( + func_name, + toNullableVec({{}, + 8, + 0, + 65535}))); + ASSERT_COLUMN_EQ( + toConst("10"), + executeFunction( + func_name, + createConstColumn(1, 8))); + + ASSERT_COLUMN_EQ( + toNullableVec({{}, + "10", + "0", + "37777777777"}), + executeFunction( + func_name, + toNullableVec({{}, + 8, + 0, + 4294967295}))); + ASSERT_COLUMN_EQ( + toConst("10"), + executeFunction( + func_name, + createConstColumn(1, 8))); + + ASSERT_COLUMN_EQ( + toNullableVec({{}, + "10", + "0", + "1777777777777777777777"}), + executeFunction( + func_name, + toNullableVec({{}, + 8, + 0, + std::numeric_limits::max()}))); + ASSERT_COLUMN_EQ( + toConst("10"), + executeFunction( + func_name, + createConstColumn(1, 8))); + + ASSERT_COLUMN_EQ( + toNullableVec({{}, + "10", + "1777777777777777777600", + "177"}), + executeFunction( + func_name, + toNullableVec({{}, + 8, + -128, + 127}))); + ASSERT_COLUMN_EQ( + toConst("10"), + executeFunction( + func_name, + createConstColumn(1, 8))); + + ASSERT_COLUMN_EQ( + toNullableVec({{}, + "10", + "1777777777777777700000", + "77777"}), + executeFunction( + func_name, + toNullableVec({{}, + 8, + std::numeric_limits::min(), + std::numeric_limits::max()}))); + ASSERT_COLUMN_EQ( + toConst("10"), + executeFunction( + func_name, + createConstColumn(1, 8))); + + ASSERT_COLUMN_EQ( + toNullableVec({{}, + "10", + "1777777777760000000000", + "17777777777"}), + executeFunction( + func_name, + toNullableVec({{}, + 8, + std::numeric_limits::min(), + std::numeric_limits::max()}))); + + ASSERT_COLUMN_EQ( + toConst("10"), + executeFunction( + func_name, + createConstColumn(1, 8))); + + ASSERT_COLUMN_EQ( + toNullableVec({{}, + "10", + "1000000000000000000000", + "777777777777777777777"}), + executeFunction( + func_name, + toNullableVec({{}, + 8, + std::numeric_limits::min(), + std::numeric_limits::max()}))); + ASSERT_COLUMN_EQ( + toConst("10"), + executeFunction( + func_name, + createConstColumn(1, 8))); +} +CATCH +} // namespace tests +} // namespace DB diff --git a/dbms/src/Functions/tests/gtest_strings_octstring.cpp b/dbms/src/Functions/tests/gtest_strings_octstring.cpp new file mode 100644 index 00000000000..64e3fe1d0a7 --- /dev/null +++ b/dbms/src/Functions/tests/gtest_strings_octstring.cpp @@ -0,0 +1,104 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +namespace DB +{ +namespace tests +{ +class OctStringTest : public DB::tests::FunctionTest +{ +protected: + static constexpr auto func_name = "octString"; + + static ColumnWithTypeAndName toConst(const String & s) + { + return createConstColumn(1, s); + } +}; + + +TEST_F(OctStringTest, Simple) +try +{ + ASSERT_COLUMN_EQ( + toNullableVec({"0", "1", "2", "10", "12"}), + executeFunction( + func_name, + toNullableVec({"0", "1", "2", "8", "10"}))); + + ASSERT_COLUMN_EQ( + toConst("10"), + executeFunction( + func_name, + toConst("8"))); +} +CATCH + +TEST_F(OctStringTest, Boundary) +try +{ + ASSERT_COLUMN_EQ( + toNullableVec({{}, + "0", + "0", + "0", + "0", + "10", + "20", + "0", + "1", + "10"}), + executeFunction( + func_name, + toNullableVec({{}, + "", + "pingcap", + " pingcap", + "ping cap", + "8pingcap", + "16 pingcap", + "pingcap16", + "1ping6 cap", + "\n\t\r\f\v8pingcap"}))); + + ASSERT_COLUMN_EQ( + toNullableVec({"0", + "0", + "0", + "0", + "1", + "10", + "1777777777777777777777", + "1777777777777777777777", + "1777777777777777777777", + "1777777777777777777777"}), + executeFunction( + func_name, + toNullableVec({"-+8", + "+-8", + "--8", + "++8", + "1.9", + "+8", + "-1", + "-1.9", + "99999999999999999999999999", + "-99999999999999999999999999"}))); +} +CATCH +} // namespace tests +} // namespace DB diff --git a/tests/fullstack-test/expr/oct_int.test b/tests/fullstack-test/expr/oct_int.test new file mode 100644 index 00000000000..df5a87f26c1 --- /dev/null +++ b/tests/fullstack-test/expr/oct_int.test @@ -0,0 +1,32 @@ +# Copyright 2022 PingCAP, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mysql> drop table if exists test.t; +mysql> create table if not exists test.t(a bigint, b double); +mysql> insert into test.t values(1, 1.9); +mysql> insert into test.t values(-1, 1.9); +mysql> insert into test.t values(-9999999999999999, -99999999999999999999); +mysql> insert into test.t values(9999999999999999, 99999999999999999999); +mysql> alter table test.t set tiflash replica 1; +func> wait_table test t + +mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select a, oct(a), b, oct(b) from test.t; ++-------------------+------------------------+----------+------------------------+ +| a | oct(a) | b | oct(b) | ++-------------------+------------------------+----------+------------------------+ +| 1 | 1 | 1.9 | 1 | +| -1 | 1777777777777777777777 | 1.9 | 1 | +| -9999999999999999 | 1777343620662017600001 | -1e+20 | 1777777777777777777777 | +| 9999999999999999 | 434157115760177777 | 1e+20 | 1777777777777777777777 | ++-------------------+------------------------+----------+------------------------+ diff --git a/tests/fullstack-test/expr/oct_string.test b/tests/fullstack-test/expr/oct_string.test new file mode 100644 index 00000000000..35cf7215019 --- /dev/null +++ b/tests/fullstack-test/expr/oct_string.test @@ -0,0 +1,53 @@ +# Copyright 2022 PingCAP, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mysql> drop table if exists test.t; +mysql> create table if not exists test.t(a varchar(255)); + +mysql> insert into test.t values("0"); +mysql> insert into test.t values("1"); +mysql> insert into test.t values("4"); +mysql> insert into test.t values("8"); +mysql> insert into test.t values("16pingcap"); +mysql> insert into test.t values(" 16pingcap"); +mysql> insert into test.t values("pingcap16"); +mysql> insert into test.t values("ping16cap"); +mysql> insert into test.t values("ping 16cap"); +mysql> insert into test.t values("1.9"); +mysql> insert into test.t values("-1"); +mysql> insert into test.t values("-1.9"); +mysql> insert into test.t values("-9999999999999999999999999"); +mysql> insert into test.t values("9999999999999999999999999"); +mysql> alter table test.t set tiflash replica 1; +func> wait_table test t + +mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select a, oct(a) from test.t; ++----------------------------+------------------------+ +| a | oct(a) | ++----------------------------+------------------------+ +| 0 | 0 | +| 1 | 1 | +| 4 | 4 | +| 8 | 10 | +| 16pingcap | 20 | +| 16pingcap | 20 | +| pingcap16 | 0 | +| ping16cap | 0 | +| ping 16cap | 0 | +| 1.9 | 1 | +| -1 | 1777777777777777777777 | +| -1.9 | 1777777777777777777777 | +| -9999999999999999999999999 | 1777777777777777777777 | +| 9999999999999999999999999 | 1777777777777777777777 | ++----------------------------+------------------------+