Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: format throw data truncated error #5272

Merged
merged 21 commits into from
Jul 11, 2022
Merged

fix: format throw data truncated error #5272

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
c99bc07
fix
xzhangxian1008 Jul 1, 2022
d21ac2f
update
xzhangxian1008 Jul 1, 2022
be0699b
Merge branch 'master' into issue_4891
xzhangxian1008 Jul 4, 2022
ba3b1c1
Merge branch 'master' into issue_4891
xzhangxian1008 Jul 4, 2022
2e49d56
Merge branch 'master' into issue_4891
xzhangxian1008 Jul 5, 2022
726e12f
update
xzhangxian1008 Jul 5, 2022
40f1a4d
format
xzhangxian1008 Jul 5, 2022
8d6c677
Merge branch 'master' into issue_4891
xzhangxian1008 Jul 5, 2022
339014a
update
xzhangxian1008 Jul 5, 2022
e931521
format
xzhangxian1008 Jul 5, 2022
3f5e6df
Merge branch 'master' of https://github.com/pingcap/tiflash into issu…
xzhangxian1008 Jul 7, 2022
58e7a98
Merge branch 'master' into issue_4891
xzhangxian1008 Jul 8, 2022
01dd580
enable ut
xzhangxian1008 Jul 8, 2022
107e110
Merge branch 'master' into issue_4891
xzhangxian1008 Jul 8, 2022
ca01708
Merge branch 'master' into issue_4891
xzhangxian1008 Jul 9, 2022
dba518e
update
xzhangxian1008 Jul 9, 2022
50f4598
update
xzhangxian1008 Jul 9, 2022
d6e8f15
udpate
xzhangxian1008 Jul 11, 2022
ee26f15
Merge branch 'master' into issue_4891
xzhangxian1008 Jul 11, 2022
aeab46a
add fullstack test
xzhangxian1008 Jul 11, 2022
ceac0fd
Merge branch 'master' into issue_4891
xzhangxian1008 Jul 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion dbms/src/Functions/FunctionsString.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4573,7 +4573,9 @@ class FormatImpl : public IFunction
using NumberFieldType = typename NumberType::FieldType;
using NumberColVec = std::conditional_t<IsDecimal<NumberFieldType>, ColumnDecimal<NumberFieldType>, ColumnVector<NumberFieldType>>;
const auto * number_raw = block.getByPosition(arguments[0]).column.get();

TiDBDecimalRoundInfo info{number_type, number_type};
info.output_prec = info.output_prec < 65 ? info.output_prec + 1 : 65;

return getPrecisionType(precision_base_type, [&](const auto & precision_type, bool) {
using PrecisionType = std::decay_t<decltype(precision_type)>;
Expand Down Expand Up @@ -4723,10 +4725,11 @@ class FormatImpl : public IFunction
static void format(
T number,
size_t max_num_decimals,
const TiDBDecimalRoundInfo & info,
TiDBDecimalRoundInfo & info,
ColumnString::Chars_t & res_data,
ColumnString::Offsets & res_offsets)
{
info.output_scale = std::min(max_num_decimals, static_cast<size_t>(info.input_scale));
auto round_number = round(number, max_num_decimals, info);
std::string round_number_str = number2Str(round_number, info);
std::string buffer = Format::apply(round_number_str, max_num_decimals);
Expand Down
19 changes: 8 additions & 11 deletions dbms/src/Functions/tests/gtest_strings_format.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ class StringFormat : public DB::tests::FunctionTest
using FieldType = DecimalField<Decimal>;
using NullableDecimal = Nullable<Decimal>;
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"0.0000", "-0.0120", "0.0120", "12,332.1000", "12,332", "12,332", "12,332.300000000000000000000000000000", "-12,332.30000", "-1,000.0", "-333.33", {}}),
createColumn<Nullable<String>>({"0.0000", "-0.0120", "0.0120", "12,332.1000", "12,332", "12,332", "12,332.300000000000000000000000000000", "-12,332.30000", "-1,000.0", "-333.33", {}, "99,999.9999000000", "100,000.000", "100,000"}),
executeFunction(
func_name,
createColumn<NullableDecimal>(
Expand All @@ -49,8 +49,11 @@ class StringFormat : public DB::tests::FunctionTest
FieldType(static_cast<Native>(-123323000), 4),
FieldType(static_cast<Native>(-9999999), 4),
FieldType(static_cast<Native>(-3333330), 4),
FieldType(static_cast<Native>(0), 0)}),
createColumn<Nullable<Int64>>({4, 4, 4, 4, 0, -1, 31, 5, 1, 2, {}})));
FieldType(static_cast<Native>(0), 0),
FieldType(static_cast<Native>(999999999), 4),
FieldType(static_cast<Native>(999999999), 4),
FieldType(static_cast<Native>(999999999), 4)}),
createColumn<Nullable<Int64>>({4, 4, 4, 4, 0, -1, 31, 5, 1, 2, {}, 10, 3, -5})));
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"12,332.100", "-12,332.300", "-1,000.000", "-333.333"}),
executeFunction(
Expand All @@ -62,8 +65,6 @@ class StringFormat : public DB::tests::FunctionTest
FieldType(static_cast<Native>(-9999999), 4),
FieldType(static_cast<Native>(-3333330), 4)}),
createConstColumn<Nullable<Int16>>(4, 3)));
/// known issue https://github.com/pingcap/tiflash/issues/4891
/*
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"-999.9999", "-1,000", "-1,000", "-999.999900000000000000000000000000", "-999.99990", "-1,000.0", "-1,000.00"}),
executeFunction(
Expand All @@ -74,15 +75,14 @@ class StringFormat : public DB::tests::FunctionTest
FieldType(static_cast<Native>(-9999999), 4)),
createColumn<Nullable<Int32>>({4, 0, -1, 31, 5, 1, 2})));
ASSERT_COLUMN_EQ(
createConstColumn<Nullable<String>>(1, "-1,000.000"),
createConstColumn<String>(1, "-1,000.000"),
executeFunction(
func_name,
createConstColumn<NullableDecimal>(
std::make_tuple(precision, 4),
1,
FieldType(static_cast<Native>(-9999999), 4)),
createConstColumn<Nullable<Int8>>(1, 3)));
*/
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"12,332.1000", "12,332", "12,332.300000000000000000000000000000", "-12,332.30000", "-1,000.0", "-333.33", {}}),
executeFunction(
Expand All @@ -108,8 +108,6 @@ class StringFormat : public DB::tests::FunctionTest
FieldType(static_cast<Native>(-9999999), 4),
FieldType(static_cast<Native>(-3333330), 4)}),
createConstColumn<Nullable<UInt16>>(4, 3)));
/// known issue https://github.com/pingcap/tiflash/issues/4891
/*
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"-999.9999", "-1,000", "-999.999900000000000000000000000000", "-999.99990", "-1,000.0", "-1,000.00"}),
executeFunction(
Expand All @@ -120,15 +118,14 @@ class StringFormat : public DB::tests::FunctionTest
FieldType(static_cast<Native>(-9999999), 4)),
createColumn<Nullable<UInt32>>({4, 0, 31, 5, 1, 2})));
ASSERT_COLUMN_EQ(
createConstColumn<Nullable<String>>(1, "-1,000.000"),
createConstColumn<String>(1, "-1,000.000"),
executeFunction(
func_name,
createConstColumn<NullableDecimal>(
std::make_tuple(precision, 4),
1,
FieldType(static_cast<Native>(-9999999), 4)),
createConstColumn<Nullable<UInt8>>(1, 3)));
*/
}

template <typename Integer>
Expand Down
49 changes: 49 additions & 0 deletions tests/fullstack-test/expr/format.test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,52 @@ int_val
1,234.000

mysql> drop table if exists test.t

mysql> create table test.t(id int, value decimal(65,4))
mysql> alter table test.t set tiflash replica 1
mysql> insert into test.t values(1,9999999999999999999999999999999999999999999999999999999999999.9999)

func> wait_table test t

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,-3) as result from test.t
result
10,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,0) as result from test.t
result
10,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,3) as result from test.t
result
10,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000.000

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,10) as result from test.t
result
9,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999.9999000000


mysql> drop table if exists test.t

mysql> create table test.t(id int, value decimal(7,4))
mysql> alter table test.t set tiflash replica 1
mysql> insert into test.t values(1,999.9999)

func> wait_table test t

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,-2) as result from test.t
result
1,000

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,0) as result from test.t
result
1,000

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,2) as result from test.t
result
1,000.00

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,10) as result from test.t
result
999.9999000000

mysql> drop table if exists test.t