From bec495665b6040cfb8d98f20fb426d9f8c66be6f Mon Sep 17 00:00:00 2001 From: xzhangxian1008 Date: Wed, 22 Jun 2022 14:03:39 +0800 Subject: [PATCH 01/11] Add gtest for Limit, TopN, Projection --- dbms/src/Flash/tests/gtest_executor.cpp | 144 +++++++++++++++++++++++ dbms/src/TestUtils/FunctionTestUtils.cpp | 4 +- 2 files changed, 147 insertions(+), 1 deletion(-) diff --git a/dbms/src/Flash/tests/gtest_executor.cpp b/dbms/src/Flash/tests/gtest_executor.cpp index 64c60f14bb6..82ef2e52140 100644 --- a/dbms/src/Flash/tests/gtest_executor.cpp +++ b/dbms/src/Flash/tests/gtest_executor.cpp @@ -85,6 +85,150 @@ try } CATCH +TEST_F(ExecutorTestRunner, Limit) +try +{ + /// Prepare column data + using ColDataType = std::optional::FieldType>; + using ColumnWithData = std::vector; + const ColumnWithData col0{"col0-0", {}, "col0-2", "col0-3", {}, "col0-5", "col0-6", "col0-7"}; + const size_t col_data_num = col0.size(); + + /// Prepare some names + const String db_name("test_db"); + const String table_name("projection_test_table"); + const String col_name("limit_col"); + + /// Create table for the test + context.addMockTable({db_name, table_name}, + {{col_name, TiDB::TP::TypeString}}, + {toNullableVec(col_name, col0)}); + + size_t concurrency = 1; + std::shared_ptr request; + ColumnsWithTypeAndName expect_cols; + + /// Check limit result with various parameters + for (size_t limit_num = 0; limit_num <= col_data_num + 3; limit_num++) + { + request = context + .scan(db_name, table_name) + .limit(limit_num) + .build(context); + + if (limit_num == 0) + expect_cols = {}; + else if (limit_num > col_data_num) + expect_cols = {toNullableVec(col_name, ColumnWithData(col0.begin(), col0.end()))}; + else + expect_cols = {toNullableVec(col_name, ColumnWithData(col0.begin(), col0.begin() + limit_num))}; + + executeStreams(request, expect_cols, concurrency); + } +} +CATCH + +TEST_F(ExecutorTestRunner, TopN) +try +{ + /// Prepare column data + using ColDataType = std::optional::FieldType>; + using ColumnWithData = std::vector; + ColumnWithData col0{"col0-0", "col0-1", "col0-2", {}, "col0-4", {}, "col0-6", "col0-7"}; + size_t col_data_num = col0.size(); + + /// Prepare some names + const String db_name("test_db"); + const String table_name("topn_test_table"); + const String col_name("topn_col"); + + /// Create table for the test + context.addMockTable({db_name, table_name}, + {{col_name, TiDB::TP::TypeString}}, + {toNullableVec(col_name, col0)}); + + size_t concurrency = 1; + std::shared_ptr request; + ColumnsWithTypeAndName expect_cols; + + bool is_desc; + + /// Check topn result with various parameters + for (size_t i = 1; i <= 1; i++) + { + is_desc = static_cast(i); /// Set descent or ascent + if (is_desc) + sort(col0.begin(), col0.end(), std::greater()); /// Sort col0 for the following comparison + else + sort(col0.begin(), col0.end()); + + for (size_t limit_num = 0; limit_num <= col_data_num + 3; limit_num++) + { + request = context + .scan(db_name, table_name) + .topN(col_name, is_desc, limit_num) + .build(context); + + if (limit_num == 0 || limit_num > col_data_num) + expect_cols = {toNullableVec(col_name, ColumnWithData(col0.begin(), col0.end()))}; + else + expect_cols = {toNullableVec(col_name, ColumnWithData(col0.begin(), col0.begin() + limit_num))}; + + executeStreams(request, expect_cols, concurrency); + } + } +} +CATCH + +TEST_F(ExecutorTestRunner, Projection) +try +{ + /// Prepare column data + using ColDataType = std::vector::FieldType>>; + const ColDataType col0{"col0-0", "col0-1", "col0-2", {}, "col0-4"}; + const ColDataType col1{"col1-0", {}, "col1-2", "col1-3", "col1-4"}; + const ColDataType col2{"col2-0", "col2-1", {}, "col2-3", "col2-4"}; + + /// Prepare some names + std::vector col_names{"proj_col0", "proj_col1", "proj_col2"}; + const String db_name("test_db"); + const String table_name("projection_test_table"); + + /// Create table for the test + context.addMockTable({db_name, table_name}, + {{col_names[0], TiDB::TP::TypeString}, + {col_names[1], TiDB::TP::TypeString}, + {col_names[2], TiDB::TP::TypeString}}, + {toNullableVec(col_names[0], col0), + toNullableVec(col_names[1], col1), + toNullableVec(col_names[2], col2)}); + + const size_t concurrency = 1; + auto get_request = [&](MockColumnNames col_names) + { + return context.scan(db_name, table_name).project(col_names).build(context); + }; + + /// Start to test projection + + auto request = get_request({col_names[2]}); + executeStreams(request, {toNullableVec(col_names[2], col2)}, concurrency); + + request = get_request({col_names[0], col_names[1]}); + executeStreams(request, + {toNullableVec(col_names[0], col0), + toNullableVec(col_names[1], col1),}, + concurrency); + + request = get_request({col_names[0], col_names[1], col_names[2]}); + executeStreams(request, + {toNullableVec(col_names[0], col0), + toNullableVec(col_names[1], col1), + toNullableVec(col_names[2], col2)}, + concurrency); +} +CATCH + TEST_F(ExecutorTestRunner, JoinWithTableScan) try { diff --git a/dbms/src/TestUtils/FunctionTestUtils.cpp b/dbms/src/TestUtils/FunctionTestUtils.cpp index 637fbf51c00..15da9a58ce9 100644 --- a/dbms/src/TestUtils/FunctionTestUtils.cpp +++ b/dbms/src/TestUtils/FunctionTestUtils.cpp @@ -108,8 +108,10 @@ void blockEqual( const Block & actual) { size_t columns = actual.columns(); + size_t expected_columns = expected.columns(); - ASSERT_TRUE(expected.columns() == columns); + ASSERT_TRUE(expected_columns == columns) + << std::string("expected: " + std::to_string(expected_columns) + ", actual:" + std::to_string(columns)); for (size_t i = 0; i < columns; ++i) { From 4e6769fedd1d8cbc444968337962c86df827b080 Mon Sep 17 00:00:00 2001 From: xzhangxian1008 Date: Wed, 22 Jun 2022 21:09:08 +0800 Subject: [PATCH 02/11] update --- dbms/src/Flash/tests/gtest_executor.cpp | 146 +----------------- dbms/src/Flash/tests/gtest_limit_executor.cpp | 75 +++++++++ .../Flash/tests/gtest_projection_executor.cpp | 77 +++++++++ dbms/src/Flash/tests/gtest_topn_executor.cpp | 84 ++++++++++ 4 files changed, 237 insertions(+), 145 deletions(-) create mode 100644 dbms/src/Flash/tests/gtest_limit_executor.cpp create mode 100644 dbms/src/Flash/tests/gtest_projection_executor.cpp create mode 100644 dbms/src/Flash/tests/gtest_topn_executor.cpp diff --git a/dbms/src/Flash/tests/gtest_executor.cpp b/dbms/src/Flash/tests/gtest_executor.cpp index 82ef2e52140..b4ba1a75563 100644 --- a/dbms/src/Flash/tests/gtest_executor.cpp +++ b/dbms/src/Flash/tests/gtest_executor.cpp @@ -85,150 +85,6 @@ try } CATCH -TEST_F(ExecutorTestRunner, Limit) -try -{ - /// Prepare column data - using ColDataType = std::optional::FieldType>; - using ColumnWithData = std::vector; - const ColumnWithData col0{"col0-0", {}, "col0-2", "col0-3", {}, "col0-5", "col0-6", "col0-7"}; - const size_t col_data_num = col0.size(); - - /// Prepare some names - const String db_name("test_db"); - const String table_name("projection_test_table"); - const String col_name("limit_col"); - - /// Create table for the test - context.addMockTable({db_name, table_name}, - {{col_name, TiDB::TP::TypeString}}, - {toNullableVec(col_name, col0)}); - - size_t concurrency = 1; - std::shared_ptr request; - ColumnsWithTypeAndName expect_cols; - - /// Check limit result with various parameters - for (size_t limit_num = 0; limit_num <= col_data_num + 3; limit_num++) - { - request = context - .scan(db_name, table_name) - .limit(limit_num) - .build(context); - - if (limit_num == 0) - expect_cols = {}; - else if (limit_num > col_data_num) - expect_cols = {toNullableVec(col_name, ColumnWithData(col0.begin(), col0.end()))}; - else - expect_cols = {toNullableVec(col_name, ColumnWithData(col0.begin(), col0.begin() + limit_num))}; - - executeStreams(request, expect_cols, concurrency); - } -} -CATCH - -TEST_F(ExecutorTestRunner, TopN) -try -{ - /// Prepare column data - using ColDataType = std::optional::FieldType>; - using ColumnWithData = std::vector; - ColumnWithData col0{"col0-0", "col0-1", "col0-2", {}, "col0-4", {}, "col0-6", "col0-7"}; - size_t col_data_num = col0.size(); - - /// Prepare some names - const String db_name("test_db"); - const String table_name("topn_test_table"); - const String col_name("topn_col"); - - /// Create table for the test - context.addMockTable({db_name, table_name}, - {{col_name, TiDB::TP::TypeString}}, - {toNullableVec(col_name, col0)}); - - size_t concurrency = 1; - std::shared_ptr request; - ColumnsWithTypeAndName expect_cols; - - bool is_desc; - - /// Check topn result with various parameters - for (size_t i = 1; i <= 1; i++) - { - is_desc = static_cast(i); /// Set descent or ascent - if (is_desc) - sort(col0.begin(), col0.end(), std::greater()); /// Sort col0 for the following comparison - else - sort(col0.begin(), col0.end()); - - for (size_t limit_num = 0; limit_num <= col_data_num + 3; limit_num++) - { - request = context - .scan(db_name, table_name) - .topN(col_name, is_desc, limit_num) - .build(context); - - if (limit_num == 0 || limit_num > col_data_num) - expect_cols = {toNullableVec(col_name, ColumnWithData(col0.begin(), col0.end()))}; - else - expect_cols = {toNullableVec(col_name, ColumnWithData(col0.begin(), col0.begin() + limit_num))}; - - executeStreams(request, expect_cols, concurrency); - } - } -} -CATCH - -TEST_F(ExecutorTestRunner, Projection) -try -{ - /// Prepare column data - using ColDataType = std::vector::FieldType>>; - const ColDataType col0{"col0-0", "col0-1", "col0-2", {}, "col0-4"}; - const ColDataType col1{"col1-0", {}, "col1-2", "col1-3", "col1-4"}; - const ColDataType col2{"col2-0", "col2-1", {}, "col2-3", "col2-4"}; - - /// Prepare some names - std::vector col_names{"proj_col0", "proj_col1", "proj_col2"}; - const String db_name("test_db"); - const String table_name("projection_test_table"); - - /// Create table for the test - context.addMockTable({db_name, table_name}, - {{col_names[0], TiDB::TP::TypeString}, - {col_names[1], TiDB::TP::TypeString}, - {col_names[2], TiDB::TP::TypeString}}, - {toNullableVec(col_names[0], col0), - toNullableVec(col_names[1], col1), - toNullableVec(col_names[2], col2)}); - - const size_t concurrency = 1; - auto get_request = [&](MockColumnNames col_names) - { - return context.scan(db_name, table_name).project(col_names).build(context); - }; - - /// Start to test projection - - auto request = get_request({col_names[2]}); - executeStreams(request, {toNullableVec(col_names[2], col2)}, concurrency); - - request = get_request({col_names[0], col_names[1]}); - executeStreams(request, - {toNullableVec(col_names[0], col0), - toNullableVec(col_names[1], col1),}, - concurrency); - - request = get_request({col_names[0], col_names[1], col_names[2]}); - executeStreams(request, - {toNullableVec(col_names[0], col0), - toNullableVec(col_names[1], col1), - toNullableVec(col_names[2], col2)}, - concurrency); -} -CATCH - TEST_F(ExecutorTestRunner, JoinWithTableScan) try { @@ -371,4 +227,4 @@ try CATCH } // namespace tests -} // namespace DB \ No newline at end of file +} // namespace DB diff --git a/dbms/src/Flash/tests/gtest_limit_executor.cpp b/dbms/src/Flash/tests/gtest_limit_executor.cpp new file mode 100644 index 00000000000..2a77d37ebad --- /dev/null +++ b/dbms/src/Flash/tests/gtest_limit_executor.cpp @@ -0,0 +1,75 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +namespace DB +{ +namespace tests +{ + +class ExecutorLimitTestRunner : public DB::tests::ExecutorTest +{ +public: + using ColDataType = std::optional::FieldType>; + using ColumnWithData = std::vector; + + void initializeContext() override + { + ExecutorTest::initializeContext(); + + context.addMockTable({db_name, table_name}, + {{col_name, TiDB::TP::TypeString}}, + {toNullableVec(col_name, col0)}); + } + + std::shared_ptr getRequest(size_t limit_num) + { + return context.scan(db_name, table_name).limit(limit_num).build(context); + } + + /// Prepare some names + const String db_name{"test_db"}; + const String table_name{"projection_test_table"}; + const String col_name{"limit_col"}; + const ColumnWithData col0{"col0-0", {}, "col0-2", "col0-3", {}, "col0-5", "col0-6", "col0-7"}; +}; + +TEST_F(ExecutorLimitTestRunner, Limit) +try +{ + std::shared_ptr request; + ColumnsWithTypeAndName expect_cols; + + /// Check limit result with various parameters + const size_t col_data_num = col0.size(); + for (size_t limit_num = 0; limit_num <= col_data_num + 3; ++limit_num) + { + request = getRequest(limit_num); + + if (limit_num == 0) + expect_cols = {}; + else if (limit_num > col_data_num) + expect_cols = {toNullableVec(col_name, ColumnWithData(col0.begin(), col0.end()))}; + else + expect_cols = {toNullableVec(col_name, ColumnWithData(col0.begin(), col0.begin() + limit_num))}; + + executeStreams(request, expect_cols); + } +} +CATCH + +} // namespace tests +} // namespace DB diff --git a/dbms/src/Flash/tests/gtest_projection_executor.cpp b/dbms/src/Flash/tests/gtest_projection_executor.cpp new file mode 100644 index 00000000000..426c4370f47 --- /dev/null +++ b/dbms/src/Flash/tests/gtest_projection_executor.cpp @@ -0,0 +1,77 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +namespace DB +{ +namespace tests +{ + +class ExecutorProjectionTestRunner : public DB::tests::ExecutorTest +{ +public: + using ColDataType = std::vector::FieldType>>; + + void initializeContext() override + { + ExecutorTest::initializeContext(); + + context.addMockTable({db_name, table_name}, + {{col_names[0], TiDB::TP::TypeString}, + {col_names[1], TiDB::TP::TypeString}, + {col_names[2], TiDB::TP::TypeString}}, + {toNullableVec(col_names[0], col0), + toNullableVec(col_names[1], col1), + toNullableVec(col_names[2], col2)}); + } + + std::shared_ptr getRequest(MockColumnNames col_names) + { + return context.scan(db_name, table_name).project(col_names).build(context); + }; + + /// Prepare column data + const ColDataType col0{"col0-0", "col0-1", "col0-2", {}, "col0-4"}; + const ColDataType col1{"col1-0", {}, "col1-2", "col1-3", "col1-4"}; + const ColDataType col2{"col2-0", "col2-1", {}, "col2-3", "col2-4"}; + + /// Prepare some names + std::vector col_names{"proj_col0", "proj_col1", "proj_col2"}; + const String db_name{"test_db"}; + const String table_name{"projection_test_table"}; +}; + +TEST_F(ExecutorProjectionTestRunner, Projection) +try +{ + auto request = getRequest({col_names[2]}); + executeStreams(request, {toNullableVec(col_names[2], col2)}); + + request = getRequest({col_names[0], col_names[1]}); + executeStreams(request, + {toNullableVec(col_names[0], col0), + toNullableVec(col_names[1], col1),}); + + request = getRequest({col_names[0], col_names[1], col_names[2]}); + executeStreams(request, + {toNullableVec(col_names[0], col0), + toNullableVec(col_names[1], col1), + toNullableVec(col_names[2], col2)}); +} +CATCH + +} // namespace tests +} // namespace DB diff --git a/dbms/src/Flash/tests/gtest_topn_executor.cpp b/dbms/src/Flash/tests/gtest_topn_executor.cpp new file mode 100644 index 00000000000..170216de594 --- /dev/null +++ b/dbms/src/Flash/tests/gtest_topn_executor.cpp @@ -0,0 +1,84 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +namespace DB +{ +namespace tests +{ + +class ExecutorTopNTestRunner : public DB::tests::ExecutorTest +{ +public: + using ColDataType = std::optional::FieldType>; + using ColumnWithData = std::vector; + + void initializeContext() override + { + ExecutorTest::initializeContext(); + + context.addMockTable({db_name, table_name}, + {{col_name, TiDB::TP::TypeString}}, + {toNullableVec(col_name, col0)}); + } + + std::shared_ptr getRequest(const String & col_name, bool is_desc, int limit_num) + { + return context.scan(db_name, table_name).topN(col_name, is_desc, limit_num).build(context); + } + + /// Prepare some names + const String db_name{"test_db"}; + const String table_name{"topn_test_table"}; + const String col_name{"topn_col"}; + ColumnWithData col0{"col0-0", "col0-1", "col0-2", {}, "col0-4", {}, "col0-6", "col0-7"}; +}; + +TEST_F(ExecutorTopNTestRunner, TopN) +try +{ + size_t col_data_num = col0.size(); + std::shared_ptr request; + ColumnsWithTypeAndName expect_cols; + bool is_desc; + + /// Check topn result with various parameters + for (size_t i = 1; i <= 1; ++i) + { + is_desc = static_cast(i); /// Set descent or ascent + if (is_desc) + sort(col0.begin(), col0.end(), std::greater()); /// Sort col0 for the following comparison + else + sort(col0.begin(), col0.end()); + + for (size_t limit_num = 0; limit_num <= col_data_num + 3; limit_num++) + { + request = getRequest(col_name, is_desc, limit_num); + + if (limit_num == 0 || limit_num > col_data_num) + expect_cols = {toNullableVec(col_name, ColumnWithData(col0.begin(), col0.end()))}; + else + expect_cols = {toNullableVec(col_name, ColumnWithData(col0.begin(), col0.begin() + limit_num))}; + + executeStreams(request, expect_cols); + } + } +} +CATCH + +} // namespace tests +} // namespace DB + From a913cf54f137b27e337895f72a21216b3f71eaf4 Mon Sep 17 00:00:00 2001 From: xzhangxian1008 Date: Wed, 22 Jun 2022 21:17:48 +0800 Subject: [PATCH 03/11] update --- dbms/src/TestUtils/FunctionTestUtils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/TestUtils/FunctionTestUtils.cpp b/dbms/src/TestUtils/FunctionTestUtils.cpp index 15da9a58ce9..d71f4f24f96 100644 --- a/dbms/src/TestUtils/FunctionTestUtils.cpp +++ b/dbms/src/TestUtils/FunctionTestUtils.cpp @@ -111,7 +111,7 @@ void blockEqual( size_t expected_columns = expected.columns(); ASSERT_TRUE(expected_columns == columns) - << std::string("expected: " + std::to_string(expected_columns) + ", actual:" + std::to_string(columns)); + << fmt::format("Expected: {}, Actual: {}", expected_columns, columns); for (size_t i = 0; i < columns; ++i) { From 3f201c149e362003d280489f6c7d55e4fc6beb57 Mon Sep 17 00:00:00 2001 From: xzhangxian1008 Date: Thu, 23 Jun 2022 09:30:37 +0800 Subject: [PATCH 04/11] update --- dbms/src/Flash/tests/gtest_limit_executor.cpp | 6 +++--- .../Flash/tests/gtest_projection_executor.cpp | 20 ++++++++++--------- dbms/src/Flash/tests/gtest_topn_executor.cpp | 9 ++++----- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/dbms/src/Flash/tests/gtest_limit_executor.cpp b/dbms/src/Flash/tests/gtest_limit_executor.cpp index 2a77d37ebad..814f5b09621 100644 --- a/dbms/src/Flash/tests/gtest_limit_executor.cpp +++ b/dbms/src/Flash/tests/gtest_limit_executor.cpp @@ -31,8 +31,8 @@ class ExecutorLimitTestRunner : public DB::tests::ExecutorTest ExecutorTest::initializeContext(); context.addMockTable({db_name, table_name}, - {{col_name, TiDB::TP::TypeString}}, - {toNullableVec(col_name, col0)}); + {{col_name, TiDB::TP::TypeString}}, + {toNullableVec(col_name, col0)}); } std::shared_ptr getRequest(size_t limit_num) @@ -65,7 +65,7 @@ try expect_cols = {toNullableVec(col_name, ColumnWithData(col0.begin(), col0.end()))}; else expect_cols = {toNullableVec(col_name, ColumnWithData(col0.begin(), col0.begin() + limit_num))}; - + executeStreams(request, expect_cols); } } diff --git a/dbms/src/Flash/tests/gtest_projection_executor.cpp b/dbms/src/Flash/tests/gtest_projection_executor.cpp index 426c4370f47..4c432a6a181 100644 --- a/dbms/src/Flash/tests/gtest_projection_executor.cpp +++ b/dbms/src/Flash/tests/gtest_projection_executor.cpp @@ -30,12 +30,12 @@ class ExecutorProjectionTestRunner : public DB::tests::ExecutorTest ExecutorTest::initializeContext(); context.addMockTable({db_name, table_name}, - {{col_names[0], TiDB::TP::TypeString}, - {col_names[1], TiDB::TP::TypeString}, - {col_names[2], TiDB::TP::TypeString}}, - {toNullableVec(col_names[0], col0), - toNullableVec(col_names[1], col1), - toNullableVec(col_names[2], col2)}); + {{col_names[0], TiDB::TP::TypeString}, + {col_names[1], TiDB::TP::TypeString}, + {col_names[2], TiDB::TP::TypeString}}, + {toNullableVec(col_names[0], col0), + toNullableVec(col_names[1], col1), + toNullableVec(col_names[2], col2)}); } std::shared_ptr getRequest(MockColumnNames col_names) @@ -62,9 +62,11 @@ try request = getRequest({col_names[0], col_names[1]}); executeStreams(request, - {toNullableVec(col_names[0], col0), - toNullableVec(col_names[1], col1),}); - + { + toNullableVec(col_names[0], col0), + toNullableVec(col_names[1], col1), + }); + request = getRequest({col_names[0], col_names[1], col_names[2]}); executeStreams(request, {toNullableVec(col_names[0], col0), diff --git a/dbms/src/Flash/tests/gtest_topn_executor.cpp b/dbms/src/Flash/tests/gtest_topn_executor.cpp index 170216de594..0c8483b14d5 100644 --- a/dbms/src/Flash/tests/gtest_topn_executor.cpp +++ b/dbms/src/Flash/tests/gtest_topn_executor.cpp @@ -31,8 +31,8 @@ class ExecutorTopNTestRunner : public DB::tests::ExecutorTest ExecutorTest::initializeContext(); context.addMockTable({db_name, table_name}, - {{col_name, TiDB::TP::TypeString}}, - {toNullableVec(col_name, col0)}); + {{col_name, TiDB::TP::TypeString}}, + {toNullableVec(col_name, col0)}); } std::shared_ptr getRequest(const String & col_name, bool is_desc, int limit_num) @@ -64,10 +64,10 @@ try else sort(col0.begin(), col0.end()); - for (size_t limit_num = 0; limit_num <= col_data_num + 3; limit_num++) + for (size_t limit_num = 0; limit_num <= col_data_num + 3; limit_num++) { request = getRequest(col_name, is_desc, limit_num); - + if (limit_num == 0 || limit_num > col_data_num) expect_cols = {toNullableVec(col_name, ColumnWithData(col0.begin(), col0.end()))}; else @@ -81,4 +81,3 @@ CATCH } // namespace tests } // namespace DB - From 8a73ed12527ffbac7513a14d801c4b10dff64952 Mon Sep 17 00:00:00 2001 From: xzhangxian1008 Date: Sun, 26 Jun 2022 22:41:06 +0800 Subject: [PATCH 05/11] add more tests --- dbms/src/Flash/tests/gtest_limit_executor.cpp | 2 + .../Flash/tests/gtest_projection_executor.cpp | 183 ++++++++++++++++-- dbms/src/Flash/tests/gtest_topn_executor.cpp | 143 +++++++++++--- dbms/src/TestUtils/FunctionTestUtils.cpp | 5 +- dbms/src/TestUtils/mockExecutor.cpp | 5 + dbms/src/TestUtils/mockExecutor.h | 6 +- 6 files changed, 296 insertions(+), 48 deletions(-) diff --git a/dbms/src/Flash/tests/gtest_limit_executor.cpp b/dbms/src/Flash/tests/gtest_limit_executor.cpp index 814f5b09621..63fc0ce2833 100644 --- a/dbms/src/Flash/tests/gtest_limit_executor.cpp +++ b/dbms/src/Flash/tests/gtest_limit_executor.cpp @@ -57,6 +57,8 @@ try const size_t col_data_num = col0.size(); for (size_t limit_num = 0; limit_num <= col_data_num + 3; ++limit_num) { + if (limit_num == col_data_num + 3) + limit_num = INT_MAX; request = getRequest(limit_num); if (limit_num == 0) diff --git a/dbms/src/Flash/tests/gtest_projection_executor.cpp b/dbms/src/Flash/tests/gtest_projection_executor.cpp index 4c432a6a181..d32a674fb37 100644 --- a/dbms/src/Flash/tests/gtest_projection_executor.cpp +++ b/dbms/src/Flash/tests/gtest_projection_executor.cpp @@ -23,7 +23,8 @@ namespace tests class ExecutorProjectionTestRunner : public DB::tests::ExecutorTest { public: - using ColDataType = std::vector::FieldType>>; + using ColDataString = std::vector::FieldType>>; + using ColDataInt32 = std::vector::FieldType>>; void initializeContext() override { @@ -32,24 +33,38 @@ class ExecutorProjectionTestRunner : public DB::tests::ExecutorTest context.addMockTable({db_name, table_name}, {{col_names[0], TiDB::TP::TypeString}, {col_names[1], TiDB::TP::TypeString}, - {col_names[2], TiDB::TP::TypeString}}, + {col_names[2], TiDB::TP::TypeString}, + {col_names[3], TiDB::TP::TypeLong}, + {col_names[4], TiDB::TP::TypeLong}}, {toNullableVec(col_names[0], col0), toNullableVec(col_names[1], col1), - toNullableVec(col_names[2], col2)}); + toNullableVec(col_names[2], col2), + toNullableVec(col_names[3], col3), + toNullableVec(col_names[4], col4)}); } - std::shared_ptr getRequest(MockColumnNames col_names) + template + std::shared_ptr getRequest(T param) { - return context.scan(db_name, table_name).project(col_names).build(context); + return context.scan(db_name, table_name).project(param).build(context); }; + void executeDifferentConcurrency(const std::shared_ptr & request, const ColumnsWithTypeAndName & expect_columns) + { + executeStreams(request, expect_columns, 1); + // for (size_t i = 1; i < 10; i += 2) + // executeStreams(request, expect_columns, i); + } + /// Prepare column data - const ColDataType col0{"col0-0", "col0-1", "col0-2", {}, "col0-4"}; - const ColDataType col1{"col1-0", {}, "col1-2", "col1-3", "col1-4"}; - const ColDataType col2{"col2-0", "col2-1", {}, "col2-3", "col2-4"}; + const ColDataString col0{"col0-0", "col0-1", "", "col0-2", {}, "col0-3", ""}; + const ColDataString col1{"col1-0", {}, "", "col1-1", "", "col1-2", "col1-3"}; + const ColDataString col2{ "", "col2-0", "col2-1", {}, "col2-3", {}, "col2-4"}; + const ColDataInt32 col3{1, {}, 0, -111111, {}, 0, 9999}; + const ColDataInt32 col4{0, 5, {}, -234, {}, 24353, 9999}; /// Prepare some names - std::vector col_names{"proj_col0", "proj_col1", "proj_col2"}; + std::vector col_names{"col0", "col1", "col2", "col3", "col4"}; const String db_name{"test_db"}; const String table_name{"projection_test_table"}; }; @@ -57,21 +72,153 @@ class ExecutorProjectionTestRunner : public DB::tests::ExecutorTest TEST_F(ExecutorProjectionTestRunner, Projection) try { - auto request = getRequest({col_names[2]}); - executeStreams(request, {toNullableVec(col_names[2], col2)}); + /// Check single column + auto request = getRequest({col_names[2]}); + executeDifferentConcurrency(request, {toNullableVec(col_names[2], col2)}); - request = getRequest({col_names[0], col_names[1]}); + /// Check multi columns + request = getRequest({col_names[0], col_names[4]}); executeStreams(request, - { - toNullableVec(col_names[0], col0), - toNullableVec(col_names[1], col1), + {toNullableVec(col_names[0], col0), + toNullableVec(col_names[4], col1), }); - - request = getRequest({col_names[0], col_names[1], col_names[2]}); + /// Check multi columns + request = getRequest({col_names[0], col_names[1], col_names[3]}); executeStreams(request, {toNullableVec(col_names[0], col0), toNullableVec(col_names[1], col1), - toNullableVec(col_names[2], col2)}); + toNullableVec(col_names[3], col2)}); + + /// Check duplicate columns + request = getRequest({col_names[1], col_names[1], col_names[1]}); + executeStreams(request, + {toNullableVec(col_names[1], col1), + toNullableVec(col_names[1], col1), + toNullableVec(col_names[1], col1)}); + + { + /// Check large number of columns + const size_t col_num = 100; + MockColumnNamesVec projection_input; + ColumnsWithTypeAndName columns; + auto expect_column = toNullableVec(col_names[1], col1); + + for (size_t i = 0; i < col_num; ++i) + { + projection_input.push_back(col_names[1]); + columns.push_back(expect_column); + } + + request = getRequest(projection_input); + executeStreams(request, columns); + } +} +CATCH + +TEST_F(ExecutorProjectionTestRunner, ProjectionFunction) +try +{ + std::shared_ptr request; + { + /// Test "equal" function + { + /// Data type: TypeString + request = getRequest({eq(col(col_names[0]), col(col_names[0]))}); + executeStreams(request, + {toNullableVec({1, 1, 1, 1, {}, 1, 1})}); + + request = getRequest({eq(col(col_names[0]), col(col_names[1]))}); + executeStreams(request, + {toNullableVec({0, {}, 1, 0, {}, 0, 0})}); + } + + { + /// Data type: TypeLong + request = getRequest({eq(col(col_names[3]), col(col_names[4]))}); + executeStreams(request, + {toNullableVec({0, {}, {}, 0, {}, 0, 1})}); + } + } + + { + /// Test "greater" function + { + /// Data type: TypeString + request = getRequest({gt(col(col_names[0]), col(col_names[1]))}); + executeStreams(request, + {toNullableVec({0, {}, 0, 0, {}, 0, 0})}); + + request = getRequest({gt(col(col_names[1]), col(col_names[0]))}); + executeStreams(request, + {toNullableVec({1, {}, 0, 1, {}, 1, 1})}); + } + + { + /// Data type: TypeLong + request = getRequest({gt(col(col_names[3]), col(col_names[4]))}); + executeStreams(request, + {toNullableVec({1, {}, {}, 0, {}, 0, 0})}); + + request = getRequest({gt(col(col_names[4]), col(col_names[3]))}); + executeStreams(request, + {toNullableVec({0, {}, {}, 1, {}, 1, 0})}); + } + } + + { + /// Test "and" function + { + /// Data type: TypeString + request = getRequest({And(col(col_names[0]), col(col_names[0]))}); + executeStreams(request, + {toNullableVec({0, 0, 0, 0, {}, 0, 0})}); + + request = getRequest({And(col(col_names[0]), col(col_names[1]))}); + executeStreams(request, + {toNullableVec({0, 0, 0, 0, 0, 0, 0})}); + } + + { + /// Data type: TypeLong + request = getRequest({And(col(col_names[3]), col(col_names[4]))}); + executeStreams(request, + {toNullableVec({0, {}, 0, 1, {}, 0, 1})}); + } + } + + { + /// Test "not" function + { + /// Data type: TypeString + request = getRequest({NOT(col(col_names[0])), NOT(col(col_names[1])), NOT(col(col_names[2]))}); + executeStreams(request, + {toNullableVec({1, 1, 1, 1, {}, 1, 1}), + toNullableVec({1, {}, 1, 1, 1, 1, 1}), + toNullableVec({1, 1, 1, {}, 1, {}, 1})}); + } + + { + /// Data type: TypeLong + request = getRequest({NOT(col(col_names[3])), NOT(col(col_names[4]))}); + executeStreams(request, + {toNullableVec({0, {}, 1, 0, {}, 1, 0}), + toNullableVec({1, 0, {}, 0, {}, 0, 0})}); + } + } + + { + /// Test "max" function + /// TODO wait for the implementation of the max aggregation + { + /// Data type: TypeString + } + + { + /// Data type: TypeLong + } + } + + /// TODO more functions } CATCH diff --git a/dbms/src/Flash/tests/gtest_topn_executor.cpp b/dbms/src/Flash/tests/gtest_topn_executor.cpp index 0c8483b14d5..8901c663ed7 100644 --- a/dbms/src/Flash/tests/gtest_topn_executor.cpp +++ b/dbms/src/Flash/tests/gtest_topn_executor.cpp @@ -23,61 +23,154 @@ namespace tests class ExecutorTopNTestRunner : public DB::tests::ExecutorTest { public: - using ColDataType = std::optional::FieldType>; - using ColumnWithData = std::vector; + using ColStringType = std::optional::FieldType>; + using ColInt32Type = std::optional::FieldType>; + using ColumnWithString = std::vector; + using ColumnWithInt32 = std::vector; void initializeContext() override { ExecutorTest::initializeContext(); + context.addMockTable({db_name, table_single_name}, + {{single_col_name, TiDB::TP::TypeString}}, + {toNullableVec(single_col_name, col0)}); + context.addMockTable({db_name, table_name}, - {{col_name, TiDB::TP::TypeString}}, - {toNullableVec(col_name, col0)}); + {{col_name[0], TiDB::TP::TypeLong}, + {col_name[1], TiDB::TP::TypeString}, + {col_name[2], TiDB::TP::TypeString}, + {col_name[3], TiDB::TP::TypeLong}}, + {toNullableVec(col_name[0], col_age), + toNullableVec(col_name[1], col_gender), + toNullableVec(col_name[2], col_country), + toNullableVec(col_name[3], c0l_salary)}); + } - std::shared_ptr getRequest(const String & col_name, bool is_desc, int limit_num) + std::shared_ptr getRequest(const String & table_name, const String & col_name, bool is_desc, int limit_num) { return context.scan(db_name, table_name).topN(col_name, is_desc, limit_num).build(context); } + std::shared_ptr getRequest(const String & table_name, MockOrderByItems order_by_items, int limit) + { + return context.scan(db_name, table_name).topN(order_by_items, limit).build(context); + } + /// Prepare some names const String db_name{"test_db"}; - const String table_name{"topn_test_table"}; - const String col_name{"topn_col"}; - ColumnWithData col0{"col0-0", "col0-1", "col0-2", {}, "col0-4", {}, "col0-6", "col0-7"}; + + const String table_single_name{"topn_single_table"}; /// For single column test + const String single_col_name{"single_col"}; + ColumnWithString col0{"col0-0", "col0-1", "col0-2", {}, "col0-4", {}, "col0-6", "col0-7"}; + + const String table_name{"clerk"}; + const std::vector col_name{"age", "gender", "country", "salary"}; + ColumnWithInt32 col_age {{}, 27, 32, 36, {}, 34}; + ColumnWithString col_gender {"female", "female", "male", "female", "male", "male"}; + ColumnWithString col_country{"korea", "usa", "usa", "china", "china", "china"}; + ColumnWithInt32 c0l_salary {1300, 0, {}, 900, {}, -300}; }; TEST_F(ExecutorTopNTestRunner, TopN) try { - size_t col_data_num = col0.size(); std::shared_ptr request; - ColumnsWithTypeAndName expect_cols; - bool is_desc; + std::vector expect_cols; - /// Check topn result with various parameters - for (size_t i = 1; i <= 1; ++i) { - is_desc = static_cast(i); /// Set descent or ascent - if (is_desc) - sort(col0.begin(), col0.end(), std::greater()); /// Sort col0 for the following comparison - else - sort(col0.begin(), col0.end()); + /// Test single column + size_t col_data_num = col0.size(); + for (size_t i = 1; i <= 1; ++i) + { + bool is_desc; + is_desc = static_cast(i); /// Set descent or ascent + if (is_desc) + sort(col0.begin(), col0.end(), std::greater()); /// Sort col0 for the following comparison + else + sort(col0.begin(), col0.end()); + + for (size_t limit_num = 0; limit_num <= col_data_num + 5; ++limit_num) + { + request = getRequest(table_single_name, single_col_name, is_desc, limit_num); + + expect_cols.clear(); + if (limit_num == 0 || limit_num > col_data_num) + expect_cols.push_back({toNullableVec(single_col_name, ColumnWithString(col0.begin(), col0.end()))}); + else + expect_cols.push_back({toNullableVec(single_col_name, ColumnWithString(col0.begin(), col0.begin() + limit_num))}); + + executeStreams(request, expect_cols[0]); + executeStreams(request, expect_cols[0], 2); + executeStreams(request, expect_cols[0], 4); + executeStreams(request, expect_cols[0], 8); + } + } + } + + { + /// Test multi-columns + expect_cols = {{toNullableVec(col_name[0], ColumnWithInt32{36, 34, 32, 27, {}, {}}), + toNullableVec(col_name[1], ColumnWithString{"female", "male", "male", "female", "male", "female"}), + toNullableVec(col_name[2], ColumnWithString{"china", "china", "usa", "usa", "china", "korea"}), + toNullableVec(col_name[3], ColumnWithInt32{900, -300, {}, 0, {}, 1300})}, + {toNullableVec(col_name[0], ColumnWithInt32{32, {}, 34, 27, 36, {}}), + toNullableVec(col_name[1], ColumnWithString{"male", "male", "male", "female", "female", "female"}), + toNullableVec(col_name[2], ColumnWithString{"usa", "china", "china", "usa", "china", "korea"}), + toNullableVec(col_name[3], ColumnWithInt32{{}, {}, -300, 0, 900, 1300})}, + {toNullableVec(col_name[0], ColumnWithInt32{34, {}, 32, 36, {}, 27}), + toNullableVec(col_name[1], ColumnWithString{"male", "male", "male", "female", "female", "female"}), + toNullableVec(col_name[2], ColumnWithString{"china", "china", "usa", "china", "korea", "usa"}), + toNullableVec(col_name[3], ColumnWithInt32{-300, {}, {}, 900, 1300, 0})}}; + + MockOrderByItems order_by_items; - for (size_t limit_num = 0; limit_num <= col_data_num + 3; limit_num++) { - request = getRequest(col_name, is_desc, limit_num); + /// select * from clerk order by age DESC, gender DESC; + order_by_items = {MockOrderByItem(col_name[0], true), MockOrderByItem(col_name[1], true)}; + request = getRequest(table_name, order_by_items, 100); + executeStreams(request, expect_cols[0]); + } - if (limit_num == 0 || limit_num > col_data_num) - expect_cols = {toNullableVec(col_name, ColumnWithData(col0.begin(), col0.end()))}; - else - expect_cols = {toNullableVec(col_name, ColumnWithData(col0.begin(), col0.begin() + limit_num))}; + { + /// select * from clerk order by gender DESC, salary ASC; + order_by_items = {MockOrderByItem(col_name[1], true), MockOrderByItem(col_name[3], false)}; + request = getRequest(table_name, order_by_items, 100); + executeStreams(request, expect_cols[1]); + } - executeStreams(request, expect_cols); + { + /// select * from clerk order by gender DESC, country ASC, salary DESC; + order_by_items = {MockOrderByItem(col_name[1], true), MockOrderByItem(col_name[2], false), MockOrderByItem(col_name[3], true)}; + request = getRequest(table_name, order_by_items, 100); + executeStreams(request, expect_cols[2]); } } } CATCH +TEST_F(ExecutorTopNTestRunner, TopNFunction) +try +{ + std::shared_ptr request; + std::vector expect_cols; + + { + /// "and" function + } + + { + /// "equal" function + } + + { + /// "greater" function + } + + /// TODO more functions... +} +CATCH + } // namespace tests } // namespace DB diff --git a/dbms/src/TestUtils/FunctionTestUtils.cpp b/dbms/src/TestUtils/FunctionTestUtils.cpp index d71f4f24f96..7fb526aeb01 100644 --- a/dbms/src/TestUtils/FunctionTestUtils.cpp +++ b/dbms/src/TestUtils/FunctionTestUtils.cpp @@ -110,14 +110,13 @@ void blockEqual( size_t columns = actual.columns(); size_t expected_columns = expected.columns(); - ASSERT_TRUE(expected_columns == columns) - << fmt::format("Expected: {}, Actual: {}", expected_columns, columns); + ASSERT_EQ(expected_columns, columns); for (size_t i = 0; i < columns; ++i) { const auto & expected_col = expected.getByPosition(i); const auto & actual_col = actual.getByPosition(i); - ASSERT_TRUE(actual_col.type->getName() == expected_col.type->getName()); + ASSERT_EQ(actual_col.type->getName(), expected_col.type->getName()); ASSERT_COLUMN_EQ(expected_col.column, actual_col.column); } } diff --git a/dbms/src/TestUtils/mockExecutor.cpp b/dbms/src/TestUtils/mockExecutor.cpp index 2cf8a939b58..9a6e92dd9c1 100644 --- a/dbms/src/TestUtils/mockExecutor.cpp +++ b/dbms/src/TestUtils/mockExecutor.cpp @@ -219,6 +219,11 @@ DAGRequestBuilder & DAGRequestBuilder::project(MockAsts exprs) } DAGRequestBuilder & DAGRequestBuilder::project(MockColumnNames col_names) +{ + return project(MockColumnNamesVec(col_names)); +} + +DAGRequestBuilder & DAGRequestBuilder::project(MockColumnNamesVec col_names) { assert(root); auto exp_list = std::make_shared(); diff --git a/dbms/src/TestUtils/mockExecutor.h b/dbms/src/TestUtils/mockExecutor.h index c11635ac93e..bad92c4226d 100644 --- a/dbms/src/TestUtils/mockExecutor.h +++ b/dbms/src/TestUtils/mockExecutor.h @@ -31,6 +31,7 @@ using MockOrderByItems = std::initializer_list; using MockPartitionByItem = std::pair; using MockPartitionByItems = std::initializer_list; using MockColumnNames = std::initializer_list; +using MockColumnNamesVec = std::vector; using MockAsts = std::initializer_list; using MockWindowFrame = mock::MockWindowFrame; @@ -84,6 +85,7 @@ class DAGRequestBuilder DAGRequestBuilder & project(const String & col_name); DAGRequestBuilder & project(MockAsts expr); DAGRequestBuilder & project(MockColumnNames col_names); + DAGRequestBuilder & project(MockColumnNamesVec col_names); DAGRequestBuilder & exchangeSender(tipb::ExchangeType exchange_type); @@ -181,8 +183,8 @@ MockWindowFrame buildDefaultRowsFrame(); #define gt(expr1, expr2) makeASTFunction("greater", (expr1), (expr2)) #define And(expr1, expr2) makeASTFunction("and", (expr1), (expr2)) #define Or(expr1, expr2) makeASTFunction("or", (expr1), (expr2)) -#define NOT(expr) makeASTFunction("not", (expr1), (expr2)) -#define Max(expr) makeASTFunction("max", expr) +#define NOT(expr) makeASTFunction("not", (expr)) +#define Max(expr) makeASTFunction("max", (expr)) /// Window functions #define RowNumber() makeASTFunction("RowNumber") #define Rank() makeASTFunction("Rank") From b0ef0b6cccab0a7d33be737d22918ea93f3ba5c7 Mon Sep 17 00:00:00 2001 From: xzhangxian1008 Date: Mon, 27 Jun 2022 09:37:36 +0800 Subject: [PATCH 06/11] update --- dbms/src/Common/typeid_cast.h | 3 +- dbms/src/Debug/astToExecutor.cpp | 4 ++- .../Flash/tests/gtest_projection_executor.cpp | 9 +----- dbms/src/Flash/tests/gtest_topn_executor.cpp | 32 +++++++++++++++++-- dbms/src/TestUtils/FunctionTestUtils.cpp | 15 +++++++++ 5 files changed, 51 insertions(+), 12 deletions(-) diff --git a/dbms/src/Common/typeid_cast.h b/dbms/src/Common/typeid_cast.h index 4cc8682ddd2..23486b6e8ad 100644 --- a/dbms/src/Common/typeid_cast.h +++ b/dbms/src/Common/typeid_cast.h @@ -49,7 +49,8 @@ std::enable_if_t, To> typeid_cast(From & from) template To typeid_cast(From * from) { - if (typeid(*from) == typeid(std::remove_pointer_t)) + if (typeid(*from) + == typeid(std::remove_pointer_t)) return static_cast(from); else return nullptr; diff --git a/dbms/src/Debug/astToExecutor.cpp b/dbms/src/Debug/astToExecutor.cpp index fec76d7a085..c71e0a65d10 100644 --- a/dbms/src/Debug/astToExecutor.cpp +++ b/dbms/src/Debug/astToExecutor.cpp @@ -1629,8 +1629,10 @@ ExecutorPtr compileProject(ExecutorPtr input, size_t & executor_index, ASTPtr se } } } - auto project = std::make_shared(executor_index, output_schema, std::move(exprs)); + for (int i = 0; i < project->exprs.size(); ++i) { + std::cout << "Output: " << project->exprs[i]->getColumnName() << std::endl; + } project->children.push_back(input); return project; } diff --git a/dbms/src/Flash/tests/gtest_projection_executor.cpp b/dbms/src/Flash/tests/gtest_projection_executor.cpp index d32a674fb37..bc34a401c04 100644 --- a/dbms/src/Flash/tests/gtest_projection_executor.cpp +++ b/dbms/src/Flash/tests/gtest_projection_executor.cpp @@ -49,13 +49,6 @@ class ExecutorProjectionTestRunner : public DB::tests::ExecutorTest return context.scan(db_name, table_name).project(param).build(context); }; - void executeDifferentConcurrency(const std::shared_ptr & request, const ColumnsWithTypeAndName & expect_columns) - { - executeStreams(request, expect_columns, 1); - // for (size_t i = 1; i < 10; i += 2) - // executeStreams(request, expect_columns, i); - } - /// Prepare column data const ColDataString col0{"col0-0", "col0-1", "", "col0-2", {}, "col0-3", ""}; const ColDataString col1{"col1-0", {}, "", "col1-1", "", "col1-2", "col1-3"}; @@ -74,7 +67,7 @@ try { /// Check single column auto request = getRequest({col_names[2]}); - executeDifferentConcurrency(request, {toNullableVec(col_names[2], col2)}); + executeStreams(request, {toNullableVec(col_names[2], col2)}); /// Check multi columns request = getRequest({col_names[0], col_names[4]}); diff --git a/dbms/src/Flash/tests/gtest_topn_executor.cpp b/dbms/src/Flash/tests/gtest_topn_executor.cpp index 8901c663ed7..6c387c76879 100644 --- a/dbms/src/Flash/tests/gtest_topn_executor.cpp +++ b/dbms/src/Flash/tests/gtest_topn_executor.cpp @@ -53,9 +53,12 @@ class ExecutorTopNTestRunner : public DB::tests::ExecutorTest return context.scan(db_name, table_name).topN(col_name, is_desc, limit_num).build(context); } - std::shared_ptr getRequest(const String & table_name, MockOrderByItems order_by_items, int limit) + std::shared_ptr getRequest(const String & table_name, MockOrderByItems order_by_items, int limit, MockAsts func_proj_ast = {}, MockColumnNames out_proj_ast = {}) { - return context.scan(db_name, table_name).topN(order_by_items, limit).build(context); + if (func_proj_ast.size() == 0) + return context.scan(db_name, table_name).topN(order_by_items, limit).build(context); + else + return context.scan(db_name, table_name).project(func_proj_ast).topN(order_by_items, limit).project(out_proj_ast).build(context); } /// Prepare some names @@ -155,17 +158,42 @@ try { std::shared_ptr request; std::vector expect_cols; + MockColumnNames output_projection{col_name[0], col_name[1], col_name[2], col_name[3]}; + MockAsts func_projection; // Do function operation for topn + auto col0_ast = col(col_name[0]); + auto col1_ast = col(col_name[1]); + auto col2_ast = col(col_name[2]); + auto col3_ast = col(col_name[3]); { /// "and" function + /// and(col0, col0) + expect_cols = {{toNullableVec(col_name[0], ColumnWithInt32{{}, 32, {}, 27, 36, 34}), + toNullableVec(col_name[1], ColumnWithString{"female", "male", "male", "female", "female", "male"}), + toNullableVec(col_name[2], ColumnWithString{"korea", "usa", "china", "usa", "china", "china"}), + toNullableVec(col_name[3], ColumnWithInt32{1300, {}, {}, 0, 900, -300})}}; + + MockOrderByItems order_by_items; + + { + /// select * from clerk order by age and salary ASC; + order_by_items = {MockOrderByItem("and(age, salary)", false)}; + auto and_ast = And(col(col_name[0]), col(col_name[3])); + func_projection = {col0_ast, col1_ast, col2_ast, col3_ast, and_ast}; + + request = getRequest(table_name, order_by_items, 100, func_projection, output_projection); + executeStreams(request, expect_cols[0]); + } } { /// "equal" function + /// equals(col0, col1) } { /// "greater" function + /// greater(col0, col1) } /// TODO more functions... diff --git a/dbms/src/TestUtils/FunctionTestUtils.cpp b/dbms/src/TestUtils/FunctionTestUtils.cpp index 7fb526aeb01..84d60e4ad5d 100644 --- a/dbms/src/TestUtils/FunctionTestUtils.cpp +++ b/dbms/src/TestUtils/FunctionTestUtils.cpp @@ -82,6 +82,21 @@ ::testing::AssertionResult columnEqual( ASSERT_EQUAL(expected->getName(), actual->getName(), "Column name mismatch"); ASSERT_EQUAL(expected->size(), actual->size(), "Column size mismatch"); + std::cout << "Expect: " << " "; + for (size_t i = 0, size = expected->size(); i < size; ++i) + { + auto expected_field = (*expected)[i]; + std::cout << expected_field.toString() << " "; + } + + std::cout << std::endl << "Actual: " << " "; + for (size_t i = 0, size = actual->size(); i < size; ++i) + { + auto actual_field = (*actual)[i]; + std::cout << actual_field.toString() << " "; + } + std::cout << std::endl; + for (size_t i = 0, size = expected->size(); i < size; ++i) { auto expected_field = (*expected)[i]; From fb65219a4f7a41d40c0da15702023b9b9f71c09c Mon Sep 17 00:00:00 2001 From: xzhangxian1008 Date: Mon, 27 Jun 2022 10:52:07 +0800 Subject: [PATCH 07/11] update --- dbms/src/Common/typeid_cast.h | 3 +- dbms/src/Debug/astToExecutor.cpp | 3 - .../Flash/tests/gtest_projection_executor.cpp | 179 +++++++++--------- dbms/src/Flash/tests/gtest_topn_executor.cpp | 72 ++++--- dbms/src/TestUtils/FunctionTestUtils.cpp | 15 -- 5 files changed, 139 insertions(+), 133 deletions(-) diff --git a/dbms/src/Common/typeid_cast.h b/dbms/src/Common/typeid_cast.h index 23486b6e8ad..4cc8682ddd2 100644 --- a/dbms/src/Common/typeid_cast.h +++ b/dbms/src/Common/typeid_cast.h @@ -49,8 +49,7 @@ std::enable_if_t, To> typeid_cast(From & from) template To typeid_cast(From * from) { - if (typeid(*from) - == typeid(std::remove_pointer_t)) + if (typeid(*from) == typeid(std::remove_pointer_t)) return static_cast(from); else return nullptr; diff --git a/dbms/src/Debug/astToExecutor.cpp b/dbms/src/Debug/astToExecutor.cpp index c71e0a65d10..7d1f3bc7209 100644 --- a/dbms/src/Debug/astToExecutor.cpp +++ b/dbms/src/Debug/astToExecutor.cpp @@ -1630,9 +1630,6 @@ ExecutorPtr compileProject(ExecutorPtr input, size_t & executor_index, ASTPtr se } } auto project = std::make_shared(executor_index, output_schema, std::move(exprs)); - for (int i = 0; i < project->exprs.size(); ++i) { - std::cout << "Output: " << project->exprs[i]->getColumnName() << std::endl; - } project->children.push_back(input); return project; } diff --git a/dbms/src/Flash/tests/gtest_projection_executor.cpp b/dbms/src/Flash/tests/gtest_projection_executor.cpp index bc34a401c04..9a6e1e755f2 100644 --- a/dbms/src/Flash/tests/gtest_projection_executor.cpp +++ b/dbms/src/Flash/tests/gtest_projection_executor.cpp @@ -43,18 +43,18 @@ class ExecutorProjectionTestRunner : public DB::tests::ExecutorTest toNullableVec(col_names[4], col4)}); } - template + template std::shared_ptr getRequest(T param) { return context.scan(db_name, table_name).project(param).build(context); }; /// Prepare column data - const ColDataString col0{"col0-0", "col0-1", "", "col0-2", {}, "col0-3", ""}; - const ColDataString col1{"col1-0", {}, "", "col1-1", "", "col1-2", "col1-3"}; - const ColDataString col2{ "", "col2-0", "col2-1", {}, "col2-3", {}, "col2-4"}; - const ColDataInt32 col3{1, {}, 0, -111111, {}, 0, 9999}; - const ColDataInt32 col4{0, 5, {}, -234, {}, 24353, 9999}; + const ColDataString col0{"col0-0", "col0-1", "", "col0-2", {}, "col0-3", ""}; + const ColDataString col1{"col1-0", {}, "", "col1-1", "", "col1-2", "col1-3"}; + const ColDataString col2{"", "col2-0", "col2-1", {}, "col2-3", {}, "col2-4"}; + const ColDataInt32 col3{1, {}, 0, -111111, {}, 0, 9999}; + const ColDataInt32 col4{0, 5, {}, -234, {}, 24353, 9999}; /// Prepare some names std::vector col_names{"col0", "col1", "col2", "col3", "col4"}; @@ -72,8 +72,9 @@ try /// Check multi columns request = getRequest({col_names[0], col_names[4]}); executeStreams(request, - {toNullableVec(col_names[0], col0), - toNullableVec(col_names[4], col1), + { + toNullableVec(col_names[0], col0), + toNullableVec(col_names[4], col1), }); /// Check multi columns request = getRequest({col_names[0], col_names[1], col_names[3]}); @@ -81,7 +82,7 @@ try {toNullableVec(col_names[0], col0), toNullableVec(col_names[1], col1), toNullableVec(col_names[3], col2)}); - + /// Check duplicate columns request = getRequest({col_names[1], col_names[1], col_names[1]}); executeStreams(request, @@ -101,7 +102,7 @@ try projection_input.push_back(col_names[1]); columns.push_back(expect_column); } - + request = getRequest(projection_input); executeStreams(request, columns); } @@ -117,101 +118,101 @@ try { /// Data type: TypeString request = getRequest({eq(col(col_names[0]), col(col_names[0]))}); - executeStreams(request, - {toNullableVec({1, 1, 1, 1, {}, 1, 1})}); - - request = getRequest({eq(col(col_names[0]), col(col_names[1]))}); - executeStreams(request, - {toNullableVec({0, {}, 1, 0, {}, 0, 0})}); - } + executeStreams(request, + {toNullableVec({1, 1, 1, 1, {}, 1, 1})}); - { - /// Data type: TypeLong - request = getRequest({eq(col(col_names[3]), col(col_names[4]))}); - executeStreams(request, - {toNullableVec({0, {}, {}, 0, {}, 0, 1})}); - } - } + request = getRequest({eq(col(col_names[0]), col(col_names[1]))}); + executeStreams(request, + {toNullableVec({0, {}, 1, 0, {}, 0, 0})}); +} + +{ + /// Data type: TypeLong + request = getRequest({eq(col(col_names[3]), col(col_names[4]))}); + executeStreams(request, + {toNullableVec({0, {}, {}, 0, {}, 0, 1})}); +} +} // namespace tests +{ + /// Test "greater" function { - /// Test "greater" function - { - /// Data type: TypeString - request = getRequest({gt(col(col_names[0]), col(col_names[1]))}); - executeStreams(request, - {toNullableVec({0, {}, 0, 0, {}, 0, 0})}); + /// Data type: TypeString + request = getRequest({gt(col(col_names[0]), col(col_names[1]))}); +executeStreams(request, + {toNullableVec({0, {}, 0, 0, {}, 0, 0})}); + +request = getRequest({gt(col(col_names[1]), col(col_names[0]))}); +executeStreams(request, + {toNullableVec({1, {}, 0, 1, {}, 1, 1})}); +} // namespace DB - request = getRequest({gt(col(col_names[1]), col(col_names[0]))}); - executeStreams(request, - {toNullableVec({1, {}, 0, 1, {}, 1, 1})}); - } +{ + /// Data type: TypeLong + request = getRequest({gt(col(col_names[3]), col(col_names[4]))}); + executeStreams(request, + {toNullableVec({1, {}, {}, 0, {}, 0, 0})}); - { - /// Data type: TypeLong - request = getRequest({gt(col(col_names[3]), col(col_names[4]))}); - executeStreams(request, - {toNullableVec({1, {}, {}, 0, {}, 0, 0})}); - - request = getRequest({gt(col(col_names[4]), col(col_names[3]))}); - executeStreams(request, - {toNullableVec({0, {}, {}, 1, {}, 1, 0})}); - } - } + request = getRequest({gt(col(col_names[4]), col(col_names[3]))}); + executeStreams(request, + {toNullableVec({0, {}, {}, 1, {}, 1, 0})}); +} +} +{ + /// Test "and" function { - /// Test "and" function - { - /// Data type: TypeString - request = getRequest({And(col(col_names[0]), col(col_names[0]))}); - executeStreams(request, - {toNullableVec({0, 0, 0, 0, {}, 0, 0})}); - - request = getRequest({And(col(col_names[0]), col(col_names[1]))}); - executeStreams(request, - {toNullableVec({0, 0, 0, 0, 0, 0, 0})}); - } + /// Data type: TypeString + request = getRequest({And(col(col_names[0]), col(col_names[0]))}); +executeStreams(request, + {toNullableVec({0, 0, 0, 0, {}, 0, 0})}); + +request = getRequest({And(col(col_names[0]), col(col_names[1]))}); +executeStreams(request, + {toNullableVec({0, 0, 0, 0, 0, 0, 0})}); +} - { - /// Data type: TypeLong - request = getRequest({And(col(col_names[3]), col(col_names[4]))}); - executeStreams(request, - {toNullableVec({0, {}, 0, 1, {}, 0, 1})}); - } - } +{ + /// Data type: TypeLong + request = getRequest({And(col(col_names[3]), col(col_names[4]))}); + executeStreams(request, + {toNullableVec({0, {}, 0, 1, {}, 0, 1})}); +} +} +{ + /// Test "not" function { - /// Test "not" function - { - /// Data type: TypeString - request = getRequest({NOT(col(col_names[0])), NOT(col(col_names[1])), NOT(col(col_names[2]))}); - executeStreams(request, - {toNullableVec({1, 1, 1, 1, {}, 1, 1}), - toNullableVec({1, {}, 1, 1, 1, 1, 1}), - toNullableVec({1, 1, 1, {}, 1, {}, 1})}); - } + /// Data type: TypeString + request = getRequest({NOT(col(col_names[0])), NOT(col(col_names[1])), NOT(col(col_names[2]))}); +executeStreams(request, + {toNullableVec({1, 1, 1, 1, {}, 1, 1}), + toNullableVec({1, {}, 1, 1, 1, 1, 1}), + toNullableVec({1, 1, 1, {}, 1, {}, 1})}); +} - { - /// Data type: TypeLong - request = getRequest({NOT(col(col_names[3])), NOT(col(col_names[4]))}); - executeStreams(request, - {toNullableVec({0, {}, 1, 0, {}, 1, 0}), - toNullableVec({1, 0, {}, 0, {}, 0, 0})}); - } - } +{ + /// Data type: TypeLong + request = getRequest({NOT(col(col_names[3])), NOT(col(col_names[4]))}); + executeStreams(request, + {toNullableVec({0, {}, 1, 0, {}, 1, 0}), + toNullableVec({1, 0, {}, 0, {}, 0, 0})}); +} +} +{ + /// Test "max" function + /// TODO wait for the implementation of the max aggregation { - /// Test "max" function - /// TODO wait for the implementation of the max aggregation - { - /// Data type: TypeString - } + /// Data type: TypeString + } - { - /// Data type: TypeLong - } + { + /// Data type: TypeLong } +} - /// TODO more functions +/// TODO more functions } CATCH diff --git a/dbms/src/Flash/tests/gtest_topn_executor.cpp b/dbms/src/Flash/tests/gtest_topn_executor.cpp index 6c387c76879..1faf935d5c1 100644 --- a/dbms/src/Flash/tests/gtest_topn_executor.cpp +++ b/dbms/src/Flash/tests/gtest_topn_executor.cpp @@ -45,7 +45,6 @@ class ExecutorTopNTestRunner : public DB::tests::ExecutorTest toNullableVec(col_name[1], col_gender), toNullableVec(col_name[2], col_country), toNullableVec(col_name[3], c0l_salary)}); - } std::shared_ptr getRequest(const String & table_name, const String & col_name, bool is_desc, int limit_num) @@ -70,10 +69,10 @@ class ExecutorTopNTestRunner : public DB::tests::ExecutorTest const String table_name{"clerk"}; const std::vector col_name{"age", "gender", "country", "salary"}; - ColumnWithInt32 col_age {{}, 27, 32, 36, {}, 34}; - ColumnWithString col_gender {"female", "female", "male", "female", "male", "male"}; - ColumnWithString col_country{"korea", "usa", "usa", "china", "china", "china"}; - ColumnWithInt32 c0l_salary {1300, 0, {}, 900, {}, -300}; + ColumnWithInt32 col_age{{}, 27, 32, 36, {}, 34}; + ColumnWithString col_gender{"female", "female", "male", "female", "male", "male"}; + ColumnWithString col_country{"korea", "usa", "usa", "china", "china", "china"}; + ColumnWithInt32 c0l_salary{1300, 0, {}, 900, {}, -300}; }; TEST_F(ExecutorTopNTestRunner, TopN) @@ -122,10 +121,10 @@ try toNullableVec(col_name[1], ColumnWithString{"male", "male", "male", "female", "female", "female"}), toNullableVec(col_name[2], ColumnWithString{"usa", "china", "china", "usa", "china", "korea"}), toNullableVec(col_name[3], ColumnWithInt32{{}, {}, -300, 0, 900, 1300})}, - {toNullableVec(col_name[0], ColumnWithInt32{34, {}, 32, 36, {}, 27}), - toNullableVec(col_name[1], ColumnWithString{"male", "male", "male", "female", "female", "female"}), - toNullableVec(col_name[2], ColumnWithString{"china", "china", "usa", "china", "korea", "usa"}), - toNullableVec(col_name[3], ColumnWithInt32{-300, {}, {}, 900, 1300, 0})}}; + {toNullableVec(col_name[0], ColumnWithInt32{34, {}, 32, 36, {}, 27}), + toNullableVec(col_name[1], ColumnWithString{"male", "male", "male", "female", "female", "female"}), + toNullableVec(col_name[2], ColumnWithString{"china", "china", "usa", "china", "korea", "usa"}), + toNullableVec(col_name[3], ColumnWithInt32{-300, {}, {}, 900, 1300, 0})}}; MockOrderByItems order_by_items; @@ -160,27 +159,26 @@ try std::vector expect_cols; MockColumnNames output_projection{col_name[0], col_name[1], col_name[2], col_name[3]}; MockAsts func_projection; // Do function operation for topn - auto col0_ast = col(col_name[0]); - auto col1_ast = col(col_name[1]); - auto col2_ast = col(col_name[2]); - auto col3_ast = col(col_name[3]); + MockOrderByItems order_by_items; + ASTPtr col0_ast = col(col_name[0]); + ASTPtr col1_ast = col(col_name[1]); + ASTPtr col2_ast = col(col_name[2]); + ASTPtr col3_ast = col(col_name[3]); + ASTPtr func_ast; { /// "and" function - /// and(col0, col0) - expect_cols = {{toNullableVec(col_name[0], ColumnWithInt32{{}, 32, {}, 27, 36, 34}), + expect_cols = {{toNullableVec(col_name[0], ColumnWithInt32{{}, {}, 32, 27, 36, 34}), toNullableVec(col_name[1], ColumnWithString{"female", "male", "male", "female", "female", "male"}), - toNullableVec(col_name[2], ColumnWithString{"korea", "usa", "china", "usa", "china", "china"}), + toNullableVec(col_name[2], ColumnWithString{"korea", "china", "usa", "usa", "china", "china"}), toNullableVec(col_name[3], ColumnWithInt32{1300, {}, {}, 0, 900, -300})}}; - - MockOrderByItems order_by_items; { - /// select * from clerk order by age and salary ASC; + /// select * from clerk order by age and salary ASC limit 100; order_by_items = {MockOrderByItem("and(age, salary)", false)}; - auto and_ast = And(col(col_name[0]), col(col_name[3])); - func_projection = {col0_ast, col1_ast, col2_ast, col3_ast, and_ast}; - + func_ast = And(col(col_name[0]), col(col_name[3])); + func_projection = {col0_ast, col1_ast, col2_ast, col3_ast, func_ast}; + request = getRequest(table_name, order_by_items, 100, func_projection, output_projection); executeStreams(request, expect_cols[0]); } @@ -188,12 +186,38 @@ try { /// "equal" function - /// equals(col0, col1) + expect_cols = {{toNullableVec(col_name[0], ColumnWithInt32{27, 36, 34, 32, {}, {}}), + toNullableVec(col_name[1], ColumnWithString{"female", "female", "male", "male", "female", "male"}), + toNullableVec(col_name[2], ColumnWithString{"usa", "china", "china", "usa", "korea", "china"}), + toNullableVec(col_name[3], ColumnWithInt32{0, 900, -300, {}, 1300, {}})}}; + + { + /// select age, salary from clerk order by age = salary DESC limit 100; + order_by_items = {MockOrderByItem("equals(age, salary)", true)}; + func_ast = eq(col(col_name[0]), col(col_name[3])); + func_projection = {col0_ast, col1_ast, col2_ast, col3_ast, func_ast}; + + request = getRequest(table_name, order_by_items, 100, func_projection, output_projection); + executeStreams(request, expect_cols[0]); + } } { /// "greater" function - /// greater(col0, col1) + expect_cols = {{toNullableVec(col_name[0], ColumnWithInt32{{}, 32, {}, 36, 27, 34}), + toNullableVec(col_name[1], ColumnWithString{"female", "male", "male", "female", "female", "male"}), + toNullableVec(col_name[2], ColumnWithString{"korea", "usa", "china", "china", "usa", "china"}), + toNullableVec(col_name[3], ColumnWithInt32{1300, {}, {}, 900, 0, -300})}}; + + { + /// select age, gender, country, salary from clerk order by age > salary ASC limit 100; + order_by_items = {MockOrderByItem("greater(age, salary)", false)}; + func_ast = gt(col(col_name[0]), col(col_name[3])); + func_projection = {col0_ast, col1_ast, col2_ast, col3_ast, func_ast}; + + request = getRequest(table_name, order_by_items, 100, func_projection, output_projection); + executeStreams(request, expect_cols[0]); + } } /// TODO more functions... diff --git a/dbms/src/TestUtils/FunctionTestUtils.cpp b/dbms/src/TestUtils/FunctionTestUtils.cpp index 84d60e4ad5d..7fb526aeb01 100644 --- a/dbms/src/TestUtils/FunctionTestUtils.cpp +++ b/dbms/src/TestUtils/FunctionTestUtils.cpp @@ -82,21 +82,6 @@ ::testing::AssertionResult columnEqual( ASSERT_EQUAL(expected->getName(), actual->getName(), "Column name mismatch"); ASSERT_EQUAL(expected->size(), actual->size(), "Column size mismatch"); - std::cout << "Expect: " << " "; - for (size_t i = 0, size = expected->size(); i < size; ++i) - { - auto expected_field = (*expected)[i]; - std::cout << expected_field.toString() << " "; - } - - std::cout << std::endl << "Actual: " << " "; - for (size_t i = 0, size = actual->size(); i < size; ++i) - { - auto actual_field = (*actual)[i]; - std::cout << actual_field.toString() << " "; - } - std::cout << std::endl; - for (size_t i = 0, size = expected->size(); i < size; ++i) { auto expected_field = (*expected)[i]; From 18bbff4dd76cdaea359ba2c6e29a1cefaab3624d Mon Sep 17 00:00:00 2001 From: xzhangxian1008 Date: Mon, 27 Jun 2022 15:11:56 +0800 Subject: [PATCH 08/11] update --- dbms/src/Debug/astToExecutor.cpp | 1 + .../Flash/tests/gtest_projection_executor.cpp | 215 +++++++++--------- dbms/src/TestUtils/FunctionTestUtils.cpp | 18 ++ 3 files changed, 124 insertions(+), 110 deletions(-) diff --git a/dbms/src/Debug/astToExecutor.cpp b/dbms/src/Debug/astToExecutor.cpp index 7d1f3bc7209..dd36f75f152 100644 --- a/dbms/src/Debug/astToExecutor.cpp +++ b/dbms/src/Debug/astToExecutor.cpp @@ -1630,6 +1630,7 @@ ExecutorPtr compileProject(ExecutorPtr input, size_t & executor_index, ASTPtr se } } auto project = std::make_shared(executor_index, output_schema, std::move(exprs)); + std::cout << "Output: " << project->exprs[0]->getColumnName() << std::endl; project->children.push_back(input); return project; } diff --git a/dbms/src/Flash/tests/gtest_projection_executor.cpp b/dbms/src/Flash/tests/gtest_projection_executor.cpp index 9a6e1e755f2..ec36816bdae 100644 --- a/dbms/src/Flash/tests/gtest_projection_executor.cpp +++ b/dbms/src/Flash/tests/gtest_projection_executor.cpp @@ -44,17 +44,33 @@ class ExecutorProjectionTestRunner : public DB::tests::ExecutorTest } template - std::shared_ptr getRequest(T param) + std::shared_ptr getRequest(T param, const String & sort_col) { - return context.scan(db_name, table_name).project(param).build(context); + /// topN is introduced, so that we can get stable results in concurrency environment. + return context.scan(db_name, table_name).project(param).topN(sort_col, false, 100).build(context); }; + void executeWithConcurrency(const std::shared_ptr & request, const ColumnsWithTypeAndName & expect_columns) + { + // executeStreams(request, expect_columns, 1); + for (size_t i = 1; i < 10; i += 2) + { + std::cout << "Expect loop " << i << std::endl; + executeStreams(request, expect_columns, i); + } + } + /// Prepare column data const ColDataString col0{"col0-0", "col0-1", "", "col0-2", {}, "col0-3", ""}; const ColDataString col1{"col1-0", {}, "", "col1-1", "", "col1-2", "col1-3"}; const ColDataString col2{"", "col2-0", "col2-1", {}, "col2-3", {}, "col2-4"}; const ColDataInt32 col3{1, {}, 0, -111111, {}, 0, 9999}; - const ColDataInt32 col4{0, 5, {}, -234, {}, 24353, 9999}; + + /** Each value in col4 should be different from each other so that topn + * could sort the columns into an unique result, or multi-results could + * be right. + */ + const ColDataInt32 col4{0, 5, -123, -234, {}, 24353, 9999}; /// Prepare some names std::vector col_names{"col0", "col1", "col2", "col3", "col4"}; @@ -65,46 +81,54 @@ class ExecutorProjectionTestRunner : public DB::tests::ExecutorTest TEST_F(ExecutorProjectionTestRunner, Projection) try { + /// Results after sorted by col4 + const ColDataString col0_sorted_asc{{}, "col0-2", "", "col0-0", "col0-1", "", "col0-3"}; + const ColDataString col1_sorted_asc{"", "col1-1", "", "col1-0", {}, "col1-3", "col1-2"}; + const ColDataString col2_sorted_asc{"col2-3", {}, "col2-1", "", "col2-0", "col2-4", {}}; + const ColDataInt32 col3_sorted_asc{{}, -111111, 0, 1, {}, 9999, 0}; + const ColDataInt32 col4_sorted_asc{{}, -234, -123, 0, 5, 9999, 24353}; + std::cout << "Expect<1>" << std::endl; /// Check single column - auto request = getRequest({col_names[2]}); - executeStreams(request, {toNullableVec(col_names[2], col2)}); - + auto request = getRequest({col_names[4]}, col_names[4]); + executeWithConcurrency(request, {toNullableVec(col_names[4], col4_sorted_asc)}); + std::cout << "Expect<2>" << std::endl; /// Check multi columns - request = getRequest({col_names[0], col_names[4]}); - executeStreams(request, - { - toNullableVec(col_names[0], col0), - toNullableVec(col_names[4], col1), - }); + request = getRequest({col_names[0], col_names[4]}, col_names[4]); + executeWithConcurrency(request, + { + toNullableVec(col_names[0], col0_sorted_asc), + toNullableVec(col_names[4], col4_sorted_asc), + }); + std::cout << "Expect<3>" << std::endl; /// Check multi columns - request = getRequest({col_names[0], col_names[1], col_names[3]}); - executeStreams(request, - {toNullableVec(col_names[0], col0), - toNullableVec(col_names[1], col1), - toNullableVec(col_names[3], col2)}); - + request = getRequest({col_names[0], col_names[1], col_names[4]}, col_names[4]); + executeWithConcurrency(request, + {toNullableVec(col_names[0], col0_sorted_asc), + toNullableVec(col_names[1], col1_sorted_asc), + toNullableVec(col_names[4], col4_sorted_asc)}); + std::cout << "Expect<4>" << std::endl; /// Check duplicate columns - request = getRequest({col_names[1], col_names[1], col_names[1]}); - executeStreams(request, - {toNullableVec(col_names[1], col1), - toNullableVec(col_names[1], col1), - toNullableVec(col_names[1], col1)}); - + request = getRequest({col_names[4], col_names[4], col_names[4]}, col_names[4]); + executeWithConcurrency(request, + {toNullableVec(col_names[4], col4_sorted_asc), + toNullableVec(col_names[4], col4_sorted_asc), + toNullableVec(col_names[4], col4_sorted_asc)}); + std::cout << "Expect<5>" << std::endl; { /// Check large number of columns const size_t col_num = 100; MockColumnNamesVec projection_input; ColumnsWithTypeAndName columns; - auto expect_column = toNullableVec(col_names[1], col1); + auto expect_column = toNullableVec(col_names[4], col4_sorted_asc); for (size_t i = 0; i < col_num; ++i) { - projection_input.push_back(col_names[1]); + projection_input.push_back(col_names[4]); columns.push_back(expect_column); } - request = getRequest(projection_input); - executeStreams(request, columns); + request = getRequest(projection_input, col_names[4]); + executeWithConcurrency(request, columns); } } CATCH @@ -113,106 +137,77 @@ TEST_F(ExecutorProjectionTestRunner, ProjectionFunction) try { std::shared_ptr request; - { - /// Test "equal" function - { - /// Data type: TypeString - request = getRequest({eq(col(col_names[0]), col(col_names[0]))}); - executeStreams(request, - {toNullableVec({1, 1, 1, 1, {}, 1, 1})}); - request = getRequest({eq(col(col_names[0]), col(col_names[1]))}); + /// Test "equal" function + std::cout << "Expect<1.1>" << std::endl; + /// Data type: TypeString + request = getRequest({eq(col(col_names[0]), col(col_names[0]))}, "equals(col0, col0)"); executeStreams(request, - {toNullableVec({0, {}, 1, 0, {}, 0, 0})}); -} - -{ + {toNullableVec({{}, 1, 1, 1, 1, 1, 1})}); + std::cout << "Expect<1.2>" << std::endl; + request = getRequest({eq(col(col_names[0]), col(col_names[1]))}, "equals(col0, col1)"); + executeStreams(request, + {toNullableVec({{}, {}, 0, 0, 0, 0, 1})}); + std::cout << "Expect<1.3>" << std::endl; /// Data type: TypeLong - request = getRequest({eq(col(col_names[3]), col(col_names[4]))}); + request = getRequest({eq(col(col_names[3]), col(col_names[4]))}, "equals(col3, col4)"); executeStreams(request, - {toNullableVec({0, {}, {}, 0, {}, 0, 1})}); -} -} // namespace tests + {toNullableVec({{}, {}, 0, 0, 0, 0, 1})}); -{ - /// Test "greater" function - { - /// Data type: TypeString - request = getRequest({gt(col(col_names[0]), col(col_names[1]))}); -executeStreams(request, - {toNullableVec({0, {}, 0, 0, {}, 0, 0})}); - -request = getRequest({gt(col(col_names[1]), col(col_names[0]))}); -executeStreams(request, - {toNullableVec({1, {}, 0, 1, {}, 1, 1})}); -} // namespace DB -{ + /// Test "greater" function + std::cout << "Expect<2.1>" << std::endl; + /// Data type: TypeString + request = getRequest({gt(col(col_names[0]), col(col_names[1]))}, "greater(col0, col1)"); + executeStreams(request, + {toNullableVec({{}, {}, 0, 0, 0, 0, 0})}); + std::cout << "Expect<2.2>" << std::endl; + request = getRequest({gt(col(col_names[1]), col(col_names[0]))}, "greater(col1, col0)"); + executeStreams(request, + {toNullableVec({{}, {}, 0, 1, 1, 1, 1})}); + std::cout << "Expect<2.3>" << std::endl; /// Data type: TypeLong - request = getRequest({gt(col(col_names[3]), col(col_names[4]))}); + request = getRequest({gt(col(col_names[3]), col(col_names[4]))}, "greater(col3, col4)"); executeStreams(request, - {toNullableVec({1, {}, {}, 0, {}, 0, 0})}); - - request = getRequest({gt(col(col_names[4]), col(col_names[3]))}); + {toNullableVec({{}, {}, 0, 0, 0, 1, 1})}); + std::cout << "Expect<2.4>" << std::endl; + request = getRequest({gt(col(col_names[4]), col(col_names[3]))}, "greater(col4, col3)"); executeStreams(request, - {toNullableVec({0, {}, {}, 1, {}, 1, 0})}); -} -} + {toNullableVec({{}, {}, 0, 0, 0, 1, 1})}); -{ - /// Test "and" function - { - /// Data type: TypeString - request = getRequest({And(col(col_names[0]), col(col_names[0]))}); -executeStreams(request, - {toNullableVec({0, 0, 0, 0, {}, 0, 0})}); - -request = getRequest({And(col(col_names[0]), col(col_names[1]))}); -executeStreams(request, - {toNullableVec({0, 0, 0, 0, 0, 0, 0})}); -} -{ + /// Test "and" function + std::cout << "Expect<3.1>" << std::endl; + /// Data type: TypeString + request = getRequest({And(col(col_names[0]), col(col_names[0]))}, "and(col0, col0)"); + executeStreams(request, + {toNullableVec({{}, 0, 0, 0, 0, 0, 0})}); + std::cout << "Expect<3.2>" << std::endl; + request = getRequest({And(col(col_names[0]), col(col_names[1]))}, "and(col0, col1)"); + executeStreams(request, + {toNullableVec({0, 0, 0, 0, 0, 0, 0})}); + std::cout << "Expect<3.3>" << std::endl; /// Data type: TypeLong - request = getRequest({And(col(col_names[3]), col(col_names[4]))}); + request = getRequest({And(col(col_names[3]), col(col_names[4]))}, "and(col3, col4)"); executeStreams(request, - {toNullableVec({0, {}, 0, 1, {}, 0, 1})}); -} -} + {toNullableVec({{}, {}, 0, 0, 0, 1, 1})}); -{ /// Test "not" function - { - /// Data type: TypeString - request = getRequest({NOT(col(col_names[0])), NOT(col(col_names[1])), NOT(col(col_names[2]))}); -executeStreams(request, - {toNullableVec({1, 1, 1, 1, {}, 1, 1}), - toNullableVec({1, {}, 1, 1, 1, 1, 1}), - toNullableVec({1, 1, 1, {}, 1, {}, 1})}); -} - -{ + std::cout << "Expect<4.1>" << std::endl; + /// Data type: TypeString + request = getRequest({NOT(col(col_names[0])), NOT(col(col_names[1])), NOT(col(col_names[2]))}, "not(col2)"); + executeStreams(request, + {toNullableVec({1, 1, 1, 1, 1, {}, 1}), + toNullableVec({1, 1, 1, {}, 1, 1, 1}), + toNullableVec({{}, {}, 1, 1, 1, 1, 1})}); + std::cout << "Expect<4.2>" << std::endl; /// Data type: TypeLong - request = getRequest({NOT(col(col_names[3])), NOT(col(col_names[4]))}); + request = getRequest({NOT(col(col_names[3])), NOT(col(col_names[4]))}, "not(col3)"); executeStreams(request, - {toNullableVec({0, {}, 1, 0, {}, 1, 0}), - toNullableVec({1, 0, {}, 0, {}, 0, 0})}); -} -} - -{ - /// Test "max" function - /// TODO wait for the implementation of the max aggregation - { - /// Data type: TypeString - } - - { - /// Data type: TypeLong - } -} + {toNullableVec({{}, {}, 0, 0, 0, 1, 1}), + toNullableVec({{}, 0, 1, 0, 0, 0, 0})}); -/// TODO more functions + /// TODO more functions... } CATCH diff --git a/dbms/src/TestUtils/FunctionTestUtils.cpp b/dbms/src/TestUtils/FunctionTestUtils.cpp index 7fb526aeb01..210fd6dc716 100644 --- a/dbms/src/TestUtils/FunctionTestUtils.cpp +++ b/dbms/src/TestUtils/FunctionTestUtils.cpp @@ -82,6 +82,24 @@ ::testing::AssertionResult columnEqual( ASSERT_EQUAL(expected->getName(), actual->getName(), "Column name mismatch"); ASSERT_EQUAL(expected->size(), actual->size(), "Column size mismatch"); + std::cout << "Expect: " + << " "; + for (size_t i = 0, size = expected->size(); i < size; ++i) + { + auto expected_field = (*expected)[i]; + std::cout << expected_field.toString() << " "; + } + + std::cout << std::endl + << "Actual: " + << " "; + for (size_t i = 0, size = actual->size(); i < size; ++i) + { + auto actual_field = (*actual)[i]; + std::cout << actual_field.toString() << " "; + } + std::cout << std::endl; + for (size_t i = 0, size = expected->size(); i < size; ++i) { auto expected_field = (*expected)[i]; From ec593ec0fa262ca7e68e748aad2eb277a463f0c7 Mon Sep 17 00:00:00 2001 From: xzhangxian1008 Date: Mon, 27 Jun 2022 15:17:34 +0800 Subject: [PATCH 09/11] update --- dbms/src/Debug/astToExecutor.cpp | 1 - .../Flash/tests/gtest_projection_executor.cpp | 36 +++++++++---------- dbms/src/TestUtils/FunctionTestUtils.cpp | 18 ---------- 3 files changed, 17 insertions(+), 38 deletions(-) diff --git a/dbms/src/Debug/astToExecutor.cpp b/dbms/src/Debug/astToExecutor.cpp index dd36f75f152..7d1f3bc7209 100644 --- a/dbms/src/Debug/astToExecutor.cpp +++ b/dbms/src/Debug/astToExecutor.cpp @@ -1630,7 +1630,6 @@ ExecutorPtr compileProject(ExecutorPtr input, size_t & executor_index, ASTPtr se } } auto project = std::make_shared(executor_index, output_schema, std::move(exprs)); - std::cout << "Output: " << project->exprs[0]->getColumnName() << std::endl; project->children.push_back(input); return project; } diff --git a/dbms/src/Flash/tests/gtest_projection_executor.cpp b/dbms/src/Flash/tests/gtest_projection_executor.cpp index ec36816bdae..7840c288abf 100644 --- a/dbms/src/Flash/tests/gtest_projection_executor.cpp +++ b/dbms/src/Flash/tests/gtest_projection_executor.cpp @@ -52,10 +52,8 @@ class ExecutorProjectionTestRunner : public DB::tests::ExecutorTest void executeWithConcurrency(const std::shared_ptr & request, const ColumnsWithTypeAndName & expect_columns) { - // executeStreams(request, expect_columns, 1); for (size_t i = 1; i < 10; i += 2) { - std::cout << "Expect loop " << i << std::endl; executeStreams(request, expect_columns, i); } } @@ -87,11 +85,11 @@ try const ColDataString col2_sorted_asc{"col2-3", {}, "col2-1", "", "col2-0", "col2-4", {}}; const ColDataInt32 col3_sorted_asc{{}, -111111, 0, 1, {}, 9999, 0}; const ColDataInt32 col4_sorted_asc{{}, -234, -123, 0, 5, 9999, 24353}; - std::cout << "Expect<1>" << std::endl; + /// Check single column auto request = getRequest({col_names[4]}, col_names[4]); executeWithConcurrency(request, {toNullableVec(col_names[4], col4_sorted_asc)}); - std::cout << "Expect<2>" << std::endl; + /// Check multi columns request = getRequest({col_names[0], col_names[4]}, col_names[4]); executeWithConcurrency(request, @@ -99,21 +97,21 @@ try toNullableVec(col_names[0], col0_sorted_asc), toNullableVec(col_names[4], col4_sorted_asc), }); - std::cout << "Expect<3>" << std::endl; + /// Check multi columns request = getRequest({col_names[0], col_names[1], col_names[4]}, col_names[4]); executeWithConcurrency(request, {toNullableVec(col_names[0], col0_sorted_asc), toNullableVec(col_names[1], col1_sorted_asc), toNullableVec(col_names[4], col4_sorted_asc)}); - std::cout << "Expect<4>" << std::endl; + /// Check duplicate columns request = getRequest({col_names[4], col_names[4], col_names[4]}, col_names[4]); executeWithConcurrency(request, {toNullableVec(col_names[4], col4_sorted_asc), toNullableVec(col_names[4], col4_sorted_asc), toNullableVec(col_names[4], col4_sorted_asc)}); - std::cout << "Expect<5>" << std::endl; + { /// Check large number of columns const size_t col_num = 100; @@ -139,16 +137,16 @@ try std::shared_ptr request; /// Test "equal" function - std::cout << "Expect<1.1>" << std::endl; + /// Data type: TypeString request = getRequest({eq(col(col_names[0]), col(col_names[0]))}, "equals(col0, col0)"); executeStreams(request, {toNullableVec({{}, 1, 1, 1, 1, 1, 1})}); - std::cout << "Expect<1.2>" << std::endl; + request = getRequest({eq(col(col_names[0]), col(col_names[1]))}, "equals(col0, col1)"); executeStreams(request, {toNullableVec({{}, {}, 0, 0, 0, 0, 1})}); - std::cout << "Expect<1.3>" << std::endl; + /// Data type: TypeLong request = getRequest({eq(col(col_names[3]), col(col_names[4]))}, "equals(col3, col4)"); executeStreams(request, @@ -156,51 +154,51 @@ try /// Test "greater" function - std::cout << "Expect<2.1>" << std::endl; + /// Data type: TypeString request = getRequest({gt(col(col_names[0]), col(col_names[1]))}, "greater(col0, col1)"); executeStreams(request, {toNullableVec({{}, {}, 0, 0, 0, 0, 0})}); - std::cout << "Expect<2.2>" << std::endl; + request = getRequest({gt(col(col_names[1]), col(col_names[0]))}, "greater(col1, col0)"); executeStreams(request, {toNullableVec({{}, {}, 0, 1, 1, 1, 1})}); - std::cout << "Expect<2.3>" << std::endl; + /// Data type: TypeLong request = getRequest({gt(col(col_names[3]), col(col_names[4]))}, "greater(col3, col4)"); executeStreams(request, {toNullableVec({{}, {}, 0, 0, 0, 1, 1})}); - std::cout << "Expect<2.4>" << std::endl; + request = getRequest({gt(col(col_names[4]), col(col_names[3]))}, "greater(col4, col3)"); executeStreams(request, {toNullableVec({{}, {}, 0, 0, 0, 1, 1})}); /// Test "and" function - std::cout << "Expect<3.1>" << std::endl; + /// Data type: TypeString request = getRequest({And(col(col_names[0]), col(col_names[0]))}, "and(col0, col0)"); executeStreams(request, {toNullableVec({{}, 0, 0, 0, 0, 0, 0})}); - std::cout << "Expect<3.2>" << std::endl; + request = getRequest({And(col(col_names[0]), col(col_names[1]))}, "and(col0, col1)"); executeStreams(request, {toNullableVec({0, 0, 0, 0, 0, 0, 0})}); - std::cout << "Expect<3.3>" << std::endl; + /// Data type: TypeLong request = getRequest({And(col(col_names[3]), col(col_names[4]))}, "and(col3, col4)"); executeStreams(request, {toNullableVec({{}, {}, 0, 0, 0, 1, 1})}); /// Test "not" function - std::cout << "Expect<4.1>" << std::endl; + /// Data type: TypeString request = getRequest({NOT(col(col_names[0])), NOT(col(col_names[1])), NOT(col(col_names[2]))}, "not(col2)"); executeStreams(request, {toNullableVec({1, 1, 1, 1, 1, {}, 1}), toNullableVec({1, 1, 1, {}, 1, 1, 1}), toNullableVec({{}, {}, 1, 1, 1, 1, 1})}); - std::cout << "Expect<4.2>" << std::endl; + /// Data type: TypeLong request = getRequest({NOT(col(col_names[3])), NOT(col(col_names[4]))}, "not(col3)"); executeStreams(request, diff --git a/dbms/src/TestUtils/FunctionTestUtils.cpp b/dbms/src/TestUtils/FunctionTestUtils.cpp index 210fd6dc716..7fb526aeb01 100644 --- a/dbms/src/TestUtils/FunctionTestUtils.cpp +++ b/dbms/src/TestUtils/FunctionTestUtils.cpp @@ -82,24 +82,6 @@ ::testing::AssertionResult columnEqual( ASSERT_EQUAL(expected->getName(), actual->getName(), "Column name mismatch"); ASSERT_EQUAL(expected->size(), actual->size(), "Column size mismatch"); - std::cout << "Expect: " - << " "; - for (size_t i = 0, size = expected->size(); i < size; ++i) - { - auto expected_field = (*expected)[i]; - std::cout << expected_field.toString() << " "; - } - - std::cout << std::endl - << "Actual: " - << " "; - for (size_t i = 0, size = actual->size(); i < size; ++i) - { - auto actual_field = (*actual)[i]; - std::cout << actual_field.toString() << " "; - } - std::cout << std::endl; - for (size_t i = 0, size = expected->size(); i < size; ++i) { auto expected_field = (*expected)[i]; From e80d61345a13de54b4138985119714d965fae4a1 Mon Sep 17 00:00:00 2001 From: xzhangxian1008 Date: Tue, 28 Jun 2022 09:15:08 +0800 Subject: [PATCH 10/11] update --- .../Flash/tests/gtest_projection_executor.cpp | 104 ++++++++++-------- 1 file changed, 58 insertions(+), 46 deletions(-) diff --git a/dbms/src/Flash/tests/gtest_projection_executor.cpp b/dbms/src/Flash/tests/gtest_projection_executor.cpp index 7840c288abf..e1bc7e6d811 100644 --- a/dbms/src/Flash/tests/gtest_projection_executor.cpp +++ b/dbms/src/Flash/tests/gtest_projection_executor.cpp @@ -70,6 +70,13 @@ class ExecutorProjectionTestRunner : public DB::tests::ExecutorTest */ const ColDataInt32 col4{0, 5, -123, -234, {}, 24353, 9999}; + /// Results after sorted by col4 + const ColDataString col0_sorted_asc{{}, "col0-2", "", "col0-0", "col0-1", "", "col0-3"}; + const ColDataString col1_sorted_asc{"", "col1-1", "", "col1-0", {}, "col1-3", "col1-2"}; + const ColDataString col2_sorted_asc{"col2-3", {}, "col2-1", "", "col2-0", "col2-4", {}}; + const ColDataInt32 col3_sorted_asc{{}, -111111, 0, 1, {}, 9999, 0}; + const ColDataInt32 col4_sorted_asc{{}, -234, -123, 0, 5, 9999, 24353}; + /// Prepare some names std::vector col_names{"col0", "col1", "col2", "col3", "col4"}; const String db_name{"test_db"}; @@ -79,13 +86,6 @@ class ExecutorProjectionTestRunner : public DB::tests::ExecutorTest TEST_F(ExecutorProjectionTestRunner, Projection) try { - /// Results after sorted by col4 - const ColDataString col0_sorted_asc{{}, "col0-2", "", "col0-0", "col0-1", "", "col0-3"}; - const ColDataString col1_sorted_asc{"", "col1-1", "", "col1-0", {}, "col1-3", "col1-2"}; - const ColDataString col2_sorted_asc{"col2-3", {}, "col2-1", "", "col2-0", "col2-4", {}}; - const ColDataInt32 col3_sorted_asc{{}, -111111, 0, 1, {}, 9999, 0}; - const ColDataInt32 col4_sorted_asc{{}, -234, -123, 0, 5, 9999, 24353}; - /// Check single column auto request = getRequest({col_names[4]}, col_names[4]); executeWithConcurrency(request, {toNullableVec(col_names[4], col4_sorted_asc)}); @@ -139,71 +139,83 @@ try /// Test "equal" function /// Data type: TypeString - request = getRequest({eq(col(col_names[0]), col(col_names[0]))}, "equals(col0, col0)"); - executeStreams(request, - {toNullableVec({{}, 1, 1, 1, 1, 1, 1})}); + request = getRequest({eq(col(col_names[0]), col(col_names[0])), col(col_names[4])}, col_names[4]); + executeWithConcurrency(request, + {toNullableVec({{}, 1, 1, 1, 1, 1, 1}), + toNullableVec(col_names[4], col4_sorted_asc)}); - request = getRequest({eq(col(col_names[0]), col(col_names[1]))}, "equals(col0, col1)"); - executeStreams(request, - {toNullableVec({{}, {}, 0, 0, 0, 0, 1})}); + request = getRequest({eq(col(col_names[0]), col(col_names[1])), col(col_names[4])}, col_names[4]); + executeWithConcurrency(request, + {toNullableVec({{}, 0, 1, 0, {}, 0, 0}), + toNullableVec(col_names[4], col4_sorted_asc)}); /// Data type: TypeLong - request = getRequest({eq(col(col_names[3]), col(col_names[4]))}, "equals(col3, col4)"); - executeStreams(request, - {toNullableVec({{}, {}, 0, 0, 0, 0, 1})}); + request = getRequest({eq(col(col_names[3]), col(col_names[4])), col(col_names[4])}, col_names[4]); + executeWithConcurrency(request, + {toNullableVec({{}, 0, 0, 0, {}, 1, 0}), + toNullableVec(col_names[4], col4_sorted_asc)}); /// Test "greater" function /// Data type: TypeString - request = getRequest({gt(col(col_names[0]), col(col_names[1]))}, "greater(col0, col1)"); - executeStreams(request, - {toNullableVec({{}, {}, 0, 0, 0, 0, 0})}); + request = getRequest({gt(col(col_names[0]), col(col_names[1])), col(col_names[4])}, col_names[4]); + executeWithConcurrency(request, + {toNullableVec({{}, 0, 0, 0, {}, 0, 0}), + toNullableVec(col_names[4], col4_sorted_asc)}); - request = getRequest({gt(col(col_names[1]), col(col_names[0]))}, "greater(col1, col0)"); - executeStreams(request, - {toNullableVec({{}, {}, 0, 1, 1, 1, 1})}); + request = getRequest({gt(col(col_names[1]), col(col_names[0])), col(col_names[4])}, col_names[4]); + executeWithConcurrency(request, + {toNullableVec({{}, 1, 0, 1, {}, 1, 1}), + toNullableVec(col_names[4], col4_sorted_asc)}); /// Data type: TypeLong - request = getRequest({gt(col(col_names[3]), col(col_names[4]))}, "greater(col3, col4)"); - executeStreams(request, - {toNullableVec({{}, {}, 0, 0, 0, 1, 1})}); + request = getRequest({gt(col(col_names[3]), col(col_names[4])), col(col_names[4])}, col_names[4]); + executeWithConcurrency(request, + {toNullableVec({{}, 0, 1, 1, {}, 0, 0}), + toNullableVec(col_names[4], col4_sorted_asc)}); - request = getRequest({gt(col(col_names[4]), col(col_names[3]))}, "greater(col4, col3)"); - executeStreams(request, - {toNullableVec({{}, {}, 0, 0, 0, 1, 1})}); + request = getRequest({gt(col(col_names[4]), col(col_names[3])), col(col_names[4])}, col_names[4]); + executeWithConcurrency(request, + {toNullableVec({{}, 1, 0, 0, {}, 0, 1}), + toNullableVec(col_names[4], col4_sorted_asc)}); /// Test "and" function /// Data type: TypeString - request = getRequest({And(col(col_names[0]), col(col_names[0]))}, "and(col0, col0)"); - executeStreams(request, - {toNullableVec({{}, 0, 0, 0, 0, 0, 0})}); + request = getRequest({And(col(col_names[0]), col(col_names[0])), col(col_names[4])}, col_names[4]); + executeWithConcurrency(request, + {toNullableVec({{}, 0, 0, 0, 0, 0, 0}), + toNullableVec(col_names[4], col4_sorted_asc)}); - request = getRequest({And(col(col_names[0]), col(col_names[1]))}, "and(col0, col1)"); - executeStreams(request, - {toNullableVec({0, 0, 0, 0, 0, 0, 0})}); + request = getRequest({And(col(col_names[0]), col(col_names[1])), col(col_names[4])}, col_names[4]); + executeWithConcurrency(request, + {toNullableVec({0, 0, 0, 0, 0, 0, 0}), + toNullableVec(col_names[4], col4_sorted_asc)}); /// Data type: TypeLong - request = getRequest({And(col(col_names[3]), col(col_names[4]))}, "and(col3, col4)"); - executeStreams(request, - {toNullableVec({{}, {}, 0, 0, 0, 1, 1})}); + request = getRequest({And(col(col_names[3]), col(col_names[4])), col(col_names[4])}, col_names[4]); + executeWithConcurrency(request, + {toNullableVec({{}, 1, 0, 0, {}, 1, 0}), + toNullableVec(col_names[4], col4_sorted_asc)}); /// Test "not" function /// Data type: TypeString - request = getRequest({NOT(col(col_names[0])), NOT(col(col_names[1])), NOT(col(col_names[2]))}, "not(col2)"); - executeStreams(request, - {toNullableVec({1, 1, 1, 1, 1, {}, 1}), - toNullableVec({1, 1, 1, {}, 1, 1, 1}), - toNullableVec({{}, {}, 1, 1, 1, 1, 1})}); + request = getRequest({NOT(col(col_names[0])), NOT(col(col_names[1])), NOT(col(col_names[2])), col(col_names[4])}, col_names[4]); + executeWithConcurrency(request, + {toNullableVec({{}, 1, 1, 1, 1, 1, 1}), + toNullableVec({1, 1, 1, 1, {}, 1, 1}), + toNullableVec({1, {}, 1, 1, 1, 1, {}}), + toNullableVec(col_names[4], col4_sorted_asc)}); /// Data type: TypeLong - request = getRequest({NOT(col(col_names[3])), NOT(col(col_names[4]))}, "not(col3)"); - executeStreams(request, - {toNullableVec({{}, {}, 0, 0, 0, 1, 1}), - toNullableVec({{}, 0, 1, 0, 0, 0, 0})}); + request = getRequest({NOT(col(col_names[3])), NOT(col(col_names[4])), col(col_names[4])}, col_names[4]); + executeWithConcurrency(request, + {toNullableVec({{}, 0, 1, 0, {}, 0, 1}), + toNullableVec({{}, 0, 0, 1, 0, 0, 0}), + toNullableVec(col_names[4], col4_sorted_asc)}); /// TODO more functions... } From ef6d9fe9a9e5337519be98cb4ed012157e661e94 Mon Sep 17 00:00:00 2001 From: xzhangxian1008 Date: Tue, 28 Jun 2022 10:41:40 +0800 Subject: [PATCH 11/11] update --- dbms/src/Flash/tests/gtest_limit_executor.cpp | 4 +- .../Flash/tests/gtest_projection_executor.cpp | 36 ++++++++--------- dbms/src/Flash/tests/gtest_topn_executor.cpp | 39 ++++++++----------- 3 files changed, 36 insertions(+), 43 deletions(-) diff --git a/dbms/src/Flash/tests/gtest_limit_executor.cpp b/dbms/src/Flash/tests/gtest_limit_executor.cpp index 63fc0ce2833..e4a3aa5db5e 100644 --- a/dbms/src/Flash/tests/gtest_limit_executor.cpp +++ b/dbms/src/Flash/tests/gtest_limit_executor.cpp @@ -35,7 +35,7 @@ class ExecutorLimitTestRunner : public DB::tests::ExecutorTest {toNullableVec(col_name, col0)}); } - std::shared_ptr getRequest(size_t limit_num) + std::shared_ptr buildDAGRequest(size_t limit_num) { return context.scan(db_name, table_name).limit(limit_num).build(context); } @@ -59,7 +59,7 @@ try { if (limit_num == col_data_num + 3) limit_num = INT_MAX; - request = getRequest(limit_num); + request = buildDAGRequest(limit_num); if (limit_num == 0) expect_cols = {}; diff --git a/dbms/src/Flash/tests/gtest_projection_executor.cpp b/dbms/src/Flash/tests/gtest_projection_executor.cpp index e1bc7e6d811..4f6401eb483 100644 --- a/dbms/src/Flash/tests/gtest_projection_executor.cpp +++ b/dbms/src/Flash/tests/gtest_projection_executor.cpp @@ -44,7 +44,7 @@ class ExecutorProjectionTestRunner : public DB::tests::ExecutorTest } template - std::shared_ptr getRequest(T param, const String & sort_col) + std::shared_ptr buildDAGRequest(T param, const String & sort_col) { /// topN is introduced, so that we can get stable results in concurrency environment. return context.scan(db_name, table_name).project(param).topN(sort_col, false, 100).build(context); @@ -87,11 +87,11 @@ TEST_F(ExecutorProjectionTestRunner, Projection) try { /// Check single column - auto request = getRequest({col_names[4]}, col_names[4]); + auto request = buildDAGRequest({col_names[4]}, col_names[4]); executeWithConcurrency(request, {toNullableVec(col_names[4], col4_sorted_asc)}); /// Check multi columns - request = getRequest({col_names[0], col_names[4]}, col_names[4]); + request = buildDAGRequest({col_names[0], col_names[4]}, col_names[4]); executeWithConcurrency(request, { toNullableVec(col_names[0], col0_sorted_asc), @@ -99,14 +99,14 @@ try }); /// Check multi columns - request = getRequest({col_names[0], col_names[1], col_names[4]}, col_names[4]); + request = buildDAGRequest({col_names[0], col_names[1], col_names[4]}, col_names[4]); executeWithConcurrency(request, {toNullableVec(col_names[0], col0_sorted_asc), toNullableVec(col_names[1], col1_sorted_asc), toNullableVec(col_names[4], col4_sorted_asc)}); /// Check duplicate columns - request = getRequest({col_names[4], col_names[4], col_names[4]}, col_names[4]); + request = buildDAGRequest({col_names[4], col_names[4], col_names[4]}, col_names[4]); executeWithConcurrency(request, {toNullableVec(col_names[4], col4_sorted_asc), toNullableVec(col_names[4], col4_sorted_asc), @@ -125,7 +125,7 @@ try columns.push_back(expect_column); } - request = getRequest(projection_input, col_names[4]); + request = buildDAGRequest(projection_input, col_names[4]); executeWithConcurrency(request, columns); } } @@ -139,18 +139,18 @@ try /// Test "equal" function /// Data type: TypeString - request = getRequest({eq(col(col_names[0]), col(col_names[0])), col(col_names[4])}, col_names[4]); + request = buildDAGRequest({eq(col(col_names[0]), col(col_names[0])), col(col_names[4])}, col_names[4]); executeWithConcurrency(request, {toNullableVec({{}, 1, 1, 1, 1, 1, 1}), toNullableVec(col_names[4], col4_sorted_asc)}); - request = getRequest({eq(col(col_names[0]), col(col_names[1])), col(col_names[4])}, col_names[4]); + request = buildDAGRequest({eq(col(col_names[0]), col(col_names[1])), col(col_names[4])}, col_names[4]); executeWithConcurrency(request, {toNullableVec({{}, 0, 1, 0, {}, 0, 0}), toNullableVec(col_names[4], col4_sorted_asc)}); /// Data type: TypeLong - request = getRequest({eq(col(col_names[3]), col(col_names[4])), col(col_names[4])}, col_names[4]); + request = buildDAGRequest({eq(col(col_names[3]), col(col_names[4])), col(col_names[4])}, col_names[4]); executeWithConcurrency(request, {toNullableVec({{}, 0, 0, 0, {}, 1, 0}), toNullableVec(col_names[4], col4_sorted_asc)}); @@ -159,23 +159,23 @@ try /// Test "greater" function /// Data type: TypeString - request = getRequest({gt(col(col_names[0]), col(col_names[1])), col(col_names[4])}, col_names[4]); + request = buildDAGRequest({gt(col(col_names[0]), col(col_names[1])), col(col_names[4])}, col_names[4]); executeWithConcurrency(request, {toNullableVec({{}, 0, 0, 0, {}, 0, 0}), toNullableVec(col_names[4], col4_sorted_asc)}); - request = getRequest({gt(col(col_names[1]), col(col_names[0])), col(col_names[4])}, col_names[4]); + request = buildDAGRequest({gt(col(col_names[1]), col(col_names[0])), col(col_names[4])}, col_names[4]); executeWithConcurrency(request, {toNullableVec({{}, 1, 0, 1, {}, 1, 1}), toNullableVec(col_names[4], col4_sorted_asc)}); /// Data type: TypeLong - request = getRequest({gt(col(col_names[3]), col(col_names[4])), col(col_names[4])}, col_names[4]); + request = buildDAGRequest({gt(col(col_names[3]), col(col_names[4])), col(col_names[4])}, col_names[4]); executeWithConcurrency(request, {toNullableVec({{}, 0, 1, 1, {}, 0, 0}), toNullableVec(col_names[4], col4_sorted_asc)}); - request = getRequest({gt(col(col_names[4]), col(col_names[3])), col(col_names[4])}, col_names[4]); + request = buildDAGRequest({gt(col(col_names[4]), col(col_names[3])), col(col_names[4])}, col_names[4]); executeWithConcurrency(request, {toNullableVec({{}, 1, 0, 0, {}, 0, 1}), toNullableVec(col_names[4], col4_sorted_asc)}); @@ -184,18 +184,18 @@ try /// Test "and" function /// Data type: TypeString - request = getRequest({And(col(col_names[0]), col(col_names[0])), col(col_names[4])}, col_names[4]); + request = buildDAGRequest({And(col(col_names[0]), col(col_names[0])), col(col_names[4])}, col_names[4]); executeWithConcurrency(request, {toNullableVec({{}, 0, 0, 0, 0, 0, 0}), toNullableVec(col_names[4], col4_sorted_asc)}); - request = getRequest({And(col(col_names[0]), col(col_names[1])), col(col_names[4])}, col_names[4]); + request = buildDAGRequest({And(col(col_names[0]), col(col_names[1])), col(col_names[4])}, col_names[4]); executeWithConcurrency(request, {toNullableVec({0, 0, 0, 0, 0, 0, 0}), toNullableVec(col_names[4], col4_sorted_asc)}); /// Data type: TypeLong - request = getRequest({And(col(col_names[3]), col(col_names[4])), col(col_names[4])}, col_names[4]); + request = buildDAGRequest({And(col(col_names[3]), col(col_names[4])), col(col_names[4])}, col_names[4]); executeWithConcurrency(request, {toNullableVec({{}, 1, 0, 0, {}, 1, 0}), toNullableVec(col_names[4], col4_sorted_asc)}); @@ -203,7 +203,7 @@ try /// Test "not" function /// Data type: TypeString - request = getRequest({NOT(col(col_names[0])), NOT(col(col_names[1])), NOT(col(col_names[2])), col(col_names[4])}, col_names[4]); + request = buildDAGRequest({NOT(col(col_names[0])), NOT(col(col_names[1])), NOT(col(col_names[2])), col(col_names[4])}, col_names[4]); executeWithConcurrency(request, {toNullableVec({{}, 1, 1, 1, 1, 1, 1}), toNullableVec({1, 1, 1, 1, {}, 1, 1}), @@ -211,7 +211,7 @@ try toNullableVec(col_names[4], col4_sorted_asc)}); /// Data type: TypeLong - request = getRequest({NOT(col(col_names[3])), NOT(col(col_names[4])), col(col_names[4])}, col_names[4]); + request = buildDAGRequest({NOT(col(col_names[3])), NOT(col(col_names[4])), col(col_names[4])}, col_names[4]); executeWithConcurrency(request, {toNullableVec({{}, 0, 1, 0, {}, 0, 1}), toNullableVec({{}, 0, 0, 1, 0, 0, 0}), diff --git a/dbms/src/Flash/tests/gtest_topn_executor.cpp b/dbms/src/Flash/tests/gtest_topn_executor.cpp index 1faf935d5c1..0e55702795d 100644 --- a/dbms/src/Flash/tests/gtest_topn_executor.cpp +++ b/dbms/src/Flash/tests/gtest_topn_executor.cpp @@ -47,12 +47,12 @@ class ExecutorTopNTestRunner : public DB::tests::ExecutorTest toNullableVec(col_name[3], c0l_salary)}); } - std::shared_ptr getRequest(const String & table_name, const String & col_name, bool is_desc, int limit_num) + std::shared_ptr buildDAGRequest(const String & table_name, const String & col_name, bool is_desc, int limit_num) { return context.scan(db_name, table_name).topN(col_name, is_desc, limit_num).build(context); } - std::shared_ptr getRequest(const String & table_name, MockOrderByItems order_by_items, int limit, MockAsts func_proj_ast = {}, MockColumnNames out_proj_ast = {}) + std::shared_ptr buildDAGRequest(const String & table_name, MockOrderByItems order_by_items, int limit, MockAsts func_proj_ast = {}, MockColumnNames out_proj_ast = {}) { if (func_proj_ast.size() == 0) return context.scan(db_name, table_name).topN(order_by_items, limit).build(context); @@ -95,7 +95,7 @@ try for (size_t limit_num = 0; limit_num <= col_data_num + 5; ++limit_num) { - request = getRequest(table_single_name, single_col_name, is_desc, limit_num); + request = buildDAGRequest(table_single_name, single_col_name, is_desc, limit_num); expect_cols.clear(); if (limit_num == 0 || limit_num > col_data_num) @@ -126,27 +126,20 @@ try toNullableVec(col_name[2], ColumnWithString{"china", "china", "usa", "china", "korea", "usa"}), toNullableVec(col_name[3], ColumnWithInt32{-300, {}, {}, 900, 1300, 0})}}; - MockOrderByItems order_by_items; - - { + std::vector order_by_items{ /// select * from clerk order by age DESC, gender DESC; - order_by_items = {MockOrderByItem(col_name[0], true), MockOrderByItem(col_name[1], true)}; - request = getRequest(table_name, order_by_items, 100); - executeStreams(request, expect_cols[0]); - } - - { + {MockOrderByItem(col_name[0], true), MockOrderByItem(col_name[1], true)}, /// select * from clerk order by gender DESC, salary ASC; - order_by_items = {MockOrderByItem(col_name[1], true), MockOrderByItem(col_name[3], false)}; - request = getRequest(table_name, order_by_items, 100); - executeStreams(request, expect_cols[1]); - } + {MockOrderByItem(col_name[1], true), MockOrderByItem(col_name[3], false)}, + /// select * from clerk order by gender DESC, country ASC, salary DESC; + {MockOrderByItem(col_name[1], true), MockOrderByItem(col_name[2], false), MockOrderByItem(col_name[3], true)}}; + size_t test_num = expect_cols.size(); + + for (size_t i = 0; i < test_num; ++i) { - /// select * from clerk order by gender DESC, country ASC, salary DESC; - order_by_items = {MockOrderByItem(col_name[1], true), MockOrderByItem(col_name[2], false), MockOrderByItem(col_name[3], true)}; - request = getRequest(table_name, order_by_items, 100); - executeStreams(request, expect_cols[2]); + request = buildDAGRequest(table_name, order_by_items[i], 100); + executeStreams(request, expect_cols[i]); } } } @@ -179,7 +172,7 @@ try func_ast = And(col(col_name[0]), col(col_name[3])); func_projection = {col0_ast, col1_ast, col2_ast, col3_ast, func_ast}; - request = getRequest(table_name, order_by_items, 100, func_projection, output_projection); + request = buildDAGRequest(table_name, order_by_items, 100, func_projection, output_projection); executeStreams(request, expect_cols[0]); } } @@ -197,7 +190,7 @@ try func_ast = eq(col(col_name[0]), col(col_name[3])); func_projection = {col0_ast, col1_ast, col2_ast, col3_ast, func_ast}; - request = getRequest(table_name, order_by_items, 100, func_projection, output_projection); + request = buildDAGRequest(table_name, order_by_items, 100, func_projection, output_projection); executeStreams(request, expect_cols[0]); } } @@ -215,7 +208,7 @@ try func_ast = gt(col(col_name[0]), col(col_name[3])); func_projection = {col0_ast, col1_ast, col2_ast, col3_ast, func_ast}; - request = getRequest(table_name, order_by_items, 100, func_projection, output_projection); + request = buildDAGRequest(table_name, order_by_items, 100, func_projection, output_projection); executeStreams(request, expect_cols[0]); } }