diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q05.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q05.plan.txt index ebed85d1de97..aeff998bfd1f 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q05.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q05.plan.txt @@ -1,57 +1,56 @@ -local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - final aggregation over (channel$gid, groupid, id$gid) - local exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) - remote exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) - partial aggregation over (channel$gid, groupid, id$gid) - local exchange (REPARTITION, ROUND_ROBIN, []) - final aggregation over (s_store_id) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [s_store_id]) - partial aggregation over (s_store_id) +remote exchange (GATHER, SINGLE, []) + final aggregation over (channel$gid, groupid, id$gid) + local exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) + remote exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) + partial aggregation over (channel$gid, groupid, id$gid) + local exchange (REPARTITION, ROUND_ROBIN, []) + final aggregation over (s_store_id) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [s_store_id]) + partial aggregation over (s_store_id) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - remote exchange (REPARTITION, ROUND_ROBIN, []) - scan store_sales - scan store_returns - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + remote exchange (REPARTITION, ROUND_ROBIN, []) + scan store_sales + scan store_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan store - final aggregation over (cp_catalog_page_id) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [cp_catalog_page_id]) - partial aggregation over (cp_catalog_page_id) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan store + final aggregation over (cp_catalog_page_id) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [cp_catalog_page_id]) + partial aggregation over (cp_catalog_page_id) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - remote exchange (REPARTITION, ROUND_ROBIN, []) - scan catalog_sales - scan catalog_returns - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + remote exchange (REPARTITION, ROUND_ROBIN, []) + scan catalog_sales + scan catalog_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan catalog_page - final aggregation over (web_site_id) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [web_site_id]) - partial aggregation over (web_site_id) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan catalog_page + final aggregation over (web_site_id) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [web_site_id]) + partial aggregation over (web_site_id) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - remote exchange (REPARTITION, ROUND_ROBIN, []) - scan web_sales - join (RIGHT, PARTITIONED): - remote exchange (REPARTITION, HASH, [ws_item_sk_216, ws_order_number_230]) - scan web_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [wr_item_sk, wr_order_number]) - scan web_returns - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + remote exchange (REPARTITION, ROUND_ROBIN, []) + scan web_sales + join (RIGHT, PARTITIONED): + remote exchange (REPARTITION, HASH, [ws_item_sk_216, ws_order_number_230]) + scan web_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [wr_item_sk, wr_order_number]) + scan web_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan web_site + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan web_site diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q14_1.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q14_1.plan.txt index 9d5966e6652b..f86567d03c0b 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q14_1.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q14_1.plan.txt @@ -1,244 +1,243 @@ -local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - final aggregation over (channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743) - local exchange (REPARTITION, HASH, [channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743]) - remote exchange (REPARTITION, HASH, [channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743]) - partial aggregation over (channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743) - local exchange (REPARTITION, ROUND_ROBIN, []) - cross join: - final aggregation over (i_brand_id, i_category_id, i_class_id) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_brand_id, i_category_id, i_class_id]) - partial aggregation over (i_brand_id, i_category_id, i_class_id) - semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, [ss_item_sk]) +remote exchange (GATHER, SINGLE, []) + final aggregation over (channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743) + local exchange (REPARTITION, HASH, [channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743]) + remote exchange (REPARTITION, HASH, [channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743]) + partial aggregation over (channel$gid, groupid, i_brand_id$gid_1742, i_category_id$gid_1744, i_class_id$gid_1743) + local exchange (REPARTITION, ROUND_ROBIN, []) + cross join: + final aggregation over (i_brand_id, i_category_id, i_class_id) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [i_brand_id, i_category_id, i_class_id]) + partial aggregation over (i_brand_id, i_category_id, i_class_id) + semijoin (PARTITIONED): + remote exchange (REPARTITION, HASH, [ss_item_sk]) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_item_sk_1]) - join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, [i_brand_id_8, i_category_id_12, i_class_id_10]) - scan item - final aggregation over (expr_216, expr_217, expr_218) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_brand_id_53, i_category_id_57, i_class_id_55]) - partial aggregation over (i_brand_id_53, i_category_id_57, i_class_id_55) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [i_item_sk_1]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, [i_brand_id_8, i_category_id_12, i_class_id_10]) + scan item + final aggregation over (expr_216, expr_217, expr_218) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [i_brand_id_53, i_category_id_57, i_class_id_55]) + partial aggregation over (i_brand_id_53, i_category_id_57, i_class_id_55) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item - remote exchange (REPARTITION, HASH, [i_brand_id_108, i_category_id_112, i_class_id_110]) - partial aggregation over (i_brand_id_108, i_category_id_112, i_class_id_110) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + remote exchange (REPARTITION, HASH, [i_brand_id_108, i_category_id_112, i_class_id_110]) + partial aggregation over (i_brand_id_108, i_category_id_112, i_class_id_110) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan catalog_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item - remote exchange (REPARTITION, HASH, [i_brand_id_167, i_category_id_171, i_class_id_169]) - partial aggregation over (i_brand_id_167, i_category_id_171, i_class_id_169) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + remote exchange (REPARTITION, HASH, [i_brand_id_167, i_category_id_171, i_class_id_169]) + partial aggregation over (i_brand_id_167, i_category_id_171, i_class_id_169) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan web_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + final aggregation over () + local exchange (GATHER, SINGLE, []) + remote exchange (GATHER, SINGLE, []) + partial aggregation over () + join (INNER, REPLICATED): + scan store_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + partial aggregation over () + join (INNER, REPLICATED): + scan catalog_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + partial aggregation over () + join (INNER, REPLICATED): + scan web_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + cross join: + final aggregation over (i_brand_id_508, i_category_id_512, i_class_id_510) local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - final aggregation over () - local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - partial aggregation over () - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - partial aggregation over () + remote exchange (REPARTITION, HASH, [i_brand_id_508, i_category_id_512, i_class_id_510]) + partial aggregation over (i_brand_id_508, i_category_id_512, i_class_id_510) + semijoin (PARTITIONED): + remote exchange (REPARTITION, HASH, [cs_item_sk_482]) + join (INNER, REPLICATED): join (INNER, REPLICATED): scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim - partial aggregation over () - join (INNER, REPLICATED): - scan web_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - cross join: - final aggregation over (i_brand_id_508, i_category_id_512, i_class_id_510) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_brand_id_508, i_category_id_512, i_class_id_510]) - partial aggregation over (i_brand_id_508, i_category_id_512, i_class_id_510) - semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, [cs_item_sk_482]) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan catalog_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [i_item_sk_552]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, [i_brand_id_559, i_category_id_563, i_class_id_561]) + scan item + final aggregation over (expr_836, expr_837, expr_838) local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan item - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_item_sk_552]) - join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, [i_brand_id_559, i_category_id_563, i_class_id_561]) - scan item - final aggregation over (expr_836, expr_837, expr_838) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_brand_id_604, i_category_id_608, i_class_id_606]) - partial aggregation over (i_brand_id_604, i_category_id_608, i_class_id_606) + remote exchange (REPARTITION, HASH, [i_brand_id_604, i_category_id_608, i_class_id_606]) + partial aggregation over (i_brand_id_604, i_category_id_608, i_class_id_606) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item - remote exchange (REPARTITION, HASH, [i_brand_id_694, i_category_id_698, i_class_id_696]) - partial aggregation over (i_brand_id_694, i_category_id_698, i_class_id_696) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + remote exchange (REPARTITION, HASH, [i_brand_id_694, i_category_id_698, i_class_id_696]) + partial aggregation over (i_brand_id_694, i_category_id_698, i_class_id_696) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan catalog_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item - remote exchange (REPARTITION, HASH, [i_brand_id_787, i_category_id_791, i_class_id_789]) - partial aggregation over (i_brand_id_787, i_category_id_791, i_class_id_789) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + remote exchange (REPARTITION, HASH, [i_brand_id_787, i_category_id_791, i_class_id_789]) + partial aggregation over (i_brand_id_787, i_category_id_791, i_class_id_789) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan web_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + final aggregation over () + local exchange (GATHER, SINGLE, []) + remote exchange (GATHER, SINGLE, []) + partial aggregation over () + join (INNER, REPLICATED): + scan store_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + partial aggregation over () + join (INNER, REPLICATED): + scan catalog_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + partial aggregation over () + join (INNER, REPLICATED): + scan web_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + cross join: + final aggregation over (i_brand_id_1135, i_category_id_1139, i_class_id_1137) local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - final aggregation over () - local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - partial aggregation over () - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - partial aggregation over () - join (INNER, REPLICATED): - scan catalog_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - partial aggregation over () + remote exchange (REPARTITION, HASH, [i_brand_id_1135, i_category_id_1139, i_class_id_1137]) + partial aggregation over (i_brand_id_1135, i_category_id_1139, i_class_id_1137) + semijoin (PARTITIONED): + remote exchange (REPARTITION, HASH, [ws_item_sk_1097]) + join (INNER, REPLICATED): join (INNER, REPLICATED): scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim - cross join: - final aggregation over (i_brand_id_1135, i_category_id_1139, i_class_id_1137) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_brand_id_1135, i_category_id_1139, i_class_id_1137]) - partial aggregation over (i_brand_id_1135, i_category_id_1139, i_class_id_1137) - semijoin (PARTITIONED): - remote exchange (REPARTITION, HASH, [ws_item_sk_1097]) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan web_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [i_item_sk_1179]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, [i_brand_id_1186, i_category_id_1190, i_class_id_1188]) + scan item + final aggregation over (expr_1463, expr_1464, expr_1465) local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan item - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_item_sk_1179]) - join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, [i_brand_id_1186, i_category_id_1190, i_class_id_1188]) - scan item - final aggregation over (expr_1463, expr_1464, expr_1465) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_brand_id_1231, i_category_id_1235, i_class_id_1233]) - partial aggregation over (i_brand_id_1231, i_category_id_1235, i_class_id_1233) + remote exchange (REPARTITION, HASH, [i_brand_id_1231, i_category_id_1235, i_class_id_1233]) + partial aggregation over (i_brand_id_1231, i_category_id_1235, i_class_id_1233) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item - remote exchange (REPARTITION, HASH, [i_brand_id_1321, i_category_id_1325, i_class_id_1323]) - partial aggregation over (i_brand_id_1321, i_category_id_1325, i_class_id_1323) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + remote exchange (REPARTITION, HASH, [i_brand_id_1321, i_category_id_1325, i_class_id_1323]) + partial aggregation over (i_brand_id_1321, i_category_id_1325, i_class_id_1323) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan catalog_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item - remote exchange (REPARTITION, HASH, [i_brand_id_1414, i_category_id_1418, i_class_id_1416]) - partial aggregation over (i_brand_id_1414, i_category_id_1418, i_class_id_1416) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + remote exchange (REPARTITION, HASH, [i_brand_id_1414, i_category_id_1418, i_class_id_1416]) + partial aggregation over (i_brand_id_1414, i_category_id_1418, i_class_id_1416) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan web_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - final aggregation over () - local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - partial aggregation over () - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - partial aggregation over () - join (INNER, REPLICATED): - scan catalog_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - partial aggregation over () - join (INNER, REPLICATED): - scan web_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + final aggregation over () + local exchange (GATHER, SINGLE, []) + remote exchange (GATHER, SINGLE, []) + partial aggregation over () + join (INNER, REPLICATED): + scan store_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + partial aggregation over () + join (INNER, REPLICATED): + scan catalog_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + partial aggregation over () + join (INNER, REPLICATED): + scan web_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q18.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q18.plan.txt index 7d4c99ffd8b9..c9d1e8288375 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q18.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q18.plan.txt @@ -1,33 +1,32 @@ -local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - final aggregation over (ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid) - local exchange (REPARTITION, HASH, [ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid]) - remote exchange (REPARTITION, HASH, [ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid]) - partial aggregation over (ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid) +remote exchange (GATHER, SINGLE, []) + final aggregation over (ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid) + local exchange (REPARTITION, HASH, [ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid]) + remote exchange (REPARTITION, HASH, [ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid]) + partial aggregation over (ca_country$gid, ca_county$gid, ca_state$gid, groupid, i_item_id$gid) + join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan catalog_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan customer_demographics + scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, [c_current_cdemo_sk]) - join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, [c_current_addr_sk]) - scan customer - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [ca_address_sk]) - scan customer_address - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [cd_demo_sk_0]) - scan customer_demographics + scan customer_demographics local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, [c_current_cdemo_sk]) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, [c_current_addr_sk]) + scan customer + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [ca_address_sk]) + scan customer_address + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [cd_demo_sk_0]) + scan customer_demographics local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q22.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q22.plan.txt index 38c414f6d58f..28ca93776bcf 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q22.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q22.plan.txt @@ -1,15 +1,14 @@ -local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - final aggregation over (groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid) - local exchange (REPARTITION, HASH, [groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid]) - remote exchange (REPARTITION, HASH, [groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid]) - partial aggregation over (groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid) +remote exchange (GATHER, SINGLE, []) + final aggregation over (groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid) + local exchange (REPARTITION, HASH, [groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid]) + remote exchange (REPARTITION, HASH, [groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid]) + partial aggregation over (groupid, i_brand$gid, i_category$gid, i_class$gid, i_product_name$gid) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan inventory - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan inventory local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan item diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q27.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q27.plan.txt index 042bd1c80518..7e3746fafa38 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q27.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q27.plan.txt @@ -1,24 +1,23 @@ -local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - final aggregation over (groupid, i_item_id$gid, s_state$gid) - local exchange (REPARTITION, HASH, [groupid, i_item_id$gid, s_state$gid]) - remote exchange (REPARTITION, HASH, [groupid, i_item_id$gid, s_state$gid]) - partial aggregation over (groupid, i_item_id$gid, s_state$gid) - join (INNER, PARTITIONED): - remote exchange (REPARTITION, HASH, [ss_item_sk]) +remote exchange (GATHER, SINGLE, []) + final aggregation over (groupid, i_item_id$gid, s_state$gid) + local exchange (REPARTITION, HASH, [groupid, i_item_id$gid, s_state$gid]) + remote exchange (REPARTITION, HASH, [groupid, i_item_id$gid, s_state$gid]) + partial aggregation over (groupid, i_item_id$gid, s_state$gid) + join (INNER, PARTITIONED): + remote exchange (REPARTITION, HASH, [ss_item_sk]) + join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan customer_demographics + scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan store + scan customer_demographics local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [i_item_sk]) - scan item + scan store + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [i_item_sk]) + scan item diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q77.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q77.plan.txt index 765e1f82b328..ea5a1a88c265 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q77.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q77.plan.txt @@ -1,82 +1,81 @@ -local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - final aggregation over (channel$gid, groupid, id$gid) - local exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) - remote exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) - partial aggregation over (channel$gid, groupid, id$gid) - local exchange (REPARTITION, ROUND_ROBIN, []) - join (LEFT, PARTITIONED): - final aggregation over (s_store_sk) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [s_store_sk]) - partial aggregation over (s_store_sk) - join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan store - final aggregation over (s_store_sk_46) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [s_store_sk_46]) - partial aggregation over (s_store_sk_46) +remote exchange (GATHER, SINGLE, []) + final aggregation over (channel$gid, groupid, id$gid) + local exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) + remote exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) + partial aggregation over (channel$gid, groupid, id$gid) + local exchange (REPARTITION, ROUND_ROBIN, []) + join (LEFT, PARTITIONED): + final aggregation over (s_store_sk) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [s_store_sk]) + partial aggregation over (s_store_sk) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan store_returns - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan store - cross join: - final aggregation over (cs_call_center_sk) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [cs_call_center_sk]) - partial aggregation over (cs_call_center_sk) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan store + final aggregation over (s_store_sk_46) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [s_store_sk_46]) + partial aggregation over (s_store_sk_46) + join (INNER, REPLICATED): join (INNER, REPLICATED): - scan catalog_sales + scan store_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan store + cross join: + final aggregation over (cs_call_center_sk) local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - final aggregation over (cr_call_center_sk) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [cr_call_center_sk]) - partial aggregation over (cr_call_center_sk) - join (INNER, REPLICATED): - scan catalog_returns - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim - join (LEFT, PARTITIONED): - final aggregation over (wp_web_page_sk) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [wp_web_page_sk]) - partial aggregation over (wp_web_page_sk) - join (INNER, REPLICATED): + remote exchange (REPARTITION, HASH, [cs_call_center_sk]) + partial aggregation over (cs_call_center_sk) + join (INNER, REPLICATED): + scan catalog_sales + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + final aggregation over (cr_call_center_sk) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [cr_call_center_sk]) + partial aggregation over (cr_call_center_sk) join (INNER, REPLICATED): - scan web_sales + scan catalog_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) scan date_dim + join (LEFT, PARTITIONED): + final aggregation over (wp_web_page_sk) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [wp_web_page_sk]) + partial aggregation over (wp_web_page_sk) + join (INNER, REPLICATED): + join (INNER, REPLICATED): + scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan web_page - final aggregation over (wp_web_page_sk_298) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [wp_web_page_sk_298]) - partial aggregation over (wp_web_page_sk_298) + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan web_page + final aggregation over (wp_web_page_sk_298) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [wp_web_page_sk_298]) + partial aggregation over (wp_web_page_sk_298) + join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - scan web_returns - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan web_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan web_page + scan date_dim + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan web_page diff --git a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q80.plan.txt b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q80.plan.txt index 99c9b4a65e45..b198e899773a 100644 --- a/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q80.plan.txt +++ b/presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q80.plan.txt @@ -1,82 +1,81 @@ -local exchange (GATHER, SINGLE, []) - remote exchange (GATHER, SINGLE, []) - final aggregation over (channel$gid, groupid, id$gid) - local exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) - remote exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) - partial aggregation over (channel$gid, groupid, id$gid) - local exchange (REPARTITION, ROUND_ROBIN, []) - final aggregation over (s_store_id) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [s_store_id]) - partial aggregation over (s_store_id) +remote exchange (GATHER, SINGLE, []) + final aggregation over (channel$gid, groupid, id$gid) + local exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) + remote exchange (REPARTITION, HASH, [channel$gid, groupid, id$gid]) + partial aggregation over (channel$gid, groupid, id$gid) + local exchange (REPARTITION, ROUND_ROBIN, []) + final aggregation over (s_store_id) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [s_store_id]) + partial aggregation over (s_store_id) + join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - join (LEFT, REPLICATED): - scan store_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan store_returns + join (LEFT, REPLICATED): + scan store_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan store_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item + scan date_dim local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan promotion + scan item local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan store - final aggregation over (cp_catalog_page_id) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [cp_catalog_page_id]) - partial aggregation over (cp_catalog_page_id) + scan promotion + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan store + final aggregation over (cp_catalog_page_id) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [cp_catalog_page_id]) + partial aggregation over (cp_catalog_page_id) + join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - join (LEFT, REPLICATED): - scan catalog_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan catalog_returns + join (LEFT, REPLICATED): + scan catalog_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan catalog_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item + scan date_dim local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan promotion + scan item local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan catalog_page - final aggregation over (web_site_id) - local exchange (GATHER, SINGLE, []) - remote exchange (REPARTITION, HASH, [web_site_id]) - partial aggregation over (web_site_id) + scan promotion + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan catalog_page + final aggregation over (web_site_id) + local exchange (GATHER, SINGLE, []) + remote exchange (REPARTITION, HASH, [web_site_id]) + partial aggregation over (web_site_id) + join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): join (INNER, REPLICATED): - join (INNER, REPLICATED): - join (LEFT, REPLICATED): - scan web_sales - local exchange (GATHER, SINGLE, []) - remote exchange (REPLICATE, BROADCAST, []) - scan web_returns + join (LEFT, REPLICATED): + scan web_sales local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan date_dim + scan web_returns local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan item + scan date_dim local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan promotion + scan item local exchange (GATHER, SINGLE, []) remote exchange (REPLICATE, BROADCAST, []) - scan web_site + scan promotion + local exchange (GATHER, SINGLE, []) + remote exchange (REPLICATE, BROADCAST, []) + scan web_site diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java index f7d4e66159a6..b7c4959bd483 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java @@ -35,8 +35,10 @@ import com.facebook.presto.sql.planner.iterative.rule.DetermineJoinDistributionType; import com.facebook.presto.sql.planner.iterative.rule.DetermineSemiJoinDistributionType; import com.facebook.presto.sql.planner.iterative.rule.EliminateCrossJoins; +import com.facebook.presto.sql.planner.iterative.rule.EvaluateZeroDistinctLimit; import com.facebook.presto.sql.planner.iterative.rule.EvaluateZeroLimit; import com.facebook.presto.sql.planner.iterative.rule.EvaluateZeroSample; +import com.facebook.presto.sql.planner.iterative.rule.EvaluateZeroTopN; import com.facebook.presto.sql.planner.iterative.rule.ExtractSpatialJoins; import com.facebook.presto.sql.planner.iterative.rule.GatherAndMergeWindows; import com.facebook.presto.sql.planner.iterative.rule.ImplementBernoulliSampleAsFilter; @@ -87,7 +89,11 @@ import com.facebook.presto.sql.planner.iterative.rule.PushTopNThroughUnion; import com.facebook.presto.sql.planner.iterative.rule.RemoveEmptyDelete; import com.facebook.presto.sql.planner.iterative.rule.RemoveFullSample; +import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantDistinctLimit; import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantIdentityProjections; +import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantLimit; +import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantSort; +import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantTopN; import com.facebook.presto.sql.planner.iterative.rule.RemoveTrivialFilters; import com.facebook.presto.sql.planner.iterative.rule.RemoveUnreferencedScalarApplyNodes; import com.facebook.presto.sql.planner.iterative.rule.RemoveUnreferencedScalarLateralNodes; @@ -180,20 +186,6 @@ public PlanOptimizers( taskCountEstimator); } - @PostConstruct - public void initialize() - { - ruleStats.export(exporter); - optimizerStats.export(exporter); - } - - @PreDestroy - public void destroy() - { - ruleStats.unexport(exporter); - optimizerStats.unexport(exporter); - } - public PlanOptimizers( Metadata metadata, SqlParser sqlParser, @@ -285,11 +277,6 @@ public PlanOptimizers( statsCalculator, estimatedExchangesCostCalculator, new CanonicalizeExpressions().rules()), - new IterativeOptimizer( - ruleStats, - statsCalculator, - estimatedExchangesCostCalculator, - ImmutableSet.of(new EvaluateZeroLimit())), new IterativeOptimizer( ruleStats, statsCalculator, @@ -301,6 +288,9 @@ public PlanOptimizers( new RemoveRedundantIdentityProjections(), new RemoveFullSample(), new EvaluateZeroSample(), + new EvaluateZeroLimit(), + new EvaluateZeroDistinctLimit(), + new EvaluateZeroTopN(), new PushLimitThroughProject(), new MergeLimits(), new MergeLimitWithSort(), @@ -310,6 +300,10 @@ public PlanOptimizers( new PushLimitThroughSemiJoin(), new PushLimitThroughUnion(), new RemoveTrivialFilters(), + new RemoveRedundantLimit(), + new RemoveRedundantSort(), + new RemoveRedundantTopN(), + new RemoveRedundantDistinctLimit(), new ImplementFilteredAggregations(), new SingleDistinctAggregationToGroupBy(), new MultipleDistinctAggregationToMarkDistinct(), @@ -447,7 +441,7 @@ public PlanOptimizers( ImmutableSet.>builder() .addAll(new PushDownDereferences(metadata).rules()) .build()), - new PruneUnreferencedOutputs()); + new PruneUnreferencedOutputs()); builder.add(new IterativeOptimizer( ruleStats, @@ -620,6 +614,20 @@ public PlanOptimizers( this.runtimeOptimizers = runtimeBuilder.build(); } + @PostConstruct + public void initialize() + { + ruleStats.export(exporter); + optimizerStats.export(exporter); + } + + @PreDestroy + public void destroy() + { + ruleStats.unexport(exporter); + optimizerStats.unexport(exporter); + } + public List getPlanningTimeOptimizers() { return planningTimeOptimizers; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroDistinctLimit.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroDistinctLimit.java new file mode 100644 index 000000000000..7ccd6f7f1b11 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroDistinctLimit.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.spi.plan.ValuesNode; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.google.common.collect.ImmutableList; + +import static com.facebook.presto.sql.planner.plan.Patterns.DistinctLimit.count; +import static com.facebook.presto.sql.planner.plan.Patterns.distinctLimit; + +public class EvaluateZeroDistinctLimit + implements Rule +{ + private static final Pattern PATTERN = distinctLimit() + .with(count().equalTo(0L)); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public Result apply(DistinctLimitNode node, Captures captures, Context context) + { + return Result.ofPlanNode(new ValuesNode(node.getId(), node.getOutputVariables(), ImmutableList.of())); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroSample.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroSample.java index bbedf63ae0a8..b4be325a541a 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroSample.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroSample.java @@ -26,6 +26,7 @@ /** * Replaces 0% sample node with empty values node. */ + public class EvaluateZeroSample implements Rule { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroTopN.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroTopN.java new file mode 100644 index 000000000000..5e5c84995e4f --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/EvaluateZeroTopN.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.spi.plan.ValuesNode; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.google.common.collect.ImmutableList; + +import static com.facebook.presto.sql.planner.plan.Patterns.TopN.count; +import static com.facebook.presto.sql.planner.plan.Patterns.topN; + +public class EvaluateZeroTopN + implements Rule +{ + private static final Pattern PATTERN = topN() + .with(count().equalTo(0L)); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public Result apply(TopNNode node, Captures captures, Context context) + { + return Result.ofPlanNode(new ValuesNode(node.getId(), node.getOutputVariables(), ImmutableList.of())); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantDistinctLimit.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantDistinctLimit.java new file mode 100644 index 000000000000..76a71b4c09cb --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantDistinctLimit.java @@ -0,0 +1,71 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.DistinctLimitNode; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.Optional; + +import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; +import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.isAtMost; +import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.isScalar; +import static com.facebook.presto.sql.planner.plan.Patterns.distinctLimit; +import static com.google.common.base.Preconditions.checkArgument; + +/** + * Replace DistinctLimit node + * 1. With a empty ValuesNode when count is 0 + * 2. With a Distinct node when the subplan is guaranteed to produce fewer rows than count + * 3. With its source when the subplan produces only one row + */ +public class RemoveRedundantDistinctLimit + implements Rule +{ + private static final Pattern PATTERN = distinctLimit(); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public Result apply(DistinctLimitNode node, Captures captures, Context context) + { + checkArgument(!node.getHashVariable().isPresent(), "HashSymbol should be empty"); + + if (isScalar(node.getSource(), context.getLookup())) { + return Result.ofPlanNode(node.getSource()); + } + if (isAtMost(node.getSource(), context.getLookup(), node.getLimit())) { + return Result.ofPlanNode(new AggregationNode( + node.getId(), + node.getSource(), + ImmutableMap.of(), + singleGroupingSet(node.getDistinctVariables()), + ImmutableList.of(), + SINGLE, + node.getHashVariable(), + Optional.empty())); + } + return Result.empty(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantLimit.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantLimit.java new file mode 100644 index 000000000000..b62a97f3544c --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantLimit.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.LimitNode; +import com.facebook.presto.sql.planner.iterative.Rule; + +import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.isAtMost; +import static com.facebook.presto.sql.planner.plan.Patterns.limit; + +/** + * Remove Limit node when the subplan is guaranteed to produce fewer rows than the limit and + * replace the plan with empty values if the limit count is 0. + */ +public class RemoveRedundantLimit + implements Rule +{ + // Applies to both LimitNode with ties and LimitNode without ties. + private static final Pattern PATTERN = limit(); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public Result apply(LimitNode limit, Captures captures, Context context) + { + if (isAtMost(limit.getSource(), context.getLookup(), limit.getCount())) { + return Result.ofPlanNode(limit.getSource()); + } + return Result.empty(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantSort.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantSort.java new file mode 100644 index 000000000000..52ec593fb6a2 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantSort.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.planner.plan.SortNode; + +import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.isAtMostScalar; +import static com.facebook.presto.sql.planner.plan.Patterns.sort; + +public class RemoveRedundantSort + implements Rule +{ + private static final Pattern PATTERN = sort(); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public Result apply(SortNode node, Captures captures, Context context) + { + if (isAtMostScalar(node.getSource(), context.getLookup())) { + return Result.ofPlanNode(node.getSource()); + } + return Result.empty(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantTopN.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantTopN.java new file mode 100644 index 000000000000..f0bcd203d214 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveRedundantTopN.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.TopNNode; +import com.facebook.presto.sql.planner.iterative.Rule; +import com.facebook.presto.sql.planner.plan.SortNode; + +import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.isAtMost; +import static com.facebook.presto.sql.planner.optimizations.QueryCardinalityUtil.isAtMostScalar; +import static com.facebook.presto.sql.planner.plan.Patterns.topN; + +/** + * Replace TopN node + * 1. With its source when the subplan is at most one row + * 2. With a Sort node when the subplan is guaranteed to produce fewer rows than N + */ +public class RemoveRedundantTopN + implements Rule +{ + private static final Pattern PATTERN = topN(); + + @Override + public Pattern getPattern() + { + return PATTERN; + } + + @Override + public Result apply(TopNNode node, Captures captures, Context context) + { + if (isAtMostScalar(node.getSource(), context.getLookup())) { + return Result.ofPlanNode(node.getSource()); + } + if (isAtMost(node.getSource(), context.getLookup(), node.getCount())) { + return Result.ofPlanNode(new SortNode(context.getIdAllocator().getNextId(), node.getSource(), node.getOrderingScheme(), false)); + } + return Result.empty(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/QueryCardinalityUtil.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/QueryCardinalityUtil.java index b1f03a51cdcf..843c8b4aac11 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/QueryCardinalityUtil.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/QueryCardinalityUtil.java @@ -24,6 +24,7 @@ import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; +import com.facebook.presto.sql.planner.plan.SampleNode; import com.google.common.collect.Range; import static com.facebook.presto.sql.planner.iterative.Lookup.noLookup; @@ -118,6 +119,15 @@ public Range visitExchange(ExchangeNode node, Void context) return Range.atLeast(0L); } + @Override + public Range visitSample(SampleNode node, Void context) + { + if (node.getSampleRatio() == 0.0) { + return Range.atMost(0L); + } + return Range.atLeast(0L); + } + @Override public Range visitProject(ProjectNode node, Void context) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java index a68b59d7fc2d..175e659f205c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java @@ -16,6 +16,7 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.matching.Property; import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.IntersectNode; import com.facebook.presto.spi.plan.LimitNode; @@ -160,6 +161,11 @@ public static Pattern union() return typeOf(UnionNode.class); } + public static Pattern distinctLimit() + { + return typeOf(DistinctLimitNode.class); + } + public static Pattern intersect() { return typeOf(IntersectNode.class); @@ -197,6 +203,14 @@ public static Property> sources() return property("sources", PlanNode::getSources); } + public static class DistinctLimit + { + public static Property count() + { + return property("limit", DistinctLimitNode::getLimit); + } + } + public static class Aggregation { public static Property> groupingColumns() @@ -266,6 +280,11 @@ public static Property step() { return property("step", TopNNode::getStep); } + + public static Property count() + { + return property("count", TopNNode::getCount); + } } public static class Values diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java index 7e0554d3062f..afba4fefeecc 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestLogicalPlanner.java @@ -17,8 +17,11 @@ import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.analyzer.FeaturesConfig.JoinDistributionType; @@ -33,6 +36,7 @@ import com.facebook.presto.sql.planner.plan.JoinNode; import com.facebook.presto.sql.planner.plan.LateralJoinNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; +import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; import com.facebook.presto.sql.tree.LongLiteral; import com.facebook.presto.tests.QueryTemplate; @@ -59,6 +63,7 @@ import static com.facebook.presto.spi.plan.AggregationNode.Step.FINAL; import static com.facebook.presto.spi.plan.AggregationNode.Step.PARTIAL; import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; +import static com.facebook.presto.sql.planner.LogicalPlanner.Stage.OPTIMIZED; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.aggregation; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.any; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.anyNot; @@ -73,6 +78,7 @@ import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.filter; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.functionCall; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.join; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.limit; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.markDistinct; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.output; @@ -582,7 +588,7 @@ public void testJoinOutputPruning() private void assertPlanContainsNoApplyOrAnyJoin(String sql) { assertFalse( - searchFrom(plan(sql, LogicalPlanner.Stage.OPTIMIZED).getRoot()) + searchFrom(plan(sql, OPTIMIZED).getRoot()) .where(isInstanceOfAny(ApplyNode.class, JoinNode.class, IndexJoinNode.class, SemiJoinNode.class, LateralJoinNode.class)) .matches(), "Unexpected node for query: " + sql); @@ -593,7 +599,7 @@ public void testCorrelatedSubqueries() { assertPlan( "SELECT orderkey FROM orders WHERE 3 = (SELECT orderkey)", - LogicalPlanner.Stage.OPTIMIZED, + OPTIMIZED, any( filter( "X = BIGINT '3'", @@ -751,7 +757,7 @@ public void testDoubleNestedCorrelatedSubqueries() assertPlan( "SELECT orderkey FROM orders o " + "WHERE 3 IN (SELECT o.custkey FROM lineitem l WHERE (SELECT l.orderkey = o.orderkey))", - LogicalPlanner.Stage.OPTIMIZED, + OPTIMIZED, anyTree( filter("OUTER_FILTER", apply(ImmutableList.of("C", "O"), @@ -1075,14 +1081,7 @@ public void testComplexOrderBy() " SUM(REDUCE(col1, ROW(0),(l, r) -> l, x -> 1)) " + " )", output( - project( - exchange( - exchange( - sort( - exchange( - project( - aggregation(ImmutableMap.of(), - project(values("col1"))))))))))); + (values("col1")))); } @Test @@ -1143,4 +1142,102 @@ public void testJoinNullFilters() ImmutableMap.of( "REGION_REGIONKEY", "regionkey")))))); } + + @Test + public void testRedundantLimitNodeRemoval() + { + String query = "SELECT count(*) FROM orders LIMIT 10"; + assertFalse( + searchFrom(plan(query, OPTIMIZED).getRoot()) + .where(LimitNode.class::isInstance) + .matches(), + format("Unexpected limit node for query: '%s'", query)); + + assertPlan( + "SELECT orderkey, count(*) FROM orders GROUP BY orderkey LIMIT 10", + output( + limit(10, + anyTree( + tableScan("orders"))))); + + assertPlan( + "SELECT * FROM (VALUES 1,2,3,4,5,6) AS t1 LIMIT 10", + output( + values(ImmutableList.of("x")))); + } + + @Test + public void testRemoveSingleRowSort() + { + String query = "SELECT count(*) FROM orders ORDER BY 1"; + assertFalse( + searchFrom(plan(query, OPTIMIZED).getRoot()) + .where(isInstanceOfAny(SortNode.class)) + .matches(), + format("Unexpected sort node for query: '%s'", query)); + + assertPlan( + "SELECT orderkey, count(*) FROM orders GROUP BY orderkey ORDER BY 1", + anyTree( + node(SortNode.class, + anyTree( + tableScan("orders"))))); + } + + @Test + public void testRedundantTopNNodeRemoval() + { + String query = "SELECT count(*) FROM orders ORDER BY 1 LIMIT 10"; + assertFalse( + searchFrom(plan(query, OPTIMIZED).getRoot()) + .where(isInstanceOfAny(TopNNode.class, SortNode.class)) + .matches(), + format("Unexpected TopN node for query: '%s'", query)); + + assertPlan( + "SELECT orderkey, count(*) FROM orders GROUP BY orderkey ORDER BY 1 LIMIT 10", + output( + node(TopNNode.class, + anyTree( + tableScan("orders"))))); + + assertPlan( + "SELECT orderkey, count(*) FROM orders GROUP BY orderkey ORDER BY 1 LIMIT 0", + output( + node(ValuesNode.class))); + + query = "SELECT * FROM (VALUES 1,2,3,4,5,6) AS t1 ORDER BY 1 LIMIT 10"; + assertPlan( + query, + output( + node(TopNNode.class, + node(TopNNode.class, + node(ValuesNode.class))))); + } + + @Test + public void testRedundantDistinctLimitNodeRemoval() + { + String query = "SELECT distinct(c) FROM (SELECT count(*) as c FROM orders) LIMIT 10"; + assertFalse( + searchFrom(plan(query, OPTIMIZED).getRoot()) + .where(isInstanceOfAny(DistinctLimitNode.class)) + .matches(), + format("Unexpected DistinctLimit node for query: '%s'", query)); + + assertPlan( + "SELECT distinct(c) FROM (SELECT count(*) as c FROM orders GROUP BY orderkey) LIMIT 10", + output( + node(DistinctLimitNode.class, + anyTree( + tableScan("orders"))))); + + assertPlan( + "SELECT distinct(id) FROM (VALUES 1, 2, 3, 4, 5, 6) as t1 (id) LIMIT 10", + output( + node(ProjectNode.class, + node(AggregationNode.class, + node(ProjectNode.class, + values(ImmutableList.of("x"))))))); + } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEvaluateZeroLimit.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEvaluateZeroLimit.java index 25539fec90b6..160254896afd 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEvaluateZeroLimit.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEvaluateZeroLimit.java @@ -51,7 +51,6 @@ public void test() ImmutableList.of( constantExpressions(BIGINT, 1L, 10L), constantExpressions(BIGINT, 2L, 11L)))))) - // TODO: verify contents .matches(values(ImmutableMap.of())); } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEvaluateZeroSample.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEvaluateZeroSample.java index 97b8745311de..b14222b46a64 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEvaluateZeroSample.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestEvaluateZeroSample.java @@ -54,7 +54,6 @@ public void test() ImmutableList.of( constantExpressions(BIGINT, 1L, 10L), constantExpressions(BIGINT, 2L, 11L)))))) - // TODO: verify contents .matches(values(ImmutableMap.of())); } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantDistinctLimit.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantDistinctLimit.java new file mode 100644 index 000000000000..2ba5e968e1df --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantDistinctLimit.java @@ -0,0 +1,68 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.ValuesNode; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.Test; + +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; + +public class TestRemoveRedundantDistinctLimit + extends BaseRuleTest +{ + @Test + public void test() + { + tester().assertThat(new RemoveRedundantDistinctLimit()) + .on(p -> + p.distinctLimit( + 10, + ImmutableList.of(p.variable("c")), + p.values(1, p.variable("c")))) + .matches(node(ValuesNode.class)); + + tester().assertThat(new RemoveRedundantDistinctLimit()) + .on(p -> + p.distinctLimit( + 10, + ImmutableList.of(p.variable("c")), + p.values(6, p.variable("c")))) + .matches( + node(AggregationNode.class, + node(ValuesNode.class))); + + tester().assertThat(new RemoveRedundantDistinctLimit()) + .on(p -> + p.distinctLimit( + 0, + ImmutableList.of(p.variable("c")), + p.values(1, p.variable("c")))) + .matches(node(ValuesNode.class)); + } + + @Test + public void doesNotFire() + { + tester().assertThat(new RemoveRedundantDistinctLimit()) + .on(p -> + p.distinctLimit( + 10, + ImmutableList.of(p.variable("c")), + p.values(100, p.variable("c")))) + .doesNotFire(); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantLimit.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantLimit.java new file mode 100644 index 000000000000..f019edad1101 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantLimit.java @@ -0,0 +1,92 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.ValuesNode; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.constantExpressions; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression; + +public class TestRemoveRedundantLimit + extends BaseRuleTest +{ + @Test + public void test() + { + tester().assertThat(new RemoveRedundantLimit()) + .on(p -> + p.limit( + 10, + p.aggregation(builder -> builder + .addAggregation(p.variable("c"), expression("count(foo)"), ImmutableList.of(BIGINT)) + .globalGrouping() + .source(p.values(p.variable("foo")))))) + .matches( + node(AggregationNode.class, + node(ValuesNode.class))); + } + + @Test + public void testRemoveLimitWithTies() + { + tester().assertThat(new RemoveRedundantLimit()) + .on(p -> { + VariableReferenceExpression c = p.variable("c"); + return p.limit( + 10, + p.values(5, c)); + }) + .matches(values("c")); + } + + @Test + public void testForZeroLimit() + { + tester().assertThat(new EvaluateZeroLimit()) + .on(p -> + p.limit( + 0, + p.filter( + expression("b > 5"), + p.values( + ImmutableList.of(p.variable("a"), p.variable("b")), + ImmutableList.of( + constantExpressions(BIGINT, 1L, 10L), + constantExpressions(BIGINT, 2L, 11L)))))) + .matches(values(ImmutableMap.of())); + } + + @Test + public void doesNotFire() + { + tester().assertThat(new EvaluateZeroLimit()) + .on(p -> + p.limit( + 10, + p.aggregation(builder -> builder + .addAggregation(p.variable("c"), expression("count(foo)"), ImmutableList.of(BIGINT)) + .singleGroupingSet(p.variable("foo")) + .source(p.values(20, p.variable("foo")))))) + .doesNotFire(); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantSort.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantSort.java new file mode 100644 index 000000000000..081d38bf3b0f --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantSort.java @@ -0,0 +1,69 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.ValuesNode; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.Test; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression; + +public class TestRemoveRedundantSort + extends BaseRuleTest +{ + @Test + public void test() + { + tester().assertThat(new RemoveRedundantSort()) + .on(p -> + p.sort( + ImmutableList.of(p.variable("c")), + p.aggregation(builder -> builder + .addAggregation(p.variable("c"), expression("count(foo)"), ImmutableList.of(BIGINT)) + .globalGrouping() + .source(p.values(p.variable("foo")))))) + .matches( + node(AggregationNode.class, + node(ValuesNode.class))); + } + + @Test + public void testForZeroCardinality() + { + tester().assertThat(new RemoveRedundantSort()) + .on(p -> + p.sort( + ImmutableList.of(p.variable("c")), + p.values(p.variable("foo")))) + .matches(node(ValuesNode.class)); + } + + @Test + public void doesNotFire() + { + tester().assertThat(new RemoveRedundantSort()) + .on(p -> + p.sort( + ImmutableList.of(p.variable("c")), + p.aggregation(builder -> builder + .addAggregation(p.variable("c"), expression("count(foo)"), ImmutableList.of(BIGINT)) + .singleGroupingSet(p.variable("foo")) + .source(p.values(20, p.variable("foo")))))) + .doesNotFire(); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantTopN.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantTopN.java new file mode 100644 index 000000000000..df3622517de1 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/TestRemoveRedundantTopN.java @@ -0,0 +1,100 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.planner.iterative.rule; + +import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.ValuesNode; +import com.facebook.presto.sql.planner.iterative.rule.test.BaseRuleTest; +import com.facebook.presto.sql.planner.plan.SortNode; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.node; +import static com.facebook.presto.sql.planner.assertions.PlanMatchPattern.values; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.constantExpressions; +import static com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression; + +public class TestRemoveRedundantTopN + extends BaseRuleTest +{ + @Test + public void test() + { + tester().assertThat(new RemoveRedundantTopN()) + .on(p -> + p.topN( + 10, + ImmutableList.of(p.variable("c")), + p.aggregation(builder -> builder + .addAggregation(p.variable("c"), expression("count(foo)"), ImmutableList.of(BIGINT)) + .globalGrouping() + .source(p.values(p.variable("foo")))))) + .matches( + node(AggregationNode.class, + node(ValuesNode.class))); + + tester().assertThat(new RemoveRedundantTopN()) + .on(p -> + p.topN( + 10, + ImmutableList.of(p.variable("a")), + p.filter( + expression("b > 5"), + p.values( + ImmutableList.of(p.variable("a"), p.variable("b")), + ImmutableList.of( + constantExpressions(BIGINT, 1L, 10L), + constantExpressions(BIGINT, 2L, 11L)))))) + .matches( + node(SortNode.class, + node(FilterNode.class, + node(ValuesNode.class)))); + } + + @Test + public void testZeroTopN() + { + tester().assertThat(new EvaluateZeroTopN()) + .on(p -> + p.topN( + 0, + ImmutableList.of(p.variable("a")), + p.filter( + expression("b > 5"), + p.values( + ImmutableList.of(p.variable("a"), p.variable("b")), + ImmutableList.of( + constantExpressions(BIGINT, 1L, 10L), + constantExpressions(BIGINT, 2L, 11L)))))) + .matches(values(ImmutableMap.of())); + } + + @Test + public void doesNotFire() + { + tester().assertThat(new RemoveRedundantTopN()) + .on(p -> + p.topN( + 10, + ImmutableList.of(p.variable("c")), + p.aggregation(builder -> builder + .addAggregation(p.variable("c"), expression("count(foo)"), ImmutableList.of(BIGINT)) + .singleGroupingSet(p.variable("foo")) + .source(p.values(20, p.variable("foo")))))) + .doesNotFire(); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java index 9f6111e6cc47..20b69c55f316 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java @@ -14,7 +14,6 @@ package com.facebook.presto.sql.planner.iterative.rule.test; import com.facebook.presto.Session; -import com.facebook.presto.common.block.SortOrder; import com.facebook.presto.common.function.OperatorType; import com.facebook.presto.common.predicate.TupleDomain; import com.facebook.presto.common.type.Type; @@ -30,6 +29,7 @@ import com.facebook.presto.spi.plan.AggregationNode.Aggregation; import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.IntersectNode; import com.facebook.presto.spi.plan.LimitNode; @@ -70,6 +70,7 @@ import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SampleNode; import com.facebook.presto.sql.planner.plan.SemiJoinNode; +import com.facebook.presto.sql.planner.plan.SortNode; import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.UnnestNode; @@ -97,6 +98,7 @@ import java.util.function.Consumer; import java.util.stream.Stream; +import static com.facebook.presto.common.block.SortOrder.ASC_NULLS_FIRST; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.VarbinaryType.VARBINARY; import static com.facebook.presto.metadata.FunctionAndTypeManager.qualifyObjectName; @@ -238,6 +240,16 @@ public EnforceSingleRowNode enforceSingleRow(PlanNode source) return new EnforceSingleRowNode(idAllocator.getNextId(), source); } + public SortNode sort(List orderBy, PlanNode source) + { + ImmutableList ordering = orderBy.stream().map(variable -> new Ordering(variable, ASC_NULLS_FIRST)).collect(toImmutableList()); + return new SortNode( + idAllocator.getNextId(), + source, + new OrderingScheme(ordering), + false); + } + public LimitNode limit(long limit, PlanNode source) { return new LimitNode(idAllocator.getNextId(), source, limit, FINAL); @@ -249,7 +261,7 @@ public TopNNode topN(long count, List orderBy, Plan idAllocator.getNextId(), source, count, - new OrderingScheme(orderBy.stream().map(variable -> new Ordering(variable, SortOrder.ASC_NULLS_FIRST)).collect(toImmutableList())), + new OrderingScheme(orderBy.stream().map(variable -> new Ordering(variable, ASC_NULLS_FIRST)).collect(toImmutableList())), TopNNode.Step.SINGLE); } @@ -302,6 +314,17 @@ public CallExpression comparison(OperatorType operatorType, RowExpression left, return call(operatorType.getOperator(), functionHandle, left.getType(), left, right); } + public DistinctLimitNode distinctLimit(long count, List distinctSymbols, PlanNode source) + { + return new DistinctLimitNode( + idAllocator.getNextId(), + source, + count, + false, + distinctSymbols, + Optional.empty()); + } + public class AggregationBuilder { private final TypeProvider types; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueries.java b/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueries.java index d19f642063fd..ee49dcde93cc 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueries.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/query/TestSubqueries.java @@ -86,16 +86,16 @@ public void testCorrelatedExistsSubqueriesWithOrPredicateAndNull() } @Test - public void testUnsupportedSubqueriesWithCoercions() + public void testSupportedSubqueriesWithCoercions() { // coercion from subquery symbol type to correlation type - assertions.assertFails( + assertions.assertQuery( "select (select count(*) from (values 1) t(a) where t.a=t2.b limit 1) from (values 1.0) t2(b)", - UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG); + "VALUES BIGINT '1'"); // coercion from t.a (null) to integer - assertions.assertFails( - "select EXISTS(select 1 from (values (null, null)) t(a, b) where t.a=t2.b GROUP BY t.b) from (values 1, 2) t2(b)", - UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG); + assertions.assertQuery( + "SELECT EXISTS(SELECT 1 FROM (VALUES ('x', 1)) u(x, cid) WHERE x = 'x' AND t.cid = cid LIMIT 1) FROM (VALUES 1) t(cid)", + "VALUES true"); } @Test @@ -104,9 +104,12 @@ public void testCorrelatedSubqueriesWithLimit() assertions.assertQuery( "select (select t.a from (values 1, 2) t(a) where t.a=t2.b limit 1) from (values 1) t2(b)", "VALUES 1"); + assertions.assertQuery( + "SELECT (SELECT t.a FROM (VALUES 1, 2) t(a) WHERE t.a=t2.b LIMIT 2) FROM (VALUES 1) t2(b)", + "VALUES 1"); // cannot enforce limit 2 on correlated subquery assertions.assertFails( - "select (select t.a from (values 1, 2) t(a) where t.a=t2.b limit 2) from (values 1) t2(b)", + "select (select t.a from (values 1, 2, 3) t(a) where t.a=t2.b limit 2) from (values 1) t2(b)", UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG); assertions.assertQuery( "select (select sum(t.a) from (values 1, 2) t(a) where t.a=t2.b group by t.a limit 2) from (values 1) t2(b)", @@ -118,10 +121,9 @@ public void testCorrelatedSubqueriesWithLimit() "select EXISTS(select 1 from (values 1, 1, 3) t(a) where t.a=t2.b limit 1) from (values 1, 2) t2(b)", "VALUES true, false", false); - // TransformCorrelatedScalarAggregationToJoin does not fire since limit is above aggregation node - assertions.assertFails( - "select (select count(*) from (values 1, 1, 3) t(a) where t.a=t2.b limit 1) from (values 1) t2(b)", - UNSUPPORTED_CORRELATED_SUBQUERY_ERROR_MSG); + assertions.assertQuery( + "SELECT (SELECT count(*) FROM (VALUES 1, 1, 3) t(a) WHERE t.a=t2.b LIMIT 1) FROM (VALUES 1) t2(b)", + "VALUES BIGINT '2'"); assertExistsRewrittenToAggregationBelowJoin( "SELECT EXISTS(SELECT 1 FROM (values ('x', 1)) u(x, cid) WHERE x = 'x' AND t.cid = cid LIMIT 1) " + "FROM (values 1) t(cid)", diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java index c8416839d8aa..679fe7aaecea 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueries.java @@ -3389,9 +3389,12 @@ public void testCorrelatedScalarSubqueries() // two level of nesting assertQuery("SELECT * FROM nation n WHERE 2 = (SELECT (SELECT 2 * n.nationkey))"); + // redundant LIMIT in subquery + assertQuery("SELECT (SELECT count(*) FROM (VALUES (7,1)) t(orderkey, value) WHERE orderkey = corr_key LIMIT 1) FROM (values 7) t(corr_key)"); + // explicit LIMIT in subquery assertQueryFails( - "SELECT (SELECT count(*) FROM (VALUES (7,1)) t(orderkey, value) WHERE orderkey = corr_key LIMIT 1) FROM (values 7) t(corr_key)", + "SELECT (SELECT count(*) FROM (VALUES (7,1)) t(orderkey, value) WHERE orderkey = corr_key GROUP BY value LIMIT 1) FROM (values 7) t(corr_key)", "line 1:9: Given correlated subquery is not supported"); }