Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: array_concat with arrays with different dimensions, add _list* aliases for _array* functions #7008

Merged
merged 5 commits into from
Jul 20, 2023
Merged

fix: array_concat with arrays with different dimensions, add _list* aliases for _array* functions #7008

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b541c49
fix: array_concat with arrays with different dimensions does not work…
izveigor Jul 18, 2023
c9db975
fix: cargo doc
izveigor Jul 18, 2023
b0e3774
Merge branch 'main' into improve_array_concat_input
izveigor Jul 19, 2023
ced6901
fix: array_concat does not work with nested empty arrays
izveigor Jul 19, 2023
68a6bed
fix: some comments
izveigor Jul 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
197 changes: 155 additions & 42 deletions datafusion/core/tests/sqllogictests/test_files/array.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ select column1, column2, column3 from arrays_values_without_nulls;
### Array function tests


## make_array
## make_array (aliases: `make_list`)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any reason to have the alias make_list, may it be confused with creating ListArray?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jayzhan211 I have identified several reasons:

  1. The idea was taken from popular database: DuckDB. So, in my opinion, users of this database can more easily adapt to our array functions. (Documentation: https://duckdb.org/docs/sql/functions/nested).
  2. array and list prefix or postfix can separate mutable or immutable objects. (See discussion: The concept of practical implementation of the array #6855 (About aliases)).
  3. Some people prefer to use one prefix/postfix over another.


# make_array scalar function #1
query ???
Expand Down Expand Up @@ -193,6 +193,12 @@ select make_array(make_array()), make_array(make_array(make_array()))
----
[[]] [[[]]]

# make_list scalar function #8 (function alias: `make_array`)
query ???
select make_list(1, 2, 3), make_list(1.0, 2.0, 3.0), make_list('h', 'e', 'l', 'l', 'o');
----
[1, 2, 3] [1.0, 2.0, 3.0] [h, e, l, l, o]

# make_array scalar function with nulls
query ???
select make_array(1, NULL, 3), make_array(NULL, 2.0, NULL), make_array('h', NULL, 'l', NULL, 'o');
Expand Down Expand Up @@ -272,7 +278,7 @@ from values;



## array_append
## array_append (aliases: `list_append`, `array_push_back`, `list_push_back`)

# array_append scalar function #2
query ?
Expand All @@ -292,6 +298,24 @@ select array_append(make_array(1, 2, 3), 4), array_append(make_array(1.0, 2.0, 3
----
[1, 2, 3, 4] [1.0, 2.0, 3.0, 4.0] [h, e, l, l, o]

# list_append scalar function #4 (function alias `array_append`)
query ???
select list_append(make_array(1, 2, 3), 4), list_append(make_array(1.0, 2.0, 3.0), 4.0), list_append(make_array('h', 'e', 'l', 'l'), 'o');
----
[1, 2, 3, 4] [1.0, 2.0, 3.0, 4.0] [h, e, l, l, o]

# array_push_back scalar function #5 (function alias `array_append`)
query ???
select array_push_back(make_array(1, 2, 3), 4), array_push_back(make_array(1.0, 2.0, 3.0), 4.0), array_push_back(make_array('h', 'e', 'l', 'l'), 'o');
----
[1, 2, 3, 4] [1.0, 2.0, 3.0, 4.0] [h, e, l, l, o]

# list_push_back scalar function #6 (function alias `array_append`)
query ???
select list_push_back(make_array(1, 2, 3), 4), list_push_back(make_array(1.0, 2.0, 3.0), 4.0), list_push_back(make_array('h', 'e', 'l', 'l'), 'o');
----
[1, 2, 3, 4] [1.0, 2.0, 3.0, 4.0] [h, e, l, l, o]

# array_append with columns
query ?
select array_append(column1, column2) from arrays_values;
Expand All @@ -317,7 +341,7 @@ select array_append(column2, 100.1), array_append(column3, '.') from arrays;
[100.1] [,, .]
[16.6, 17.7, 18.8, 100.1] [.]

## array_prepend
## array_prepend (aliases: `list_prepend`, `array_push_front`, `list_push_front`)

# array_prepend scalar function #1
query ?
Expand All @@ -337,6 +361,24 @@ select array_prepend(1, make_array(2, 3, 4)), array_prepend(1.0, make_array(2.0,
----
[1, 2, 3, 4] [1.0, 2.0, 3.0, 4.0] [h, e, l, l, o]

# list_prepend scalar function #4 (function alias `array_prepend`)
query ???
select list_prepend(1, make_array(2, 3, 4)), list_prepend(1.0, make_array(2.0, 3.0, 4.0)), list_prepend('h', make_array('e', 'l', 'l', 'o'));
----
[1, 2, 3, 4] [1.0, 2.0, 3.0, 4.0] [h, e, l, l, o]

# array_push_front scalar function #4 (function alias `array_prepend`)
query ???
select array_push_front(1, make_array(2, 3, 4)), array_push_front(1.0, make_array(2.0, 3.0, 4.0)), array_push_front('h', make_array('e', 'l', 'l', 'o'));
----
[1, 2, 3, 4] [1.0, 2.0, 3.0, 4.0] [h, e, l, l, o]

# list_push_front scalar function #4 (function alias `array_prepend`)
query ???
select list_push_front(1, make_array(2, 3, 4)), list_push_front(1.0, make_array(2.0, 3.0, 4.0)), list_push_front('h', make_array('e', 'l', 'l', 'o'));
----
[1, 2, 3, 4] [1.0, 2.0, 3.0, 4.0] [h, e, l, l, o]

# array_prepend with columns
query ?
select array_prepend(column2, column1) from arrays_values;
Expand Down Expand Up @@ -382,7 +424,7 @@ select array_fill(1, make_array())
----
[]

## array_concat
## array_concat (aliases: `array_cat`, `list_concat`, `list_cat`)

# array_concat scalar function #1
query ??
Expand Down Expand Up @@ -420,30 +462,60 @@ select array_concat(make_array(), make_array(2, 3));
----
[2, 3]

# array_cat scalar function #7 (function alias `array_concat`)
query ??
select array_cat(make_array(1, 2, 3), make_array(4, 5, 6), make_array(7, 8, 9)), array_cat(make_array([1], [2]), make_array([3], [4]));
----
[1, 2, 3, 4, 5, 6, 7, 8, 9] [[1], [2], [3], [4]]

# list_concat scalar function #8 (function alias `array_concat`)
query ??
select list_concat(make_array(1, 2, 3), make_array(4, 5, 6), make_array(7, 8, 9)), list_concat(make_array([1], [2]), make_array([3], [4]));
----
[1, 2, 3, 4, 5, 6, 7, 8, 9] [[1], [2], [3], [4]]

# list_cat scalar function #9 (function alias `array_concat`)
query ??
select list_cat(make_array(1, 2, 3), make_array(4, 5, 6), make_array(7, 8, 9)), list_cat(make_array([1], [2]), make_array([3], [4]));
----
[1, 2, 3, 4, 5, 6, 7, 8, 9] [[1], [2], [3], [4]]

# array_concat with different dimensions #1 (2D + 1D)
query ?
select array_concat(make_array([1,2], [3,4]), make_array(5, 6))
select array_concat(make_array([1,2], [3,4]), make_array(5, 6));
----
[[1, 2], [3, 4], [5, 6]]

# array_concat with different dimensions #2 (1D + 2D)
query ?
select array_concat(make_array(5, 6), make_array([1,2], [3,4]))
select array_concat(make_array(5, 6), make_array([1,2], [3,4]));
----
[[5, 6], [1, 2], [3, 4]]

# array_concat with different dimensions #3 (2D + 1D + 1D)
query ?
select array_concat(make_array([1,2], [3,4]), make_array(5, 6), make_array(7,8))
select array_concat(make_array([1,2], [3,4]), make_array(5, 6), make_array(7,8));
----
[[1, 2], [3, 4], [5, 6], [7, 8]]

# array_concat with different dimensions #4 (1D + 2D + 3D)
query ?
select array_concat(make_array(10, 20), make_array([30, 40]), make_array([[50, 60]]))
select array_concat(make_array(10, 20), make_array([30, 40]), make_array([[50, 60]]));
----
[[[10, 20]], [[30, 40]], [[50, 60]]]

# array_concat with different dimensions #5 (2D + 1D + 3D)
query ?
select array_concat(make_array([30, 40]), make_array(10, 20), make_array([[50, 60]]));
----
[[[30, 40]], [[10, 20]], [[50, 60]]]

# array_concat with different dimensions #6 (2D + 1D + 3D + 4D + 3D)
query ?
select array_concat(make_array([30, 40]), make_array(10, 20), make_array([[50, 60]]), make_array([[[70, 80]]]), make_array([[80, 40]]));
----
[[[[30, 40]]], [[[10, 20]]], [[[50, 60]]], [[[70, 80]]], [[[80, 40]]]]

# array_concat column-wise #1
query ?
select array_concat(column1, make_array(0)) from arrays_values_without_nulls;
Expand Down Expand Up @@ -471,19 +543,6 @@ select array_concat(make_array(column2), make_array(column3)) from arrays_values
[23, 3]
[34, 4]

# array_concat column-wise #4
query ?
select array_concat(column1, column2) from arrays_values;
----
[, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1]
[11, 12, 13, 14, 15, 16, 17, 18, , 20, 12]
[21, 22, 23, , 25, 26, 27, 28, 29, 30, 23]
[31, 32, 33, 34, 35, , 37, 38, 39, 40, 34]
[44]
[41, 42, 43, 44, 45, 46, 47, 48, 49, 50, ]
[51, 52, , 54, 55, 56, 57, 58, 59, 60, 55]
[61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 66]

# array_concat column-wise #5
query ?
select array_concat(make_array(column2), make_array(0)) from arrays_values;
Expand Down Expand Up @@ -559,17 +618,6 @@ NULL
# query error DataFusion error: Arrow error: Invalid argument error: column types must match schema types, expected List\(Field \{ name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: \{\} \}\) but found List\(Field \{ name: "item", data_type: List\(Field \{ name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: \{\} \}\), nullable: true, dict_id: 0, dict_is_ordered: false, metadata: \{\} \}\) at column index 0
# select array_concat(make_array(column3), column4) from arrays_values_v2;

# array_concat column-wise #11 (1D + Integers)
query ?
select array_concat(column2, column3) from arrays_values_v2;
----
[4, 5, , 12]
[7, , 8, 13]
[14]
[, 21, ]
[]
[]

# TODO: Panic at 'range end index 3 out of range for slice of length 2'
# array_concat column-wise #12 (2D + 1D)
# query
Expand All @@ -586,7 +634,7 @@ select array_concat(make_array(column3), column1, column2) from arrays_values_v2
[, 11, 12]
[]

## array_position
## array_position (aliases: `list_position`, `array_indexof`, `list_indexof`)

# array_position scalar function #1
query III
Expand All @@ -600,6 +648,24 @@ select array_position(['h', 'e', 'l', 'l', 'o'], 'l', 4), array_position([1, 2,
----
4 5 2

# list_position scalar function #3 (function alias `array_position`)
query III
select list_position(['h', 'e', 'l', 'l', 'o'], 'l'), list_position([1, 2, 3, 4, 5], 5), list_position([1, 1, 1], 1);
----
3 5 1

# array_indexof scalar function #4 (function alias `array_position`)
query III
select array_indexof(['h', 'e', 'l', 'l', 'o'], 'l'), array_indexof([1, 2, 3, 4, 5], 5), array_indexof([1, 1, 1], 1);
----
3 5 1

# list_indexof scalar function #5 (function alias `array_position`)
query III
select list_indexof(['h', 'e', 'l', 'l', 'o'], 'l'), list_indexof([1, 2, 3, 4, 5], 5), list_indexof([1, 1, 1], 1);
----
3 5 1

# array_position with columns
query II
select array_position(column1, column2), array_position(column1, column2, column3) from arrays_values_without_nulls;
Expand All @@ -618,14 +684,20 @@ NULL NULL
NULL NULL
NULL NULL

## array_positions
## array_positions (aliases: `list_positions`)

# array_positions scalar function
# array_positions scalar function #1
query ???
select array_positions(['h', 'e', 'l', 'l', 'o'], 'l'), array_positions([1, 2, 3, 4, 5], 5), array_positions([1, 1, 1], 1);
----
[3, 4] [5] [1, 2, 3]

# list_positions scalar function #2 (function alias `array_positions`)
query ???
select list_positions(['h', 'e', 'l', 'l', 'o'], 'l'), list_positions([1, 2, 3, 4, 5], 5), list_positions([1, 1, 1], 1);
----
[3, 4] [5] [1, 2, 3]

# array_positions with columns
query ?
select array_positions(column1, column2) from arrays_values_without_nulls;
Expand All @@ -652,9 +724,9 @@ select array_replace(make_array(1, 2, 3, 4), 2, 3), array_replace(make_array(1,
----
[1, 3, 3, 4] [1, 0, 0, 5, 0, 6, 7] [1, 2, 3]

## array_to_string
## array_to_string (aliases: `list_to_string`, `array_join`, `list_join`)

# array_to_string scalar function
# array_to_string scalar function #1
query TTT
select array_to_string(['h', 'e', 'l', 'l', 'o'], ','), array_to_string([1, 2, 3, 4, 5], '-'), array_to_string([1.0, 2.0, 3.0], '|');
----
Expand All @@ -672,6 +744,24 @@ select array_to_string(make_array(), ',')
----
(empty)

# list_to_string scalar function #4 (function alias `array_to_string`)
query TTT
select list_to_string(['h', 'e', 'l', 'l', 'o'], ','), list_to_string([1, 2, 3, 4, 5], '-'), list_to_string([1.0, 2.0, 3.0], '|');
----
h,e,l,l,o 1-2-3-4-5 1|2|3

# array_join scalar function #5 (function alias `array_to_string`)
query TTT
select array_join(['h', 'e', 'l', 'l', 'o'], ','), array_join([1, 2, 3, 4, 5], '-'), array_join([1.0, 2.0, 3.0], '|');
----
h,e,l,l,o 1-2-3-4-5 1|2|3

# list_join scalar function #6 (function alias `list_join`)
query TTT
select list_join(['h', 'e', 'l', 'l', 'o'], ','), list_join([1, 2, 3, 4, 5], '-'), list_join([1.0, 2.0, 3.0], '|');
----
h,e,l,l,o 1-2-3-4-5 1|2|3

# array_to_string scalar function with nulls #1
query TTT
select array_to_string(make_array('h', NULL, 'l', NULL, 'o'), ','), array_to_string(make_array(1, NULL, 3, NULL, 5), '-'), array_to_string(make_array(NULL, 2.0, 3.0), '|');
Expand Down Expand Up @@ -792,9 +882,9 @@ select trim_array(make_array(), 0), trim_array(make_array(), 1)
----
[] []

## array_length
## array_length (aliases: `list_length`)

# array_length scalar function
# array_length scalar function #1
query III
select array_length(make_array(1, 2, 3, 4, 5)), array_length(make_array(1, 2, 3)), array_length(make_array([1, 2], [3, 4], [5, 6]));
----
Expand Down Expand Up @@ -824,6 +914,12 @@ select array_length(make_array()), array_length(make_array(), 1), array_length(m
----
0 0 NULL

# list_length scalar function #6 (function alias `array_length`)
query III
select list_length(make_array(1, 2, 3, 4, 5)), list_length(make_array(1, 2, 3)), list_length(make_array([1, 2], [3, 4], [5, 6]));
----
5 3 3

# array_length with columns
query I
select array_length(column1, column3) from arrays_values;
Expand All @@ -850,7 +946,7 @@ NULL 10
NULL 10
NULL 10

## array_dims
## array_dims (aliases: `list_dims`)

# array_dims scalar function
query ???
Expand All @@ -870,6 +966,12 @@ select array_dims(make_array()), array_dims(make_array(make_array()))
----
NULL [1, 0]

# list_dims scalar function #4 (function alias `array_dims`)
query ???
select list_dims(make_array(1, 2, 3)), list_dims(make_array([1, 2], [3, 4])), list_dims(make_array([[[[1], [2]]]]));
----
[3] [2, 2] [1, 1, 1, 2, 1]

# array_dims with columns
query ???
select array_dims(column1), array_dims(column2), array_dims(column3) from arrays;
Expand All @@ -882,9 +984,9 @@ NULL [3] [4]
[2, 2] NULL [1]
[2, 2] [3] NULL

## array_ndims
## array_ndims (aliases: `list_ndims`)

# array_ndims scalar function
# array_ndims scalar function #1
query III
select array_ndims(make_array(1, 2, 3)), array_ndims(make_array([1, 2], [3, 4])), array_ndims(make_array([[[[1], [2]]]]));
----
Expand All @@ -902,6 +1004,17 @@ select array_ndims(make_array()), array_ndims(make_array(make_array()))
----
NULL 2

# list_ndims scalar function #4 (function alias `array_ndims`)
query III
select list_ndims(make_array(1, 2, 3)), list_ndims(make_array([1, 2], [3, 4])), list_ndims(make_array([[[[1], [2]]]]));
----
1 2 5

query II
select array_ndims(make_array()), array_ndims(make_array(make_array()))
----
NULL 2

# array_ndims with columns
query III
select array_ndims(column1), array_ndims(column2), array_ndims(column3) from arrays;
Expand Down
Loading