Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Query: Fixes ORDER BY undefined (and mixed type primitives) continuation token support #2103

Merged
merged 13 commits into from
Jan 22, 2021
Merged

Query: Fixes ORDER BY undefined (and mixed type primitives) continuation token support #2103

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
889f0b7
drafted out support for order by undefined
bchong95 Jan 5, 2021
ef0efe4
Merge branch 'master' into users/brchon/OrderByUndefinedContinuationT…
bchong95 Jan 5, 2021
1c8466a
drafted out test but need to handle descending cases
bchong95 Jan 6, 2021
2ff00e7
Merge branch 'users/brchon/OrderByUndefinedContinuationTokens' of htt…
bchong95 Jan 6, 2021
c283b5b
got single order by with undefined working
bchong95 Jan 8, 2021
78a564f
got multi order by case working
bchong95 Jan 9, 2021
79f61df
comparison on arrays and objects are undefined
bchong95 Jan 11, 2021
224cc68
Merge branch 'master' into users/brchon/OrderByUndefinedContinuationT…
bchong95 Jan 11, 2021
378902b
Merge branch 'master' into users/brchon/OrderByUndefinedContinuationT…
bchong95 Jan 12, 2021
cfa05a6
Merge branch 'master' into users/brchon/OrderByUndefinedContinuationT…
bchong95 Jan 15, 2021
acd9139
Merge branch 'master' into users/brchon/OrderByUndefinedContinuationT…
sboshra Jan 22, 2021
8808821
resolved iteration comments
bchong95 Jan 22, 2021
0623899
Merge branch 'master' into users/brchon/OrderByUndefinedContinuationT…
bchong95 Jan 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
267 changes: 240 additions & 27 deletions ...src/Query/Core/Pipeline/CrossPartition/OrderBy/OrderByCrossPartitionQueryPipelineStage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ namespace Microsoft.Azure.Cosmos.Query.Core.Pipeline.CrossPartition.OrderBy
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Linq.Expressions;
using System.Net;
using System.Runtime.CompilerServices;
using System.Text;
Expand Down Expand Up @@ -66,6 +67,8 @@ private static class Expressions
public const string EqualTo = "=";
public const string GreaterThan = ">";
public const string GreaterThanOrEqualTo = ">=";
public const string True = "true";
public const string False = "false";
}

private OrderByCrossPartitionQueryPipelineStage(
Expand Down Expand Up @@ -287,7 +290,7 @@ private async ValueTask<bool> MoveNextAsync_InitializeAsync_HandleSplitAsync(
IReadOnlyList<FeedRangeEpk> childRanges = await this.documentContainer.GetChildRangeAsync(
uninitializedEnumerator.Range,
trace,
this.cancellationToken);
this.cancellationToken);
if (childRanges.Count == 0)
{
throw new InvalidOperationException("Got back no children");
Expand Down Expand Up @@ -816,15 +819,46 @@ private static (string leftFilter, string targetFilter, string rightFilter) GetF
(OrderByColumn orderByColumn, CosmosElement orderByItem) = columnAndItems.Span[0];
(string expression, SortOrder sortOrder) = (orderByColumn.Expression, orderByColumn.SortOrder);

StringBuilder sb = new StringBuilder();
CosmosElementToQueryLiteral cosmosElementToQueryLiteral = new CosmosElementToQueryLiteral(sb);
orderByItem.Accept(cosmosElementToQueryLiteral);
AppendToBuilders(builders, "( ");

// We need to add the filter for within the same type.
if (orderByItem != default)
{
StringBuilder sb = new StringBuilder();
CosmosElementToQueryLiteral cosmosElementToQueryLiteral = new CosmosElementToQueryLiteral(sb);
orderByItem.Accept(cosmosElementToQueryLiteral);

string orderByItemToString = sb.ToString();

left.Append($"{expression} {(sortOrder == SortOrder.Descending ? Expressions.LessThan : Expressions.GreaterThan)} {orderByItemToString}");
target.Append($"{expression} {(sortOrder == SortOrder.Descending ? Expressions.LessThanOrEqualTo : Expressions.GreaterThanOrEqualTo)} {orderByItemToString}");
right.Append($"{expression} {(sortOrder == SortOrder.Descending ? Expressions.LessThanOrEqualTo : Expressions.GreaterThanOrEqualTo)} {orderByItemToString}");
}
else
{
// User is ordering by undefined, so we need to avoid a null reference exception.

string orderByItemToString = sb.ToString();
// What we really want is to support expression > undefined,
// but the engine evaluates to undefined instead of true or false,
// so we work around this by using the IS_DEFINED() system function.

ComparisionWithUndefinedFilters filters = new ComparisionWithUndefinedFilters(expression);
left.Append($"{(sortOrder == SortOrder.Descending ? filters.LessThan : filters.GreaterThan)}");
target.Append($"{(sortOrder == SortOrder.Descending ? filters.LessThanOrEqualTo : filters.GreaterThanOrEqualTo)}");
right.Append($"{(sortOrder == SortOrder.Descending ? filters.LessThanOrEqualTo : filters.GreaterThanOrEqualTo)}");
}

left.Append($"{expression} {(sortOrder == SortOrder.Descending ? Expressions.LessThan : Expressions.GreaterThan)} {orderByItemToString}");
target.Append($"{expression} {(sortOrder == SortOrder.Descending ? Expressions.LessThanOrEqualTo : Expressions.GreaterThanOrEqualTo)} {orderByItemToString}");
right.Append($"{expression} {(sortOrder == SortOrder.Descending ? Expressions.LessThanOrEqualTo : Expressions.GreaterThanOrEqualTo)} {orderByItemToString}");
// Now we need to include all the types that match the sort order.
ReadOnlyMemory<string> isDefinedFunctions = orderByItem == default
? CosmosElementToIsSystemFunctionsVisitor.VisitUndefined(sortOrder == SortOrder.Ascending)
: orderByItem.Accept(CosmosElementToIsSystemFunctionsVisitor.Singleton, sortOrder == SortOrder.Ascending);
foreach (string isDefinedFunction in isDefinedFunctions.Span)
{
AppendToBuilders(builders, " OR ");
AppendToBuilders(builders, $"{isDefinedFunction}({expression})");
}

AppendToBuilders(builders, " )");
}
else
{
Expand Down Expand Up @@ -872,37 +906,105 @@ private static (string leftFilter, string targetFilter, string rightFilter) GetF
CosmosElement orderByItem = columnAndItemPrefix[index].orderByItem;
bool lastItem = index == prefixLength - 1;

// Append Expression
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, expression);
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, " ");
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, "(");

// Append binary operator
if (lastItem)
bool wasInequality;
// We need to add the filter for within the same type.
if (orderByItem == default)
{
string inequality = sortOrder == SortOrder.Descending ? Expressions.LessThan : Expressions.GreaterThan;
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, inequality);
if (lastPrefix)
ComparisionWithUndefinedFilters filters = new ComparisionWithUndefinedFilters(expression);

// Refer to the logic from single order by for how we are handling order by undefined
if (lastItem)
{
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, string.Empty, Expressions.EqualTo, Expressions.EqualTo);
if (lastPrefix)
{
if (sortOrder == SortOrder.Descending)
{
// <, <=, <=
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, filters.LessThan, filters.LessThanOrEqualTo, filters.LessThanOrEqualTo);
}
else
{
// >, >=, >=
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, filters.GreaterThan, filters.GreaterThanOrEqualTo, filters.GreaterThanOrEqualTo);
}
}
else
{
if (sortOrder == SortOrder.Descending)
{
// <, <, <
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, filters.LessThan, filters.LessThan, filters.LessThan);
}
else
{
// >, >, >
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, filters.GreaterThan, filters.GreaterThan, filters.GreaterThan);
}
}

wasInequality = true;
}
else
{
// =, =, =
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, filters.EqualTo);
wasInequality = false;
}
}
else
{
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, Expressions.EqualTo);
// Append Expression
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, expression);
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, " ");

// Append Binary Operator
if (lastItem)
{
string inequality = sortOrder == SortOrder.Descending ? Expressions.LessThan : Expressions.GreaterThan;
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, inequality);
if (lastPrefix)
{
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, string.Empty, Expressions.EqualTo, Expressions.EqualTo);
}

wasInequality = true;
}
else
{
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, Expressions.EqualTo);
wasInequality = false;
}

// Append OrderBy Item
StringBuilder sb = new StringBuilder();
CosmosElementToQueryLiteral cosmosElementToQueryLiteral = new CosmosElementToQueryLiteral(sb);
orderByItem.Accept(cosmosElementToQueryLiteral);
string orderByItemToString = sb.ToString();
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, " ");
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, orderByItemToString);
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, " ");
}

if (wasInequality)
{
// Now we need to include all the types that match the sort order.
ReadOnlyMemory<string> isDefinedFunctions = orderByItem == default
? CosmosElementToIsSystemFunctionsVisitor.VisitUndefined(sortOrder == SortOrder.Ascending)
: orderByItem.Accept(CosmosElementToIsSystemFunctionsVisitor.Singleton, sortOrder == SortOrder.Ascending);
foreach (string isDefinedFunction in isDefinedFunctions.Span)
{
AppendToBuilders(builders, " OR ");
AppendToBuilders(builders, $"{isDefinedFunction}({expression}) ");
}
}

// Append SortOrder
StringBuilder sb = new StringBuilder();
CosmosElementToQueryLiteral cosmosElementToQueryLiteral = new CosmosElementToQueryLiteral(sb);
orderByItem.Accept(cosmosElementToQueryLiteral);
string orderByItemToString = sb.ToString();
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, " ");
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, orderByItemToString);
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, " ");
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, ")");

if (!lastItem)
{
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, "AND ");
OrderByCrossPartitionQueryPipelineStage.AppendToBuilders(builders, " AND ");
}
}

Expand Down Expand Up @@ -1058,5 +1160,116 @@ public void SetCancellationToken(CancellationToken cancellationToken)
enumeratorAndToken.Item1.SetCancellationToken(cancellationToken);
}
}

private sealed class CosmosElementToIsSystemFunctionsVisitor : ICosmosElementVisitor<bool, ReadOnlyMemory<string>>
{
public static readonly CosmosElementToIsSystemFunctionsVisitor Singleton = new CosmosElementToIsSystemFunctionsVisitor();

private static class IsSystemFunctions
{
public const string Undefined = "not IS_DEFINED";
public const string Null = "IS_NULL";
public const string Boolean = "IS_BOOLEAN";
public const string Number = "IS_NUMBER";
public const string String = "IS_STRING";
public const string Array = "IS_ARRAY";
public const string Object = "IS_OBJECT";
}

private static readonly ReadOnlyMemory<string> SystemFunctionSortOrder = new string[]
{
IsSystemFunctions.Undefined,
IsSystemFunctions.Null,
IsSystemFunctions.Boolean,
IsSystemFunctions.Number,
IsSystemFunctions.String,
IsSystemFunctions.Array,
IsSystemFunctions.Object,
};

private static class SortOrder
{
public const int Undefined = 0;
public const int Null = 1;
public const int Boolean = 2;
public const int Number = 3;
public const int String = 4;
public const int Array = 5;
public const int Object = 6;
}

private CosmosElementToIsSystemFunctionsVisitor()
{
}

public ReadOnlyMemory<string> Visit(CosmosArray cosmosArray, bool isAscending)
{
return GetIsDefinedFunctions(SortOrder.Array, isAscending);
}

public ReadOnlyMemory<string> Visit(CosmosBinary cosmosBinary, bool isAscending)
{
throw new NotImplementedException();
}

public ReadOnlyMemory<string> Visit(CosmosBoolean cosmosBoolean, bool isAscending)
{
return GetIsDefinedFunctions(SortOrder.Boolean, isAscending);
}

public ReadOnlyMemory<string> Visit(CosmosGuid cosmosGuid, bool isAscending)
{
throw new NotImplementedException();
}

public ReadOnlyMemory<string> Visit(CosmosNull cosmosNull, bool isAscending)
{
return GetIsDefinedFunctions(SortOrder.Null, isAscending);
}

public ReadOnlyMemory<string> Visit(CosmosNumber cosmosNumber, bool isAscending)
{
return GetIsDefinedFunctions(SortOrder.Number, isAscending);
}

public ReadOnlyMemory<string> Visit(CosmosObject cosmosObject, bool isAscending)
{
return GetIsDefinedFunctions(SortOrder.Object, isAscending);
}

public ReadOnlyMemory<string> Visit(CosmosString cosmosString, bool isAscending)
{
return GetIsDefinedFunctions(SortOrder.String, isAscending);
}

public static ReadOnlyMemory<string> VisitUndefined(bool isAscending)
{
return isAscending ? SystemFunctionSortOrder.Slice(start: 1) : ReadOnlyMemory<string>.Empty;
}

private static ReadOnlyMemory<string> GetIsDefinedFunctions(int index, bool isAscending)
{
return isAscending ? SystemFunctionSortOrder.Slice(index + 1) : SystemFunctionSortOrder.Slice(start: 0, index);
}
}

private readonly struct ComparisionWithUndefinedFilters
{
public ComparisionWithUndefinedFilters(
string expression)
{
this.LessThan = "false";
this.LessThanOrEqualTo = $"NOT IS_DEFINED({expression})";
this.EqualTo = $"NOT IS_DEFINED({expression})";
this.GreaterThan = $"IS_DEFINED({expression})";
this.GreaterThanOrEqualTo = "true";
}

public string LessThan { get; }
public string LessThanOrEqualTo { get; }
public string EqualTo { get; }
public string GreaterThan { get; }
public string GreaterThanOrEqualTo { get; }
}
}
}
1 change: 1 addition & 0 deletions ...ery/Core/Pipeline/CrossPartition/OrderBy/OrderByQueryPartitionRangePageAsyncEnumerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ protected override Task<TryCatch<OrderByQueryPage>> GetNextPageAsync(ITrace trac
TryCatch<QueryPage> monadicQueryPage = antecedent.Result;
if (monadicQueryPage.Failed)
{
Console.WriteLine(this.SqlQuerySpec);
return TryCatch<OrderByQueryPage>.FromException(monadicQueryPage.Exception);
}

Expand Down
Loading