From 3e7c8b1d0537484b1dceeddf0b1fa58a3ac0a932 Mon Sep 17 00:00:00 2001 From: Santosh Kulkarni <66682828+kr-santosh@users.noreply.github.com> Date: Fri, 22 Apr 2022 11:58:08 +0530 Subject: [PATCH 1/6] Fixes issue with stream being disposed off after the first read. --- .../src/CosmosSqlQuerySpecJsonConverter.cs | 13 ++++---- .../CosmosItemTests.cs | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/CosmosSqlQuerySpecJsonConverter.cs b/Microsoft.Azure.Cosmos/src/CosmosSqlQuerySpecJsonConverter.cs index c782605dfa..b13eae0b6e 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosSqlQuerySpecJsonConverter.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosSqlQuerySpecJsonConverter.cs @@ -44,11 +44,14 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s // if the SqlParameter has stream value we dont pass it through the custom serializer. if (sqlParameter.Value is SerializedParameterValue serializedEncryptedData) { - using (StreamReader streamReader = new StreamReader(serializedEncryptedData.valueStream)) - { - string parameterValue = streamReader.ReadToEnd(); - writer.WriteRawValue(parameterValue); - } + StreamReader streamReader = new StreamReader(serializedEncryptedData.valueStream); + string parameterValue = streamReader.ReadToEnd(); + writer.WriteRawValue(parameterValue); + + // once read need to move back the pointer to start.If it gets reused. + // Query plan will be serialized multiple times for cross partition queries and for scenarios where the query plan can not be generated locally. + // Note: this is a stream passed by user. + serializedEncryptedData.valueStream.Position = 0; } else { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs index dfc5e2b3e2..9d52f10f12 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs @@ -1091,6 +1091,36 @@ public async Task QueryStreamValueTest() Assert.AreEqual(pageCount, fromStreamCount); } + // get result across pages,multiple requests by setting MaxItemCount to 1. + foreach (QueryDefinition queryDefinition in queryDefinitions) + { + toStreamCount = 0; + fromStreamCount = 0; + + List allItems = new List(); + int pageCount = 0; + using (FeedIterator feedIterator = containerSerializer.GetItemQueryIterator( + queryDefinition: queryDefinition, + requestOptions: new QueryRequestOptions { MaxItemCount = 1 })) + { + while (feedIterator.HasMoreResults) + { + // Only need once to verify correct serialization of the query definition + FeedResponse response = await feedIterator.ReadNextAsync(this.cancellationToken); + Assert.AreEqual(response.Count, response.Count()); + allItems.AddRange(response); + pageCount++; + } + } + + Assert.AreEqual(2, allItems.Count, $"missing query results. Only found: {allItems.Count} items for query:{queryDefinition.ToSqlQuerySpec().QueryText}"); + + // There should be no call to custom serializer since the parameter values are already serialized. + Assert.AreEqual(0, toStreamCount, $"missing to stream call. Expected: 0 , Actual: {toStreamCount} for query:{queryDefinition.ToSqlQuerySpec().QueryText}"); + Assert.AreEqual(pageCount, fromStreamCount); + } + + // Standard Cosmos Serializer Used CosmosClient clientStandardSerializer = TestCommon.CreateCosmosClient(useCustomSeralizer: false); From 72263136139dc08e210323ece6345027dbb7fd3d Mon Sep 17 00:00:00 2001 From: Santosh Kulkarni <66682828+kr-santosh@users.noreply.github.com> Date: Fri, 22 Apr 2022 17:18:50 +0530 Subject: [PATCH 2/6] Fix as per review comment. --- .../src/CosmosSqlQuerySpecJsonConverter.cs | 13 +++++-------- .../src/Query/v3Query/QueryDefinition.cs | 11 +++++++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/CosmosSqlQuerySpecJsonConverter.cs b/Microsoft.Azure.Cosmos/src/CosmosSqlQuerySpecJsonConverter.cs index b13eae0b6e..b6dc550b1e 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosSqlQuerySpecJsonConverter.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosSqlQuerySpecJsonConverter.cs @@ -44,14 +44,11 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s // if the SqlParameter has stream value we dont pass it through the custom serializer. if (sqlParameter.Value is SerializedParameterValue serializedEncryptedData) { - StreamReader streamReader = new StreamReader(serializedEncryptedData.valueStream); - string parameterValue = streamReader.ReadToEnd(); - writer.WriteRawValue(parameterValue); - - // once read need to move back the pointer to start.If it gets reused. - // Query plan will be serialized multiple times for cross partition queries and for scenarios where the query plan can not be generated locally. - // Note: this is a stream passed by user. - serializedEncryptedData.valueStream.Position = 0; + using (StreamReader streamReader = new StreamReader(new MemoryStream(serializedEncryptedData.byteArrayValue))) + { + string parameterValue = streamReader.ReadToEnd(); + writer.WriteRawValue(parameterValue); + } } else { diff --git a/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs b/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs index c1eed49cb3..c0d15725d8 100644 --- a/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs +++ b/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs @@ -132,10 +132,17 @@ public QueryDefinition WithParameter(string name, object value) /// An instance of . public QueryDefinition WithParameterStream(string name, Stream valueStream) { + if (!(valueStream is MemoryStream memoryStream)) + { + memoryStream = new MemoryStream(); + valueStream.CopyTo(memoryStream); + memoryStream.Position = 0; + } + // pack it into an internal type for identification. SerializedParameterValue serializedParameterValue = new SerializedParameterValue { - valueStream = valueStream + byteArrayValue = memoryStream.ToArray() }; return this.WithParameter(name, serializedParameterValue); @@ -200,6 +207,6 @@ IEnumerator IEnumerable.GetEnumerator() internal struct SerializedParameterValue { - internal Stream valueStream; + internal byte[] byteArrayValue; } } From b521b14a3880d2f5997572365198f2ef3f522583 Mon Sep 17 00:00:00 2001 From: Santosh Kulkarni <66682828+kr-santosh@users.noreply.github.com> Date: Fri, 22 Apr 2022 17:23:12 +0530 Subject: [PATCH 3/6] Update QueryDefinition.cs --- Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs b/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs index c0d15725d8..3ccd80cc19 100644 --- a/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs +++ b/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs @@ -136,7 +136,6 @@ public QueryDefinition WithParameterStream(string name, Stream valueStream) { memoryStream = new MemoryStream(); valueStream.CopyTo(memoryStream); - memoryStream.Position = 0; } // pack it into an internal type for identification. From 4ac6c660eec3542ec41eca0b84bacbff0201e621 Mon Sep 17 00:00:00 2001 From: Santosh Kulkarni <66682828+kr-santosh@users.noreply.github.com> Date: Mon, 25 Apr 2022 17:27:10 +0530 Subject: [PATCH 4/6] Fixes --- .../src/CosmosSqlQuerySpecJsonConverter.cs | 6 +----- .../src/Query/v3Query/QueryDefinition.cs | 10 +++++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/CosmosSqlQuerySpecJsonConverter.cs b/Microsoft.Azure.Cosmos/src/CosmosSqlQuerySpecJsonConverter.cs index b6dc550b1e..cc4dd8f9c9 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosSqlQuerySpecJsonConverter.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosSqlQuerySpecJsonConverter.cs @@ -44,11 +44,7 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s // if the SqlParameter has stream value we dont pass it through the custom serializer. if (sqlParameter.Value is SerializedParameterValue serializedEncryptedData) { - using (StreamReader streamReader = new StreamReader(new MemoryStream(serializedEncryptedData.byteArrayValue))) - { - string parameterValue = streamReader.ReadToEnd(); - writer.WriteRawValue(parameterValue); - } + writer.WriteRawValue(serializedEncryptedData.rawSerializedJsonValue); } else { diff --git a/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs b/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs index 3ccd80cc19..c67a67b090 100644 --- a/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs +++ b/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs @@ -132,16 +132,16 @@ public QueryDefinition WithParameter(string name, object value) /// An instance of . public QueryDefinition WithParameterStream(string name, Stream valueStream) { - if (!(valueStream is MemoryStream memoryStream)) + string rawSerializedJsonValue = null; + using (StreamReader streamReader = new StreamReader(valueStream)) { - memoryStream = new MemoryStream(); - valueStream.CopyTo(memoryStream); + rawSerializedJsonValue = streamReader.ReadToEnd(); } // pack it into an internal type for identification. SerializedParameterValue serializedParameterValue = new SerializedParameterValue { - byteArrayValue = memoryStream.ToArray() + rawSerializedJsonValue = rawSerializedJsonValue }; return this.WithParameter(name, serializedParameterValue); @@ -206,6 +206,6 @@ IEnumerator IEnumerable.GetEnumerator() internal struct SerializedParameterValue { - internal byte[] byteArrayValue; + internal string rawSerializedJsonValue; } } From 110e7964036abe0dd6abf7995da1d2fbd58d3a7b Mon Sep 17 00:00:00 2001 From: Santosh Kulkarni <66682828+kr-santosh@users.noreply.github.com> Date: Mon, 25 Apr 2022 17:33:41 +0530 Subject: [PATCH 5/6] Update QueryDefinition.cs --- Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs b/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs index c67a67b090..cbb7f3b37d 100644 --- a/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs +++ b/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs @@ -137,7 +137,7 @@ public QueryDefinition WithParameterStream(string name, Stream valueStream) { rawSerializedJsonValue = streamReader.ReadToEnd(); } - + // pack it into an internal type for identification. SerializedParameterValue serializedParameterValue = new SerializedParameterValue { From bdd1555b5e2b204d47eac939d938236b69276e18 Mon Sep 17 00:00:00 2001 From: Santosh Kulkarni <66682828+kr-santosh@users.noreply.github.com> Date: Mon, 25 Apr 2022 17:34:48 +0530 Subject: [PATCH 6/6] Update QueryDefinition.cs --- Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs b/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs index cbb7f3b37d..c67a67b090 100644 --- a/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs +++ b/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryDefinition.cs @@ -137,7 +137,7 @@ public QueryDefinition WithParameterStream(string name, Stream valueStream) { rawSerializedJsonValue = streamReader.ReadToEnd(); } - + // pack it into an internal type for identification. SerializedParameterValue serializedParameterValue = new SerializedParameterValue {