Retrieve original file properties when requested (#3360)

* Retrieve original file properties when requested

* Add comment

* run idp tests in CIrun

* fix sproc

* fix sproc

* Revert

* no dicom update category

* Missed to check in gen file

* Fix unit tests

* Add comment

* Fix unit test

* Fix int tests

* Change to initialVersion

* Fix build issue

build/common/run-e2e-connected-store-tests.yml

@@ -9,7 +9,7 @@ steps:
     downloadPath: '$(System.ArtifactsDirectory)'
     artifactName: 'IntegrationTests'
 
-- script: dotnet test "Microsoft.Health.Dicom.Web.Tests.E2E.dll" --filter "Category!=bvt-dp & Category!=dicomupdate" --logger trx --results-directory "$(Agent.TempDirectory)/TestResults" -e DicomServer__Features__EnableExternalStore="true"
+- script: dotnet test "Microsoft.Health.Dicom.Web.Tests.E2E.dll" --filter "Category!=bvt-dp" --logger trx --results-directory "$(Agent.TempDirectory)/TestResults" -e DicomServer__Features__EnableExternalStore="true"
   displayName: 'dotnet test Microsoft.Health.Dicom.Web.Tests.E2E.dll with EnableExternalStore'
   workingDirectory: '$(System.ArtifactsDirectory)/IntegrationTests/Microsoft.Health.Dicom.Web.Tests.E2E'
   condition: eq(${{ parameters.externalStorageTests }}, true)

src/Microsoft.Health.Dicom.Core.UnitTests/Features/Export/IdentifierExportSourceTests.cs

@@ -1,4 +1,4 @@
-// -------------------------------------------------------------------------------------------------
+// -------------------------------------------------------------------------------------------------
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
 // -------------------------------------------------------------------------------------------------
@@ -60,24 +60,24 @@ public async Task GivenInstances_WhenEnumerating_ThenYieldValues()
             new InstanceMetadata(new VersionedInstanceIdentifier("1000", "2000", "3000", 1, _partition), new InstanceProperties()),
         };
         _store
-            .GetInstanceIdentifierWithPropertiesAsync(_partition, "10", null, null, tokenSource.Token)
+            .GetInstanceIdentifierWithPropertiesAsync(_partition, "10", null, null, false, tokenSource.Token)
             .Returns(expected[..3]);
         _store
-            .GetInstanceIdentifierWithPropertiesAsync(_partition, "11", null, null, tokenSource.Token)
+            .GetInstanceIdentifierWithPropertiesAsync(_partition, "11", null, null, false, tokenSource.Token)
             .Returns(Array.Empty<InstanceMetadata>());
 
         _store
-            .GetInstanceIdentifierWithPropertiesAsync(_partition, "100", "200", null, tokenSource.Token)
+            .GetInstanceIdentifierWithPropertiesAsync(_partition, "100", "200", null, false, tokenSource.Token)
             .Returns(expected[3..6]);
         _store
-            .GetInstanceIdentifierWithPropertiesAsync(_partition, "100", "201", null, tokenSource.Token)
+            .GetInstanceIdentifierWithPropertiesAsync(_partition, "100", "201", null, false, tokenSource.Token)
             .Returns(Array.Empty<InstanceMetadata>());
 
         _store
-            .GetInstanceIdentifierWithPropertiesAsync(_partition, "1000", "2000", "3000", tokenSource.Token)
+            .GetInstanceIdentifierWithPropertiesAsync(_partition, "1000", "2000", "3000", false, tokenSource.Token)
             .Returns(new[] { expected[6] });
         _store
-            .GetInstanceIdentifierWithPropertiesAsync(_partition, "1000", "2000", "3001", tokenSource.Token)
+            .GetInstanceIdentifierWithPropertiesAsync(_partition, "1000", "2000", "3001", false, tokenSource.Token)
             .Returns(Array.Empty<InstanceMetadata>());
 
         // Enumerate
@@ -88,22 +88,22 @@ public async Task GivenInstances_WhenEnumerating_ThenYieldValues()
         // Check Results
         await _store
             .Received(1)
-            .GetInstanceIdentifierWithPropertiesAsync(_partition, "10", null, null, tokenSource.Token);
+            .GetInstanceIdentifierWithPropertiesAsync(_partition, "10", null, null, false, tokenSource.Token);
         await _store
             .Received(1)
-            .GetInstanceIdentifierWithPropertiesAsync(_partition, "11", null, null, tokenSource.Token);
+            .GetInstanceIdentifierWithPropertiesAsync(_partition, "11", null, null, false, tokenSource.Token);
         await _store
             .Received(1)
-            .GetInstanceIdentifierWithPropertiesAsync(_partition, "100", "200", null, tokenSource.Token);
+            .GetInstanceIdentifierWithPropertiesAsync(_partition, "100", "200", null, false, tokenSource.Token);
         await _store
             .Received(1)
-            .GetInstanceIdentifierWithPropertiesAsync(_partition, "100", "201", null, tokenSource.Token);
+            .GetInstanceIdentifierWithPropertiesAsync(_partition, "100", "201", null, false, tokenSource.Token);
         await _store
             .Received(1)
-            .GetInstanceIdentifierWithPropertiesAsync(_partition, "1000", "2000", "3000", tokenSource.Token);
+            .GetInstanceIdentifierWithPropertiesAsync(_partition, "1000", "2000", "3000", false, tokenSource.Token);
         await _store
             .Received(1)
-            .GetInstanceIdentifierWithPropertiesAsync(_partition, "1000", "2000", "3001", tokenSource.Token);
+            .GetInstanceIdentifierWithPropertiesAsync(_partition, "1000", "2000", "3001", false, tokenSource.Token);
 
         Assert.Same(expected[0], actual[0].Instance);
         Assert.Same(expected[1], actual[1].Instance);

src/Microsoft.Health.Dicom.Core.UnitTests/Features/Retrieve/RetrieveMetadataServiceTests.cs

@@ -156,7 +156,8 @@ public async Task GivenRetrieveInstanceMetadataRequestWithOriginalVersionForStud
     {
         IReadOnlyList<InstanceMetadata> versionedInstanceIdentifiers = SetupInstanceIdentifiersList(
             ResourceType.Study,
-            instanceProperty: new InstanceProperties() { OriginalVersion = 5 });
+            instanceProperty: new InstanceProperties() { OriginalVersion = 5 },
+            isInitialVersion: true);
 
         _metadataStore.GetInstanceMetadataAsync(5, DefaultCancellationToken).Returns(new DicomDataset());
 
@@ -226,7 +227,7 @@ public async Task GivenRetrieveInstanceMetadataRequestForInstance_WhenIsSuccessf
         Assert.Equal(1, _dicomRequestContextAccessor.RequestContext.PartCount);
     }
 
-    private List<InstanceMetadata> SetupInstanceIdentifiersList(ResourceType resourceType, Partition partition = null, InstanceProperties instanceProperty = null)
+    private List<InstanceMetadata> SetupInstanceIdentifiersList(ResourceType resourceType, Partition partition = null, InstanceProperties instanceProperty = null, bool isInitialVersion = false)
     {
         var dicomInstanceIdentifiersList = new List<InstanceMetadata>();
 
@@ -238,16 +239,16 @@ private List<InstanceMetadata> SetupInstanceIdentifiersList(ResourceType resourc
             case ResourceType.Study:
                 dicomInstanceIdentifiersList.Add(new InstanceMetadata(new VersionedInstanceIdentifier(_studyInstanceUid, TestUidGenerator.Generate(), TestUidGenerator.Generate(), version: 0), instanceProperty));
                 dicomInstanceIdentifiersList.Add(new InstanceMetadata(new VersionedInstanceIdentifier(_studyInstanceUid, TestUidGenerator.Generate(), TestUidGenerator.Generate(), version: 1), instanceProperty));
-                _instanceStore.GetInstanceIdentifierWithPropertiesAsync(partition, _studyInstanceUid, cancellationToken: DefaultCancellationToken).Returns(dicomInstanceIdentifiersList);
+                _instanceStore.GetInstanceIdentifierWithPropertiesAsync(Arg.Is<Partition>(x => x.Key == partition.Key), _studyInstanceUid, isInitialVersion: isInitialVersion, cancellationToken: DefaultCancellationToken).Returns(dicomInstanceIdentifiersList);
                 break;
             case ResourceType.Series:
                 dicomInstanceIdentifiersList.Add(new InstanceMetadata(new VersionedInstanceIdentifier(_studyInstanceUid, _seriesInstanceUid, TestUidGenerator.Generate(), version: 0), instanceProperty));
                 dicomInstanceIdentifiersList.Add(new InstanceMetadata(new VersionedInstanceIdentifier(_studyInstanceUid, _seriesInstanceUid, TestUidGenerator.Generate(), version: 1), instanceProperty));
-                _instanceStore.GetInstanceIdentifierWithPropertiesAsync(partition, _studyInstanceUid, _seriesInstanceUid, cancellationToken: DefaultCancellationToken).Returns(dicomInstanceIdentifiersList);
+                _instanceStore.GetInstanceIdentifierWithPropertiesAsync(Arg.Is<Partition>(x => x.Key == partition.Key), _studyInstanceUid, _seriesInstanceUid, isInitialVersion: isInitialVersion, cancellationToken: DefaultCancellationToken).Returns(dicomInstanceIdentifiersList);
                 break;
             case ResourceType.Instance:
                 dicomInstanceIdentifiersList.Add(new InstanceMetadata(new VersionedInstanceIdentifier(_studyInstanceUid, _seriesInstanceUid, _sopInstanceUid, version: 0), instanceProperty));
-                _instanceStore.GetInstanceIdentifierWithPropertiesAsync(partition, _studyInstanceUid, _seriesInstanceUid, _sopInstanceUid, DefaultCancellationToken).Returns(dicomInstanceIdentifiersList);
+                _instanceStore.GetInstanceIdentifierWithPropertiesAsync(Arg.Is<Partition>(x => x.Key == partition.Key), _studyInstanceUid, _seriesInstanceUid, _sopInstanceUid, isInitialVersion: isInitialVersion, DefaultCancellationToken).Returns(dicomInstanceIdentifiersList);
                 break;
         }
 

src/Microsoft.Health.Dicom.Core.UnitTests/Features/Retrieve/RetrieveRenderedServiceTests.cs

@@ -400,7 +400,7 @@ private List<InstanceMetadata> SetupInstanceIdentifiersList(Partition partition
         partition ??= Partition.Default;
 
         dicomInstanceIdentifiersList.Add(new InstanceMetadata(new VersionedInstanceIdentifier(_studyInstanceUid, _firstSeriesInstanceUid, TestUidGenerator.Generate(), 0, partition), instanceProperty));
-        _instanceStore.GetInstanceIdentifierWithPropertiesAsync(dicomInstanceIdentifiersList[0].VersionedInstanceIdentifier.Partition, _studyInstanceUid, _firstSeriesInstanceUid, _sopInstanceUid, DefaultCancellationToken).Returns(dicomInstanceIdentifiersList);
+        _instanceStore.GetInstanceIdentifierWithPropertiesAsync(dicomInstanceIdentifiersList[0].VersionedInstanceIdentifier.Partition, _studyInstanceUid, _firstSeriesInstanceUid, _sopInstanceUid, false, DefaultCancellationToken).Returns(dicomInstanceIdentifiersList);
 
         return dicomInstanceIdentifiersList;
     }

src/Microsoft.Health.Dicom.Core.UnitTests/Features/Retrieve/RetrieveResourceServiceTests.cs

@@ -168,7 +168,8 @@ public async Task GivenStoredInstancesWithOriginalVersion_WhenRetrieveRequestFor
         // Add multiple instances to validate that we return the requested instance and ignore the other(s).
         List<InstanceMetadata> versionedInstanceIdentifiers = SetupInstanceIdentifiersList(
             ResourceType.Study,
-            instanceProperty: new InstanceProperties() { HasFrameMetadata = true, OriginalVersion = 5 });
+            instanceProperty: new InstanceProperties() { HasFrameMetadata = true, OriginalVersion = 5 },
+            isOriginalVersion: true);
 
         _fileStore.GetFilePropertiesAsync(Arg.Any<long>(), Partition.Default, null, DefaultCancellationToken).Returns(new FileProperties { ContentLength = 1000 });
 
@@ -218,7 +219,8 @@ public async Task GivenInstancesWithOriginalVersion_WhenRetrieveRequestForStudyF
                 OriginalVersion = originalVersion,
                 TransferSyntaxUid = "1.2.840.10008.1.2.4.90",
                 FileProperties = fileProperties
-            });
+            },
+            isOriginalVersion: true);
         _fileStore.GetFilePropertiesAsync(originalVersion, versionedInstanceIdentifiers.First().VersionedInstanceIdentifier.Partition, fileProperties, DefaultCancellationToken)
             .Returns(new FileProperties { ContentLength = new RetrieveConfiguration().MaxDicomFileSize });
 
@@ -257,7 +259,7 @@ public async Task GivenSpecificTransferSyntax_WhenRetrieveRequestForStudy_ThenIn
     {
         // Add multiple instances to validate that we return the requested instance and ignore the other(s).
         var instanceMetadata = new InstanceMetadata(new VersionedInstanceIdentifier(_studyInstanceUid, _firstSeriesInstanceUid, TestUidGenerator.Generate(), 0, Partition.Default), new InstanceProperties());
-        _instanceStore.GetInstanceIdentifierWithPropertiesAsync(_dicomRequestContextAccessor.RequestContext.DataPartition, _studyInstanceUid, null, null, DefaultCancellationToken).Returns(new[] { instanceMetadata });
+        _instanceStore.GetInstanceIdentifierWithPropertiesAsync(_dicomRequestContextAccessor.RequestContext.DataPartition, _studyInstanceUid, null, null, false, DefaultCancellationToken).Returns(new[] { instanceMetadata });
 
         // For each instance identifier, set up the fileStore to return a stream containing a file associated with the identifier.
         var streamsAndStoredFile = await RetrieveHelpers.StreamAndStoredFileFromDataset(RetrieveHelpers.GenerateDatasetsFromIdentifiers(instanceMetadata.VersionedInstanceIdentifier), _recyclableMemoryStreamManager);
@@ -829,7 +831,7 @@ await _fileStore.GetFileFrameAsync(
             Arg.Any<CancellationToken>());
     }
 
-    private List<InstanceMetadata> SetupInstanceIdentifiersList(ResourceType resourceType, Partition partition = null, InstanceProperties instanceProperty = null, bool withFileProperties = false)
+    private List<InstanceMetadata> SetupInstanceIdentifiersList(ResourceType resourceType, Partition partition = null, InstanceProperties instanceProperty = null, bool withFileProperties = false, bool isOriginalVersion = false)
     {
         var dicomInstanceIdentifiersList = new List<InstanceMetadata>();
 
@@ -842,18 +844,18 @@ private List<InstanceMetadata> SetupInstanceIdentifiersList(ResourceType resourc
                 dicomInstanceIdentifiersList.Add(new InstanceMetadata(new VersionedInstanceIdentifier(_studyInstanceUid, _firstSeriesInstanceUid, TestUidGenerator.Generate(), 0, partition), instanceProperty));
                 dicomInstanceIdentifiersList.Add(new InstanceMetadata(new VersionedInstanceIdentifier(_studyInstanceUid, _firstSeriesInstanceUid, TestUidGenerator.Generate(), 1, partition), instanceProperty));
                 dicomInstanceIdentifiersList.Add(new InstanceMetadata(new VersionedInstanceIdentifier(_studyInstanceUid, _secondSeriesInstanceUid, TestUidGenerator.Generate(), 2, partition), instanceProperty));
-                _instanceStore.GetInstanceIdentifierWithPropertiesAsync(partition, _studyInstanceUid, null, null, DefaultCancellationToken).Returns(dicomInstanceIdentifiersList);
+                _instanceStore.GetInstanceIdentifierWithPropertiesAsync(partition, _studyInstanceUid, null, null, isOriginalVersion, DefaultCancellationToken).Returns(dicomInstanceIdentifiersList);
                 break;
             case ResourceType.Series:
                 dicomInstanceIdentifiersList.Add(new InstanceMetadata(new VersionedInstanceIdentifier(_studyInstanceUid, _firstSeriesInstanceUid, TestUidGenerator.Generate(), 0, partition), instanceProperty));
                 dicomInstanceIdentifiersList.Add(new InstanceMetadata(new VersionedInstanceIdentifier(_studyInstanceUid, _firstSeriesInstanceUid, TestUidGenerator.Generate(), 1, partition), instanceProperty));
-                _instanceStore.GetInstanceIdentifierWithPropertiesAsync(partition, _studyInstanceUid, _firstSeriesInstanceUid, null, DefaultCancellationToken).Returns(dicomInstanceIdentifiersList);
+                _instanceStore.GetInstanceIdentifierWithPropertiesAsync(partition, _studyInstanceUid, _firstSeriesInstanceUid, null, isOriginalVersion, DefaultCancellationToken).Returns(dicomInstanceIdentifiersList);
                 break;
             case ResourceType.Instance:
             case ResourceType.Frames:
                 dicomInstanceIdentifiersList.Add(new InstanceMetadata(new VersionedInstanceIdentifier(_studyInstanceUid, _firstSeriesInstanceUid, _sopInstanceUid, 3, partition), instanceProperty));
                 dicomInstanceIdentifiersList.Add(new InstanceMetadata(new VersionedInstanceIdentifier(_studyInstanceUid, _firstSeriesInstanceUid, TestUidGenerator.Generate(), 4, partition), instanceProperty));
-                _instanceStore.GetInstanceIdentifierWithPropertiesAsync(partition, _studyInstanceUid, _firstSeriesInstanceUid, _sopInstanceUid, DefaultCancellationToken).Returns(dicomInstanceIdentifiersList.SkipLast(1).ToList());
+                _instanceStore.GetInstanceIdentifierWithPropertiesAsync(Arg.Is<Partition>(x => x.Key == partition.Key), _studyInstanceUid, _firstSeriesInstanceUid, _sopInstanceUid, isOriginalVersion, DefaultCancellationToken).Returns(dicomInstanceIdentifiersList.SkipLast(1).ToList());
                 var identifier = new InstanceIdentifier(_studyInstanceUid, _firstSeriesInstanceUid, _sopInstanceUid, partition);
                 _instanceMetadataCache.GetAsync(Arg.Any<object>(), identifier, Arg.Any<Func<InstanceIdentifier, CancellationToken, Task<InstanceMetadata>>>(), Arg.Any<CancellationToken>()).Returns(dicomInstanceIdentifiersList.First());
                 break;

src/Microsoft.Health.Dicom.Core/Features/Export/IdentifierExportSource.cs

@@ -1,4 +1,4 @@
-// -------------------------------------------------------------------------------------------------
+// -------------------------------------------------------------------------------------------------
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
 // -------------------------------------------------------------------------------------------------
@@ -48,6 +48,7 @@ public async IAsyncEnumerator<ReadResult> GetAsyncEnumerator(CancellationToken c
                 identifier.StudyInstanceUid,
                 identifier.SeriesInstanceUid,
                 identifier.SopInstanceUid,
+                isInitialVersion: false,
                 cancellationToken);
 
             if (instances.Count == 0)

src/Microsoft.Health.Dicom.Core/Features/Retrieve/IInstanceStore.cs

@@ -112,13 +112,15 @@ Task<IReadOnlyList<WatermarkRange>> GetInstanceBatchesAsync(
     /// <param name="studyInstanceUid">The study identifier.</param>
     /// <param name="seriesInstanceUid">The series identifier.</param>
     /// <param name="sopInstanceUid">The instance identifier.</param>
+    /// <param name="isInitialVersion">True, if requesting original version. If original version is null it will uses the version.</param>
     /// <param name="cancellationToken">An optional cancellation token.</param>
     /// <returns>Instance identifiers.</returns>
     Task<IReadOnlyList<InstanceMetadata>> GetInstanceIdentifierWithPropertiesAsync(
         Partition partition,
         string studyInstanceUid,
         string seriesInstanceUid = null,
         string sopInstanceUid = null,
+        bool isInitialVersion = false,
         CancellationToken cancellationToken = default);
 
     /// <summary>

src/Microsoft.Health.Dicom.Core/Features/Retrieve/InstanceStoreExtensions.cs

@@ -24,11 +24,12 @@ public static async Task<IReadOnlyList<InstanceMetadata>> GetInstancesWithProper
             string studyInstanceUid,
             string seriesInstanceUid,
             string sopInstanceUid,
-            CancellationToken cancellationToken)
+            bool isInitialVersion = false,
+            CancellationToken cancellationToken = default)
     {
         EnsureArg.IsNotNull(instanceStore, nameof(instanceStore));
 
-        IReadOnlyList<InstanceMetadata> instancesToRetrieve = await instanceStore.GetInstanceIdentifierWithPropertiesAsync(partition, studyInstanceUid, seriesInstanceUid, sopInstanceUid, cancellationToken);
+        IReadOnlyList<InstanceMetadata> instancesToRetrieve = await instanceStore.GetInstanceIdentifierWithPropertiesAsync(partition, studyInstanceUid, seriesInstanceUid, sopInstanceUid, isInitialVersion, cancellationToken);
 
         if (!instancesToRetrieve.Any())
         {

src/Microsoft.Health.Dicom.Core/Features/Retrieve/RetrieveMetadataService.cs

@@ -56,6 +56,7 @@ public async Task<RetrieveMetadataResponse> RetrieveStudyInstanceMetadataAsync(s
             studyInstanceUid,
             seriesInstanceUid: null,
             sopInstanceUid: null,
+            isOriginalVersionRequested,
             cancellationToken);
 
         string eTag = _eTagGenerator.GetETag(ResourceType.Study, retrieveInstances);
@@ -71,6 +72,7 @@ public async Task<RetrieveMetadataResponse> RetrieveSeriesInstanceMetadataAsync(
                 studyInstanceUid,
                 seriesInstanceUid,
                 sopInstanceUid: null,
+                isOriginalVersionRequested,
                 cancellationToken);
 
         string eTag = _eTagGenerator.GetETag(ResourceType.Series, retrieveInstances);
@@ -86,6 +88,7 @@ public async Task<RetrieveMetadataResponse> RetrieveSopInstanceMetadataAsync(str
             studyInstanceUid,
             seriesInstanceUid,
             sopInstanceUid,
+            isOriginalVersionRequested,
             cancellationToken);
 
         string eTag = _eTagGenerator.GetETag(ResourceType.Instance, retrieveInstances);