From fe0431c70fceba4dd652f53a8c9da4fb82ed0fd2 Mon Sep 17 00:00:00 2001 From: Esma Dautovic Date: Thu, 25 Jan 2024 15:34:43 -0800 Subject: [PATCH] add function to get sum and total indexed --- .../Features/Common/IndexedFileProperties.cs | 22 +++++ .../Features/Store/IIndexDataStore.cs | 7 ++ .../IndexMetricsCollectionFunctionTests.cs | 79 ++++++++++++++++ .../IndexMetricsCollectionFunction.cs | 80 ++++++++++++++++ .../IndexMetricsCollectionOptions.cs | 28 ++++++ .../Telemetry/IndexMetricsCollectionMeter.cs | 43 +++++++++ .../ServiceCollectionExtensions.cs | 2 + .../Features/Schema/Migrations/57.diff.sql | 15 +++ .../Features/Schema/Migrations/57.sql | 11 +++ .../Schema/Model/VLatest.Generated.cs | 14 +++ ...otalAndSumContentLengthIndexedAsyncV57.sql | 19 ++++ .../Features/Store/SqlIndexDataStore.cs | 6 ++ .../Features/Store/SqlIndexDataStoreV1.cs | 5 + .../Features/Store/SqlIndexDataStoreV57.cs | 51 ++++++++++ .../DicomSqlServerRegistrationExtensions.cs | 1 + .../IndexedFilePropertiesCollection.cs | 13 +++ .../Persistence/IndexedFilePropertiesTests.cs | 94 +++++++++++++++++++ .../Persistence/SqlDataStoreTestsFixture.cs | 1 + 18 files changed, 491 insertions(+) create mode 100644 src/Microsoft.Health.Dicom.Core/Features/Common/IndexedFileProperties.cs create mode 100644 src/Microsoft.Health.Dicom.Functions.UnitTests/IndexMetricsCollection/IndexMetricsCollectionFunctionTests.cs create mode 100644 src/Microsoft.Health.Dicom.Functions/IndexMetricsCollection/IndexMetricsCollectionFunction.cs create mode 100644 src/Microsoft.Health.Dicom.Functions/IndexMetricsCollection/IndexMetricsCollectionOptions.cs create mode 100644 src/Microsoft.Health.Dicom.Functions/IndexMetricsCollection/Telemetry/IndexMetricsCollectionMeter.cs create mode 100644 src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Sql/Sprocs/GetTotalAndSumContentLengthIndexedAsyncV57.sql create mode 100644 src/Microsoft.Health.Dicom.SqlServer/Features/Store/SqlIndexDataStoreV57.cs create mode 100644 test/Microsoft.Health.Dicom.Tests.Integration/IndexedFilePropertiesCollection.cs create mode 100644 test/Microsoft.Health.Dicom.Tests.Integration/Persistence/IndexedFilePropertiesTests.cs diff --git a/src/Microsoft.Health.Dicom.Core/Features/Common/IndexedFileProperties.cs b/src/Microsoft.Health.Dicom.Core/Features/Common/IndexedFileProperties.cs new file mode 100644 index 0000000000..5ce5b9f5a3 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Core/Features/Common/IndexedFileProperties.cs @@ -0,0 +1,22 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Dicom.Core.Features.Common; + +/// +/// Metadata on FileProperty table in database +/// +public class IndexedFileProperties +{ + /// + /// Total indexed FileProperty in database + /// + public int TotalIndexed { get; init; } + + /// + /// Total sum of all ContentLength rows in FileProperty table + /// + public long TotalSum { get; init; } +} diff --git a/src/Microsoft.Health.Dicom.Core/Features/Store/IIndexDataStore.cs b/src/Microsoft.Health.Dicom.Core/Features/Store/IIndexDataStore.cs index 7c82c3b2c8..33c7a763a2 100644 --- a/src/Microsoft.Health.Dicom.Core/Features/Store/IIndexDataStore.cs +++ b/src/Microsoft.Health.Dicom.Core/Features/Store/IIndexDataStore.cs @@ -185,4 +185,11 @@ public interface IIndexDataStore /// The cancellation token. /// A task that with list of instance metadata with new watermark. Task UpdateFilePropertiesContentLengthAsync(IReadOnlyDictionary filePropertiesByWatermark, CancellationToken cancellationToken = default); + + /// + /// Retrieves total count in FileProperty table and summation of all content length values across FileProperty table. + /// + /// The cancellation token. + /// A task that gets the count + Task GetIndexedFilePropertiesAsync(CancellationToken cancellationToken = default); } diff --git a/src/Microsoft.Health.Dicom.Functions.UnitTests/IndexMetricsCollection/IndexMetricsCollectionFunctionTests.cs b/src/Microsoft.Health.Dicom.Functions.UnitTests/IndexMetricsCollection/IndexMetricsCollectionFunctionTests.cs new file mode 100644 index 0000000000..65e18c66fc --- /dev/null +++ b/src/Microsoft.Health.Dicom.Functions.UnitTests/IndexMetricsCollection/IndexMetricsCollectionFunctionTests.cs @@ -0,0 +1,79 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Health.Dicom.Core.Configs; +using Microsoft.Health.Dicom.Core.Features.Common; +using Microsoft.Health.Dicom.Core.Features.Store; +using Microsoft.Health.Dicom.Core.Features.Telemetry; +using Microsoft.Health.Dicom.Functions.IndexMetricsCollection; +using Microsoft.Health.Dicom.Functions.IndexMetricsCollection.Telemetry; +using NSubstitute; +using NSubstitute.ExceptionExtensions; +using OpenTelemetry; +using OpenTelemetry.Metrics; +using Xunit; + +namespace Microsoft.Health.Dicom.Functions.UnitTests.IndexMetricsCollection; + +public class IndexMetricsCollectionFunctionTests +{ + private readonly IndexMetricsCollectionFunction _collectionFunction; + private readonly IIndexDataStore _indexStore; + private readonly IndexMetricsCollectionMeter _meter; + private List _exportedItems; + private MeterProvider _meterProvider; + private readonly TimerInfo _timer; + + public IndexMetricsCollectionFunctionTests() + { + _meter = new IndexMetricsCollectionMeter(); + _indexStore = Substitute.For(); + _collectionFunction = new IndexMetricsCollectionFunction( + _indexStore, + Options.Create(new FeatureConfiguration { EnableExternalStore = true, }), + _meter); + _timer = Substitute.For(default, default, default); + } + + private void InitializeMetricExporter() + { + _exportedItems = new List(); + _meterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter($"{OpenTelemetryLabels.BaseMeterName}.{IndexMetricsCollectionMeter.MeterName}") + .AddInMemoryExporter(_exportedItems) + .Build(); + } + + [Fact] + public async Task GivenIndexMetricsCollectionFunction_WhenRun_ThenIndexMetricsCollectionsCompletedCounterIsIncremented() + { + InitializeMetricExporter(); + _indexStore.GetIndexedFilePropertiesAsync().ReturnsForAnyArgs(new IndexedFileProperties()); + + await _collectionFunction.Run(_timer, NullLogger.Instance); + + _meterProvider.ForceFlush(); + Assert.Single(_exportedItems); + Assert.Equal(nameof(_meter.IndexMetricsCollectionsCompletedCounter), _exportedItems[0].Name); + } + + [Fact] + public async Task GivenIndexMetricsCollectionFunction_WhenRunException_ThenIndexMetricsCollectionsCompletedCounterIsNotIncremented() + { + InitializeMetricExporter(); + _indexStore.GetIndexedFilePropertiesAsync().ThrowsForAnyArgs(new Exception()); + + await Assert.ThrowsAsync(async () => await _collectionFunction.Run(_timer, NullLogger.Instance)); + + _meterProvider.ForceFlush(); + Assert.Empty(_exportedItems); + } +} diff --git a/src/Microsoft.Health.Dicom.Functions/IndexMetricsCollection/IndexMetricsCollectionFunction.cs b/src/Microsoft.Health.Dicom.Functions/IndexMetricsCollection/IndexMetricsCollectionFunction.cs new file mode 100644 index 0000000000..7a35209441 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Functions/IndexMetricsCollection/IndexMetricsCollectionFunction.cs @@ -0,0 +1,80 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Diagnostics; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Azure.WebJobs; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.Core; +using Microsoft.Health.Dicom.Core.Configs; +using Microsoft.Health.Dicom.Core.Features.Common; +using Microsoft.Health.Dicom.Core.Features.Store; +using Microsoft.Health.Dicom.Functions.IndexMetricsCollection.Telemetry; + +namespace Microsoft.Health.Dicom.Functions.IndexMetricsCollection; + +/// +/// A function for collecting index metrics +/// +public class IndexMetricsCollectionFunction +{ + private readonly IIndexDataStore _indexDataStore; + private readonly IndexMetricsCollectionMeter _meter; + private readonly bool _externalStoreEnabled; + private readonly bool _enableDataPartitions; + + public IndexMetricsCollectionFunction( + IIndexDataStore indexDataStore, + IOptions featureConfiguration, + IndexMetricsCollectionMeter meter) + { + EnsureArg.IsNotNull(featureConfiguration, nameof(featureConfiguration)); + _indexDataStore = EnsureArg.IsNotNull(indexDataStore, nameof(indexDataStore)); + _meter = EnsureArg.IsNotNull(meter, nameof(meter)); + _externalStoreEnabled = featureConfiguration.Value.EnableExternalStore; + _enableDataPartitions = featureConfiguration.Value.EnableDataPartitions; + } + + + /// + /// Asynchronously collects index metrics. + /// + /// The timer which tracks the invocation schedule. + /// A diagnostic logger. + /// A task that represents the asynchronous metrics collection operation. + [FunctionName(nameof(IndexMetricsCollectionFunction))] + public async Task Run( + [TimerTrigger(IndexMetricsCollectionOptions.Frequency)] TimerInfo invocationTimer, + ILogger log) + { + EnsureArg.IsNotNull(invocationTimer, nameof(invocationTimer)); + EnsureArg.IsNotNull(log, nameof(log)); + + log.LogInformation("Collecting a daily summation starting. At: {Timestamp}", Clock.UtcNow); + if (invocationTimer.IsPastDue) + { + log.LogWarning("Current function invocation is running late."); + } + + Stopwatch stopwatch = new Stopwatch(); + stopwatch.Start(); + + IndexedFileProperties indexedFileProperties = await _indexDataStore.GetIndexedFilePropertiesAsync(); + + stopwatch.Stop(); + + log.LogInformation("Collecting a daily summation time taken: {ElapsedTime} ms with ExternalStoreEnabled: {ExternalStoreEnabled} and DataPartitionsEnabled: {PartitionsEnabled}", stopwatch.ElapsedMilliseconds, _externalStoreEnabled, _enableDataPartitions); + + log.LogInformation("DICOM telemetry - total files indexed: {0} with ExternalStoreEnabled: {ExternalStoreEnabled} and DataPartitionsEnabled: {PartitionsEnabled}", indexedFileProperties.TotalIndexed, _externalStoreEnabled, _enableDataPartitions); + + log.LogInformation("DICOM telemetry - total content length indexed: {0} with ExternalStoreEnabled: {ExternalStoreEnabled} and DataPartitionsEnabled: {PartitionsEnabled}", indexedFileProperties.TotalSum, _externalStoreEnabled, _enableDataPartitions); + + log.LogInformation("Collecting a daily summation completed. with ExternalStoreEnabled: {ExternalStoreEnabled} and DataPartitionsEnabled: {PartitionsEnabled}", _externalStoreEnabled, _enableDataPartitions); + + _meter.IndexMetricsCollectionsCompletedCounter.Add(1, IndexMetricsCollectionMeter.CreateTelemetryDimension(_externalStoreEnabled, _enableDataPartitions)); + } +} \ No newline at end of file diff --git a/src/Microsoft.Health.Dicom.Functions/IndexMetricsCollection/IndexMetricsCollectionOptions.cs b/src/Microsoft.Health.Dicom.Functions/IndexMetricsCollection/IndexMetricsCollectionOptions.cs new file mode 100644 index 0000000000..77dd26947d --- /dev/null +++ b/src/Microsoft.Health.Dicom.Functions/IndexMetricsCollection/IndexMetricsCollectionOptions.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + + +using System.ComponentModel.DataAnnotations; + +namespace Microsoft.Health.Dicom.Functions.IndexMetricsCollection; + +/// +/// +/// +public static class IndexMetricsCollectionOptions +{ + /// + /// The default section name for in a configuration. + /// + public const string SectionName = "IndexMetricsCollection"; + + /// + /// Gets or sets the cron expression that indicates how frequently to run the index metrics collection function. + /// + /// A value cron expression + [Required] + public const string Frequency = "* * * * *"; // Every day at midnight + // public const string Frequency = "0 0 * * *"; // Every day at midnight +} diff --git a/src/Microsoft.Health.Dicom.Functions/IndexMetricsCollection/Telemetry/IndexMetricsCollectionMeter.cs b/src/Microsoft.Health.Dicom.Functions/IndexMetricsCollection/Telemetry/IndexMetricsCollectionMeter.cs new file mode 100644 index 0000000000..cf801e3341 --- /dev/null +++ b/src/Microsoft.Health.Dicom.Functions/IndexMetricsCollection/Telemetry/IndexMetricsCollectionMeter.cs @@ -0,0 +1,43 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using Microsoft.Health.Dicom.Core.Features.Telemetry; + +namespace Microsoft.Health.Dicom.Functions.IndexMetricsCollection.Telemetry; + +public sealed class IndexMetricsCollectionMeter : IDisposable +{ + private readonly Meter _meter; + internal const string MeterName = "IndexMetricsCollection"; + + public IndexMetricsCollectionMeter() + { + _meter = new Meter($"{OpenTelemetryLabels.BaseMeterName}.{MeterName}", "1.0"); + + IndexMetricsCollectionsCompletedCounter = + _meter.CreateCounter( + nameof(IndexMetricsCollectionsCompletedCounter), + description: "Represents a successful run of the index metrics collection function."); + } + + + public static KeyValuePair[] CreateTelemetryDimension(bool externalStoreEnabled, bool dataPartitionsEnabled) => + new[] + { + new KeyValuePair("ExternalStoreEnabled", externalStoreEnabled), + new KeyValuePair("DataPartitionsEnabled", dataPartitionsEnabled), + }; + + /// + /// Represents a successful run of the index metrics collection function + /// + public Counter IndexMetricsCollectionsCompletedCounter { get; } + + public void Dispose() + => _meter.Dispose(); +} \ No newline at end of file diff --git a/src/Microsoft.Health.Dicom.Functions/Registration/ServiceCollectionExtensions.cs b/src/Microsoft.Health.Dicom.Functions/Registration/ServiceCollectionExtensions.cs index efb42dfcf8..294f12923b 100644 --- a/src/Microsoft.Health.Dicom.Functions/Registration/ServiceCollectionExtensions.cs +++ b/src/Microsoft.Health.Dicom.Functions/Registration/ServiceCollectionExtensions.cs @@ -24,6 +24,7 @@ using Microsoft.Health.Dicom.Functions.DataCleanup; using Microsoft.Health.Dicom.Functions.Export; using Microsoft.Health.Dicom.Functions.Indexing; +using Microsoft.Health.Dicom.Functions.IndexMetricsCollection.Telemetry; using Microsoft.Health.Dicom.Functions.Update; using Microsoft.Health.Dicom.SqlServer.Registration; using Microsoft.Health.Extensions.DependencyInjection; @@ -70,6 +71,7 @@ public static IDicomFunctionsBuilder ConfigureFunctions( .ConfigureDurableFunctionSerialization() .AddJsonSerializerOptions(o => o.ConfigureDefaultDicomSettings()) .AddSingleton() + .AddSingleton() .AddSingleton()); } diff --git a/src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Migrations/57.diff.sql b/src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Migrations/57.diff.sql index 584db3127c..96ed66bffc 100644 --- a/src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Migrations/57.diff.sql +++ b/src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Migrations/57.diff.sql @@ -1,4 +1,19 @@ SET XACT_ABORT ON + +BEGIN TRANSACTION +GO +CREATE OR ALTER PROCEDURE dbo.GetTotalAndSumContentLengthIndexedAsyncV57 + AS +BEGIN + SET NOCOUNT ON; + SET XACT_ABORT ON; +SELECT count(*), + SUM(ContentLength) +FROM dbo.FileProperty; +END +GO + +COMMIT TRANSACTION IF NOT EXISTS ( diff --git a/src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Migrations/57.sql b/src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Migrations/57.sql index b9a6102443..964344068c 100644 --- a/src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Migrations/57.sql +++ b/src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Migrations/57.sql @@ -2746,6 +2746,17 @@ BEGIN AND i.PartitionKey = sv.PartitionKey; END +GO +CREATE OR ALTER PROCEDURE dbo.GetTotalAndSumContentLengthIndexedAsyncV57 +AS +BEGIN + SET NOCOUNT ON; + SET XACT_ABORT ON; + SELECT count(*), + SUM(ContentLength) + FROM dbo.FileProperty; +END + GO CREATE OR ALTER PROCEDURE dbo.GetWorkitemMetadata @partitionKey INT, @workitemUid VARCHAR (64), @procedureStepStateTagPath VARCHAR (64) diff --git a/src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Model/VLatest.Generated.cs b/src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Model/VLatest.Generated.cs index 0258190559..6c1a91207e 100644 --- a/src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Model/VLatest.Generated.cs +++ b/src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Model/VLatest.Generated.cs @@ -87,6 +87,7 @@ internal class VLatest internal readonly static GetPartitionsProcedure GetPartitions = new GetPartitionsProcedure(); internal readonly static GetSeriesResultProcedure GetSeriesResult = new GetSeriesResultProcedure(); internal readonly static GetStudyResultProcedure GetStudyResult = new GetStudyResultProcedure(); + internal readonly static GetTotalAndSumContentLengthIndexedAsyncV57Procedure GetTotalAndSumContentLengthIndexedAsyncV57 = new GetTotalAndSumContentLengthIndexedAsyncV57Procedure(); internal readonly static GetWorkitemMetadataProcedure GetWorkitemMetadata = new GetWorkitemMetadataProcedure(); internal readonly static GetWorkitemQueryTagsProcedure GetWorkitemQueryTags = new GetWorkitemQueryTagsProcedure(); internal readonly static IIndexInstanceCoreV9Procedure IIndexInstanceCoreV9 = new IIndexInstanceCoreV9Procedure(); @@ -2154,6 +2155,19 @@ internal GetStudyResultTableValuedParameters(global::System.Collections.Generic. internal global::System.Collections.Generic.IEnumerable WatermarkTableType { get; } } + internal class GetTotalAndSumContentLengthIndexedAsyncV57Procedure : StoredProcedure + { + internal GetTotalAndSumContentLengthIndexedAsyncV57Procedure() : base("dbo.GetTotalAndSumContentLengthIndexedAsyncV57") + { + } + + public void PopulateCommand(SqlCommandWrapper command) + { + command.CommandType = global::System.Data.CommandType.StoredProcedure; + command.CommandText = "dbo.GetTotalAndSumContentLengthIndexedAsyncV57"; + } + } + internal class GetWorkitemMetadataProcedure : StoredProcedure { internal GetWorkitemMetadataProcedure() : base("dbo.GetWorkitemMetadata") diff --git a/src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Sql/Sprocs/GetTotalAndSumContentLengthIndexedAsyncV57.sql b/src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Sql/Sprocs/GetTotalAndSumContentLengthIndexedAsyncV57.sql new file mode 100644 index 0000000000..0ae3508170 --- /dev/null +++ b/src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Sql/Sprocs/GetTotalAndSumContentLengthIndexedAsyncV57.sql @@ -0,0 +1,19 @@ +/***************************************************************************************/ +-- STORED PROCEDURE +-- GetTotalAndSumContentLengthIndexedAsyncV57 +-- +-- FIRST SCHEMA VERSION +-- 57 +-- +-- DESCRIPTION +-- Retrieves total sum of content length across all FileProperty rows +-- +/***************************************************************************************/ +CREATE OR ALTER PROCEDURE dbo.GetTotalAndSumContentLengthIndexedAsyncV57 + AS +BEGIN + SET NOCOUNT ON + SET XACT_ABORT ON + +SELECT count(*), SUM(ContentLength) FROM dbo.FileProperty +END \ No newline at end of file diff --git a/src/Microsoft.Health.Dicom.SqlServer/Features/Store/SqlIndexDataStore.cs b/src/Microsoft.Health.Dicom.SqlServer/Features/Store/SqlIndexDataStore.cs index 1b6276d028..c5d2d73c14 100644 --- a/src/Microsoft.Health.Dicom.SqlServer/Features/Store/SqlIndexDataStore.cs +++ b/src/Microsoft.Health.Dicom.SqlServer/Features/Store/SqlIndexDataStore.cs @@ -133,4 +133,10 @@ public async Task UpdateFilePropertiesContentLengthAsync( ISqlIndexDataStore store = await _cache.GetAsync(cancellationToken: cancellationToken); await store.UpdateFilePropertiesContentLengthAsync(filePropertiesByWatermark, cancellationToken); } + + public async Task GetIndexedFilePropertiesAsync(CancellationToken cancellationToken = default) + { + ISqlIndexDataStore store = await _cache.GetAsync(cancellationToken: cancellationToken); + return await store.GetIndexedFilePropertiesAsync(cancellationToken); + } } diff --git a/src/Microsoft.Health.Dicom.SqlServer/Features/Store/SqlIndexDataStoreV1.cs b/src/Microsoft.Health.Dicom.SqlServer/Features/Store/SqlIndexDataStoreV1.cs index c9f7a8df89..3f97cefe69 100644 --- a/src/Microsoft.Health.Dicom.SqlServer/Features/Store/SqlIndexDataStoreV1.cs +++ b/src/Microsoft.Health.Dicom.SqlServer/Features/Store/SqlIndexDataStoreV1.cs @@ -378,4 +378,9 @@ public virtual Task UpdateFilePropertiesContentLengthAsync( { throw new BadRequestException(DicomSqlServerResource.SchemaVersionNeedsToBeUpgraded); } + + public virtual Task GetIndexedFilePropertiesAsync(CancellationToken cancellationToken = default) + { + throw new BadRequestException(DicomSqlServerResource.SchemaVersionNeedsToBeUpgraded); + } } diff --git a/src/Microsoft.Health.Dicom.SqlServer/Features/Store/SqlIndexDataStoreV57.cs b/src/Microsoft.Health.Dicom.SqlServer/Features/Store/SqlIndexDataStoreV57.cs new file mode 100644 index 0000000000..132549e922 --- /dev/null +++ b/src/Microsoft.Health.Dicom.SqlServer/Features/Store/SqlIndexDataStoreV57.cs @@ -0,0 +1,51 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Data.SqlClient; +using Microsoft.Health.Dicom.Core.Exceptions; +using Microsoft.Health.Dicom.Core.Features.Common; +using Microsoft.Health.Dicom.SqlServer.Features.Schema; +using Microsoft.Health.Dicom.SqlServer.Features.Schema.Model; +using Microsoft.Health.SqlServer.Features.Client; + +namespace Microsoft.Health.Dicom.SqlServer.Features.Store; + +internal class SqlIndexDataStoreV57 : SqlIndexDataStoreV55 +{ + public SqlIndexDataStoreV57(SqlConnectionWrapperFactory sqlConnectionWrapperFactory) + : base(sqlConnectionWrapperFactory) + { + } + + public override SchemaVersion Version => SchemaVersion.V57; + + public override async Task GetIndexedFilePropertiesAsync(CancellationToken cancellationToken = default) + { + using (SqlConnectionWrapper sqlConnectionWrapper = await SqlConnectionWrapperFactory.ObtainSqlConnectionWrapperAsync(cancellationToken, true)) + using (SqlCommandWrapper sqlCommandWrapper = sqlConnectionWrapper.CreateRetrySqlCommand()) + { + VLatest.GetTotalAndSumContentLengthIndexedAsyncV57.PopulateCommand(sqlCommandWrapper); + + try + { + using (SqlDataReader sqlDataReader = await sqlCommandWrapper.ExecuteReaderAsync(cancellationToken)) + { + await sqlDataReader.ReadAsync(cancellationToken); + return new IndexedFileProperties + { + TotalIndexed = (int)sqlDataReader[0], + TotalSum = await sqlDataReader.IsDBNullAsync(1, cancellationToken) ? 0 : (long)sqlDataReader[1], + }; + } + } + catch (SqlException ex) + { + throw new DataStoreException(ex); + } + } + } +} diff --git a/src/Microsoft.Health.Dicom.SqlServer/Registration/DicomSqlServerRegistrationExtensions.cs b/src/Microsoft.Health.Dicom.SqlServer/Registration/DicomSqlServerRegistrationExtensions.cs index df2c212e94..7e87bcfbd7 100644 --- a/src/Microsoft.Health.Dicom.SqlServer/Registration/DicomSqlServerRegistrationExtensions.cs +++ b/src/Microsoft.Health.Dicom.SqlServer/Registration/DicomSqlServerRegistrationExtensions.cs @@ -161,6 +161,7 @@ private static IServiceCollection AddSqlIndexDataStores(this IServiceCollection services.TryAddEnumerable(ServiceDescriptor.Scoped()); services.TryAddEnumerable(ServiceDescriptor.Scoped()); services.TryAddEnumerable(ServiceDescriptor.Scoped()); + services.TryAddEnumerable(ServiceDescriptor.Scoped()); return services; } diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/IndexedFilePropertiesCollection.cs b/test/Microsoft.Health.Dicom.Tests.Integration/IndexedFilePropertiesCollection.cs new file mode 100644 index 0000000000..5a43a8db1e --- /dev/null +++ b/test/Microsoft.Health.Dicom.Tests.Integration/IndexedFilePropertiesCollection.cs @@ -0,0 +1,13 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Xunit; + +namespace Microsoft.Health.Dicom.Tests.Integration; + +[CollectionDefinition("Indexed File Properties Collection", DisableParallelization = true)] +public class IndexedFilePropertiesCollection +{ +} diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/IndexedFilePropertiesTests.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/IndexedFilePropertiesTests.cs new file mode 100644 index 0000000000..da8af84566 --- /dev/null +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/IndexedFilePropertiesTests.cs @@ -0,0 +1,94 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using EnsureThat; +using FellowOakDicom; +using Microsoft.Health.Dicom.Core.Features.Common; +using Microsoft.Health.Dicom.Core.Features.Partitioning; +using Microsoft.Health.Dicom.Core.Features.Store; +using Microsoft.Health.Dicom.Tests.Common; +using Microsoft.Health.Dicom.Tests.Common.Extensions; +using Xunit; + +namespace Microsoft.Health.Dicom.Tests.Integration.Persistence; + +/// +/// Can not be run in parallel as each inserts data and each queries all of these inserts for testing. +/// +[Collection("Indexed File Properties Collection")] +public class IndexedFilePropertiesTests : IClassFixture +{ + private readonly IIndexDataStore _indexDataStore; + private readonly SqlDataStoreTestsFixture _fixture; + + private static readonly string StudyInstanceUid = TestUidGenerator.Generate(); + private readonly FileProperties _fileProperties = new() { Path = "path.dcm", ETag = "E1230", ContentLength = 10 }; + + public IndexedFilePropertiesTests(SqlDataStoreTestsFixture fixture) + { + _fixture = EnsureArg.IsNotNull(fixture, nameof(fixture)); + _indexDataStore = EnsureArg.IsNotNull(fixture?.IndexDataStore, nameof(fixture.IndexDataStore)); + } + + [Fact] + public async Task GivenNoFilePropertiesIndexed_WhenGetIndexedFilePropertiesAsync_ExpectCorrectTotalsRetrieved() + { + IndexedFileProperties indexedFileProperties = await _indexDataStore.GetIndexedFilePropertiesAsync(); + + Assert.Equal(0, indexedFileProperties.TotalSum); + Assert.Equal(0, indexedFileProperties.TotalIndexed); + } + + [Fact] + public async Task GivenFilePropertiesIndexed_WhenGetIndexedFilePropertiesAsync_ExpectCorrectTotalsRetrieved() + { + var properties = new List + { + await CreateRandomInstanceAsync(fileProperties: _fileProperties), + await CreateRandomInstanceAsync(fileProperties: _fileProperties), + await CreateRandomInstanceAsync(fileProperties: _fileProperties), + await CreateRandomInstanceAsync(fileProperties: _fileProperties), + }; + + IndexedFileProperties indexedFileProperties = await _indexDataStore.GetIndexedFilePropertiesAsync(); + + Assert.Equal(properties.Sum(x => x.ContentLength), indexedFileProperties.TotalSum); + Assert.Equal(properties.Count, indexedFileProperties.TotalIndexed); + } + + [Fact] + public async Task GivenFilePropertiesIndexedWithZeroes_WhenGetIndexedFilePropertiesAsync_ExpectCorrectTotalsRetrieved() + { + FileProperties zeroLengthFileProperties = new FileProperties { Path = "zeroLength.dcm", ETag = "E1230", ContentLength = 0 }; + + var properties = new List + { + await CreateRandomInstanceAsync(fileProperties: _fileProperties), + await CreateRandomInstanceAsync(fileProperties: zeroLengthFileProperties), + await CreateRandomInstanceAsync(fileProperties: _fileProperties), + }; + + IndexedFileProperties indexedFileProperties = await _indexDataStore.GetIndexedFilePropertiesAsync(); + + Assert.Equal(properties.Sum(x => x.ContentLength), indexedFileProperties.TotalSum); + Assert.Equal(properties.Count, indexedFileProperties.TotalIndexed); + } + + private async Task CreateRandomInstanceAsync(FileProperties fileProperties, Partition partition = null) + { + DicomDataset dataset = Samples.CreateRandomInstanceDataset(StudyInstanceUid); + partition ??= Partition.Default; + + long watermark = await _indexDataStore.BeginCreateInstanceIndexAsync(partition, dataset); + + await _indexDataStore.EndCreateInstanceIndexAsync(partition.Key, dataset, watermark, fileProperties); + + return fileProperties; + } + +} \ No newline at end of file diff --git a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/SqlDataStoreTestsFixture.cs b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/SqlDataStoreTestsFixture.cs index 2ca522d9e5..cba1d267eb 100644 --- a/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/SqlDataStoreTestsFixture.cs +++ b/test/Microsoft.Health.Dicom.Tests.Integration/Persistence/SqlDataStoreTestsFixture.cs @@ -121,6 +121,7 @@ internal SqlDataStoreTestsFixture(string databaseName, SchemaInformation schemaI new SqlIndexDataStoreV52(SqlConnectionWrapperFactory), new SqlIndexDataStoreV54(SqlConnectionWrapperFactory), new SqlIndexDataStoreV55(SqlConnectionWrapperFactory), + new SqlIndexDataStoreV57(SqlConnectionWrapperFactory), }), NullLogger.Instance);