add function to get sum and total indexed

src/Microsoft.Health.Dicom.Core/Features/Common/IndexedFileProperties.cs

@@ -0,0 +1,22 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+namespace Microsoft.Health.Dicom.Core.Features.Common;
+
+/// <summary>
+/// Metadata on FileProperty table in database
+/// </summary>
+public class IndexedFileProperties
+{
+    /// <summary>
+    /// Total indexed FileProperty in database
+    /// </summary>
+    public int TotalIndexed { get; init; }
+
+    /// <summary>
+    /// Total sum of all ContentLength rows in FileProperty table
+    /// </summary>
+    public long TotalSum { get; init; }
+}

src/Microsoft.Health.Dicom.Core/Features/Store/IIndexDataStore.cs

@@ -185,4 +185,11 @@ public interface IIndexDataStore
     /// <param name="cancellationToken">The cancellation token.</param>
     /// <returns>A task that with list of instance metadata with new watermark.</returns>
     Task UpdateFilePropertiesContentLengthAsync(IReadOnlyDictionary<long, FileProperties> filePropertiesByWatermark, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Retrieves total count in FileProperty table and summation of all content length values across FileProperty table.
+    /// </summary>
+    /// <param name="cancellationToken">The cancellation token.</param>
+    /// <returns>A task that gets the count</returns>
+    Task<IndexedFileProperties> GetIndexedFilePropertiesAsync(CancellationToken cancellationToken = default);
 }

src/Microsoft.Health.Dicom.Functions.UnitTests/IndexMetricsCollection/IndexMetricsCollectionFunctionTests.cs

@@ -0,0 +1,79 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+using Microsoft.Health.Dicom.Core.Configs;
+using Microsoft.Health.Dicom.Core.Features.Common;
+using Microsoft.Health.Dicom.Core.Features.Store;
+using Microsoft.Health.Dicom.Core.Features.Telemetry;
+using Microsoft.Health.Dicom.Functions.IndexMetricsCollection;
+using Microsoft.Health.Dicom.Functions.IndexMetricsCollection.Telemetry;
+using NSubstitute;
+using NSubstitute.ExceptionExtensions;
+using OpenTelemetry;
+using OpenTelemetry.Metrics;
+using Xunit;
+
+namespace Microsoft.Health.Dicom.Functions.UnitTests.IndexMetricsCollection;
+
+public class IndexMetricsCollectionFunctionTests
+{
+    private readonly IndexMetricsCollectionFunction _collectionFunction;
+    private readonly IIndexDataStore _indexStore;
+    private readonly IndexMetricsCollectionMeter _meter;
+    private List<Metric> _exportedItems;
+    private MeterProvider _meterProvider;
+    private readonly TimerInfo _timer;
+
+    public IndexMetricsCollectionFunctionTests()
+    {
+        _meter = new IndexMetricsCollectionMeter();
+        _indexStore = Substitute.For<IIndexDataStore>();
+        _collectionFunction = new IndexMetricsCollectionFunction(
+            _indexStore,
+            Options.Create(new FeatureConfiguration { EnableExternalStore = true, }),
+            _meter);
+        _timer = Substitute.For<TimerInfo>(default, default, default);
+    }
+
+    private void InitializeMetricExporter()
+    {
+        _exportedItems = new List<Metric>();
+        _meterProvider = Sdk.CreateMeterProviderBuilder()
+            .AddMeter($"{OpenTelemetryLabels.BaseMeterName}.{IndexMetricsCollectionMeter.MeterName}")
+            .AddInMemoryExporter(_exportedItems)
+            .Build();
+    }
+
+    [Fact]
+    public async Task GivenIndexMetricsCollectionFunction_WhenRun_ThenIndexMetricsCollectionsCompletedCounterIsIncremented()
+    {
+        InitializeMetricExporter();
+        _indexStore.GetIndexedFilePropertiesAsync().ReturnsForAnyArgs(new IndexedFileProperties());
+
+        await _collectionFunction.Run(_timer, NullLogger.Instance);
+
+        _meterProvider.ForceFlush();
+        Assert.Single(_exportedItems);
+        Assert.Equal(nameof(_meter.IndexMetricsCollectionsCompletedCounter), _exportedItems[0].Name);
+    }
+
+    [Fact]
+    public async Task GivenIndexMetricsCollectionFunction_WhenRunException_ThenIndexMetricsCollectionsCompletedCounterIsNotIncremented()
+    {
+        InitializeMetricExporter();
+        _indexStore.GetIndexedFilePropertiesAsync().ThrowsForAnyArgs(new Exception());
+
+        await Assert.ThrowsAsync<Exception>(async () => await _collectionFunction.Run(_timer, NullLogger.Instance));
+
+        _meterProvider.ForceFlush();
+        Assert.Empty(_exportedItems);
+    }
+}

src/Microsoft.Health.Dicom.Functions/IndexMetricsCollection/IndexMetricsCollectionFunction.cs

@@ -0,0 +1,80 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System.Diagnostics;
+using System.Threading.Tasks;
+using EnsureThat;
+using Microsoft.Azure.WebJobs;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Microsoft.Health.Core;
+using Microsoft.Health.Dicom.Core.Configs;
+using Microsoft.Health.Dicom.Core.Features.Common;
+using Microsoft.Health.Dicom.Core.Features.Store;
+using Microsoft.Health.Dicom.Functions.IndexMetricsCollection.Telemetry;
+
+namespace Microsoft.Health.Dicom.Functions.IndexMetricsCollection;
+
+/// <summary>
+///  A function for collecting index metrics
+/// </summary>
+public class IndexMetricsCollectionFunction
+{
+    private readonly IIndexDataStore _indexDataStore;
+    private readonly IndexMetricsCollectionMeter _meter;
+    private readonly bool _externalStoreEnabled;
+    private readonly bool _enableDataPartitions;
+
+    public IndexMetricsCollectionFunction(
+        IIndexDataStore indexDataStore,
+        IOptions<FeatureConfiguration> featureConfiguration,
+        IndexMetricsCollectionMeter meter)
+    {
+        EnsureArg.IsNotNull(featureConfiguration, nameof(featureConfiguration));
+        _indexDataStore = EnsureArg.IsNotNull(indexDataStore, nameof(indexDataStore));
+        _meter = EnsureArg.IsNotNull(meter, nameof(meter));
+        _externalStoreEnabled = featureConfiguration.Value.EnableExternalStore;
+        _enableDataPartitions = featureConfiguration.Value.EnableDataPartitions;
+    }
+
+
+    /// <summary>
+    /// Asynchronously collects index metrics.
+    /// </summary>
+    /// <param name="invocationTimer">The timer which tracks the invocation schedule.</param>
+    /// <param name="log">A diagnostic logger.</param>
+    /// <returns>A task that represents the asynchronous metrics collection operation.</returns>
+    [FunctionName(nameof(IndexMetricsCollectionFunction))]
+    public async Task Run(
+        [TimerTrigger(IndexMetricsCollectionOptions.Frequency)] TimerInfo invocationTimer,
+        ILogger log)
+    {
+        EnsureArg.IsNotNull(invocationTimer, nameof(invocationTimer));
+        EnsureArg.IsNotNull(log, nameof(log));
+
+        log.LogInformation("Collecting a daily summation starting. At: {Timestamp}", Clock.UtcNow);
+        if (invocationTimer.IsPastDue)
+        {
+            log.LogWarning("Current function invocation is running late.");
+        }
+
+        Stopwatch stopwatch = new Stopwatch();
+        stopwatch.Start();
+
+        IndexedFileProperties indexedFileProperties = await _indexDataStore.GetIndexedFilePropertiesAsync();
+
+        stopwatch.Stop();
+
+        log.LogInformation("Collecting a daily summation time taken: {ElapsedTime} ms with ExternalStoreEnabled: {ExternalStoreEnabled} and DataPartitionsEnabled: {PartitionsEnabled}", stopwatch.ElapsedMilliseconds, _externalStoreEnabled, _enableDataPartitions);
+
+        log.LogInformation("DICOM telemetry - total files indexed: {0} with ExternalStoreEnabled: {ExternalStoreEnabled} and DataPartitionsEnabled: {PartitionsEnabled}", indexedFileProperties.TotalIndexed, _externalStoreEnabled, _enableDataPartitions);
+
+        log.LogInformation("DICOM telemetry - total content length indexed: {0} with ExternalStoreEnabled: {ExternalStoreEnabled} and DataPartitionsEnabled: {PartitionsEnabled}", indexedFileProperties.TotalSum, _externalStoreEnabled, _enableDataPartitions);
+
+        log.LogInformation("Collecting a daily summation completed. with ExternalStoreEnabled: {ExternalStoreEnabled} and DataPartitionsEnabled: {PartitionsEnabled}", _externalStoreEnabled, _enableDataPartitions);
+
+        _meter.IndexMetricsCollectionsCompletedCounter.Add(1, IndexMetricsCollectionMeter.CreateTelemetryDimension(_externalStoreEnabled, _enableDataPartitions));
+    }
+}

src/Microsoft.Health.Dicom.Functions/IndexMetricsCollection/IndexMetricsCollectionOptions.cs

@@ -0,0 +1,28 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+
+using System.ComponentModel.DataAnnotations;
+
+namespace Microsoft.Health.Dicom.Functions.IndexMetricsCollection;
+
+/// <summary>
+/// 
+/// </summary>
+public static class IndexMetricsCollectionOptions
+{
+    /// <summary>
+    /// The default section name for <see cref="IndexMetricsCollectionOptions"/> in a configuration.
+    /// </summary>
+    public const string SectionName = "IndexMetricsCollection";
+
+    /// <summary>
+    /// Gets or sets the cron expression that indicates how frequently to run the index metrics collection function.
+    /// </summary>
+    /// <value>A value cron expression</value>
+    [Required]
+    public const string Frequency = "* * * * *"; // Every day at midnight
+    // public const string Frequency = "0 0 * * *"; // Every day at midnight
+}

src/Microsoft.Health.Dicom.Functions/IndexMetricsCollection/Telemetry/IndexMetricsCollectionMeter.cs

@@ -0,0 +1,43 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Metrics;
+using Microsoft.Health.Dicom.Core.Features.Telemetry;
+
+namespace Microsoft.Health.Dicom.Functions.IndexMetricsCollection.Telemetry;
+
+public sealed class IndexMetricsCollectionMeter : IDisposable
+{
+    private readonly Meter _meter;
+    internal const string MeterName = "IndexMetricsCollection";
+
+    public IndexMetricsCollectionMeter()
+    {
+        _meter = new Meter($"{OpenTelemetryLabels.BaseMeterName}.{MeterName}", "1.0");
+
+        IndexMetricsCollectionsCompletedCounter =
+            _meter.CreateCounter<long>(
+                nameof(IndexMetricsCollectionsCompletedCounter),
+                description: "Represents a successful run of the index metrics collection function.");
+    }
+
+
+    public static KeyValuePair<string, object>[] CreateTelemetryDimension(bool externalStoreEnabled, bool dataPartitionsEnabled) =>
+        new[]
+        {
+            new KeyValuePair<string, object>("ExternalStoreEnabled", externalStoreEnabled),
+            new KeyValuePair<string, object>("DataPartitionsEnabled", dataPartitionsEnabled),
+        };
+
+    /// <summary>
+    /// Represents a successful run of the index metrics collection function
+    /// </summary>
+    public Counter<long> IndexMetricsCollectionsCompletedCounter { get; }
+
+    public void Dispose()
+        => _meter.Dispose();
+}

src/Microsoft.Health.Dicom.Functions/Registration/ServiceCollectionExtensions.cs

@@ -24,6 +24,7 @@
 using Microsoft.Health.Dicom.Functions.DataCleanup;
 using Microsoft.Health.Dicom.Functions.Export;
 using Microsoft.Health.Dicom.Functions.Indexing;
+using Microsoft.Health.Dicom.Functions.IndexMetricsCollection.Telemetry;
 using Microsoft.Health.Dicom.Functions.Update;
 using Microsoft.Health.Dicom.SqlServer.Registration;
 using Microsoft.Health.Extensions.DependencyInjection;
@@ -70,6 +71,7 @@ public static IDicomFunctionsBuilder ConfigureFunctions(
             .ConfigureDurableFunctionSerialization()
             .AddJsonSerializerOptions(o => o.ConfigureDefaultDicomSettings())
             .AddSingleton<UpdateMeter>()
+            .AddSingleton<IndexMetricsCollectionMeter>()
             .AddSingleton<IAuditLogger, AuditLogger>());
     }
 

src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Migrations/57.diff.sql

@@ -1,4 +1,19 @@
 SET XACT_ABORT ON
+
+BEGIN TRANSACTION
+GO
+CREATE OR ALTER PROCEDURE dbo.GetTotalAndSumContentLengthIndexedAsyncV57
+    AS
+BEGIN
+    SET NOCOUNT ON;
+    SET XACT_ABORT ON;
+SELECT count(*),
+       SUM(ContentLength)
+FROM   dbo.FileProperty;
+END
+GO
+
+COMMIT TRANSACTION
 
 IF NOT EXISTS 
 (

src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Migrations/57.sql

@@ -2746,6 +2746,17 @@ BEGIN
               AND i.PartitionKey = sv.PartitionKey;
 END
 
+GO
+CREATE OR ALTER PROCEDURE dbo.GetTotalAndSumContentLengthIndexedAsyncV57
+AS
+BEGIN
+    SET NOCOUNT ON;
+    SET XACT_ABORT ON;
+    SELECT count(*),
+           SUM(ContentLength)
+    FROM   dbo.FileProperty;
+END
+
 GO
 CREATE OR ALTER PROCEDURE dbo.GetWorkitemMetadata
 @partitionKey INT, @workitemUid VARCHAR (64), @procedureStepStateTagPath VARCHAR (64)

src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Model/VLatest.Generated.cs

@@ -87,6 +87,7 @@ internal class VLatest
         internal readonly static GetPartitionsProcedure GetPartitions = new GetPartitionsProcedure();
         internal readonly static GetSeriesResultProcedure GetSeriesResult = new GetSeriesResultProcedure();
         internal readonly static GetStudyResultProcedure GetStudyResult = new GetStudyResultProcedure();
+        internal readonly static GetTotalAndSumContentLengthIndexedAsyncV57Procedure GetTotalAndSumContentLengthIndexedAsyncV57 = new GetTotalAndSumContentLengthIndexedAsyncV57Procedure();
         internal readonly static GetWorkitemMetadataProcedure GetWorkitemMetadata = new GetWorkitemMetadataProcedure();
         internal readonly static GetWorkitemQueryTagsProcedure GetWorkitemQueryTags = new GetWorkitemQueryTagsProcedure();
         internal readonly static IIndexInstanceCoreV9Procedure IIndexInstanceCoreV9 = new IIndexInstanceCoreV9Procedure();
@@ -2154,6 +2155,19 @@ internal GetStudyResultTableValuedParameters(global::System.Collections.Generic.
             internal global::System.Collections.Generic.IEnumerable<WatermarkTableTypeRow> WatermarkTableType { get; }
         }
 
+        internal class GetTotalAndSumContentLengthIndexedAsyncV57Procedure : StoredProcedure
+        {
+            internal GetTotalAndSumContentLengthIndexedAsyncV57Procedure() : base("dbo.GetTotalAndSumContentLengthIndexedAsyncV57")
+            {
+            }
+
+            public void PopulateCommand(SqlCommandWrapper command)
+            {
+                command.CommandType = global::System.Data.CommandType.StoredProcedure;
+                command.CommandText = "dbo.GetTotalAndSumContentLengthIndexedAsyncV57";
+            }
+        }
+
         internal class GetWorkitemMetadataProcedure : StoredProcedure
         {
             internal GetWorkitemMetadataProcedure() : base("dbo.GetWorkitemMetadata")

src/Microsoft.Health.Dicom.SqlServer/Features/Schema/Sql/Sprocs/GetTotalAndSumContentLengthIndexedAsyncV57.sql

@@ -0,0 +1,19 @@
+/***************************************************************************************/
+-- STORED PROCEDURE
+--     GetTotalAndSumContentLengthIndexedAsyncV57
+--
+-- FIRST SCHEMA VERSION
+--     57
+--
+-- DESCRIPTION
+--     Retrieves total sum of content length across all FileProperty rows
+--
+/***************************************************************************************/
+CREATE OR ALTER PROCEDURE dbo.GetTotalAndSumContentLengthIndexedAsyncV57
+    AS
+BEGIN
+    SET NOCOUNT     ON
+    SET XACT_ABORT  ON
+
+SELECT count(*), SUM(ContentLength) FROM dbo.FileProperty
+END

src/Microsoft.Health.Dicom.SqlServer/Features/Store/SqlIndexDataStore.cs

@@ -133,4 +133,10 @@ public async Task UpdateFilePropertiesContentLengthAsync(
         ISqlIndexDataStore store = await _cache.GetAsync(cancellationToken: cancellationToken);
         await store.UpdateFilePropertiesContentLengthAsync(filePropertiesByWatermark, cancellationToken);
     }
+
+    public async Task<IndexedFileProperties> GetIndexedFilePropertiesAsync(CancellationToken cancellationToken = default)
+    {
+        ISqlIndexDataStore store = await _cache.GetAsync(cancellationToken: cancellationToken);
+        return await store.GetIndexedFilePropertiesAsync(cancellationToken);
+    }
 }

src/Microsoft.Health.Dicom.SqlServer/Features/Store/SqlIndexDataStoreV1.cs

@@ -378,4 +378,9 @@ public virtual Task UpdateFilePropertiesContentLengthAsync(
     {
         throw new BadRequestException(DicomSqlServerResource.SchemaVersionNeedsToBeUpgraded);
     }
+
+    public virtual Task<IndexedFileProperties> GetIndexedFilePropertiesAsync(CancellationToken cancellationToken = default)
+    {
+        throw new BadRequestException(DicomSqlServerResource.SchemaVersionNeedsToBeUpgraded);
+    }
 }