Simplifying perf improvement PR (#199)

* Simplifying PR * Fixing linting error
microsoft · Jun 18, 2022 · 1087bf8 · 1087bf8
1 parent cc4bd71
commit 1087bf8
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 1 deletion.
diff --git a/src/lib/Microsoft.Health.Fhir.Ingest.Template/MeasurementExtractor.cs b/src/lib/Microsoft.Health.Fhir.Ingest.Template/MeasurementExtractor.cs
@@ -126,12 +126,18 @@ protected virtual IEnumerable<JToken> MatchTypeTokens(JObject token)
             EnsureArg.IsNotNull(token, nameof(token));
             var evaluator = CreateRequiredExpressionEvaluator(Template.TypeMatchExpression, nameof(Template.TypeMatchExpression));
 
+            JObject tokenClone = null;
+
             foreach (var extractedToken in evaluator.SelectTokens(token))
             {
                 // Add the extracted data as an element of the original data.
                 // This allows subsequent expressions access to data from the original event data
+                if (tokenClone == null)
+                {
+                    tokenClone = new JObject(token);
+                }
 
-                var tokenClone = token.DeepClone() as JObject;
+                tokenClone.Remove(MatchedToken);
                 tokenClone.Add(MatchedToken, extractedToken);
                 yield return tokenClone;
             }

diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementEventNormalizationService.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Service/MeasurementEventNormalizationService.cs
@@ -6,6 +6,7 @@
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -117,10 +118,19 @@ private async Task StartConsumer(ISourceBlock<EventData> producer, IEnumerableAs
 
                         try
                         {
+                            var stopWatch = new Stopwatch();
+                            stopWatch.Start();
                             foreach (var measurement in _contentTemplate.GetMeasurements(token))
                             {
                                 measurement.IngestionTimeUtc = evt.SystemProperties.EnqueuedTimeUtc;
                                 createdMeasurements.Add((partitionId, measurement));
+
+                                stopWatch.Stop();
+                                _log.LogMetric(
+                                    IomtMetrics.NormalizedEventGenerationTimeMs(partitionId),
+                                    stopWatch.ElapsedMilliseconds);
+                                stopWatch.Reset();
+                                stopWatch.Start();
                             }
                         }
                         catch (Exception ex)

diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/Metrics/IomtMetricDefinition.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/Metrics/IomtMetricDefinition.cs
@@ -24,6 +24,8 @@ private IomtMetricDefinition(string metricName)
 
         public static IomtMetricDefinition NormalizedEvent { get; } = new IomtMetricDefinition(nameof(NormalizedEvent));
 
+        public static IomtMetricDefinition NormalizedEventGenerationTimeMs { get; } = new IomtMetricDefinition(nameof(NormalizedEventGenerationTimeMs));
+
         public static IomtMetricDefinition Measurement { get; } = new IomtMetricDefinition(nameof(Measurement));
 
         public static IomtMetricDefinition MeasurementGroup { get; } = new IomtMetricDefinition(nameof(MeasurementGroup));

diff --git a/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/Metrics/IomtMetrics.cs b/src/lib/Microsoft.Health.Fhir.Ingest/Telemetry/Metrics/IomtMetrics.cs
@@ -159,5 +159,16 @@ public static Metric HandledException(string exceptionName, string connectorStag
         {
             return exceptionName.ToErrorMetric(connectorStage, ErrorType.GeneralError, ErrorSeverity.Critical);
         }
+
+        /// <summary>
+        /// The time it takes to generate a Normalized Event.
+        /// </summary>
+        /// <param name="partitionId">The partition id of the events being consumed from the event hub partition </param>
+        public static Metric NormalizedEventGenerationTimeMs(string partitionId = null)
+        {
+            return IomtMetricDefinition.NormalizedEventGenerationTimeMs
+                .CreateBaseMetric(Category.Traffic, ConnectorOperation.Normalization)
+                .AddDimension(_partitionDimension, partitionId);
+        }
     }
 }