[monitor-opentelemetry] Live Metrics Filtering (for charts)

sdk/monitor/monitor-opentelemetry/src/metrics/quickpulse/export/exporter.ts

@@ -15,6 +15,7 @@ import {
   MonitoringDataPoint,
   PublishOptionalParams,
   PublishResponse,
+  CollectionConfigurationError,
 } from "../../../generated";
 import { getTransmissionTime, resourceMetricsToQuickpulseDataPoint } from "../utils";
 
@@ -27,6 +28,9 @@ export class QuickpulseMetricExporter implements PushMetricExporter {
   private getDocumentsFn: () => DocumentIngress[];
   // Monitoring data point with common properties
   private baseMonitoringDataPoint: MonitoringDataPoint;
+  private etag: string;
+  private getErrorsFn: () => CollectionConfigurationError[];
+  private getDerivedMetricValuesFn: () => Map<string, number>;
 
   /**
    * Initializes a new instance of the AzureMonitorMetricExporter class.
@@ -43,6 +47,9 @@ export class QuickpulseMetricExporter implements PushMetricExporter {
     this.postCallback = options.postCallback;
     this.getDocumentsFn = options.getDocumentsFn;
     this.baseMonitoringDataPoint = options.baseMonitoringDataPoint;
+    this.getErrorsFn = options.getErrorsFn;
+    this.etag = "";
+    this.getDerivedMetricValuesFn = options.getDerivedMetricValuesFn;
     diag.debug("QuickpulseMetricExporter was successfully setup");
   }
 
@@ -62,14 +69,16 @@ export class QuickpulseMetricExporter implements PushMetricExporter {
         metrics,
         this.baseMonitoringDataPoint,
         this.getDocumentsFn(),
+        this.getErrorsFn(),
+        this.getDerivedMetricValuesFn(),
       ),
       transmissionTime: getTransmissionTime(),
+      configurationEtag: this.etag,
     };
     // Supress tracing until OpenTelemetry Metrics SDK support it
     await context.with(suppressTracing(context.active()), async () => {
       try {
-        const postResponse = await this.sender.publish(optionalParams);
-        this.postCallback(postResponse);
+        this.postCallback(await this.sender.publish(optionalParams));
         resultCallback({ code: ExportResultCode.SUCCESS });
       } catch (error) {
         this.postCallback(undefined);
@@ -112,4 +121,8 @@ export class QuickpulseMetricExporter implements PushMetricExporter {
   public getSender(): QuickpulseSender {
     return this.sender;
   }
+
+  public setEtag(etag: string): void {
+    this.etag = etag;
+  }
 }

sdk/monitor/monitor-opentelemetry/src/metrics/quickpulse/filtering/collectionConfigurationErrorTracker.ts

@@ -0,0 +1,44 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { CollectionConfigurationError } from "../../../generated";
+
+export class CollectionConfigurationErrorTracker {
+  /**
+   * This list represents the errors that are found when the response from a ping or post is initially parsed.
+   * The errors in this list are expected to stay the same across multiple post requests of the same configuration
+   * id, and so will only be changed when a new configuration gets parsed.
+   */
+  private validationTimeErrors: CollectionConfigurationError[] = [];
+  /**
+   * This list represents errors that can't be caught while parsing the response - such as validation errors that would occur when
+   * analyzing customDimensions present in incoming spans/logs, or when creating a projection. These errors aren't expected to be the
+   * same across post requests of the same configuration id and so is expected to be regenerated for every post request.
+   */
+  private runTimeErrors: CollectionConfigurationError[] = [];
+
+  constructor() {
+    this.validationTimeErrors = [];
+    this.runTimeErrors = [];
+  }
+
+  public addValidationError(error: CollectionConfigurationError): void {
+    this.validationTimeErrors.push(error);
+  }
+
+  public addRunTimeError(error: CollectionConfigurationError): void {
+    this.runTimeErrors.push(error);
+  }
+
+  public getErrors(): CollectionConfigurationError[] {
+    return this.validationTimeErrors.concat(this.runTimeErrors);
+  }
+
+  public clearRunTimeErrors(): void {
+    this.runTimeErrors = [];
+  }
+
+  public clearValidationTimeErrors(): void {
+    this.validationTimeErrors = [];
+  }
+}

sdk/monitor/monitor-opentelemetry/src/metrics/quickpulse/filtering/filter.ts

@@ -0,0 +1,180 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import {
+  DerivedMetricInfo,
+  FilterInfo,
+  KnownPredicateType,
+  FilterConjunctionGroupInfo,
+} from "../../../generated";
+import {
+  RequestData,
+  TelemetryData,
+  DependencyData,
+  ExceptionData,
+  TraceData,
+  KnownDependencyColumns,
+  KnownRequestColumns,
+} from "../types";
+import {
+  isRequestData,
+  isDependencyData,
+  isExceptionData,
+  isTraceData,
+  getMsFromFilterTimestampString,
+} from "../utils";
+
+export class Filter {
+  public renameExceptionFieldNamesForFiltering(
+    filterConjunctionGroupInfo: FilterConjunctionGroupInfo,
+  ): void {
+    filterConjunctionGroupInfo.filters.forEach((filter) => {
+      if (filter.fieldName.startsWith("Exception.")) {
+        filter.fieldName = filter.fieldName.replace("Exception.", "");
+      }
+    });
+  }
+
+  public checkMetricFilters(derivedMetricInfo: DerivedMetricInfo, data: TelemetryData): boolean {
+    if (derivedMetricInfo.filterGroups.length === 0) {
+      // This should never happen - even when a user does not add filter pills to the derived metric,
+      // the filterGroups array should have one filter group with an empty array of filters.
+      return true;
+    }
+    // Haven't yet seen any case where there is more than one filter group in a derived metric info.
+    // Just to be safe, handling the multiple filter conjunction group case as an or operation.
+    let matched = false;
+    derivedMetricInfo.filterGroups.forEach((filterConjunctionGroup) => {
+      matched = matched || this.checkFilterConjunctionGroup(filterConjunctionGroup.filters, data);
+    });
+    return matched;
+  }
+
+  /* public static checkDocumentFilters(documentStreamInfo: DocumentStreamInfo, data: TelemetryData): boolean {
+    return true; // to be implemented
+  }*/
+
+  private checkFilterConjunctionGroup(filters: FilterInfo[], data: TelemetryData): boolean {
+    // All of the filters need to match for this to return true (and operation).
+    for (const filter of filters) {
+      if (!this.checkFilter(filter, data)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  private checkFilter(filter: FilterInfo, data: TelemetryData): boolean {
+    if (filter.fieldName === "*") {
+      // Any field
+      return this.checkAnyFieldFilter(filter, data);
+    } else if (filter.fieldName.startsWith("CustomDimensions.")) {
+      return this.checkCustomDimFilter(filter, data);
+    } else {
+      let dataValue: string | number | boolean | Map<string, string>;
+      // use filter.fieldname to get the property of data to query
+      if (isRequestData(data)) {
+        dataValue = data[filter.fieldName as keyof RequestData];
+      } else if (isDependencyData(data)) {
+        dataValue = data[filter.fieldName as keyof DependencyData];
+      } else if (isExceptionData(data)) {
+        dataValue = data[filter.fieldName as keyof ExceptionData];
+      } else if (isTraceData(data)) {
+        dataValue = data[filter.fieldName as keyof TraceData];
+      } else {
+        return false; // should not reach here
+      }
+
+      if (filter.fieldName === KnownRequestColumns.Success.toString()) {
+        if (filter.predicate === KnownPredicateType.Equal.toString()) {
+          return dataValue === (filter.comparand.toLowerCase() === "true");
+        } else if (filter.predicate === KnownPredicateType.NotEqual.toString()) {
+          return dataValue !== (filter.comparand.toLowerCase() === "true");
+        }
+      } else if (
+        filter.fieldName === KnownDependencyColumns.ResultCode.toString() ||
+        filter.fieldName === KnownRequestColumns.ResponseCode.toString() ||
+        filter.fieldName === KnownDependencyColumns.Duration.toString()
+      ) {
+        const comparand: number =
+          filter.fieldName === KnownDependencyColumns.Duration.toString()
+            ? getMsFromFilterTimestampString(filter.comparand)
+            : parseFloat(filter.comparand);
+        switch (filter.predicate) {
+          case KnownPredicateType.Equal.toString():
+            return dataValue === comparand;
+          case KnownPredicateType.NotEqual.toString():
+            return dataValue !== comparand;
+          case KnownPredicateType.GreaterThan.toString():
+            return (dataValue as number) > comparand;
+          case KnownPredicateType.GreaterThanOrEqual.toString():
+            return (dataValue as number) >= comparand;
+          case KnownPredicateType.LessThan.toString():
+            return (dataValue as number) < comparand;
+          case KnownPredicateType.LessThanOrEqual.toString():
+            return (dataValue as number) <= comparand;
+          default:
+            return false;
+        }
+      } else {
+        // string fields
+        return this.stringCompare(dataValue as string, filter.comparand, filter.predicate);
+      }
+    }
+    return false;
+  }
+
+  private checkAnyFieldFilter(filter: FilterInfo, data: TelemetryData): boolean {
+    const properties: string[] = Object.keys(data);
+    // At this point, the only predicates possible to pass in are Contains and DoesNotContain
+    // At config validation time the predicate is checked to be one of these two.
+    for (const property of properties) {
+      if (property === "CustomDimensions") {
+        for (const value of data.CustomDimensions.values()) {
+          if (this.stringCompare(value, filter.comparand, filter.predicate)) {
+            return true;
+          }
+        }
+      } else {
+        // @ts-expect-error - data can be any type of telemetry data and we know property is a valid key
+        const value: string = String(data[property]);
+        if (this.stringCompare(value, filter.comparand, filter.predicate)) {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
+  private checkCustomDimFilter(filter: FilterInfo, data: TelemetryData): boolean {
+    const fieldName: string = filter.fieldName.replace("CustomDimensions.", "");
+    let value: string | undefined;
+    if (data.CustomDimensions.has(fieldName)) {
+      value = data.CustomDimensions.get(fieldName) as string;
+    } else {
+      return false; // the asked for field is not present in the custom dimensions
+    }
+    return this.stringCompare(value, filter.comparand, filter.predicate);
+  }
+
+  private stringCompare(dataValue: string, comparand: string, predicate: string): boolean {
+    switch (predicate) {
+      case KnownPredicateType.Equal.toString():
+        return dataValue === comparand;
+      case KnownPredicateType.NotEqual.toString():
+        return dataValue !== comparand;
+      case KnownPredicateType.Contains.toString(): {
+        const lowerDataValue = dataValue.toLowerCase();
+        const lowerComparand = comparand.toLowerCase();
+        return lowerDataValue.includes(lowerComparand);
+      }
+      case KnownPredicateType.DoesNotContain.toString(): {
+        const lowerDataValue = dataValue.toLowerCase();
+        const lowerComparand = comparand.toLowerCase();
+        return !lowerDataValue.includes(lowerComparand);
+      }
+      default:
+        return false;
+    }
+  }
+}