[Azure Monitor] Add Standard Metrics support (Azure#26281)

Adding support for Standard Metrics
Adding Stastbeat features
Adding unit tests
Add support for APPLICATIONINSIGHTS_CONFIGURATION_CONTENT

Author: hectorhdzg

common/config/rush/pnpm-lock.yaml

@@ -72,6 +72,7 @@
     "@microsoft/api-extractor": "^7.31.1",
     "@types/mocha": "^7.0.2",
     "@types/node": "^14.0.0",
+    "@types/sinon": "10.0.15",
     "dotenv": "^16.0.0",
     "eslint": "^8.0.0",
     "eslint-plugin-node": "^11.1.0",

sdk/monitor/monitor-opentelemetry/package.json

@@ -11,6 +11,11 @@ import { TraceHandler } from "./traces/handler";
 import { Logger as InternalLogger } from "./shared/logging";
 import { AzureMonitorOpenTelemetryOptions } from "./shared/types";
 import { LogHandler } from "./logs";
+import {
+  AZURE_MONITOR_STATSBEAT_FEATURES,
+  StatsbeatFeature,
+  StatsbeatInstrumentation,
+} from "./types";
 
 /**
  * Azure Monitor OpenTelemetry Client
@@ -35,6 +40,7 @@ export class AzureMonitorOpenTelemetryClient {
         "Connection String not found, please provide it before starting Azure Monitor OpenTelemetry Client."
       );
     }
+    this._setStatsbeatFeatures();
     this._metricHandler = new MetricHandler(this._config);
     this._traceHandler = new TraceHandler(this._config, this._metricHandler);
     this._logHandler = new LogHandler(this._config, this._metricHandler);
@@ -100,5 +106,37 @@ export class AzureMonitorOpenTelemetryClient {
   public async shutdown(): Promise<void> {
     this._traceHandler.shutdown();
     this._metricHandler.shutdown();
+    this._logHandler.shutdown();
+  }
+
+  private _setStatsbeatFeatures() {
+    let instrumentationBitMap = 0;
+    if (this._config.instrumentationOptions?.azureSdk?.enabled) {
+      instrumentationBitMap |= StatsbeatInstrumentation.AZURE_CORE_TRACING;
+    }
+    if (this._config.instrumentationOptions?.mongoDb?.enabled) {
+      instrumentationBitMap |= StatsbeatInstrumentation.MONGODB;
+    }
+    if (this._config.instrumentationOptions?.mySql?.enabled) {
+      instrumentationBitMap |= StatsbeatInstrumentation.MYSQL;
+    }
+    if (this._config.instrumentationOptions?.postgreSql?.enabled) {
+      instrumentationBitMap |= StatsbeatInstrumentation.POSTGRES;
+    }
+    if (this._config.instrumentationOptions?.redis?.enabled) {
+      instrumentationBitMap |= StatsbeatInstrumentation.REDIS;
+    }
+
+    let featureBitMap = 0;
+    featureBitMap |= StatsbeatFeature.DISTRO;
+
+    try {
+      process.env[AZURE_MONITOR_STATSBEAT_FEATURES] = JSON.stringify({
+        instrumentation: instrumentationBitMap,
+        feature: featureBitMap,
+      });
+    } catch (error) {
+      InternalLogger.getInstance().error("Failed call to JSON.stringify.", error);
+    }
   }
 }

sdk/monitor/monitor-opentelemetry/src/client.ts

@@ -10,6 +10,7 @@ import {
 import { LoggerProviderConfig } from "@opentelemetry/sdk-logs/build/src/types";
 import { AzureMonitorOpenTelemetryConfig } from "../shared/config";
 import { MetricHandler } from "../metrics/handler";
+import { AzureLogRecordProcessor } from "./logRecordProcessor";
 
 /**
  * Azure Monitor OpenTelemetry Log Handler
@@ -21,13 +22,14 @@ export class LogHandler {
   private _logRecordProcessor: SimpleLogRecordProcessor;
   private _config: AzureMonitorOpenTelemetryConfig;
   private _metricHandler?: MetricHandler;
+  private _azureLogProccessor: AzureLogRecordProcessor;
 
   /**
    * Initializes a new instance of the TraceHandler class.
    * @param _config - Distro configuration.
    * @param _metricHandler - MetricHandler.
    */
-  constructor(config: AzureMonitorOpenTelemetryConfig, metricHandler?: MetricHandler) {
+  constructor(config: AzureMonitorOpenTelemetryConfig, metricHandler: MetricHandler) {
     this._config = config;
     this._metricHandler = metricHandler;
     const loggerProviderConfig: LoggerProviderConfig = {
@@ -37,10 +39,9 @@ export class LogHandler {
     this._exporter = new ConsoleLogRecordExporter();
     this._logRecordProcessor = new SimpleLogRecordProcessor(this._exporter);
     this._loggerProvider.addLogRecordProcessor(this._logRecordProcessor);
+    this._azureLogProccessor = new AzureLogRecordProcessor(this._metricHandler);
+    this._loggerProvider.addLogRecordProcessor(this._azureLogProccessor);
     this._logger = this._loggerProvider.getLogger("AzureMonitorLogger", undefined) as OtelLogger;
-    if (this._metricHandler) {
-      // TODO: Use metric handler to track standard metrics
-    }
   }
 
   /**

sdk/monitor/monitor-opentelemetry/src/logs/handler.ts

@@ -0,0 +1,29 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+import { MetricHandler } from "../metrics/handler";
+import { LogRecord, LogRecordProcessor } from "@opentelemetry/sdk-logs";
+
+/**
+ * Azure Monitor LogRecord Processor.
+ * @internal
+ */
+export class AzureLogRecordProcessor implements LogRecordProcessor {
+  private readonly _metricHandler: MetricHandler;
+
+  constructor(metricHandler: MetricHandler) {
+    this._metricHandler = metricHandler;
+  }
+
+  public onEmit(logRecord: LogRecord): void {
+    this._metricHandler.recordLog(logRecord);
+  }
+
+  public forceFlush(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  public shutdown(): Promise<void> {
+    return Promise.resolve();
+  }
+}

sdk/monitor/monitor-opentelemetry/src/logs/logRecordProcessor.ts

@@ -12,6 +12,8 @@ import {
 import { AzureMonitorOpenTelemetryConfig } from "../shared/config";
 import { PerformanceCounterMetrics } from "./performanceCounters";
 import { StandardMetrics } from "./standardMetrics";
+import { ReadableSpan, Span } from "@opentelemetry/sdk-trace-base";
+import { LogRecord } from "@opentelemetry/sdk-logs";
 
 /**
  * Azure Monitor OpenTelemetry Metric Handler
@@ -67,18 +69,17 @@ export class MetricHandler {
     return this._meter;
   }
 
-  /**
-   *Get StandardMetric handler
-   */
-  public getStandardMetrics(): StandardMetrics | undefined {
-    return this._standardMetrics;
+  public markSpanAsProcessed(span: Span): void {
+    this._standardMetrics?.markSpanAsProcessed(span);
   }
 
-  /**
-   *Get PerformanceCounter handler
-   */
-  public getPerformanceCounterMetrics(): PerformanceCounterMetrics | undefined {
-    return this._perfCounterMetrics;
+  public recordSpan(span: ReadableSpan): void {
+    this._standardMetrics?.recordSpan(span);
+    this._perfCounterMetrics?.recordSpan(span);
+  }
+
+  public recordLog(logRecord: LogRecord): void {
+    this._standardMetrics?.recordLog(logRecord);
   }
 
   /**

sdk/monitor/monitor-opentelemetry/src/metrics/handler.ts

@@ -9,8 +9,8 @@ import {
 } from "@opentelemetry/sdk-metrics";
 import { AzureMonitorOpenTelemetryConfig } from "../shared/config";
 import { AzureMonitorMetricExporter } from "@azure/monitor-opentelemetry-exporter";
-import { Attributes, Histogram, Meter, SpanKind, ValueType } from "@opentelemetry/api";
-import { ReadableSpan, Span } from "@opentelemetry/sdk-trace-base";
+import { Attributes, Counter, Histogram, Meter, SpanKind, ValueType } from "@opentelemetry/api";
+import { ReadableSpan, Span, TimedEvent } from "@opentelemetry/sdk-trace-base";
 import {
   SemanticAttributes,
   SemanticResourceAttributes,
@@ -20,20 +20,24 @@ import {
   MetricRequestDimensions,
   StandardMetricBaseDimensions,
 } from "./types";
+import { LogRecord } from "@opentelemetry/sdk-logs";
+import { Resource } from "@opentelemetry/resources";
 
 /**
  * Azure Monitor Standard Metrics
  * @internal
  */
 export class StandardMetrics {
+  private _config: AzureMonitorOpenTelemetryConfig;
   private _collectionInterval = 60000; // 60 seconds
   private _meterProvider: MeterProvider;
   private _azureExporter: AzureMonitorMetricExporter;
   private _metricReader: PeriodicExportingMetricReader;
   private _meter: Meter;
   private _incomingRequestDurationHistogram: Histogram;
   private _outgoingRequestDurationHistogram: Histogram;
-  private _config: AzureMonitorOpenTelemetryConfig;
+  private _exceptionsCounter: Counter;
+  private _tracesCounter: Counter;
 
   /**
    * Initializes a new instance of the StandardMetrics class.
@@ -54,11 +58,24 @@ export class StandardMetrics {
     this._metricReader = new PeriodicExportingMetricReader(metricReaderOptions);
     this._meterProvider.addMetricReader(this._metricReader);
     this._meter = this._meterProvider.getMeter("AzureMonitorStandardMetricsMeter");
-    this._incomingRequestDurationHistogram = this._meter.createHistogram("REQUEST_DURATION", {
-      valueType: ValueType.DOUBLE,
+    this._incomingRequestDurationHistogram = this._meter.createHistogram(
+      "azureMonitor.http.requestDuration",
+      {
+        valueType: ValueType.DOUBLE,
+      }
+    );
+    this._outgoingRequestDurationHistogram = this._meter.createHistogram(
+      "azureMonitor.http.dependencyDuration",
+      {
+        valueType: ValueType.DOUBLE,
+      }
+    );
+
+    this._exceptionsCounter = this._meter.createCounter("azureMonitor.exceptionCount", {
+      valueType: ValueType.INT,
     });
-    this._outgoingRequestDurationHistogram = this._meter.createHistogram("DEPENDENCY_DURATION", {
-      valueType: ValueType.DOUBLE,
+    this._tracesCounter = this._meter.createCounter("azureMonitor.traceCount", {
+      valueType: ValueType.INT,
     });
   }
 
@@ -108,10 +125,36 @@ export class StandardMetrics {
         this._getDependencyDimensions(span)
       );
     }
+    if (span.events) {
+      span.events.forEach((event: TimedEvent) => {
+        event.attributes = event.attributes || {};
+        if (event.name === "exception") {
+          event.attributes["_MS.ProcessedByMetricExtractors"] = "(Name:'Exceptions', Ver:'1.1')";
+          this._exceptionsCounter.add(1, this._getExceptionDimensions(span.resource));
+        } else {
+          event.attributes["_MS.ProcessedByMetricExtractors"] = "(Name:'Traces', Ver:'1.1')";
+          this._tracesCounter.add(1, this._getTraceDimensions(span.resource));
+        }
+      });
+    }
+  }
+
+  /**
+   * Record LogRecord metrics, add attribute so data is not aggregated again in ingestion
+   * @internal
+   */
+  public recordLog(logRecord: LogRecord): void {
+    if (this._isExceptionTelemetry(logRecord)) {
+      logRecord.setAttribute("_MS.ProcessedByMetricExtractors", "(Name:'Exceptions', Ver:'1.1')");
+      this._exceptionsCounter.add(1, this._getExceptionDimensions(logRecord.resource));
+    } else if (this._isTraceTelemetry(logRecord)) {
+      logRecord.setAttribute("_MS.ProcessedByMetricExtractors", "(Name:'Traces', Ver:'1.1')");
+      this._tracesCounter.add(1, this._getTraceDimensions(logRecord.resource));
+    }
   }
 
   private _getRequestDimensions(span: ReadableSpan): Attributes {
-    const dimensions: MetricRequestDimensions = this._getBaseDimensions(span);
+    const dimensions: MetricRequestDimensions = this._getBaseDimensions(span.resource);
     dimensions.metricId = "requests/duration";
     const statusCode = String(span.attributes["http.status_code"]);
     dimensions.requestResultCode = statusCode;
@@ -120,7 +163,7 @@ export class StandardMetrics {
   }
 
   private _getDependencyDimensions(span: ReadableSpan): Attributes {
-    const dimensions: MetricDependencyDimensions = this._getBaseDimensions(span);
+    const dimensions: MetricDependencyDimensions = this._getBaseDimensions(span.resource);
     dimensions.metricId = "dependencies/duration";
     const statusCode = String(span.attributes["http.status_code"]);
     dimensions.dependencyTarget = this._getDependencyTarget(span.attributes);
@@ -130,11 +173,23 @@ export class StandardMetrics {
     return dimensions as Attributes;
   }
 
-  private _getBaseDimensions(span: ReadableSpan): StandardMetricBaseDimensions {
+  private _getExceptionDimensions(resource: Resource): Attributes {
+    const dimensions: StandardMetricBaseDimensions = this._getBaseDimensions(resource);
+    dimensions.metricId = "exceptions/count";
+    return dimensions as Attributes;
+  }
+
+  private _getTraceDimensions(resource: Resource): Attributes {
+    const dimensions: StandardMetricBaseDimensions = this._getBaseDimensions(resource);
+    dimensions.metricId = "traces/count";
+    return dimensions as Attributes;
+  }
+
+  private _getBaseDimensions(resource: Resource): StandardMetricBaseDimensions {
     const dimensions: StandardMetricBaseDimensions = {};
     dimensions.IsAutocollected = "True";
-    if (span.resource) {
-      const spanResourceAttributes = span.resource.attributes;
+    if (resource) {
+      const spanResourceAttributes = resource.attributes;
       const serviceName = spanResourceAttributes[SemanticResourceAttributes.SERVICE_NAME];
       const serviceNamespace = spanResourceAttributes[SemanticResourceAttributes.SERVICE_NAMESPACE];
       if (serviceName) {
@@ -173,4 +228,32 @@ export class StandardMetrics {
     }
     return "";
   }
+
+  private _isExceptionTelemetry(logRecord: LogRecord) {
+    const baseType = logRecord.attributes["_MS.baseType"];
+    // If Application Insights Legacy logs
+    if (baseType && baseType === "ExceptionData") {
+      return true;
+    } else if (
+      logRecord.attributes[SemanticAttributes.EXCEPTION_MESSAGE] ||
+      logRecord.attributes[SemanticAttributes.EXCEPTION_TYPE]
+    ) {
+      return true;
+    }
+    return false;
+  }
+
+  private _isTraceTelemetry(logRecord: LogRecord) {
+    const baseType = logRecord.attributes["_MS.baseType"];
+    // If Application Insights Legacy logs
+    if (baseType && baseType === "MessageData") {
+      return true;
+    } else if (
+      !logRecord.attributes[SemanticAttributes.EXCEPTION_MESSAGE] &&
+      !logRecord.attributes[SemanticAttributes.EXCEPTION_TYPE]
+    ) {
+      return true;
+    }
+    return false;
+  }
 }