extra logging for PartialSuccess (Azure#37915)

Author: Timothy Mothra

sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/Diagnostics/AzureMonitorExporterEventSource.cs

@@ -5,6 +5,7 @@
 using System.Diagnostics.Tracing;
 using System.Runtime.CompilerServices;
 using Azure.Monitor.OpenTelemetry.Exporter.Internals.ConnectionString;
+using Azure.Monitor.OpenTelemetry.Exporter.Models;
 
 namespace Azure.Monitor.OpenTelemetry.Exporter.Internals.Diagnostics
 {
@@ -355,5 +356,32 @@ public void FailedToDeserializeIngestionResponse(Exception ex)
 
         [Event(36, Message = "Version string exceeds expected length. This is only for internal telemetry and can safely be ignored. Type Name: {0}. Version: {1}", Level = EventLevel.Verbose)]
         public void VersionStringUnexpectedLength(string typeName, string value) => WriteEvent(36, typeName, value);
+
+        [Event(37, Message = "Failed to delete telemetry from storage. This may result in duplicate telemetry if the file is processed again. Not user actionable.", Level = EventLevel.Warning)]
+        public void DeletedFailed() => WriteEvent(37);
+
+        [NonEvent]
+        public void PartialContentResponseInvalid(int totalTelemetryItems, TelemetryErrorDetails error)
+        {
+            if (IsEnabled(EventLevel.Warning))
+            {
+                PartialContentResponseInvalid(totalTelemetryItems, error.Index?.ToString() ?? "N/A", error.StatusCode?.ToString() ?? "N/A", error.Message);
+            }
+        }
+
+        [Event(38, Message = "Received a partial success from ingestion that does not match telemetry that was sent. Total telemetry items sent: {0}. Error Index: {1}. Error StatusCode: {2}. Error Message: {3}", Level = EventLevel.Warning)]
+        public void PartialContentResponseInvalid(int totalTelemetryItems, string errorIndex, string errorStatusCode, string errorMessage) => WriteEvent(38, totalTelemetryItems, errorIndex, errorStatusCode, errorMessage);
+
+        [NonEvent]
+        public void PartialContentResponseUnhandled(TelemetryErrorDetails error)
+        {
+            if (IsEnabled(EventLevel.Warning))
+            {
+                PartialContentResponseUnhandled(error.StatusCode?.ToString() ?? "N/A", error.Message);
+            }
+        }
+
+        [Event(39, Message = "Received a partial success from ingestion. This status code is not handled and telemetry will be lost. Error StatusCode: {0}. Error Message: {1}", Level = EventLevel.Warning)]
+        public void PartialContentResponseUnhandled(string errorStatusCode, string errorMessage) => WriteEvent(39, errorStatusCode, errorMessage);
     }
 }

sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/HttpPipelineHelper.cs

@@ -113,6 +113,12 @@ internal static bool TryGetRequestContent(RequestContent? content, [NotNullWhen(
             return true;
         }
 
+        /// <summary>
+        /// Parse a PartialSuccess response from ingestion.
+        /// </summary>
+        /// <param name="trackResponse"><see cref="TrackResponse"/> is the parsed response from ingestion.</param>
+        /// <param name="content"><see cref="RequestContent"/> that was sent to ingestion.</param>
+        /// <returns>Telemetry that will be tried.</returns>
         internal static byte[]? GetPartialContentForRetry(TrackResponse trackResponse, RequestContent? content)
         {
             if (content == null)
@@ -123,14 +129,16 @@ internal static bool TryGetRequestContent(RequestContent? content, [NotNullWhen(
             string? partialContent = null;
             if (TryGetRequestContent(content, out var requestContent))
             {
-                var fullContent = Encoding.UTF8.GetString(requestContent).Split('\n');
+                var telemetryItems = Encoding.UTF8.GetString(requestContent).Split('\n');
                 foreach (var error in trackResponse.Errors)
                 {
                     if (error != null && error.Index != null)
                     {
-                        if (error.Index >= fullContent.Length || error.Index < 0)
+                        int errorIndex = (int)error.Index;
+
+                        if (errorIndex >= telemetryItems.Length || errorIndex < 0)
                         {
-                            // TODO: log
+                            AzureMonitorExporterEventSource.Log.PartialContentResponseInvalid(telemetryItems.Length, error);
                             continue;
                         }
 
@@ -142,13 +150,17 @@ internal static bool TryGetRequestContent(RequestContent? content, [NotNullWhen(
                         {
                             if (string.IsNullOrEmpty(partialContent))
                             {
-                                partialContent = fullContent[(int)error.Index];
+                                partialContent = telemetryItems[errorIndex];
                             }
                             else
                             {
-                                partialContent += '\n' + fullContent[(int)error.Index];
+                                partialContent += '\n' + telemetryItems[errorIndex];
                             }
                         }
+                        else
+                        {
+                            AzureMonitorExporterEventSource.Log.PartialContentResponseUnhandled(error);
+                        }
                     }
                 }
             }

sdk/monitor/Azure.Monitor.OpenTelemetry.Exporter/src/Internals/TransmitFromStorageHandler.cs

@@ -61,8 +61,11 @@ internal void TransmitFromStorage(object? sender, ElapsedEventArgs? e)
 
                             // In case if the delete fails, there is a possibility
                             // that the current batch will be transmitted more than once resulting in duplicates.
-                            // TODO: TryDelete returns a boolean, should we log when this occurs?
-                            blob.TryDelete();
+                            var deleteSucceeded = blob.TryDelete();
+                            if (!deleteSucceeded)
+                            {
+                                AzureMonitorExporterEventSource.Log.DeletedFailed();
+                            }
                         }
                         else
                         {