Add support for structured metadata in logs

README.md

@@ -25,6 +25,7 @@ Glory to Ukraine! 🇺🇦
 - [Quickstart](#quickstart)
 - [Custom HTTP Client](#custom-http-client)
 - [Sending json content to Loki](#sending-json-content-to-loki)
+- [Structured Metadata](#structured-metadata)
 - [Inspiration and Credits](#inspiration-and-credits)
 
 ## What is this sink and Loki?
@@ -37,6 +38,7 @@ You can find more information about what Loki is over on [Grafana's website here
 - Formats and batches log entries to Loki via HTTP (using actual API)
 - Global and contextual labels support
 - Flexible Loki labels configuration possibilities
+- Structured metadata support for enhanced log context
 - Collision avoiding mechanism for labels
 - Integration with Serilog.Settings.Configuration
 - Customizable HTTP clients
@@ -166,5 +168,76 @@ Example configuration:
 }
 ```
 
+### Structured Metadata
+
+Loki supports [structured metadata](https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs) - additional key-value pairs that can be attached to each log line without being indexed as labels. This is useful for high-cardinality data like trace IDs, user IDs, or request IDs that you want to query but don't want to use as labels.
+
+> **Requirements:**
+> - Structured metadata requires **Loki 1.2.0 or newer**
+> - You must [enable structured metadata](https://grafana.com/docs/loki/latest/get-started/labels/structured-metadata/#enable-or-disable-structured-metadata) in your Loki configuration by setting `allow_structured_metadata: true` in `limits_config` (enabled by default in Loki 3.0+)
+>
+> Example Loki configuration:
+> ```yaml
+> limits_config:
+>   allow_structured_metadata: true
+> ```
+
+You can configure which log event properties should be extracted as structured metadata using the `propertiesAsStructuredMetadata` parameter:
+
+```csharp
+Log.Logger = new LoggerConfiguration()
+    .WriteTo.GrafanaLoki(
+        "http://localhost:3100",
+        propertiesAsStructuredMetadata: new[] { "trace_id", "user_id", "request_id" })
+    .CreateLogger();
+
+Log.Information("User {user_id} performed action with trace {trace_id}", "user123", "abc-xyz-123");
+```
+
+This produces a log entry with structured metadata attached:
+```json
+{
+  "streams": [{
+    "stream": {},
+    "values": [
+      ["1234567890000000000", "{\"Message\":\"...\",\"level\":\"info\"}", {"trace_id":"abc-xyz-123","user_id":"user123"}]
+    ]
+  }]
+}
+```
+
+By default, properties extracted as structured metadata are removed from the log line JSON to avoid duplication. To keep them in both places, set `leaveStructuredMetadataPropertiesIntact` to `true`:
+
+```csharp
+Log.Logger = new LoggerConfiguration()
+    .WriteTo.GrafanaLoki(
+        "http://localhost:3100",
+        propertiesAsStructuredMetadata: new[] { "trace_id", "user_id" },
+        leaveStructuredMetadataPropertiesIntact: true)
+    .CreateLogger();
+```
+
+Configuration via `appsettings.json`:
+
+```json
+{
+  "Serilog": {
+    "Using": ["Serilog.Sinks.Grafana.Loki"],
+    "WriteTo": [
+      {
+        "Name": "GrafanaLoki",
+        "Args": {
+          "uri": "http://localhost:3100",
+          "propertiesAsStructuredMetadata": ["trace_id", "user_id", "request_id"],
+          "leaveStructuredMetadataPropertiesIntact": false
+        }
+      }
+    ]
+  }
+}
+```
+
+For more details on all configuration parameters, see the [Application Settings wiki page](https://github.com/mishamyte/serilog-sinks-grafana-loki/wiki/Application-settings).
+
 ### Inspiration and Credits
-- [Serilog.Sinks.Loki](https://github.com/JosephWoodward/Serilog-Sinks-Loki)
+- [Serilog.Sinks.Loki](https://github.com/JosephWoodward/Serilog-Sinks-Loki)

sample/Serilog.Sinks.Grafana.Loki.SampleWebApp/appsettings.json

@@ -33,6 +33,10 @@
           ],
           "propertiesAsLabels": [
             "app"
+          ],
+          "propertiesAsStructuredMetadata": [
+            "RequestId",
+            "RequestPath"
           ]
         }
       }

src/Serilog.Sinks.Grafana.Loki/LoggerConfigurationLokiExtensions.cs

@@ -44,6 +44,10 @@ public static class LoggerConfigurationLokiExtensions
     /// <param name="propertiesAsLabels">
     /// The list of properties, which would be mapped to the labels.
     /// </param>
+    /// <param name="propertiesAsStructuredMetadata">
+    /// The list of properties, which would be mapped to structured metadata.
+    /// See <a href="https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs">docs</a>.
+    /// </param>
     /// <param name="credentials">
     /// Auth <see cref="LokiCredentials"/>.
     /// </param>
@@ -81,12 +85,16 @@ public static class LoggerConfigurationLokiExtensions
     /// <param name="leavePropertiesIntact">
     /// Leaves the list of properties intact after extracting the labels specified in propertiesAsLabels.
     /// </param>
+    /// <param name="leaveStructuredMetadataPropertiesIntact">
+    /// Leaves the list of properties intact after extracting the structured metadata specified in propertiesAsStructuredMetadata.
+    /// </param>
     /// <returns>Logger configuration, allowing configuration to continue.</returns>
     public static LoggerConfiguration GrafanaLoki(
         this LoggerSinkConfiguration sinkConfiguration,
         string uri,
         IEnumerable<LokiLabel>? labels = null,
         IEnumerable<string>? propertiesAsLabels = null,
+        IEnumerable<string>? propertiesAsStructuredMetadata = null,
         LokiCredentials? credentials = null,
         string? tenant = null,
         LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum,
@@ -97,7 +105,8 @@ public static LoggerConfiguration GrafanaLoki(
         ILokiHttpClient? httpClient = null,
         IReservedPropertyRenamingStrategy? reservedPropertyRenamingStrategy = null,
         bool useInternalTimestamp = false,
-        bool leavePropertiesIntact = false)
+        bool leavePropertiesIntact = false,
+        bool leaveStructuredMetadataPropertiesIntact = false)
     {
         if (sinkConfiguration == null)
         {
@@ -116,8 +125,10 @@ public static LoggerConfiguration GrafanaLoki(
             reservedPropertyRenamingStrategy,
             labels,
             propertiesAsLabels,
+            propertiesAsStructuredMetadata,
             useInternalTimestamp,
-            leavePropertiesIntact);
+            leavePropertiesIntact,
+            leaveStructuredMetadataPropertiesIntact);
 
         var sink = new LokiSink(
             LokiRoutesBuilder.BuildLogsEntriesRoute(uri),

src/Serilog.Sinks.Grafana.Loki/LokiBatchFormatter.cs

@@ -34,8 +34,10 @@ internal class LokiBatchFormatter : ILokiBatchFormatter
     private readonly IEnumerable<LokiLabel> _globalLabels;
     private readonly IReservedPropertyRenamingStrategy _renamingStrategy;
     private readonly IEnumerable<string> _propertiesAsLabels;
+    private readonly IEnumerable<string> _propertiesAsStructuredMetadata;
 
     private readonly bool _leavePropertiesIntact;
+    private readonly bool _leaveStructuredMetadataPropertiesIntact;
     private readonly bool _useInternalTimestamp;
 
     /// <summary>
@@ -51,24 +53,34 @@ internal class LokiBatchFormatter : ILokiBatchFormatter
     /// <param name="propertiesAsLabels">
     /// The list of properties, which would be mapped to the labels.
     /// </param>
+    /// <param name="propertiesAsStructuredMetadata">
+    /// The list of properties, which would be mapped to structured metadata.
+    /// </param>
     /// <param name="useInternalTimestamp">
     /// Compute internal timestamp
     /// </param>
     /// <param name="leavePropertiesIntact">
     /// Leave the list of properties intact after extracting the labels specified in propertiesAsLabels.
     /// </param>
+    /// <param name="leaveStructuredMetadataPropertiesIntact">
+    /// Leave the list of properties intact after extracting the structured metadata specified in propertiesAsStructuredMetadata.
+    /// </param>
     public LokiBatchFormatter(
         IReservedPropertyRenamingStrategy renamingStrategy,
         IEnumerable<LokiLabel>? globalLabels = null,
         IEnumerable<string>? propertiesAsLabels = null,
+        IEnumerable<string>? propertiesAsStructuredMetadata = null,
         bool useInternalTimestamp = false,
-        bool leavePropertiesIntact = false)
+        bool leavePropertiesIntact = false,
+        bool leaveStructuredMetadataPropertiesIntact = false)
     {
         _renamingStrategy = renamingStrategy;
         _globalLabels = globalLabels ?? Enumerable.Empty<LokiLabel>();
         _propertiesAsLabels = propertiesAsLabels ?? Enumerable.Empty<string>();
+        _propertiesAsStructuredMetadata = propertiesAsStructuredMetadata ?? Enumerable.Empty<string>();
         _useInternalTimestamp = useInternalTimestamp;
         _leavePropertiesIntact = leavePropertiesIntact;
+        _leaveStructuredMetadataPropertiesIntact = leaveStructuredMetadataPropertiesIntact;
     }
 
     /// <summary>
@@ -179,9 +191,41 @@ private void GenerateEntry(
             timestamp = lokiLogEvent.InternalTimestamp;
         }
 
+        // Extract structured metadata
+        Dictionary<string, string>? structuredMetadata = null;
+        if (_propertiesAsStructuredMetadata.Any())
+        {
+            structuredMetadata = new Dictionary<string, string>();
+            var propertiesToExtract = logEvent.Properties
+                .Where(kvp => _propertiesAsStructuredMetadata.Contains(kvp.Key))
+                .ToList();
+
+            foreach (var property in propertiesToExtract)
+            {
+                // Remove quotes from the value string representation
+                var value = property.Value.ToString().Replace("\"", string.Empty);
+                structuredMetadata[property.Key] = value;
+
+                // Remove the property from the log event if configured to do so
+                if (!_leaveStructuredMetadataPropertiesIntact)
+                {
+                    logEvent.RemovePropertyIfPresent(property.Key);
+                }
+            }
+        }
+
         formatter.Format(logEvent, buffer);
 
-        stream.AddEntry(timestamp, buffer.ToString().TrimEnd('\r', '\n'));
+        var entry = buffer.ToString().TrimEnd('\r', '\n');
+
+        if (structuredMetadata != null && structuredMetadata.Count > 0)
+        {
+            stream.AddEntry(timestamp, entry, structuredMetadata);
+        }
+        else
+        {
+            stream.AddEntry(timestamp, entry);
+        }
     }
 
     private (Dictionary<string, string> Labels, LokiLogEvent LokiLogEvent) GenerateLabels(LokiLogEvent lokiLogEvent)

src/Serilog.Sinks.Grafana.Loki/Models/LokiSerializationContext.cs

@@ -13,6 +13,9 @@
 namespace Serilog.Sinks.Grafana.Loki.Models;
 
 [JsonSerializable(typeof(LokiBatch))]
+[JsonSerializable(typeof(Dictionary<string, string>))]
+[JsonSerializable(typeof(object[]))]
+[JsonSerializable(typeof(string))]
 internal sealed partial class LokiSerializationContext : JsonSerializerContext
 {
 }

src/Serilog.Sinks.Grafana.Loki/Models/LokiStream.cs

@@ -19,7 +19,7 @@ internal class LokiStream
     public Dictionary<string, string> Labels { get; } = new();
 
     [JsonPropertyName("values")]
-    public IList<IList<string>> Entries { get; set; } = new List<IList<string>>();
+    public IList<IList<object>> Entries { get; set; } = new List<IList<object>>();
 
     public void AddLabel(string key, string value)
     {
@@ -28,6 +28,17 @@ public void AddLabel(string key, string value)
 
     public void AddEntry(DateTimeOffset timestamp, string entry)
     {
-        Entries.Add(new[] {timestamp.ToUnixNanosecondsString(), entry});
+        Entries.Add(new object[] {timestamp.ToUnixNanosecondsString(), entry});
+    }
+
+    public void AddEntry(DateTimeOffset timestamp, string entry, Dictionary<string, string>? structuredMetadata)
+    {
+        if (structuredMetadata == null || structuredMetadata.Count == 0)
+        {
+            AddEntry(timestamp, entry);
+            return;
+        }
+
+        Entries.Add(new object[] {timestamp.ToUnixNanosecondsString(), entry, structuredMetadata});
     }
 }

test/Serilog.Sinks.Grafana.Loki.Tests/IntegrationTests/LokiJsonTextFormatterRequestPayloadTests.cs

@@ -443,4 +443,55 @@ public void LevelPropertyShouldBeRenamed()
                         TimeStampReplacement));
             });
     }
+
+    [Fact]
+    public void PropertiesAsStructuredMetadataShouldBeSerializedCorrectly()
+    {
+        var logger = new LoggerConfiguration()
+            .WriteTo.GrafanaLoki(
+                "https://loki:3100",
+                propertiesAsStructuredMetadata: new[] { "trace_id", "user_id" },
+                httpClient: _client)
+            .CreateLogger();
+
+        logger.Information("User action: {Action} by {trace_id} for {user_id}", "login", "0242ac120002", "superUser123");
+        logger.Dispose();
+
+        _client.Content.ShouldMatchApproved(
+            c =>
+            {
+                c.SubFolder(ApprovalsFolderName);
+                c.WithScrubber(
+                    s => Regex.Replace(
+                        s,
+                        TimeStampRegEx,
+                        TimeStampReplacement));
+            });
+    }
+
+    [Fact]
+    public void StructuredMetadataWithLeavePropertiesIntactShouldKeepProperties()
+    {
+        var logger = new LoggerConfiguration()
+            .WriteTo.GrafanaLoki(
+                "https://loki:3100",
+                propertiesAsStructuredMetadata: new[] { "trace_id", "user_id" },
+                leaveStructuredMetadataPropertiesIntact: true,
+                httpClient: _client)
+            .CreateLogger();
+
+        logger.Information("User action: {Action} by {trace_id} for {user_id}", "login", "0242ac120002", "superUser123");
+        logger.Dispose();
+
+        _client.Content.ShouldMatchApproved(
+            c =>
+            {
+                c.SubFolder(ApprovalsFolderName);
+                c.WithScrubber(
+                    s => Regex.Replace(
+                        s,
+                        TimeStampRegEx,
+                        TimeStampReplacement));
+            });
+    }
 }