diff --git a/README.md b/README.md
index 1a930dc..649a1a7 100644
--- a/README.md
+++ b/README.md
@@ -25,6 +25,7 @@ Glory to Ukraine! πΊπ¦
- [Quickstart](#quickstart)
- [Custom HTTP Client](#custom-http-client)
- [Sending json content to Loki](#sending-json-content-to-loki)
+- [Structured Metadata](#structured-metadata)
- [Inspiration and Credits](#inspiration-and-credits)
## What is this sink and Loki?
@@ -37,6 +38,7 @@ You can find more information about what Loki is over on [Grafana's website here
- Formats and batches log entries to Loki via HTTP (using actual API)
- Global and contextual labels support
- Flexible Loki labels configuration possibilities
+- Structured metadata support for enhanced log context
- Collision avoiding mechanism for labels
- Integration with Serilog.Settings.Configuration
- Customizable HTTP clients
@@ -166,5 +168,76 @@ Example configuration:
}
```
+### Structured Metadata
+
+Loki supports [structured metadata](https://grafana.com/docs/loki/latest/reference/loki-http-api/#ingest-logs) - additional key-value pairs that can be attached to each log line without being indexed as labels. This is useful for high-cardinality data like trace IDs, user IDs, or request IDs that you want to query but don't want to use as labels.
+
+> **Requirements:**
+> - Structured metadata requires **Loki 1.2.0 or newer**
+> - You must [enable structured metadata](https://grafana.com/docs/loki/latest/get-started/labels/structured-metadata/#enable-or-disable-structured-metadata) in your Loki configuration by setting `allow_structured_metadata: true` in `limits_config` (enabled by default in Loki 3.0+)
+>
+> Example Loki configuration:
+> ```yaml
+> limits_config:
+> allow_structured_metadata: true
+> ```
+
+You can configure which log event properties should be extracted as structured metadata using the `propertiesAsStructuredMetadata` parameter:
+
+```csharp
+Log.Logger = new LoggerConfiguration()
+ .WriteTo.GrafanaLoki(
+ "http://localhost:3100",
+ propertiesAsStructuredMetadata: new[] { "trace_id", "user_id", "request_id" })
+ .CreateLogger();
+
+Log.Information("User {user_id} performed action with trace {trace_id}", "user123", "abc-xyz-123");
+```
+
+This produces a log entry with structured metadata attached:
+```json
+{
+ "streams": [{
+ "stream": {},
+ "values": [
+ ["1234567890000000000", "{\"Message\":\"...\",\"level\":\"info\"}", {"trace_id":"abc-xyz-123","user_id":"user123"}]
+ ]
+ }]
+}
+```
+
+By default, properties extracted as structured metadata are removed from the log line JSON to avoid duplication. To keep them in both places, set `leaveStructuredMetadataPropertiesIntact` to `true`:
+
+```csharp
+Log.Logger = new LoggerConfiguration()
+ .WriteTo.GrafanaLoki(
+ "http://localhost:3100",
+ propertiesAsStructuredMetadata: new[] { "trace_id", "user_id" },
+ leaveStructuredMetadataPropertiesIntact: true)
+ .CreateLogger();
+```
+
+Configuration via `appsettings.json`:
+
+```json
+{
+ "Serilog": {
+ "Using": ["Serilog.Sinks.Grafana.Loki"],
+ "WriteTo": [
+ {
+ "Name": "GrafanaLoki",
+ "Args": {
+ "uri": "http://localhost:3100",
+ "propertiesAsStructuredMetadata": ["trace_id", "user_id", "request_id"],
+ "leaveStructuredMetadataPropertiesIntact": false
+ }
+ }
+ ]
+ }
+}
+```
+
+For more details on all configuration parameters, see the [Application Settings wiki page](https://github.com/mishamyte/serilog-sinks-grafana-loki/wiki/Application-settings).
+
### Inspiration and Credits
-- [Serilog.Sinks.Loki](https://github.com/JosephWoodward/Serilog-Sinks-Loki)
\ No newline at end of file
+- [Serilog.Sinks.Loki](https://github.com/JosephWoodward/Serilog-Sinks-Loki)
diff --git a/sample/Serilog.Sinks.Grafana.Loki.SampleWebApp/appsettings.json b/sample/Serilog.Sinks.Grafana.Loki.SampleWebApp/appsettings.json
index a40c2c4..f764009 100644
--- a/sample/Serilog.Sinks.Grafana.Loki.SampleWebApp/appsettings.json
+++ b/sample/Serilog.Sinks.Grafana.Loki.SampleWebApp/appsettings.json
@@ -33,6 +33,10 @@
],
"propertiesAsLabels": [
"app"
+ ],
+ "propertiesAsStructuredMetadata": [
+ "RequestId",
+ "RequestPath"
]
}
}
diff --git a/src/Serilog.Sinks.Grafana.Loki/LoggerConfigurationLokiExtensions.cs b/src/Serilog.Sinks.Grafana.Loki/LoggerConfigurationLokiExtensions.cs
index 3c63746..8a5666d 100644
--- a/src/Serilog.Sinks.Grafana.Loki/LoggerConfigurationLokiExtensions.cs
+++ b/src/Serilog.Sinks.Grafana.Loki/LoggerConfigurationLokiExtensions.cs
@@ -44,6 +44,10 @@ public static class LoggerConfigurationLokiExtensions
///
/// The list of properties, which would be mapped to the labels.
///
+ ///
+ /// The list of properties, which would be mapped to structured metadata.
+ /// See docs.
+ ///
///
/// Auth .
///
@@ -81,12 +85,16 @@ public static class LoggerConfigurationLokiExtensions
///
/// Leaves the list of properties intact after extracting the labels specified in propertiesAsLabels.
///
+ ///
+ /// Leaves the list of properties intact after extracting the structured metadata specified in propertiesAsStructuredMetadata.
+ ///
/// Logger configuration, allowing configuration to continue.
public static LoggerConfiguration GrafanaLoki(
this LoggerSinkConfiguration sinkConfiguration,
string uri,
IEnumerable? labels = null,
IEnumerable? propertiesAsLabels = null,
+ IEnumerable? propertiesAsStructuredMetadata = null,
LokiCredentials? credentials = null,
string? tenant = null,
LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum,
@@ -97,7 +105,8 @@ public static LoggerConfiguration GrafanaLoki(
ILokiHttpClient? httpClient = null,
IReservedPropertyRenamingStrategy? reservedPropertyRenamingStrategy = null,
bool useInternalTimestamp = false,
- bool leavePropertiesIntact = false)
+ bool leavePropertiesIntact = false,
+ bool leaveStructuredMetadataPropertiesIntact = false)
{
if (sinkConfiguration == null)
{
@@ -116,8 +125,10 @@ public static LoggerConfiguration GrafanaLoki(
reservedPropertyRenamingStrategy,
labels,
propertiesAsLabels,
+ propertiesAsStructuredMetadata,
useInternalTimestamp,
- leavePropertiesIntact);
+ leavePropertiesIntact,
+ leaveStructuredMetadataPropertiesIntact);
var sink = new LokiSink(
LokiRoutesBuilder.BuildLogsEntriesRoute(uri),
diff --git a/src/Serilog.Sinks.Grafana.Loki/LokiBatchFormatter.cs b/src/Serilog.Sinks.Grafana.Loki/LokiBatchFormatter.cs
index bade389..0eb1eb5 100644
--- a/src/Serilog.Sinks.Grafana.Loki/LokiBatchFormatter.cs
+++ b/src/Serilog.Sinks.Grafana.Loki/LokiBatchFormatter.cs
@@ -34,8 +34,10 @@ internal class LokiBatchFormatter : ILokiBatchFormatter
private readonly IEnumerable _globalLabels;
private readonly IReservedPropertyRenamingStrategy _renamingStrategy;
private readonly IEnumerable _propertiesAsLabels;
+ private readonly IEnumerable _propertiesAsStructuredMetadata;
private readonly bool _leavePropertiesIntact;
+ private readonly bool _leaveStructuredMetadataPropertiesIntact;
private readonly bool _useInternalTimestamp;
///
@@ -51,24 +53,34 @@ internal class LokiBatchFormatter : ILokiBatchFormatter
///
/// The list of properties, which would be mapped to the labels.
///
+ ///
+ /// The list of properties, which would be mapped to structured metadata.
+ ///
///
/// Compute internal timestamp
///
///
/// Leave the list of properties intact after extracting the labels specified in propertiesAsLabels.
///
+ ///
+ /// Leave the list of properties intact after extracting the structured metadata specified in propertiesAsStructuredMetadata.
+ ///
public LokiBatchFormatter(
IReservedPropertyRenamingStrategy renamingStrategy,
IEnumerable? globalLabels = null,
IEnumerable? propertiesAsLabels = null,
+ IEnumerable? propertiesAsStructuredMetadata = null,
bool useInternalTimestamp = false,
- bool leavePropertiesIntact = false)
+ bool leavePropertiesIntact = false,
+ bool leaveStructuredMetadataPropertiesIntact = false)
{
_renamingStrategy = renamingStrategy;
_globalLabels = globalLabels ?? Enumerable.Empty();
_propertiesAsLabels = propertiesAsLabels ?? Enumerable.Empty();
+ _propertiesAsStructuredMetadata = propertiesAsStructuredMetadata ?? Enumerable.Empty();
_useInternalTimestamp = useInternalTimestamp;
_leavePropertiesIntact = leavePropertiesIntact;
+ _leaveStructuredMetadataPropertiesIntact = leaveStructuredMetadataPropertiesIntact;
}
///
@@ -179,9 +191,41 @@ private void GenerateEntry(
timestamp = lokiLogEvent.InternalTimestamp;
}
+ // Extract structured metadata
+ Dictionary? structuredMetadata = null;
+ if (_propertiesAsStructuredMetadata.Any())
+ {
+ structuredMetadata = new Dictionary();
+ var propertiesToExtract = logEvent.Properties
+ .Where(kvp => _propertiesAsStructuredMetadata.Contains(kvp.Key))
+ .ToList();
+
+ foreach (var property in propertiesToExtract)
+ {
+ // Remove quotes from the value string representation
+ var value = property.Value.ToString().Replace("\"", string.Empty);
+ structuredMetadata[property.Key] = value;
+
+ // Remove the property from the log event if configured to do so
+ if (!_leaveStructuredMetadataPropertiesIntact)
+ {
+ logEvent.RemovePropertyIfPresent(property.Key);
+ }
+ }
+ }
+
formatter.Format(logEvent, buffer);
- stream.AddEntry(timestamp, buffer.ToString().TrimEnd('\r', '\n'));
+ var entry = buffer.ToString().TrimEnd('\r', '\n');
+
+ if (structuredMetadata != null && structuredMetadata.Count > 0)
+ {
+ stream.AddEntry(timestamp, entry, structuredMetadata);
+ }
+ else
+ {
+ stream.AddEntry(timestamp, entry);
+ }
}
private (Dictionary Labels, LokiLogEvent LokiLogEvent) GenerateLabels(LokiLogEvent lokiLogEvent)
diff --git a/src/Serilog.Sinks.Grafana.Loki/Models/LokiSerializationContext.cs b/src/Serilog.Sinks.Grafana.Loki/Models/LokiSerializationContext.cs
index 58a12d9..085de1a 100644
--- a/src/Serilog.Sinks.Grafana.Loki/Models/LokiSerializationContext.cs
+++ b/src/Serilog.Sinks.Grafana.Loki/Models/LokiSerializationContext.cs
@@ -13,6 +13,9 @@
namespace Serilog.Sinks.Grafana.Loki.Models;
[JsonSerializable(typeof(LokiBatch))]
+[JsonSerializable(typeof(Dictionary))]
+[JsonSerializable(typeof(object[]))]
+[JsonSerializable(typeof(string))]
internal sealed partial class LokiSerializationContext : JsonSerializerContext
{
}
diff --git a/src/Serilog.Sinks.Grafana.Loki/Models/LokiStream.cs b/src/Serilog.Sinks.Grafana.Loki/Models/LokiStream.cs
index 0492910..0ad20d1 100644
--- a/src/Serilog.Sinks.Grafana.Loki/Models/LokiStream.cs
+++ b/src/Serilog.Sinks.Grafana.Loki/Models/LokiStream.cs
@@ -19,7 +19,7 @@ internal class LokiStream
public Dictionary Labels { get; } = new();
[JsonPropertyName("values")]
- public IList> Entries { get; set; } = new List>();
+ public IList> Entries { get; set; } = new List>();
public void AddLabel(string key, string value)
{
@@ -28,6 +28,17 @@ public void AddLabel(string key, string value)
public void AddEntry(DateTimeOffset timestamp, string entry)
{
- Entries.Add(new[] {timestamp.ToUnixNanosecondsString(), entry});
+ Entries.Add(new object[] {timestamp.ToUnixNanosecondsString(), entry});
+ }
+
+ public void AddEntry(DateTimeOffset timestamp, string entry, Dictionary? structuredMetadata)
+ {
+ if (structuredMetadata == null || structuredMetadata.Count == 0)
+ {
+ AddEntry(timestamp, entry);
+ return;
+ }
+
+ Entries.Add(new object[] {timestamp.ToUnixNanosecondsString(), entry, structuredMetadata});
}
}
\ No newline at end of file
diff --git a/test/Serilog.Sinks.Grafana.Loki.Tests/IntegrationTests/LokiJsonTextFormatterRequestPayloadTests.cs b/test/Serilog.Sinks.Grafana.Loki.Tests/IntegrationTests/LokiJsonTextFormatterRequestPayloadTests.cs
index ca88251..3fc2771 100644
--- a/test/Serilog.Sinks.Grafana.Loki.Tests/IntegrationTests/LokiJsonTextFormatterRequestPayloadTests.cs
+++ b/test/Serilog.Sinks.Grafana.Loki.Tests/IntegrationTests/LokiJsonTextFormatterRequestPayloadTests.cs
@@ -443,4 +443,55 @@ public void LevelPropertyShouldBeRenamed()
TimeStampReplacement));
});
}
+
+ [Fact]
+ public void PropertiesAsStructuredMetadataShouldBeSerializedCorrectly()
+ {
+ var logger = new LoggerConfiguration()
+ .WriteTo.GrafanaLoki(
+ "https://loki:3100",
+ propertiesAsStructuredMetadata: new[] { "trace_id", "user_id" },
+ httpClient: _client)
+ .CreateLogger();
+
+ logger.Information("User action: {Action} by {trace_id} for {user_id}", "login", "0242ac120002", "superUser123");
+ logger.Dispose();
+
+ _client.Content.ShouldMatchApproved(
+ c =>
+ {
+ c.SubFolder(ApprovalsFolderName);
+ c.WithScrubber(
+ s => Regex.Replace(
+ s,
+ TimeStampRegEx,
+ TimeStampReplacement));
+ });
+ }
+
+ [Fact]
+ public void StructuredMetadataWithLeavePropertiesIntactShouldKeepProperties()
+ {
+ var logger = new LoggerConfiguration()
+ .WriteTo.GrafanaLoki(
+ "https://loki:3100",
+ propertiesAsStructuredMetadata: new[] { "trace_id", "user_id" },
+ leaveStructuredMetadataPropertiesIntact: true,
+ httpClient: _client)
+ .CreateLogger();
+
+ logger.Information("User action: {Action} by {trace_id} for {user_id}", "login", "0242ac120002", "superUser123");
+ logger.Dispose();
+
+ _client.Content.ShouldMatchApproved(
+ c =>
+ {
+ c.SubFolder(ApprovalsFolderName);
+ c.WithScrubber(
+ s => Regex.Replace(
+ s,
+ TimeStampRegEx,
+ TimeStampReplacement));
+ });
+ }
}
\ No newline at end of file