From 1947737db8f45401cbb14c301cd446128a07ce57 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Fri, 14 Nov 2025 12:27:17 -0800 Subject: [PATCH 1/3] implement resource attributes for geneva logs --- .../CHANGELOG.md | 7 + .../GenevaExporterOptions.cs | 2 + .../GenevaLogExporter.cs | 7 +- .../Internal/MsgPack/MsgPackLogExporter.cs | 140 ++++++++--- .../Exporter/LogExporterBenchmarks.cs | 23 +- .../Exporter/TLDLogExporterBenchmarks.cs | 19 +- .../GenevaLogExporterTests.cs | 226 ++++++++++++++++-- .../LogSerializationTests.cs | 3 +- .../MsgPackLogExporterTests.cs | 5 +- 9 files changed, 351 insertions(+), 81 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md index bb63d18174..e7f7a805ce 100644 --- a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md @@ -2,6 +2,13 @@ ## Unreleased +* Support for specifying resource attributes, including + `service.name`, `service.instanceId`, and custom attributes in log fields. + ([#TODO](https://github.com/open-telemetry/opentelemetry-dotnet-contrib)) +* Logs: Custom log fields take precedence over prepopulated fields, + preventing duplicate keys in exported logs. + ([#TODO](https://github.com/open-telemetry/opentelemetry-dotnet-contrib)) + ## 1.14.0 Released 2025-Nov-13 diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaExporterOptions.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaExporterOptions.cs index 14a2419285..1314e3147d 100644 --- a/src/OpenTelemetry.Exporter.Geneva/GenevaExporterOptions.cs +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaExporterOptions.cs @@ -100,6 +100,8 @@ public IReadOnlyDictionary? TableNameMappings /// /// Gets or sets prepopulated fields. + /// + /// Pre-populated fields are fields that are added as dedicated fields to every record, unless it would conflict with a log or trace field that is marked as a custom field. /// public IReadOnlyDictionary PrepopulatedFields { diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs index 2faab4661a..be75191637 100644 --- a/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs @@ -88,9 +88,14 @@ public GenevaLogExporter(GenevaExporterOptions options) throw new NotSupportedException($"Protocol '{connectionStringBuilder.Protocol}' is not supported"); } + Resources.Resource ResourceProvider() + { + return connectionStringBuilder.HonorResourceAttributes ? this.ParentProvider.GetResource() : Resources.Resource.Empty; + } + if (useMsgPackExporter) { - var msgPackLogExporter = new MsgPackLogExporter(options); + var msgPackLogExporter = new MsgPackLogExporter(options, ResourceProvider); this.IsUsingUnixDomainSocket = msgPackLogExporter.IsUsingUnixDomainSocket; this.exportLogRecord = msgPackLogExporter.Export; this.exporter = msgPackLogExporter; diff --git a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs index e3953e8235..b65d1632f8 100644 --- a/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs +++ b/src/OpenTelemetry.Exporter.Geneva/Internal/MsgPack/MsgPackLogExporter.cs @@ -12,6 +12,7 @@ using OpenTelemetry.Exporter.Geneva.Transports; using OpenTelemetry.Internal; using OpenTelemetry.Logs; +using OpenTelemetry.Resources; namespace OpenTelemetry.Exporter.Geneva.MsgPack; @@ -33,16 +34,20 @@ internal sealed class MsgPackLogExporter : MsgPackExporter, IDisposable #if NET private readonly FrozenSet? customFields; - private readonly FrozenDictionary? prepopulatedFields; #else private readonly HashSet? customFields; - private readonly Dictionary? prepopulatedFields; #endif private readonly ExceptionStackExportMode exportExceptionStack; - private readonly List? prepopulatedFieldKeys; private readonly byte[] bufferEpilogue; private readonly IDataTransport dataTransport; + private readonly Func resourceProvider; + + // These are values that are always added to the body as dedicated fields + private readonly Dictionary prepopulatedFields; + + // These are values that are always added to env_properties + private readonly Dictionary propertiesEntries; private readonly int stringFieldSizeLimitCharCount; // the maximum string size limit for MsgPack strings // This is used for Scopes @@ -50,9 +55,12 @@ internal sealed class MsgPackLogExporter : MsgPackExporter, IDisposable private bool isDisposed; - public MsgPackLogExporter(GenevaExporterOptions options) + public MsgPackLogExporter(GenevaExporterOptions options, Func resourceProvider) { Guard.ThrowIfNull(options); + Guard.ThrowIfNull(resourceProvider); + + this.resourceProvider = resourceProvider; this.tableNameSerializer = new(options, defaultTableName: "Log"); this.exportExceptionStack = options.ExceptionStackExportMode; @@ -88,21 +96,17 @@ public MsgPackLogExporter(GenevaExporterOptions options) } this.stringFieldSizeLimitCharCount = connectionStringBuilder.PrivatePreviewLogMessagePackStringSizeLimit; + + this.propertiesEntries = []; + + this.prepopulatedFields = new Dictionary(options.PrepopulatedFields.Count, StringComparer.Ordinal); + if (options.PrepopulatedFields != null) { - this.prepopulatedFieldKeys = []; - var tempPrepopulatedFields = new Dictionary(options.PrepopulatedFields.Count, StringComparer.Ordinal); foreach (var kv in options.PrepopulatedFields) { - tempPrepopulatedFields[kv.Key] = kv.Value; - this.prepopulatedFieldKeys.Add(kv.Key); + this.prepopulatedFields[kv.Key] = kv.Value; } - -#if NET - this.prepopulatedFields = tempPrepopulatedFields.ToFrozenDictionary(StringComparer.Ordinal); -#else - this.prepopulatedFields = tempPrepopulatedFields; -#endif } // TODO: Validate custom fields (reserved name? etc). @@ -174,19 +178,59 @@ public void Dispose() this.isDisposed = true; } + internal void AddResourceAttributesToPrepopulated() + { + // This function needs to be idempotent + + foreach (var entry in this.resourceProvider().Attributes) + { + string key = entry.Key; + bool isDedicatedField = false; + if (entry.Value is string) + { + switch (key) + { + case "service.name": + key = Schema.V40.PartA.Extensions.Cloud.Role; + isDedicatedField = true; + break; + case "service.instanceId": + key = Schema.V40.PartA.Extensions.Cloud.RoleInstance; + isDedicatedField = true; + break; + } + } + + if (isDedicatedField || this.customFields == null || this.customFields.Contains(key)) + { + if (!this.prepopulatedFields.ContainsKey(key)) + { + this.prepopulatedFields.Add(key, entry.Value); + } + } + else + { + if (!this.propertiesEntries.ContainsKey(key)) + { + this.propertiesEntries.Add(key, entry.Value); + } + } + } + } + internal ArraySegment SerializeLogRecord(LogRecord logRecord) { // `LogRecord.State` and `LogRecord.StateValues` were marked Obsolete in https://github.com/open-telemetry/opentelemetry-dotnet/pull/4334 #pragma warning disable 0618 - IReadOnlyList>? listKvp; + IReadOnlyList>? logFields; if (logRecord.StateValues != null) { - listKvp = logRecord.StateValues; + logFields = logRecord.StateValues; } else { // Attempt to see if State could be ROL_KVP. - listKvp = logRecord.State as IReadOnlyList>; + logFields = logRecord.State as IReadOnlyList> ?? []; } #pragma warning restore 0618 @@ -194,7 +238,7 @@ internal ArraySegment SerializeLogRecord(LogRecord logRecord) /* Fluentd Forward Mode: [ - "Log", + "Log", // (or category name) [ [ , { "env_ver": "4.0", ... } ] ], @@ -227,15 +271,20 @@ internal ArraySegment SerializeLogRecord(LogRecord logRecord) ushort cntFields = 0; var idxMapSizePatch = cursor - 2; - if (this.prepopulatedFieldKeys != null) + this.AddResourceAttributesToPrepopulated(); + + foreach (var entry in this.prepopulatedFields) { - for (var i = 0; i < this.prepopulatedFieldKeys.Count; i++) + // A prepopulated entry should not be added if the same key exists in the log, + // and customFields configuration would make it a dedicated field. + if ((this.customFields == null || this.customFields.Contains(entry.Key)) + && logFields.Any(kvp => kvp.Key == entry.Key)) { - var key = this.prepopulatedFieldKeys[i]; - var value = this.prepopulatedFields![key]; - cursor = AddPartAField(buffer, cursor, key, value); - cntFields += 1; + continue; } + + cursor = AddPartAField(buffer, cursor, entry.Key, entry.Value); + cntFields += 1; } // Part A - core envelope @@ -295,10 +344,8 @@ internal ArraySegment SerializeLogRecord(LogRecord logRecord) var hasEnvProperties = false; var bodyPopulated = false; var namePopulated = false; - for (var i = 0; i < listKvp?.Count; i++) + foreach (var entry in logFields) { - var entry = listKvp[i]; - // Iteration #1 - Get those fields which become dedicated columns // i.e all Part B fields and opt-in Part C fields. if (entry.Key == "{OriginalFormat}") @@ -366,27 +413,44 @@ internal ArraySegment SerializeLogRecord(LogRecord logRecord) cursor = dataForScopes.Cursor; cntFields = dataForScopes.FieldsCount; - if (hasEnvProperties) + if (hasEnvProperties || this.propertiesEntries.Count > 0) { - // Iteration #2 - Get all "other" fields and collapse them into single field - // named "env_properties". + // Anything that's not a dedicated field gets put into a part C field called "env_properties". ushort envPropertiesCount = 0; cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "env_properties"); cursor = MessagePackSerializer.WriteMapHeader(buffer, cursor, ushort.MaxValue); var idxMapSizeEnvPropertiesPatch = cursor - 2; - for (var i = 0; i < listKvp!.Count; i++) + + if (hasEnvProperties) { - var entry = listKvp[i]; - if (entry.Key == "{OriginalFormat}" || this.customFields!.Contains(entry.Key)) + foreach (var entry in logFields) { - continue; + if (entry.Key == "{OriginalFormat}" || this.customFields!.Contains(entry.Key)) + { + continue; + } + else + { + cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, entry.Key, this.stringFieldSizeLimitCharCount); + cursor = MessagePackSerializer.Serialize(buffer, cursor, entry.Value); + envPropertiesCount += 1; + } } - else + } + + foreach (var entry in this.propertiesEntries) + { + // A prepopulated env_properties entry should not be added if the same key exists in the log, + // and lack of customFields configuration would place it in env_properties. + if (this.customFields != null && !this.customFields.Contains(entry.Key) + && logFields.Any(kvp => kvp.Key == entry.Key)) { - cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, entry.Key, this.stringFieldSizeLimitCharCount); - cursor = MessagePackSerializer.Serialize(buffer, cursor, entry.Value); - envPropertiesCount += 1; + continue; } + + cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, entry.Key); + cursor = MessagePackSerializer.Serialize(buffer, cursor, entry.Value); + envPropertiesCount += 1; } // Prepare state for scopes diff --git a/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/LogExporterBenchmarks.cs b/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/LogExporterBenchmarks.cs index 1c1bef453d..477d1109d8 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/LogExporterBenchmarks.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/LogExporterBenchmarks.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging; using OpenTelemetry.Exporter.Geneva.MsgPack; using OpenTelemetry.Logs; +using OpenTelemetry.Resources; /* BenchmarkDotNet v0.13.10, Windows 11 (10.0.23424.1000) @@ -74,16 +75,18 @@ public LogExporterBenchmarks() // For msgpack serialization + export this.logRecord = GenerateTestLogRecord(); this.batch = GenerateTestLogRecordBatch(); - this.exporter = new MsgPackLogExporter(new GenevaExporterOptions - { - ConnectionString = "EtwSession=OpenTelemetry", - PrepopulatedFields = new Dictionary - { - ["cloud.role"] = "BusyWorker", - ["cloud.roleInstance"] = "CY1SCH030021417", - ["cloud.roleVer"] = "9.0.15289.2", - }, - }); + this.exporter = new MsgPackLogExporter( + new GenevaExporterOptions + { + ConnectionString = "EtwSession=OpenTelemetry", + PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }, + }, + () => Resource.Empty); } [Benchmark] diff --git a/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/TLDLogExporterBenchmarks.cs b/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/TLDLogExporterBenchmarks.cs index 4256741e44..8808c1d016 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/TLDLogExporterBenchmarks.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/TLDLogExporterBenchmarks.cs @@ -6,6 +6,7 @@ using OpenTelemetry.Exporter.Geneva.MsgPack; using OpenTelemetry.Exporter.Geneva.Tld; using OpenTelemetry.Logs; +using OpenTelemetry.Resources; using OpenTelemetry.Trace; /* @@ -37,16 +38,18 @@ public class TLDLogExporterBenchmarks public TLDLogExporterBenchmarks() { - this.msgPackExporter = new MsgPackLogExporter(new GenevaExporterOptions - { - ConnectionString = "EtwSession=OpenTelemetry", - PrepopulatedFields = new Dictionary + this.msgPackExporter = new MsgPackLogExporter( + new GenevaExporterOptions { - ["cloud.role"] = "BusyWorker", - ["cloud.roleInstance"] = "CY1SCH030021417", - ["cloud.roleVer"] = "9.0.15289.2", + ConnectionString = "EtwSession=OpenTelemetry", + PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }, }, - }); + () => Resource.Empty); this.tldExporter = new TldLogExporter(new GenevaExporterOptions() { diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs index f1e07b7138..d367b98aa2 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs @@ -13,6 +13,7 @@ using Microsoft.Extensions.Logging; using OpenTelemetry.Exporter.Geneva.MsgPack; using OpenTelemetry.Logs; +using OpenTelemetry.Resources; using OpenTelemetry.Tests; using Xunit; @@ -179,7 +180,7 @@ public void TableNameMappingTest(params string[] category) .AddFilter("*", LogLevel.Trace)); // Enable all LogLevels // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); ILogger logger; object fluentdData; @@ -298,7 +299,7 @@ public void PassThruTableMappingsWhenTheRuleIsEnabled() .AddFilter("*", LogLevel.Trace)); // Enable all LogLevels // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); ILogger passThruTableMappingsLogger, userInitializedTableMappingsLogger; var m_buffer = MsgPackLogExporter.Buffer; @@ -473,7 +474,7 @@ public void SerializeILoggerScopes(bool hasCustomFields) Assert.Single(exportedItems); var logRecord = exportedItems[0]; - this.AssertFluentdForwardModeForLogRecord(exporterOptions, fluentdData, logRecord); + this.AssertFluentdForwardModeForLogRecord(exporterOptions, Resource.Empty, fluentdData, logRecord); } finally { @@ -532,7 +533,7 @@ public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage) .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -666,9 +667,12 @@ public void SerializationTestWithILoggerLogWithTemplates(bool hasTableNameMappin ["cloud.role"] = "BusyWorker", ["cloud.roleInstance"] = "CY1SCH030021417", ["cloud.roleVer"] = "9.0.15289.2", + ["prepopulated"] = "prepopulated field", }, }; + var resource = ResourceBuilder.CreateEmpty().AddAttributes([new KeyValuePair("resourceAttr", "from resource")]).Build(); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; @@ -713,7 +717,7 @@ public void SerializationTestWithILoggerLogWithTemplates(bool hasTableNameMappin .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => resource); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -764,7 +768,154 @@ public void SerializationTestWithILoggerLogWithTemplates(bool hasTableNameMappin { _ = exporter.SerializeLogRecord(logRecord); var fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); - this.AssertFluentdForwardModeForLogRecord(exporterOptions, fluentdData, logRecord); + this.AssertFluentdForwardModeForLogRecord(exporterOptions, resource, fluentdData, logRecord); + } + } + finally + { + server?.Dispose(); + try + { + File.Delete(path); + } + catch + { + } + } + } + + [Theory] + [InlineData(false, true, true, false)] + [InlineData(false, true, true, true)] + [InlineData(true, false, true, false)] + [InlineData(true, false, true, true)] + [InlineData(true, true, false, false)] + [InlineData(true, true, false, true)] + [InlineData(true, true, true, false)] + [InlineData(true, true, true, true)] + public void SerializationTestWithDuplicateFields(bool conflictingPrepopulatedField, bool conflictingResourceAttribute, bool conflictingLogField, bool isCustomField) + { + var path = string.Empty; + Socket server = null; + var logRecordList = new List(); + try + { + var exporterOptions = new GenevaExporterOptions(); + if (conflictingPrepopulatedField) + { + exporterOptions.PrepopulatedFields = new Dictionary + { + ["Conflict"] = "prepopulated field", + }; + } + + var resource = Resource.Empty; + if (conflictingResourceAttribute) + { + resource = ResourceBuilder.CreateEmpty().AddAttributes([new KeyValuePair("Conflict", "resource attribute")]).Build(); + } + + exporterOptions.CustomFields = []; + if (isCustomField) + { + exporterOptions.CustomFields = ["Conflict"]; + } + + using var loggerFactory = LoggerFactory.Create(builder => builder + .AddOpenTelemetry(options => options.AddInMemoryExporter(logRecordList)) + .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels + var logger = loggerFactory.CreateLogger(); + + if (conflictingLogField) + { + logger.Log(LogLevel.Trace, 101, "Log a message with a {Conflict} with other fields", "log field"); + } + else + { + logger.Log(LogLevel.Trace, 101, "Log a normal message without conflict"); + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; + } + else + { + path = GenerateTempFilePath(); + exporterOptions.ConnectionString = "Endpoint=unix:" + path; + var endpoint = new UnixDomainSocketEndPoint(path); + server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server.Bind(endpoint); + server.Listen(1); + } + + using var exporter = new MsgPackLogExporter(exporterOptions, () => resource); + var m_buffer = MsgPackLogExporter.Buffer; + Assert.Single(logRecordList); + var serializedLog = exporter.SerializeLogRecord(logRecordList[0]); + var fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); + var signal = (fluentdData as object[])[0] as string; + var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; + var timeStamp = (DateTime)(TimeStampAndMappings as object[])[0]; + var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; + var env_properties = mapping.GetValueOrDefault("env_properties") as Dictionary ?? []; + + void AssertField(bool isDedicated, string fieldValue) + { + if (isDedicated) + { + Assert.Contains("Conflict", mapping); + Assert.Equal(fieldValue, mapping["Conflict"]); + } + else + { + Assert.Contains("Conflict", env_properties); + Assert.Equal(fieldValue, env_properties["Conflict"]); + } + } + + if (isCustomField) + { + // If Conflict is marked as a custom field, it should never appear in env_properties + Assert.DoesNotContain("Conflict", env_properties); + + // If Conflict is marked as a custom field, + // then the conflict will occur in a dedicated field. + // Log field has highest precedence, followed by prepopulated field + if (conflictingLogField) + { + AssertField(true, "log field"); + } + else if (conflictingPrepopulatedField) + { + AssertField(true, "prepopulated field"); + } + + // no need to check resource attribute because it will never win a conflict + } + else + { + // Prepopulated fields are unaffected by not being marked as custom fields + if (conflictingPrepopulatedField) + { + AssertField(true, "prepopulated field"); + } + else + { + Assert.DoesNotContain("Conflict", mapping); + } + + // If Conflict is not marked as a custom field, + // log fields and resource attributes conflict in env_properties, + // and log fields have precedence + if (conflictingLogField) + { + AssertField(false, "log field"); + } + else if (conflictingResourceAttribute) + { + AssertField(false, "resource attribute"); + } } } finally @@ -844,16 +995,18 @@ public void SuccessfulExport_Linux() serverSocket.ReceiveTimeout = 10000; // Create a test exporter to get MessagePack byte data for validation of the data received via Socket. - using var exporter = new MsgPackLogExporter(new GenevaExporterOptions - { - ConnectionString = "Endpoint=unix:" + path, - PrepopulatedFields = new Dictionary + using var exporter = new MsgPackLogExporter( + new GenevaExporterOptions { - ["cloud.role"] = "BusyWorker", - ["cloud.roleInstance"] = "CY1SCH030021417", - ["cloud.roleVer"] = "9.0.15289.2", + ConnectionString = "Endpoint=unix:" + path, + PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }, }, - }); + () => Resource.Empty); // Emit a LogRecord and grab a copy of internal buffer for validation. var logger = loggerFactory.CreateLogger(); @@ -938,7 +1091,7 @@ public void SerializationTestForException() .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -1031,7 +1184,7 @@ public void SerializationTestForEventName(EventNameExportMode eventNameExportMod })); // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -1158,7 +1311,7 @@ public void SerializationTestForPartBName(bool hasCustomFields, bool hasNameInCu })); // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -1273,7 +1426,7 @@ public void SerializationTestForEventId() .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter var logger = loggerFactory.CreateLogger(); @@ -1508,7 +1661,7 @@ private static object GetField(object fluentdData, string key) return mapping.TryGetValue(key, out var value) ? value : null; } - private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporterOptions, object fluentdData, LogRecord logRecord) + private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporterOptions, Resource resource, object fluentdData, LogRecord logRecord) { /* Fluentd Forward Mode: [ @@ -1559,7 +1712,7 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter foreach (var item in exporterOptions.PrepopulatedFields) { var partAValue = item.Value as string; - var partAKey = MsgPackExporter.V40_PART_A_MAPPING[item.Key]; + var partAKey = MsgPackExporter.V40_PART_A_MAPPING.GetValueOrDefault(item.Key, item.Key); Assert.Equal(partAValue, mapping[partAKey]); } @@ -1636,14 +1789,45 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter } else if (exporterOptions.CustomFields == null || exporterOptions.CustomFields.Contains(item.Key)) { + // It should be found as a custom field + if (item.Value != null) + { + Assert.Equal(item.Value, mapping[item.Key]); + } + + if (envPropertiesMapping != null) + { + Assert.DoesNotContain(item.Key, envPropertiesMapping.Keys); + } + } + else + { + // It should be found in env_properties + Assert.Equal(item.Value, envPropertiesMapping[item.Key]); + Assert.DoesNotContain(item.Key, mapping); + } + } + + foreach (var item in resource.Attributes) + { + if (exporterOptions.CustomFields == null || exporterOptions.CustomFields.Contains(item.Key)) + { + // It should be found as a custom field if (item.Value != null) { Assert.Equal(item.Value, mapping[item.Key]); } + + if (envPropertiesMapping != null) + { + Assert.DoesNotContain(item.Key, envPropertiesMapping.Keys); + } } else { + // It should be found in env_properties Assert.Equal(item.Value, envPropertiesMapping[item.Key]); + Assert.DoesNotContain(item.Key, mapping); } } } @@ -1653,7 +1837,7 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter Assert.Equal(logRecord.EventId.Id, int.Parse(mapping["eventId"].ToString(), CultureInfo.InvariantCulture)); } - // Epilouge + // Epilogue Assert.Equal("DateTime", timeFormat["TimeFormat"]); } diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/LogSerializationTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/LogSerializationTests.cs index 0c9758db87..ca2ecd2868 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/LogSerializationTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/LogSerializationTests.cs @@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging; using OpenTelemetry.Exporter.Geneva.MsgPack; using OpenTelemetry.Logs; +using OpenTelemetry.Resources; using Xunit; namespace OpenTelemetry.Exporter.Geneva.Tests; @@ -110,7 +111,7 @@ private static Dictionary GetExportedFieldsAfterLogging(Action Resource.Empty); _ = exporter.SerializeLogRecord(logRecordList[0]); var fluentdData = MessagePack.MessagePackSerializer.Deserialize(MsgPackLogExporter.Buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Options); diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/MsgPackLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/MsgPackLogExporterTests.cs index e42e4427e3..3d2a63f95a 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/MsgPackLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/MsgPackLogExporterTests.cs @@ -4,6 +4,7 @@ using System.Net.Sockets; using System.Runtime.InteropServices; using OpenTelemetry.Exporter.Geneva.MsgPack; +using OpenTelemetry.Resources; using Xunit; namespace OpenTelemetry.Exporter.Geneva.Tests; @@ -55,7 +56,7 @@ public void StringSizeLimit_Default_Success() { ConnectionString = this.connectionString, }; - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); } [Fact] @@ -65,7 +66,7 @@ public void StringSizeLimit_Valid_Success() { ConnectionString = this.connectionString + ";PrivatePreviewLogMessagePackStringSizeLimit=65360", }; - using var exporter = new MsgPackLogExporter(exporterOptions); + using var exporter = new MsgPackLogExporter(exporterOptions, () => Resource.Empty); } private static string GenerateTempFilePath() From 522cf919c005b8525dd0e87017bf57a17b73c6f8 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Fri, 14 Nov 2025 12:37:24 -0800 Subject: [PATCH 2/3] fix indentation --- .../Exporter/LogExporterBenchmarks.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/LogExporterBenchmarks.cs b/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/LogExporterBenchmarks.cs index 477d1109d8..7c539c56ed 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/LogExporterBenchmarks.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Benchmarks/Exporter/LogExporterBenchmarks.cs @@ -77,16 +77,16 @@ public LogExporterBenchmarks() this.batch = GenerateTestLogRecordBatch(); this.exporter = new MsgPackLogExporter( new GenevaExporterOptions + { + ConnectionString = "EtwSession=OpenTelemetry", + PrepopulatedFields = new Dictionary { - ConnectionString = "EtwSession=OpenTelemetry", - PrepopulatedFields = new Dictionary - { - ["cloud.role"] = "BusyWorker", - ["cloud.roleInstance"] = "CY1SCH030021417", - ["cloud.roleVer"] = "9.0.15289.2", - }, + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", }, - () => Resource.Empty); + }, + () => Resource.Empty); } [Benchmark] From 85e9c5366253c7fba47bbcf48bef8c4320c6a0f6 Mon Sep 17 00:00:00 2001 From: Matthew Sainsbury Date: Fri, 14 Nov 2025 13:09:51 -0800 Subject: [PATCH 3/3] add tests for cloud extension --- .../GenevaLogExporterTests.cs | 42 ++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs index d367b98aa2..beffbbb9d3 100644 --- a/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs +++ b/test/OpenTelemetry.Exporter.Geneva.Tests/GenevaLogExporterTests.cs @@ -664,14 +664,16 @@ public void SerializationTestWithILoggerLogWithTemplates(bool hasTableNameMappin { PrepopulatedFields = new Dictionary { - ["cloud.role"] = "BusyWorker", - ["cloud.roleInstance"] = "CY1SCH030021417", ["cloud.roleVer"] = "9.0.15289.2", ["prepopulated"] = "prepopulated field", }, }; - var resource = ResourceBuilder.CreateEmpty().AddAttributes([new KeyValuePair("resourceAttr", "from resource")]).Build(); + var resource = ResourceBuilder.CreateEmpty().AddAttributes([ + new KeyValuePair("resourceAttr", "from resource"), + new KeyValuePair("service.name", "BusyWorker"), + new KeyValuePair("service.instanceId", "CY1SCH030021417")]) + .Build(); if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { @@ -858,7 +860,11 @@ public void SerializationTestWithDuplicateFields(bool conflictingPrepopulatedFie var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; var timeStamp = (DateTime)(TimeStampAndMappings as object[])[0]; var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; - var env_properties = mapping.GetValueOrDefault("env_properties") as Dictionary ?? []; + var env_properties = new Dictionary(); + if (mapping.ContainsKey("env_properties")) + { + env_properties = mapping["env_properties"] as Dictionary ?? []; + } void AssertField(bool isDedicated, string fieldValue) { @@ -1712,7 +1718,12 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter foreach (var item in exporterOptions.PrepopulatedFields) { var partAValue = item.Value as string; - var partAKey = MsgPackExporter.V40_PART_A_MAPPING.GetValueOrDefault(item.Key, item.Key); + var partAKey = item.Key; + if (MsgPackExporter.V40_PART_A_MAPPING.ContainsKey(item.Key)) + { + partAKey = MsgPackExporter.V40_PART_A_MAPPING[item.Key]; + } + Assert.Equal(partAValue, mapping[partAKey]); } @@ -1737,6 +1748,21 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter Assert.Equal(logRecord.Exception.Message, mapping["env_ex_msg"]); } + // Part A cloud extensions + var serviceNameField = resource.Attributes.FirstOrDefault(attr => attr.Key == "service.name"); + if (serviceNameField.Key == "service.name" && !exporterOptions.PrepopulatedFields.ContainsKey("cloud.role")) + { + Assert.Contains("env_cloud_role", mapping); + Assert.Equal(serviceNameField.Value, mapping["env_cloud_role"]); + } + + var serviceInstanceField = resource.Attributes.FirstOrDefault(attr => attr.Key == "service.instanceId"); + if (serviceInstanceField.Key == "service.instanceId" && !exporterOptions.PrepopulatedFields.ContainsKey("cloud.roleInstance")) + { + Assert.Contains("env_cloud_roleInstance", mapping); + Assert.Equal(serviceInstanceField.Value, mapping["env_cloud_roleInstance"]); + } + // Part B fields // `LogRecord.LogLevel` was marked Obsolete in https://github.com/open-telemetry/opentelemetry-dotnet/pull/4568 @@ -1810,6 +1836,12 @@ private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporter foreach (var item in resource.Attributes) { + if (item.Key == "service.name" || item.Key == "service.instanceId") + { + // these ones are already checked. + continue; + } + if (exporterOptions.CustomFields == null || exporterOptions.CustomFields.Contains(item.Key)) { // It should be found as a custom field