Tell collector to watch the .log.1 files to avoid missing records around rotation time.

Add comments around each components filelog receiver settings.

Author: dsessler7

internal/collector/patroni.go

@@ -39,7 +39,14 @@ func EnablePatroniLogging(ctx context.Context,
 		// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/receiver/filelogreceiver#readme
 		outConfig.Receivers["filelog/patroni_jsonlog"] = map[string]any{
 			// Read the JSON files and keep track of what has been processed.
-			"include": []string{directory + "/*.log"},
+			// When patroni rotates its log files, it renames the old .log file
+			// to .log.1. We want the collector to ingest logs from both files
+			// as it is possible that patroni will continue to write a log
+			// record or two to the old file while rotation is occurring. The
+			// collector knows not to create duplicate logs.
+			"include": []string{
+				directory + "/*.log", directory + "/*.log.1",
+			},
 			"storage": "file_storage/patroni_logs",
 
 			"operators": []map[string]any{

internal/collector/patroni_test.go

@@ -88,6 +88,7 @@ receivers:
   filelog/patroni_jsonlog:
     include:
     - /pgdata/patroni/log/*.log
+    - /pgdata/patroni/log/*.log.1
     operators:
     - from: body
       to: body.original
@@ -183,6 +184,7 @@ receivers:
   filelog/patroni_jsonlog:
     include:
     - /pgdata/patroni/log/*.log
+    - /pgdata/patroni/log/*.log.1
     operators:
     - from: body
       to: body.original

internal/collector/pgadmin.go

@@ -29,6 +29,11 @@ func EnablePgAdminLogging(ctx context.Context, spec *v1beta1.InstrumentationSpec
 		"fsync":            true,
 	}
 
+	// PgAdmin/gunicorn logs are rotated by python -- python tries to emit a log
+	// and if the file needs to rotate, it rotates first and then emits the log.
+	// The collector therefore only needs to watch the single active log for
+	// each component.
+	// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/receiver/filelogreceiver#readme
 	otelConfig.Receivers["filelog/pgadmin"] = map[string]any{
 		"include": []string{"/var/lib/pgadmin/logs/pgadmin.log"},
 		"storage": "file_storage/pgadmin_data_logs",

internal/collector/pgbackrest.go

@@ -55,8 +55,13 @@ func NewConfigForPgBackrestRepoHostPod(
 		// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/receiver/filelogreceiver#readme
 		config.Receivers["filelog/pgbackrest_log"] = map[string]any{
 			// Read the files and keep track of what has been processed.
+			// We use logrotate to rotate the pgbackrest logs which renames the
+			// old .log file to .log.1. We want the collector to ingest logs from
+			// both files as it is possible that pgbackrest will continue to write
+			// a log record or two to the old file while rotation is occurring.
+			// The collector knows not to create duplicate logs.
 			"include": []string{
-				directory + "/*.log",
+				directory + "/*.log", directory + "/*.log.1",
 			},
 			"storage": "file_storage/pgbackrest_logs",
 			// pgBackRest prints logs with a log prefix, which includes a timestamp

internal/collector/pgbackrest_test.go

@@ -95,6 +95,7 @@ receivers:
   filelog/pgbackrest_log:
     include:
     - /pgbackrest/repo1/log/*.log
+    - /pgbackrest/repo1/log/*.log.1
     multiline:
       line_start_pattern: ^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}|^-{19}
     storage: file_storage/pgbackrest_logs
@@ -195,6 +196,7 @@ receivers:
   filelog/pgbackrest_log:
     include:
     - /pgbackrest/repo1/log/*.log
+    - /pgbackrest/repo1/log/*.log.1
     multiline:
       line_start_pattern: ^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}|^-{19}
     storage: file_storage/pgbackrest_logs

internal/collector/postgres.go

@@ -142,6 +142,7 @@ func EnablePostgresLogging(
 		// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/receiver/filelogreceiver#readme
 		outConfig.Receivers["filelog/postgres_csvlog"] = map[string]any{
 			// Read the CSV files and keep track of what has been processed.
+			// The wildcard covers all potential log file names.
 			"include": []string{directory + "/*.csv"},
 			"storage": "file_storage/postgres_logs",
 
@@ -173,6 +174,7 @@ func EnablePostgresLogging(
 		// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/receiver/filelogreceiver#readme
 		outConfig.Receivers["filelog/postgres_jsonlog"] = map[string]any{
 			// Read the JSON files and keep track of what has been processed.
+			// The wildcard covers all potential log file names.
 			"include": []string{directory + "/*.json"},
 			"storage": "file_storage/postgres_logs",
 
@@ -238,8 +240,17 @@ func EnablePostgresLogging(
 			"fsync":            true,
 		}
 
+		// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/receiver/filelogreceiver#readme
 		outConfig.Receivers["filelog/pgbackrest_log"] = map[string]any{
-			"include": []string{naming.PGBackRestPGDataLogPath + "/*.log"},
+			// We use logrotate to rotate the pgbackrest logs which renames the
+			// old .log file to .log.1. We want the collector to ingest logs from
+			// both files as it is possible that pgbackrest will continue to write
+			// a log record or two to the old file while rotation is occurring.
+			// The collector knows not to create duplicate logs.
+			"include": []string{
+				naming.PGBackRestPGDataLogPath + "/*.log",
+				naming.PGBackRestPGDataLogPath + "/*.log.1",
+			},
 			"storage": "file_storage/pgbackrest_logs",
 
 			// pgBackRest prints logs with a log prefix, which includes a timestamp

internal/collector/postgres_test.go

@@ -197,6 +197,7 @@ receivers:
   filelog/pgbackrest_log:
     include:
     - /pgdata/pgbackrest/log/*.log
+    - /pgdata/pgbackrest/log/*.log.1
     multiline:
       line_start_pattern: ^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}|^-{19}
     storage: file_storage/pgbackrest_logs
@@ -438,6 +439,7 @@ receivers:
   filelog/pgbackrest_log:
     include:
     - /pgdata/pgbackrest/log/*.log
+    - /pgdata/pgbackrest/log/*.log.1
     multiline:
       line_start_pattern: ^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}|^-{19}
     storage: file_storage/pgbackrest_logs