Skip to content

Commit 08ab9a4

Browse files
dsessler7cbandy
dsessler7
authored and
cbandy
committed
Parse Patroni logs
Issue: PGO-2059
1 parent 9fcef77 commit 08ab9a4

File tree

5 files changed

+179
-0
lines changed

5 files changed

+179
-0
lines changed

‎internal/collector/patroni.go‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,96 @@ import (
88
"context"
99

1010
"github.com/crunchydata/postgres-operator/internal/feature"
11+
"github.com/crunchydata/postgres-operator/internal/naming"
1112
"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
1213
)
1314

15+
func EnablePatroniLogging(ctx context.Context,
16+
inCluster *v1beta1.PostgresCluster,
17+
outConfig *Config,
18+
) {
19+
if feature.Enabled(ctx, feature.OpenTelemetryLogs) {
20+
directory := naming.PatroniPGDataLogPath
21+
22+
// Keep track of what log records and files have been processed.
23+
// Use a subdirectory of the logs directory to stay within the same failure domain.
24+
//
25+
// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/extension/storage/filestorage#readme
26+
outConfig.Extensions["file_storage/patroni_logs"] = map[string]any{
27+
"directory": directory + "/receiver",
28+
"create_directory": true,
29+
"fsync": true,
30+
}
31+
32+
// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/receiver/filelogreceiver#readme
33+
outConfig.Receivers["filelog/patroni_jsonlog"] = map[string]any{
34+
// Read the JSON files and keep track of what has been processed.
35+
"include": []string{directory + "/*.log"},
36+
"storage": "file_storage/patroni_logs",
37+
38+
"operators": []map[string]any{
39+
{"type": "move", "from": "body", "to": "body.original"},
40+
},
41+
}
42+
43+
// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/processor/transformprocessor#readme
44+
outConfig.Processors["transform/patroni_logs"] = map[string]any{
45+
"log_statements": []map[string]any{{
46+
"context": "log",
47+
"statements": []string{
48+
`set(instrumentation_scope.name, "patroni")`,
49+
50+
// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/ottlfuncs#parsejson
51+
`set(cache, ParseJSON(body["original"]))`,
52+
53+
// The log severity is in the "levelname" field.
54+
// https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitytext
55+
`set(severity_text, cache["levelname"])`,
56+
57+
// Map Patroni (python) "logging levels" to OpenTelemetry severity levels.
58+
//
59+
// https://docs.python.org/3.6/library/logging.html#logging-levels
60+
// https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber
61+
// https://github.com/open-telemetry/opentelemetry-python/blob/v1.29.0/opentelemetry-api/src/opentelemetry/_logs/severity/__init__.py
62+
// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/contexts/ottllog#enums
63+
`set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG"`,
64+
`set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO"`,
65+
`set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING"`,
66+
`set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR"`,
67+
`set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "CRITICAL"`,
68+
69+
// Parse the "asctime" field into the record timestamp.
70+
// The format is neither RFC 3339 nor ISO 8601:
71+
//
72+
// The date and time are separated by a single space U+0020,
73+
// followed by a comma U+002C, then milliseconds.
74+
//
75+
// https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/stanza/docs/types/timestamp.md
76+
// https://docs.python.org/3.6/library/logging.html#logging.LogRecord
77+
`set(time, Time(cache["asctime"], "%F %T,%L"))`,
78+
79+
// Keep the unparsed log record in a standard attribute, and replace
80+
// the log record body with the message field.
81+
//
82+
// https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/general/logs.md
83+
`set(attributes["log.record.original"], body["original"])`,
84+
`set(body, cache["message"])`,
85+
},
86+
}},
87+
}
88+
89+
outConfig.Pipelines["logs/patroni"] = Pipeline{
90+
Extensions: []ComponentID{"file_storage/patroni_logs"},
91+
Receivers: []ComponentID{"filelog/patroni_jsonlog"},
92+
Processors: []ComponentID{
93+
"transform/patroni_logs",
94+
SubSecondBatchProcessor,
95+
},
96+
Exporters: []ComponentID{DebugExporter},
97+
}
98+
}
99+
}
100+
14101
func EnablePatroniMetrics(ctx context.Context,
15102
inCluster *v1beta1.PostgresCluster,
16103
outConfig *Config,

‎internal/collector/patroni_test.go‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
// Copyright 2024 - 2025 Crunchy Data Solutions, Inc.
2+
//
3+
// SPDX-License-Identifier: Apache-2.0
4+
5+
package collector
6+
7+
import (
8+
"context"
9+
"testing"
10+
11+
"gotest.tools/v3/assert"
12+
13+
"github.com/crunchydata/postgres-operator/internal/feature"
14+
"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
15+
)
16+
17+
func TestEnablePatroniLogging(t *testing.T) {
18+
t.Run("Enabled", func(t *testing.T) {
19+
gate := feature.NewGate()
20+
assert.NilError(t, gate.SetFromMap(map[string]bool{
21+
feature.OpenTelemetryLogs: true,
22+
}))
23+
ctx := feature.NewContext(context.Background(), gate)
24+
25+
config := NewConfig()
26+
27+
EnablePatroniLogging(ctx, new(v1beta1.PostgresCluster), config)
28+
29+
result, err := config.ToYAML()
30+
assert.NilError(t, err)
31+
assert.DeepEqual(t, result, `# Generated by postgres-operator. DO NOT EDIT.
32+
# Your changes will not be saved.
33+
exporters:
34+
debug:
35+
verbosity: detailed
36+
extensions:
37+
file_storage/patroni_logs:
38+
create_directory: true
39+
directory: /pgdata/patroni/log/receiver
40+
fsync: true
41+
processors:
42+
batch/1s:
43+
timeout: 1s
44+
batch/200ms:
45+
timeout: 200ms
46+
groupbyattrs/compact: {}
47+
transform/patroni_logs:
48+
log_statements:
49+
- context: log
50+
statements:
51+
- set(instrumentation_scope.name, "patroni")
52+
- set(cache, ParseJSON(body["original"]))
53+
- set(severity_text, cache["levelname"])
54+
- set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG"
55+
- set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO"
56+
- set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING"
57+
- set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR"
58+
- set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "CRITICAL"
59+
- set(time, Time(cache["asctime"], "%F %T,%L"))
60+
- set(attributes["log.record.original"], body["original"])
61+
- set(body, cache["message"])
62+
receivers:
63+
filelog/patroni_jsonlog:
64+
include:
65+
- /pgdata/patroni/log/*.log
66+
operators:
67+
- from: body
68+
to: body.original
69+
type: move
70+
storage: file_storage/patroni_logs
71+
service:
72+
extensions:
73+
- file_storage/patroni_logs
74+
pipelines:
75+
logs/patroni:
76+
exporters:
77+
- debug
78+
processors:
79+
- transform/patroni_logs
80+
- batch/200ms
81+
receivers:
82+
- filelog/patroni_jsonlog
83+
`)
84+
})
85+
}

‎internal/collector/postgres.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ func NewConfigForPostgresPod(ctx context.Context,
2323
) *Config {
2424
config := NewConfig()
2525

26+
EnablePatroniLogging(ctx, inCluster, config)
2627
EnablePatroniMetrics(ctx, inCluster, config)
2728
EnablePostgresLogging(ctx, inCluster, config, outParameters)
2829

‎internal/patroni/config.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,11 @@ func clusterYAML(
168168
// defaults to "INFO"
169169
"level": cluster.Spec.Patroni.Logging.Level,
170170

171+
// Setting group read permissions so that the OTel filelog receiver can
172+
// read the log files.
173+
// NOTE: This log configuration setting is only available in Patroni v4
174+
"mode": "0660",
175+
171176
// There will only be two log files. Cannot set to 1 or the logs won't rotate.
172177
// - https://github.com/python/cpython/blob/3.11/Lib/logging/handlers.py#L134
173178
"file_num": 1,

‎internal/patroni/config_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,7 @@ log:
191191
file_num: 1
192192
file_size: 500
193193
level: DEBUG
194+
mode: "0660"
194195
type: json
195196
postgresql:
196197
authentication:

0 commit comments

Comments
 (0)