Skip to content

Commit 8330267

Browse files
philrhurstphilrhurst
authored
Merge branch 'CrunchyData:main' into pgadmin-oauth-secrets
2 parents e5a6a74 + d7f4913 commit 8330267

38 files changed

+1054
-262
lines changed

‎config/crd/bases/postgres-operator.crunchydata.com_pgadmins.yaml‎

Lines changed: 77 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1629,6 +1629,38 @@ spec:
16291629
description: Config is the place for users to configure exporters
16301630
and provide files.
16311631
properties:
1632+
detectors:
1633+
description: |-
1634+
Resource detectors add identifying attributes to logs and metrics. These run in the order they are defined.
1635+
More info: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/processor/resourcedetectionprocessor#readme
1636+
items:
1637+
properties:
1638+
attributes:
1639+
additionalProperties:
1640+
type: boolean
1641+
description: |-
1642+
Attributes to use from this detector. Detectors usually add every attribute
1643+
they know automatically. Names omitted here behave according to detector defaults.
1644+
maxProperties: 30
1645+
minProperties: 1
1646+
type: object
1647+
x-kubernetes-map-type: atomic
1648+
name:
1649+
description: 'Name of the resource detector to enable:
1650+
`aks`, `eks`, `gcp`, etc.'
1651+
maxLength: 20
1652+
minLength: 1
1653+
type: string
1654+
required:
1655+
- name
1656+
type: object
1657+
x-kubernetes-map-type: atomic
1658+
maxItems: 10
1659+
minItems: 1
1660+
type: array
1661+
x-kubernetes-list-map-keys:
1662+
- name
1663+
x-kubernetes-list-type: map
16321664
exporters:
16331665
description: |-
16341666
Exporters allows users to configure OpenTelemetry exporters that exist
@@ -1961,7 +1993,9 @@ spec:
19611993
- path
19621994
type: object
19631995
type: object
1996+
minItems: 1
19641997
type: array
1998+
x-kubernetes-list-type: atomic
19651999
type: object
19662000
image:
19672001
description: |-
@@ -1972,13 +2006,53 @@ spec:
19722006
description: Logs is the place for users to configure the log
19732007
collection.
19742008
properties:
2009+
batches:
2010+
description: Log records are exported in small batches. Set
2011+
this field to change their size and frequency.
2012+
properties:
2013+
maxDelay:
2014+
default: 200ms
2015+
description: |-
2016+
Maximum time to wait before exporting a log record. Higher numbers
2017+
allow more records to be deduplicated and compressed before export.
2018+
format: duration
2019+
maxLength: 20
2020+
minLength: 1
2021+
pattern: ^((PT)?( *[0-9]+ *(?i:(ms|s|m)|(milli|sec|min)s?))+|0)$
2022+
type: string
2023+
x-kubernetes-validations:
2024+
- rule: duration("0") <= self && self <= duration("5m")
2025+
maxRecords:
2026+
description: |-
2027+
Maximum number of records to include in an exported batch. When present,
2028+
batches this size are sent without any further delay.
2029+
format: int32
2030+
minimum: 1
2031+
type: integer
2032+
minRecords:
2033+
default: 8192
2034+
description: |-
2035+
Number of records to wait for before exporting a batch. Higher numbers
2036+
allow more records to be deduplicated and compressed before export.
2037+
format: int32
2038+
minimum: 0
2039+
type: integer
2040+
type: object
2041+
x-kubernetes-map-type: atomic
2042+
x-kubernetes-validations:
2043+
- message: to disable batching, both minRecords and maxDelay
2044+
must be zero
2045+
rule: (has(self.minRecords) && self.minRecords == 0) ==
2046+
(has(self.maxDelay) && self.maxDelay == duration('0'))
2047+
- message: minRecords cannot be larger than maxRecords
2048+
rule: '!has(self.maxRecords) || self.minRecords <= self.maxRecords'
19752049
exporters:
1976-
description: |-
1977-
Exporters allows users to specify which exporters they want to use in
1978-
the logs pipeline.
2050+
description: The names of exporters that should send logs.
19792051
items:
19802052
type: string
2053+
minItems: 1
19812054
type: array
2055+
x-kubernetes-list-type: set
19822056
retentionPeriod:
19832057
description: |-
19842058
How long to retain log files locally. An RFC 3339 duration or a number

‎config/crd/bases/postgres-operator.crunchydata.com_postgresclusters.yaml‎

Lines changed: 77 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11181,6 +11181,38 @@ spec:
1118111181
description: Config is the place for users to configure exporters
1118211182
and provide files.
1118311183
properties:
11184+
detectors:
11185+
description: |-
11186+
Resource detectors add identifying attributes to logs and metrics. These run in the order they are defined.
11187+
More info: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/processor/resourcedetectionprocessor#readme
11188+
items:
11189+
properties:
11190+
attributes:
11191+
additionalProperties:
11192+
type: boolean
11193+
description: |-
11194+
Attributes to use from this detector. Detectors usually add every attribute
11195+
they know automatically. Names omitted here behave according to detector defaults.
11196+
maxProperties: 30
11197+
minProperties: 1
11198+
type: object
11199+
x-kubernetes-map-type: atomic
11200+
name:
11201+
description: 'Name of the resource detector to enable:
11202+
`aks`, `eks`, `gcp`, etc.'
11203+
maxLength: 20
11204+
minLength: 1
11205+
type: string
11206+
required:
11207+
- name
11208+
type: object
11209+
x-kubernetes-map-type: atomic
11210+
maxItems: 10
11211+
minItems: 1
11212+
type: array
11213+
x-kubernetes-list-map-keys:
11214+
- name
11215+
x-kubernetes-list-type: map
1118411216
exporters:
1118511217
description: |-
1118611218
Exporters allows users to configure OpenTelemetry exporters that exist
@@ -11513,7 +11545,9 @@ spec:
1151311545
- path
1151411546
type: object
1151511547
type: object
11548+
minItems: 1
1151611549
type: array
11550+
x-kubernetes-list-type: atomic
1151711551
type: object
1151811552
image:
1151911553
description: |-
@@ -11524,13 +11558,53 @@ spec:
1152411558
description: Logs is the place for users to configure the log
1152511559
collection.
1152611560
properties:
11561+
batches:
11562+
description: Log records are exported in small batches. Set
11563+
this field to change their size and frequency.
11564+
properties:
11565+
maxDelay:
11566+
default: 200ms
11567+
description: |-
11568+
Maximum time to wait before exporting a log record. Higher numbers
11569+
allow more records to be deduplicated and compressed before export.
11570+
format: duration
11571+
maxLength: 20
11572+
minLength: 1
11573+
pattern: ^((PT)?( *[0-9]+ *(?i:(ms|s|m)|(milli|sec|min)s?))+|0)$
11574+
type: string
11575+
x-kubernetes-validations:
11576+
- rule: duration("0") <= self && self <= duration("5m")
11577+
maxRecords:
11578+
description: |-
11579+
Maximum number of records to include in an exported batch. When present,
11580+
batches this size are sent without any further delay.
11581+
format: int32
11582+
minimum: 1
11583+
type: integer
11584+
minRecords:
11585+
default: 8192
11586+
description: |-
11587+
Number of records to wait for before exporting a batch. Higher numbers
11588+
allow more records to be deduplicated and compressed before export.
11589+
format: int32
11590+
minimum: 0
11591+
type: integer
11592+
type: object
11593+
x-kubernetes-map-type: atomic
11594+
x-kubernetes-validations:
11595+
- message: to disable batching, both minRecords and maxDelay
11596+
must be zero
11597+
rule: (has(self.minRecords) && self.minRecords == 0) ==
11598+
(has(self.maxDelay) && self.maxDelay == duration('0'))
11599+
- message: minRecords cannot be larger than maxRecords
11600+
rule: '!has(self.maxRecords) || self.minRecords <= self.maxRecords'
1152711601
exporters:
11528-
description: |-
11529-
Exporters allows users to specify which exporters they want to use in
11530-
the logs pipeline.
11602+
description: The names of exporters that should send logs.
1153111603
items:
1153211604
type: string
11605+
minItems: 1
1153311606
type: array
11607+
x-kubernetes-list-type: set
1153411608
retentionPeriod:
1153511609
description: |-
1153611610
How long to retain log files locally. An RFC 3339 duration or a number

‎internal/collector/config.go‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,58 @@ func NewConfig(spec *v1beta1.InstrumentationSpec) *Config {
114114
Pipelines: map[PipelineID]Pipeline{},
115115
}
116116

117+
// Configure a batch processor for logs according to the API spec.
118+
// Use API defaults for any unspecified fields.
119+
{
120+
var batches v1beta1.OpenTelemetryLogsBatchSpec
121+
if spec != nil && spec.Logs != nil && spec.Logs.Batches != nil {
122+
spec.Logs.Batches.DeepCopyInto(&batches)
123+
}
124+
batches.Default()
125+
126+
// https://pkg.go.dev/go.opentelemetry.io/collector/processor/batchprocessor#section-readme
127+
processor := map[string]any{}
128+
if batches.MaxDelay != nil {
129+
processor["timeout"] = batches.MaxDelay.AsDuration().Duration.String()
130+
}
131+
if batches.MaxRecords != nil {
132+
processor["send_batch_max_size"] = *batches.MaxRecords
133+
}
134+
if batches.MinRecords != nil {
135+
processor["send_batch_size"] = *batches.MinRecords
136+
}
137+
config.Processors[LogsBatchProcessor] = processor
138+
}
139+
140+
// Create a resource detection processor according to the API spec.
141+
// When nothing is specified, the processor does nothing.
142+
{
143+
// https://pkg.go.dev/github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor#section-readme
144+
processor := map[string]any{"override": false, "timeout": "30s"}
145+
146+
if spec != nil && spec.Config != nil {
147+
names := make([]string, len(spec.Config.Detectors))
148+
for i, detector := range spec.Config.Detectors {
149+
names[i] = detector.Name
150+
151+
if len(detector.Attributes) > 0 {
152+
attributes := make(map[string]any, len(detector.Attributes))
153+
for k, v := range detector.Attributes {
154+
attributes[k] = map[string]any{"enabled": v}
155+
}
156+
processor[detector.Name] = map[string]any{
157+
"resource_attributes": attributes,
158+
}
159+
}
160+
}
161+
processor["detectors"] = names
162+
} else {
163+
processor["detectors"] = []string{}
164+
}
165+
166+
config.Processors[ResourceDetectionProcessor] = processor
167+
}
168+
117169
// If there are exporters defined in the spec, add them to the config.
118170
if spec != nil && spec.Config != nil && spec.Config.Exporters != nil {
119171
for k, v := range spec.Config.Exporters {

‎internal/collector/config_test.go‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import (
99

1010
"gotest.tools/v3/assert"
1111

12+
"github.com/crunchydata/postgres-operator/internal/testing/cmp"
13+
"github.com/crunchydata/postgres-operator/internal/testing/require"
1214
"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
1315
)
1416

@@ -27,7 +29,14 @@ processors:
2729
timeout: 1s
2830
batch/200ms:
2931
timeout: 200ms
32+
batch/logs:
33+
send_batch_size: 8192
34+
timeout: 200ms
3035
groupbyattrs/compact: {}
36+
resourcedetection:
37+
detectors: []
38+
override: false
39+
timeout: 30s
3140
receivers: {}
3241
service:
3342
extensions: []
@@ -55,13 +64,86 @@ processors:
5564
timeout: 1s
5665
batch/200ms:
5766
timeout: 200ms
67+
batch/logs:
68+
send_batch_size: 8192
69+
timeout: 200ms
5870
groupbyattrs/compact: {}
71+
resourcedetection:
72+
detectors: []
73+
override: false
74+
timeout: 30s
5975
receivers: {}
6076
service:
6177
extensions: []
6278
pipelines: {}
6379
`)
6480
})
81+
82+
t.Run("LogsBatches", func(t *testing.T) {
83+
var spec *v1beta1.InstrumentationSpec
84+
require.UnmarshalInto(t, &spec, `{
85+
logs: {
86+
batches: {
87+
maxDelay: 5min 12sec,
88+
maxRecords: 123,
89+
minRecords: 45,
90+
},
91+
},
92+
}`)
93+
94+
result, err := NewConfig(spec).ToYAML()
95+
assert.NilError(t, err)
96+
assert.Assert(t, cmp.Contains(result, `
97+
batch/logs:
98+
send_batch_max_size: 123
99+
send_batch_size: 45
100+
timeout: 5m12s
101+
`))
102+
103+
t.Run("Disable", func(t *testing.T) {
104+
var spec *v1beta1.InstrumentationSpec
105+
require.UnmarshalInto(t, &spec, `{
106+
logs: {
107+
batches: { minRecords: 0, maxDelay: "0" },
108+
},
109+
}`)
110+
111+
result, err := NewConfig(spec).ToYAML()
112+
assert.NilError(t, err)
113+
assert.Assert(t, cmp.Contains(result, `
114+
batch/logs:
115+
send_batch_size: 0
116+
timeout: 0s
117+
`))
118+
})
119+
})
120+
121+
t.Run("Detectors", func(t *testing.T) {
122+
var spec *v1beta1.InstrumentationSpec
123+
require.UnmarshalInto(t, &spec, `{
124+
config: {
125+
detectors: [
126+
{ name: gcp },
127+
{ name: aks, attributes: { k8s.cluster.name: true } },
128+
],
129+
},
130+
}`)
131+
132+
result, err := NewConfig(spec).ToYAML()
133+
assert.NilError(t, err)
134+
assert.Assert(t, cmp.Contains(result, `
135+
resourcedetection:
136+
aks:
137+
resource_attributes:
138+
k8s.cluster.name:
139+
enabled: true
140+
detectors:
141+
- gcp
142+
- aks
143+
override: false
144+
timeout: 30s
145+
`))
146+
})
65147
}
66148

67149
func TestGenerateLogrotateConfig(t *testing.T) {

‎internal/collector/naming.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,14 @@ package collector
66

77
const CompactingProcessor = "groupbyattrs/compact"
88
const DebugExporter = "debug"
9+
const LogsBatchProcessor = "batch/logs"
910
const OneSecondBatchProcessor = "batch/1s"
1011
const SubSecondBatchProcessor = "batch/200ms"
1112
const Prometheus = "prometheus"
1213
const PGBouncerMetrics = "metrics/pgbouncer"
1314
const PostgresMetrics = "metrics/postgres"
1415
const PatroniMetrics = "metrics/patroni"
16+
const ResourceDetectionProcessor = "resourcedetection"
1517

1618
const SqlQuery = "sqlquery"
1719

0 commit comments

Comments
 (0)