Skip to content

Otel custom queries #4126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 11, 2025
Merged

Otel custom queries #4126

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 69 additions & 0 deletions config/crd/bases/postgres-operator.crunchydata.com_pgadmins.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2048,6 +2048,75 @@ spec:
- message: must be at least one hour
rule: duration("1h") <= self && self <= duration("8760h")
type: object
metrics:
description: Metrics is the place for users to configure metrics
collection.
properties:
customQueries:
description: |-
Where users can turn off built-in metrics and also provide their own
custom queries.
properties:
add:
description: User defined queries and metrics.
items:
properties:
collectionInterval:
default: 5s
description: How often the queries should be run.
format: duration
maxLength: 20
minLength: 1
pattern: ^((PT)?( *[0-9]+ *(?i:(ms|s|m)|(milli|sec|min)s?))+|0)$
type: string
x-kubernetes-validations:
- rule: duration("0") <= self && self <= duration("60m")
name:
description: |-
The name of this batch of queries, which will be used in naming the OTel
SqlQuery receiver.
maxLength: 20
pattern: ^[^\pZ\pC\pS]+$
type: string
queries:
description: A ConfigMap holding the yaml file that
contains the queries.
properties:
key:
description: Name of the data field within the
ConfigMap.
maxLength: 253
minLength: 1
pattern: ^[-._a-zA-Z0-9]+$
type: string
x-kubernetes-validations:
- message: cannot be "." or start with ".."
rule: self != "." && !self.startsWith("..")
name:
description: Name of the ConfigMap.
maxLength: 253
minLength: 1
pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?([.][a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
type: string
required:
- key
- name
type: object
x-kubernetes-map-type: atomic
required:
- name
- queries
type: object
type: array
remove:
description: |-
A list of built-in queries that should be removed. If all queries for a
given SQL statement are removed, the SQL statement will no longer be run.
items:
type: string
type: array
type: object
type: object
resources:
description: Resources holds the resource requirements for the
collector container.
Expand Down
69 changes: 69 additions & 0 deletions config/crd/bases/postgres-operator.crunchydata.com_postgresclusters.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11695,6 +11695,75 @@ spec:
- message: must be at least one hour
rule: duration("1h") <= self && self <= duration("8760h")
type: object
metrics:
description: Metrics is the place for users to configure metrics
collection.
properties:
customQueries:
description: |-
Where users can turn off built-in metrics and also provide their own
custom queries.
properties:
add:
description: User defined queries and metrics.
items:
properties:
collectionInterval:
default: 5s
description: How often the queries should be run.
format: duration
maxLength: 20
minLength: 1
pattern: ^((PT)?( *[0-9]+ *(?i:(ms|s|m)|(milli|sec|min)s?))+|0)$
type: string
x-kubernetes-validations:
- rule: duration("0") <= self && self <= duration("60m")
name:
description: |-
The name of this batch of queries, which will be used in naming the OTel
SqlQuery receiver.
maxLength: 20
pattern: ^[^\pZ\pC\pS]+$
type: string
queries:
description: A ConfigMap holding the yaml file that
contains the queries.
properties:
key:
description: Name of the data field within the
ConfigMap.
maxLength: 253
minLength: 1
pattern: ^[-._a-zA-Z0-9]+$
type: string
x-kubernetes-validations:
- message: cannot be "." or start with ".."
rule: self != "." && !self.startsWith("..")
name:
description: Name of the ConfigMap.
maxLength: 253
minLength: 1
pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?([.][a-z0-9]([-a-z0-9]*[a-z0-9])?)*$
type: string
required:
- key
- name
type: object
x-kubernetes-map-type: atomic
required:
- name
- queries
type: object
type: array
remove:
description: |-
A list of built-in queries that should be removed. If all queries for a
given SQL statement are removed, the SQL statement will no longer be run.
items:
type: string
type: array
type: object
type: object
resources:
description: Resources holds the resource requirements for the
collector container.
Expand Down
56 changes: 39 additions & 17 deletions internal/collector/instance.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,12 @@ func AddToPod(
spec *v1beta1.InstrumentationSpec,
pullPolicy corev1.PullPolicy,
inInstanceConfigMap *corev1.ConfigMap,
outPod *corev1.PodSpec,
template *corev1.PodTemplateSpec,
volumeMounts []corev1.VolumeMount,
sqlQueryPassword string,
logDirectories []string,
includeLogrotate bool,
thisPodServesMetrics bool,
) {
if spec == nil ||
!(feature.Enabled(ctx, feature.OpenTelemetryLogs) ||
Expand Down Expand Up @@ -76,14 +77,13 @@ func AddToPod(
}},
}

// If the user has specified files to be mounted in the spec, add them to the projected config volume
if spec != nil && spec.Config != nil && spec.Config.Files != nil {
configVolume.Projected.Sources = append(configVolume.Projected.Sources, spec.Config.Files...)
// If the user has specified files to be mounted in the spec, add them to
// the projected config volume
if spec.Config != nil && spec.Config.Files != nil {
configVolume.Projected.Sources = append(configVolume.Projected.Sources,
spec.Config.Files...)
}

// Add configVolume to the pod's volumes
outPod.Volumes = append(outPod.Volumes, configVolume)

// Create collector container
container := corev1.Container{
Name: naming.ContainerCollector,
Expand Down Expand Up @@ -113,6 +113,28 @@ func AddToPod(
VolumeMounts: append(volumeMounts, configVolumeMount),
}

// If metrics feature is enabled and this Pod serves metrics, add the
// Prometheus port to this container
if feature.Enabled(ctx, feature.OpenTelemetryMetrics) && thisPodServesMetrics {
container.Ports = []corev1.ContainerPort{{
ContainerPort: int32(PrometheusPort),
Name: "otel-metrics",
Protocol: corev1.ProtocolTCP,
}}

// If the user has specified custom queries to add, put the queries
// file(s) in the projected config volume
if spec.Metrics != nil && spec.Metrics.CustomQueries != nil &&
spec.Metrics.CustomQueries.Add != nil {
for _, querySet := range spec.Metrics.CustomQueries.Add {
projection := querySet.Queries.AsProjection(querySet.Name +
"/" + querySet.Queries.Key)
configVolume.Projected.Sources = append(configVolume.Projected.Sources,
corev1.VolumeProjection{ConfigMap: &projection})
}
}
}

// If this is a pod that uses logrotate for log rotation, add config volume
// and mount for logrotate config
if includeLogrotate {
Expand All @@ -136,18 +158,17 @@ func AddToPod(
}},
}
container.VolumeMounts = append(container.VolumeMounts, logrotateConfigVolumeMount)
outPod.Volumes = append(outPod.Volumes, logrotateConfigVolume)
template.Spec.Volumes = append(template.Spec.Volumes, logrotateConfigVolume)
}

if feature.Enabled(ctx, feature.OpenTelemetryMetrics) {
container.Ports = []corev1.ContainerPort{{
ContainerPort: int32(8889),
Name: "otel-metrics",
Protocol: corev1.ProtocolTCP,
}}
}
// Add configVolume to the Pod's volumes and add the collector container to
// the Pod's containers
template.Spec.Volumes = append(template.Spec.Volumes, configVolume)
template.Spec.Containers = append(template.Spec.Containers, container)

outPod.Containers = append(outPod.Containers, container)
// add the OTel collector label to the Pod
initialize.Labels(template)
template.Labels[naming.LabelCollectorDiscovery] = "true"
}

// startCommand generates the command script used by the collector container
Expand Down Expand Up @@ -192,7 +213,8 @@ while read -r -t 5 -u "${fd}" ||:; do
done
`, mkdirScript, configDirectory, logrotateCommand)

wrapper := `monitor() {` + startScript + `}; export directory="$1"; export -f monitor; exec -a "$0" bash -ceu monitor`
wrapper := `monitor() {` + startScript +
`}; export directory="$1"; export -f monitor; exec -a "$0" bash -ceu monitor`

return []string{"bash", "-ceu", "--", wrapper, "collector", configDirectory}
}
1 change: 1 addition & 0 deletions internal/collector/naming.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ const LogsBatchProcessor = "batch/logs"
const OneSecondBatchProcessor = "batch/1s"
const SubSecondBatchProcessor = "batch/200ms"
const Prometheus = "prometheus"
const PrometheusPort = 9187
const PGBouncerMetrics = "metrics/pgbouncer"
const PostgresMetrics = "metrics/postgres"
const PatroniMetrics = "metrics/patroni"
Expand Down
3 changes: 2 additions & 1 deletion internal/collector/patroni.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ package collector
import (
"context"
"slices"
"strconv"

"github.com/crunchydata/postgres-operator/internal/feature"
"github.com/crunchydata/postgres-operator/internal/naming"
Expand Down Expand Up @@ -136,7 +137,7 @@ func EnablePatroniMetrics(ctx context.Context,
if feature.Enabled(ctx, feature.OpenTelemetryMetrics) {
// Add Prometheus exporter
outConfig.Exporters[Prometheus] = map[string]any{
"endpoint": "0.0.0.0:9187",
"endpoint": "0.0.0.0:" + strconv.Itoa(PrometheusPort),
}

// Add Prometheus Receiver
Expand Down
6 changes: 4 additions & 2 deletions internal/collector/pgbouncer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"encoding/json"
"fmt"
"slices"
"strconv"

"github.com/crunchydata/postgres-operator/internal/feature"
"github.com/crunchydata/postgres-operator/internal/naming"
Expand Down Expand Up @@ -174,13 +175,14 @@ func EnablePgBouncerMetrics(ctx context.Context, config *Config, sqlQueryUsernam
if feature.Enabled(ctx, feature.OpenTelemetryMetrics) {
// Add Prometheus exporter
config.Exporters[Prometheus] = map[string]any{
"endpoint": "0.0.0.0:9187",
"endpoint": "0.0.0.0:" + strconv.Itoa(PrometheusPort),
}

// Add SqlQuery Receiver
config.Receivers[SqlQuery] = map[string]any{
"driver": "postgres",
"datasource": fmt.Sprintf(`host=localhost dbname=pgbouncer port=5432 user=%s password=${env:PGPASSWORD}`,
"datasource": fmt.Sprintf(
`host=localhost dbname=pgbouncer port=5432 user=%s password=${env:PGPASSWORD}`,
sqlQueryUsername),
"queries": slices.Clone(pgBouncerMetricsQueries),
}
Expand Down
Loading