From cbaf8693cf75720806e213c149d4c6eea70327bc Mon Sep 17 00:00:00 2001 From: Ben Blattberg Date: Fri, 7 Mar 2025 13:37:40 -0600 Subject: [PATCH 1/8] collector util func Check feature gates and check spec --- internal/collector/instance.go | 4 +- internal/collector/util.go | 56 +++++++++++++++++++ .../standalone_pgadmin/configmap.go | 3 +- .../standalone_pgadmin/statefulset.go | 3 +- internal/pgmonitor/postgres.go | 14 +++-- 5 files changed, 67 insertions(+), 13 deletions(-) create mode 100644 internal/collector/util.go diff --git a/internal/collector/instance.go b/internal/collector/instance.go index 970f9c9109..5ad515a2a5 100644 --- a/internal/collector/instance.go +++ b/internal/collector/instance.go @@ -49,9 +49,7 @@ func AddToPod( logDirectories []string, includeLogrotate bool, ) { - if spec == nil || - !(feature.Enabled(ctx, feature.OpenTelemetryLogs) || - feature.Enabled(ctx, feature.OpenTelemetryMetrics)) { + if !OpenTelemetryLogsOrMetricsEnabled(ctx, spec) { return } diff --git a/internal/collector/util.go b/internal/collector/util.go new file mode 100644 index 0000000000..136ae3fb44 --- /dev/null +++ b/internal/collector/util.go @@ -0,0 +1,56 @@ +// Copyright 2025 Crunchy Data Solutions, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +package collector + +import ( + "context" + + "github.com/crunchydata/postgres-operator/internal/feature" + "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" +) + +type CrunchyCRD interface { + *v1beta1.PostgresCluster | *v1beta1.PGAdmin | *v1beta1.InstrumentationSpec +} + +func OpenTelemetrySpecPresent[T CrunchyCRD](object T) bool { + + switch v := any(object).(type) { + case *v1beta1.InstrumentationSpec: + return true + case *v1beta1.PostgresCluster: + return v.Spec.Instrumentation != nil + case *v1beta1.PGAdmin: + return v.Spec.Instrumentation != nil + default: + return false + } + +} + +func OpenTelemetryLogsOrMetricsEnabled[T CrunchyCRD]( + ctx context.Context, + object T, +) bool { + return OpenTelemetrySpecPresent(object) && + (feature.Enabled(ctx, feature.OpenTelemetryLogs) || + feature.Enabled(ctx, feature.OpenTelemetryMetrics)) +} + +func OpenTelemetryLogsEnabled[T CrunchyCRD]( + ctx context.Context, + object T, +) bool { + return OpenTelemetrySpecPresent(object) && + feature.Enabled(ctx, feature.OpenTelemetryLogs) +} + +func OpenTelemetryMetricsEnabled[T CrunchyCRD]( + ctx context.Context, + object T, +) bool { + return OpenTelemetrySpecPresent(object) && + feature.Enabled(ctx, feature.OpenTelemetryMetrics) +} diff --git a/internal/controller/standalone_pgadmin/configmap.go b/internal/controller/standalone_pgadmin/configmap.go index 72a95b14db..5078e0e9fa 100644 --- a/internal/controller/standalone_pgadmin/configmap.go +++ b/internal/controller/standalone_pgadmin/configmap.go @@ -19,7 +19,6 @@ import ( "github.com/pkg/errors" "github.com/crunchydata/postgres-operator/internal/collector" - "github.com/crunchydata/postgres-operator/internal/feature" "github.com/crunchydata/postgres-operator/internal/initialize" "github.com/crunchydata/postgres-operator/internal/naming" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" @@ -73,7 +72,7 @@ func configmap(ctx context.Context, pgadmin *v1beta1.PGAdmin, gunicornRetentionPeriod = "D" ) // If OTel logs feature gate is enabled, we want to change the pgAdmin/gunicorn logging - if feature.Enabled(ctx, feature.OpenTelemetryLogs) && pgadmin.Spec.Instrumentation != nil { + if collector.OpenTelemetryLogsEnabled(ctx, pgadmin) { logRetention = true // If the user has set a retention period, we will use those values for log rotation, diff --git a/internal/controller/standalone_pgadmin/statefulset.go b/internal/controller/standalone_pgadmin/statefulset.go index c75668defc..a2163118da 100644 --- a/internal/controller/standalone_pgadmin/statefulset.go +++ b/internal/controller/standalone_pgadmin/statefulset.go @@ -17,7 +17,6 @@ import ( "github.com/crunchydata/postgres-operator/internal/collector" "github.com/crunchydata/postgres-operator/internal/controller/postgrescluster" - "github.com/crunchydata/postgres-operator/internal/feature" "github.com/crunchydata/postgres-operator/internal/initialize" "github.com/crunchydata/postgres-operator/internal/naming" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" @@ -122,7 +121,7 @@ func statefulset( pod(pgadmin, configmap, &sts.Spec.Template.Spec, dataVolume) - if pgadmin.Spec.Instrumentation != nil && feature.Enabled(ctx, feature.OpenTelemetryLogs) { + if collector.OpenTelemetryLogsEnabled(ctx, pgadmin) { // Logs for gunicorn and pgadmin write to /var/lib/pgadmin/logs // so the collector needs access to that that path. dataVolumeMount := corev1.VolumeMount{ diff --git a/internal/pgmonitor/postgres.go b/internal/pgmonitor/postgres.go index 1d7817c9a3..4dd6ab8fb2 100644 --- a/internal/pgmonitor/postgres.go +++ b/internal/pgmonitor/postgres.go @@ -8,12 +8,11 @@ import ( "context" "strings" - corev1 "k8s.io/api/core/v1" - - "github.com/crunchydata/postgres-operator/internal/feature" + "github.com/crunchydata/postgres-operator/internal/collector" "github.com/crunchydata/postgres-operator/internal/logging" "github.com/crunchydata/postgres-operator/internal/postgres" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" + corev1 "k8s.io/api/core/v1" ) const ( @@ -24,7 +23,8 @@ const ( // PostgreSQLHBAs provides the Postgres HBA rules for allowing the monitoring // exporter to be accessible func PostgreSQLHBAs(ctx context.Context, inCluster *v1beta1.PostgresCluster, outHBAs *postgres.HBAs) { - if ExporterEnabled(ctx, inCluster) || feature.Enabled(ctx, feature.OpenTelemetryMetrics) { + if ExporterEnabled(ctx, inCluster) || + collector.OpenTelemetryMetricsEnabled(ctx, inCluster) { // Limit the monitoring user to local connections using SCRAM. outHBAs.Mandatory = append(outHBAs.Mandatory, postgres.NewHBA().TCP().Users(MonitoringUser).Method("scram-sha-256").Network("127.0.0.0/8"), @@ -34,9 +34,11 @@ func PostgreSQLHBAs(ctx context.Context, inCluster *v1beta1.PostgresCluster, out } // PostgreSQLParameters provides additional required configuration parameters -// that Postgres needs to support monitoring +// that Postgres needs to support monitoring for both pgMonitor and OTel func PostgreSQLParameters(ctx context.Context, inCluster *v1beta1.PostgresCluster, outParameters *postgres.Parameters) { - if ExporterEnabled(ctx, inCluster) || feature.Enabled(ctx, feature.OpenTelemetryMetrics) { + if ExporterEnabled(ctx, inCluster) || + collector.OpenTelemetryMetricsEnabled(ctx, inCluster) { + // Exporter expects that shared_preload_libraries are installed // pg_stat_statements: https://access.crunchydata.com/documentation/pgmonitor/latest/exporter/ // pgnodemx: https://github.com/CrunchyData/pgnodemx From 9b551e8dccaea7bc505a0c018201677fb30d99e7 Mon Sep 17 00:00:00 2001 From: Ben Blattberg Date: Fri, 7 Mar 2025 14:18:40 -0600 Subject: [PATCH 2/8] update the others --- internal/collector/naming.go | 1 + internal/collector/patroni.go | 5 +- internal/collector/pgadmin.go | 208 +++++++++--------- internal/collector/pgbackrest.go | 3 +- internal/collector/pgbouncer.go | 11 +- internal/collector/postgres.go | 3 +- internal/collector/postgres_metrics.go | 9 +- internal/collector/util.go | 2 +- .../controller/postgrescluster/cluster.go | 4 +- .../controller/postgrescluster/instance.go | 6 +- .../controller/postgrescluster/pgbackrest.go | 2 +- .../controller/postgrescluster/pgbouncer.go | 7 +- .../controller/postgrescluster/pgmonitor.go | 17 +- internal/pgbackrest/config.go | 8 +- internal/pgbouncer/config.go | 6 +- internal/pgbouncer/reconcile.go | 2 +- 16 files changed, 145 insertions(+), 149 deletions(-) diff --git a/internal/collector/naming.go b/internal/collector/naming.go index 964d3d4d13..c293ccf7b1 100644 --- a/internal/collector/naming.go +++ b/internal/collector/naming.go @@ -14,6 +14,7 @@ const PGBouncerMetrics = "metrics/pgbouncer" const PostgresMetrics = "metrics/postgres" const PatroniMetrics = "metrics/patroni" const ResourceDetectionProcessor = "resourcedetection" +const MonitoringUser = "ccp_monitoring" const SqlQuery = "sqlquery" diff --git a/internal/collector/patroni.go b/internal/collector/patroni.go index 60305b458b..8b6ddeee5d 100644 --- a/internal/collector/patroni.go +++ b/internal/collector/patroni.go @@ -8,7 +8,6 @@ import ( "context" "slices" - "github.com/crunchydata/postgres-operator/internal/feature" "github.com/crunchydata/postgres-operator/internal/naming" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -22,7 +21,7 @@ func EnablePatroniLogging(ctx context.Context, spec = inCluster.Spec.Instrumentation.Logs } - if feature.Enabled(ctx, feature.OpenTelemetryLogs) { + if OpenTelemetryLogsEnabled(ctx, inCluster) { directory := naming.PatroniPGDataLogPath // Keep track of what log records and files have been processed. @@ -133,7 +132,7 @@ func EnablePatroniMetrics(ctx context.Context, inCluster *v1beta1.PostgresCluster, outConfig *Config, ) { - if feature.Enabled(ctx, feature.OpenTelemetryMetrics) { + if OpenTelemetryMetricsEnabled(ctx, inCluster) { // Add Prometheus exporter outConfig.Exporters[Prometheus] = map[string]any{ "endpoint": "0.0.0.0:9187", diff --git a/internal/collector/pgadmin.go b/internal/collector/pgadmin.go index e22ed621f0..ff1cca4eb6 100644 --- a/internal/collector/pgadmin.go +++ b/internal/collector/pgadmin.go @@ -10,7 +10,6 @@ import ( corev1 "k8s.io/api/core/v1" - "github.com/crunchydata/postgres-operator/internal/feature" "github.com/crunchydata/postgres-operator/internal/naming" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -18,112 +17,113 @@ import ( func EnablePgAdminLogging(ctx context.Context, spec *v1beta1.InstrumentationSpec, configmap *corev1.ConfigMap, ) error { - if !feature.Enabled(ctx, feature.OpenTelemetryLogs) { - return nil - } - otelConfig := NewConfig(spec) - - otelConfig.Extensions["file_storage/pgadmin_data_logs"] = map[string]any{ - "directory": "/var/lib/pgadmin/logs/receiver", - "create_directory": false, - "fsync": true, - } - - otelConfig.Receivers["filelog/pgadmin"] = map[string]any{ - "include": []string{"/var/lib/pgadmin/logs/pgadmin.log"}, - "storage": "file_storage/pgadmin_data_logs", - } - otelConfig.Receivers["filelog/gunicorn"] = map[string]any{ - "include": []string{"/var/lib/pgadmin/logs/gunicorn.log"}, - "storage": "file_storage/pgadmin_data_logs", - } - - otelConfig.Processors["resource/pgadmin"] = map[string]any{ - "attributes": []map[string]any{ - // Container and Namespace names need no escaping because they are DNS labels. - // Pod names need no escaping because they are DNS subdomains. - // - // https://kubernetes.io/docs/concepts/overview/working-with-objects/names - // https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/resource/k8s.md - // https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/general/logs.md - {"action": "insert", "key": "k8s.container.name", "value": naming.ContainerPGAdmin}, - {"action": "insert", "key": "k8s.namespace.name", "value": "${env:K8S_POD_NAMESPACE}"}, - {"action": "insert", "key": "k8s.pod.name", "value": "${env:K8S_POD_NAME}"}, - }, - } - - otelConfig.Processors["transform/pgadmin_log"] = map[string]any{ - "log_statements": []map[string]any{ - { - "context": "log", - "statements": []string{ - // Keep the unparsed log record in a standard attribute, and replace - // the log record body with the message field. - // - // https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/general/logs.md - `set(attributes["log.record.original"], body)`, - `set(cache, ParseJSON(body))`, - `merge_maps(attributes, ExtractPatterns(cache["message"], "(?P[A-Z]{3}.*?[\\d]{3})"), "insert")`, - `set(body, cache["message"])`, - - // Set instrumentation scope to the "name" from each log record. - `set(instrumentation_scope.name, cache["name"])`, - - // https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitytext - `set(severity_text, cache["level"])`, - `set(time_unix_nano, Int(cache["time"]*1000000000))`, - - // Map pgAdmin "logging levels" to OpenTelemetry severity levels. - // - // https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber - // https://opentelemetry.io/docs/specs/otel/logs/data-model-appendix/#appendix-b-severitynumber-example-mappings - // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/contexts/ottllog#enums - `set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG"`, - `set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO"`, - `set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING"`, - `set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR"`, - `set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "CRITICAL"`, + var err error + if OpenTelemetryLogsEnabled(ctx, spec) { + + otelConfig := NewConfig(spec) + + otelConfig.Extensions["file_storage/pgadmin_data_logs"] = map[string]any{ + "directory": "/var/lib/pgadmin/logs/receiver", + "create_directory": false, + "fsync": true, + } + + otelConfig.Receivers["filelog/pgadmin"] = map[string]any{ + "include": []string{"/var/lib/pgadmin/logs/pgadmin.log"}, + "storage": "file_storage/pgadmin_data_logs", + } + otelConfig.Receivers["filelog/gunicorn"] = map[string]any{ + "include": []string{"/var/lib/pgadmin/logs/gunicorn.log"}, + "storage": "file_storage/pgadmin_data_logs", + } + + otelConfig.Processors["resource/pgadmin"] = map[string]any{ + "attributes": []map[string]any{ + // Container and Namespace names need no escaping because they are DNS labels. + // Pod names need no escaping because they are DNS subdomains. + // + // https://kubernetes.io/docs/concepts/overview/working-with-objects/names + // https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/resource/k8s.md + // https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/general/logs.md + {"action": "insert", "key": "k8s.container.name", "value": naming.ContainerPGAdmin}, + {"action": "insert", "key": "k8s.namespace.name", "value": "${env:K8S_POD_NAMESPACE}"}, + {"action": "insert", "key": "k8s.pod.name", "value": "${env:K8S_POD_NAME}"}, + }, + } + + otelConfig.Processors["transform/pgadmin_log"] = map[string]any{ + "log_statements": []map[string]any{ + { + "context": "log", + "statements": []string{ + // Keep the unparsed log record in a standard attribute, and replace + // the log record body with the message field. + // + // https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/general/logs.md + `set(attributes["log.record.original"], body)`, + `set(cache, ParseJSON(body))`, + `merge_maps(attributes, ExtractPatterns(cache["message"], "(?P[A-Z]{3}.*?[\\d]{3})"), "insert")`, + `set(body, cache["message"])`, + + // Set instrumentation scope to the "name" from each log record. + `set(instrumentation_scope.name, cache["name"])`, + + // https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitytext + `set(severity_text, cache["level"])`, + `set(time_unix_nano, Int(cache["time"]*1000000000))`, + + // Map pgAdmin "logging levels" to OpenTelemetry severity levels. + // + // https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber + // https://opentelemetry.io/docs/specs/otel/logs/data-model-appendix/#appendix-b-severitynumber-example-mappings + // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/contexts/ottllog#enums + `set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG"`, + `set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO"`, + `set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING"`, + `set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR"`, + `set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "CRITICAL"`, + }, }, }, - }, - } - - // If there are exporters to be added to the logs pipelines defined in - // the spec, add them to the pipeline. Otherwise, add the DebugExporter. - exporters := []ComponentID{DebugExporter} - if spec != nil && spec.Logs != nil && spec.Logs.Exporters != nil { - exporters = slices.Clone(spec.Logs.Exporters) - } - - otelConfig.Pipelines["logs/pgadmin"] = Pipeline{ - Extensions: []ComponentID{"file_storage/pgadmin_data_logs"}, - Receivers: []ComponentID{"filelog/pgadmin"}, - Processors: []ComponentID{ - "resource/pgadmin", - "transform/pgadmin_log", - ResourceDetectionProcessor, - LogsBatchProcessor, - CompactingProcessor, - }, - Exporters: exporters, - } - - otelConfig.Pipelines["logs/gunicorn"] = Pipeline{ - Extensions: []ComponentID{"file_storage/pgadmin_data_logs"}, - Receivers: []ComponentID{"filelog/gunicorn"}, - Processors: []ComponentID{ - "resource/pgadmin", - "transform/pgadmin_log", - ResourceDetectionProcessor, - LogsBatchProcessor, - CompactingProcessor, - }, - Exporters: exporters, - } + } + + // If there are exporters to be added to the logs pipelines defined in + // the spec, add them to the pipeline. Otherwise, add the DebugExporter. + exporters := []ComponentID{DebugExporter} + if spec != nil && spec.Logs != nil && spec.Logs.Exporters != nil { + exporters = slices.Clone(spec.Logs.Exporters) + } + + otelConfig.Pipelines["logs/pgadmin"] = Pipeline{ + Extensions: []ComponentID{"file_storage/pgadmin_data_logs"}, + Receivers: []ComponentID{"filelog/pgadmin"}, + Processors: []ComponentID{ + "resource/pgadmin", + "transform/pgadmin_log", + ResourceDetectionProcessor, + LogsBatchProcessor, + CompactingProcessor, + }, + Exporters: exporters, + } + + otelConfig.Pipelines["logs/gunicorn"] = Pipeline{ + Extensions: []ComponentID{"file_storage/pgadmin_data_logs"}, + Receivers: []ComponentID{"filelog/gunicorn"}, + Processors: []ComponentID{ + "resource/pgadmin", + "transform/pgadmin_log", + ResourceDetectionProcessor, + LogsBatchProcessor, + CompactingProcessor, + }, + Exporters: exporters, + } - otelYAML, err := otelConfig.ToYAML() - if err == nil { - configmap.Data["collector.yaml"] = otelYAML + otelYAML, err := otelConfig.ToYAML() + if err == nil { + configmap.Data["collector.yaml"] = otelYAML + } } return err } diff --git a/internal/collector/pgbackrest.go b/internal/collector/pgbackrest.go index 569748ed9c..009ec0c825 100644 --- a/internal/collector/pgbackrest.go +++ b/internal/collector/pgbackrest.go @@ -11,7 +11,6 @@ import ( "fmt" "slices" - "github.com/crunchydata/postgres-operator/internal/feature" "github.com/crunchydata/postgres-operator/internal/naming" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -29,7 +28,7 @@ func NewConfigForPgBackrestRepoHostPod( ) *Config { config := NewConfig(spec) - if feature.Enabled(ctx, feature.OpenTelemetryLogs) { + if OpenTelemetryLogsEnabled(ctx, spec) { var directory string for _, repo := range repos { diff --git a/internal/collector/pgbouncer.go b/internal/collector/pgbouncer.go index f1f150f6f4..4e161ac373 100644 --- a/internal/collector/pgbouncer.go +++ b/internal/collector/pgbouncer.go @@ -11,7 +11,6 @@ import ( "fmt" "slices" - "github.com/crunchydata/postgres-operator/internal/feature" "github.com/crunchydata/postgres-operator/internal/naming" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -39,7 +38,7 @@ func NewConfigForPgBouncerPod( config := NewConfig(cluster.Spec.Instrumentation) EnablePgBouncerLogging(ctx, cluster, config) - EnablePgBouncerMetrics(ctx, config, sqlQueryUsername) + EnablePgBouncerMetrics(ctx, cluster, config, sqlQueryUsername) return config } @@ -55,7 +54,7 @@ func EnablePgBouncerLogging(ctx context.Context, spec = inCluster.Spec.Instrumentation.Logs } - if feature.Enabled(ctx, feature.OpenTelemetryLogs) { + if OpenTelemetryLogsEnabled(ctx, inCluster) { directory := naming.PGBouncerLogPath // Keep track of what log records and files have been processed. @@ -170,8 +169,10 @@ func EnablePgBouncerLogging(ctx context.Context, // EnablePgBouncerMetrics adds necessary configuration to the collector config to scrape // metrics from pgBouncer when the OpenTelemetryMetrics feature flag is enabled. -func EnablePgBouncerMetrics(ctx context.Context, config *Config, sqlQueryUsername string) { - if feature.Enabled(ctx, feature.OpenTelemetryMetrics) { +func EnablePgBouncerMetrics(ctx context.Context, inCluster *v1beta1.PostgresCluster, + config *Config, sqlQueryUsername string) { + + if OpenTelemetryMetricsEnabled(ctx, inCluster) { // Add Prometheus exporter config.Exporters[Prometheus] = map[string]any{ "endpoint": "0.0.0.0:9187", diff --git a/internal/collector/postgres.go b/internal/collector/postgres.go index cfc0b88245..5d419f85ea 100644 --- a/internal/collector/postgres.go +++ b/internal/collector/postgres.go @@ -15,7 +15,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/crunchydata/postgres-operator/internal/feature" "github.com/crunchydata/postgres-operator/internal/naming" "github.com/crunchydata/postgres-operator/internal/postgres" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" @@ -86,7 +85,7 @@ func EnablePostgresLogging( spec = inCluster.Spec.Instrumentation.Logs } - if inCluster != nil && feature.Enabled(ctx, feature.OpenTelemetryLogs) { + if OpenTelemetryLogsEnabled(ctx, inCluster) { directory := postgres.LogDirectory() version := inCluster.Spec.PostgresVersion diff --git a/internal/collector/postgres_metrics.go b/internal/collector/postgres_metrics.go index 5d56afbf00..81793ade85 100644 --- a/internal/collector/postgres_metrics.go +++ b/internal/collector/postgres_metrics.go @@ -11,8 +11,7 @@ import ( "fmt" "slices" - "github.com/crunchydata/postgres-operator/internal/feature" - "github.com/crunchydata/postgres-operator/internal/pgmonitor" + // "github.com/crunchydata/postgres-operator/internal/pgmonitor" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -37,7 +36,7 @@ var gtePG16 json.RawMessage var ltPG16 json.RawMessage func EnablePostgresMetrics(ctx context.Context, inCluster *v1beta1.PostgresCluster, config *Config) { - if feature.Enabled(ctx, feature.OpenTelemetryMetrics) { + if OpenTelemetryMetricsEnabled(ctx, inCluster) { // We must create a copy of the fiveSecondMetrics variable, otherwise we // will continually append to it and blow up our ConfigMap fiveSecondMetricsClone := slices.Clone(fiveSecondMetrics) @@ -61,7 +60,7 @@ func EnablePostgresMetrics(ctx context.Context, inCluster *v1beta1.PostgresClust config.Receivers[FiveSecondSqlQuery] = map[string]any{ "driver": "postgres", - "datasource": fmt.Sprintf(`host=localhost dbname=postgres port=5432 user=%s password=${env:PGPASSWORD}`, pgmonitor.MonitoringUser), + "datasource": fmt.Sprintf(`host=localhost dbname=postgres port=5432 user=%s password=${env:PGPASSWORD}`, MonitoringUser), "collection_interval": "5s", // Give Postgres time to finish setup. "initial_delay": "10s", @@ -70,7 +69,7 @@ func EnablePostgresMetrics(ctx context.Context, inCluster *v1beta1.PostgresClust config.Receivers[FiveMinuteSqlQuery] = map[string]any{ "driver": "postgres", - "datasource": fmt.Sprintf(`host=localhost dbname=postgres port=5432 user=%s password=${env:PGPASSWORD}`, pgmonitor.MonitoringUser), + "datasource": fmt.Sprintf(`host=localhost dbname=postgres port=5432 user=%s password=${env:PGPASSWORD}`, MonitoringUser), "collection_interval": "300s", // Give Postgres time to finish setup. "initial_delay": "10s", diff --git a/internal/collector/util.go b/internal/collector/util.go index 136ae3fb44..72cf8641ef 100644 --- a/internal/collector/util.go +++ b/internal/collector/util.go @@ -19,7 +19,7 @@ func OpenTelemetrySpecPresent[T CrunchyCRD](object T) bool { switch v := any(object).(type) { case *v1beta1.InstrumentationSpec: - return true + return v != nil case *v1beta1.PostgresCluster: return v.Spec.Instrumentation != nil case *v1beta1.PGAdmin: diff --git a/internal/controller/postgrescluster/cluster.go b/internal/controller/postgrescluster/cluster.go index ead4881b1e..2ceb30453a 100644 --- a/internal/controller/postgrescluster/cluster.go +++ b/internal/controller/postgrescluster/cluster.go @@ -15,7 +15,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "github.com/crunchydata/postgres-operator/internal/feature" + "github.com/crunchydata/postgres-operator/internal/collector" "github.com/crunchydata/postgres-operator/internal/initialize" "github.com/crunchydata/postgres-operator/internal/naming" "github.com/crunchydata/postgres-operator/internal/patroni" @@ -75,7 +75,7 @@ func (r *Reconciler) patroniLogSize(ctx context.Context, cluster *v1beta1.Postgr sizeInBytes = 25000000 } return sizeInBytes - } else if feature.Enabled(ctx, feature.OpenTelemetryLogs) { + } else if collector.OpenTelemetryLogsEnabled(ctx, cluster) { return 25000000 } return 0 diff --git a/internal/controller/postgrescluster/instance.go b/internal/controller/postgrescluster/instance.go index 5c9786459d..867955517f 100644 --- a/internal/controller/postgrescluster/instance.go +++ b/internal/controller/postgrescluster/instance.go @@ -1202,7 +1202,7 @@ func (r *Reconciler) reconcileInstance( // If either OpenTelemetry feature is enabled, we want to add the collector config to the pod if err == nil && - (feature.Enabled(ctx, feature.OpenTelemetryLogs) || feature.Enabled(ctx, feature.OpenTelemetryMetrics)) { + collector.OpenTelemetryLogsOrMetricsEnabled(ctx, cluster) { // If the OpenTelemetryMetrics feature is enabled, we need to get the pgpassword from the // monitoring user secret @@ -1428,8 +1428,8 @@ func (r *Reconciler) reconcileInstanceConfigMap( // If OTel logging or metrics is enabled, add collector config if err == nil && - (feature.Enabled(ctx, feature.OpenTelemetryLogs) || - feature.Enabled(ctx, feature.OpenTelemetryMetrics)) { + collector.OpenTelemetryLogsOrMetricsEnabled(ctx, cluster) { + err = collector.AddToConfigMap(ctx, otelConfig, instanceConfigMap) // Add pgbackrest logrotate if OpenTelemetryLogs is enabled and diff --git a/internal/controller/postgrescluster/pgbackrest.go b/internal/controller/postgrescluster/pgbackrest.go index 54068193af..a79e705f46 100644 --- a/internal/controller/postgrescluster/pgbackrest.go +++ b/internal/controller/postgrescluster/pgbackrest.go @@ -694,7 +694,7 @@ func (r *Reconciler) generateRepoHostIntent(ctx context.Context, postgresCluster // If OpenTelemetryLogs is enabled, we want to add the collector to the pod // and also add the RepoVolumes to the container. - if postgresCluster.Spec.Instrumentation != nil && feature.Enabled(ctx, feature.OpenTelemetryLogs) { + if collector.OpenTelemetryLogsEnabled(ctx, postgresCluster) { collector.AddToPod(ctx, postgresCluster.Spec.Instrumentation, postgresCluster.Spec.ImagePullPolicy, &corev1.ConfigMap{ObjectMeta: naming.PGBackRestConfig(postgresCluster)}, &repo.Spec.Template.Spec, []corev1.VolumeMount{}, "", diff --git a/internal/controller/postgrescluster/pgbouncer.go b/internal/controller/postgrescluster/pgbouncer.go index d5a935bbf3..3acd22876c 100644 --- a/internal/controller/postgrescluster/pgbouncer.go +++ b/internal/controller/postgrescluster/pgbouncer.go @@ -19,7 +19,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/crunchydata/postgres-operator/internal/collector" - "github.com/crunchydata/postgres-operator/internal/feature" "github.com/crunchydata/postgres-operator/internal/initialize" "github.com/crunchydata/postgres-operator/internal/logging" "github.com/crunchydata/postgres-operator/internal/naming" @@ -99,13 +98,11 @@ func (r *Reconciler) reconcilePGBouncerConfigMap( pgbouncer.ConfigMap(ctx, cluster, configmap) } // If OTel logging or metrics is enabled, add collector config - if otelConfig != nil && - (feature.Enabled(ctx, feature.OpenTelemetryLogs) || - feature.Enabled(ctx, feature.OpenTelemetryMetrics)) { + if collector.OpenTelemetryLogsOrMetricsEnabled(ctx, cluster) { err = collector.AddToConfigMap(ctx, otelConfig, configmap) } // If OTel logging is enabled, add logrotate config - if err == nil && otelConfig != nil && feature.Enabled(ctx, feature.OpenTelemetryLogs) { + if err == nil && collector.OpenTelemetryLogsEnabled(ctx, cluster) { logrotateConfig := collector.LogrotateConfig{ LogFiles: []string{naming.PGBouncerFullLogPath}, PostrotateScript: collector.PGBouncerPostRotateScript, diff --git a/internal/controller/postgrescluster/pgmonitor.go b/internal/controller/postgrescluster/pgmonitor.go index 84b955559a..48d15d1e6d 100644 --- a/internal/controller/postgrescluster/pgmonitor.go +++ b/internal/controller/postgrescluster/pgmonitor.go @@ -16,6 +16,7 @@ import ( corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/crunchydata/postgres-operator/internal/collector" "github.com/crunchydata/postgres-operator/internal/config" "github.com/crunchydata/postgres-operator/internal/feature" "github.com/crunchydata/postgres-operator/internal/initialize" @@ -62,7 +63,7 @@ func (r *Reconciler) reconcilePGMonitorExporter(ctx context.Context, // the `EnableExporterInPostgreSQL` funcs; that way we are always running // that function against an updated and running pod. - if pgmonitor.ExporterEnabled(ctx, cluster) || feature.Enabled(ctx, feature.OpenTelemetryMetrics) { + if pgmonitor.ExporterEnabled(ctx, cluster) || collector.OpenTelemetryMetricsEnabled(ctx, cluster) { sql, err := os.ReadFile(fmt.Sprintf("%s/pg%d/setup.sql", pgmonitor.GetQueriesConfigDir(ctx), cluster.Spec.PostgresVersion)) if err != nil { return err @@ -99,7 +100,7 @@ func (r *Reconciler) reconcilePGMonitorExporter(ctx context.Context, return pgmonitor.EnableExporterInPostgreSQL(ctx, exec, monitoringSecret, pgmonitor.ExporterDB, setup) } - if !pgmonitor.ExporterEnabled(ctx, cluster) && !feature.Enabled(ctx, feature.OpenTelemetryMetrics) { + if !pgmonitor.ExporterEnabled(ctx, cluster) && !collector.OpenTelemetryMetricsEnabled(ctx, cluster) { action = func(ctx context.Context, exec postgres.Executor) error { return pgmonitor.DisableMonitoringUserInPostgres(ctx, exec) } @@ -161,7 +162,7 @@ func (r *Reconciler) reconcileMonitoringSecret( // is enabled to determine when monitoring secret should be created, // since our implementation of the SqlQuery receiver in the OTel Collector // uses the monitoring user as well. - if !pgmonitor.ExporterEnabled(ctx, cluster) && !feature.Enabled(ctx, feature.OpenTelemetryMetrics) { + if !pgmonitor.ExporterEnabled(ctx, cluster) && !collector.OpenTelemetryMetricsEnabled(ctx, cluster) { if err == nil { err = errors.WithStack(r.deleteControlled(ctx, cluster, existing)) } @@ -234,7 +235,7 @@ func addPGMonitorExporterToInstancePodSpec( template *corev1.PodTemplateSpec, exporterQueriesConfig, exporterWebConfig *corev1.ConfigMap) { - if !pgmonitor.ExporterEnabled(ctx, cluster) || feature.Enabled(ctx, feature.OpenTelemetryMetrics) { + if !pgmonitor.ExporterEnabled(ctx, cluster) || collector.OpenTelemetryMetricsEnabled(ctx, cluster) { return } @@ -374,7 +375,7 @@ func addPGMonitorExporterToInstancePodSpec( func (r *Reconciler) reconcileExporterWebConfig(ctx context.Context, cluster *v1beta1.PostgresCluster) (*corev1.ConfigMap, error) { - if feature.Enabled(ctx, feature.OpenTelemetryMetrics) { + if collector.OpenTelemetryMetricsEnabled(ctx, cluster) { return nil, nil } @@ -384,7 +385,9 @@ func (r *Reconciler) reconcileExporterWebConfig(ctx context.Context, return nil, err } - if !pgmonitor.ExporterEnabled(ctx, cluster) || feature.Enabled(ctx, feature.OpenTelemetryMetrics) || cluster.Spec.Monitoring.PGMonitor.Exporter.CustomTLSSecret == nil { + if !pgmonitor.ExporterEnabled(ctx, cluster) || + collector.OpenTelemetryMetricsEnabled(ctx, cluster) || + cluster.Spec.Monitoring.PGMonitor.Exporter.CustomTLSSecret == nil { // We could still have a NotFound error here so check the err. // If no error that means the configmap is found and needs to be deleted if err == nil { @@ -441,7 +444,7 @@ func (r *Reconciler) reconcileExporterQueriesConfig(ctx context.Context, return nil, err } - if !pgmonitor.ExporterEnabled(ctx, cluster) || feature.Enabled(ctx, feature.OpenTelemetryMetrics) { + if !pgmonitor.ExporterEnabled(ctx, cluster) || collector.OpenTelemetryMetricsEnabled(ctx, cluster) { // We could still have a NotFound error here so check the err. // If no error that means the configmap is found and needs to be deleted if err == nil { diff --git a/internal/pgbackrest/config.go b/internal/pgbackrest/config.go index c14a264ce3..498be32d3b 100644 --- a/internal/pgbackrest/config.go +++ b/internal/pgbackrest/config.go @@ -17,7 +17,6 @@ import ( "github.com/crunchydata/postgres-operator/internal/collector" "github.com/crunchydata/postgres-operator/internal/config" - "github.com/crunchydata/postgres-operator/internal/feature" "github.com/crunchydata/postgres-operator/internal/initialize" "github.com/crunchydata/postgres-operator/internal/naming" "github.com/crunchydata/postgres-operator/internal/postgres" @@ -131,8 +130,8 @@ func CreatePGBackRestConfigMapIntent(ctx context.Context, postgresCluster *v1bet ).String() if RepoHostVolumeDefined(postgresCluster) && - (feature.Enabled(ctx, feature.OpenTelemetryLogs) || - feature.Enabled(ctx, feature.OpenTelemetryMetrics)) { + collector.OpenTelemetryLogsOrMetricsEnabled(ctx, postgresCluster) { + err = collector.AddToConfigMap(ctx, collector.NewConfigForPgBackrestRepoHostPod( ctx, postgresCluster.Spec.Instrumentation, @@ -141,8 +140,7 @@ func CreatePGBackRestConfigMapIntent(ctx context.Context, postgresCluster *v1bet // If OTel logging is enabled, add logrotate config for the RepoHost if err == nil && - postgresCluster.Spec.Instrumentation != nil && - feature.Enabled(ctx, feature.OpenTelemetryLogs) { + collector.OpenTelemetryLogsEnabled(ctx, postgresCluster) { var pgBackRestLogPath string for _, repo := range postgresCluster.Spec.Backups.PGBackRest.Repos { if repo.Volume != nil { diff --git a/internal/pgbouncer/config.go b/internal/pgbouncer/config.go index 257dc63dbd..99bcac0399 100644 --- a/internal/pgbouncer/config.go +++ b/internal/pgbouncer/config.go @@ -12,7 +12,7 @@ import ( corev1 "k8s.io/api/core/v1" - "github.com/crunchydata/postgres-operator/internal/feature" + "github.com/crunchydata/postgres-operator/internal/collector" "github.com/crunchydata/postgres-operator/internal/naming" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -127,13 +127,13 @@ func clusterINI(ctx context.Context, cluster *v1beta1.PostgresCluster) string { } // If OpenTelemetryLogs feature is enabled, enable logging to file - if feature.Enabled(ctx, feature.OpenTelemetryLogs) { + if collector.OpenTelemetryLogsEnabled(ctx, cluster) { global["logfile"] = naming.PGBouncerLogPath + "/pgbouncer.log" } // When OTel metrics are enabled, allow pgBouncer's postgres user // to run read-only console queries on pgBouncer's virtual db - if feature.Enabled(ctx, feature.OpenTelemetryMetrics) { + if collector.OpenTelemetryMetricsEnabled(ctx, cluster) { global["stats_users"] = PostgresqlUser } diff --git a/internal/pgbouncer/reconcile.go b/internal/pgbouncer/reconcile.go index 4181cea478..1b8adef739 100644 --- a/internal/pgbouncer/reconcile.go +++ b/internal/pgbouncer/reconcile.go @@ -190,7 +190,7 @@ func Pod( outPod.Volumes = []corev1.Volume{configVolume} - if feature.Enabled(ctx, feature.OpenTelemetryLogs) || feature.Enabled(ctx, feature.OpenTelemetryMetrics) { + if collector.OpenTelemetryLogsOrMetricsEnabled(ctx, inCluster) { collector.AddToPod(ctx, inCluster.Spec.Instrumentation, inCluster.Spec.ImagePullPolicy, inConfigMap, outPod, []corev1.VolumeMount{configVolumeMount}, string(inSecret.Data["pgbouncer-password"]), []string{naming.PGBouncerLogPath}, true) From 1bcbbaca251f3e009c45f53feb2558f0873b2114 Mon Sep 17 00:00:00 2001 From: Ben Blattberg Date: Fri, 7 Mar 2025 14:19:00 -0600 Subject: [PATCH 3/8] exporterenabled just checks that --- internal/pgmonitor/util.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/pgmonitor/util.go b/internal/pgmonitor/util.go index 32cf222448..72f528ffa3 100644 --- a/internal/pgmonitor/util.go +++ b/internal/pgmonitor/util.go @@ -8,7 +8,6 @@ import ( "context" "os" - "github.com/crunchydata/postgres-operator/internal/feature" "github.com/crunchydata/postgres-operator/internal/logging" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -37,8 +36,5 @@ func ExporterEnabled(ctx context.Context, cluster *v1beta1.PostgresCluster) bool if cluster.Spec.Monitoring.PGMonitor.Exporter == nil { return false } - if feature.Enabled(ctx, feature.OpenTelemetryMetrics) { - return false - } return true } From 4955e4cbf6967a21aa8dd3e84373626cc40347fa Mon Sep 17 00:00:00 2001 From: Ben Blattberg Date: Fri, 7 Mar 2025 14:21:38 -0600 Subject: [PATCH 4/8] update pgadmin collector test --- internal/collector/pgadmin_test.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/internal/collector/pgadmin_test.go b/internal/collector/pgadmin_test.go index c4d5acfab6..e7512b4f54 100644 --- a/internal/collector/pgadmin_test.go +++ b/internal/collector/pgadmin_test.go @@ -12,7 +12,6 @@ import ( corev1 "k8s.io/api/core/v1" "github.com/crunchydata/postgres-operator/internal/collector" - pgadmin "github.com/crunchydata/postgres-operator/internal/controller/standalone_pgadmin" "github.com/crunchydata/postgres-operator/internal/feature" "github.com/crunchydata/postgres-operator/internal/initialize" "github.com/crunchydata/postgres-operator/internal/testing/cmp" @@ -44,7 +43,7 @@ collector.yaml: | extensions: file_storage/pgadmin_data_logs: create_directory: false - directory: `+pgadmin.LogDirectoryAbsolutePath+`/receiver + directory: /var/lib/pgadmin/logs/receiver fsync: true processors: batch/1s: @@ -90,11 +89,11 @@ collector.yaml: | receivers: filelog/gunicorn: include: - - `+pgadmin.GunicornLogFileAbsolutePath+` + - /var/lib/pgadmin/logs/gunicorn.log storage: file_storage/pgadmin_data_logs filelog/pgadmin: include: - - `+pgadmin.LogFileAbsolutePath+` + - /var/lib/pgadmin/logs/pgadmin.log storage: file_storage/pgadmin_data_logs service: extensions: @@ -165,7 +164,7 @@ collector.yaml: | extensions: file_storage/pgadmin_data_logs: create_directory: false - directory: `+pgadmin.LogDirectoryAbsolutePath+`/receiver + directory: /var/lib/pgadmin/logs/receiver fsync: true processors: batch/1s: @@ -211,11 +210,11 @@ collector.yaml: | receivers: filelog/gunicorn: include: - - `+pgadmin.GunicornLogFileAbsolutePath+` + - /var/lib/pgadmin/logs/gunicorn.log storage: file_storage/pgadmin_data_logs filelog/pgadmin: include: - - `+pgadmin.LogFileAbsolutePath+` + - /var/lib/pgadmin/logs/pgadmin.log storage: file_storage/pgadmin_data_logs service: extensions: From 6da49fc6d6ebfcb9a733b85b59f6808394c9ffc5 Mon Sep 17 00:00:00 2001 From: Ben Blattberg Date: Fri, 7 Mar 2025 14:41:50 -0600 Subject: [PATCH 5/8] update tests --- internal/collector/patroni_test.go | 9 ++++++++- internal/collector/pgadmin_test.go | 9 ++++++++- internal/collector/pgbackrest_test.go | 9 ++++++++- internal/collector/pgbouncer_test.go | 10 ++++++++-- internal/collector/postgres_test.go | 6 ++++++ internal/controller/postgrescluster/cluster_test.go | 5 +++++ 6 files changed, 43 insertions(+), 5 deletions(-) diff --git a/internal/collector/patroni_test.go b/internal/collector/patroni_test.go index e2d3a84e58..2f73374109 100644 --- a/internal/collector/patroni_test.go +++ b/internal/collector/patroni_test.go @@ -11,6 +11,7 @@ import ( "gotest.tools/v3/assert" "github.com/crunchydata/postgres-operator/internal/feature" + "github.com/crunchydata/postgres-operator/internal/testing/require" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -23,8 +24,14 @@ func TestEnablePatroniLogging(t *testing.T) { ctx := feature.NewContext(context.Background(), gate) config := NewConfig(nil) + cluster := new(v1beta1.PostgresCluster) + require.UnmarshalInto(t, &cluster.Spec, `{ + instrumentation: { + logs: { retentionPeriod: 5h }, + }, + }`) - EnablePatroniLogging(ctx, new(v1beta1.PostgresCluster), config) + EnablePatroniLogging(ctx, cluster, config) result, err := config.ToYAML() assert.NilError(t, err) diff --git a/internal/collector/pgadmin_test.go b/internal/collector/pgadmin_test.go index e7512b4f54..5de808ace7 100644 --- a/internal/collector/pgadmin_test.go +++ b/internal/collector/pgadmin_test.go @@ -30,7 +30,14 @@ func TestEnablePgAdminLogging(t *testing.T) { configmap := new(corev1.ConfigMap) initialize.Map(&configmap.Data) - err := collector.EnablePgAdminLogging(ctx, nil, configmap) + retentionPeriod, err := v1beta1.NewDuration("12 hours") + assert.NilError(t, err) + instrumentation := v1beta1.InstrumentationSpec{ + Logs: &v1beta1.InstrumentationLogsSpec{ + RetentionPeriod: retentionPeriod, + }, + } + err = collector.EnablePgAdminLogging(ctx, &instrumentation, configmap) assert.NilError(t, err) assert.Assert(t, cmp.MarshalMatches(configmap.Data, ` diff --git a/internal/collector/pgbackrest_test.go b/internal/collector/pgbackrest_test.go index f1ebf14e4f..2a1bbe2511 100644 --- a/internal/collector/pgbackrest_test.go +++ b/internal/collector/pgbackrest_test.go @@ -27,8 +27,15 @@ func TestNewConfigForPgBackrestRepoHostPod(t *testing.T) { Volume: new(v1beta1.RepoPVC), }, } + retentionPeriod, err := v1beta1.NewDuration("12 hours") + assert.NilError(t, err) + instrumentation := v1beta1.InstrumentationSpec{ + Logs: &v1beta1.InstrumentationLogsSpec{ + RetentionPeriod: retentionPeriod, + }, + } - config := NewConfigForPgBackrestRepoHostPod(ctx, nil, repos) + config := NewConfigForPgBackrestRepoHostPod(ctx, &instrumentation, repos) result, err := config.ToYAML() assert.NilError(t, err) diff --git a/internal/collector/pgbouncer_test.go b/internal/collector/pgbouncer_test.go index df8427fbbd..74aed710da 100644 --- a/internal/collector/pgbouncer_test.go +++ b/internal/collector/pgbouncer_test.go @@ -11,6 +11,7 @@ import ( "gotest.tools/v3/assert" "github.com/crunchydata/postgres-operator/internal/feature" + "github.com/crunchydata/postgres-operator/internal/testing/require" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -23,8 +24,13 @@ func TestEnablePgBouncerLogging(t *testing.T) { ctx := feature.NewContext(context.Background(), gate) config := NewConfig(nil) - - EnablePgBouncerLogging(ctx, new(v1beta1.PostgresCluster), config) + cluster := new(v1beta1.PostgresCluster) + require.UnmarshalInto(t, &cluster.Spec, `{ + instrumentation: { + logs: { retentionPeriod: 5h }, + }, + }`) + EnablePgBouncerLogging(ctx, cluster, config) result, err := config.ToYAML() assert.NilError(t, err) diff --git a/internal/collector/postgres_test.go b/internal/collector/postgres_test.go index a6736d66cc..3bdf33c61a 100644 --- a/internal/collector/postgres_test.go +++ b/internal/collector/postgres_test.go @@ -12,6 +12,7 @@ import ( "github.com/crunchydata/postgres-operator/internal/feature" "github.com/crunchydata/postgres-operator/internal/postgres" + "github.com/crunchydata/postgres-operator/internal/testing/require" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -25,6 +26,11 @@ func TestEnablePostgresLogging(t *testing.T) { cluster := new(v1beta1.PostgresCluster) cluster.Spec.PostgresVersion = 99 + require.UnmarshalInto(t, &cluster.Spec, `{ + instrumentation: { + logs: { retentionPeriod: 5h }, + }, + }`) config := NewConfig(nil) params := postgres.NewParameterSet() diff --git a/internal/controller/postgrescluster/cluster_test.go b/internal/controller/postgrescluster/cluster_test.go index 6882cfa27b..a38a128086 100644 --- a/internal/controller/postgrescluster/cluster_test.go +++ b/internal/controller/postgrescluster/cluster_test.go @@ -870,6 +870,11 @@ func TestPatroniLogSize(t *testing.T) { reconciler := &Reconciler{Recorder: recorder} cluster.Spec.Patroni = nil + require.UnmarshalInto(t, &cluster.Spec, `{ + instrumentation: { + logs: { retentionPeriod: 5h }, + }, + }`) size := reconciler.patroniLogSize(ctx, &cluster) From 579f0011f99b2dc7601afae0ba6d7388a0818f35 Mon Sep 17 00:00:00 2001 From: Ben Blattberg Date: Fri, 7 Mar 2025 14:51:24 -0600 Subject: [PATCH 6/8] test, lint update --- internal/pgmonitor/postgres.go | 3 ++- internal/pgmonitor/util_test.go | 9 --------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/internal/pgmonitor/postgres.go b/internal/pgmonitor/postgres.go index 4dd6ab8fb2..3ef83cd2e0 100644 --- a/internal/pgmonitor/postgres.go +++ b/internal/pgmonitor/postgres.go @@ -8,11 +8,12 @@ import ( "context" "strings" + corev1 "k8s.io/api/core/v1" + "github.com/crunchydata/postgres-operator/internal/collector" "github.com/crunchydata/postgres-operator/internal/logging" "github.com/crunchydata/postgres-operator/internal/postgres" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" - corev1 "k8s.io/api/core/v1" ) const ( diff --git a/internal/pgmonitor/util_test.go b/internal/pgmonitor/util_test.go index e83bbb3730..a7758d0da4 100644 --- a/internal/pgmonitor/util_test.go +++ b/internal/pgmonitor/util_test.go @@ -10,7 +10,6 @@ import ( "gotest.tools/v3/assert" - "github.com/crunchydata/postgres-operator/internal/feature" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -27,12 +26,4 @@ func TestExporterEnabled(t *testing.T) { cluster.Spec.Monitoring.PGMonitor.Exporter = &v1beta1.ExporterSpec{} assert.Assert(t, ExporterEnabled(ctx, cluster)) - - gate := feature.NewGate() - assert.NilError(t, gate.SetFromMap(map[string]bool{ - feature.OpenTelemetryMetrics: true, - })) - ctx = feature.NewContext(ctx, gate) - cluster.Spec.Monitoring.PGMonitor.Exporter = &v1beta1.ExporterSpec{} - assert.Assert(t, !ExporterEnabled(ctx, cluster)) } From 447360bed8fd650e99eab39b7050986e5bb6d44f Mon Sep 17 00:00:00 2001 From: Ben Blattberg Date: Tue, 11 Mar 2025 16:22:30 -0500 Subject: [PATCH 7/8] missed ref to pgmonitor when resolving --- internal/collector/postgres_metrics.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/collector/postgres_metrics.go b/internal/collector/postgres_metrics.go index abf653c010..4530c431a3 100644 --- a/internal/collector/postgres_metrics.go +++ b/internal/collector/postgres_metrics.go @@ -12,7 +12,6 @@ import ( "slices" "strconv" - "github.com/crunchydata/postgres-operator/internal/feature" "github.com/crunchydata/postgres-operator/internal/logging" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -171,7 +170,7 @@ func EnablePostgresMetrics(ctx context.Context, inCluster *v1beta1.PostgresClust "driver": "postgres", "datasource": fmt.Sprintf( `host=localhost dbname=postgres port=5432 user=%s password=${env:PGPASSWORD}`, - pgmonitor.MonitoringUser), + MonitoringUser), "collection_interval": querySet.CollectionInterval, // Give Postgres time to finish setup. "initial_delay": "10s", From d211384b88d83655d2356db7ceb27dc56be757ea Mon Sep 17 00:00:00 2001 From: Ben Blattberg Date: Wed, 12 Mar 2025 13:39:24 -0500 Subject: [PATCH 8/8] PR feedback --- internal/collector/pgadmin.go | 209 +++++++++--------- internal/collector/pgadmin_test.go | 13 +- internal/collector/pgbackrest_test.go | 14 +- .../controller/standalone_pgadmin/pod_test.go | 11 +- 4 files changed, 120 insertions(+), 127 deletions(-) diff --git a/internal/collector/pgadmin.go b/internal/collector/pgadmin.go index ff1cca4eb6..1f82115703 100644 --- a/internal/collector/pgadmin.go +++ b/internal/collector/pgadmin.go @@ -17,113 +17,114 @@ import ( func EnablePgAdminLogging(ctx context.Context, spec *v1beta1.InstrumentationSpec, configmap *corev1.ConfigMap, ) error { - var err error - if OpenTelemetryLogsEnabled(ctx, spec) { - - otelConfig := NewConfig(spec) - - otelConfig.Extensions["file_storage/pgadmin_data_logs"] = map[string]any{ - "directory": "/var/lib/pgadmin/logs/receiver", - "create_directory": false, - "fsync": true, - } - - otelConfig.Receivers["filelog/pgadmin"] = map[string]any{ - "include": []string{"/var/lib/pgadmin/logs/pgadmin.log"}, - "storage": "file_storage/pgadmin_data_logs", - } - otelConfig.Receivers["filelog/gunicorn"] = map[string]any{ - "include": []string{"/var/lib/pgadmin/logs/gunicorn.log"}, - "storage": "file_storage/pgadmin_data_logs", - } - - otelConfig.Processors["resource/pgadmin"] = map[string]any{ - "attributes": []map[string]any{ - // Container and Namespace names need no escaping because they are DNS labels. - // Pod names need no escaping because they are DNS subdomains. - // - // https://kubernetes.io/docs/concepts/overview/working-with-objects/names - // https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/resource/k8s.md - // https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/general/logs.md - {"action": "insert", "key": "k8s.container.name", "value": naming.ContainerPGAdmin}, - {"action": "insert", "key": "k8s.namespace.name", "value": "${env:K8S_POD_NAMESPACE}"}, - {"action": "insert", "key": "k8s.pod.name", "value": "${env:K8S_POD_NAME}"}, - }, - } - - otelConfig.Processors["transform/pgadmin_log"] = map[string]any{ - "log_statements": []map[string]any{ - { - "context": "log", - "statements": []string{ - // Keep the unparsed log record in a standard attribute, and replace - // the log record body with the message field. - // - // https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/general/logs.md - `set(attributes["log.record.original"], body)`, - `set(cache, ParseJSON(body))`, - `merge_maps(attributes, ExtractPatterns(cache["message"], "(?P[A-Z]{3}.*?[\\d]{3})"), "insert")`, - `set(body, cache["message"])`, - - // Set instrumentation scope to the "name" from each log record. - `set(instrumentation_scope.name, cache["name"])`, - - // https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitytext - `set(severity_text, cache["level"])`, - `set(time_unix_nano, Int(cache["time"]*1000000000))`, - - // Map pgAdmin "logging levels" to OpenTelemetry severity levels. - // - // https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber - // https://opentelemetry.io/docs/specs/otel/logs/data-model-appendix/#appendix-b-severitynumber-example-mappings - // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/contexts/ottllog#enums - `set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG"`, - `set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO"`, - `set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING"`, - `set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR"`, - `set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "CRITICAL"`, - }, + if !OpenTelemetryLogsEnabled(ctx, spec) { + return nil + } + + otelConfig := NewConfig(spec) + + otelConfig.Extensions["file_storage/pgadmin_data_logs"] = map[string]any{ + "directory": "/var/lib/pgadmin/logs/receiver", + "create_directory": false, + "fsync": true, + } + + otelConfig.Receivers["filelog/pgadmin"] = map[string]any{ + "include": []string{"/var/lib/pgadmin/logs/pgadmin.log"}, + "storage": "file_storage/pgadmin_data_logs", + } + otelConfig.Receivers["filelog/gunicorn"] = map[string]any{ + "include": []string{"/var/lib/pgadmin/logs/gunicorn.log"}, + "storage": "file_storage/pgadmin_data_logs", + } + + otelConfig.Processors["resource/pgadmin"] = map[string]any{ + "attributes": []map[string]any{ + // Container and Namespace names need no escaping because they are DNS labels. + // Pod names need no escaping because they are DNS subdomains. + // + // https://kubernetes.io/docs/concepts/overview/working-with-objects/names + // https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/resource/k8s.md + // https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/general/logs.md + {"action": "insert", "key": "k8s.container.name", "value": naming.ContainerPGAdmin}, + {"action": "insert", "key": "k8s.namespace.name", "value": "${env:K8S_POD_NAMESPACE}"}, + {"action": "insert", "key": "k8s.pod.name", "value": "${env:K8S_POD_NAME}"}, + }, + } + + otelConfig.Processors["transform/pgadmin_log"] = map[string]any{ + "log_statements": []map[string]any{ + { + "context": "log", + "statements": []string{ + // Keep the unparsed log record in a standard attribute, and replace + // the log record body with the message field. + // + // https://github.com/open-telemetry/semantic-conventions/blob/v1.29.0/docs/general/logs.md + `set(attributes["log.record.original"], body)`, + `set(cache, ParseJSON(body))`, + `merge_maps(attributes, ExtractPatterns(cache["message"], "(?P[A-Z]{3}.*?[\\d]{3})"), "insert")`, + `set(body, cache["message"])`, + + // Set instrumentation scope to the "name" from each log record. + `set(instrumentation_scope.name, cache["name"])`, + + // https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitytext + `set(severity_text, cache["level"])`, + `set(time_unix_nano, Int(cache["time"]*1000000000))`, + + // Map pgAdmin "logging levels" to OpenTelemetry severity levels. + // + // https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber + // https://opentelemetry.io/docs/specs/otel/logs/data-model-appendix/#appendix-b-severitynumber-example-mappings + // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/-/pkg/ottl/contexts/ottllog#enums + `set(severity_number, SEVERITY_NUMBER_DEBUG) where severity_text == "DEBUG"`, + `set(severity_number, SEVERITY_NUMBER_INFO) where severity_text == "INFO"`, + `set(severity_number, SEVERITY_NUMBER_WARN) where severity_text == "WARNING"`, + `set(severity_number, SEVERITY_NUMBER_ERROR) where severity_text == "ERROR"`, + `set(severity_number, SEVERITY_NUMBER_FATAL) where severity_text == "CRITICAL"`, }, }, - } - - // If there are exporters to be added to the logs pipelines defined in - // the spec, add them to the pipeline. Otherwise, add the DebugExporter. - exporters := []ComponentID{DebugExporter} - if spec != nil && spec.Logs != nil && spec.Logs.Exporters != nil { - exporters = slices.Clone(spec.Logs.Exporters) - } - - otelConfig.Pipelines["logs/pgadmin"] = Pipeline{ - Extensions: []ComponentID{"file_storage/pgadmin_data_logs"}, - Receivers: []ComponentID{"filelog/pgadmin"}, - Processors: []ComponentID{ - "resource/pgadmin", - "transform/pgadmin_log", - ResourceDetectionProcessor, - LogsBatchProcessor, - CompactingProcessor, - }, - Exporters: exporters, - } - - otelConfig.Pipelines["logs/gunicorn"] = Pipeline{ - Extensions: []ComponentID{"file_storage/pgadmin_data_logs"}, - Receivers: []ComponentID{"filelog/gunicorn"}, - Processors: []ComponentID{ - "resource/pgadmin", - "transform/pgadmin_log", - ResourceDetectionProcessor, - LogsBatchProcessor, - CompactingProcessor, - }, - Exporters: exporters, - } + }, + } + + // If there are exporters to be added to the logs pipelines defined in + // the spec, add them to the pipeline. Otherwise, add the DebugExporter. + exporters := []ComponentID{DebugExporter} + if spec != nil && spec.Logs != nil && spec.Logs.Exporters != nil { + exporters = slices.Clone(spec.Logs.Exporters) + } - otelYAML, err := otelConfig.ToYAML() - if err == nil { - configmap.Data["collector.yaml"] = otelYAML - } + otelConfig.Pipelines["logs/pgadmin"] = Pipeline{ + Extensions: []ComponentID{"file_storage/pgadmin_data_logs"}, + Receivers: []ComponentID{"filelog/pgadmin"}, + Processors: []ComponentID{ + "resource/pgadmin", + "transform/pgadmin_log", + ResourceDetectionProcessor, + LogsBatchProcessor, + CompactingProcessor, + }, + Exporters: exporters, } + + otelConfig.Pipelines["logs/gunicorn"] = Pipeline{ + Extensions: []ComponentID{"file_storage/pgadmin_data_logs"}, + Receivers: []ComponentID{"filelog/gunicorn"}, + Processors: []ComponentID{ + "resource/pgadmin", + "transform/pgadmin_log", + ResourceDetectionProcessor, + LogsBatchProcessor, + CompactingProcessor, + }, + Exporters: exporters, + } + + otelYAML, err := otelConfig.ToYAML() + if err == nil { + configmap.Data["collector.yaml"] = otelYAML + } + return err } diff --git a/internal/collector/pgadmin_test.go b/internal/collector/pgadmin_test.go index 5de808ace7..e5db11f587 100644 --- a/internal/collector/pgadmin_test.go +++ b/internal/collector/pgadmin_test.go @@ -30,14 +30,11 @@ func TestEnablePgAdminLogging(t *testing.T) { configmap := new(corev1.ConfigMap) initialize.Map(&configmap.Data) - retentionPeriod, err := v1beta1.NewDuration("12 hours") - assert.NilError(t, err) - instrumentation := v1beta1.InstrumentationSpec{ - Logs: &v1beta1.InstrumentationLogsSpec{ - RetentionPeriod: retentionPeriod, - }, - } - err = collector.EnablePgAdminLogging(ctx, &instrumentation, configmap) + var instrumentation *v1beta1.InstrumentationSpec + require.UnmarshalInto(t, &instrumentation, `{ + logs: { retentionPeriod: 12h }, + }`) + err := collector.EnablePgAdminLogging(ctx, instrumentation, configmap) assert.NilError(t, err) assert.Assert(t, cmp.MarshalMatches(configmap.Data, ` diff --git a/internal/collector/pgbackrest_test.go b/internal/collector/pgbackrest_test.go index 2a1bbe2511..e8a5a4d2dd 100644 --- a/internal/collector/pgbackrest_test.go +++ b/internal/collector/pgbackrest_test.go @@ -11,6 +11,7 @@ import ( "gotest.tools/v3/assert" "github.com/crunchydata/postgres-operator/internal/feature" + "github.com/crunchydata/postgres-operator/internal/testing/require" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -27,15 +28,12 @@ func TestNewConfigForPgBackrestRepoHostPod(t *testing.T) { Volume: new(v1beta1.RepoPVC), }, } - retentionPeriod, err := v1beta1.NewDuration("12 hours") - assert.NilError(t, err) - instrumentation := v1beta1.InstrumentationSpec{ - Logs: &v1beta1.InstrumentationLogsSpec{ - RetentionPeriod: retentionPeriod, - }, - } + var instrumentation *v1beta1.InstrumentationSpec + require.UnmarshalInto(t, &instrumentation, `{ + logs: { retentionPeriod: 12h }, + }`) - config := NewConfigForPgBackrestRepoHostPod(ctx, &instrumentation, repos) + config := NewConfigForPgBackrestRepoHostPod(ctx, instrumentation, repos) result, err := config.ToYAML() assert.NilError(t, err) diff --git a/internal/controller/standalone_pgadmin/pod_test.go b/internal/controller/standalone_pgadmin/pod_test.go index b414a7bab0..bc8a32da49 100644 --- a/internal/controller/standalone_pgadmin/pod_test.go +++ b/internal/controller/standalone_pgadmin/pod_test.go @@ -16,6 +16,7 @@ import ( "github.com/crunchydata/postgres-operator/internal/initialize" "github.com/crunchydata/postgres-operator/internal/kubernetes" "github.com/crunchydata/postgres-operator/internal/testing/cmp" + "github.com/crunchydata/postgres-operator/internal/testing/require" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -211,13 +212,9 @@ volumes: pgadmin.Spec.Resources.Requests = corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("100m"), } - retentionPeriod, err := v1beta1.NewDuration("12 hours") - assert.NilError(t, err) - pgadmin.Spec.Instrumentation = &v1beta1.InstrumentationSpec{ - Logs: &v1beta1.InstrumentationLogsSpec{ - RetentionPeriod: retentionPeriod, - }, - } + require.UnmarshalInto(t, &pgadmin.Spec.Instrumentation, `{ + logs: { retentionPeriod: 12h }, + }`) call()