Add kuttl test

Author: benjaminjb

testing/kuttl/e2e/otel-logging-and-metrics/11--add-logs-exporter.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- files/11--add-per-db-metrics.yaml

testing/kuttl/e2e/otel-logging-and-metrics/11--add-per-db-metrics-to-postgrescluster.yaml

@@ -0,0 +1,32 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+# First, check that all containers in the instance pod are ready.
+# Then, grab the collector metrics output and check that the per-db metrics
+# are present for the single added target.
+- script: |
+    retry() { bash -ceu 'printf "$1\nSleeping...\n" && sleep 5' - "$@"; }
+    check_containers_ready() { bash -ceu 'echo "$1" | jq -e ".[] | select(.type==\"ContainersReady\") | .status==\"True\""' - "$@"; }
+    contains() { bash -ceu '[[ "$1" == *"$2"* ]]' - "$@"; }
+
+    pod=$(kubectl get pods -o name -n "${NAMESPACE}" \
+      -l postgres-operator.crunchydata.com/cluster=otel-cluster,postgres-operator.crunchydata.com/data=postgres)
+    [ "$pod" = "" ] && retry "Pod not found" && exit 1
+
+    condition_json=$(kubectl get "${pod}" -n "${NAMESPACE}" -o jsonpath="{.status.conditions}")
+    [ "$condition_json" = "" ] && retry "conditions not found" && exit 1
+    { check_containers_ready "$condition_json"; } || {
+      retry "containers not ready"
+      exit 1
+    }
+
+    scrape_metrics=$(kubectl exec "${pod}" -c collector -n "${NAMESPACE}" -- \
+      curl --insecure --silent http://localhost:9187/metrics)
+    { contains "${scrape_metrics}" 'ccp_table_size_bytes{dbname="pikachu"'; } || {
+      retry "ccp_table_size_bytes not found for pikachu"
+      exit 1
+    }
+    { ! contains "${scrape_metrics}" 'ccp_table_size_bytes{dbname="onix"'; } || {
+      retry "ccp_table_size_bytes found for onix"
+      exit 1
+    }

testing/kuttl/e2e/otel-logging-and-metrics/12-assert-per-db-queries.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- files/13--add-per-db-metrics.yaml

testing/kuttl/e2e/otel-logging-and-metrics/13--add-second-per-db-metrics-to-postgrescluster.yaml

@@ -0,0 +1,32 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+# First, check that all containers in the instance pod are ready.
+# Then, grab the collector metrics output and check that the per-db metrics
+# are present for both added targets.
+- script: |
+    retry() { bash -ceu 'printf "$1\nSleeping...\n" && sleep 5' - "$@"; }
+    check_containers_ready() { bash -ceu 'echo "$1" | jq -e ".[] | select(.type==\"ContainersReady\") | .status==\"True\""' - "$@"; }
+    contains() { bash -ceu '[[ "$1" == *"$2"* ]]' - "$@"; }
+
+    pod=$(kubectl get pods -o name -n "${NAMESPACE}" \
+      -l postgres-operator.crunchydata.com/cluster=otel-cluster,postgres-operator.crunchydata.com/data=postgres)
+    [ "$pod" = "" ] && retry "Pod not found" && exit 1
+
+    condition_json=$(kubectl get "${pod}" -n "${NAMESPACE}" -o jsonpath="{.status.conditions}")
+    [ "$condition_json" = "" ] && retry "conditions not found" && exit 1
+    { check_containers_ready "$condition_json"; } || {
+      retry "containers not ready"
+      exit 1
+    }
+
+    scrape_metrics=$(kubectl exec "${pod}" -c collector -n "${NAMESPACE}" -- \
+      curl --insecure --silent http://localhost:9187/metrics)
+    { contains "${scrape_metrics}" 'ccp_table_size_bytes{dbname="pikachu"'; } || {
+      retry "ccp_table_size_bytes not found for pikachu"
+      exit 1
+    }
+    { contains "${scrape_metrics}" 'ccp_table_size_bytes{dbname="onix"'; } || {
+      retry "ccp_table_size_bytes not found for onix"
+      exit 1
+    }

testing/kuttl/e2e/otel-logging-and-metrics/13--cluster-no-backups.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- files/15--remove-per-db-metrics.yaml

testing/kuttl/e2e/otel-logging-and-metrics/14-assert-per-db-queries-for-multiple-targets.yaml

@@ -0,0 +1,32 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+# First, check that all containers in the instance pod are ready.
+# Then, grab the collector metrics output and check that the per-db metrics
+# are absent from the targets since they've been removed.
+- script: |
+    retry() { bash -ceu 'printf "$1\nSleeping...\n" && sleep 5' - "$@"; }
+    check_containers_ready() { bash -ceu 'echo "$1" | jq -e ".[] | select(.type==\"ContainersReady\") | .status==\"True\""' - "$@"; }
+    contains() { bash -ceu '[[ "$1" == *"$2"* ]]' - "$@"; }
+
+    pod=$(kubectl get pods -o name -n "${NAMESPACE}" \
+      -l postgres-operator.crunchydata.com/cluster=otel-cluster,postgres-operator.crunchydata.com/data=postgres)
+    [ "$pod" = "" ] && retry "Pod not found" && exit 1
+
+    condition_json=$(kubectl get "${pod}" -n "${NAMESPACE}" -o jsonpath="{.status.conditions}")
+    [ "$condition_json" = "" ] && retry "conditions not found" && exit 1
+    { check_containers_ready "$condition_json"; } || {
+      retry "containers not ready"
+      exit 1
+    }
+
+    scrape_metrics=$(kubectl exec "${pod}" -c collector -n "${NAMESPACE}" -- \
+      curl --insecure --silent http://localhost:9187/metrics)
+    { ! contains "${scrape_metrics}" 'ccp_table_size_bytes{dbname="pikachu"'; } || {
+      retry "ccp_table_size_bytes found for pikachu"
+      exit 1
+    }
+    { ! contains "${scrape_metrics}" 'ccp_table_size_bytes{dbname="onix"'; } || {
+      retry "ccp_table_size_bytes found for onix"
+      exit 1
+    }

testing/kuttl/e2e/otel-logging-and-metrics/15--remove-per-db-metrics-from-postgrescluster.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- files/17--add-custom-queries-per-db.yaml
+assert:
+- files/17-custom-queries-per-db-added.yaml

testing/kuttl/e2e/otel-logging-and-metrics/16-assert-per-db-query-removed.yaml

@@ -0,0 +1,42 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+# First, check that all containers in the instance pod are ready.
+# Then, grab the collector metrics output and check that the two metrics that we
+# checked for earlier are no longer there.
+# Then, check that the two custom metrics that we added are present
+# only for the targets that were specified.
+- script: |
+    retry() { bash -ceu 'printf "$1\nSleeping...\n" && sleep 5' - "$@"; }
+    check_containers_ready() { bash -ceu 'echo "$1" | jq -e ".[] | select(.type==\"ContainersReady\") | .status==\"True\""' - "$@"; }
+    contains() { bash -ceu '[[ "$1" == *"$2"* ]]' - "$@"; }
+
+    pod=$(kubectl get pods -o name -n "${NAMESPACE}" \
+      -l postgres-operator.crunchydata.com/cluster=otel-cluster,postgres-operator.crunchydata.com/data=postgres)
+    [ "$pod" = "" ] && retry "Pod not found" && exit 1
+
+    condition_json=$(kubectl get "${pod}" -n "${NAMESPACE}" -o jsonpath="{.status.conditions}")
+    [ "$condition_json" = "" ] && retry "conditions not found" && exit 1
+    { check_containers_ready "$condition_json"; } || {
+      retry "containers not ready"
+      exit 1
+    }
+
+    scrape_metrics=$(kubectl exec "${pod}" -c collector -n "${NAMESPACE}" -- \
+      curl --insecure --silent http://localhost:9187/metrics)
+    { contains "${scrape_metrics}" 'ccp_table_size_bytes_1{dbname="pikachu"'; } || {
+      retry "custom metric not found for pikachu db"
+      exit 1
+    }
+    { ! contains "${scrape_metrics}" 'ccp_table_size_bytes_1{dbname="onix"'; } || {
+      retry "custom metric found for onix db"
+      exit 1
+    }
+    { contains "${scrape_metrics}" 'ccp_table_size_bytes_2{dbname="onix"'; } || {
+      retry "custom metric not found for onix db"
+      exit 1
+    }
+    { ! contains "${scrape_metrics}" 'ccp_table_size_bytes_2{dbname="pikachu"'; } || {
+      retry "custom metric found for pikachu db"
+      exit 1
+    }