Move pgbackrest-restore test to Kyverno Chainsaw

This test has a number of scripts and jobs that pass and share data.
Chainsaw's bindings and templates are a nice way to break this test up,
and its "catch" operations provide good context when a step fails.

Tested with Kyverno Chainsaw 0.2.12

See: https://kyverno.github.io/chainsaw/main

Author: cbandy

.github/workflows/test.yaml

@@ -144,10 +144,12 @@ jobs:
             --env 'RELATED_IMAGE_COLLECTOR=registry.developers.crunchydata.com/crunchydata/postgres-operator:ubi9-5.8.2-0' \
             --env 'PGO_FEATURE_GATES=TablespaceVolumes=true,OpenTelemetryLogs=true,OpenTelemetryMetrics=true' \
             --name 'postgres-operator' localhost/postgres-operator
-      - name: Install kuttl
-        run: |
-          curl -Lo /usr/local/bin/kubectl-kuttl https://github.com/kudobuilder/kuttl/releases/download/v0.13.0/kubectl-kuttl_0.13.0_linux_x86_64
-          chmod +x /usr/local/bin/kubectl-kuttl
+
+      - run: |
+          make check-chainsaw && exit
+          failed=$?
+          echo '::group::PGO logs'; docker logs 'postgres-operator'; echo '::endgroup::'
+          exit $failed
 
       - run: make generate-kuttl
         env:
@@ -161,8 +163,6 @@ jobs:
           failed=$?
           echo '::group::PGO logs'; docker logs 'postgres-operator'; echo '::endgroup::'
           exit $failed
-        env:
-          KUTTL: kubectl-kuttl
 
       - name: Stop PGO
         run: docker stop 'postgres-operator' || true

Makefile

@@ -12,6 +12,8 @@ GO_TEST ?= $(GO) test
 CONTROLLER ?= $(GO) tool sigs.k8s.io/controller-tools/cmd/controller-gen
 
 # Run tests using the latest tools.
+CHAINSAW ?= $(GO) run github.com/kyverno/chainsaw@latest
+CHAINSAW_TEST ?= $(CHAINSAW) test
 ENVTEST ?= $(GO) run sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
 KUTTL ?= $(GO) run github.com/kudobuilder/kuttl/cmd/kubectl-kuttl@latest
 KUTTL_TEST ?= $(KUTTL) test
@@ -170,6 +172,17 @@ check-envtest-existing: createnamespaces
 		$(GO_TEST) -count=1 -cover -p=1 ./...
 	kubectl delete -k ./config/dev
 
+# Expects operator to be running
+#
+# Chainsaw runs with a single kubectl context named "chainsaw".
+# If you experience `cluster "minikube" does not exist`, try `MINIKUBE_PROFILE=chainsaw`.
+#
+# https://kyverno.github.io/chainsaw/latest/operations/script#kubeconfig
+#
+.PHONY: check-chainsaw
+check-chainsaw:
+	$(CHAINSAW_TEST) --config testing/chainsaw/e2e/config.yaml --values testing/chainsaw/e2e/values.yaml testing/chainsaw/e2e
+
 # Expects operator to be running
 #
 # KUTTL runs with a single kubectl context named "cluster".

testing/chainsaw/e2e/config.yaml

@@ -0,0 +1,11 @@
+apiVersion: chainsaw.kyverno.io/v1alpha2
+kind: Configuration
+metadata:
+  name: end-to-end
+spec:
+  namespace:
+    template:
+      metadata:
+        labels: { postgres-operator-test: chainsaw }
+  timeouts:
+    assert: 3m

testing/chainsaw/e2e/pgbackrest-restore/01-create-cluster.yaml

@@ -0,0 +1,55 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: 01-create-cluster
+spec:
+  try:
+    -
+      description: >
+        Create a cluster with a single pgBackRest repository
+        and some parameters that require attention during PostgreSQL recovery
+      apply:
+        resource:
+          apiVersion: postgres-operator.crunchydata.com/v1beta1
+          kind: PostgresCluster
+          metadata:
+            name: original
+          spec:
+            postgresVersion: ($postgres.version)
+            config:
+              parameters:
+                archive_timeout: 15
+                checkpoint_timeout: 30
+                max_connections: 200
+            instances:
+              - dataVolumeClaimSpec: ($volume)
+                replicas: 2
+            backups:
+              pgbackrest:
+                manual:
+                  repoName: repo1
+                repos:
+                  - name: repo1
+                    volume:
+                      volumeClaimSpec: ($volume)
+
+    -
+      description: >
+        Wait for the replica backup to complete
+      assert:
+        resource:
+          apiVersion: postgres-operator.crunchydata.com/v1beta1
+          kind: PostgresCluster
+          metadata:
+            name: original
+          status:
+            pgbackrest:
+              repos:
+              - name: repo1
+                replicaCreateBackupComplete: true
+                stanzaCreated: true
+
+  catch:
+    - podLogs:
+        selector: postgres-operator.crunchydata.com/cluster in (original)
+        tail: 50

testing/chainsaw/e2e/pgbackrest-restore/02-create-data.yaml

@@ -0,0 +1,52 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: 02-create-data
+spec:
+  try:
+    -
+      description: >
+        Create some data that will be restored
+      apply:
+        resource:
+          apiVersion: batch/v1
+          kind: Job
+          metadata:
+            name: original-data
+          spec:
+            backoffLimit: 3
+            template:
+              spec:
+                restartPolicy: Never
+                containers:
+                  - name: psql
+                    image: ($psql.image)
+                    env:
+                      - ($psql.connect)
+                      - name: PGURI
+                        valueFrom: { secretKeyRef: { name: original-pguser-original, key: uri } }
+                    command:
+                      - psql
+                      - $(PGURI)
+                      - --set=ON_ERROR_STOP=1
+                      - --command
+                      - |
+                        CREATE SCHEMA IF NOT EXISTS "original";
+                        CREATE TABLE important (data) AS VALUES ('treasure');
+
+    - assert:
+        resource:
+          apiVersion: batch/v1
+          kind: Job
+          metadata:
+            name: original-data
+          status:
+            succeeded: 1
+
+  catch:
+    -
+      description: >
+        Read all log lines from the job pods
+      podLogs:
+        selector: batch.kubernetes.io/job-name in (original-data)
+        tail: -1

testing/chainsaw/e2e/pgbackrest-restore/03-create-backup.yaml

@@ -0,0 +1,33 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: 03-create-backup
+spec:
+  try:
+    -
+      description: >
+        Annotate the cluster to trigger a backup
+      patch:
+        resource:
+          apiVersion: postgres-operator.crunchydata.com/v1beta1
+          kind: PostgresCluster
+          metadata:
+            name: original
+            annotations:
+              postgres-operator.crunchydata.com/pgbackrest-backup: one
+
+    -
+      description: >
+        Wait for it to complete
+      assert:
+        resource:
+          apiVersion: batch/v1
+          kind: Job
+          metadata:
+            annotations:
+              postgres-operator.crunchydata.com/pgbackrest-backup: one
+            labels:
+              postgres-operator.crunchydata.com/cluster: original
+              postgres-operator.crunchydata.com/pgbackrest-backup: manual
+          status:
+            succeeded: 1

testing/chainsaw/e2e/pgbackrest-restore/11-update-cluster.yaml

@@ -0,0 +1,83 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: 11-update-cluster
+spec:
+  try:
+    -
+      description: >
+        Find the primary pod
+      command:
+        outputs:
+          - name: primary
+            value: (trim_space($stdout))
+        entrypoint: kubectl
+        args:
+          - get
+          - pod
+          - --namespace=${NAMESPACE}
+          - --output=name
+          - --selector
+          - >-
+            postgres-operator.crunchydata.com/cluster=original,
+            postgres-operator.crunchydata.com/role=master
+
+    -
+      description: >
+        Read the timestamp at which PostgreSQL last started
+      command:
+        outputs:
+          - name: start_before
+            value: (trim_space($stdout))
+        env:
+          - name: PRIMARY
+            value: ($primary)
+        entrypoint: kubectl
+        args:
+          - exec
+          - --namespace=${NAMESPACE}
+          - ${PRIMARY}
+          - --
+          - psql
+          - -qAt
+          - --command
+          - SELECT pg_postmaster_start_time()
+
+    -
+      description: >
+        Update the cluster with parameters that require attention during recovery
+      patch:
+        resource:
+          apiVersion: postgres-operator.crunchydata.com/v1beta1
+          kind: PostgresCluster
+          metadata:
+            name: original
+          spec:
+            config:
+              parameters:
+                max_connections: 1000
+
+    -
+      description: >
+        Wait for Postgres to restart
+      script:
+        skipCommandOutput: true
+        timeout: 30s
+        env:
+          - name: BEFORE
+            value: ($start_before)
+          - name: PRIMARY
+            value: ($primary)
+        content: |
+          while true; do
+            START=$(
+              kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \
+                -- psql -qAt --command 'SELECT pg_postmaster_start_time()'
+            )
+            if [ "${START}" ] && [ "${START}" != "${BEFORE}" ]; then break; else sleep 1; fi
+          done
+          echo "${START} != ${BEFORE}"
+
+          # Reset counters in the "pg_stat_archiver" view.
+          kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \
+            -- psql -qb --command "SELECT pg_stat_reset_shared('archiver')" --output /dev/null

testing/chainsaw/e2e/pgbackrest-restore/12-update-data.yaml

@@ -0,0 +1,74 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: 12-update-data
+spec:
+  try:
+    -
+      description: >
+        Add more data to the WAL archive
+      apply:
+        resource:
+          apiVersion: batch/v1
+          kind: Job
+          metadata:
+            name: original-more-data
+          spec:
+            backoffLimit: 3
+            template:
+              spec:
+                restartPolicy: Never
+                containers:
+                  - name: psql
+                    image: ($psql.image)
+                    env:
+                      - ($psql.connect)
+                      - name: PGURI
+                        valueFrom: { secretKeyRef: { name: original-pguser-original, key: uri } }
+
+                    command:
+                      - psql
+                      - $(PGURI)
+                      - --set=ON_ERROR_STOP=1
+                      - --command
+                      - |
+                        INSERT INTO important (data) VALUES ('water'), ('socks');
+
+    - assert:
+        resource:
+          apiVersion: batch/v1
+          kind: Job
+          metadata:
+            name: original-more-data
+          status:
+            succeeded: 1
+
+    -
+      description: >
+        Wait for the data to be sent to the WAL archive
+      script:
+        skipCommandOutput: true
+        content: |
+          PRIMARY=$(
+            kubectl get pod --namespace "${NAMESPACE}" \
+              --output name --selector '
+                postgres-operator.crunchydata.com/cluster=original,
+                postgres-operator.crunchydata.com/role=master'
+          )
+
+          kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \
+            -- psql --command 'SELECT pg_switch_wal()' --pset footer=off
+
+          # A prior step reset the "pg_stat_archiver" counters, so anything more than zero should suffice.
+          while [ 0 = "$(
+            kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \
+              -- psql -qAt --command 'SELECT archived_count FROM pg_stat_archiver'
+          )" ]; do sleep 1; done
+
+  catch:
+    -
+      description: >
+        Read all log lines from the job pods
+      podLogs:
+        selector: batch.kubernetes.io/job-name in (original-more-data)
+        tail: -1