test: Extend the snapshot-s3 test to a complete backup and restore test

tests/templates/kuttl/snapshot-s3/01-rbac.yaml → tests/templates/kuttl/backup-restore/01-rbac.yaml

@@ -17,6 +17,13 @@ rules:
       - privileged
     verbs:
       - use
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      # The jobs creating the admin certificates need the permission to insert them into Secrets.
+      - create
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

tests/templates/kuttl/backup-restore/03-create-truststore.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: secrets.stackable.tech/v1alpha1
+kind: TrustStore
+metadata:
+  name: truststore-pem
+spec:
+  secretClassName: tls
+  format: tls-pem
+  targetKind: ConfigMap

tests/templates/kuttl/backup-restore/10-install-s3-credentials-secret.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: s3-credentials
+stringData:
+  s3.client.default.access_key: openSearchAccessKey
+  s3.client.default.secret_key: openSearchSecretKey

tests/templates/kuttl/snapshot-s3/11_minio-values.yaml.j2 → tests/templates/kuttl/backup-restore/11_minio-values.yaml

@@ -25,7 +25,7 @@ persistence:
 provisioning:
   enabled: true
   buckets:
-    - name: opensearch
+    - name: opensearch-data
   users:
     - username: integrationtest
       password: integrationtest

tests/templates/kuttl/backup-restore/20-assert.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: create-opensearch-1-admin-certificate
+status:
+  succeeded: 1
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: opensearch-1-admin-certificate

tests/templates/kuttl/backup-restore/20-create-opensearch-1-admin-certificate.yaml

@@ -0,0 +1,57 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: create-opensearch-1-admin-certificate
+spec:
+  template:
+    spec:
+      containers:
+        - name: create-opensearch-1-admin-certificate
+          image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+          command:
+            - /stackable/scripts/create-opensearch-1-admin-certificate.sh
+          volumeMounts:
+            - name: script
+              mountPath: /stackable/scripts
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            runAsNonRoot: true
+          resources:
+            requests:
+              memory: 128Mi
+              cpu: 100m
+            limits:
+              memory: 128Mi
+              cpu: 400m
+      volumes:
+        - name: script
+          configMap:
+            name: create-opensearch-1-admin-certificate-script
+            defaultMode: 0o770
+      serviceAccountName: test-service-account
+      securityContext:
+        fsGroup: 1000
+      restartPolicy: OnFailure
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: create-opensearch-1-admin-certificate-script
+data:
+  create-opensearch-1-admin-certificate.sh: |
+    #!/usr/bin/env sh
+
+    openssl req \
+      -x509 \
+      -nodes \
+      -subj=/CN=opensearch-1-admin-certificate \
+      -out=tls.crt \
+      -keyout=tls.key
+
+    kubectl create secret generic opensearch-1-admin-certificate \
+        --from-file=tls.crt \
+        --from-file=tls.key

tests/templates/kuttl/snapshot-s3/20-assert.yaml.j2 → tests/templates/kuttl/backup-restore/21-assert.yaml

@@ -6,7 +6,7 @@ timeout: 600
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: opensearch-nodes-default
+  name: opensearch-1-nodes-default
 status:
   readyReplicas: 1
   replicas: 1

tests/templates/kuttl/snapshot-s3/20-install-opensearch.yaml.j2 → tests/templates/kuttl/backup-restore/21-install-opensearch-1.yaml.j2

@@ -2,7 +2,7 @@
 apiVersion: opensearch.stackable.tech/v1alpha1
 kind: OpenSearchCluster
 metadata:
-  name: opensearch
+  name: opensearch-1
 spec:
   image:
 {% if test_scenario['values']['opensearch'].find(",") > 0 %}
@@ -33,6 +33,8 @@ spec:
         # not be created even if enough disk space would be available.
         cluster.routing.allocation.disk.threshold_enabled: "false"
         plugins.security.allow_default_init_securityindex: "true"
+        plugins.security.authcz.admin_dn: CN=opensearch-1-admin-certificate
+        plugins.security.restapi.roles_enabled: all_access
         plugins.security.ssl.transport.enabled: "true"
         plugins.security.ssl.transport.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt
         plugins.security.ssl.transport.pemkey_filepath: /stackable/opensearch/config/tls/tls.key
@@ -96,6 +98,34 @@ spec:
                 mountPath: /etc/pki/ca-trust/source/anchors/s3-ca.crt
                 subPath: tls.crt
                 readOnly: true
+          - name: init-tls
+{% if test_scenario['values']['opensearch'].find(",") > 0 %}
+            image: "{{ test_scenario['values']['opensearch'].split(',')[1] }}"
+{% else %}
+            image: oci.stackable.tech/sdp/opensearch:{{ test_scenario['values']['opensearch'].split(',')[0] }}-stackable{{ test_scenario['values']['release'] }}
+{% endif %}
+            command:
+              - /bin/bash
+              - -euxo
+              - pipefail
+              - -c
+            args:
+              - |
+                cp /stackable/opensearch/config/tls/tls.* /stackable/opensearch/config/tls-concatenated
+                cat \
+                    /stackable/opensearch/config/tls/ca.crt \
+                    /stackable/opensearch/config/tls-admin/tls.crt > \
+                    /stackable/opensearch/config/tls-concatenated/ca.crt
+            volumeMounts:
+              - name: tls
+                mountPath: /stackable/opensearch/config/tls
+                readOnly: true
+              - name: admin-certificate
+                mountPath: /stackable/opensearch/config/tls-admin
+                readOnly: true
+              - name: tls-concatenated
+                mountPath: /stackable/opensearch/config/tls-concatenated
+                readOnly: false
         containers:
           - name: opensearch
             volumeMounts:
@@ -106,7 +136,7 @@ spec:
                 mountPath: /etc/pki/java/cacerts
                 subPath: java/cacerts
                 readOnly: true
-              - name: tls
+              - name: tls-concatenated
                 mountPath: /stackable/opensearch/config/tls
                 readOnly: true
               - name: keystore
@@ -125,9 +155,13 @@ spec:
             secret:
               secretName: minio-ca-crt
               defaultMode: 0o660
+          - name: admin-certificate
+            secret:
+              secretName: opensearch-1-admin-certificate
+              defaultMode: 0o660
           - name: security-config
             secret:
-              secretName: opensearch-security-config
+              secretName: opensearch-1-security-config
               defaultMode: 0o660
           - name: system-trust-store
             emptyDir:
@@ -137,7 +171,7 @@ spec:
               volumeClaimTemplate:
                 metadata:
                   annotations:
-                    secrets.stackable.tech/scope: node,pod,service=opensearch,service=opensearch-nodes-default-headless
+                    secrets.stackable.tech/scope: node,pod,service=opensearch-1,service=opensearch-1-nodes-default,service=opensearch-1-nodes-default-headless,listener-volume=listener
                     secrets.stackable.tech/class: tls
                 spec:
                   storageClassName: secrets.stackable.tech
@@ -146,11 +180,14 @@ spec:
                   resources:
                     requests:
                       storage: "1"
+          - name: tls-concatenated
+            emptyDir:
+              sizeLimit: 1Mi
 ---
 apiVersion: v1
 kind: Secret
 metadata:
-  name: opensearch-security-config
+  name: opensearch-1-security-config
 stringData:
   action_groups.yml: |
     ---

tests/templates/kuttl/backup-restore/22-assert.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: create-testuser
+status:
+  succeeded: 1

tests/templates/kuttl/backup-restore/22-create-testuser.yaml

@@ -0,0 +1,93 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: create-testuser
+spec:
+  template:
+    spec:
+      containers:
+        - name: create-testuser
+          image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
+          command:
+            - /bin/bash
+            - -euxo
+            - pipefail
+            - -c
+          args:
+            - |
+              pip install opensearch-py==3.0.0
+              python scripts/create-testuser.py
+          env:
+            # required for pip install
+            - name: HOME
+              value: /stackable
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          volumeMounts:
+            - name: script
+              mountPath: /stackable/scripts
+            - name: tls
+              mountPath: /stackable/tls
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            runAsNonRoot: true
+          resources:
+            requests:
+              memory: 128Mi
+              cpu: 100m
+            limits:
+              memory: 128Mi
+              cpu: 400m
+      volumes:
+        - name: script
+          configMap:
+            name: create-testuser-script
+        - name: tls
+          configMap:
+            name: truststore-pem
+      serviceAccountName: test-service-account
+      securityContext:
+        fsGroup: 1000
+      restartPolicy: OnFailure
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: create-testuser-script
+data:
+  create-testuser.py: |
+    import os
+    from opensearchpy import OpenSearch
+    from opensearchpy.exceptions import RequestError
+
+    namespace = os.environ['NAMESPACE']
+
+    # Login as admin
+    client = OpenSearch(
+      http_auth=('admin', 'AJVFsGJBbpT6mChn'),
+      hosts=[{
+        'host': f'opensearch-1-nodes-default.{namespace}.svc.cluster.local',
+        'port': 9200
+      }],
+      http_compress=True,
+      use_ssl=True,
+      verify_certs=True,
+      ca_certs='/stackable/tls/ca.crt'
+    )
+
+    # Create a test user and grant all access
+    response = client.security.create_user(
+      username='testuser',
+      body={
+        'password': 'L9hUHtLVVEsrcLzZ',
+        'opendistro_security_roles': ['all_access']
+      }
+    )
+
+    print(f'Creating test user; {response=}')

tests/templates/kuttl/backup-restore/23-assert.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: create-data
+status:
+  succeeded: 1