Add Patroni failsafe_mode parameter (#2076)

This commit adds support of a not-yet-released Patroni feature that allows postgres to run as primary in case of a failed leader lock update.
* Add Patroni 'failsafe_mode' local parameter (enable for a single PG cluster)
* Allow configuring Patroni 'failsafe_mode' parameter globally

Author: hughcapet

charts/postgres-operator/crds/operatorconfigurations.yaml

@@ -633,6 +633,12 @@ spec:
                     type: string
                     pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
                     default: "100Mi"
+              patroni:
+                type: object
+                properties:
+                  failsafe_mode:
+                    type: boolean
+                    default: false
           status:
             type: object
             additionalProperties:

charts/postgres-operator/crds/postgresqls.yaml

@@ -320,6 +320,8 @@ spec:
               patroni:
                 type: object
                 properties:
+                  failsafe_mode:
+                    type: boolean
                   initdb:
                     type: object
                     additionalProperties:

charts/postgres-operator/values.yaml

@@ -409,6 +409,10 @@ configConnectionPooler:
   connection_pooler_default_cpu_limit: "1"
   connection_pooler_default_memory_limit: 100Mi
 
+configPatroni:
+  # enable Patroni DCS failsafe_mode feature
+  failsafe_mode: false
+
 # Zalando's internal CDC stream feature
 enableStreams: false
 

docs/reference/cluster_manifest.md

@@ -316,6 +316,9 @@ explanation of `ttl` and `loop_wait` parameters.
 
 * **synchronous_node_count**
   Patroni `synchronous_node_count` parameter value. Note, this option is only available for Spilo images with Patroni 2.0+. The default is set to `1`. Optional.
+
+* **failsafe_mode**
+  Patroni `failsafe_mode` parameter value. If enabled, allows Patroni to cope with DCS outages and avoid leader demotion. Note, this option is currently not included in any Patroni release. The default is set to `false`. Optional.
 
 ## Postgres container resources
 

e2e/tests/test_e2e.py

@@ -407,7 +407,8 @@ def test_config_update(self):
                     "ttl": 29,
                     "loop_wait": 9,
                     "retry_timeout": 9,
-                    "synchronous_mode": True
+                    "synchronous_mode": True,
+                    "failsafe_mode": True,
                  }
             }
         }
@@ -434,6 +435,8 @@ def compare_config():
                             "retry_timeout not updated")
                 self.assertEqual(desired_config["synchronous_mode"], effective_config["synchronous_mode"],
                             "synchronous_mode not updated")
+                self.assertEqual(desired_config["failsafe_mode"], effective_config["failsafe_mode"],
+                            "failsafe_mode not updated")
                 return True
 
             # check if Patroni config has been updated

manifests/complete-postgres-manifest.yaml

@@ -109,6 +109,7 @@ spec:
       cpu: 500m
       memory: 500Mi
   patroni:
+    failsafe_mode: false
     initdb:
       encoding: "UTF8"
       locale: "en_US.UTF-8"

manifests/configmap.yaml

@@ -47,6 +47,7 @@ data:
   enable_master_load_balancer: "false"
   enable_master_pooler_load_balancer: "false"
   enable_password_rotation: "false"
+  # enable_patroni_failsafe_mode: "false"
   enable_pgversion_env_var: "true"
   # enable_pod_antiaffinity: "false"
   # enable_pod_disruption_budget: "true"

manifests/operatorconfiguration.crd.yaml

@@ -631,6 +631,12 @@ spec:
                     type: string
                     pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$'
                     default: "100Mi"
+              patroni:
+                type: object
+                properties:
+                  failsafe_mode:
+                    type: boolean
+                    default: false
           status:
             type: object
             additionalProperties:

manifests/postgresql-operator-default-configuration.yaml

@@ -198,3 +198,5 @@ configuration:
     connection_pooler_number_of_instances: 2
     # connection_pooler_schema: "pooler"
     # connection_pooler_user: "pooler"
+  patroni:
+    # failsafe_mode: "false"

manifests/postgresql.crd.yaml

@@ -318,6 +318,8 @@ spec:
               patroni:
                 type: object
                 properties:
+                  failsafe_mode:
+                    type: boolean
                   initdb:
                     type: object
                     additionalProperties: