feat: Added support for ConfigMap reference-based queryables configuration (#360)

Author: emmanuelmathot

CHANGELOG.md

@@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added support for ConfigMap reference-based queryables configuration in addition to file-based queryables. Queryables can now be sourced from external ConfigMaps using `configMapRef`, from chart files using `file`, or a combination of both [#360](https://github.com/developmentseed/eoapi-k8s/pull/360)
+
 ## Changed
 
 - Unified scripts and removed Makefile, combined all into one CLI command `eoapi-cli` [#359](https://github.com/developmentseed/eoapi-k8s/pull/359)
@@ -42,7 +44,7 @@ stac:
 ### Added
 
 - Exposed PgSTAC configuration options in Helm chart values (`pgstacBootstrap.settings.pgstacSettings`). These are dynamically applied via templated SQL during bootstrap. [#340](https://github.com/developmentseed/eoapi-k8s/pull/340)
-  - Added `queue_timeout`, `use_queue`, and `update_collection_extent` settings for database performance tuning
+- Added `queue_timeout`, `use_queue`, and `update_collection_extent` settings for database performance tuning
   - Made existing context settings configurable (`context`, `context_estimated_count`, `context_estimated_cost`, `context_stats_ttl`)
   - Automatic queue processor CronJob created when `use_queue` is "true" (configurable schedule via `queueProcessor.schedule`)
   - Automatic extent updater CronJob created when `update_collection_extent` is "false" (configurable schedule via `extentUpdater.schedule`)

charts/eoapi/initdb-data/queryables/test-queryables.json

@@ -1,10 +1,39 @@
 {
-  "$schema": "https://json-schema.org/draft/2019-09/schema",
-  "$id": "https://stac-extensions.github.io/item-search/v1.0.0/schema.json",
-  "title": "Test Queryables",
-  "description": "Test queryables for eoapi",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "http://localhost/stac/queryables",
+  "title": "STAC Queryables.",
   "type": "object",
   "properties": {
+    "id": {
+      "$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id",
+      "title": "Item ID",
+      "description": "Item identifier"
+    },
+    "datetime": {
+      "type": "string",
+      "title": "Acquired",
+      "format": "date-time",
+      "pattern": "(\\+00:00|Z)$",
+      "description": "Datetime"
+    },
+    "geometry": {
+      "$ref": "https://geojson.org/schema/Feature.json",
+      "title": "Item Geometry",
+      "description": "Item Geometry"
+    },
+    "platform": {
+      "description": "Platform or satellite name",
+      "type": "string",
+      "title": "Platform"
+    },
+    "instruments": {
+      "description": "Instrument(s) used",
+      "type": "array",
+      "title": "Instruments",
+      "items": {
+        "type": "string"
+      }
+    },
     "eo:cloud_cover": {
       "description": "Estimate of cloud cover as a percentage (0-100) of the entire scene",
       "type": "number",
@@ -25,19 +54,6 @@
       "title": "Sun Elevation",
       "minimum": -90,
       "maximum": 90
-    },
-    "platform": {
-      "description": "Platform or satellite name",
-      "type": "string",
-      "title": "Platform"
-    },
-    "instruments": {
-      "description": "Instrument(s) used",
-      "type": "array",
-      "title": "Instruments",
-      "items": {
-        "type": "string"
-      }
     }
   },
   "additionalProperties": true

charts/eoapi/profiles/experimental.yaml

@@ -68,7 +68,8 @@ pgstacBootstrap:
 
     # Queryables configuration for testing
     queryables:
-      - file: "initdb-data/queryables/test-queryables.json"
+      - name: test-queryables.json
+        file: "initdb-data/queryables/test-queryables.json"
         indexFields: ["platform", "instruments"]
         deleteMissing: true
 

charts/eoapi/templates/database/pgstacbootstrap/configmap.yaml

@@ -48,6 +48,13 @@ data:
 {{- end }}
 ---
 {{- if .Values.pgstacBootstrap.settings.queryables }}
+{{- $hasFileBasedQueryables := false }}
+{{- range $config := .Values.pgstacBootstrap.settings.queryables }}
+  {{- if $config.file }}
+    {{- $hasFileBasedQueryables = true }}
+  {{- end }}
+{{- end }}
+{{- if $hasFileBasedQueryables }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -58,10 +65,12 @@ metadata:
     helm.sh/hook-delete-policy: "before-hook-creation,hook-succeeded"
 data:
   {{- range $config := .Values.pgstacBootstrap.settings.queryables }}
-  {{- $filename := splitList "/" $config.file | last }}
-  {{ $filename }}: |
+  {{- if $config.file }}
+  {{ $config.name }}: |
     {{- $.Files.Get $config.file | nindent 4 }}
   {{- end }}
+  {{- end }}
+{{- end }}
 {{- end }}
 {{- end }}
 ---

charts/eoapi/templates/database/pgstacbootstrap/job.yaml

@@ -197,15 +197,14 @@ spec:
               # Load queryables configurations
               echo "Loading queryables configurations..."
               {{- range $idx, $config := .Values.pgstacBootstrap.settings.queryables }}
-              {{- $filename := splitList "/" $config.file | last }}
-              echo "Processing queryables file: {{ $filename }}"
+              echo "Processing queryables: {{ $config.name }}"
               pypgstac load-queryables \
               {{- if $config.deleteMissing }}
                 --delete-missing \
               {{- end }}
               {{- if $config.collections }}
                 # Complex escaping needed due to multiple interpretation layers:
-                # 1. Helm processes the template: \\\" becomes \" in the rendered YAML
+                # 1. Helm processes the template: \\\\\" becomes \" in the rendered YAML
                 # 2. YAML processes the string: \" becomes " in the shell script
                 # 3. Shell receives: --collection-ids '["collection1","collection2"]'
                 # 4. pypgstac gets a proper JSON array string as expected
@@ -214,21 +213,48 @@ spec:
               {{- if $config.indexFields }}
                 --index-fields {{ join "," $config.indexFields }} \
               {{- end }}
-                "/opt/queryables/{{ $filename }}"
+                "/opt/queryables/{{ $config.name }}"
               {{- end }}
 
               echo "Queryables loading complete"
           resources:
             {{- toYaml .Values.pgstacBootstrap.settings.resources | nindent 12 }}
           volumeMounts:
-            - mountPath: /opt/queryables
-              name: {{ .Release.Name }}-queryables-volume
+            {{- range $idx, $config := .Values.pgstacBootstrap.settings.queryables }}
+            {{- if $config.file }}
+            - mountPath: /opt/queryables/{{ $config.name }}
+              name: {{ $.Release.Name }}-queryables-volume
+              subPath: {{ $config.name }}
+            {{- else if $config.configMapRef }}
+            - mountPath: /opt/queryables/{{ $config.name }}
+              name: {{ $.Release.Name }}-queryables-configmapref-{{ $idx }}
+              subPath: {{ $config.configMapRef.key }}
+            {{- end }}
+            {{- end }}
           env:
             {{- include "eoapi.postgresqlEnv" . | nindent 12 }}
       volumes:
+        {{- $hasFileBasedQueryables := false }}
+        {{- range $config := .Values.pgstacBootstrap.settings.queryables }}
+          {{- if $config.file }}
+            {{- $hasFileBasedQueryables = true }}
+          {{- end }}
+        {{- end }}
+        {{- if $hasFileBasedQueryables }}
         - name: {{ .Release.Name }}-queryables-volume
           configMap:
             name: {{ .Release.Name }}-pgstac-queryables-config
+        {{- end }}
+        {{- range $idx, $config := .Values.pgstacBootstrap.settings.queryables }}
+          {{- if $config.configMapRef }}
+        - name: {{ $.Release.Name }}-queryables-configmapref-{{ $idx }}
+          configMap:
+            name: {{ $config.configMapRef.name }}
+            items:
+              - key: {{ $config.configMapRef.key }}
+                path: {{ $config.configMapRef.key }}
+          {{- end }}
+        {{- end }}
       {{- with .Values.pgstacBootstrap.settings.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}