feat: improved multi-cluster handling, added thanos storage manifests

updated docs as well

Signed-off-by: Ali Mukadam <ali.mukadam@oracle.com>

Author: hyder

admin.tf

@@ -103,5 +103,5 @@ resource "oci_objectstorage_bucket" "thanos_admin" {
 
   provider = oci.sydney
 
-  count = lookup(var.thanos, "enabled", "false") ? 1 : 0
+  count = tobool(lookup(var.thanos, "enabled", "false")) ? 1 : 0
 }

docs/src/advanced/thanos.md

@@ -34,7 +34,7 @@ When the above is configured, they will be generated and added to the Custom Res
 1. For each cluster, use the following to configure your user principal authentication for Thanos:
 
 ``` yaml, editable
-{{#include ../../../modules/verrazzano/resources/thanos-storage.yaml.example:4:}}
+{{#include ../../../modules/verrazzano/resources/thanos-storage.yaml.example:1:}}
 ```
 2. Save the file as storage.yaml
 
@@ -54,7 +54,6 @@ TODO
 Before enabling Thanos, ensure the following secret is created:
 
 ``` bash
-
 kubectl create namespace verrazzano-monitoring
 kubectl create secret generic objstore-config -n verrazzano-monitoring --from-file=objstore.yml=storage.yaml
 ```

docs/src/multi/multi.md

@@ -9,6 +9,4 @@ whereas a managed cluster typically consists only of the application workload, t
 ![Managed Cluster](../images/managedcluster.svg)
 
   - [With Public Endpoints](./pub-ep.md)
-  - [With Private Endpoints](./pri-ep.md)
-  - [With Public Load Balancers](./pub-lb.md)
-  - [With Private Load Balancers](./pri-lb.md)
+  - [With Private Endpoints](./pri-ep.md)

docs/src/multi/pri-ep.md

@@ -335,7 +335,26 @@ for cluster in admin phoenix ; do
 done
 ```
 
-5. Install the Verrazzano Admin cluster:
+5. If you are using Thanos, configure your secret for OCI Object Storage now. For each enabled cluster, a storage file is created. Edit them and enter the following and create a secret:
+
+  - compartment_ocid
+  - region
+  - tenancy_ocid
+  - user_ocid
+  - fingerprint
+  - and the private key
+
+```
+cd /home/opc/vz/clusters
+for cluster in admin phoenix; do
+  kubectx $cluster
+  kubectl create namespace verrazzano-monitoring
+  kubectl create secret generic objstore-config -n verrazzano-monitoring --from-file=objstore.yml=thanos_$cluster_storage.yaml
+done
+
+```  
+
+6. Install the Verrazzano Admin cluster:
 
 ```
 cd /home/opc/vz/clusters
@@ -345,7 +364,7 @@ bash install_vz_cluster_admin.sh
 The Admin cluster has more components to install and takes longer, so we install it separately. This allows us to install the managed clusters in parallel.
 ```
 
-6. While the Admin cluster is being installed in the background, you can install the managed clusters in parallel:
+7. While the Admin cluster is being installed in the background, you can install the managed clusters in parallel:
 
 ```bash, editable
 cd /home/opc/vz/clusters
@@ -354,7 +373,7 @@ for cluster in phoenix ; do
 done
 ```
 
-7. Wait for Verrazzano to be installed in all clusters:
+8. Wait for Verrazzano to be installed in all clusters:
 
 ```
 # check managed clusters' status
@@ -365,7 +384,7 @@ kubectx admin
 kubectl wait --timeout=20m --for=condition=InstallComplete verrazzano/admin
 ```
 
-8. Create the certificates secrets for each managed cluster:
+9. Create the certificates secrets for each managed cluster:
 
 ```bash, editable
 cd /home/opc/vz/certs
@@ -374,14 +393,14 @@ for cluster in phoenix; do
 done
 ```
 
-9. Create the ConfigMap for the API Server:
+10. Create the ConfigMap for the API Server:
 
 ```
 cd /home/opc/vz/cm
 bash create_api_cm.sh
 ```
 
-10. Create the Verrazzano managed cluster objects for each managed cluster:    
+11. Create the Verrazzano managed cluster objects for each managed cluster:    
 
 ```bash, editable
 cd /home/opc/vz/clusters
@@ -390,7 +409,7 @@ for cluster in phoenix; do
 done
 ```
 
-11. Register all the managed clusters:
+12. Register all the managed clusters:
 
 ```bash, editable
 for cluster in phoenix; do

docs/src/multi/pub-ep.md

@@ -295,7 +295,7 @@ Replace phoenix with a list of your clusters e.g.
 `for cluster in admin sanjose chicago ;...`
 ```
 
-2. Check if the operator has installed sucessfully in all clusters:
+2. Check if the operator has installed successfully in all clusters:
 
 ```bash, editable
 for cluster in admin phoenix; do
@@ -319,7 +319,26 @@ for cluster in admin phoenix ; do
 done
 ```
 
-5. Install the Verrazzano Admin cluster:
+5. If you are using Thanos, configure your secret for OCI Object Storage now. For each enabled cluster, a storage file is created. Edit them and enter the following and create a secret:
+
+  - compartment_ocid
+  - region
+  - tenancy_ocid
+  - user_ocid
+  - fingerprint
+  - and the private key
+
+```
+cd /home/opc/vz/clusters
+for cluster in admin phoenix; do
+  kubectx $cluster
+  kubectl create namespace verrazzano-monitoring
+  kubectl create secret generic objstore-config -n verrazzano-monitoring --from-file=objstore.yml=thanos_$cluster_storage.yaml
+done
+
+```  
+
+6. Install the Verrazzano Admin cluster:
 
 ```
 cd /home/opc/vz/clusters
@@ -329,7 +348,7 @@ bash install_vz_cluster_admin.sh
 The Admin cluster has more components to install and takes longer, so we install it separately. This allows us to install the managed clusters in parallel.
 ```
 
-6. While the Admin cluster is being installed in the background, you can install the managed clusters in parallel:
+7. While the Admin cluster is being installed in the background, you can install the managed clusters in parallel:
 
 ```bash, editable
 cd /home/opc/vz/clusters
@@ -338,7 +357,7 @@ for cluster in phoenix ; do
 done
 ```
 
-7. Wait for Verrazzano to be installed in all clusters:
+8. Wait for Verrazzano to be installed in all clusters:
 
 ```
 # check managed clusters' status
@@ -349,7 +368,7 @@ kubectx admin
 kubectl wait --timeout=20m --for=condition=InstallComplete verrazzano/admin
 ```
 
-8. Create the certificates secrets for each managed cluster:
+9. Create the certificates secrets for each managed cluster:
 
 ```bash, editable
 cd /home/opc/vz/certs
@@ -358,14 +377,14 @@ for cluster in phoenix; do
 done
 ```
 
-9. Create the ConfigMap for the API Server:
+10. Create the ConfigMap for the API Server:
 
 ```
 cd /home/opc/vz/cm
 bash create_api_cm.sh
 ```
 
-10. Create the Verrazzano managed cluster objects for each managed cluster:    
+11. Create the Verrazzano managed cluster objects for each managed cluster:    
 
 ```bash, editable
 cd /home/opc/vz/clusters
@@ -374,7 +393,7 @@ for cluster in phoenix; do
 done
 ```
 
-11. Register all the managed clusters:
+12. Register all the managed clusters:
 
 ```bash, editable
 for cluster in phoenix; do

docs/src/single/dev.md

@@ -127,14 +127,37 @@ echo >> oci.yaml
 bash create_oci_secret_admin.sh
 ```
 
-5. Install Verrazzano:
+5. If you are using Thanos, configure your secret for OCI Object Storage now.
+
+```
+cd /home/opc/vz/clusters
+```
+
+6. Edit `thanos_admin_storage.yaml` file and enter the following parameters:
+
+  - compartment_ocid
+  - region
+  - tenancy_ocid
+  - user_ocid
+  - fingerprint
+  - and the private key
+
+7. Create the secret:
+
+```
+
+kubectl create namespace verrazzano-monitoring
+kubectl create secret generic objstore-config -n verrazzano-monitoring --from-file=objstore.yml=thanos_admin_storage.yaml
+```
+
+8. Install Verrazzano:
 
 ```
 cd /home/opc/vz/clusters
 bash install_vz_cluster_admin.sh
 ```
 
-6. Wait for Verrazzano to be installed:
+9. Wait for Verrazzano to be installed:
 
 ```
 bash vz_status.sh

docs/src/single/production.md

@@ -126,14 +126,37 @@ echo >> oci.yaml
 bash create_oci_secret_admin.sh
 ```
 
-5. Install Verrazzano:
+5. If you are using Thanos, configure your secret for OCI Object Storage now.
+
+```
+cd /home/opc/vz/clusters
+```
+
+6. Edit `thanos_admin_storage.yaml` file and enter the following parameters:
+
+  - compartment_ocid
+  - region
+  - tenancy_ocid
+  - user_ocid
+  - fingerprint
+  - and the private key
+
+7. Create the secret:
+
+```
+
+kubectl create namespace verrazzano-monitoring
+kubectl create secret generic objstore-config -n verrazzano-monitoring --from-file=objstore.yml=thanos_admin_storage.yaml
+```
+
+8. Install Verrazzano:
 
 ```
 cd /home/opc/vz/clusters
 bash install_vz_cluster_admin.sh
 ```
 
-6. Wait for Verrazzano to be installed:
+9. Wait for Verrazzano to be installed:
 
 ```
 bash vz_status.sh

locals.tf

@@ -56,4 +56,6 @@ locals {
     sanjose   = "us-sanjose-1"
     toronto   = "ca-toronto-1"
   }
+
+  managed_clusters = { for k,v in module.clusters.cluster_ids: k => v if v != null }
 }

main.tf

@@ -94,7 +94,8 @@ module "verrazzano" {
   verrazzano_control_plane = var.verrazzano_control_plane
   verrazzano_data_plane    = var.verrazzano_data_plane
   verrazzano_load_balancer = var.verrazzano_load_balancer
-  cluster_ids              = merge({ lookup(var.admin_region, "admin_name", "admin") = module.admin.cluster_id }, module.clusters.cluster_ids)
+  all_cluster_ids          = merge({ lookup(var.admin_region, "admin_name", "admin") = module.admin.cluster_id }, local.managed_clusters)
+  managed_cluster_ids      = local.managed_clusters
   int_nsg_ids              = merge({ lookup(var.admin_region, "admin_name", "admin") = lookup(module.admin.nsg_ids, "int_lb") }, module.clusters.int_nsg_ids)
   int_lb_subnet_ids        = merge({ lookup(var.admin_region, "admin_name", "admin") = lookup(module.admin.subnet_ids, "int_lb") }, module.clusters.int_lb_subnet_ids)
   pub_nsg_ids              = merge({ lookup(var.admin_region, "admin_name", "admin") = lookup(module.admin.nsg_ids, "pub_lb") }, module.clusters.pub_nsg_ids)

modules/clusters/australia.tf

@@ -94,7 +94,7 @@ resource "oci_objectstorage_bucket" "thanos_melbourne" {
 
   provider = oci.melbourne
 
-  count = tobool(lookup(var.clusters, "melbourne")) && lookup(var.thanos, "enabled", "false") ? 1 : 0
+  count = tobool(lookup(var.clusters, "melbourne", "false")) && tobool(lookup(var.thanos, "enabled", "false")) ? 1 : 0
 }
 
 module "sydney" {
@@ -181,7 +181,16 @@ module "sydney" {
     oci.home = oci.home
   }
 
-  count = tobool(lookup(var.clusters, "sydney")) ? 1 : 0
+  count = tobool(lookup(var.clusters, "sydney", )) ? 1 : 0
 
 }
 
+resource "oci_objectstorage_bucket" "thanos_sydney" {
+  compartment_id = var.compartment_id
+  name           = "syd-${lookup(var.thanos, "bucket_name", "thanos")}"
+  namespace      = lookup(var.thanos, "bucket_namespace")
+
+  provider = oci.sydney
+
+  count = tobool(lookup(var.clusters, "sydney", "false")) && tobool(lookup(var.thanos, "enabled", "false")) ? 1 : 0
+}