feat: added support for Thanos and Cluster API

hyder · hyder · commit 47ab4de13053 · 2023-07-10T15:51:24.000+10:00
Signed-off-by: Ali Mukadam &lt;ali.mukadam@oracle.com&gt;
diff --git a/admin.tf b/admin.tf
@@ -95,3 +95,13 @@ module "admin" {
     oci.home = oci.home
   }
 }
+
+resource "oci_objectstorage_bucket" "thanos_admin" {
+  compartment_id = var.compartment_id
+  name           = "${lookup(var.admin_region, "admin_name")}-${lookup(var.thanos, "bucket_name", "thanos")}"
+  namespace      = lookup(var.thanos, "bucket_namespace")
+
+  provider = oci.sydney
+
+  count = lookup(var.thanos, "enabled", "false") ? 1 : 0
+}
diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md
@@ -16,4 +16,7 @@
   - [Control plane](./advanced/controlplane.md)
   - [Data plane](./advanced/dataplane.md)
   - [DNS](./advanced/dns.md)
+  - [Observability]()
+    - [Prometheus]()
+    - [Thanos](./advanced/thanos.md)
 - [Terraform Options](./terraformoptions.md)
diff --git a/docs/src/advanced/dataplane.md b/docs/src/advanced/dataplane.md
@@ -1,11 +1,11 @@
 # Data plane
 
-The data plane is where the workload are run. This is usually done as part of the service mesh (Istio). 
+The data plane is where the workloads are run. This is usually done as part of the service mesh (Istio). 
 
 Istio usually has an ingress gateway that allows incoming traffic into the mesh.
 
 On OCI, the service mesh is front-ended by an OCI Load Balancer and Istio's ingress gateway.
-the ingress gateway is front-ended by an OCI Load Balancer. As such a number of configuration options are possible depending on the use case:
+As such a number of configuration options are possible depending on the use case:
 
 1. access: the control plane can be made public or private. By default, it is public.
 2. shape: the load balancer shape can be configured including the bandwidth, security posture
diff --git a/docs/src/advanced/thanos.md b/docs/src/advanced/thanos.md
@@ -0,0 +1,62 @@
+# Thanos
+
+Verrazzano includes Thanos, an open source CNCF-projet that provides the following features:
+
+- Global Query across multiple clusters
+- Cheap, long time metrics storage using Object Storage
+- Downsampling and compaction
+- A Prometheus-compatible API
+
+## Configuring Thanos
+
+Configure the following parameters to use Thanos:
+
+``` yaml, editable
+prometheus            = true
+
+prometheus_operator   = true
+
+thanos = {
+  bucket           = "vzthanos"
+  bucket_namespace = "<replace-me>" 
+  enabled          = "true"
+  integration      = "sidecar"
+  storage_gateway  = "true"
+}  
+```
+
+When the above is configured, they will be generated and added to the Custom Resource of each Verrazzano instance.
+
+## Configuring OCI Authentication
+
+### User principal
+
+1. For each cluster, use the following to configure your user principal authentication for Thanos:
+
+``` yaml, editable
+{{#include ../../../modules/verrazzano/resources/thanos-storage.yaml.example:4:}}
+```
+2. Save the file as storage.yaml
+
+```admonish important
+If you are using multiple clusters and your clusters are in different regions, ensure:
+
+1. each cluster has its own storage configuration
+2. you replace the region value in the region parameter above
+```
+
+### Instance principal
+
+TODO
+
+## Create the secret
+
+Before enabling Thanos, ensure the following secret is created:
+
+``` bash
+
+kubectl create namespace verrazzano-monitoring
+kubectl create secret generic objstore-config -n verrazzano-monitoring --from-file=objstore.yml=storage.yaml
+```
+
+You can now enable Thanos.
diff --git a/main.tf b/main.tf
@@ -31,6 +31,8 @@ module "clusters" {
 
   nodepools = var.nodepools
 
+  thanos = var.thanos
+
   providers = {
     oci.home         = oci.home,
     oci.johannesburg = oci.johannesburg,
@@ -92,13 +94,14 @@ module "verrazzano" {
   verrazzano_control_plane = var.verrazzano_control_plane
   verrazzano_data_plane    = var.verrazzano_data_plane
   verrazzano_load_balancer = var.verrazzano_load_balancer
-  cluster_ids              = merge({ "admin" = module.admin.cluster_id }, module.clusters.cluster_ids)
-  int_nsg_ids              = merge({ "admin" = lookup(module.admin.nsg_ids, "int_lb") }, module.clusters.int_nsg_ids)
-  int_lb_subnet_ids        = merge({ "admin" = lookup(module.admin.subnet_ids, "int_lb") }, module.clusters.int_lb_subnet_ids)
-  pub_nsg_ids              = merge({ "admin" = lookup(module.admin.nsg_ids, "pub_lb") }, module.clusters.pub_nsg_ids)
+  cluster_ids              = merge({ lookup(var.admin_region, "admin_name", "admin") = module.admin.cluster_id }, module.clusters.cluster_ids)
+  int_nsg_ids              = merge({ lookup(var.admin_region, "admin_name", "admin") = lookup(module.admin.nsg_ids, "int_lb") }, module.clusters.int_nsg_ids)
+  int_lb_subnet_ids        = merge({ lookup(var.admin_region, "admin_name", "admin") = lookup(module.admin.subnet_ids, "int_lb") }, module.clusters.int_lb_subnet_ids)
+  pub_nsg_ids              = merge({ lookup(var.admin_region, "admin_name", "admin") = lookup(module.admin.nsg_ids, "pub_lb") }, module.clusters.pub_nsg_ids)
 
   # verrazzano components
   argocd                = var.argocd
+  cluster_api           = var.cluster_api
   coherence             = var.coherence
   configure_dns         = var.configure_dns
   console               = var.console
@@ -112,6 +115,7 @@ module "verrazzano" {
   prometheus            = var.prometheus
   prometheus_operator   = var.prometheus_operator
   rancher               = var.rancher
+  thanos                = var.thanos
   velero                = var.velero
   weblogic_operator     = var.weblogic_operator
 
diff --git a/modules/clusters/australia.tf b/modules/clusters/australia.tf
@@ -87,6 +87,16 @@ module "melbourne" {
 
 }
 
+resource "oci_objectstorage_bucket" "thanos_melbourne" {
+  compartment_id = var.compartment_id
+  name           = "mel-${lookup(var.thanos, "bucket_name", "thanos")}"
+  namespace      = lookup(var.thanos, "bucket_namespace")
+
+  provider = oci.melbourne
+
+  count = tobool(lookup(var.clusters, "melbourne")) && lookup(var.thanos, "enabled", "false") ? 1 : 0
+}
+
 module "sydney" {
   source  = "oracle-terraform-modules/oke/oci"
   version = "4.5.9"
@@ -152,9 +162,9 @@ module "sydney" {
 
 
   # node pools
-  kubeproxy_mode = "ipvs"  
-  node_pools = local.managed_nodepools
-  cloudinit_nodepool_common = var.cloudinit_nodepool_common  
+  kubeproxy_mode            = "ipvs"
+  node_pools                = local.managed_nodepools
+  cloudinit_nodepool_common = var.cloudinit_nodepool_common
 
   node_pool_image_type = "oke"
 
diff --git a/modules/clusters/outputs.tf b/modules/clusters/outputs.tf
@@ -143,7 +143,7 @@ output "int_lb_subnet_ids" {
     # "jeddah"       = coalesce(lookup(module.jeddah[0].subnet_ids,"int_lb"))
     # "jerusalem"    = coalesce(lookup(module.jerusalem[0].subnet_ids,"int_lb"))
     "melbourne"    = coalesce(lookup(module.melbourne[0].subnet_ids,"int_lb"))
-    "sydney"       = coalesce(lookup(module.sydney[0].subnet_ids,"int_lb"))
+    # "sydney"       = coalesce(lookup(module.sydney[0].subnet_ids,"int_lb"))
     # "santiago"     = coalesce(lookup(module.santiago[0].subnet_ids,"int_lb"))
     # "saupaulo"     = coalesce(lookup(module.saupaulo[0].subnet_ids,"int_lb"))
     # "vinhedo"      = coalesce(lookup(module.vinhedo[0].subnet_ids,"int_lb"))
diff --git a/modules/clusters/variables.tf b/modules/clusters/variables.tf
@@ -61,4 +61,8 @@ variable "nodepools" {
 
 variable "cloudinit_nodepool_common" {
   type        = string
+}
+
+variable "thanos" {
+  type        = map(string)
 }
diff --git a/modules/verrazzano/install_vz.tf b/modules/verrazzano/install_vz.tf
@@ -16,12 +16,12 @@ resource "null_resource" "install_vz_admin" {
 
   provisioner "file" {
     content     = local.vz_admin_template
-    destination = "/home/opc/vz/clusters/vz_admin.yaml"
+    destination = "/home/opc/vz/clusters/vz_${local.admin_region_name}.yaml"
   }
 
   provisioner "file" {
     content     = local.install_admin_script
-    destination = "/home/opc/vz/clusters/install_vz_cluster_admin.sh"
+    destination = "/home/opc/vz/clusters/install_vz_cluster_${local.admin_region_name}.sh"
   }
 
   provisioner "file" {
diff --git a/modules/verrazzano/resources/thanos-storage.yaml.example b/modules/verrazzano/resources/thanos-storage.yaml.example
@@ -0,0 +1,13 @@
+type: OCI
+config:
+  provider: "raw"
+  bucket: "thanos"
+  compartment_ocid: "ocid1.compartment.oc1....."
+  region: "us-ashburn-1"
+  tenancy_ocid: "ocid1.tenancy.oc1....."
+  user_ocid: "ocid1.user.oc1....."
+  fingerprint: "12:d3:4c:..."
+  privatekey: |
+    -----BEGIN RSA PRIVATE KEY-----
+    ...
+    -----END RSA PRIVATE KEY-----
diff --git a/modules/verrazzano/resources/vz_admin.template.yaml b/modules/verrazzano/resources/vz_admin.template.yaml
@@ -1,13 +1,15 @@
 apiVersion: install.verrazzano.io/v1beta1
 kind: Verrazzano
 metadata:
-  name: admin
+  name: ${cluster}
 spec:
   profile: ${profile}
   environmentName: ${environment}
   components:
     argoCD:
       enabled: ${argocd}
+    clusterAPI:
+      enabled: ${cluster_api}  
     clusterOperator:
       overrides:
       - values:
@@ -21,7 +23,7 @@ spec:
     coherenceOperator:
       enabled: ${coherence}
     console:
-      enabled: true
+      enabled: false
     dns:
       oci:
         ociConfigSecret: oci
@@ -120,6 +122,23 @@ spec:
       enabled: ${prometheus}
     prometheusOperator:
       enabled: ${prometheus_operator}
+      overrides:
+      - values:
+          prometheus:
+            thanos:
+              integration: ${thanos_integration}
+            prometheusSpec:
+              thanos:
+                objectStorageConfig:
+                  name: objstore-config
+                  key: objstore.yml
+    thanos:
+      enabled: ${thanos_enabled}
+      overrides:
+      - values:
+          existingObjstoreSecret: objstore-config
+          storagegateway:
+            enabled: ${storage_gateway}      
     rancher:
       enabled: ${rancher}
     velero:
diff --git a/modules/verrazzano/resources/vz_mc.template.yaml b/modules/verrazzano/resources/vz_mc.template.yaml
@@ -100,6 +100,23 @@ spec:
       enabled: ${prometheus}
     prometheusOperator:
       enabled: ${prometheus_operator}
+      overrides:
+      - values:
+          prometheus:
+            thanos:
+              integration: ${thanos_integration}
+            prometheusSpec:
+              thanos:
+                objectStorageConfig:
+                  name: objstore-config
+                  key: objstore.yml
+    thanos:
+      enabled: ${thanos_enabled}
+      overrides:
+      - values:
+          existingObjstoreSecret: objstore-config
+          storagegateway:
+            enabled: ${storage_gateway}
     velero:
       enabled: ${velero}
     weblogicOperator:
diff --git a/modules/verrazzano/scripts/create_api_cm.template.sh b/modules/verrazzano/scripts/create_api_cm.template.sh
@@ -2,7 +2,7 @@
 # Copyright (c) 2023 Oracle Corporation and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
 
-kubectx admin
+kubectx ${admin_ctx}
 API_SERVER="$(kubectl config view --minify -o json | jq -r '.clusters[0].cluster.server')"
 
 sed -i -e "s?API_SERVER?$(kubectl config view --minify -o json | jq -r '.clusters[0].cluster.server')?g" $HOME/vz/cm/api_cm.yaml
diff --git a/modules/verrazzano/scripts/create_cert_secret.template.sh b/modules/verrazzano/scripts/create_cert_secret.template.sh
@@ -16,7 +16,7 @@ if [ ! -f $HOME/vz/certs/vz_cert_${cluster}.completed ]; then
   --from-literal=cacrt="$CA_CERT" \
   --dry-run=client -o yaml > $HOME/vz/certs/${cluster}-secret.yaml
 
-  kubectx admin
+  kubectx ${admin_ctx}
   kubectl apply -f $HOME/vz/certs/${cluster}-secret.yaml  
 else
   echo "Cert secret already created in ${cluster}"
diff --git a/modules/verrazzano/scripts/create_vmc.template.sh b/modules/verrazzano/scripts/create_vmc.template.sh
@@ -4,7 +4,7 @@
 
 if [ ! -f $HOME/vz/clusters/vmc_${cluster}.completed ]; then
   echo "Creating VMC in admin for ${cluster}"
-  kubectx admin
+  kubectx ${admin_ctx}
   sed -i -e "s?CLUSTER?${cluster}?g" $HOME/vz/clusters/vmc_${cluster}.yaml
   kubectl apply -f $HOME/vz/clusters/vmc_${cluster}.yaml
   kubectl wait --for=condition=Ready vmc ${cluster} -n verrazzano-mc
diff --git a/modules/verrazzano/scripts/install_vz_admin.template.sh b/modules/verrazzano/scripts/install_vz_admin.template.sh
@@ -4,6 +4,6 @@
 
 if [ ! -f $HOME/vz/clusters/vz_admin.completed ]; then
   echo "Installing Verrazzano in Admin cluster"
-  kubectx admin
-  kubectl apply -f $HOME/vz/clusters/vz_admin.yaml
+  kubectx ${admin_ctx}
+  kubectl apply -f $HOME/vz/clusters/vz_${admin_ctx}.yaml
 fi
diff --git a/modules/verrazzano/scripts/register_vmc.template.sh b/modules/verrazzano/scripts/register_vmc.template.sh
@@ -4,15 +4,15 @@
 
 if [ ! -f $HOME/vz/clusters/${cluster}.registered ];  then
   echo "Generating registration manifest for ${cluster}"
-  kubectx admin
+  kubectx ${admin_ctx}
 
   kubectl get secret verrazzano-cluster-${cluster}-manifest \
     -n verrazzano-mc \
     -o jsonpath={.data.yaml} | base64 --decode > $HOME/vz/clusters/register_${cluster}.yaml
 
   kubectx ${cluster}
   kubectl apply -f $HOME/vz/clusters/register_${cluster}.yaml
-  echo "${cluster} registered with admin."
+  echo "${cluster} registered with ${admin_ctx}."
   touch $HOME/vz/clusters/${cluster}.registered
 else
   echo "${cluster} already registered"
diff --git a/modules/verrazzano/templates.tf b/modules/verrazzano/templates.tf
diff --git a/modules/verrazzano/variables.tf b/modules/verrazzano/variables.tf
diff --git a/terraform.tfvars.example b/terraform.tfvars.example
diff --git a/variables.tf b/variables.tf

Original file line number	Diff line number	Diff line change
`@@ -95,3 +95,13 @@ module "admin" {`
`95`	`95`	`oci.home = oci.home`
`96`	`96`	`}`
`97`	`97`	`}`
	`98`	`+`
	`99`	`+resource "oci_objectstorage_bucket" "thanos_admin" {`
	`100`	`+ compartment_id = var.compartment_id`
	`101`	`+ name = "${lookup(var.admin_region, "admin_name")}-${lookup(var.thanos, "bucket_name", "thanos")}"`
	`102`	`+ namespace = lookup(var.thanos, "bucket_namespace")`
	`103`	`+`
	`104`	`+ provider = oci.sydney`
	`105`	`+`
	`106`	`+ count = lookup(var.thanos, "enabled", "false") ? 1 : 0`
	`107`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -61,4 +61,8 @@ variable "nodepools" {`
`61`	`61`
`62`	`62`	`variable "cloudinit_nodepool_common" {`
`63`	`63`	`type = string`
	`64`	`+}`
	`65`	`+`
	`66`	`+variable "thanos" {`
	`67`	`+ type = map(string)`
`64`	`68`	`}`
Original file line number	Diff line number	Diff line change
`@@ -16,12 +16,12 @@ resource "null_resource" "install_vz_admin" {`
`16`	`16`
`17`	`17`	`provisioner "file" {`
`18`	`18`	`content = local.vz_admin_template`
`19`		`- destination = "/home/opc/vz/clusters/vz_admin.yaml"`
	`19`	`+ destination = "/home/opc/vz/clusters/vz_${local.admin_region_name}.yaml"`
`20`	`20`	`}`
`21`	`21`
`22`	`22`	`provisioner "file" {`
`23`	`23`	`content = local.install_admin_script`
`24`		`- destination = "/home/opc/vz/clusters/install_vz_cluster_admin.sh"`
	`24`	`+ destination = "/home/opc/vz/clusters/install_vz_cluster_${local.admin_region_name}.sh"`
`25`	`25`	`}`
`26`	`26`
`27`	`27`	`provisioner "file" {`