feat: cross-cluster mesh

Signed-off-by: Ali Mukadam <ali.mukadam@oracle.com>

Author: hyder

main.tf

@@ -94,6 +94,7 @@ module "verrazzano" {
   verrazzano_load_balancer = var.verrazzano_load_balancer
   cluster_ids              = merge({ "admin" = module.admin.cluster_id }, module.clusters.cluster_ids)
   int_nsg_ids              = merge({ "admin" = lookup(module.admin.nsg_ids, "int_lb") }, module.clusters.int_nsg_ids)
+  int_lb_subnet_ids        = merge({ "admin" = lookup(module.admin.subnet_ids, "int_lb") }, module.clusters.int_lb_subnet_ids)
   pub_nsg_ids              = merge({ "admin" = lookup(module.admin.nsg_ids, "pub_lb") }, module.clusters.pub_nsg_ids)
 
   # verrazzano components

modules/clusters/africa.tf

@@ -74,7 +74,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 

modules/clusters/australia.tf

@@ -55,7 +55,7 @@ module "melbourne" {
   # oke cluster options
   allow_worker_ssh_access     = false
   cluster_name                = "melbourne"
-  cni_type                    = var.preferred_cni  
+  cni_type                    = var.preferred_cni
   control_plane_type          = var.oke_control_plane
   control_plane_allowed_cidrs = ["0.0.0.0/0"]
   kubernetes_version          = var.kubernetes_version
@@ -64,8 +64,8 @@ module "melbourne" {
 
 
   # node pools
-  kubeproxy_mode = "ipvs"  
-  node_pools = local.managed_nodepools
+  kubeproxy_mode            = "ipvs"
+  node_pools                = local.managed_nodepools
   cloudinit_nodepool_common = var.cloudinit_nodepool_common
 
   node_pool_image_type = "oke"
@@ -74,7 +74,7 @@ module "melbourne" {
   load_balancers            = "both"
   preferred_load_balancer   = "public"
   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-  internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+  internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
   public_lb_allowed_ports   = [80, 443]
 
@@ -87,91 +87,91 @@ module "melbourne" {
 
 }
 
-# module "sydney" {
-#   source  = "oracle-terraform-modules/oke/oci"
-#   version = "4.5.9"
-
-#   home_region = var.home_region
-#   region      = local.regions["sydney"]
-
-#   tenancy_id = var.tenancy_id
-
-#   # general oci parameters
-#   compartment_id = var.compartment_id
-#   label_prefix   = var.label_prefix
-
-#   # ssh keys
-#   ssh_private_key_path = "~/.ssh/id_rsa"
-#   ssh_public_key_path  = "~/.ssh/id_rsa.pub"
-
-#   # networking
-#   create_drg       = true
-#   drg_display_name = "sydney"
-
-#   remote_peering_connections = var.connectivity_mode == "mesh" ? { for k, v in merge({ "admin" = true }, var.clusters) : "rpc-to-${k}" => {} if tobool(v) && k != "sydney" } : { "rpc-to-admin" : {} }
-
-#   nat_gateway_route_rules = concat([
-#     {
-#       destination       = lookup(var.admin_region, "vcn_cidr")
-#       destination_type  = "CIDR_BLOCK"
-#       network_entity_id = "drg"
-#       description       = "To Admin"
-#     }], var.connectivity_mode == "mesh" ?
-#     [for c in keys(var.clusters) :
-#       {
-#         destination       = lookup(lookup(var.cidrs, c), "vcn")
-#         destination_type  = "CIDR_BLOCK"
-#         network_entity_id = "drg"
-#         description       = "Routing to allow connectivity to ${title(c)} cluster"
-#     } if tobool(lookup(var.clusters, c) && c != "sydney")] : []
-#   )
-
-#   vcn_cidrs     = [lookup(lookup(var.cidrs, lower("sydney")), "vcn")]
-#   vcn_dns_label = "sydney"
-#   vcn_name      = "sydney"
-
-#   # bastion host
-#   create_bastion_host = false
-#   upgrade_bastion     = false
-
-#   # operator host
-#   create_operator                    = false
-#   upgrade_operator                   = false
-#   enable_operator_instance_principal = false
-
-
-#   # oke cluster options
-#   allow_worker_ssh_access     = false
-#   cluster_name                = "sydney"
-#   cni_type                    = var.preferred_cni
-#   control_plane_type          = var.oke_control_plane
-#   control_plane_allowed_cidrs = ["0.0.0.0/0"]
-#   kubernetes_version          = var.kubernetes_version
-#   pods_cidr                   = lookup(lookup(var.cidrs, lower("sydney")), "pods")
-#   services_cidr               = lookup(lookup(var.cidrs, lower("sydney")), "services")
-
-
-#   # node pools
-#   kubeproxy_mode = "ipvs"  
-#   node_pools = local.managed_nodepools
-#   cloudinit_nodepool_common = var.cloudinit_nodepool_common  
-
-#   node_pool_image_type = "oke"
-
-#   # oke load balancers
-#   load_balancers            = "both"
-#   preferred_load_balancer   = "public"
-#   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
-#   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
-#   public_lb_allowed_ports   = [80, 443]
-
-#   providers = {
-#     oci      = oci.sydney
-#     oci.home = oci.home
-#   }
-
-#   count = tobool(lookup(var.clusters, "sydney")) ? 1 : 0
-
-# }
+module "sydney" {
+  source  = "oracle-terraform-modules/oke/oci"
+  version = "4.5.9"
+
+  home_region = var.home_region
+  region      = local.regions["sydney"]
+
+  tenancy_id = var.tenancy_id
+
+  # general oci parameters
+  compartment_id = var.compartment_id
+  label_prefix   = var.label_prefix
+
+  # ssh keys
+  ssh_private_key_path = "~/.ssh/id_rsa"
+  ssh_public_key_path  = "~/.ssh/id_rsa.pub"
+
+  # networking
+  create_drg       = true
+  drg_display_name = "sydney"
+
+  remote_peering_connections = var.connectivity_mode == "mesh" ? { for k, v in merge({ "admin" = true }, var.clusters) : "rpc-to-${k}" => {} if tobool(v) && k != "sydney" } : { "rpc-to-admin" : {} }
+
+  nat_gateway_route_rules = concat([
+    {
+      destination       = lookup(var.admin_region, "vcn_cidr")
+      destination_type  = "CIDR_BLOCK"
+      network_entity_id = "drg"
+      description       = "To Admin"
+    }], var.connectivity_mode == "mesh" ?
+    [for c in keys(var.clusters) :
+      {
+        destination       = lookup(lookup(var.cidrs, c), "vcn")
+        destination_type  = "CIDR_BLOCK"
+        network_entity_id = "drg"
+        description       = "Routing to allow connectivity to ${title(c)} cluster"
+    } if tobool(lookup(var.clusters, c) && c != "sydney")] : []
+  )
+
+  vcn_cidrs     = [lookup(lookup(var.cidrs, lower("sydney")), "vcn")]
+  vcn_dns_label = "sydney"
+  vcn_name      = "sydney"
+
+  # bastion host
+  create_bastion_host = false
+  upgrade_bastion     = false
+
+  # operator host
+  create_operator                    = false
+  upgrade_operator                   = false
+  enable_operator_instance_principal = false
+
+
+  # oke cluster options
+  allow_worker_ssh_access     = false
+  cluster_name                = "sydney"
+  cni_type                    = var.preferred_cni
+  control_plane_type          = var.oke_control_plane
+  control_plane_allowed_cidrs = ["0.0.0.0/0"]
+  kubernetes_version          = var.kubernetes_version
+  pods_cidr                   = lookup(lookup(var.cidrs, lower("sydney")), "pods")
+  services_cidr               = lookup(lookup(var.cidrs, lower("sydney")), "services")
+
+
+  # node pools
+  kubeproxy_mode = "ipvs"  
+  node_pools = local.managed_nodepools
+  cloudinit_nodepool_common = var.cloudinit_nodepool_common  
+
+  node_pool_image_type = "oke"
+
+  # oke load balancers
+  load_balancers            = "both"
+  preferred_load_balancer   = "public"
+  internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
+  internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
+  public_lb_allowed_cidrs   = ["0.0.0.0/0"]
+  public_lb_allowed_ports   = [80, 443]
+
+  providers = {
+    oci      = oci.sydney
+    oci.home = oci.home
+  }
+
+  count = tobool(lookup(var.clusters, "sydney")) ? 1 : 0
+
+}
 

modules/clusters/brazil.tf

@@ -74,7 +74,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 
@@ -160,7 +160,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 

modules/clusters/canada.tf

@@ -74,7 +74,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 
@@ -160,7 +160,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 

modules/clusters/europe.tf

@@ -74,7 +74,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 
@@ -161,7 +161,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 
@@ -248,7 +248,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 
@@ -334,7 +334,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 
@@ -420,7 +420,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 
@@ -506,7 +506,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 

modules/clusters/france.tf

@@ -74,7 +74,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 
@@ -160,7 +160,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 

modules/clusters/india.tf

@@ -74,7 +74,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 
@@ -160,7 +160,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 

modules/clusters/japan.tf

@@ -74,7 +74,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 
@@ -160,7 +160,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 

modules/clusters/korea.tf

@@ -74,7 +74,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]
 
@@ -160,7 +160,7 @@
 #   load_balancers            = "both"
 #   preferred_load_balancer   = "public"
 #   internal_lb_allowed_cidrs = [lookup(var.admin_region, "vcn_cidr")]
-#   internal_lb_allowed_ports = var.connectivity_mode == "mesh" ? [80, 443, 15012, 15017, 15021, 15443] : [80, 443]
+#   internal_lb_allowed_ports = [80, 443, 15012, 15017, 15021, 15443]
 #   public_lb_allowed_cidrs   = ["0.0.0.0/0"]
 #   public_lb_allowed_ports   = [80, 443]