Automation Toolkit Release v2024.4.2

Author: xs2suruchi

cd3_automation_toolkit/Database/create_terraform_adb.py

@@ -180,8 +180,8 @@ def create_terraform_adb(inputfile, outdir, service_dir, prefix, ct):
                         k +=1
                 else:
                     wl_str = ""
-                tempdict = {'whitelisted_ips': wl_str,'network_compartment_id': network_compartment_id, 'vcn_name': vcn_name,
-                            'subnet_id': subnet_id }
+                #tempdict = {'whitelisted_ips': wl_str,'network_compartment_id': network_compartment_id, 'vcn_name': vcn_name,'subnet_id': subnet_id }
+                tempdict = {'whitelisted_ips': wl_str }
                 tempStr.update(tempdict)
 
 

cd3_automation_toolkit/Database/templates/adb-template

@@ -43,7 +43,7 @@ adb = {
             database_edition         = "{{ database_edition }}" #Only for BYOL license model
             {% endif %}
 
-            {% if data_storage_size_in_tbs == "" %}
+            {% if data_storage_size_in_tb == "" %}
             data_storage_size_in_tbs = {{ data_storage_size_in_tb }}
             {% endif %}
             db_version               = "19c"

cd3_automation_toolkit/Identity/Users/export_users_nonGreenField.py

@@ -112,7 +112,7 @@ def export_users(inputfile, outdir, service_dir, config, signer, ct,export_domai
             domain_name = domain_key.split("@")[1]
             domain_client = oci.identity_domains.IdentityDomainsClient(config=config, signer=signer,
                                                                        service_endpoint=idcs_endpoint)
-            users = domain_client.list_users()
+            users = domain_client.list_users(limit=100000) # change this to pagination once api supports
             index = 0
             for user in users.data.resources:
                 defined_tags_info = user.urn_ietf_params_scim_schemas_oracle_idcs_extension_oci_tags

cd3_automation_toolkit/Network/BaseNetwork/create_major_objects.py

@@ -243,7 +243,10 @@ def create_drg_and_attachments(inputfile, outdir):
 
                 if columnname == "DRG Name":
                     drg_name = columnvalue
-                    drg_tf_name = commonTools.check_tf_variable(drg_name)
+                    if ("ocid1.drg.oc" not in drg_name):
+                        drg_tf_name = commonTools.check_tf_variable(drg_name)
+                    else:
+                        drg_tf_name = drg_name
                     tempdict['drg_tf_name'] = drg_tf_name
 
                 if (columnname == 'Attached To'):
@@ -327,6 +330,8 @@ def create_drg_and_attachments(inputfile, outdir):
                     #if it is Auto Generated RT(during export) dont attach any RT to DRG attachment
                     if(columnvalue in commonTools.drg_auto_RTs):
                         drg_rt_tf_name = ''
+                    elif("ocid1.drgroutetable.oc1" in columnvalue):
+                        drg_rt_tf_name = columnvalue
                     elif(columnvalue!=''):
                         drg_rt_tf_name = commonTools.check_tf_variable(drg_name + "_" + columnvalue)
                     tempStr['drg_rt_tf_name'] = drg_rt_tf_name
@@ -340,7 +345,9 @@ def create_drg_and_attachments(inputfile, outdir):
                 drgstr_skeleton = drg_template.render(count=0)[:-1]
                 region_included_drg.append(region)
             tempStr['drg_version'] = drg_versions[region, drg_name]
-            drgstr = drg_template.render(tempStr)
+            drgstr=''
+            if ("ocid1.drg.oc" not in drg_tf_name):
+                drgstr = drg_template.render(tempStr)
 
             if(attachedto=="attached"):
                 drg_attach = drg_attach_template.render(tempStr)
@@ -360,7 +367,8 @@ def create_drg_and_attachments(inputfile, outdir):
             if region in region_included_drg:
                 if(drg_attach_tfStr[region]!=''):
                     drg_attach_tfStr[region] = drg_attach_skeleton + drg_attach_tfStr[region]
-                drg_tfStr[region] = drgstr_skeleton + drg_tfStr[region]
+                if(drg_tfStr[region] != ''):
+                    drg_tfStr[region] = drgstr_skeleton + drg_tfStr[region]
 
     def processVCN(tempStr):
         rt_tf_name = ''

cd3_automation_toolkit/Network/BaseNetwork/create_terraform_route.py

@@ -218,6 +218,9 @@ def create_terraform_drg_route(inputfile, outdir, service_dir, prefix, ct, non_g
             # Dont create any route table or route distribution name if using Auto Generated ones
             if (DRG_RT in commonTools.drg_auto_RTs and DRG_RD in commonTools.drg_auto_RDs):
                 continue
+            # Dont create any oute table or route distribution name if OCID is goven in DRG RT Name
+            if ("ocid1.drgroutetable.oc" in DRG_RT):
+                continue
 
             region = region.strip().lower()
             if region not in ct.all_regions:

cd3_automation_toolkit/Network/BaseNetwork/exportRoutetable.py

@@ -12,23 +12,39 @@ def get_network_entity_name(config,signer,network_identity_id):
     vcn1 = VirtualNetworkClient(config=config, retry_strategy=oci.retry.DEFAULT_RETRY_STRATEGY,signer=signer)
     if('internetgateway' in network_identity_id):
         igw=vcn1.get_internet_gateway(network_identity_id)
-        network_identity_name = "igw:"+igw.data.display_name
+        network_entity_comp_id=igw.data.compartment_id
+        if network_entity_comp_id in export_compartment_ids:
+            network_identity_name = "igw:"+igw.data.display_name
+        else:
+            network_identity_name = "igw:" + igw.data.id
         return  network_identity_name
 
     elif ('servicegateway' in network_identity_id):
         sgw = vcn1.get_service_gateway(network_identity_id)
-        network_identity_name = "sgw:"+sgw.data.display_name
+        network_entity_comp_id = sgw.data.compartment_id
+        if network_entity_comp_id in export_compartment_ids:
+            network_identity_name = "sgw:" + sgw.data.display_name
+        else:
+            network_identity_name = "sgw:"+sgw.data.id
         return network_identity_name
 
 
     elif ('natgateway' in network_identity_id):
         ngw = vcn1.get_nat_gateway(network_identity_id)
-        network_identity_name = "ngw:"+ngw.data.display_name
+        network_entity_comp_id = ngw.data.compartment_id
+        if network_entity_comp_id in export_compartment_ids:
+            network_identity_name = "ngw:" + ngw.data.display_name
+        else:
+            network_identity_name = "ngw:"+ngw.data.id
         return network_identity_name
 
     elif ('localpeeringgateway' in network_identity_id):
         lpg = vcn1.get_local_peering_gateway(network_identity_id)
-        network_identity_name = "lpg:"+lpg.data.display_name
+        network_entity_comp_id = lpg.data.compartment_id
+        if network_entity_comp_id in export_compartment_ids:
+            network_identity_name = "lpg:" + lpg.data.display_name
+        else:
+            network_identity_name = "lpg:"+lpg.data.id
         return network_identity_name
     elif ('drgattachment' in network_identity_id):
         drg_attach = vcn1.get_drg_attachment(network_identity_id)
@@ -46,7 +62,11 @@ def get_network_entity_name(config,signer,network_identity_id):
         return network_identity_name
     elif ('drg' in network_identity_id):
         drg = vcn1.get_drg(network_identity_id)
-        network_identity_name = "drg:"+drg.data.display_name
+        network_entity_comp_id = drg.data.compartment_id
+        if network_entity_comp_id in export_compartment_ids:
+            network_identity_name = "drg:" + drg.data.display_name
+        else:
+            network_identity_name = "drg:"+drg.data.id
         return network_identity_name
 
         """
@@ -282,6 +302,7 @@ def export_routetable(inputfile, outdir, service_dir,config1,signer1, ct, export
     config=config1
     global signer,tf_or_tofu
     signer=signer1
+    global export_compartment_ids
 
     tf_or_tofu = ct.tf_or_tofu
     tf_state_list = [tf_or_tofu, "state", "list"]
@@ -316,6 +337,10 @@ def export_routetable(inputfile, outdir, service_dir,config1,signer1, ct, export
                                         "import_commands_network_routerules.sh")
             importCommands[reg] = ''
 
+    export_compartment_ids = []
+    for comp in export_compartments:
+        export_compartment_ids.append(ct.ntk_compartment_ids[comp])
+
     for reg in export_regions:
         config.__setitem__("region", commonTools().region_dict[reg])
         state = {'path': f'{outdir}/{reg}/{service_dir}', 'resources': []}

cd3_automation_toolkit/Network/BaseNetwork/export_network_nonGreenField.py

@@ -22,14 +22,18 @@
 
 
 def print_drgv2(values_for_column_drgv2, region, comp_name, vcn_info, drg_info, drg_attachment_info, drg_rt_info,
-                import_drg_route_distribution_info, drg_route_distribution_statements):
+                import_drg_route_distribution_info, drg_route_distribution_statements,write_drg_ocids):
     for col_header in values_for_column_drgv2.keys():
         if (col_header == "Region"):
             values_for_column_drgv2[col_header].append(region)
         elif (col_header == "Compartment Name"):
             values_for_column_drgv2[col_header].append(comp_name)
         elif (col_header == "DRG Name"):
-            values_for_column_drgv2[col_header].append(drg_info.display_name)
+            if write_drg_ocids == True:
+                values_for_column_drgv2[col_header].append(drg_info.id)
+            else:
+                values_for_column_drgv2[col_header].append(drg_info.display_name)
+
         elif (col_header == "Attached To"):
             if (drg_attachment_info is None):
                 values_for_column_drgv2[col_header].append('')
@@ -53,12 +57,19 @@ def print_drgv2(values_for_column_drgv2, region, comp_name, vcn_info, drg_info,
             if (drg_rt_info == None):
                 values_for_column_drgv2[col_header].append("")
             else:
-                values_for_column_drgv2[col_header].append(drg_rt_info.display_name)
+                if write_drg_ocids==True:
+                    values_for_column_drgv2[col_header].append(drg_rt_info.id)
+                else:
+                    values_for_column_drgv2[col_header].append(drg_rt_info.display_name)
+
         elif (col_header == 'Import DRG Route Distribution Name'):
             if import_drg_route_distribution_info == None:
                 values_for_column_drgv2[col_header].append("")
             else:
-                values_for_column_drgv2[col_header].append(import_drg_route_distribution_info.display_name)
+                if write_drg_ocids == True:
+                    values_for_column_drgv2[col_header].append(import_drg_route_distribution_info.id)
+                else:
+                    values_for_column_drgv2[col_header].append(import_drg_route_distribution_info.display_name)
         elif (col_header == "Import DRG Route Distribution Statements"):
             statement_val = ''
             if (drg_route_distribution_statements == None):
@@ -90,7 +101,7 @@ def print_drgv2(values_for_column_drgv2, region, comp_name, vcn_info, drg_info,
 
 
 def print_vcns(values_for_column_vcns, region, comp_name, vnc,vcn_info, drg_attachment_info, igw_info, ngw_info, sgw_info,
-               lpg_display_names,state):
+               lpg_display_names,state,write_drg_ocids):
     drg_info=None
     for col_header in values_for_column_vcns.keys():
 
@@ -106,10 +117,17 @@ def print_vcns(values_for_column_vcns, region, comp_name, vnc,vcn_info, drg_atta
                     values_for_column_vcns[col_header].append("n")
                 else:
                     route_table_id = drg_attachment_info.route_table_id
-                    if (route_table_id is not None):
-                        val = drg_info.display_name + "::" + vnc.get_route_table(route_table_id).data.display_name
+                    if write_drg_ocids == True:
+                        if (route_table_id is not None):
+                            val = drg_info.id + "::" + vnc.get_route_table(route_table_id).data.display_name
+                        else:
+                            val = drg_info.id
                     else:
-                        val = drg_info.display_name
+                        if (route_table_id is not None):
+                            val = drg_info.display_name + "::" + vnc.get_route_table(route_table_id).data.display_name
+                        else:
+                            val = drg_info.display_name
+
                     values_for_column_vcns[col_header].append(val)
             else:
                 values_for_column_vcns[col_header].append("n")
@@ -620,6 +638,10 @@ def export_major_objects(inputfile, outdir, service_dir, config, signer, ct, exp
 
     print("Tabs- VCNs and DRGs would be overwritten during export process!!!\n")
 
+    export_compartment_ids = []
+    for comp in export_compartments:
+        export_compartment_ids.append(ct.ntk_compartment_ids[comp])
+
     # Fetch DRGs
     for reg in export_regions:
         current_region = reg
@@ -637,6 +659,7 @@ def export_major_objects(inputfile, outdir, service_dir, config, signer, ct, exp
                                                                            ntk_compartment_name],
                                                                        attachment_type="ALL")  # ,lifecycle_state ="ATTACHED")#,attachment_type="ALL")
             rpc_execution = True
+            write_drg_ocids=False
             for drg_attachment_info in DRG_Attachments.data:
                 if (drg_attachment_info.lifecycle_state != "ATTACHED"):
                     continue
@@ -647,6 +670,11 @@ def export_major_objects(inputfile, outdir, service_dir, config, signer, ct, exp
                 # Attachment Data
                 drg_display_name = drg_info.display_name
                 drg_comp_id = drg_info.compartment_id
+
+                if drg_comp_id not in export_compartment_ids:
+                    drg_display_name=drg_id
+                    write_drg_ocids=True
+
                 for key, val in ct.ntk_compartment_ids.items():
                     if val == drg_comp_id:
                         if ("::" in key):
@@ -660,7 +688,7 @@ def export_major_objects(inputfile, outdir, service_dir, config, signer, ct, exp
                 if (drg_id not in drg_ocid):
                     oci_obj_names[reg].write("\nDRG Version::::" + drg_display_name + "::::" + drg_version)
                     tf_resource = f'module.drgs[\\"{tf_name}\\"].oci_core_drg.drg'
-                    if tf_resource not in state["resources"]:
+                    if tf_resource not in state["resources"] and write_drg_ocids == False:
                         importCommands[reg].write( f'\n{tf_or_tofu} import "{tf_resource}" {str(drg_info.id)}')
                     drg_ocid.append(drg_id)
 
@@ -700,15 +728,17 @@ def export_major_objects(inputfile, outdir, service_dir, config, signer, ct, exp
                         drg_route_table_info = vnc.get_drg_route_table(drg_route_table_id).data
 
                         import_drg_route_distribution_id = drg_route_table_info.import_drg_route_distribution_id
+
                         if (import_drg_route_distribution_id != None):
                             import_drg_route_distribution_info = vnc.get_drg_route_distribution(
                                 import_drg_route_distribution_id).data
+
                             drg_route_distribution_statements = vnc.list_drg_route_distribution_statements(
                                 import_drg_route_distribution_info.id)
 
                             tf_name = commonTools.check_tf_variable(
                                 drg_display_name + "_" + import_drg_route_distribution_info.display_name)
-                            if (import_drg_route_distribution_info.display_name not in commonTools.drg_auto_RDs):
+                            if (import_drg_route_distribution_info.display_name not in commonTools.drg_auto_RDs and "ocid1.drg.oc" not in drg_display_name):
                                 tf_resource = f'module.drg-route-distributions[\\"{tf_name}\\"].oci_core_drg_route_distribution.drg_route_distribution'
                                 if tf_resource not in state["resources"]:
                                     importCommands[reg].write(f'\n{tf_or_tofu} import "{tf_resource}" {str(import_drg_route_distribution_info.id)}')
@@ -722,7 +752,7 @@ def export_major_objects(inputfile, outdir, service_dir, config, signer, ct, exp
 
                     print_drgv2(values_for_column_drgv2, region, drg_comp_name, vcn_info, drg_info, drg_attachment_info,
                                 drg_route_table_info, import_drg_route_distribution_info,
-                                drg_route_distribution_statements)
+                                drg_route_distribution_statements,write_drg_ocids)
 
                 # RPC
                 elif attach_type.upper() == "REMOTE_PEERING_CONNECTION" and rpc_execution:
@@ -747,7 +777,7 @@ def export_major_objects(inputfile, outdir, service_dir, config, signer, ct, exp
 
                             tf_name = commonTools.check_tf_variable(
                                 drg_display_name + "_" + import_drg_route_distribution_info.display_name)
-                            if (import_drg_route_distribution_info.display_name not in commonTools.drg_auto_RDs):
+                            if (import_drg_route_distribution_info.display_name not in commonTools.drg_auto_RDs and write_drg_ocids == False):
                                 tf_resource = f'module.drg-route-distributions[\\"{tf_name}\\"].oci_core_drg_route_distribution.drg_route_distribution'
                                 if tf_resource not in state["resources"]:
                                     importCommands[reg].write(f'\n{tf_or_tofu} import "{tf_resource}" {str(import_drg_route_distribution_info.id)}')
@@ -784,6 +814,13 @@ def export_major_objects(inputfile, outdir, service_dir, config, signer, ct, exp
                 drg_info = vnc.get_drg(drg_id).data
                 drg_display_name = drg_info.display_name
 
+                #Do not process if DRG (and its RTs/RDs are in different compartment than the export_compartments list
+                drg_comp_id=drg_info.compartment_id
+                if drg_comp_id not in export_compartment_ids:
+                    continue
+
+                write_drg_ocids=False
+
                 if drg_info.default_drg_route_tables is not None:
                     DRG_RTs = oci.pagination.list_call_get_all_results(vnc.list_drg_route_tables,
                                                                        drg_id=drg_id)
@@ -820,7 +857,7 @@ def export_major_objects(inputfile, outdir, service_dir, config, signer, ct, exp
                         print_drgv2(values_for_column_drgv2, region, drg_comp_name, vcn_info, drg_info,
                                     drg_attachment_info, drg_route_table_info,
                                     import_drg_route_distribution_info,
-                                    drg_route_distribution_statements)
+                                    drg_route_distribution_statements,write_drg_ocids)
 
     commonTools.write_to_cd3(values_for_column_drgv2, cd3file, "DRGs")
     print("RPCs exported to CD3\n")
@@ -861,6 +898,14 @@ def export_major_objects(inputfile, outdir, service_dir, config, signer, ct, exp
                     if (drg_attachment_info.lifecycle_state != "ATTACHED"):
                         continue
 
+                write_drg_ocids=False
+                if drg_attachment_info != None:
+                    drg_id = drg_attachment_info.drg_id
+                    drg_info = vnc.get_drg(drg_id).data
+                    drg_comp_id=drg_info.compartment_id
+                    if drg_comp_id not in export_compartment_ids:
+                        write_drg_ocids= True
+
                 # igw_display_name = "n"
                 IGWs = oci.pagination.list_call_get_all_results(vnc.list_internet_gateways,
                                                                 compartment_id=ct.ntk_compartment_ids[
@@ -933,7 +978,7 @@ def export_major_objects(inputfile, outdir, service_dir, config, signer, ct, exp
 
                 # Fill VCNs Tab
                 print_vcns(values_for_column_vcns, region, ntk_compartment_name, vnc,vcn_info, drg_attachment_info, igw_info, ngw_info,
-                           sgw_info, lpg_display_names,state)
+                           sgw_info, lpg_display_names,state,write_drg_ocids)
 
     commonTools.write_to_cd3(values_for_column_vcns, cd3file, "VCNs")
     print("VCNs exported to CD3\n")