Cadl automation pipeline (Azure#26912)

* cdoe

* fix

* code

* fix

* code

* meta.json

* meta.json

* compatible

Author: msyyc

tools/azure-sdk-tools/packaging_tools/generate_utils.py

@@ -6,7 +6,7 @@
 
 from azure_devtools.ci_tools.git_tools import get_add_diff_file_list
 from pathlib import Path
-from subprocess import check_call
+from subprocess import check_call, getoutput
 from typing import List, Dict, Any
 from glob import glob
 import yaml
@@ -52,26 +52,32 @@ def init_new_service(package_name, folder_name):
 
 def update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme):
 
-    readme_file = str(Path(spec_folder, input_readme))
-    global_conf = config["meta"]
-    local_conf = config.get("projects", {}).get(readme_file, {})
-
-    if "resource-manager" in input_readme:
-        cmd = ["autorest", input_readme]
-    else:
-        # autorest for DPG will be executed in package folder like: sdk/deviceupdate/azure-iot-deviceupdate/swagger
-        cmd = ["autorest", _DPG_README]
-    cmd += build_autorest_options(global_conf, local_conf)
-
-    # metadata
     metadata = {
-        "autorest": global_conf["autorest_options"]["version"],
-        "use": global_conf["autorest_options"]["use"],
         "commit": data["headSha"],
         "repository_url": data["repoHttpsUrl"],
-        "autorest_command": " ".join(cmd),
-        "readme": input_readme,
     }
+    if "meta" in config:
+        readme_file = str(Path(spec_folder, input_readme))
+        global_conf = config["meta"]
+        local_conf = config.get("projects", {}).get(readme_file, {})
+
+        if "resource-manager" in input_readme:
+            cmd = ["autorest", input_readme]
+        else:
+            # autorest for DPG will be executed in package folder like: sdk/deviceupdate/azure-iot-deviceupdate/swagger
+            cmd = ["autorest", _DPG_README]
+        cmd += build_autorest_options(global_conf, local_conf)
+
+        # metadata
+        metadata.update({
+            "autorest": global_conf["autorest_options"]["version"],
+            "use": global_conf["autorest_options"]["use"],
+            "autorest_command": " ".join(cmd),
+            "readme": input_readme,
+        })
+    else:
+        metadata["cadl_src"] = input_readme
+        metadata.update(config)
 
     _LOGGER.info("Metadata json:\n {}".format(json.dumps(metadata, indent=2)))
 
@@ -322,3 +328,40 @@ def format_samples(sdk_code_path) -> None:
             fw.write(file_content)
 
     _LOGGER.info(f"format generated_samples successfully")
+
+def gen_cadl(cadl_relative_path: str, spec_folder: str) -> Dict[str, Any]:
+    # update config file
+    cadl_python = "@azure-tools/cadl-python"
+    project_yaml_path = Path(spec_folder) / cadl_relative_path / "cadl-project.yaml"
+    with open(project_yaml_path, "r") as file_in:
+        project_yaml = yaml.safe_load(file_in)
+    if not project_yaml.get("emitters", {}).get(cadl_python):
+        return
+    if not project_yaml["emitters"][cadl_python].get("sdk-folder"):
+        raise Exception("no sdk-folder is defined")
+    output_path = Path(os.getcwd()) / project_yaml["emitters"][cadl_python]["sdk-folder"]
+    if not output_path.exists():
+        os.makedirs(output_path)
+
+    project_yaml["emitters"][cadl_python].pop("sdk-folder")
+    project_yaml["emitters"][cadl_python]["output-path"] = str(output_path)
+    with open(project_yaml_path, "w") as file_out:
+        yaml.safe_dump(project_yaml, file_out)
+
+    # npm install tool
+    origin_path = os.getcwd()
+    os.chdir(Path(spec_folder) / cadl_relative_path)
+    check_call("npm install", shell=True)
+
+    # generate code
+    check_call(f"npx cadl compile . --emit {cadl_python}", shell=True)
+    if Path(output_path / "output.yaml").exists():
+        os.remove(Path(output_path / "output.yaml"))
+
+    # get version of @azure-tools/cadl-python used in generation
+    cadl_python_version = getoutput(f"npm view {cadl_python} version").split('\n')[-1]
+
+    # return to original folder
+    os.chdir(origin_path)
+
+    return {cadl_python: cadl_python_version}

tools/azure-sdk-tools/packaging_tools/sdk_generator.py

@@ -16,6 +16,7 @@
     format_samples,
     gen_dpg,
     dpg_relative_folder,
+    gen_cadl
 )
 
 _LOGGER = logging.getLogger(__name__)
@@ -30,20 +31,28 @@ def main(generate_input, generate_output):
     result = {}
     python_tag = data.get("python_tag")
     package_total = set()
+    spec_word = "readmeMd"
     if "relatedReadmeMdFiles" in data:
         readme_files = data["relatedReadmeMdFiles"]
-    else:
+    elif "relatedReadmeMdFile" in data:
         input_readme = data["relatedReadmeMdFile"]
         if "specification" in spec_folder:
             spec_folder = str(Path(spec_folder.split("specification")[0]))
         if "specification" not in input_readme:
             input_readme = str(Path("specification") / input_readme)
         readme_files = [input_readme]
+    else:
+        # ["specification/confidentialledger/ConfientialLedger"]
+        if isinstance(data["relatedCadlProjectFolder"], str):
+            readme_files = [data["relatedCadlProjectFolder"]]
+        else:
+            readme_files = data["relatedCadlProjectFolder"]
+        spec_word = "cadlProject"
 
     for input_readme in readme_files:
-        relative_path_readme = str(Path(spec_folder, input_readme))
         _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin")
         if "resource-manager" in input_readme:
+            relative_path_readme = str(Path(spec_folder, input_readme))
             config = generate(
                 CONFIG_FILE,
                 sdk_folder,
@@ -53,8 +62,10 @@ def main(generate_input, generate_output):
                 force_generation=True,
                 python_tag=python_tag,
             )
-        else:
+        elif "data-plane" in input_readme:
             config = gen_dpg(input_readme, data.get("autorestConfig", ""), dpg_relative_folder(spec_folder))
+        else:
+            config = gen_cadl(input_readme, spec_folder)
         package_names = get_package_names(sdk_folder)
         _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]")
 
@@ -68,12 +79,12 @@ def main(generate_input, generate_output):
                 package_entry = {}
                 package_entry["packageName"] = package_name
                 package_entry["path"] = [folder_name]
-                package_entry["readmeMd"] = [input_readme]
+                package_entry[spec_word] = [input_readme]
                 package_entry["tagIsStable"] = not judge_tag_preview(sdk_code_path)
                 result[package_name] = package_entry
             else:
                 result[package_name]["path"].append(folder_name)
-                result[package_name]["readmeMd"].append(input_readme)
+                result[package_name][spec_word].append(input_readme)
 
             # Generate some necessary file for new service
             init_new_service(package_name, folder_name)
@@ -102,7 +113,7 @@ def main(generate_input, generate_output):
     # remove duplicates
     for value in result.values():
         value["path"] = list(set(value["path"]))
-        value["readmeMd"] = list(set(value["readmeMd"]))
+        value[spec_word] = list(set(value[spec_word]))
 
     with open(generate_output, "w") as writer:
         json.dump(result, writer)