Skip to content

Commit 266c382

Browse files
zzvswxyzzvswxy
authored
[feature]Support codegen service for python SDK (Azure#22783)
* feature: support codegen pipeline Signed-off-by: zhouzheng <576014838@qq.com> * change file name as reviewed suggestions Signed-off-by: zhouzheng <576014838@qq.com> * change file names Signed-off-by: zhouzheng <576014838@qq.com> * sdk automation only need one arg Signed-off-by: zhouzheng <576014838@qq.com> * bugfix: fix wrong path and make sdk_init.sh support no args Signed-off-by: zhouzheng <576014838@qq.com> Signed-off-by: Zhou Zheng <zhouzheng@microsoft.com>
1 parent 89d3d9f commit 266c382

File tree

10 files changed

+377
-133
lines changed

10 files changed

+377
-133
lines changed

‎codegen_to_sdk_config.json‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"init": {
3+
"initScript": {
4+
"path": "./scripts/sdk_init.sh"
5+
}
6+
},
7+
8+
"generateAndBuild": {
9+
"generateAndBuildScript": {
10+
"path": "./scripts/sdk_generate.sh",
11+
"stderr": {
12+
"storeAllLog": true
13+
},
14+
"stdout": {
15+
"storeLogByFilter": "^\\[Autorest\\]"
16+
}
17+
}
18+
}
19+
}

‎scripts/sdk_generate.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/bin/bash
2+
3+
TMPDIR="$(dirname $(dirname $(readlink -f "$0")))/venv"
4+
export TMPDIR
5+
VIRTUAL_ENV=$TMPDIR/venv-sdk
6+
echo "$VIRTUAL_ENV"
7+
export VIRTUAL_ENV
8+
PATH="$VIRTUAL_ENV/bin:$PATH"
9+
export PATH
10+
11+
# node version degrade
12+
sudo npm install -g n
13+
sudo n 14.15.0
14+
echo "$PATH"
15+
export PATH="/usr/local/n/versions/node/14.15.0/bin:$PATH"
16+
17+
TEMP_FILE="$TMPDIR/venv-sdk/auto_temp.json"
18+
# generate code
19+
python -m packaging_tools.sdk_generator "$1" "$TEMP_FILE" 2>&1
20+
echo "[Generate] codegen done!!!"
21+
if [ ! -f "$TEMP_FILE" ]; then
22+
echo "[Autorest]$TEMP_FILE does not exist!!!Error happened during codegen"
23+
exit 1
24+
fi
25+
26+
# package
27+
python -m packaging_tools.sdk_package "$TEMP_FILE" "$2" 2>&1
28+
echo "[Generate] generate done!!!"
29+
if [ ! -f "$2" ]; then
30+
echo "[Autorest]$2 does not exist!!!Error happened during package"
31+
exit 1
32+
fi

‎scripts/sdk_init.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/bin/bash
2+
3+
# install python3.8
4+
sudo apt-get install python3.8
5+
sudo apt-get install python3.8-venv
6+
7+
# init env
8+
TMPDIR="$(dirname $(dirname $(readlink -f "$0")))/venv"
9+
export TMPDIR
10+
echo "$TMPDIR"
11+
rm -rf $TMPDIR/venv-sdk
12+
VIRTUAL_ENV=$TMPDIR/venv-sdk
13+
export VIRTUAL_ENV
14+
PATH="$VIRTUAL_ENV/bin:$PATH"
15+
export PATH
16+
echo "$PATH"
17+
python3.8 -m venv $TMPDIR/venv-sdk
18+
python -m pip install -U pip
19+
python scripts/dev_setup.py -p azure-core
20+
21+
if [ x"$1" = x ]; then
22+
echo "[Generate] init success!!!"
23+
exit 0
24+
fi
25+
26+
echo "{}" >> $1
27+
echo "[Generate] init success!!!！"

‎tools/azure-sdk-tools/packaging_tools/auto_codegen.py‎

Lines changed: 2 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -1,96 +1,14 @@
11
import argparse
22
import json
33
import logging
4-
import os
54
from pathlib import Path
6-
import re
75
from subprocess import check_call
86

9-
from .swaggertosdk.SwaggerToSdkCore import (
10-
read_config,
11-
CONFIG_FILE,
12-
)
13-
from azure_devtools.ci_tools.git_tools import get_add_diff_file_list
14-
from .swaggertosdk.autorest_tools import build_autorest_options
7+
from .swaggertosdk.SwaggerToSdkCore import (CONFIG_FILE,)
158
from .generate_sdk import generate
9+
from .generate_utils import get_package_names, init_new_service, update_servicemetadata
1610

1711
_LOGGER = logging.getLogger(__name__)
18-
_SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII)
19-
20-
DEFAULT_DEST_FOLDER = "./dist"
21-
22-
23-
def get_package_names(sdk_folder):
24-
files = get_add_diff_file_list(sdk_folder)
25-
matches = {_SDK_FOLDER_RE.search(f) for f in files}
26-
package_names = {match.groups() for match in matches if match is not None}
27-
return package_names
28-
29-
30-
def init_new_service(package_name, folder_name):
31-
setup = Path(folder_name, package_name, "setup.py")
32-
if not setup.exists():
33-
check_call(f"python -m packaging_tools --build-conf {package_name} -o {folder_name}", shell=True)
34-
ci = Path(folder_name, "ci.yml")
35-
if not ci.exists():
36-
with open("ci_template.yml", "r") as file_in:
37-
content = file_in.readlines()
38-
name = package_name.replace("azure-", "").replace("mgmt-", "")
39-
content = [line.replace("MyService", name) for line in content]
40-
with open(str(ci), "w") as file_out:
41-
file_out.writelines(content)
42-
43-
44-
def update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme):
45-
46-
readme_file = str(Path(spec_folder, input_readme))
47-
global_conf = config["meta"]
48-
local_conf = config["projects"][readme_file]
49-
50-
cmd = ["autorest", input_readme]
51-
cmd += build_autorest_options(global_conf, local_conf)
52-
53-
# metadata
54-
metadata = {
55-
"autorest": global_conf["autorest_options"]["version"],
56-
"use": global_conf["autorest_options"]["use"],
57-
"commit": data["headSha"],
58-
"repository_url": data["repoHttpsUrl"],
59-
"autorest_command": " ".join(cmd),
60-
"readme": input_readme,
61-
}
62-
63-
_LOGGER.info("Metadata json:\n {}".format(json.dumps(metadata, indent=2)))
64-
65-
package_folder = Path(sdk_folder, folder_name, package_name).expanduser()
66-
if not os.path.exists(package_folder):
67-
_LOGGER.info(f"Package folder doesn't exist: {package_folder}")
68-
_LOGGER.info("Failed to save metadata.")
69-
return
70-
71-
metadata_file_path = os.path.join(package_folder, "_meta.json")
72-
with open(metadata_file_path, "w") as writer:
73-
json.dump(metadata, writer, indent=2)
74-
_LOGGER.info(f"Saved metadata to {metadata_file_path}")
75-
76-
# Check whether MANIFEST.in includes _meta.json
77-
require_meta = "include _meta.json\n"
78-
manifest_file = os.path.join(package_folder, "MANIFEST.in")
79-
if not os.path.exists(manifest_file):
80-
_LOGGER.info(f"MANIFEST.in doesn't exist: {manifest_file}")
81-
return
82-
83-
includes = []
84-
write_flag = False
85-
with open(manifest_file, "r") as f:
86-
includes = f.readlines()
87-
if require_meta not in includes:
88-
includes = [require_meta] + includes
89-
write_flag = True
90-
91-
if write_flag:
92-
with open(manifest_file, "w") as f:
93-
f.write("".join(includes))
9412

9513

9614
def main(generate_input, generate_output):

‎tools/azure-sdk-tools/packaging_tools/auto_package.py‎

Lines changed: 2 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,13 @@
11
import argparse
22
import json
3-
import glob
43
import logging
54
import os
65
from pathlib import Path
7-
import re
8-
from subprocess import check_call
96

107
from azure_devtools.ci_tools.git_tools import get_diff_file_list
11-
from .change_log import main as change_log_main
8+
from .package_utils import create_package, change_log_generate, extract_breaking_change
129

1310
_LOGGER = logging.getLogger(__name__)
14-
_SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII)
15-
16-
DEFAULT_DEST_FOLDER = "./dist"
17-
18-
19-
def create_package(name, dest_folder=DEFAULT_DEST_FOLDER):
20-
# a package will exist in either one, or the other folder. this is why we can resolve both at the same time.
21-
absdirs = [
22-
os.path.dirname(package)
23-
for package in (glob.glob("{}/setup.py".format(name)) + glob.glob("sdk/*/{}/setup.py".format(name)))
24-
]
25-
26-
absdirpath = os.path.abspath(absdirs[0])
27-
check_call(["python", "setup.py", "bdist_wheel", "-d", dest_folder], cwd=absdirpath)
28-
check_call(["python", "setup.py", "sdist", "--format", "zip", "-d", dest_folder], cwd=absdirpath)
29-
30-
31-
def get_package_names(sdk_folder):
32-
files = get_diff_file_list(sdk_folder)
33-
matches = {_SDK_FOLDER_RE.search(f) for f in files}
34-
package_names = {match.groups() for match in matches if match is not None}
35-
return package_names
36-
37-
38-
def change_log_generate(package_name, last_version):
39-
from pypi_tools.pypi import PyPIClient
40-
41-
client = PyPIClient()
42-
try:
43-
last_version[-1] = str(client.get_ordered_versions(package_name)[-1])
44-
except:
45-
return " - Initial Release"
46-
else:
47-
return change_log_main(f"{package_name}:pypi", f"{package_name}:latest")
48-
49-
50-
def _extract_breaking_change(changelog):
51-
log = changelog.split("\n")
52-
breaking_change = []
53-
for i in range(0, len(log)):
54-
if log[i].find("Breaking changes") > -1:
55-
breaking_change = log[min(i + 2, len(log) - 1) :]
56-
break
57-
return sorted([x.replace(" - ", "") for x in breaking_change])
5811

5912

6013
def main(generate_input, generate_output):
@@ -74,7 +27,7 @@ def main(generate_input, generate_output):
7427
package["changelog"] = {
7528
"content": md_output,
7629
"hasBreakingChange": "Breaking changes" in md_output,
77-
"breakingChangeItems": _extract_breaking_change(md_output),
30+
"breakingChangeItems": extract_breaking_change(md_output),
7831
}
7932
package["version"] = last_version[-1]
8033

‎tools/azure-sdk-tools/packaging_tools/generate_utils.py‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
import json
2+
import logging
3+
import os
4+
import re
5+
6+
from azure_devtools.ci_tools.git_tools import get_add_diff_file_list
7+
from pathlib import Path
8+
from subprocess import check_call
9+
10+
from .swaggertosdk.autorest_tools import build_autorest_options
11+
12+
_LOGGER = logging.getLogger(__name__)
13+
_SDK_FOLDER_RE = re.compile(r"^(sdk/[\w-]+)/(azure[\w-]+)/", re.ASCII)
14+
15+
DEFAULT_DEST_FOLDER = "./dist"
16+
17+
18+
def get_package_names(sdk_folder):
19+
files = get_add_diff_file_list(sdk_folder)
20+
matches = {_SDK_FOLDER_RE.search(f) for f in files}
21+
package_names = {match.groups() for match in matches if match is not None}
22+
return package_names
23+
24+
25+
def init_new_service(package_name, folder_name):
26+
setup = Path(folder_name, package_name, "setup.py")
27+
if not setup.exists():
28+
check_call(f"python -m packaging_tools --build-conf {package_name} -o {folder_name}", shell=True)
29+
ci = Path(folder_name, "ci.yml")
30+
if not ci.exists():
31+
with open("ci_template.yml", "r") as file_in:
32+
content = file_in.readlines()
33+
name = package_name.replace("azure-", "").replace("mgmt-", "")
34+
content = [line.replace("MyService", name) for line in content]
35+
with open(str(ci), "w") as file_out:
36+
file_out.writelines(content)
37+
38+
39+
def update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme):
40+
41+
readme_file = str(Path(spec_folder, input_readme))
42+
global_conf = config["meta"]
43+
local_conf = config["projects"][readme_file]
44+
45+
cmd = ["autorest", input_readme]
46+
cmd += build_autorest_options(global_conf, local_conf)
47+
48+
# metadata
49+
metadata = {
50+
"autorest": global_conf["autorest_options"]["version"],
51+
"use": global_conf["autorest_options"]["use"],
52+
"commit": data["headSha"],
53+
"repository_url": data["repoHttpsUrl"],
54+
"autorest_command": " ".join(cmd),
55+
"readme": input_readme,
56+
}
57+
58+
_LOGGER.info("Metadata json:\n {}".format(json.dumps(metadata, indent=2)))
59+
60+
package_folder = Path(sdk_folder, folder_name, package_name).expanduser()
61+
if not os.path.exists(package_folder):
62+
_LOGGER.info(f"Package folder doesn't exist: {package_folder}")
63+
_LOGGER.info("Failed to save metadata.")
64+
return
65+
66+
metadata_file_path = os.path.join(package_folder, "_meta.json")
67+
with open(metadata_file_path, "w") as writer:
68+
json.dump(metadata, writer, indent=2)
69+
_LOGGER.info(f"Saved metadata to {metadata_file_path}")
70+
71+
# Check whether MANIFEST.in includes _meta.json
72+
require_meta = "include _meta.json\n"
73+
manifest_file = os.path.join(package_folder, "MANIFEST.in")
74+
if not os.path.exists(manifest_file):
75+
_LOGGER.info(f"MANIFEST.in doesn't exist: {manifest_file}")
76+
return
77+
78+
includes = []
79+
write_flag = False
80+
with open(manifest_file, "r") as f:
81+
includes = f.readlines()
82+
if require_meta not in includes:
83+
includes = [require_meta] + includes
84+
write_flag = True
85+
86+
if write_flag:
87+
with open(manifest_file, "w") as f:
88+
f.write("".join(includes))

‎tools/azure-sdk-tools/packaging_tools/package_utils.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
import glob
2+
import os
3+
4+
from subprocess import check_call
5+
6+
from .change_log import main as change_log_main
7+
8+
DEFAULT_DEST_FOLDER = "./dist"
9+
10+
11+
def create_package(name, dest_folder=DEFAULT_DEST_FOLDER):
12+
# a package will exist in either one, or the other folder. this is why we can resolve both at the same time.
13+
absdirs = [
14+
os.path.dirname(package)
15+
for package in (glob.glob("{}/setup.py".format(name)) + glob.glob("sdk/*/{}/setup.py".format(name)))
16+
]
17+
18+
absdirpath = os.path.abspath(absdirs[0])
19+
check_call(["python", "setup.py", "bdist_wheel", "-d", dest_folder], cwd=absdirpath)
20+
check_call(["python", "setup.py", "sdist", "--format", "zip", "-d", dest_folder], cwd=absdirpath)
21+
22+
23+
def change_log_generate(package_name, last_version):
24+
from pypi_tools.pypi import PyPIClient
25+
26+
client = PyPIClient()
27+
try:
28+
last_version[-1] = str(client.get_ordered_versions(package_name)[-1])
29+
except:
30+
return " - Initial Release"
31+
else:
32+
return change_log_main(f"{package_name}:pypi", f"{package_name}:latest")
33+
34+
35+
def extract_breaking_change(changelog):
36+
log = changelog.split("\n")
37+
breaking_change = []
38+
for i in range(0, len(log)):
39+
if log[i].find("Breaking changes") > -1:
40+
breaking_change = log[min(i + 2, len(log) - 1) :]
41+
break
42+
return sorted([x.replace(" - ", "") for x in breaking_change])

0 commit comments

Comments
 (0)