Skip to content

Commit 37380a4

Browse files
00Kai000Kai0
authored
Upload metadata function (Azure#17108)
1 parent 110ca79 commit 37380a4

File tree

4 files changed

+218
-1
lines changed

4 files changed

+218
-1
lines changed

‎sdk/template/azure-template/MANIFEST.in‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
include _meta.json
12
include *.md
23
include azure/__init__.py
34
recursive-include tests *.py

‎tools/azure-sdk-tools/packaging_tools/auto_codegen.py‎

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
import argparse
22
import json
33
import logging
4+
import os
45
from pathlib import Path
56
import re
67
from subprocess import check_call
78

89
from .swaggertosdk.SwaggerToSdkCore import (
10+
read_config,
911
CONFIG_FILE,
1012
)
1113
from azure_devtools.ci_tools.git_tools import get_diff_file_list
14+
from .swaggertosdk.autorest_tools import build_autorest_options
1215
from .generate_sdk import generate
1316

1417
_LOGGER = logging.getLogger(__name__)
@@ -37,6 +40,57 @@ def init_new_service(package_name, folder_name):
3740
with open(str(ci), 'w') as file_out:
3841
file_out.writelines(content)
3942

43+
def update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme):
44+
45+
readme_file = str(Path(spec_folder, input_readme))
46+
global_conf = config["meta"]
47+
local_conf = config["projects"][readme_file]
48+
49+
cmd = ["autorest", input_readme]
50+
cmd += build_autorest_options(global_conf, local_conf)
51+
52+
# metadata
53+
metadata = {
54+
"autorest": global_conf["autorest_options"]["version"],
55+
"use": global_conf["autorest_options"]["use"],
56+
"commit": data["headSha"],
57+
"repository_url": data["repoHttpsUrl"],
58+
"autorest_command": " ".join(cmd),
59+
"readme": input_readme
60+
}
61+
62+
_LOGGER.info("Metadata json:\n {}".format(json.dumps(metadata, indent=2)))
63+
64+
package_folder = Path(sdk_folder, folder_name, package_name).expanduser()
65+
if not os.path.exists(package_folder):
66+
_LOGGER.info(f"Package folder doesn't exist: {package_folder}")
67+
_LOGGER.info("Failed to save metadata.")
68+
return
69+
70+
metadata_file_path = os.path.join(package_folder, "_meta.json")
71+
with open(metadata_file_path, "w") as writer:
72+
json.dump(metadata, writer, indent=2)
73+
_LOGGER.info(f"Saved metadata to {metadata_file_path}")
74+
75+
# Check whether MANIFEST.in includes _meta.json
76+
require_meta = "include _meta.json\n"
77+
manifest_file = os.path.join(package_folder, "MANIFEST.in")
78+
if not os.path.exists(manifest_file):
79+
_LOGGER.info(f"MANIFEST.in doesn't exist: {manifest_file}")
80+
return
81+
82+
includes = []
83+
write_flag = False
84+
with open(manifest_file, "r") as f:
85+
includes = f.readlines()
86+
if require_meta not in includes:
87+
includes = [require_meta] + includes
88+
write_flag = True
89+
90+
if write_flag:
91+
with open(manifest_file, "w") as f:
92+
f.write("".join(includes))
93+
4094

4195
def main(generate_input, generate_output):
4296
with open(generate_input, "r") as reader:
@@ -49,7 +103,7 @@ def main(generate_input, generate_output):
49103
for input_readme in data["relatedReadmeMdFiles"]:
50104
relative_path_readme = str(Path(spec_folder, input_readme))
51105
_LOGGER.info(f'[CODEGEN]({input_readme})codegen begin')
52-
generate(CONFIG_FILE,
106+
config = generate(CONFIG_FILE,
53107
sdk_folder,
54108
[],
55109
relative_path_readme,
@@ -59,6 +113,7 @@ def main(generate_input, generate_output):
59113
package_names = get_package_names(sdk_folder)
60114
_LOGGER.info(f'[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]')
61115

116+
62117
for folder_name, package_name in package_names:
63118
if package_name in package_total:
64119
continue
@@ -77,6 +132,12 @@ def main(generate_input, generate_output):
77132
# Generate some necessary file for new service
78133
init_new_service(package_name, folder_name)
79134

135+
# Update metadata
136+
try:
137+
update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme)
138+
except Exception as e:
139+
_LOGGER.info(str(e))
140+
80141
# Setup package locally
81142
check_call(f'pip install --ignore-requires-python -e {str(Path(sdk_folder, folder_name, package_name))}',
82143
shell=True)

‎tools/azure-sdk-tools/packaging_tools/generate_sdk.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ def generate(config_path, sdk_folder, project_pattern, readme, restapi_git_folde
8282
local_conf,
8383
autorest_bin
8484
)
85+
return config
8586

8687

8788
def generate_main():

‎tools/azure-sdk-tools/tests/test_servicemetadata.py‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
import os
2+
import json
3+
import tempfile
4+
import unittest
5+
from pathlib import Path
6+
7+
import pytest
8+
from packaging_tools.auto_codegen import update_servicemetadata
9+
10+
11+
"""
12+
Create metadata file
13+
14+
Update metadata file
15+
16+
Update MANIFETS.IN
17+
18+
No need to update MANIFETS.IN
19+
"""
20+
21+
MANIFEST_TEMP = """recursive-include tests *.py *.yaml
22+
include *.md
23+
include azure/__init__.py
24+
include azure/mgmt/__init__.py
25+
"""
26+
27+
class TestServiceMetadata(unittest.TestCase):
28+
29+
def setUp(self):
30+
self.sdk_folder = "sdk"
31+
self.data = {
32+
"specFolder": "../azure-rest-api-specs",
33+
"headSha": "e295fe97eee3709668d3e5f7f8b434026b814ef9",
34+
"headRef": "master",
35+
"repoHttpsUrl": "https://github.com/Azure/azure-rest-api-specs",
36+
}
37+
self.config = {
38+
"meta": {
39+
"autorest_options": {
40+
"version": "3.0.6369",
41+
"use": "@autorest/python@5.4.3",
42+
"python": "",
43+
"python-mode": "update",
44+
"sdkrel:python-sdks-folder": "./sdk/.",
45+
"multiapi": "",
46+
"track2": ""
47+
},
48+
"advanced_options": {
49+
"create_sdk_pull_requests": True,
50+
"sdk_generation_pull_request_base": "integration_branch"
51+
},
52+
"repotag": "azure-sdk-for-python-track2",
53+
"version": "0.2.0"
54+
},
55+
"projects": {
56+
"./azure-rest-api-specs/specification/operationalinsights/resource-manager/readme.md": {}
57+
}
58+
59+
}
60+
self.folder_name = "monitor"
61+
self.package_name = "azure-mgmt-monitor"
62+
self.spec_folder = "./"
63+
self.input_readme = "azure-rest-api-specs/specification/operationalinsights/resource-manager/readme.md"
64+
65+
def test_metadata(self):
66+
67+
with tempfile.TemporaryDirectory() as temp_dir:
68+
# Init directories
69+
self.sdk_folder = str(Path(temp_dir, "sdk"))
70+
self.spec_folder = str(Path(temp_dir, "spec"))
71+
os.makedirs(self.sdk_folder)
72+
os.makedirs(self.spec_folder)
73+
74+
readme_file = str(Path(self.spec_folder, self.input_readme))
75+
self.config["projects"][readme_file] = {}
76+
77+
package_folder = Path(self.sdk_folder, self.folder_name, self.package_name).expanduser()
78+
metadata_file_path = os.path.join(package_folder, "_meta.json")
79+
manifest_file = os.path.join(package_folder, "MANIFEST.in")
80+
os.makedirs(package_folder)
81+
82+
# Test MANIFEST.in does not exist
83+
update_servicemetadata(
84+
sdk_folder=self.sdk_folder,
85+
data=self.data,
86+
config=self.config,
87+
folder_name=self.folder_name,
88+
package_name=self.package_name,
89+
spec_folder=self.spec_folder,
90+
input_readme=self.input_readme
91+
)
92+
93+
assert os.path.isfile(metadata_file_path) == True
94+
# Do not create MANIFEST.in, if it does not exist
95+
assert os.path.isfile(manifest_file) == False
96+
97+
# Test update metadata
98+
with open(metadata_file_path, "w") as f:
99+
json.dump({"autorest": "3.0.0"}, f, indent=2)
100+
update_servicemetadata(
101+
sdk_folder=self.sdk_folder,
102+
data=self.data,
103+
config=self.config,
104+
folder_name=self.folder_name,
105+
package_name=self.package_name,
106+
spec_folder=self.spec_folder,
107+
input_readme=self.input_readme
108+
)
109+
110+
assert os.path.isfile(metadata_file_path) == True
111+
assert os.path.isfile(manifest_file) == False
112+
with open(metadata_file_path, "r") as f:
113+
md = json.load(f)
114+
assert md["autorest"] == "3.0.6369"
115+
116+
# Test update MANIFEST.in
117+
with open(manifest_file, "w") as f:
118+
f.write(MANIFEST_TEMP)
119+
update_servicemetadata(
120+
sdk_folder=self.sdk_folder,
121+
data=self.data,
122+
config=self.config,
123+
folder_name=self.folder_name,
124+
package_name=self.package_name,
125+
spec_folder=self.spec_folder,
126+
input_readme=self.input_readme
127+
)
128+
129+
assert os.path.isfile(metadata_file_path) == True
130+
assert os.path.isfile(manifest_file) == True
131+
with open(manifest_file, "r") as f:
132+
meta_line = "include _meta.json\n"
133+
line = f.readline()
134+
assert meta_line == line
135+
136+
# Test update MANIFEST.in again
137+
update_servicemetadata(
138+
sdk_folder=self.sdk_folder,
139+
data=self.data,
140+
config=self.config,
141+
folder_name=self.folder_name,
142+
package_name=self.package_name,
143+
spec_folder=self.spec_folder,
144+
input_readme=self.input_readme
145+
)
146+
147+
assert os.path.isfile(metadata_file_path) == True
148+
assert os.path.isfile(manifest_file) == True
149+
with open(manifest_file, "r") as f:
150+
meta_line = "include _meta.json\n"
151+
line = f.readline()
152+
second_line = f.readline()
153+
assert meta_line == line
154+
assert meta_line != second_line

0 commit comments

Comments
 (0)