Skip to content

Commit 189bd8c

Browse files
weidongxu-microsoftweidongxu-microsoft
authored
LLC, improve codegen script (Azure#26042)
* add artifact-id option * support title in script * add TITLE in pipeline * refactor, data-plane CI also handle external files like CI and root POM * upgrade codegen * update script help
1 parent 45971e4 commit 189bd8c

File tree

5 files changed

+218
-154
lines changed

5 files changed

+218
-154
lines changed

‎eng/mgmt/automation/generate.py‎

Lines changed: 8 additions & 142 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,15 @@
1111
import requests
1212
import tempfile
1313
import subprocess
14-
import collections
1514
import urllib.parse
1615
from typing import Tuple
1716

1817
pwd = os.getcwd()
1918
os.chdir(os.path.abspath(os.path.dirname(sys.argv[0])))
2019
from parameters import *
20+
from utils import update_service_ci_and_pom
21+
from utils import update_root_pom
22+
from utils import update_version
2123
from generate_data import sdk_automation as sdk_automation_data
2224
os.chdir(pwd)
2325

@@ -76,9 +78,12 @@ def generate(
7678
logging.error('[GENERATE] Autorest fail')
7779
return False
7880

79-
update_service_ci_and_pom(sdk_root, service)
81+
module = ARTIFACT_FORMAT.format(service)
82+
group = GROUP_ID
83+
output_folder = OUTPUT_FOLDER_FORMAT.format(service)
84+
update_service_ci_and_pom(sdk_root, group, service, module)
8085
update_root_pom(sdk_root, service)
81-
update_version(sdk_root, service)
86+
update_version(sdk_root, output_folder)
8287

8388
return True
8489

@@ -189,145 +194,6 @@ def compare_with_maven_package(sdk_root, service, stable_version,
189194
os.remove(old_jar)
190195

191196

192-
def add_module_to_modules(modules: str, module: str) -> str:
193-
post_module = re.search(r'([^\S\n\r]*)</modules>', modules)
194-
indent = post_module.group(1)
195-
indent += ' '
196-
197-
all_module = set(re.findall(r'<module>(.*?)</module>', modules))
198-
all_module.add(module)
199-
all_module = [
200-
indent + POM_MODULE_FORMAT.format(module)
201-
for module in sorted(all_module)
202-
]
203-
204-
return '<modules>\n' + ''.join(all_module) + post_module.group()
205-
206-
207-
def add_module_to_default_profile(pom: str, module: str) -> Tuple[bool, str]:
208-
for profile in re.finditer(r'<profile>[\s\S]*?</profile>', pom):
209-
profile_value = profile.group()
210-
if re.search(r'<id>default</id>', profile_value):
211-
if len(re.findall('<modules>', profile_value)) > 1:
212-
logging.error(
213-
'[POM][Profile][Skip] find more than one <modules> in <profile> default'
214-
)
215-
return (False, '')
216-
modules = re.search(r'<modules>[\s\S]*</modules>', profile_value)
217-
if not modules:
218-
logging.error(
219-
'[POM][Profile][Skip] Cannot find <modules> in <profile> default'
220-
)
221-
return (False, '')
222-
modules_update = add_module_to_modules(modules.group(), module)
223-
pre_modules = pom[:profile.start() + modules.start()]
224-
post_modules = pom[profile.start() + modules.end():]
225-
return (True, pre_modules + modules_update + post_modules)
226-
logging.error(
227-
'[POM][Profile][Skip] cannot find <profile> with <id> default')
228-
return (False, '')
229-
230-
231-
def add_module_to_pom(pom: str, module: str) -> Tuple[bool, str]:
232-
if pom.find('<module>{0}</module>'.format(module)) >= 0:
233-
logging.info('[POM][Skip] pom already has module {0}'.format(module))
234-
return (True, pom)
235-
236-
if len(re.findall('<modules>', pom)) > 1:
237-
if pom.find('<profiles>') >= 0:
238-
return add_module_to_default_profile(pom, module)
239-
logging.error('[POM][Skip] find more than one <modules> in pom')
240-
return (False, '')
241-
242-
modules = re.search(r'<modules>[\s\S]*</modules>', pom)
243-
if not modules:
244-
logging.error('[POM][Skip] Cannot find <modules> in pom')
245-
return (False, '')
246-
247-
modules_update = add_module_to_modules(modules.group(), module)
248-
pre_modules = pom[:modules.start()]
249-
post_modules = pom[modules.end():]
250-
return (True, pre_modules + modules_update + post_modules)
251-
252-
253-
def update_root_pom(sdk_root: str, service: str):
254-
pom_file = os.path.join(sdk_root, 'pom.xml')
255-
if not os.path.exists(pom_file):
256-
logging.error('[POM][Skip] cannot find root pom')
257-
return
258-
259-
module = 'sdk/{0}'.format(service)
260-
with open(pom_file, 'r') as fin:
261-
pom = fin.read()
262-
263-
logging.info('[POM][Process] dealing with root pom')
264-
success, pom = add_module_to_pom(pom, module)
265-
if success:
266-
with open(pom_file, 'w') as fout:
267-
fout.write(pom)
268-
logging.info('[POM][Success] Write to root pom')
269-
270-
271-
def update_service_ci_and_pom(sdk_root: str, service: str):
272-
folder = os.path.join(sdk_root, 'sdk/{0}'.format(service))
273-
module = ARTIFACT_FORMAT.format(service)
274-
ci_yml_file = os.path.join(folder, 'ci.yml')
275-
pom_xml_file = os.path.join(folder, 'pom.xml')
276-
277-
if os.path.exists(ci_yml_file):
278-
with open(ci_yml_file, 'r') as fin:
279-
ci_yml = yaml.safe_load(fin)
280-
sdk_type: str = ci_yml.get('extends', dict()).get('parameters', dict()).get('SDKType', '')
281-
if type(sdk_type) == str and sdk_type.lower() == 'data':
282-
os.rename(ci_yml_file, os.path.join(os.path.dirname(ci_yml_file), 'ci.data.yml'))
283-
ci_yml = yaml.safe_load(CI_FORMAT.format(service, module))
284-
else:
285-
ci_yml = yaml.safe_load(CI_FORMAT.format(service, module))
286-
287-
if not (type(ci_yml.get('extends')) == dict and
288-
type(ci_yml['extends'].get('parameters')) == dict and
289-
type(ci_yml['extends']['parameters'].get('Artifacts')) == list):
290-
logging.error('[CI][Skip] Unexpected ci.yml format')
291-
else:
292-
artifacts: list = ci_yml['extends']['parameters']['Artifacts']
293-
for artifact in artifacts:
294-
if (artifact.get('name') == module and
295-
artifact.get('groupId') == GROUP_ID):
296-
logging.info(
297-
'[CI][Skip] ci.yml already has module {0}'.format(module))
298-
break
299-
else:
300-
artifacts.append({
301-
'name': module,
302-
'groupId': GROUP_ID,
303-
'safeName': module.replace('-', '')
304-
})
305-
ci_yml_str = yaml.dump(ci_yml,
306-
sort_keys = False,
307-
Dumper = ListIndentDumper)
308-
ci_yml_str = re.sub('(\n\S)', r'\n\1', ci_yml_str)
309-
310-
with open(ci_yml_file, 'w') as fout:
311-
fout.write(CI_HEADER)
312-
fout.write(ci_yml_str)
313-
logging.info('[CI][Success] Write to ci.yml')
314-
315-
if os.path.exists(pom_xml_file):
316-
with open(pom_xml_file, 'r') as fin:
317-
pom_xml = fin.read()
318-
else:
319-
pom_xml = POM_FORMAT.format(service = service,
320-
group_id = GROUP_ID,
321-
artifact_id = module)
322-
323-
logging.info('[POM][Process] dealing with pom.xml')
324-
success, pom_xml = add_module_to_pom(pom_xml, module)
325-
if success:
326-
with open(pom_xml_file, 'w') as fout:
327-
fout.write(pom_xml)
328-
logging.info('[POM][Success] Write to pom.xml')
329-
330-
331197
def get_version(
332198
sdk_root: str,
333199
service: str,

‎eng/mgmt/automation/generate_data.py‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99
from typing import List
1010

1111
from parameters import *
12+
from utils import update_service_ci_and_pom
13+
from utils import update_root_pom
14+
from utils import update_version
1215

1316

1417
LLC_ARGUMENTS = '--java --low-level-client --sdk-integration --generate-samples'
@@ -75,6 +78,7 @@ def generate(
7578
module: str,
7679
credential_types: str,
7780
credential_scopes: str,
81+
title: str,
7882
autorest: str,
7983
use: str,
8084
autorest_options: str = '',
@@ -95,19 +99,28 @@ def generate(
9599

96100
input_arguments = '--input-file={0}'.format(input_file)
97101

102+
artifact_arguments = '--artifact-id={0}'.format(module)
103+
if title:
104+
artifact_arguments += ' --title={0}'.format(title)
105+
98106
command = 'autorest --version={0} --use={1} --java.azure-libraries-for-java-folder={2} --java.output-folder={3} --java.namespace={4} {5}'.format(
99107
autorest,
100108
use,
101109
os.path.abspath(sdk_root),
102110
os.path.abspath(output_dir),
103111
namespace,
104-
' '.join((LLC_ARGUMENTS, input_arguments, credential_arguments, autorest_options)),
112+
' '.join((LLC_ARGUMENTS, input_arguments, credential_arguments, artifact_arguments, autorest_options)),
105113
)
106114
logging.info(command)
107115
if os.system(command) != 0:
108116
logging.error('[GENERATE] Autorest fail')
109117
return False
110118

119+
group = "com.azure"
120+
update_service_ci_and_pom(sdk_root, group, service, module)
121+
update_root_pom(sdk_root, service)
122+
update_version(sdk_root, output_dir)
123+
111124
return True
112125

113126

@@ -134,31 +147,37 @@ def parse_args() -> argparse.Namespace:
134147
parser.add_argument(
135148
'--input-file',
136149
required=True,
137-
help='URL to OpenAPI 2.0 specification JSON as input file',
150+
help='URL to OpenAPI 2.0 specification JSON as input file.',
138151
)
139152
parser.add_argument(
140153
'--service',
141154
required=True,
142-
help='Service name under sdk/, sample: storage',
155+
help='Service name under sdk/. Sample: storage',
143156
)
144157
parser.add_argument(
145158
'--module',
146159
required=True,
147-
help='Module name under sdk/<service>/, sample: azure-storage-blob',
160+
help='Module name under sdk/<service>/. Sample: azure-storage-blob',
148161
)
149162
parser.add_argument(
150163
'--credential-types',
151164
required=True,
152-
help='Credential types, '
165+
help='Credential types. '
153166
'Sample: "tokencredential" for AAD credential for OAuth 2.0 authentication; '
154167
'"azurekeycredential" for Azure key credential',
155168
)
156169
parser.add_argument(
157170
'--credential-scopes',
158171
required=False,
159-
help='OAuth 2.0 scopes when credential-types includes tokencredential, '
172+
help='OAuth 2.0 scopes when credential-types includes "tokencredential". '
160173
'Sample: https://storage.azure.com/.default',
161174
)
175+
parser.add_argument(
176+
'--title',
177+
required=False,
178+
help='The name of the client. The name should always ends with "Client". '
179+
'Sample: BlobClient, which makes BlobClientBuilder as builder class',
180+
)
162181
parser.add_argument(
163182
'-u',
164183
'--use',

‎eng/mgmt/automation/generation_data.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ steps:
3333
- bash: |
3434
export PATH=$JAVA_HOME_11_X64/bin:$PATH
3535
java -version
36-
./eng/mgmt/automation/generate_data.py --input-file="$(INPUT_FILE)" --service="$(SERVICE)" --module="$(MODULE)" --credential-types="$(CREDENTIAL_TYPES)" --credential-scopes="$(CREDENTIAL_SCOPES)"
36+
./eng/mgmt/automation/generate_data.py --input-file="$(INPUT_FILE)" --service="$(SERVICE)" --module="$(MODULE)" --credential-types="$(CREDENTIAL_TYPES)" --credential-scopes="$(CREDENTIAL_SCOPES)" --title="$(TITLE)"
3737
displayName: Generation
3838

3939
- template: /eng/common/pipelines/templates/steps/create-pull-request.yml

‎eng/mgmt/automation/parameters.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
SDK_ROOT = '../../../' # related to file dir
1818
AUTOREST_CORE_VERSION = '3.6.6'
19-
AUTOREST_JAVA = '@autorest/java@4.0.42'
19+
AUTOREST_JAVA = '@autorest/java@4.0.43'
2020
DEFAULT_VERSION = '1.0.0-beta.1'
2121
GROUP_ID = 'com.azure.resourcemanager'
2222
API_SPECS_FILE = 'api-specs.yaml'
@@ -46,10 +46,10 @@
4646
paths:
4747
include:
4848
- sdk/{0}/ci.yml
49-
- sdk/{0}/azure-resourcemanager-{0}/
49+
- sdk/{0}/{1}/
5050
exclude:
5151
- sdk/{0}/pom.xml
52-
- sdk/{0}/azure-resourcemanager-{0}/pom.xml
52+
- sdk/{0}/{1}/pom.xml
5353
5454
pr:
5555
branches:
@@ -61,10 +61,10 @@
6161
paths:
6262
include:
6363
- sdk/{0}/ci.yml
64-
- sdk/{0}/azure-resourcemanager-{0}/
64+
- sdk/{0}/{1}/
6565
exclude:
6666
- sdk/{0}/pom.xml
67-
- sdk/{0}/azure-resourcemanager-{0}/pom.xml
67+
- sdk/{0}/{1}/pom.xml
6868
6969
extends:
7070
template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml

0 commit comments

Comments
 (0)