[Release helper] Update (Azure#22196)

* update for Go

* check tag consistency and optize reply

* add write.md

* update go js readme

* update

* Update common.py

* Update common.py

* update assignee for JS

* update

* update

* Update common.py

* Update common.py

* Update release_helper.yml for Azure Pipelines

* Update common.py

* update

* Update common.py

* Update common.py

* Update main.py

* update

* update bot advice

* update

* update

* update excel

* update output function

* add exception handle for bad credential

* fix static varaible in class

* update auto_assignee algorithm

* update bot token to have a try

* update bot token

* force to single process

Co-authored-by: Zed <601306339@qq.com>
Co-authored-by: Yiming Lei <59104634+RAY-316@users.noreply.github.com>

Author: msyyc

scripts/release_helper/common.py

@@ -1,52 +1,62 @@
 from datetime import date, datetime
 from typing import Set, List, Dict
 import os
-from utils import IssuePackage, REQUEST_REPO, AUTO_ASSIGN_LABEL, AUTO_PARSE_LABEL, get_origin_link_and_tag
+from utils import IssuePackage, REQUEST_REPO, AUTO_ASSIGN_LABEL, AUTO_PARSE_LABEL, get_origin_link_and_tag,\
+    MULTI_LINK_LABEL
 import re
 import logging
 import time
+from random import randint
 from github import Github
 from github.Repository import Repository
-import subprocess as sp
 
 _LOG = logging.getLogger(__name__)
 
 # assignee dict which will be assigned to handle issues
 _LANGUAGE_OWNER = {'msyyc'}
 
 # 'github assignee': 'token'
-_ASSIGNEE_TOKEN = {'msyyc': os.getenv('PYTHON_MSYYC_TOKEN')}
+_ASSIGNEE_TOKEN = {'msyyc': os.getenv('AZURESDK_BOT_TOKEN')}
 
 _SWAGGER_URL = 'https://github.com/Azure/azure-rest-api-specs/blob/main/specification'
 _SWAGGER_PULL = 'https://github.com/Azure/azure-rest-api-specs/pull'
 
 
-_SWAGGER_URL = 'https://github.com/Azure/azure-rest-api-specs/blob/main/specification'
-_SWAGGER_PULL = 'https://github.com/Azure/azure-rest-api-specs/pull'
-
 class IssueProcess:
+    """
     # won't be changed anymore after __init__
     request_repo_dict = {}  # request repo instance generated by different token
     owner = ''  # issue owner
     assignee_candidates = {}  # assignee candidates who will be assigned to handle issue
     language_owner = {}  # language owner who may handle issue
 
     # will be changed by order
-    issue = None  # issue that needs to handle
+    issue_package = None  # issue that needs to handle
     assignee = ''
-    bot = []  # bot advice to help SDK owner
+    bot_advice = []  # bot advice to help SDK owner
     target_readme_tag = ''  # swagger content that customers want
     readme_link = ''  # https link which swagger definition is in
     default_readme_tag = ''  # configured in `README.md`
+    package_name = ''  # target package name
+    target_date = ''  # target release date asked by customer
+    date_from_target = 0
+    """
 
-    def __init__(self, issue: IssuePackage, request_repo_dict: Dict[str, Repository],
+    def __init__(self, issue_package: IssuePackage, request_repo_dict: Dict[str, Repository],
                  assignee_candidates: Set[str], language_owner: Set[str]):
-        self.issue_package = issue
+        self.issue_package = issue_package
         self.request_repo_dict = request_repo_dict
-        self.assignee = issue.issue.assignee.login
-        self.owner = issue.issue.user.login
+        self.assignee = issue_package.issue.assignee.login
+        self.owner = issue_package.issue.user.login
         self.assignee_candidates = assignee_candidates
         self.language_owner = language_owner
+        self.bot_advice = []
+        self.target_readme_tag = ''
+        self.readme_link = ''
+        self.default_readme_tag = ''
+        self.package_name = ''
+        self.target_date = ''
+        self.date_from_target = 0
 
     def get_issue_body(self) -> List[str]:
         return [i for i in self.issue_package.issue.body.split("\n") if i]
@@ -82,8 +92,7 @@ def get_readme_from_pr_link(self, link: str) -> str:
             pr = f"{_SWAGGER_PULL}/{pr_number}"
             self.comment(
                 f'Hi, @{self.assignee}, by parsing {pr}, there are multi service link: {multi_link}. Please decide which one is the right.')
-
-            self.bot.append('multi readme link!')
+            self.add_label(MULTI_LINK_LABEL)
             raise Exception(f'multi link in "{pr}"')
 
         return readme_link[0]
@@ -190,7 +199,7 @@ def auto_assign(self) -> None:
             return
         # assign averagely
         assignees = list(self.assignee_candidates)
-        random_idx = int(str(time.time())[-1]) % len(assignees) if len(assignees) > 1 else 0
+        random_idx = randint(0, len(assignees) - 1) if len(assignees) > 1 else 0
         assignee = assignees[random_idx]
 
         # update assignee
@@ -203,83 +212,127 @@ def auto_assign(self) -> None:
             self.update_issue_instance()
         self.add_label(AUTO_ASSIGN_LABEL)
 
-    def bot_advice(self):
-        latest_comments = ''
+    def new_issue_policy(self):
+        new_issue_advice = 'new issue.'
+        if self.issue_package.issue.comments == 0:
+            self.bot_advice.append(new_issue_advice)
+        else:
+            # issue that no comment from language owner will also be treated as new issue
+            comment_from_owner = set(comment.user.login for comment in self.issue_package.issue.get_comments()
+                                     if comment.user.login in self.language_owner)
+            if not comment_from_owner:
+                self.bot_advice.append(new_issue_advice)
+
+    def new_comment_policy(self):
+        if self.issue_package.issue.comments == 0:
+            return
         comments = [(comment.updated_at.timestamp(), comment.user.login) for comment in
                     self.issue_package.issue.get_comments()]
         comments.sort()
-        if comments:
-            latest_comments = comments[-1][1]
-        if self.issue_package.issue.comments == 0:
-            self.bot = 'new issue ! <br>'
-        elif latest_comments not in self.language_owner:
-            self.bot = 'new comment.  <br>'
+        latest_comments = comments[-1][1]
+        if latest_comments not in self.language_owner:
+            self.bot_advice.append('new comment.')
+
+    def multi_link_policy(self):
+        if MULTI_LINK_LABEL in self.issue_package.labels_name:
+            self.bot_advice.append('multi readme link!')
+
+    def remind_logic(self) -> bool:
+        return abs(self.date_from_target) <= 2
+
+    def print_date_from_target_date(self) -> str:
+        return str(self.date_from_target) if self.remind_logic() else ''
+
+    def date_remind_policy(self):
+        if self.remind_logic():
+            self.bot_advice.append('close to release date. ')
+
+    def auto_bot_advice(self):
+        self.new_issue_policy()
+        self.new_comment_policy()
+        self.multi_link_policy()
+        self.date_remind_policy()
+
+    def get_target_date(self):
+        body = self.get_issue_body()
+        try:
+            self.target_date = [line.split(':')[-1].strip() for line in body if 'Target release date' in line][0]
+            self.date_from_target = int((time.mktime(time.strptime(self.target_date, '%Y-%m-%d')) - time.time()) / 3600 / 24)
+        except Exception:
+            self.target_date = 'fail to get.'
+            self.date_from_target = 1000  # make a ridiculous data to remind failure when error happens
 
     def run(self) -> None:
         # common part(don't change the order)
         self.auto_assign()  # necessary flow
         self.auto_parse()  # necessary flow
-        self.bot_advice()
+        self.get_target_date()
+        self.auto_bot_advice()  # make sure this is the last step
 
 
 class Common:
-    """ The class defines some function for all languages to reference """
+    """ The class defines some function for all languages to reference
     issues_package = None  # issues that need to handle
     request_repo_dict = {}  # request repo instance generated by different token
     assignee_candidates = {}  # assignee candidates who will be assigned to handle issue
     language_owner = {}  # language owner who may handle issue
+    result = []
+    file_out_name = ''  # file that storages issue status
+    """
 
-    def __init__(self, issues: List[IssuePackage], assignee_token: Dict[str, str], language_owner: Set[str]):
-        self.issues_package = issues
+    def __init__(self, issues_package: List[IssuePackage], assignee_token: Dict[str, str], language_owner: Set[str]):
+        self.issues_package = issues_package
         self.assignee_candidates = set(assignee_token.keys())
         self.language_owner = language_owner
         # arguments add to language.md
         self.file_out_name = 'common.md'
         self.target_release_date = ''
         self.date_from_target = ''
         self.package_name = ''
+        self.result = []
+        self.request_repo_dict = {}
 
         for assignee in assignee_token:
             self.request_repo_dict[assignee] = Github(assignee_token[assignee]).get_repo(REQUEST_REPO)
 
-    def output_md(self, items):
+    def output(self):
         with open(self.file_out_name, 'w') as file_out:
-            file_out.write(
-                '| issue | author | package | assignee | bot advice | created date of issue | target release date | date from target |\n')
+            file_out.write('| issue | author | package | assignee | bot advice | created date of issue | target release date | date from target |\n')
             file_out.write('| ------ | ------ | ------ | ------ | ------ | ------ | ------ | :-----: |\n')
-            file_out.writelines([self.output_python(item) for item in items])
+            for item in self.result:
+                try:
+                    item_status = Common.output_md(item)
+                    file_out.write(item_status)
+                except Exception as e:
+                    _LOG.error(f'Error happened during output result of handled issue {item.issue_package.issue.number}: {e}')
 
-    def output_python(self, item):
+    @staticmethod
+    def output_md(item: IssueProcess):
         create_date = str(date.fromtimestamp(item.issue_package.issue.created_at.timestamp()).strftime('%m-%d'))
+        target_date = str(datetime.strptime(item.target_date, "%Y-%m-%d").strftime('%m-%d'))
 
         return '| [#{}]({}) | {} | {} | {} | {} | {} | {} | {} |\n'.format(
             item.issue_package.issue.html_url.split('/')[-1],
             item.issue_package.issue.html_url,
             item.issue_package.issue.user.login,
-            self.package_name,
-            item.issue_package.issue.assignee.login,
-            item.bot,
+            item.package_name,
+            item.assignee,
+            ' '.join(item.bot_advice),
             create_date,
-            self.target_release_date,
-            self.date_from_target
+            target_date,
+            item.print_date_from_target_date()
         )
 
-    @staticmethod
-    def push_md_to_storage():
-        cmd_list = ['git add .', 'git commit -m \"update excel\"', 'git push -f origin HEAD']
-        [sp.check_call(cmd, shell=True) for cmd in cmd_list]
-
     def run(self):
         items = []
         for item in self.issues_package:
             issue = IssueProcess(item, self.request_repo_dict, self.assignee_candidates, self.language_owner)
             try:
                 issue.run()
-                items.append(issue)
+                self.result.append(issue)
             except Exception as e:
                 _LOG.error(f'Error happened during handling issue {item.issue.number}: {e}')
-        self.output_md(items)
-
+        self.output()
 
 
 def common_process(issues: List[IssuePackage]):

scripts/release_helper/go.py

@@ -6,7 +6,7 @@
 _GO_OWNER = {'ArcturusZhang'}
 
 # 'github assignee': 'token'
-_ASSIGNEE_TOKEN_GO = {'ArcturusZhang': os.getenv('GO_DAPENGZHANG_TOKEN')}
+_ASSIGNEE_TOKEN_GO = {'ArcturusZhang': os.getenv('AZURESDK_BOT_TOKEN')}
 
 
 class IssueProcessGo(IssueProcess):

scripts/release_helper/java.py

@@ -8,9 +8,9 @@
 
 # 'github assignee': 'token'
 _ASSIGNEE_TOKEN_JAVA = {
-    'weidongxu-microsoft': os.getenv('JAVA_WEIDONGXU_TOKEN'),
-    'haolingdong-msft': os.getenv('JAVA_WEIDONGXU_TOKEN'),
-    'XiaofeiCao': os.getenv('JAVA_WEIDONGXU_TOKEN'),
+    'weidongxu-microsoft': os.getenv('AZURESDK_BOT_TOKEN'),
+    'haolingdong-msft': os.getenv('AZURESDK_BOT_TOKEN'),
+    'XiaofeiCao': os.getenv('AZURESDK_BOT_TOKEN'),
 }
 
 

scripts/release_helper/js.py

@@ -6,7 +6,7 @@
 _JS_OWNER = {'qiaozha', 'lirenhe'}
 
 # 'github assignee': 'token'
-_ASSIGNEE_TOKEN_JS = {'qiaozha': os.getenv('JS_QIAOQIAO_TOKEN')}
+_ASSIGNEE_TOKEN_JS = {'qiaozha': os.getenv('AZURESDK_BOT_TOKEN')}
 
 
 class IssueProcessJs(IssueProcess):

scripts/release_helper/main.py

@@ -6,6 +6,7 @@
 from java import java_process
 from js import js_process
 from common import common_process, Common
+import subprocess as sp
 
 
 import os
@@ -53,9 +54,15 @@ def main():
     language = os.getenv('LANGUAGE')
     languages = {_CONVERT[language]: _LANGUAGES[_CONVERT[language]]} if language in _CONVERT else _LANGUAGES
     for language in languages:
-        language_issues = select_language_issues(issues, language)
-        languages[language](language_issues)
-    Common.push_md_to_storage()
+        try:
+            language_issues = select_language_issues(issues, language)
+            languages[language](language_issues)
+        except Exception as e:
+            _LOG.error(f'Error happened during handling {language} issue: {e}')
+
+    # output
+    cmd_list = ['git add -u', 'git commit -m \"update excel\"', 'git push -f origin HEAD']
+    [sp.call(cmd, shell=True) for cmd in cmd_list]
 
 
 if __name__ == '__main__':

scripts/release_helper/release_helper.yml

@@ -15,12 +15,18 @@ schedules:
       - main
     always: true
 
+variables:
+  - group: Release Secrets for GitHub
+  - group: SDK Release Helper
+  - group: Azure SDK Auto Release Pipeline Secrets
+
+
 jobs:
 - job: ReleaseHelper
   displayName: ReleaseHelper Python 3.8
   timeoutInMinutes: 30
   strategy:
-    maxParallel: 3
+    maxParallel: 1
   pool:
     vmImage: 'ubuntu-20.04'
   steps:
@@ -44,11 +50,10 @@ jobs:
         export PYTHON_BIGCAT_TOKEN=$(Jiefeng-GitToken)
         export PYTHON_ZED_TOKEN=$(Zed-GitToken)
         export PYTHON_MSYYC_TOKEN=$(Yuchao-GitToken)
-        export JAVA_WEIDONGXU_TOKEN=$(Weidong-GitToken)
-        export GO_DAPENGZHANG_TOKEN=$(Dapeng-GitToken)
-        export JS_QIAOQIAO_TOKEN=$(QiaoQiao-GitToken)
+        export AZURESDK_BOT_TOKEN=$(azuresdk-github-pat)
         export LANGUAGE=$(RUN_LANGUAGE)
 
+
         # create virtual env
         python -m venv venv-sdk
         source venv-sdk/bin/activate

scripts/release_helper/utils.py

@@ -7,6 +7,7 @@
 REST_REPO = 'Azure/azure-rest-api-specs'
 AUTO_ASSIGN_LABEL = 'assigned'
 AUTO_PARSE_LABEL = 'auto-link'
+MULTI_LINK_LABEL = 'MultiLink'
 
 _LOG = logging.getLogger(__name__)