generator release-v2: add feature of automatically create release pull request from release request (Azure#18455)

* Auto SDK Release

* add comment

* goimports -w

* format

* resolve comment

* Hi

* recover

* Use the 'attention' label to determine that the PR has been generated

* attention label replace to 'PR ready' label

* execute .ps1

* add issue 'PR ready' label

* add issue lable

* fix

* resolve comment

* Handle multi-package releasee

* fix

* fix

* remove isPrReady func

* fix

* issue label

* rename to Invoke-MgmtTestGen.ps1

* fix

* git push msg

* fix GetRemoteUserName func

* fix get pullrequest html_url

* fix

* fix html_url

Author: Alancere

eng/tools/generator/cmd/issue/issueCmd.go

@@ -179,22 +179,43 @@ func (c *commandContext) listSpecifiedIssues(ids []int) ([]*github.Issue, error)
 	return issues, nil
 }
 
-func issueHasLabel(issue *github.Issue, label string) bool {
+func issueHasLabel(issue *github.Issue, label IssueLabel) bool {
 	if issue == nil {
 		return false
 	}
 
 	for _, l := range issue.Labels {
-		if l.GetName() == label {
+		if IssueLabel(l.GetName()) == label {
 			return true
 		}
 	}
 
 	return false
 }
 
+type IssueLabel string
+
+const (
+	GoLabel              IssueLabel = "GO"
+	AutoLinkLabel        IssueLabel = "auto-link"
+	PRreadyLabel         IssueLabel = "PRready"
+	InconsistentTagLabel IssueLabel = "Inconsistent tag"
+)
+
 func isGoReleaseRequest(issue *github.Issue) bool {
-	return issueHasLabel(issue, "Go")
+	return issueHasLabel(issue, GoLabel)
+}
+
+func isAutoLink(issue *github.Issue) bool {
+	return issueHasLabel(issue, AutoLinkLabel)
+}
+
+func isPRReady(issue *github.Issue) bool {
+	return issueHasLabel(issue, PRreadyLabel)
+}
+
+func isInconsistentTag(issue *github.Issue) bool {
+	return issueHasLabel(issue, InconsistentTagLabel)
 }
 
 func (c *commandContext) parseIssues(issues []*github.Issue) ([]request.Request, error) {
@@ -204,6 +225,18 @@ func (c *commandContext) parseIssues(issues []*github.Issue) ([]request.Request,
 		if issue == nil {
 			continue
 		}
+		if isPRReady(issue) {
+			continue
+		}
+		if !isAutoLink(issue) {
+			continue
+		}
+		if isInconsistentTag(issue) {
+			log.Printf("[ERROR] %s Readme tag is inconsistent with default tag\n", issue.GetHTMLURL())
+			errResult = multierror.Append(errResult, fmt.Errorf("%s: readme tag is inconsistent with default tag", issue.GetHTMLURL()))
+			continue
+		}
+
 		log.Printf("Parsing issue %s (%s)", issue.GetHTMLURL(), issue.GetTitle())
 		req, err := request.ParseIssue(c.ctx, c.client, *issue, request.ParsingOptions{
 			IncludeDataPlaneRequests: c.flags.IncludeDataPlaneRequests,

eng/tools/generator/cmd/issue/link/consts.go

@@ -18,4 +18,8 @@ const (
 	FilePrefix = "blob/"
 	// CommitPrefix ...
 	CommitPrefix = "commit/"
+	// SDKRepo ...
+	SDKRepo = "azure-sdk-for-go"
+	// ReleaseIssueRepo SDK Release Request Repo ...
+	ReleaseIssueRepo = "sdk-release-request"
 )

eng/tools/generator/cmd/issue/request/handlers.go

@@ -40,10 +40,10 @@ func handlePRNotMerged(_ context.Context, _ *query.Client, reqIssue ReleaseReque
 
 func getTrack(issue ReleaseRequestIssue) Track {
 	for _, l := range issue.Labels {
-		if l != nil && l.GetName() == "Track2" {
-			return Track2
+		if l != nil && l.GetName() == "Track1" {
+			return Track1
 		}
 	}
 
-	return Track1
+	return Track2
 }

eng/tools/generator/cmd/v2/common/cmdProcessor.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"log"
 	"os/exec"
+	"regexp"
+	"strings"
 )
 
 // execute `go generate` command and fetch result
@@ -23,7 +25,7 @@ func ExecuteGoGenerate(path string) error {
 
 // execute `pwsh Invoke-MgmtTestgen` command and fetch result
 func ExecuteExampleGenerate(path, packagePath string) error {
-	cmd := exec.Command("pwsh", "../../../../eng/scripts/Invoke-MgmtTestGen.ps1", "-skipBuild", "-cleanGenerated", "-format", "-tidy", "-generateExample", packagePath)
+	cmd := exec.Command("pwsh", "../../../../eng/scripts/Invoke-MgmtTestgen.ps1", "-skipBuild", "-cleanGenerated", "-format", "-tidy", "-generateExample", packagePath)
 	cmd.Dir = path
 	output, err := cmd.CombinedOutput()
 	log.Printf("Result of `pwsh Invoke-MgmtTestgen` execution: \n%s", string(output))
@@ -51,3 +53,57 @@ func ExecuteGoimports(path string) error {
 	}
 	return nil
 }
+
+func ExecuteGitPush(path, remoteName, branchName string) (string, error) {
+	refName := fmt.Sprintf(branchName + ":" + branchName)
+	cmd := exec.Command("git", "push", remoteName, refName)
+	cmd.Dir = path
+	msg, err := cmd.CombinedOutput()
+	if err != nil {
+		return string(msg), err
+	}
+	return "", nil
+}
+
+func ExecuteCreatePullRequest(path, repoOwner, repoName, prOwner, prBranch, prTitle, prBody, authToken string) (string, error) {
+	cmd := exec.Command("pwsh", "./eng/common/scripts/Submit-PullRequest.ps1", "-RepoOwner", repoOwner, "-RepoName", repoName, "-BaseBranch", "main", "-PROwner", prOwner, "-PRBranch", prBranch, "-AuthToken", authToken, "-PRTitle", prTitle, "-PRBody", prBody)
+	cmd.Dir = path
+	output, err := cmd.CombinedOutput()
+	log.Printf("Result of `pwsh Submit-PullRequest` execution: \n%s", string(output))
+	if err != nil {
+		return "", fmt.Errorf("failed to execute `pwsh Submit-PullRequest` '%s': %+v", string(output), err)
+	}
+
+	match := regexp.MustCompile(`html_url[ ]*:.*`).FindString(string(output))
+	_, after, _ := strings.Cut(match, ":")
+	index := strings.Index(after, "https")
+	return after[index:], nil
+}
+
+func ExecuteAddIssueComment(path, repoOwner, repoName, issueNumber, comment, authToken string) error {
+	cmd := exec.Command("pwsh", "./eng/common/scripts/Add-IssueComment.ps1", "-RepoOwner", repoOwner, "-RepoName", repoName, "-IssueNumber", issueNumber, "-Comment", comment, "-AuthToken", authToken)
+	cmd.Dir = path
+	output, err := cmd.CombinedOutput()
+	log.Printf("Result of `pwsh Add-IssueComment` execution: \n%s", string(output))
+	if err != nil {
+		return fmt.Errorf("failed to execute `pwsh Add-IssueComment` '%s': %+v", string(output), err)
+	}
+	return nil
+}
+
+func ExecuteAddIssueLabels(path, repoOwner, repoName, issueNumber, authToken string, labels []string) error {
+	var l string
+	if len(labels) == 1 {
+		l = labels[0]
+	} else {
+		l = strings.Join(labels, ",")
+	}
+	cmd := exec.Command("pwsh", "./eng/common/scripts/Add-IssueLabels.ps1", "-RepoOwner", repoOwner, "-RepoName", repoName, "-IssueNumber", issueNumber, "-Labels", l, "-AuthToken", authToken)
+	cmd.Dir = path
+	output, err := cmd.CombinedOutput()
+	log.Printf("Result of `pwsh Add-IssueLabels` execution: \n%s", string(output))
+	if err != nil {
+		return fmt.Errorf("failed to execute `pwsh Add-IssueLabels` '%s': %+v", string(output), err)
+	}
+	return nil
+}

eng/tools/generator/cmd/v2/common/fileProcessor.go

@@ -39,8 +39,10 @@ var (
 )
 
 type PackageInfo struct {
-	Name   string
-	Config string
+	Name        string
+	Config      string
+	SpecName    string
+	RequestLink string
 }
 
 // reads from readme.go.md, parses the `track2` section to get module and package name
@@ -78,6 +80,10 @@ func ReadV2ModuleNameToGetNamespace(path string) (map[string][]PackageInfo, erro
 		return nil, fmt.Errorf("last `track2` section does not properly end")
 	}
 
+	s := strings.ReplaceAll(path, "\\", "/")
+	s1 := strings.Split(s, "/")
+	specName := s1[len(s1)-3]
+
 	for i := range start {
 		// get the content of the `track2` section
 		section := lines[start[i]+1 : end[i]]
@@ -95,7 +101,7 @@ func ReadV2ModuleNameToGetNamespace(path string) (map[string][]PackageInfo, erro
 				for _, matchResult := range matchResults {
 					packageConfig = matchResult[1] + ": true"
 				}
-				result[modules[2]] = append(result[modules[2]], PackageInfo{Name: namespaceName, Config: packageConfig})
+				result[modules[2]] = append(result[modules[2]], PackageInfo{Name: namespaceName, Config: packageConfig, SpecName: specName})
 			}
 		}
 	}

eng/tools/generator/cmd/v2/release/releaseCmd.go

@@ -4,30 +4,39 @@
 package release
 
 import (
+	"errors"
 	"fmt"
 	"log"
+	"path"
+	"strings"
 	"time"
 
+	"github.com/Azure/azure-sdk-for-go/eng/tools/generator/cmd/issue/link"
 	"github.com/Azure/azure-sdk-for-go/eng/tools/generator/cmd/v2/common"
+	"github.com/Azure/azure-sdk-for-go/eng/tools/generator/config"
+	"github.com/Azure/azure-sdk-for-go/eng/tools/generator/config/validate"
 	"github.com/Azure/azure-sdk-for-go/eng/tools/generator/flags"
+	"github.com/Azure/azure-sdk-for-go/eng/tools/generator/repo"
+	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
 
 var (
 	releaseBranchNamePattern = "release-%s-%s-%s-%v"
+	confirmComment           = "Release [PR](%s) is ready. Please check whether the package works well ([doc for testing with the generated Go SDK](https://github.com/Azure/azure-rest-api-specs/blob/main/documentation/code-gen/configure-go-sdk.md#test-with-the-generated-go-sdk)). If you are not a Golang user, you can mainly check whether the changelog meets your requirements."
 )
 
 // Release command
 func Command() *cobra.Command {
 	releaseCmd := &cobra.Command{
-		Use:   "release-v2 <azure-sdk-for-go directory/commitid> <azure-rest-api-specs directory/commitid> <rp-name> [namespaceName]",
+		Use:   "release-v2 <azure-sdk-for-go directory/commitid> <azure-rest-api-specs directory/commitid> <rp-name/release-request-json-config> [namespaceName]",
 		Short: "Generate a v2 release of azure-sdk-for-go",
 		Long: `This command will generate a new v2 release for azure-sdk-for-go with given rp name and namespace name.
 
 azure-sdk-for-go directory/commitid: the directory path of the azure-sdk-for-go with git control or just a commitid for remote repo
 azure-rest-api-specs directory: the directory path of the azure-rest-api-specs with git control or just a commitid for remote repo
-rp-name: name of resource provider to be released, same for the swagger folder name
+rp-name/release-request-json-config: name of resource provider to be released (same for the swagger folder name) or release request json config file path generated by 'generator issue'
 namespaceName: name of namespace to be released, default value is arm+rp-name
 
 `,
@@ -64,6 +73,7 @@ type Flags struct {
 	SkipGenerateExample bool
 	PackageConfig       string
 	GoVersion           string
+	Token               string
 }
 
 func BindFlags(flagSet *pflag.FlagSet) {
@@ -77,6 +87,7 @@ func BindFlags(flagSet *pflag.FlagSet) {
 	flagSet.Bool("skip-generate-example", false, "Skip generate example for SDK in the same time")
 	flagSet.String("package-config", "", "Additional config for package")
 	flagSet.String("go-version", "1.18", "Go version")
+	flagSet.StringP("token", "t", "", "Specify the personal access token of Github")
 }
 
 func ParseFlags(flagSet *pflag.FlagSet) Flags {
@@ -91,6 +102,7 @@ func ParseFlags(flagSet *pflag.FlagSet) Flags {
 		SkipGenerateExample: flags.GetBool(flagSet, "skip-generate-example"),
 		PackageConfig:       flags.GetString(flagSet, "package-config"),
 		GoVersion:           flags.GetString(flagSet, "go-version"),
+		Token:               flags.GetString(flagSet, "token"),
 	}
 }
 
@@ -111,6 +123,14 @@ func (c *commandContext) execute(sdkRepoParam, specRepoParam string) error {
 		return err
 	}
 
+	if path.Ext(c.rpName) == ".json" {
+		return c.generateFromRequest(sdkRepo, specRepoParam, specCommitHash)
+	} else {
+		return c.generate(sdkRepo, specCommitHash)
+	}
+}
+
+func (c *commandContext) generate(sdkRepo repo.SDKRepository, specCommitHash string) error {
 	log.Printf("Release generation for rp: %s, namespace: %s", c.rpName, c.namespaceName)
 	generateCtx := common.GenerateContext{
 		SDKPath:        sdkRepo.Root(),
@@ -155,3 +175,103 @@ func (c *commandContext) execute(sdkRepoParam, specRepoParam string) error {
 
 	return nil
 }
+
+func (c *commandContext) generateFromRequest(sdkRepo repo.SDKRepository, specRepoParam, specCommitHash string) error {
+	var pullRequestUrls = make(map[string]string)
+	var pushBranch = make(map[string]string)
+	forkRemote, err := repo.GetForkRemote(sdkRepo)
+	if err != nil {
+		return err
+	}
+
+	log.Printf("track2 parsing the config...")
+	cfg, err := config.ParseConfig(c.rpName)
+	if err != nil {
+		return fmt.Errorf("parse config err: %v", err)
+	}
+	log.Printf("Configuration: %s", cfg.String())
+	armServices, err := validate.ParseTrack2(cfg, specRepoParam)
+	if err != nil {
+		return err
+	}
+	for arm, packageInfos := range armServices {
+		for _, info := range packageInfos {
+			originalHead, err := sdkRepo.Head()
+			if err != nil {
+				return err
+			}
+
+			// run generator
+			c.rpName = arm
+			c.namespaceName = info.Name
+			c.flags.SpecRPName = info.SpecName
+			err = c.generate(sdkRepo, specCommitHash)
+			if err != nil {
+				return err
+			}
+
+			// get current branch name
+			generateHead, err := sdkRepo.Head()
+			if err != nil {
+				return err
+			}
+			pushBranch[generateHead.Name().Short()] = info.RequestLink
+
+			log.Printf("git checkout %v", originalHead.Name().Short())
+			if err := sdkRepo.Checkout(&repo.CheckoutOptions{
+				Branch: plumbing.ReferenceName(originalHead.Name().Short()),
+				Force:  true,
+			}); err != nil {
+				return err
+			}
+		}
+	}
+
+	for branch := range pushBranch {
+		log.Printf("Fixes: %s\n", branch)
+	}
+
+	if c.flags.Token != "" {
+		for branchName, issue := range pushBranch {
+			log.Printf("git push fork %s\n", branchName)
+			msg, err := common.ExecuteGitPush(sdkRepo.Root(), forkRemote.Config().Name, branchName)
+			if err != nil {
+				return fmt.Errorf("git push fork error:%v,msg:%s", err, msg)
+			}
+
+			githubUserName := repo.GetRemoteUserName(forkRemote)
+			if githubUserName == "" {
+				return errors.New("github user name not exist")
+			}
+
+			log.Printf("%s: create pull request...\n", branchName)
+			pullRequestUrl, err := common.ExecuteCreatePullRequest(sdkRepo.Root(), link.SpecOwner, link.SDKRepo, githubUserName, branchName, repo.ReleaseTitle(branchName), issue, c.flags.Token)
+			if err != nil {
+				return err
+			}
+			pullRequestUrls[branchName] = pullRequestUrl
+
+			log.Printf("Leave a comment in %s...\n", issue)
+			issueNumber := strings.Split(issue, "/")
+			err = common.ExecuteAddIssueComment(sdkRepo.Root(), link.SpecOwner, link.ReleaseIssueRepo, issueNumber[len(issueNumber)-1], fmt.Sprintf(confirmComment, pullRequestUrl), c.flags.Token)
+			if err != nil {
+				return err
+			}
+
+			log.Printf("Add Labels...\n")
+			err = common.ExecuteAddIssueLabels(sdkRepo.Root(), link.SpecOwner, link.ReleaseIssueRepo, issueNumber[len(issueNumber)-1], c.flags.Token, []string{"PRready"})
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	if len(pullRequestUrls) != 0 {
+		log.Println("Fixes:")
+		for branch, url := range pullRequestUrls {
+			log.Printf("%s : %s", branch, url)
+		}
+	}
+
+	return nil
+}