Update CI logic (Azure#18980)

wyunchi-ms · web-flow · commit 7e41f3af825f · 2022-07-20T13:41:47.000+08:00
* trigger CI

* trigger CI

* trigger CI

Co-authored-by: wyunchi-ms &lt;yunwang@microsoft.com&gt;
diff --git a/.ci-config.json b/.ci-config.json
@@ -284,5 +284,20 @@
         "test:all"
       ]
     }
+  ],
+  "selectModuleList": [
+    "Accounts",
+    "Aks",
+    "ApplicationInsights",
+    "Compute",
+    "Functions",
+    "KeyVault",
+    "KubernetersConfiguration",
+    "Network",
+    "PostgreSql",
+    "Purview",
+    "Resources",
+    "Storage",
+    "Websites"
   ]
 }
diff --git a/tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/CIFilterTask.cs b/tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/CIFilterTask.cs
@@ -193,7 +193,8 @@ private List<string> GetDependentModuleList(string moduleName, Dictionary<string
         // Run a selected module list instead of run all the modules to speed up the CI process.
         private List<string> GetSelectedModuleList()
         {
-            return new List<string>(Environment.GetEnvironmentVariable("SELECTEDMODULELIST").Split(';'));
+            CIPhaseFilterConfig config = GetCIPhaseFilterConfig();
+            return config.SelectModuleList;
         }
 
         private List<string> GetTestCsprojList(string moduleName, Dictionary<string, string[]> csprojMap)
@@ -251,7 +252,7 @@ private Dictionary<string, HashSet<string>> CalculateInfluencedModuleInfoForEach
             {
                 List<string> phaseList = new List<string>();
                 bool isMatched = false;
-                string machedModuleName = "";
+                string matchedModuleName = "";
                 foreach ((Regex regex, List<string> phaseConfigList) in ruleList)
                 {
                     var regexResult = regex.Match(filePath);
@@ -261,7 +262,7 @@ private Dictionary<string, HashSet<string>> CalculateInfluencedModuleInfoForEach
                         isMatched = true;
                         if (regexResult.Groups[MODULE_NAME_PLACEHOLDER].Success)
                         {
-                            machedModuleName = regexResult.Groups[MODULE_NAME_PLACEHOLDER].Value;
+                            matchedModuleName = regexResult.Groups[MODULE_NAME_PLACEHOLDER].Value;
                         }
                         Console.WriteLine(string.Format("File {0} match rule: {1} and phaseConfig is: [{2}]", filePath, regex.ToString(), string.Join(", ", phaseConfigList)));
                         break;
@@ -294,7 +295,7 @@ private Dictionary<string, HashSet<string>> CalculateInfluencedModuleInfoForEach
                         }
                         else
                         {
-                            string moduleName = machedModuleName == "" ? filePath.Split('/')[1] : machedModuleName;
+                            string moduleName = matchedModuleName == "" ? filePath.Split('/')[1] : matchedModuleName;
                             if (scope.Equals(SingleModule))
                             {
                                 scopes.Add(moduleName);
@@ -411,7 +412,7 @@ private Dictionary<string, HashSet<string>> CalculateCsprojForBuildAndTest(Dicti
             return influencedModuleInfo;
         }
 
-        private bool ProcessFileChanged(Dictionary<string, string[]> csprojMap)
+        private CIPhaseFilterConfig GetCIPhaseFilterConfig()
         {
             string configPath = Path.GetFullPath(TaskMappingConfigName);
             if (!File.Exists(configPath))
@@ -420,8 +421,14 @@ private bool ProcessFileChanged(Dictionary<string, string[]> csprojMap)
             }
             string content = File.ReadAllText(configPath);
 
-            CIPhaseFilterConfig config = JsonConvert.DeserializeObject<CIPhaseFilterConfig>(content);
-            List<(Regex, List<string>)> ruleList = config.Rules.Select(rule => (new Regex(string.Join("|", rule.Patterns.Select(ProcessSinglePattern))), rule.Phases)).ToList();
+            return JsonConvert.DeserializeObject<CIPhaseFilterConfig>(content);
+        }
+
+        private bool ProcessFileChanged(Dictionary<string, string[]> csprojMap)
+        {
+
+            CIPhaseFilterConfig config = GetCIPhaseFilterConfig();
+            List<(Regex, List<string>)> ruleList = config.Rules.Select(rule => (new Regex(string.Join("|", rule.Patterns.Select(ProcessSinglePattern).ToList())), rule.Phases)).ToList();
 
             DateTime startTime = DateTime.Now;
 
@@ -456,13 +463,28 @@ public override bool Execute()
             }
             else if (FilesChanged != null)
             {
-                if (FilesChanged.Length > 0 && FilesChanged.Length < OCTOKIT_CHANGED_FILE_LIMIT)
+                if (FilesChanged.Length <= 0)
                 {
-                    return ProcessFileChanged(csprojMap);
+                    return false;
+                }
+                else if (FilesChanged.Length >= OCTOKIT_CHANGED_FILE_LIMIT)
+                {
+                    var selectedModuleList = GetSelectedModuleList();
+                    Dictionary<string, HashSet<string>> influencedModuleInfo = new Dictionary<string, HashSet<string>>
+                    {
+                        [BUILD_PHASE] = new HashSet<string>(selectedModuleList),
+                        [ANALYSIS_BREAKING_CHANGE_PHASE] = new HashSet<string>(selectedModuleList),
+                        [ANALYSIS_DEPENDENCY_PHASE] = new HashSet<string>(selectedModuleList),
+                        [ANALYSIS_HELP_PHASE] = new HashSet<string>(selectedModuleList),
+                        [ANALYSIS_SIGNATURE_PHASE] = new HashSet<string>(selectedModuleList),
+                        [TEST_PHASE] = new HashSet<string>(selectedModuleList)
+                    };
+                    FilterTaskResult.PhaseInfo = CalculateCsprojForBuildAndTest(influencedModuleInfo, csprojMap);
+                    return true;
                 }
                 else
                 {
-                    return false;
+                    return ProcessFileChanged(csprojMap);
                 }
             }
             return true;
diff --git a/tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/CIPhaseFilterConfig.cs b/tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/CIPhaseFilterConfig.cs
@@ -19,6 +19,7 @@ namespace Microsoft.WindowsAzure.Build.Tasks
     class CIPhaseFilterConfig
     {
         public List<Rule> Rules { get; set; }
+        public List<string> SelectModuleList { get; set; }
     }
 
     class Rule
diff --git a/tools/PrepareAutorestModule.ps1 b/tools/PrepareAutorestModule.ps1
@@ -47,7 +47,8 @@ foreach ($file in $ChangedFiles)
 if ($ModuleSet.Contains($ALL_MODULE))
 {
     $Null = $ModuleSet.Remove($ALL_MODULE)
-    $SelectedModuleList = (Get-ChildItem "$PSScriptRoot\..\src\").Name | Where-Object { (Get-Item env:SELECTEDMODULELIST).Value.Split(';') -contains $_ }
+    $CIConfig = Get-Content "$PSScriptRoot\..\.ci-config.json" | ConvertFrom-Json
+    $SelectedModuleList = (Get-ChildItem "$PSScriptRoot\..\src\").Name | Where-Object { $CIConfig.selectModuleList -contains $_ }
     $Null = $ModuleSet.Add($SelectedModuleList)
     $ModuleList = $ModuleSet | Where-Object { $SKIP_MODULES -notcontains $_ }
 }

Original file line number	Diff line number	Diff line change
`@@ -284,5 +284,20 @@`
`284`	`284`	`"test:all"`
`285`	`285`	`]`
`286`	`286`	`}`
	`287`	`+ ],`
	`288`	`+ "selectModuleList": [`
	`289`	`+ "Accounts",`
	`290`	`+ "Aks",`
	`291`	`+ "ApplicationInsights",`
	`292`	`+ "Compute",`
	`293`	`+ "Functions",`
	`294`	`+ "KeyVault",`
	`295`	`+ "KubernetersConfiguration",`
	`296`	`+ "Network",`
	`297`	`+ "PostgreSql",`
	`298`	`+ "Purview",`
	`299`	`+ "Resources",`
	`300`	`+ "Storage",`
	`301`	`+ "Websites"`
`287`	`302`	`]`
`288`	`303`	`}`
Original file line number	Diff line number	Diff line change
`@@ -193,7 +193,8 @@ private List<string> GetDependentModuleList(string moduleName, Dictionary<string`
`193`	`193`	`// Run a selected module list instead of run all the modules to speed up the CI process.`
`194`	`194`	`private List<string> GetSelectedModuleList()`
`195`	`195`	`{`
`196`		`- return new List<string>(Environment.GetEnvironmentVariable("SELECTEDMODULELIST").Split(';'));`
	`196`	`+ CIPhaseFilterConfig config = GetCIPhaseFilterConfig();`
	`197`	`+ return config.SelectModuleList;`
`197`	`198`	`}`
`198`	`199`
`199`	`200`	`private List<string> GetTestCsprojList(string moduleName, Dictionary<string, string[]> csprojMap)`
`@@ -251,7 +252,7 @@ private Dictionary<string, HashSet<string>> CalculateInfluencedModuleInfoForEach`
`251`	`252`	`{`
`252`	`253`	`List<string> phaseList = new List<string>();`
`253`	`254`	`bool isMatched = false;`
`254`		`- string machedModuleName = "";`
	`255`	`+ string matchedModuleName = "";`
`255`	`256`	`foreach ((Regex regex, List<string> phaseConfigList) in ruleList)`
`256`	`257`	`{`
`257`	`258`	`var regexResult = regex.Match(filePath);`
`@@ -261,7 +262,7 @@ private Dictionary<string, HashSet<string>> CalculateInfluencedModuleInfoForEach`
`261`	`262`	`isMatched = true;`
`262`	`263`	`if (regexResult.Groups[MODULE_NAME_PLACEHOLDER].Success)`
`263`	`264`	`{`
`264`		`- machedModuleName = regexResult.Groups[MODULE_NAME_PLACEHOLDER].Value;`
	`265`	`+ matchedModuleName = regexResult.Groups[MODULE_NAME_PLACEHOLDER].Value;`
`265`	`266`	`}`
`266`	`267`	`Console.WriteLine(string.Format("File {0} match rule: {1} and phaseConfig is: [{2}]", filePath, regex.ToString(), string.Join(", ", phaseConfigList)));`
`267`	`268`	`break;`
`@@ -294,7 +295,7 @@ private Dictionary<string, HashSet<string>> CalculateInfluencedModuleInfoForEach`
`294`	`295`	`}`
`295`	`296`	`else`
`296`	`297`	`{`
`297`		`- string moduleName = machedModuleName == "" ? filePath.Split('/')[1] : machedModuleName;`
	`298`	`+ string moduleName = matchedModuleName == "" ? filePath.Split('/')[1] : matchedModuleName;`
`298`	`299`	`if (scope.Equals(SingleModule))`
`299`	`300`	`{`
`300`	`301`	`scopes.Add(moduleName);`
`@@ -411,7 +412,7 @@ private Dictionary<string, HashSet<string>> CalculateCsprojForBuildAndTest(Dicti`
`411`	`412`	`return influencedModuleInfo;`
`412`	`413`	`}`
`413`	`414`
`414`		`- private bool ProcessFileChanged(Dictionary<string, string[]> csprojMap)`
	`415`	`+ private CIPhaseFilterConfig GetCIPhaseFilterConfig()`
`415`	`416`	`{`
`416`	`417`	`string configPath = Path.GetFullPath(TaskMappingConfigName);`
`417`	`418`	`if (!File.Exists(configPath))`
`@@ -420,8 +421,14 @@ private bool ProcessFileChanged(Dictionary<string, string[]> csprojMap)`
`420`	`421`	`}`
`421`	`422`	`string content = File.ReadAllText(configPath);`
`422`	`423`
`423`		`- CIPhaseFilterConfig config = JsonConvert.DeserializeObject<CIPhaseFilterConfig>(content);`
`424`		`- List<(Regex, List<string>)> ruleList = config.Rules.Select(rule => (new Regex(string.Join("\|", rule.Patterns.Select(ProcessSinglePattern))), rule.Phases)).ToList();`
	`424`	`+ return JsonConvert.DeserializeObject<CIPhaseFilterConfig>(content);`
	`425`	`+ }`
	`426`	`+`
	`427`	`+ private bool ProcessFileChanged(Dictionary<string, string[]> csprojMap)`
	`428`	`+ {`
	`429`	`+`
	`430`	`+ CIPhaseFilterConfig config = GetCIPhaseFilterConfig();`
	`431`	`+ List<(Regex, List<string>)> ruleList = config.Rules.Select(rule => (new Regex(string.Join("\|", rule.Patterns.Select(ProcessSinglePattern).ToList())), rule.Phases)).ToList();`
`425`	`432`
`426`	`433`	`DateTime startTime = DateTime.Now;`
`427`	`434`
`@@ -456,13 +463,28 @@ public override bool Execute()`
`456`	`463`	`}`
`457`	`464`	`else if (FilesChanged != null)`
`458`	`465`	`{`
`459`		`- if (FilesChanged.Length > 0 && FilesChanged.Length < OCTOKIT_CHANGED_FILE_LIMIT)`
	`466`	`+ if (FilesChanged.Length <= 0)`
`460`	`467`	`{`
`461`		`- return ProcessFileChanged(csprojMap);`
	`468`	`+ return false;`
	`469`	`+ }`
	`470`	`+ else if (FilesChanged.Length >= OCTOKIT_CHANGED_FILE_LIMIT)`
	`471`	`+ {`
	`472`	`+ var selectedModuleList = GetSelectedModuleList();`
	`473`	`+ Dictionary<string, HashSet<string>> influencedModuleInfo = new Dictionary<string, HashSet<string>>`
	`474`	`+ {`
	`475`	`+ [BUILD_PHASE] = new HashSet<string>(selectedModuleList),`
	`476`	`+ [ANALYSIS_BREAKING_CHANGE_PHASE] = new HashSet<string>(selectedModuleList),`
	`477`	`+ [ANALYSIS_DEPENDENCY_PHASE] = new HashSet<string>(selectedModuleList),`
	`478`	`+ [ANALYSIS_HELP_PHASE] = new HashSet<string>(selectedModuleList),`
	`479`	`+ [ANALYSIS_SIGNATURE_PHASE] = new HashSet<string>(selectedModuleList),`
	`480`	`+ [TEST_PHASE] = new HashSet<string>(selectedModuleList)`
	`481`	`+ };`
	`482`	`+ FilterTaskResult.PhaseInfo = CalculateCsprojForBuildAndTest(influencedModuleInfo, csprojMap);`
	`483`	`+ return true;`
`462`	`484`	`}`
`463`	`485`	`else`
`464`	`486`	`{`
`465`		`- return false;`
	`487`	`+ return ProcessFileChanged(csprojMap);`
`466`	`488`	`}`
`467`	`489`	`}`
`468`	`490`	`return true;`
Original file line number	Diff line number	Diff line change
`@@ -19,6 +19,7 @@ namespace Microsoft.WindowsAzure.Build.Tasks`
`19`	`19`	`class CIPhaseFilterConfig`
`20`	`20`	`{`
`21`	`21`	`public List<Rule> Rules { get; set; }`
	`22`	`+ public List<string> SelectModuleList { get; set; }`
`22`	`23`	`}`
`23`	`24`
`24`	`25`	`class Rule`
Original file line number	Diff line number	Diff line change
`@@ -47,7 +47,8 @@ foreach ($file in $ChangedFiles)`
`47`	`47`	`if ($ModuleSet.Contains($ALL_MODULE))`
`48`	`48`	`{`
`49`	`49`	`$Null = $ModuleSet.Remove($ALL_MODULE)`
`50`		`- $SelectedModuleList = (Get-ChildItem "$PSScriptRoot\..\src\").Name \| Where-Object { (Get-Item env:SELECTEDMODULELIST).Value.Split(';') -contains $_ }`
	`50`	`+ $CIConfig = Get-Content "$PSScriptRoot\..\.ci-config.json" \| ConvertFrom-Json`
	`51`	`+ $SelectedModuleList = (Get-ChildItem "$PSScriptRoot\..\src\").Name \| Where-Object { $CIConfig.selectModuleList -contains $_ }`
`51`	`52`	`$Null = $ModuleSet.Add($SelectedModuleList)`
`52`	`53`	`$ModuleList = $ModuleSet \| Where-Object { $SKIP_MODULES -notcontains $_ }`
`53`	`54`	`}`