From 6c2b7584cc9ed3a19dcee41e00a18d1256003cbf Mon Sep 17 00:00:00 2001 From: Dev-Op Date: Tue, 30 Sep 2025 18:31:58 +0200 Subject: [PATCH 1/6] add support for nested task groups --- .../AzureDevOpsPipelineProcessor.cs | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs b/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs index ea80a25fe..2a6878e5a 100644 --- a/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs +++ b/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs @@ -80,6 +80,7 @@ private async System.Threading.Tasks.Task MigratePipelinesAsync() if (Options.MigrateTaskGroups) { taskGroupMappings = await CreateTaskGroupDefinitionsAsync(); + taskGroupMappings = await CreateTaskGroupDefinitionsAsync(serviceConnectionMappings); } if (Options.MigrateBuildPipelines) { @@ -202,6 +203,7 @@ private IEnumerable FilterOutExistingTaskGroups(IEnumerable /// List of filtered Definitions private IEnumerable FilterOutIncompatibleTaskGroups(IEnumerable filteredTaskGroups, IEnumerable availableTasks) + private IEnumerable FilterOutIncompatibleTaskGroups(IEnumerable filteredTaskGroups, IEnumerable availableTasks, IEnumerable taskGroupMappings) { var objectsToMigrate = filteredTaskGroups.Where(g => { @@ -212,6 +214,15 @@ private IEnumerable FilterOutIncompatibleTaskGroups(IEnumerable m.SourceId == t.Task.Id)) + { + return true; + } + } + missingTasksNames.Add(t.DisplayName); return false; }); @@ -312,6 +323,7 @@ private async Task> CreateBuildPipelinesAsync(IEnumerable> CreateServiceConnectionsAsync() } private async Task> CreateTaskGroupDefinitionsAsync() + private async Task> CreateTaskGroupDefinitionsAsync(IEnumerable serviceConnectionMappings) { Log.LogInformation($"Processing Taskgroups.."); @@ -625,9 +639,88 @@ private async Task> CreateTaskGroupDefinitionsAsync() var filteredTaskGroups = FilterOutExistingTaskGroups(sourceDefinitions, targetDefinitions); filteredTaskGroups = FilterOutIncompatibleTaskGroups(filteredTaskGroups, availableTasks).ToList(); + var existingMappings = FindExistingMappings(sourceDefinitions, targetDefinitions, new List()); + + Log.LogInformation($"Phase 1 - Unnested Taskgroups"); + var unnestedTaskGroups = filteredTaskGroups.Where(g => g.Tasks.All(t => t.Task.DefinitionType.ToLower() != "metaTask".ToLower())); + existingMappings = await CreateTaskGroupsAsync(serviceConnectionMappings, targetDefinitions, availableTasks, unnestedTaskGroups, existingMappings); + + Log.LogInformation($"Phase 2 - Nested Taskgroups"); + var nestedTaskGroups = filteredTaskGroups.Where(g => g.Tasks.Any(t => t.Task.DefinitionType.ToLower() == "metaTask".ToLower())).ToList(); + var taskGroupsToMigrate = new List(); + + do + { + // We need to process the nested taskgroups in a loop, because they can contain other nested taskgroups. + taskGroupsToMigrate.Clear(); + foreach (var taskGroup in nestedTaskGroups) + { + var nestedTaskGroup = taskGroup.Tasks.Where(t => t.Task.DefinitionType.ToLower() == "metaTask".ToLower()).Select(t => t.Task).ToList(); + if (nestedTaskGroup.All(t => existingMappings.Any(m => t.Id == m.SourceId))) + { + taskGroupsToMigrate.Add(taskGroup); + } + } + + nestedTaskGroups = nestedTaskGroups.Where(g => !taskGroupsToMigrate.Any(t => t.Id == g.Id)).ToList(); + existingMappings = await CreateTaskGroupsAsync(serviceConnectionMappings, targetDefinitions, availableTasks, taskGroupsToMigrate, existingMappings); + } while (nestedTaskGroups.Any() && taskGroupsToMigrate.Any()); + + return existingMappings; + } + + private async Task> CreateTaskGroupsAsync(IEnumerable serviceConnectionMappings, IEnumerable targetDefinitions, IEnumerable availableTasks, IEnumerable filteredTaskGroups, IEnumerable existingMappings) + { + filteredTaskGroups = FilterOutIncompatibleTaskGroups(filteredTaskGroups, availableTasks, existingMappings).ToList(); + var rootSourceDefinitions = SortDefinitionsByVersion(filteredTaskGroups).First(); var updatedSourceDefinitions = SortDefinitionsByVersion(filteredTaskGroups).Last(); + foreach (var definitionToBeMigrated in rootSourceDefinitions) + { + if (serviceConnectionMappings is not null) + { + foreach (var task in definitionToBeMigrated.Tasks) + { + var newInputs = new Dictionary(); + foreach (var input in (IDictionary)task.Inputs) + { + var mapping = serviceConnectionMappings.FirstOrDefault(d => d.SourceId == input.Value.ToString()); + if (mapping != null) + { + newInputs.Add(input.Key, mapping.TargetId); + } + } + + foreach (var input in newInputs) + { + ((IDictionary)task.Inputs).Remove(input.Key); + ((IDictionary)task.Inputs).Add(input.Key, input.Value); + } + } + } + + if (existingMappings is not null) + { + foreach (var task in definitionToBeMigrated.Tasks) + { + if (task.Task.DefinitionType.ToLower() != "metaTask".ToLower()) + { + continue; + } + var mapping = existingMappings.FirstOrDefault(d => d.SourceId == task.Task.Id); + if (mapping == null) + { + Log.LogWarning("Can't find taskgroup {MissingTaskGroupId} in the target collection.", task.Task.Id); + } + else + { + task.Task.Id = mapping.TargetId; + } + } + } + } + var mappings = await Target.CreateApiDefinitionsAsync(rootSourceDefinitions); targetDefinitions = await Target.GetApiDefinitionsAsync(queryForDetails: false); @@ -636,6 +729,7 @@ private async Task> CreateTaskGroupDefinitionsAsync() targetDefinitions = await Target.GetApiDefinitionsAsync(queryForDetails: false); mappings.AddRange(FindExistingMappings(sourceDefinitions, targetDefinitions.Where(d => d.Name != null), mappings)); + mappings.AddRange(existingMappings); return mappings; } From 05170b5aaf43daf8b425bcff88915fa244a8ac17 Mon Sep 17 00:00:00 2001 From: Dev-Op Date: Sun, 12 Oct 2025 13:03:34 +0200 Subject: [PATCH 2/6] Update src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../Processors/AzureDevOpsPipelineProcessor.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs b/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs index 2a6878e5a..39b78b64b 100644 --- a/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs +++ b/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs @@ -79,7 +79,6 @@ private async System.Threading.Tasks.Task MigratePipelinesAsync() } if (Options.MigrateTaskGroups) { - taskGroupMappings = await CreateTaskGroupDefinitionsAsync(); taskGroupMappings = await CreateTaskGroupDefinitionsAsync(serviceConnectionMappings); } if (Options.MigrateBuildPipelines) From 3f7b3a86645030b9d63572a1e02e51a552f75b50 Mon Sep 17 00:00:00 2001 From: Dev-Op Date: Sun, 12 Oct 2025 13:03:48 +0200 Subject: [PATCH 3/6] Update src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../Processors/AzureDevOpsPipelineProcessor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs b/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs index 39b78b64b..01658987a 100644 --- a/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs +++ b/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs @@ -202,7 +202,7 @@ private IEnumerable FilterOutExistingTaskGroups(IEnumerable /// List of filtered Definitions private IEnumerable FilterOutIncompatibleTaskGroups(IEnumerable filteredTaskGroups, IEnumerable availableTasks) - private IEnumerable FilterOutIncompatibleTaskGroups(IEnumerable filteredTaskGroups, IEnumerable availableTasks, IEnumerable taskGroupMappings) + private IEnumerable FilterOutIncompatibleTaskGroupsWithMappings(IEnumerable filteredTaskGroups, IEnumerable availableTasks, IEnumerable taskGroupMappings) { var objectsToMigrate = filteredTaskGroups.Where(g => { From 7384a69bf1b08cfe5a40e76f3bffaee747a1eafa Mon Sep 17 00:00:00 2001 From: Dev-Op Date: Sun, 12 Oct 2025 13:04:56 +0200 Subject: [PATCH 4/6] Update src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../Processors/AzureDevOpsPipelineProcessor.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs b/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs index 01658987a..e87683089 100644 --- a/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs +++ b/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs @@ -321,7 +321,6 @@ private async Task> CreateBuildPipelinesAsync(IEnumerable Date: Sun, 12 Oct 2025 13:05:28 +0200 Subject: [PATCH 5/6] Update src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../Processors/AzureDevOpsPipelineProcessor.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs b/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs index e87683089..d71892185 100644 --- a/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs +++ b/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs @@ -626,7 +626,6 @@ private async Task> CreateServiceConnectionsAsync() return mappings; } - private async Task> CreateTaskGroupDefinitionsAsync() private async Task> CreateTaskGroupDefinitionsAsync(IEnumerable serviceConnectionMappings) { Log.LogInformation($"Processing Taskgroups.."); From 591a12cb7c9eacaf68f08af6a11e220401d09e03 Mon Sep 17 00:00:00 2001 From: Dev-Op Date: Fri, 17 Oct 2025 14:06:18 +0200 Subject: [PATCH 6/6] fix issues from automatic PR add IsMetaTask method for multiple used checks --- .../AzureDevOpsPipelineProcessor.cs | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs b/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs index d71892185..284f12602 100644 --- a/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs +++ b/src/MigrationTools.Clients.AzureDevops.Rest/Processors/AzureDevOpsPipelineProcessor.cs @@ -200,8 +200,8 @@ private IEnumerable FilterOutExistingTaskGroups(IEnumerable /// /// + /// /// List of filtered Definitions - private IEnumerable FilterOutIncompatibleTaskGroups(IEnumerable filteredTaskGroups, IEnumerable availableTasks) private IEnumerable FilterOutIncompatibleTaskGroupsWithMappings(IEnumerable filteredTaskGroups, IEnumerable availableTasks, IEnumerable taskGroupMappings) { var objectsToMigrate = filteredTaskGroups.Where(g => @@ -287,6 +287,11 @@ not null when typeof(DefinitionType) == typeof(ReleaseDefinition) => definitionN .ToList(); } + private static bool IsMetaTask(string definitionType) + { + return string.Equals(definitionType, "metaTask", StringComparison.OrdinalIgnoreCase); + } + private async Task> CreateBuildPipelinesAsync(IEnumerable TaskGroupMapping = null, IEnumerable VariableGroupMapping = null, IEnumerable serviceConnectionMappings = null) { Log.LogInformation("Processing Build Pipelines.."); @@ -321,7 +326,7 @@ private async Task> CreateBuildPipelinesAsync(IEnumerable> CreateTaskGroupDefinitionsAsync(IEnumer var targetDefinitions = await Target.GetApiDefinitionsAsync(queryForDetails: false); var availableTasks = await Target.GetApiDefinitionsAsync(queryForDetails: false); var filteredTaskGroups = FilterOutExistingTaskGroups(sourceDefinitions, targetDefinitions); - filteredTaskGroups = FilterOutIncompatibleTaskGroups(filteredTaskGroups, availableTasks).ToList(); + filteredTaskGroups = FilterOutIncompatibleTaskGroupsWithMappings(filteredTaskGroups, availableTasks, null).ToList(); var existingMappings = FindExistingMappings(sourceDefinitions, targetDefinitions, new List()); Log.LogInformation($"Phase 1 - Unnested Taskgroups"); - var unnestedTaskGroups = filteredTaskGroups.Where(g => g.Tasks.All(t => t.Task.DefinitionType.ToLower() != "metaTask".ToLower())); + var unnestedTaskGroups = filteredTaskGroups.Where(g => g.Tasks.All(t => !IsMetaTask(t.Task.DefinitionType))); existingMappings = await CreateTaskGroupsAsync(serviceConnectionMappings, targetDefinitions, availableTasks, unnestedTaskGroups, existingMappings); Log.LogInformation($"Phase 2 - Nested Taskgroups"); - var nestedTaskGroups = filteredTaskGroups.Where(g => g.Tasks.Any(t => t.Task.DefinitionType.ToLower() == "metaTask".ToLower())).ToList(); + var nestedTaskGroups = filteredTaskGroups.Where(g => g.Tasks.Any(t => IsMetaTask(t.Task.DefinitionType))).ToList(); var taskGroupsToMigrate = new List(); do @@ -652,7 +656,7 @@ private async Task> CreateTaskGroupDefinitionsAsync(IEnumer taskGroupsToMigrate.Clear(); foreach (var taskGroup in nestedTaskGroups) { - var nestedTaskGroup = taskGroup.Tasks.Where(t => t.Task.DefinitionType.ToLower() == "metaTask".ToLower()).Select(t => t.Task).ToList(); + var nestedTaskGroup = taskGroup.Tasks.Where(t => IsMetaTask(t.Task.DefinitionType)).Select(t => t.Task).ToList(); if (nestedTaskGroup.All(t => existingMappings.Any(m => t.Id == m.SourceId))) { taskGroupsToMigrate.Add(taskGroup); @@ -668,7 +672,7 @@ private async Task> CreateTaskGroupDefinitionsAsync(IEnumer private async Task> CreateTaskGroupsAsync(IEnumerable serviceConnectionMappings, IEnumerable targetDefinitions, IEnumerable availableTasks, IEnumerable filteredTaskGroups, IEnumerable existingMappings) { - filteredTaskGroups = FilterOutIncompatibleTaskGroups(filteredTaskGroups, availableTasks, existingMappings).ToList(); + filteredTaskGroups = FilterOutIncompatibleTaskGroupsWithMappings(filteredTaskGroups, availableTasks, existingMappings).ToList(); var rootSourceDefinitions = SortDefinitionsByVersion(filteredTaskGroups).First(); var updatedSourceDefinitions = SortDefinitionsByVersion(filteredTaskGroups).Last(); @@ -701,7 +705,7 @@ private async Task> CreateTaskGroupsAsync(IEnumerable> CreateTaskGroupsAsync(IEnumerable(queryForDetails: false); - mappings.AddRange(FindExistingMappings(sourceDefinitions, targetDefinitions.Where(d => d.Name != null), mappings)); + mappings.AddRange(FindExistingMappings(rootSourceDefinitions, targetDefinitions.Where(d => d.Name != null), mappings)); mappings.AddRange(existingMappings); return mappings; }