diff --git a/test/e2e/actuals_test.go b/test/e2e/actuals_test.go index 3766bb61b..e72f17e63 100644 --- a/test/e2e/actuals_test.go +++ b/test/e2e/actuals_test.go @@ -18,6 +18,8 @@ package e2e import ( "fmt" + "strings" + "time" "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo/v2" @@ -982,12 +984,39 @@ func safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResourceIdentifiers [ } func workNamespaceRemovedFromClusterActual(cluster *framework.Cluster) func() error { - client := cluster.KubeClient - ns := appNamespace() return func() error { - if err := client.Get(ctx, types.NamespacedName{Name: ns.Name}, &corev1.Namespace{}); !errors.IsNotFound(err) { - return fmt.Errorf("work namespace %s still exists or an unexpected error occurred: %w", ns.Name, err) + if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: ns.Name}, &ns); !errors.IsNotFound(err) { + if err == nil { + // List all pods and configmaps in the namespace + podList := &corev1.PodList{} + if listErr := cluster.KubeClient.List(ctx, podList, client.InNamespace(ns.Name)); listErr != nil { + return fmt.Errorf("work namespace %s still exists on cluster %s but failed to list pods: %w, ns deletion timestamp: %v, current timestamp: %v", + ns.Name, cluster.ClusterName, listErr, ns.GetDeletionTimestamp(), time.Now()) + } + configMapList := &corev1.ConfigMapList{} + if listErr := cluster.KubeClient.List(ctx, configMapList, client.InNamespace(ns.Name)); listErr != nil { + return fmt.Errorf("work namespace %s still exists on cluster %s but failed to list configmaps: %w, ns deletion timestamp: %v, current timestamp: %v", + ns.Name, cluster.ClusterName, listErr, ns.GetDeletionTimestamp(), time.Now()) + } + + // Build resource status summary + var resourceStatus strings.Builder + resourceStatus.WriteString(fmt.Sprintf("\nList pods and configmaps in namespace %s:\n", ns.Name)) + for _, item := range podList.Items { + resourceStatus.WriteString(fmt.Sprintf("- Pod: %s/%s\n", item.GetNamespace(), item.GetName())) + resourceStatus.WriteString(fmt.Sprintf(" DeletionTimestamp: %v\n", item.GetDeletionTimestamp())) + resourceStatus.WriteString(fmt.Sprintf(" Finalizers: %v\n", item.GetFinalizers())) + } + for _, item := range configMapList.Items { + resourceStatus.WriteString(fmt.Sprintf("- ConfigMap: %s/%s\n", item.GetNamespace(), item.GetName())) + resourceStatus.WriteString(fmt.Sprintf(" DeletionTimestamp: %v\n", item.GetDeletionTimestamp())) + resourceStatus.WriteString(fmt.Sprintf(" Finalizers: %v\n", item.GetFinalizers())) + } + return fmt.Errorf("work namespace %s still exists on cluster %s: deletion timestamp: %v, current timestamp: %v, resources: %s", + ns.Name, cluster.ClusterName, ns.GetDeletionTimestamp(), time.Now(), resourceStatus.String()) + } + return fmt.Errorf("getting work namespace %s failed: %w", ns.Name, err) } return nil } diff --git a/test/e2e/placement_eviction_test.go b/test/e2e/placement_eviction_test.go index b38e331ff..d93ad71d9 100644 --- a/test/e2e/placement_eviction_test.go +++ b/test/e2e/placement_eviction_test.go @@ -159,10 +159,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding, taint clus }) It("should ensure no resources exist on evicted member cluster with taint", func() { - for _, cluster := range taintClusters { - resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) - Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") - } + checkIfRemovedWorkResourcesFromMemberClusters(taintClusters) }) It("should update cluster resource placement status as expected", func() { @@ -583,10 +580,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll C }) It("should ensure no resources exist on evicted member cluster with taint", func() { - for _, cluster := range taintClusters { - resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) - Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") - } + checkIfRemovedWorkResourcesFromMemberClusters(taintClusters) }) It("should update cluster resource placement status as expected", func() { @@ -776,10 +770,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP }) It("should ensure no resources exist on evicted member cluster with taint", func() { - for _, cluster := range taintClusters { - resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) - Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") - } + checkIfRemovedWorkResourcesFromMemberClusters(taintClusters) }) It("should update cluster resource placement status as expected", func() { @@ -885,10 +876,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP }) It("should ensure no resources exist on evicted member cluster with taint", func() { - for _, cluster := range taintClusters { - resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) - Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") - } + checkIfRemovedWorkResourcesFromMemberClusters(taintClusters) }) It("should update cluster resource placement status as expected", func() { @@ -1170,10 +1158,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP }) It("should ensure no resources exist on evicted member cluster with taint", func() { - for _, cluster := range taintClusters { - resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) - Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") - } + checkIfRemovedWorkResourcesFromMemberClusters(taintClusters) }) It("should update cluster resource placement status as expected", func() { @@ -1279,10 +1264,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP }) It("should ensure no resources exist on evicted member cluster with taint", func() { - for _, cluster := range taintClusters { - resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) - Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") - } + checkIfRemovedWorkResourcesFromMemberClusters(taintClusters) }) It("should update cluster resource placement status as expected", func() { diff --git a/test/e2e/placement_pickall_test.go b/test/e2e/placement_pickall_test.go index 5f1c0d354..b682f5f75 100644 --- a/test/e2e/placement_pickall_test.go +++ b/test/e2e/placement_pickall_test.go @@ -293,8 +293,7 @@ var _ = Describe("placing resources using a CRP of PickAll placement type", func }) It("should remove resources on previously matched clusters", func() { - resourceRemovedActual := workNamespaceRemovedFromClusterActual(memberCluster1EastProd) - Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove resources on previously matched clusters") + checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{memberCluster1EastProd}) }) AfterAll(func() { @@ -918,8 +917,7 @@ var _ = Describe("placing resources using a CRP of PickAll placement type", func }) It("should remove resources on previously matched clusters", func() { - resourceRemovedActual := workNamespaceRemovedFromClusterActual(memberCluster2EastCanary) - Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove resources on previously matched clusters") + checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{memberCluster2EastCanary}) }) AfterAll(func() { diff --git a/test/e2e/placement_pickfixed_test.go b/test/e2e/placement_pickfixed_test.go index 0719e61a4..32225b435 100644 --- a/test/e2e/placement_pickfixed_test.go +++ b/test/e2e/placement_pickfixed_test.go @@ -137,8 +137,7 @@ var _ = Describe("placing resources using a CRP of PickFixed placement type", fu }) It("should remove resources from previously specified clusters", func() { - resourceRemovedActual := workNamespaceRemovedFromClusterActual(memberCluster1EastProd) - Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove resources from previously specified clusters") + checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{memberCluster1EastProd}) }) AfterAll(func() { diff --git a/test/e2e/placement_pickn_test.go b/test/e2e/placement_pickn_test.go index e1ebcd51c..cffc3e9ae 100644 --- a/test/e2e/placement_pickn_test.go +++ b/test/e2e/placement_pickn_test.go @@ -145,8 +145,7 @@ var _ = Describe("placing resources using a CRP of PickN placement", func() { }) It("should remove resources from the downscaled clusters", func() { - resourceRemovedActual := workNamespaceRemovedFromClusterActual(memberCluster2EastCanary) - Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove resources from the downscaled clusters") + checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{memberCluster2EastCanary}) }) AfterAll(func() { @@ -411,8 +410,7 @@ var _ = Describe("placing resources using a CRP of PickN placement", func() { }) It("should remove resources from the unpicked clusters", func() { - resourceRemovedActual := workNamespaceRemovedFromClusterActual(memberCluster2EastCanary) - Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove resources from the downscaled clusters") + checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{memberCluster2EastCanary}) }) AfterAll(func() { @@ -548,10 +546,7 @@ var _ = Describe("placing resources using a CRP of PickN placement", func() { It("should remove resources from the downscaled clusters", func() { downscaledClusters := []*framework.Cluster{memberCluster3WestProd, memberCluster2EastCanary} - for _, cluster := range downscaledClusters { - resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) - Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove resources from the downscaled clusters") - } + checkIfRemovedWorkResourcesFromMemberClusters(downscaledClusters) }) It("should update CRP status as expected", func() { diff --git a/test/e2e/taint_toleration_test.go b/test/e2e/taint_toleration_test.go index 0f5718e7a..7024b91f3 100644 --- a/test/e2e/taint_toleration_test.go +++ b/test/e2e/taint_toleration_test.go @@ -114,10 +114,7 @@ var _ = Describe("placing resources using a cluster resource placement with no p }) It("should ensure no resources exist on member clusters with taint", func() { - for _, cluster := range taintClusters { - resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) - Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") - } + checkIfRemovedWorkResourcesFromMemberClusters(taintClusters) }) It("should place resources on the selected cluster without taint", func() { @@ -190,10 +187,7 @@ var _ = Describe("placing resources using a cluster resource placement with no p }) It("should ensure no resources exist on member clusters with taint", func() { - for _, cluster := range taintClusters { - resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) - Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") - } + checkIfRemovedWorkResourcesFromMemberClusters(taintClusters) }) It("should place resources on the selected cluster without taint", func() { @@ -307,10 +301,7 @@ var _ = Describe("picking N clusters with affinities and topology spread constra }) It("should ensure no resources exist on member clusters with untolerated taint", func() { - for _, cluster := range unSelectedClusters { - resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster) - Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster") - } + checkIfRemovedWorkResourcesFromMemberClusters(unSelectedClusters) }) AfterAll(func() { diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index a2679cc37..8b6252494 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -754,7 +754,7 @@ func checkIfRemovedWorkResourcesFromMemberClusters(clusters []*framework.Cluster memberCluster := clusters[idx] workResourcesRemovedActual := workNamespaceRemovedFromClusterActual(memberCluster) - Eventually(workResourcesRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove work resources from member cluster %s", memberCluster.ClusterName) + Eventually(workResourcesRemovedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove work resources from member cluster %s", memberCluster.ClusterName) } }