diff --git a/apis/placement/v1beta1/stageupdate_types.go b/apis/placement/v1beta1/stageupdate_types.go index 1f03fb625..fa92786d2 100644 --- a/apis/placement/v1beta1/stageupdate_types.go +++ b/apis/placement/v1beta1/stageupdate_types.go @@ -186,8 +186,8 @@ type UpdateRunSpec struct { // The resource snapshot index of the selected resources to be updated across clusters. // The index represents a group of resource snapshots that includes all the resources a ResourcePlacement selected. - // +kubebuilder:validation:Required // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="resourceSnapshotIndex is immutable" + // +kubebuilder:validation:Optional ResourceSnapshotIndex string `json:"resourceSnapshotIndex"` // The name of the update strategy that specifies the stages and the sequence @@ -374,6 +374,11 @@ type UpdateRunStatus struct { // +kubebuilder:validation:Optional PolicyObservedClusterCount int `json:"policyObservedClusterCount,omitempty"` + // ResourceSnapshotIndexUsed records the resource snapshot index that the update run is based on. + // The index represents the same resource snapshots as specified in the spec field, or the latest. + // +kubbebuilder:validation:Optional + ResourceSnapshotIndexUsed string `json:"resourceSnapshotIndexUsed,omitempty"` + // ApplyStrategy is the apply strategy that the stagedUpdateRun is using. // It is the same as the apply strategy in the CRP when the staged update run starts. // The apply strategy is not updated during the update run even if it changes in the CRP. diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml index bf75d613d..a2ddf03ec 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml @@ -1200,7 +1200,6 @@ spec: type: string required: - placementName - - resourceSnapshotIndex - stagedRolloutStrategyName type: object x-kubernetes-validations: @@ -1895,6 +1894,11 @@ spec: All clusters involved in the update run are selected from the list of clusters scheduled by the CRP according to the current policy. type: string + resourceSnapshotIndexUsed: + description: |- + ResourceSnapshotIndexUsed records the resource snapshot index that the update run is based on. + The index represents the same resource snapshots as specified in the spec field, or the latest. + type: string stagedUpdateStrategySnapshot: description: |- UpdateStrategySnapshot is the snapshot of the UpdateStrategy used for the update run. diff --git a/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml b/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml index 10fe5f738..0db56cd16 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml @@ -120,7 +120,6 @@ spec: type: string required: - placementName - - resourceSnapshotIndex - stagedRolloutStrategyName type: object x-kubernetes-validations: @@ -815,6 +814,11 @@ spec: All clusters involved in the update run are selected from the list of clusters scheduled by the CRP according to the current policy. type: string + resourceSnapshotIndexUsed: + description: |- + ResourceSnapshotIndexUsed records the resource snapshot index that the update run is based on. + The index represents the same resource snapshots as specified in the spec field, or the latest. + type: string stagedUpdateStrategySnapshot: description: |- UpdateStrategySnapshot is the snapshot of the UpdateStrategy used for the update run. diff --git a/pkg/controllers/updaterun/execution.go b/pkg/controllers/updaterun/execution.go index c69e34f11..87f5eaf4b 100644 --- a/pkg/controllers/updaterun/execution.go +++ b/pkg/controllers/updaterun/execution.go @@ -96,7 +96,7 @@ func (r *Reconciler) executeUpdatingStage( updateRunSpec := updateRun.GetUpdateRunSpec() updatingStageStatus := &updateRunStatus.StagesStatus[updatingStageIndex] // The parse error is ignored because the initialization should have caught it. - resourceIndex, _ := strconv.Atoi(updateRunSpec.ResourceSnapshotIndex) + resourceIndex, _ := strconv.Atoi(updateRunStatus.ResourceSnapshotIndexUsed) resourceSnapshotName := fmt.Sprintf(placementv1beta1.ResourceSnapshotNameFmt, updateRunSpec.PlacementName, resourceIndex) updateRunRef := klog.KObj(updateRun) // Create the map of the toBeUpdatedBindings. diff --git a/pkg/controllers/updaterun/execution_integration_test.go b/pkg/controllers/updaterun/execution_integration_test.go index 9b3e06833..980283497 100644 --- a/pkg/controllers/updaterun/execution_integration_test.go +++ b/pkg/controllers/updaterun/execution_integration_test.go @@ -160,7 +160,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded and the execution started") - initialized := generateSucceededInitializationStatus(crp, updateRun, policySnapshot, updateStrategy, clusterResourceOverride) + initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -521,7 +521,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded and the execution started") - initialized := generateSucceededInitializationStatus(crp, updateRun, policySnapshot, updateStrategy, clusterResourceOverride) + initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -680,7 +680,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded and the execution started") - initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, policySnapshot, updateStrategy) + initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -774,7 +774,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded and the execution started") - initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, policySnapshot, updateStrategy) + initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -883,7 +883,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded and the execution started") - initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, policySnapshot, updateStrategy) + initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -1014,7 +1014,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded and the execution started") - initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, policySnapshot, updateStrategy) + initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -1106,7 +1106,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded and the execution started") - initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, policySnapshot, updateStrategy) + initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -1163,7 +1163,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded and the execution started") - initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, policySnapshot, updateStrategy) + initialized := generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") }) diff --git a/pkg/controllers/updaterun/initialization.go b/pkg/controllers/updaterun/initialization.go index 45a4ba48a..1296008c2 100644 --- a/pkg/controllers/updaterun/initialization.go +++ b/pkg/controllers/updaterun/initialization.go @@ -471,30 +471,10 @@ func validateAfterStageTask(tasks []placementv1beta1.StageTask) error { func (r *Reconciler) recordOverrideSnapshots(ctx context.Context, placementKey types.NamespacedName, updateRun placementv1beta1.UpdateRunObj) error { updateRunRef := klog.KObj(updateRun) updateRunSpec := updateRun.GetUpdateRunSpec() - placementName := placementKey.Name - snapshotIndex, err := strconv.Atoi(updateRunSpec.ResourceSnapshotIndex) - if err != nil || snapshotIndex < 0 { - err := controller.NewUserError(fmt.Errorf("invalid resource snapshot index `%s` provided, expected an integer >= 0", updateRunSpec.ResourceSnapshotIndex)) - klog.ErrorS(err, "Failed to parse the resource snapshot index", "updateRun", updateRunRef) - // no more retries here. - return fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) - } - - resourceSnapshotList, err := controller.ListAllResourceSnapshotWithAnIndex(ctx, r.Client, updateRunSpec.ResourceSnapshotIndex, placementName, placementKey.Namespace) + resourceSnapshotObjs, err := r.getResourceSnapshotObjs(ctx, placementKey, updateRun) if err != nil { - klog.ErrorS(err, "Failed to list the resourceSnapshots associated with the placement", - "placement", placementKey, "resourceSnapshotIndex", snapshotIndex, "updateRun", updateRunRef) - // err can be retried. - return controller.NewAPIServerError(true, err) - } - - resourceSnapshotObjs := resourceSnapshotList.GetResourceSnapshotObjs() - if len(resourceSnapshotObjs) == 0 { - err := controller.NewUserError(fmt.Errorf("no resourceSnapshots with index `%d` found for placement `%s`", snapshotIndex, placementKey)) - klog.ErrorS(err, "No specified resourceSnapshots found", "updateRun", updateRunRef) - // no more retries here. - return fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) + return err } // Look for the master resourceSnapshot. @@ -509,12 +489,18 @@ func (r *Reconciler) recordOverrideSnapshots(ctx context.Context, placementKey t // No masterResourceSnapshot found. if masterResourceSnapshot == nil { - err := controller.NewUnexpectedBehaviorError(fmt.Errorf("no master resourceSnapshot found for placement `%s` with index `%d`", placementKey, snapshotIndex)) + err := controller.NewUnexpectedBehaviorError(fmt.Errorf("no master resourceSnapshot found for placement %s", placementKey)) klog.ErrorS(err, "Failed to find master resourceSnapshot", "updateRun", updateRunRef) // no more retries here. return fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) } - klog.V(2).InfoS("Found master resourceSnapshot", "placement", placementKey, "index", snapshotIndex, "updateRun", updateRunRef) + + klog.V(2).InfoS("Found master resourceSnapshot", "placement", placementKey, "masterResourceSnapshot", masterResourceSnapshot.GetName(), "updateRun", updateRunRef) + + // Record the resource snapshot index used. + updateRunStatus := updateRun.GetUpdateRunStatus() + updateRunStatus.ResourceSnapshotIndexUsed = masterResourceSnapshot.GetLabels()[placementv1beta1.ResourceIndexLabel] + updateRun.SetUpdateRunStatus(*updateRunStatus) resourceSnapshotRef := klog.KObj(masterResourceSnapshot) // Fetch all the matching overrides. @@ -526,7 +512,6 @@ func (r *Reconciler) recordOverrideSnapshots(ctx context.Context, placementKey t } // Pick the overrides associated with each target cluster. - updateRunStatus := updateRun.GetUpdateRunStatus() for _, stageStatus := range updateRunStatus.StagesStatus { for i := range stageStatus.Clusters { clusterStatus := &stageStatus.Clusters[i] @@ -543,6 +528,58 @@ func (r *Reconciler) recordOverrideSnapshots(ctx context.Context, placementKey t return nil } +// getResourceSnapshotObjs retrieves the list of resource snapshot objects from the specified ResourceSnapshotIndex. +// If ResourceSnapshotIndex is unspecified, it returns the list of latest resource snapshots. +func (r *Reconciler) getResourceSnapshotObjs(ctx context.Context, placementKey types.NamespacedName, updateRun placementv1beta1.UpdateRunObj) ([]placementv1beta1.ResourceSnapshotObj, error) { + updateRunRef := klog.KObj(updateRun) + updateRunSpec := updateRun.GetUpdateRunSpec() + var resourceSnapshotObjs []placementv1beta1.ResourceSnapshotObj + if updateRunSpec.ResourceSnapshotIndex != "" { + snapshotIndex, err := strconv.Atoi(updateRunSpec.ResourceSnapshotIndex) + if err != nil || snapshotIndex < 0 { + err := controller.NewUserError(fmt.Errorf("invalid resource snapshot index `%s` provided, expected an integer >= 0", updateRunSpec.ResourceSnapshotIndex)) + klog.ErrorS(err, "Failed to parse the resource snapshot index", "updateRun", updateRunRef) + // no more retries here. + return nil, fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) + } + + resourceSnapshotList, err := controller.ListAllResourceSnapshotWithAnIndex(ctx, r.Client, updateRunSpec.ResourceSnapshotIndex, placementKey.Name, placementKey.Namespace) + if err != nil { + klog.ErrorS(err, "Failed to list the resourceSnapshots associated with the placement", + "placement", placementKey, "resourceSnapshotIndex", snapshotIndex, "updateRun", updateRunRef) + // list err can be retried. + return nil, controller.NewAPIServerError(true, err) + } + + resourceSnapshotObjs = resourceSnapshotList.GetResourceSnapshotObjs() + if len(resourceSnapshotObjs) == 0 { + err := controller.NewUserError(fmt.Errorf("no resourceSnapshots with index `%d` found for placement `%s`", snapshotIndex, placementKey)) + klog.ErrorS(err, "No specified resourceSnapshots found", "updateRun", updateRunRef) + // no more retries here. + return resourceSnapshotObjs, fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) + } + return resourceSnapshotObjs, nil + } + + klog.V(2).InfoS("No resource snapshot index specified, fetching latest resource snapshots", "placement", placementKey, "updateRun", updateRunRef) + latestResourceSnapshots, err := controller.ListLatestResourceSnapshots(ctx, r.Client, placementKey) + if err != nil { + klog.ErrorS(err, "Failed to list the latest resourceSnapshots associated with the placement", + "placement", placementKey, "updateRun", updateRunRef) + // list err can be retried. + return nil, controller.NewAPIServerError(true, err) + } + + resourceSnapshotObjs = latestResourceSnapshots.GetResourceSnapshotObjs() + if len(resourceSnapshotObjs) == 0 { + err := fmt.Errorf("no latest resourceSnapshots found for placement `%s`. This might be a transient state, need retry", placementKey) + klog.ErrorS(err, "No latest resourceSnapshots found for placement. This might be transient, need retry", "placement", placementKey, "updateRun", updateRunRef) + // retryable error. + return resourceSnapshotObjs, err + } + return resourceSnapshotObjs, nil +} + // recordInitializationSucceeded records the successful initialization condition in the UpdateRun status. func (r *Reconciler) recordInitializationSucceeded(ctx context.Context, updateRun placementv1beta1.UpdateRunObj) error { updateRunStatus := updateRun.GetUpdateRunStatus() diff --git a/pkg/controllers/updaterun/initialization_integration_test.go b/pkg/controllers/updaterun/initialization_integration_test.go index 8bfab157a..9a4507eee 100644 --- a/pkg/controllers/updaterun/initialization_integration_test.go +++ b/pkg/controllers/updaterun/initialization_integration_test.go @@ -57,7 +57,7 @@ var _ = Describe("Updaterun initialization tests", func() { var resourceBindings []*placementv1beta1.ClusterResourceBinding var targetClusters []*clusterv1beta1.MemberCluster var unscheduledClusters []*clusterv1beta1.MemberCluster - var resourceSnapshot *placementv1beta1.ClusterResourceSnapshot + var resourceSnapshot, resourceSnapshot2, resourceSnapshot3 *placementv1beta1.ClusterResourceSnapshot var clusterResourceOverride *placementv1beta1.ClusterResourceOverrideSnapshot BeforeEach(func() { @@ -75,6 +75,8 @@ var _ = Describe("Updaterun initialization tests", func() { resourceBindings, targetClusters, unscheduledClusters = generateTestClusterResourceBindingsAndClusters(1) policySnapshot = generateTestClusterSchedulingPolicySnapshot(1, len(targetClusters)) resourceSnapshot = generateTestClusterResourceSnapshot() + resourceSnapshot2 = generateTestClusterResourceSnapshot() + resourceSnapshot3 = generateTestClusterResourceSnapshot() // Set smaller wait time for testing stageUpdatingWaitTime = time.Second * 3 @@ -113,9 +115,13 @@ var _ = Describe("Updaterun initialization tests", func() { Expect(k8sClient.Delete(ctx, updateStrategy)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) updateStrategy = nil - By("Deleting the clusterResourceSnapshot") + By("Deleting the clusterResourceSnapshots") Expect(k8sClient.Delete(ctx, resourceSnapshot)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) resourceSnapshot = nil + Expect(k8sClient.Delete(ctx, resourceSnapshot2)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) + resourceSnapshot2 = nil + Expect(k8sClient.Delete(ctx, resourceSnapshot3)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) + resourceSnapshot3 = nil By("Deleting the clusterResourceOverride") Expect(k8sClient.Delete(ctx, clusterResourceOverride)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) @@ -654,7 +660,8 @@ var _ = Describe("Updaterun initialization tests", func() { return err } - want := generateSucceededInitializationStatus(crp, updateRun, policySnapshot, updateStrategy, clusterResourceOverride) + // no resource snapshot created in this test + want := generateSucceededInitializationStatus(crp, updateRun, "", policySnapshot, updateStrategy, clusterResourceOverride) // No clusters should be selected in the first stage. want.StagesStatus[0].Clusters = []placementv1beta1.ClusterUpdatingStatus{} // All clusters should be selected in the second stage and sorted by name. @@ -696,7 +703,8 @@ var _ = Describe("Updaterun initialization tests", func() { return err } - want := generateSucceededInitializationStatus(crp, updateRun, policySnapshot, updateStrategy, clusterResourceOverride) + // no resource snapshot created in this test + want := generateSucceededInitializationStatus(crp, updateRun, "", policySnapshot, updateStrategy, clusterResourceOverride) for i := range want.StagesStatus[0].Clusters { // Remove the CROs, as they are not added in this test. want.StagesStatus[0].Clusters[i].ClusterResourceOverrideSnapshots = nil @@ -783,6 +791,31 @@ var _ = Describe("Updaterun initialization tests", func() { validateUpdateRunMetricsEmitted(generateInitializationFailedMetric(updateRun)) }) + It("Should NOT fail to initialize if the specified resource snapshot is not found when no resource index specified - no resourceSnapshots at all", func() { + By("Creating a new clusterStagedUpdateRun without specifying resourceSnapshotIndex") + updateRun.Spec.ResourceSnapshotIndex = "" + Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) + + By("Validating the initialization did not fail due to resourceSnapshot not found (retryable error)") + // Populate the cache first. + Eventually(func() error { + if err := k8sClient.Get(ctx, updateRunNamespacedName, updateRun); err != nil { + return err + } + return nil + }, timeout, interval).Should(Succeed(), "failed to get the updateRun") + Consistently(func() error { + if err := k8sClient.Get(ctx, updateRunNamespacedName, updateRun); err != nil { + return err + } + initCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionInitialized)) + if initCond != nil { + return fmt.Errorf("got initialization condition: %v, want nil", initCond) + } + return nil + }, duration, interval).Should(Succeed(), "the initialization should keep retrying, not failed") + }) + It("Should fail to initialize if the specified resource snapshot is not found - no CRP label found", func() { By("Creating a new resource snapshot associated with another CRP") resourceSnapshot.Labels[placementv1beta1.PlacementTrackingLabel] = "not-exist-crp" @@ -828,6 +861,29 @@ var _ = Describe("Updaterun initialization tests", func() { validateUpdateRunMetricsEmitted(generateInitializationFailedMetric(updateRun)) }) + It("Should select latest resource snapshot in the status when no resource index defined", func() { + By("Creating a new resource snapshot") + Expect(k8sClient.Create(ctx, resourceSnapshot)).To(Succeed()) + + By("Creating a new cluster resource override") + Expect(k8sClient.Create(ctx, clusterResourceOverride)).To(Succeed()) + + By("Creating a new clusterStagedUpdateRun without specifying resourceSnapshotIndex") + updateRun.Spec.ResourceSnapshotIndex = "" + Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) + + By("Validating the clusterStagedUpdateRun stats") + initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) + want := generateExecutionStartedStatus(updateRun, initialized) + validateClusterStagedUpdateRunStatus(ctx, updateRun, want, "") + + By("Validating the clusterStagedUpdateRun initialized consistently") + validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, want, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + }) + It("Should put related ClusterResourceOverrides in the status", func() { By("Creating a new resource snapshot") Expect(k8sClient.Create(ctx, resourceSnapshot)).To(Succeed()) @@ -839,7 +895,42 @@ var _ = Describe("Updaterun initialization tests", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the clusterStagedUpdateRun stats") - initialized := generateSucceededInitializationStatus(crp, updateRun, policySnapshot, updateStrategy, clusterResourceOverride) + initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) + want := generateExecutionStartedStatus(updateRun, initialized) + validateClusterStagedUpdateRunStatus(ctx, updateRun, want, "") + + By("Validating the clusterStagedUpdateRun initialized consistently") + validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, want, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) + }) + + It("Should pick latest master resource snapshot if multiple snapshots", func() { + By("Creating a new resource snapshot") + resourceSnapshot.Labels[placementv1beta1.IsLatestSnapshotLabel] = "false" + Expect(k8sClient.Create(ctx, resourceSnapshot)).To(Succeed()) + + By("Creating a another new resource snapshot") + resourceSnapshot2.Name = testCRPName + "-1-snapshot" + resourceSnapshot2.Labels[placementv1beta1.IsLatestSnapshotLabel] = "false" + resourceSnapshot2.Labels[placementv1beta1.ResourceIndexLabel] = "1" + Expect(k8sClient.Create(ctx, resourceSnapshot2)).To(Succeed()) + + By("Creating a latest master resource snapshot") + resourceSnapshot3.Name = testCRPName + "-2-snapshot" + resourceSnapshot3.Labels[placementv1beta1.ResourceIndexLabel] = "2" + Expect(k8sClient.Create(ctx, resourceSnapshot3)).To(Succeed()) + + By("Creating a new cluster resource override") + Expect(k8sClient.Create(ctx, clusterResourceOverride)).To(Succeed()) + + By("Creating a new clusterStagedUpdateRun without specifying resourceSnapshotIndex") + updateRun.Spec.ResourceSnapshotIndex = "" + Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) + + By("Validating the clusterStagedUpdateRun status") + initialized := generateSucceededInitializationStatus(crp, updateRun, "2", policySnapshot, updateStrategy, clusterResourceOverride) want := generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, want, "") @@ -872,6 +963,7 @@ func validateFailedInitCondition(ctx context.Context, updateRun *placementv1beta func generateSucceededInitializationStatus( crp *placementv1beta1.ClusterResourcePlacement, updateRun *placementv1beta1.ClusterStagedUpdateRun, + resourceSnapshotIndex string, policySnapshot *placementv1beta1.ClusterSchedulingPolicySnapshot, updateStrategy *placementv1beta1.ClusterStagedUpdateStrategy, clusterResourceOverride *placementv1beta1.ClusterResourceOverrideSnapshot, @@ -879,6 +971,7 @@ func generateSucceededInitializationStatus( status := &placementv1beta1.UpdateRunStatus{ PolicySnapshotIndexUsed: policySnapshot.Labels[placementv1beta1.PolicyIndexLabel], PolicyObservedClusterCount: 10, + ResourceSnapshotIndexUsed: resourceSnapshotIndex, ApplyStrategy: crp.Spec.Strategy.ApplyStrategy.DeepCopy(), UpdateStrategySnapshot: &updateStrategy.Spec, StagesStatus: []placementv1beta1.StageUpdatingStatus{ @@ -933,12 +1026,14 @@ func generateSucceededInitializationStatus( func generateSucceededInitializationStatusForSmallClusters( crp *placementv1beta1.ClusterResourcePlacement, updateRun *placementv1beta1.ClusterStagedUpdateRun, + resourceSnapshotIndex string, policySnapshot *placementv1beta1.ClusterSchedulingPolicySnapshot, updateStrategy *placementv1beta1.ClusterStagedUpdateStrategy, ) *placementv1beta1.UpdateRunStatus { status := &placementv1beta1.UpdateRunStatus{ PolicySnapshotIndexUsed: policySnapshot.Labels[placementv1beta1.PolicyIndexLabel], PolicyObservedClusterCount: 3, + ResourceSnapshotIndexUsed: resourceSnapshotIndex, ApplyStrategy: crp.Spec.Strategy.ApplyStrategy.DeepCopy(), UpdateStrategySnapshot: &updateStrategy.Spec, StagesStatus: []placementv1beta1.StageUpdatingStatus{ diff --git a/pkg/controllers/updaterun/validation_integration_test.go b/pkg/controllers/updaterun/validation_integration_test.go index de1cd71e8..bcc255473 100644 --- a/pkg/controllers/updaterun/validation_integration_test.go +++ b/pkg/controllers/updaterun/validation_integration_test.go @@ -110,7 +110,7 @@ var _ = Describe("UpdateRun validation tests", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded") - initialized := generateSucceededInitializationStatus(crp, updateRun, policySnapshot, updateStrategy, clusterResourceOverride) + initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") }) @@ -442,7 +442,7 @@ var _ = Describe("UpdateRun validation tests", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization succeeded") - initialized := generateSucceededInitializationStatus(crp, updateRun, policySnapshot, updateStrategy, clusterResourceOverride) + initialized := generateSucceededInitializationStatus(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy, clusterResourceOverride) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") }) diff --git a/test/e2e/actuals_test.go b/test/e2e/actuals_test.go index 889f1abe3..344faf6f4 100644 --- a/test/e2e/actuals_test.go +++ b/test/e2e/actuals_test.go @@ -2092,6 +2092,7 @@ func updateRunSucceedConditions(generation int64) []metav1.Condition { func clusterStagedUpdateRunStatusSucceededActual( updateRunName string, + wantResourceIndex string, wantPolicyIndex string, wantClusterCount int, wantApplyStrategy *placementv1beta1.ApplyStrategy, @@ -2109,6 +2110,7 @@ func clusterStagedUpdateRunStatusSucceededActual( wantStatus := placementv1beta1.UpdateRunStatus{ PolicySnapshotIndexUsed: wantPolicyIndex, + ResourceSnapshotIndexUsed: wantResourceIndex, PolicyObservedClusterCount: wantClusterCount, ApplyStrategy: wantApplyStrategy.DeepCopy(), UpdateStrategySnapshot: wantStrategySpec, @@ -2126,7 +2128,7 @@ func clusterStagedUpdateRunStatusSucceededActual( func stagedUpdateRunStatusSucceededActual( updateRunName, namespace string, - wantPolicyIndex string, + wantResourceIndex, wantPolicyIndex string, wantClusterCount int, wantApplyStrategy *placementv1beta1.ApplyStrategy, wantStrategySpec *placementv1beta1.UpdateStrategySpec, @@ -2143,6 +2145,7 @@ func stagedUpdateRunStatusSucceededActual( wantStatus := placementv1beta1.UpdateRunStatus{ PolicySnapshotIndexUsed: wantPolicyIndex, + ResourceSnapshotIndexUsed: wantResourceIndex, PolicyObservedClusterCount: wantClusterCount, ApplyStrategy: wantApplyStrategy.DeepCopy(), UpdateStrategySnapshot: wantStrategySpec, diff --git a/test/e2e/cluster_staged_updaterun_test.go b/test/e2e/cluster_staged_updaterun_test.go index 9880c2eef..5f93a4254 100644 --- a/test/e2e/cluster_staged_updaterun_test.go +++ b/test/e2e/cluster_staged_updaterun_test.go @@ -63,6 +63,175 @@ var _ = Describe("test CRP rollout with staged update run", func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) strategyName := fmt.Sprintf(clusterStagedUpdateRunStrategyNameTemplate, GinkgoParallelProcess()) + Context("Test resource rollout with staged update run with latest resource snapshot when not specified", Ordered, func() { + updateRunNames := []string{} + var strategy *placementv1beta1.ClusterStagedUpdateStrategy + var oldConfigMap, newConfigMap corev1.ConfigMap + + BeforeAll(func() { + // Create a test namespace and a configMap inside it on the hub cluster. + createWorkResources() + + // Create the CRP with external rollout strategy. + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: workResourceSelector(), + Strategy: placementv1beta1.RolloutStrategy{ + Type: placementv1beta1.ExternalRolloutStrategyType, + }, + }, + } + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") + + // Create the clusterStagedUpdateStrategy. + strategy = createClusterStagedUpdateStrategySucceed(strategyName) + + for i := 0; i < 2; i++ { + updateRunNames = append(updateRunNames, fmt.Sprintf(clusterStagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) + } + + oldConfigMap = appConfigMap() + newConfigMap = appConfigMap() + newConfigMap.Data["data"] = testConfigMapDataValue + }) + + AfterAll(func() { + // Remove the custom deletion blocker finalizer from the CRP. + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + + // Remove all the clusterStagedUpdateRuns. + for _, name := range updateRunNames { + ensureClusterStagedUpdateRunDeletion(name) + } + + // Delete the clusterStagedUpdateStrategy. + ensureClusterUpdateRunStrategyDeletion(strategyName) + }) + + It("Should not rollout any resources to member clusters as there's no update run yet", checkIfRemovedWorkResourcesFromAllMemberClustersConsistently) + + It("Should have the latest resource snapshot", func() { + validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex1st) + }) + + It("Should successfully schedule the crp", func() { + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) + }) + + It("Should update crp status as pending rollout", func() { + crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", "", ""}, []bool{false, false, false}, nil, nil) + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) + }) + + It("Should create a cluster staged update run successfully", func() { + By("Create a cluster staged update run without specifying resource snapshot index") + createClusterStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunNames[0], crpName, strategyName) + }) + + It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[1]}) + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + + By("Validating crp status as member-cluster-2 updated") + crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) + + validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary) + }) + + It("Should rollout resources to member-cluster-1 first because of its name", func() { + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0]}) + }) + + It("Should rollout resources to all the members and complete the cluster staged update run successfully", func() { + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) + }) + + It("Should update crp status as completed", func() { + crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, + []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) + }) + + It("Should update the configmap successfully on hub but not change member clusters", func() { + updateConfigMapSucceed(&newConfigMap) + + for _, cluster := range allMemberClusters { + configMapActual := configMapPlacedOnClusterActual(cluster, &oldConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", oldConfigMap.Name) + } + }) + + It("Should not update crp status, should still be completed", func() { + crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, + []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) + Consistently(crpStatusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep CRP %s status as expected", crpName) + }) + + It("Should create a new latest resource snapshot", func() { + crsList := &placementv1beta1.ClusterResourceSnapshotList{} + Eventually(func() error { + if err := hubClient.List(ctx, crsList, client.MatchingLabels{placementv1beta1.PlacementTrackingLabel: crpName, placementv1beta1.IsLatestSnapshotLabel: "true"}); err != nil { + return fmt.Errorf("failed to list the resourcesnapshot: %w", err) + } + if len(crsList.Items) != 1 { + return fmt.Errorf("got %d latest resourcesnapshots, want 1", len(crsList.Items)) + } + if crsList.Items[0].Labels[placementv1beta1.ResourceIndexLabel] != resourceSnapshotIndex2nd { + return fmt.Errorf("got resource snapshot index %s, want %s", crsList.Items[0].Labels[placementv1beta1.ResourceIndexLabel], resourceSnapshotIndex2nd) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed get the new latest resourcensnapshot") + }) + + It("Should create a new cluster staged update run successfully", func() { + By("Create a new cluster staged update run without specifying resource snapshot index") + createClusterStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunNames[1], crpName, strategyName) + }) + + It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { + By("Verify that the new configmap is updated on member-cluster-2") + configMapActual := configMapPlacedOnClusterActual(allMemberClusters[1], &newConfigMap) + Eventually(configMapActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update to the new configmap %s on cluster %s", newConfigMap.Name, allMemberClusterNames[1]) + By("Verify that the configmap is not updated on member-cluster-1 and member-cluster-3") + for _, cluster := range []*framework.Cluster{allMemberClusters[0], allMemberClusters[2]} { + configMapActual := configMapPlacedOnClusterActual(cluster, &oldConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", newConfigMap.Name) + } + + By("Validating crp status as member-cluster-2 updated") + crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, + []string{resourceSnapshotIndex1st, resourceSnapshotIndex2nd, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) + + validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary) + }) + + It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) + By("Verify that new the configmap is updated on all member clusters") + for idx := range allMemberClusters { + configMapActual := configMapPlacedOnClusterActual(allMemberClusters[idx], &newConfigMap) + Eventually(configMapActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update to the new configmap %s on cluster %s as expected", newConfigMap.Name, allMemberClusterNames[idx]) + } + }) + + It("Should update crp status as completed", func() { + crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex2nd, true, allMemberClusterNames, + []string{resourceSnapshotIndex2nd, resourceSnapshotIndex2nd, resourceSnapshotIndex2nd}, []bool{true, true, true}, nil, nil) + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) + }) + }) + Context("Test resource rollout and rollback with staged update run", Ordered, func() { updateRunNames := []string{} var strategy *placementv1beta1.ClusterStagedUpdateStrategy @@ -149,7 +318,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to all the members and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -214,7 +383,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) By("Verify that new the configmap is updated on all member clusters") for idx := range allMemberClusters { @@ -252,7 +421,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollback resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) for idx := range allMemberClusters { configMapActual := configMapPlacedOnClusterActual(allMemberClusters[idx], &oldConfigMap) @@ -348,7 +517,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-1 too but not member-cluster-3 and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) @@ -398,7 +567,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-3 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -445,7 +614,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should remove resources on member-cluster-1 and member-cluster-2 and complete the cluster staged update run successfully", func() { // need to go through two stages - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil) Eventually(csurSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[2]) checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) @@ -537,7 +706,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-3 and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[2]}) checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) @@ -586,7 +755,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-1 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -632,7 +801,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should remove resources on member-cluster-1 and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) Eventually(csurSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[2]) checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{allMemberClusters[0]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[1], allMemberClusters[2]}) @@ -804,7 +973,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, wantCROs, wantROs) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, wantCROs, wantROs) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -904,7 +1073,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should report diff for member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) }) @@ -1015,7 +1184,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunName, envCanary) // Verify complete rollout - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) // Verify new configmap is on all member clusters @@ -1141,7 +1310,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should complete the staged update run after approval", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1214,7 +1383,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunName, envCanary) - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) @@ -1340,7 +1509,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) By("Validating staged update run has succeeded") - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) By("Validating CRP status as completed") @@ -1392,7 +1561,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label }) It("Should complete the second staged update run and complete the CRP", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1], allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1], allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames[1:], @@ -1440,7 +1609,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label }) It("Should complete the staged update run, complete CRP, and rollout resources to all member clusters", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, @@ -1483,7 +1652,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label }) It("Should complete the staged update run, complete CRP, and rollout updated resources to all member clusters", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex2nd, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex2nd, true, allMemberClusterNames, @@ -1522,7 +1691,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label }) It("Should complete the staged update run, complete CRP, and re-place resources to all member clusters", func() { - csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, @@ -1673,6 +1842,19 @@ func createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapsho Expect(hubClient.Create(ctx, updateRun)).To(Succeed(), "Failed to create ClusterStagedUpdateRun %s", updateRunName) } +func createClusterStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunName, crpName, strategyName string) { + updateRun := &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRunName, + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: crpName, + StagedUpdateStrategyName: strategyName, + }, + } + Expect(hubClient.Create(ctx, updateRun)).To(Succeed(), "Failed to create ClusterStagedUpdateRun %s", updateRunName) +} + func validateAndApproveClusterApprovalRequests(updateRunName, stageName string) { Eventually(func() error { appReqList := &placementv1beta1.ClusterApprovalRequestList{} diff --git a/test/e2e/staged_updaterun_test.go b/test/e2e/staged_updaterun_test.go index b5a5428b1..277d0221a 100644 --- a/test/e2e/staged_updaterun_test.go +++ b/test/e2e/staged_updaterun_test.go @@ -58,6 +58,171 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) + Context("Test resource rollout with staged update run with latest resource snapshot when not specified", Ordered, func() { + updateRunNames := []string{} + var strategy *placementv1beta1.StagedUpdateStrategy + var oldConfigMap, newConfigMap corev1.ConfigMap + + BeforeAll(func() { + // Create the RP with external rollout strategy. + rp := &placementv1beta1.ResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: rpName, + Namespace: testNamespace, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: configMapSelector(), + Strategy: placementv1beta1.RolloutStrategy{ + Type: placementv1beta1.ExternalRolloutStrategyType, + }, + }, + } + Expect(hubClient.Create(ctx, rp)).To(Succeed(), "Failed to create RP") + + // Create the stagedUpdateStrategy. + strategy = createStagedUpdateStrategySucceed(strategyName, testNamespace) + + for i := 0; i < 2; i++ { + updateRunNames = append(updateRunNames, fmt.Sprintf(stagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) + } + + oldConfigMap = appConfigMap() + newConfigMap = appConfigMap() + newConfigMap.Data["data"] = testConfigMapDataValue + }) + + AfterAll(func() { + // Remove the custom deletion blocker finalizer from the RP. + ensureRPAndRelatedResourcesDeleted(types.NamespacedName{Name: rpName, Namespace: testNamespace}, allMemberClusters) + + // Remove all the stagedUpdateRuns. + for _, name := range updateRunNames { + ensureStagedUpdateRunDeletion(name, testNamespace) + } + + // Delete the stagedUpdateStrategy. + ensureStagedUpdateRunStrategyDeletion(strategyName, testNamespace) + }) + + It("Should not rollout any resources to member clusters as there's no update run yet", checkIfRemovedConfigMapFromAllMemberClustersConsistently) + + It("Should have the latest resource snapshot", func() { + validateLatestResourceSnapshot(rpName, testNamespace, resourceSnapshotIndex1st) + }) + + It("Should successfully schedule the rp", func() { + validateLatestSchedulingPolicySnapshot(rpName, testNamespace, policySnapshotIndex1st, 3) + }) + + It("Should update rp status as pending rollout", func() { + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", "", ""}, []bool{false, false, false}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) + }) + + It("Should create a staged update run successfully", func() { + createStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunNames[0], testNamespace, rpName, strategyName) + }) + + It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[1]}) + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + + By("Validating rp status as member-cluster-2 updated") + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) + + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary) + }) + + It("Should rollout resources to member-cluster-1 first because of its name", func() { + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0]}) + }) + + It("Should rollout resources to all the members and complete the staged update run successfully", func() { + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) + }) + + It("Should update rp status as completed", func() { + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(appConfigMapIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, + []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) + }) + + It("Should update the configmap successfully on hub but not change member clusters", func() { + updateConfigMapSucceed(&newConfigMap) + + for _, cluster := range allMemberClusters { + configMapActual := configMapPlacedOnClusterActual(cluster, &oldConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", oldConfigMap.Name) + } + }) + + It("Should not update rp status, should still be completed", func() { + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(appConfigMapIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, + []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) + Consistently(rpStatusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep RP %s/%s status as expected", testNamespace, rpName) + }) + + It("Should create a new latest resource snapshot", func() { + rsList := &placementv1beta1.ResourceSnapshotList{} + Eventually(func() error { + if err := hubClient.List(ctx, rsList, client.InNamespace(testNamespace), client.MatchingLabels{placementv1beta1.PlacementTrackingLabel: rpName, placementv1beta1.IsLatestSnapshotLabel: "true"}); err != nil { + return fmt.Errorf("failed to list the resourcesnapshot: %w", err) + } + if len(rsList.Items) != 1 { + return fmt.Errorf("got %d latest resourcesnapshots, want 1", len(rsList.Items)) + } + if rsList.Items[0].Labels[placementv1beta1.ResourceIndexLabel] != resourceSnapshotIndex2nd { + return fmt.Errorf("got resource snapshot index %s, want %s", rsList.Items[0].Labels[placementv1beta1.ResourceIndexLabel], resourceSnapshotIndex2nd) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed get the new latest resourcensnapshot") + }) + + It("Should create a new staged update run successfully", func() { + createStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunNames[1], testNamespace, rpName, strategyName) + }) + + It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { + By("Verify that the new configmap is updated on member-cluster-2") + configMapActual := configMapPlacedOnClusterActual(allMemberClusters[1], &newConfigMap) + Eventually(configMapActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update to the new configmap %s on cluster %s", newConfigMap.Name, allMemberClusterNames[1]) + By("Verify that the configmap is not updated on member-cluster-1 and member-cluster-3") + for _, cluster := range []*framework.Cluster{allMemberClusters[0], allMemberClusters[2]} { + configMapActual := configMapPlacedOnClusterActual(cluster, &oldConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", newConfigMap.Name) + } + + By("Validating rp status as member-cluster-2 updated") + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, + []string{resourceSnapshotIndex1st, resourceSnapshotIndex2nd, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) + + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary) + }) + + It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) + By("Verify that new the configmap is updated on all member clusters") + for idx := range allMemberClusters { + configMapActual := configMapPlacedOnClusterActual(allMemberClusters[idx], &newConfigMap) + Eventually(configMapActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update to the new configmap %s on cluster %s as expected", newConfigMap.Name, allMemberClusterNames[idx]) + } + }) + + It("Should update rp status as completed", func() { + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(appConfigMapIdentifiers(), resourceSnapshotIndex2nd, true, allMemberClusterNames, + []string{resourceSnapshotIndex2nd, resourceSnapshotIndex2nd, resourceSnapshotIndex2nd}, []bool{true, true, true}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) + }) + }) + Context("Test resource rollout and rollback with staged update run", Ordered, func() { updateRunNames := []string{} var strategy *placementv1beta1.StagedUpdateStrategy @@ -142,7 +307,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to all the members and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -207,7 +372,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) By("Verify that new the configmap is updated on all member clusters") for idx := range allMemberClusters { @@ -245,7 +410,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollback resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) for idx := range allMemberClusters { configMapActual := configMapPlacedOnClusterActual(allMemberClusters[idx], &oldConfigMap) @@ -339,7 +504,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-1 too but not member-cluster-3 and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) @@ -389,7 +554,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-3 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -436,7 +601,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem It("Should remove resources on member-cluster-1 and member-cluster-2 and complete the staged update run successfully", func() { // need to go through two stages - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil) Eventually(surSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[2]) checkIfRemovedConfigMapFromMemberClusters([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) @@ -526,7 +691,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-3 and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[2]}) checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) @@ -575,7 +740,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-1 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -621,7 +786,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should remove resources on member-cluster-1 and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) Eventually(surSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[2]) checkIfRemovedConfigMapFromMemberClusters([]*framework.Cluster{allMemberClusters[0]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[1], allMemberClusters[2]}) @@ -766,7 +931,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, wantROs) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, wantROs) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -860,7 +1025,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should report diff for member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) }) @@ -969,7 +1134,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary) // Verify complete rollout. - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex2nd, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) // Verify new configmap is on all member clusters. @@ -1045,7 +1210,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary) - surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunName, testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) @@ -1206,6 +1371,20 @@ func createStagedUpdateRunSucceed(updateRunName, namespace, rpName, resourceSnap Expect(hubClient.Create(ctx, updateRun)).To(Succeed(), "Failed to create StagedUpdateRun %s", updateRunName) } +func createStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunName, namespace, rpName, strategyName string) { + updateRun := &placementv1beta1.StagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRunName, + Namespace: namespace, + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: rpName, + StagedUpdateStrategyName: strategyName, + }, + } + Expect(hubClient.Create(ctx, updateRun)).To(Succeed(), "Failed to create StagedUpdateRun %s", updateRunName) +} + func validateAndApproveNamespacedApprovalRequests(updateRunName, namespace, stageName string) { Eventually(func() error { appReqList := &placementv1beta1.ApprovalRequestList{}