From eede6d9b61fca39eb5077f1e715294bd493b10b9 Mon Sep 17 00:00:00 2001 From: Wantong Jiang Date: Mon, 21 Apr 2025 23:43:21 +0000 Subject: [PATCH 1/2] feat: add metrics for updaterun Signed-off-by: Wantong Jiang --- pkg/controllers/updaterun/controller.go | 42 +++++- .../updaterun/controller_integration_test.go | 126 ++++++++++++++++++ pkg/controllers/updaterun/execution.go | 68 +++++++++- .../updaterun/execution_integration_test.go | 124 +++++++++++++++++ .../initialization_integration_test.go | 50 +++++++ .../updaterun/validation_integration_test.go | 31 +++++ pkg/utils/condition/condition.go | 6 + pkg/utils/controller/metrics/metrics.go | 7 + 8 files changed, 449 insertions(+), 5 deletions(-) diff --git a/pkg/controllers/updaterun/controller.go b/pkg/controllers/updaterun/controller.go index 32c489010..8fdf7234e 100644 --- a/pkg/controllers/updaterun/controller.go +++ b/pkg/controllers/updaterun/controller.go @@ -21,8 +21,10 @@ import ( "context" "errors" "fmt" + "strconv" "time" + "github.com/prometheus/client_golang/prometheus" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -42,6 +44,7 @@ import ( "github.com/kubefleet-dev/kubefleet/pkg/utils" "github.com/kubefleet-dev/kubefleet/pkg/utils/condition" "github.com/kubefleet-dev/kubefleet/pkg/utils/controller" + "github.com/kubefleet-dev/kubefleet/pkg/utils/controller/metrics" "github.com/kubefleet-dev/kubefleet/pkg/utils/informer" ) @@ -95,6 +98,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim return runtime.Result{}, err } + // Emit the update run status metric based on status conditions in the updateRun. + defer emitUpdateRunStatusMetric(&updateRun) + var updatingStageIndex int var toBeUpdatedBindings, toBeDeletedBindings []*placementv1beta1.ClusterResourceBinding var err error @@ -169,13 +175,17 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim // We delete all the dependent resources, including approvalRequest objects, of the clusterStagedUpdateRun object. func (r *Reconciler) handleDelete(ctx context.Context, updateRun *placementv1beta1.ClusterStagedUpdateRun) (bool, time.Duration, error) { runObjRef := klog.KObj(updateRun) - // delete all the associated approvalRequests. + // Delete all the associated approvalRequests. approvalRequest := &placementv1beta1.ClusterApprovalRequest{} if err := r.Client.DeleteAllOf(ctx, approvalRequest, client.MatchingLabels{placementv1beta1.TargetUpdateRunLabel: updateRun.GetName()}); err != nil { klog.ErrorS(err, "Failed to delete all associated approvalRequests", "clusterStagedUpdateRun", runObjRef) return false, 0, controller.NewAPIServerError(false, err) } klog.V(2).InfoS("Deleted all approvalRequests associated with the clusterStagedUpdateRun", "clusterStagedUpdateRun", runObjRef) + + // Delete the update run status metric. + metrics.FleetUpdateRunStatusLastTimestampSeconds.DeletePartialMatch(prometheus.Labels{"name": updateRun.GetName()}) + controllerutil.RemoveFinalizer(updateRun, placementv1beta1.ClusterStagedUpdateRunFinalizer) if err := r.Client.Update(ctx, updateRun); err != nil { klog.ErrorS(err, "Failed to remove updateRun finalizer", "clusterStagedUpdateRun", runObjRef) @@ -301,3 +311,33 @@ func handleClusterApprovalRequest(oldObj, newObj client.Object, q workqueue.Type NamespacedName: types.NamespacedName{Name: updateRun}, }) } + +// emitUpdateRunStatusMetric emits the update run status metric based on status conditions in the updateRun. +func emitUpdateRunStatusMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) { + generation := updateRun.Generation + genStr := strconv.FormatInt(generation, 10) + + succeedCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionSucceeded)) + if succeedCond != nil && succeedCond.ObservedGeneration == generation { + metrics.FleetUpdateRunStatusLastTimestampSeconds.WithLabelValues(updateRun.Name, genStr, + string(placementv1beta1.StagedUpdateRunConditionSucceeded), string(succeedCond.Status), succeedCond.Reason).SetToCurrentTime() + return + } + + progressingCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) + if progressingCond != nil && progressingCond.ObservedGeneration == generation { + metrics.FleetUpdateRunStatusLastTimestampSeconds.WithLabelValues(updateRun.Name, genStr, + string(placementv1beta1.StagedUpdateRunConditionProgressing), string(progressingCond.Status), progressingCond.Reason).SetToCurrentTime() + return + } + + initializedCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionInitialized)) + if initializedCond != nil && initializedCond.ObservedGeneration == generation { + metrics.FleetUpdateRunStatusLastTimestampSeconds.WithLabelValues(updateRun.Name, genStr, + string(placementv1beta1.StagedUpdateRunConditionInitialized), string(initializedCond.Status), initializedCond.Reason).SetToCurrentTime() + return + } + + // We should rarely reach here, it can only happen when updating updateRun status fails. + klog.V(2).InfoS("There's no valid status condition on updateRun, status updating failed possibly", "updateRun", klog.KObj(updateRun)) +} diff --git a/pkg/controllers/updaterun/controller_integration_test.go b/pkg/controllers/updaterun/controller_integration_test.go index ad6d1b16f..fe8f892ce 100644 --- a/pkg/controllers/updaterun/controller_integration_test.go +++ b/pkg/controllers/updaterun/controller_integration_test.go @@ -22,8 +22,11 @@ import ( "strconv" "time" + "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" + prometheusclientmodel "github.com/prometheus/client_model/go" rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -32,6 +35,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -40,6 +44,8 @@ import ( placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" "github.com/kubefleet-dev/kubefleet/pkg/utils" "github.com/kubefleet-dev/kubefleet/pkg/utils/condition" + "github.com/kubefleet-dev/kubefleet/pkg/utils/controller/metrics" + metricsutils "github.com/kubefleet-dev/kubefleet/test/utils/metrics" ) const ( @@ -69,6 +75,7 @@ var ( testCROName string updateRunNamespacedName types.NamespacedName testNamespace []byte + customRegistry *prometheus.Registry ) var _ = Describe("Test the clusterStagedUpdateRun controller", func() { @@ -80,6 +87,15 @@ var _ = Describe("Test the clusterStagedUpdateRun controller", func() { testUpdateStrategyName = "updatestrategy-" + utils.RandStr() testCROName = "cro-" + utils.RandStr() updateRunNamespacedName = types.NamespacedName{Name: testUpdateRunName} + + customRegistry = initializeUpdateRunMetricsRegistry() + }) + + AfterEach(func() { + By("Checking the update run status metrics are removed") + // No metrics are emitted as all are removed after updateRun is deleted. + validateUpdateRunMetricsEmitted(customRegistry) + unregisterUpdateRunMetrics(customRegistry) }) Context("Test reconciling a clusterStagedUpdateRun", func() { @@ -223,6 +239,114 @@ var _ = Describe("Test the clusterStagedUpdateRun controller", func() { }) }) +func initializeUpdateRunMetricsRegistry() *prometheus.Registry { + // Create a test registry + customRegistry := prometheus.NewRegistry() + Expect(customRegistry.Register(metrics.FleetUpdateRunStatusLastTimestampSeconds)).Should(Succeed()) + // Reset metrics before each test + metrics.FleetUpdateRunStatusLastTimestampSeconds.Reset() + return customRegistry +} + +func unregisterUpdateRunMetrics(registry *prometheus.Registry) { + Expect(registry.Unregister(metrics.FleetUpdateRunStatusLastTimestampSeconds)).Should(BeTrue()) +} + +// validateUpdateRunMetricsEmitted validates the update run status metrics are emitted and are emitted in the correct order. +func validateUpdateRunMetricsEmitted(registry *prometheus.Registry, wantMetrics ...*prometheusclientmodel.Metric) { + Eventually(func() error { + metricFamilies, err := registry.Gather() + if err != nil { + return fmt.Errorf("failed to gather metrics: %w", err) + } + var gotMetrics []*prometheusclientmodel.Metric + for _, mf := range metricFamilies { + if mf.GetName() == "fleet_workload_update_run_status_last_timestamp_seconds" { + gotMetrics = mf.GetMetric() + } + } + + if diff := cmp.Diff(gotMetrics, wantMetrics, metricsutils.MetricsCmpOptions...); diff != "" { + return fmt.Errorf("update run status metrics mismatch (-got, +want):\n%s", diff) + } + + return nil + }, timeout, interval).Should(Succeed(), "failed to validate the update run status metrics") +} + +func generateMetricsLabels( + updateRun *placementv1beta1.ClusterStagedUpdateRun, + condition, status, reason string, +) []*prometheusclientmodel.LabelPair { + return []*prometheusclientmodel.LabelPair{ + {Name: ptr.To("name"), Value: &updateRun.Name}, + {Name: ptr.To("generation"), Value: ptr.To(strconv.FormatInt(updateRun.Generation, 10))}, + {Name: ptr.To("condition"), Value: ptr.To(condition)}, + {Name: ptr.To("status"), Value: ptr.To(status)}, + {Name: ptr.To("reason"), Value: ptr.To(reason)}, + } +} + +func generateInitializationFailedMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { + return &prometheusclientmodel.Metric{ + Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionInitialized), + string(metav1.ConditionFalse), condition.UpdateRunInitializeFailedReason), + Gauge: &prometheusclientmodel.Gauge{ + Value: ptr.To(float64(time.Now().UnixNano()) / 1e9), + }, + } +} + +func generateProgressingMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { + return &prometheusclientmodel.Metric{ + Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionProgressing), + string(metav1.ConditionTrue), condition.UpdateRunStartedReason), + Gauge: &prometheusclientmodel.Gauge{ + Value: ptr.To(float64(time.Now().UnixNano()) / 1e9), + }, + } +} + +func generateWaitingMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { + return &prometheusclientmodel.Metric{ + Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionProgressing), + string(metav1.ConditionFalse), condition.UpdateRunWaitingReason), + Gauge: &prometheusclientmodel.Gauge{ + Value: ptr.To(float64(time.Now().UnixNano()) / 1e9), + }, + } +} + +func generateStuckMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { + return &prometheusclientmodel.Metric{ + Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionProgressing), + string(metav1.ConditionFalse), condition.UpdateRunStuckReason), + Gauge: &prometheusclientmodel.Gauge{ + Value: ptr.To(float64(time.Now().UnixNano()) / 1e9), + }, + } +} + +func generateFailedMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { + return &prometheusclientmodel.Metric{ + Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionSucceeded), + string(metav1.ConditionFalse), condition.UpdateRunFailedReason), + Gauge: &prometheusclientmodel.Gauge{ + Value: ptr.To(float64(time.Now().UnixNano()) / 1e9), + }, + } +} + +func generateSucceededMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { + return &prometheusclientmodel.Metric{ + Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionSucceeded), + string(metav1.ConditionTrue), condition.UpdateRunSucceededReason), + Gauge: &prometheusclientmodel.Gauge{ + Value: ptr.To(float64(time.Now().UnixNano()) / 1e9), + }, + } +} + func generateTestClusterStagedUpdateRun() *placementv1beta1.ClusterStagedUpdateRun { return &placementv1beta1.ClusterStagedUpdateRun{ ObjectMeta: metav1.ObjectMeta{ @@ -564,6 +688,8 @@ func generateFalseCondition(obj client.Object, condType any) metav1.Condition { reason = condition.UpdateRunInitializeFailedReason case placementv1beta1.StagedUpdateRunConditionSucceeded: reason = condition.UpdateRunFailedReason + case placementv1beta1.StagedUpdateRunConditionProgressing: + reason = condition.UpdateRunWaitingReason } typeStr = string(cond) case placementv1beta1.StageUpdatingConditionType: diff --git a/pkg/controllers/updaterun/execution.go b/pkg/controllers/updaterun/execution.go index 0c28a8e1e..151193f46 100644 --- a/pkg/controllers/updaterun/execution.go +++ b/pkg/controllers/updaterun/execution.go @@ -44,6 +44,10 @@ var ( // stageUpdatingWaitTime is the time to wait before rechecking the stage update status. // Put it as a variable for convenient testing. stageUpdatingWaitTime = 60 * time.Second + + // updateRunStuckThreshold is the time to wait on a single cluster update before marking update run as stuck. + // TODO(wantjian): make this configurable + updateRunStuckThreshold = 5 * time.Minute ) // execute executes the update run by updating the clusters in the updating stage specified by updatingStageIndex. @@ -55,8 +59,9 @@ func (r *Reconciler) execute( updatingStageIndex int, toBeUpdatedBindings, toBeDeletedBindings []*placementv1beta1.ClusterResourceBinding, ) (bool, time.Duration, error) { - // Mark the update run as started regardless of whether it's already marked. - markUpdateRunStarted(updateRun) + // Before the execute returns, mark the update run as started in memory if it's not marked as waiting or stuck. + // If the update run actually succeeds or fails, it will be remarked later in controller.go. + defer markUpdateRunStartedIfNotWaitingOrStuck(updateRun) if updatingStageIndex < len(updateRun.Status.StagesStatus) { updatingStage := &updateRun.Status.StagesStatus[updatingStageIndex] @@ -172,10 +177,19 @@ func (r *Reconciler) executeUpdatingStage( markClusterUpdatingFailed(clusterStatus, updateRun.Generation, unexpectedErr.Error()) return 0, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) } + finished, updateErr := checkClusterUpdateResult(binding, clusterStatus, updatingStageStatus, updateRun) if finished { finishedClusterCount++ + unmarkUpdateRunStuck(updateRun) continue + } else { + // If cluster update has been running for more than "updateRunStuckThreshold", mark the update run as stuck. + timeElapsed := time.Since(clusterStartedCond.LastTransitionTime.Time) + if timeElapsed > updateRunStuckThreshold { + klog.V(2).InfoS("Time waiting for cluster update to finish passes threshold, mark the update run as stuck", "time elapsed", timeElapsed, "threshold", updateRunStuckThreshold, "cluster", clusterStatus.ClusterName, "stage", updatingStageStatus.StageName, "clusterStagedUpdateRun", updateRunRef) + markUpdateRunStuck(updateRun, updatingStageStatus.StageName, clusterStatus.ClusterName) + } } // No need to continue as we only support one cluster updating at a time for now. return clusterUpdatingWaitTime, updateErr @@ -183,6 +197,7 @@ func (r *Reconciler) executeUpdatingStage( if finishedClusterCount == len(updatingStageStatus.Clusters) { // All the clusters in the stage have been updated. + markUpdateRunWaiting(updateRun, updatingStageStatus.StageName) markStageUpdatingWaiting(updatingStageStatus, updateRun.Generation) klog.V(2).InfoS("The stage has finished all cluster updating", "stage", updatingStageStatus.StageName, "clusterStagedUpdateRun", updateRunRef) // Check if the after stage tasks are ready. @@ -191,6 +206,7 @@ func (r *Reconciler) executeUpdatingStage( return 0, err } if approved { + unmarkUpdateRunWaiting(updateRun) markStageUpdatingSucceeded(updatingStageStatus, updateRun.Generation) // No need to wait to get to the next stage. return 0, nil @@ -448,8 +464,14 @@ func checkClusterUpdateResult( return false, nil } -// markUpdateRunStarted marks the update run as started in memory. -func markUpdateRunStarted(updateRun *placementv1beta1.ClusterStagedUpdateRun) { +// markUpdateRunStartedIfNotWaitingOrStuck marks the update run as started in memory if it's not marked as waiting or stuck already. +func markUpdateRunStartedIfNotWaitingOrStuck(updateRun *placementv1beta1.ClusterStagedUpdateRun) { + progressingCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) + if condition.IsConditionStatusFalse(progressingCond, updateRun.Generation) && + (progressingCond.Reason == condition.UpdateRunWaitingReason || progressingCond.Reason == condition.UpdateRunStuckReason) { + // The updateRun is waiting or stuck, no need to mark it as started. + return + } meta.SetStatusCondition(&updateRun.Status.Conditions, metav1.Condition{ Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), Status: metav1.ConditionTrue, @@ -459,6 +481,44 @@ func markUpdateRunStarted(updateRun *placementv1beta1.ClusterStagedUpdateRun) { }) } +// markUpdateRunStuck marks the updateRun as stuck in memory. +func markUpdateRunStuck(updateRun *placementv1beta1.ClusterStagedUpdateRun, stageName, clusterName string) { + meta.SetStatusCondition(&updateRun.Status.Conditions, metav1.Condition{ + Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), + Status: metav1.ConditionFalse, + ObservedGeneration: updateRun.Generation, + Reason: condition.UpdateRunStuckReason, + Message: fmt.Sprintf("The updateRun is stuck waiting for cluster %s in stage %s to finish updating, please check crp status for potential errors", clusterName, stageName), + }) +} + +// unmarkUpdateRunStuck unmarks the updateRun as stuck in memory. +func unmarkUpdateRunStuck(updateRun *placementv1beta1.ClusterStagedUpdateRun) { + progressingCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) + if condition.IsConditionStatusFalse(progressingCond, updateRun.Generation) && progressingCond.Reason == condition.UpdateRunStuckReason { + meta.RemoveStatusCondition(&updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) + } +} + +// markUpdateRunWaiting marks the updateRun as waiting in memory. +func markUpdateRunWaiting(updateRun *placementv1beta1.ClusterStagedUpdateRun, stageName string) { + meta.SetStatusCondition(&updateRun.Status.Conditions, metav1.Condition{ + Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), + Status: metav1.ConditionFalse, + ObservedGeneration: updateRun.Generation, + Reason: condition.UpdateRunWaitingReason, + Message: fmt.Sprintf("The updateRun is waiting for after-stage tasks in stage %s to complete", stageName), + }) +} + +// unmarkUpdateRunWaiting unmarks the updateRun as waiting in memory. +func unmarkUpdateRunWaiting(updateRun *placementv1beta1.ClusterStagedUpdateRun) { + progressingCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) + if condition.IsConditionStatusFalse(progressingCond, updateRun.Generation) && progressingCond.Reason == condition.UpdateRunWaitingReason { + meta.RemoveStatusCondition(&updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) + } +} + // markStageUpdatingStarted marks the stage updating status as started in memory. func markStageUpdatingStarted(stageUpdatingStatus *placementv1beta1.StageUpdatingStatus, generation int64) { if stageUpdatingStatus.StartTime == nil { diff --git a/pkg/controllers/updaterun/execution_integration_test.go b/pkg/controllers/updaterun/execution_integration_test.go index 49e763de3..0a2c6e8cd 100644 --- a/pkg/controllers/updaterun/execution_integration_test.go +++ b/pkg/controllers/updaterun/execution_integration_test.go @@ -26,6 +26,7 @@ import ( "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -50,6 +51,7 @@ var _ = Describe("UpdateRun execution tests", func() { var resourceSnapshot *placementv1beta1.ClusterResourceSnapshot var clusterResourceOverride *placementv1alpha1.ClusterResourceOverrideSnapshot var wantStatus *placementv1beta1.StagedUpdateRunStatus + var customRegistry *prometheus.Registry BeforeEach(OncePerOrdered, func() { testUpdateRunName = "updaterun-" + utils.RandStr() @@ -105,6 +107,8 @@ var _ = Describe("UpdateRun execution tests", func() { stageUpdatingWaitTime = time.Second * 3 clusterUpdatingWaitTime = time.Second * 2 + customRegistry = initializeUpdateRunMetricsRegistry() + By("Creating a new clusterResourcePlacement") Expect(k8sClient.Create(ctx, crp)).To(Succeed()) @@ -182,6 +186,11 @@ var _ = Describe("UpdateRun execution tests", func() { By("Deleting the clusterResourceOverride") Expect(k8sClient.Delete(ctx, clusterResourceOverride)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) clusterResourceOverride = nil + + By("Checking update run status metrics are removed") + // No metrics are emitted as all are removed after updateRun is deleted. + validateUpdateRunMetricsEmitted(customRegistry) + unregisterUpdateRunMetrics(customRegistry) }) Context("Cluster staged update run should update clusters one by one - strategy with double afterStageTasks", Ordered, func() { @@ -193,6 +202,9 @@ var _ = Describe("UpdateRun execution tests", func() { initialized := generateSucceededInitializationStatus(crp, updateRun, policySnapshot, updateStrategy, clusterResourceOverride) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun)) }) It("Should mark the 1st cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -211,6 +223,9 @@ var _ = Describe("UpdateRun execution tests", func() { By("Validating the 1st stage has startTime set") Expect(updateRun.Status.StagesStatus[0].StartTime).ShouldNot(BeNil()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun)) }) It("Should mark the 2nd cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -226,6 +241,9 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.StagesStatus[0].Clusters[1].Conditions = append(wantStatus.StagesStatus[0].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun)) }) It("Should mark the 3rd cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -241,6 +259,9 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[0].Clusters[3].Conditions = append(wantStatus.StagesStatus[0].Clusters[3].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun)) }) It("Should mark the 4th cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -256,6 +277,9 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.StagesStatus[0].Clusters[3].Conditions = append(wantStatus.StagesStatus[0].Clusters[3].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[0].Clusters[4].Conditions = append(wantStatus.StagesStatus[0].Clusters[4].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun)) }) It("Should mark the 5th cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -272,7 +296,11 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.StagesStatus[0].Conditions[0] = generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) // The progressing condition now becomes false with waiting reason. wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.AfterStageTaskConditionApprovalRequestCreated)) + wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun), generateWaitingMetric(updateRun)) }) It("Should complete the 1st stage after wait time passed and approval request approved and move on to the 2nd stage", func() { @@ -323,6 +351,7 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.StagesStatus[1].Conditions = append(wantStatus.StagesStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) // 1st cluster in 2nd stage started. wantStatus.StagesStatus[1].Clusters[0].Conditions = append(wantStatus.StagesStatus[1].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + wantStatus.Conditions[1] = generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Validating the 1st stage has endTime set") @@ -337,6 +366,9 @@ var _ = Describe("UpdateRun execution tests", func() { By("Validating the creation time of the approval request is before the complete time of the timedwait task") approvalCreateTime := meta.FindStatusCondition(updateRun.Status.StagesStatus[0].AfterStageTaskStatus[1].Conditions, string(placementv1beta1.AfterStageTaskConditionApprovalRequestCreated)).LastTransitionTime.Time Expect(approvalCreateTime.Before(waitEndTime)).Should(BeTrue()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 1st cluster in the 2nd stage as succeeded after marking the binding available", func() { @@ -355,6 +387,9 @@ var _ = Describe("UpdateRun execution tests", func() { By("Validating the 2nd stage has startTime set") Expect(updateRun.Status.StagesStatus[0].StartTime).ShouldNot(BeNil()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 2nd cluster in the 2nd stage as succeeded after marking the binding available", func() { @@ -370,6 +405,9 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.StagesStatus[1].Clusters[1].Conditions = append(wantStatus.StagesStatus[1].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[1].Clusters[2].Conditions = append(wantStatus.StagesStatus[1].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 3rd cluster in the 2nd stage as succeeded after marking the binding available", func() { @@ -385,6 +423,9 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.StagesStatus[1].Clusters[2].Conditions = append(wantStatus.StagesStatus[1].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[1].Clusters[3].Conditions = append(wantStatus.StagesStatus[1].Clusters[3].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 4th cluster in the 2nd stage as succeeded after marking the binding available", func() { @@ -400,6 +441,9 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.StagesStatus[1].Clusters[3].Conditions = append(wantStatus.StagesStatus[1].Clusters[3].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[1].Clusters[4].Conditions = append(wantStatus.StagesStatus[1].Clusters[4].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 5th cluster in the 2nd stage as succeeded after marking the binding available", func() { @@ -416,7 +460,11 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.StagesStatus[1].Conditions[0] = generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) // The progressing condition now becomes false with waiting reason. wantStatus.StagesStatus[1].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[1].AfterStageTaskStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.AfterStageTaskConditionApprovalRequestCreated)) + wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun), generateWaitingMetric(updateRun)) }) It("Should complete the 2nd stage after both after stage tasks are completed and move on to the delete stage", func() { @@ -460,6 +508,7 @@ var _ = Describe("UpdateRun execution tests", func() { generateTrueCondition(updateRun, placementv1beta1.AfterStageTaskConditionWaitTimeElapsed)) wantStatus.StagesStatus[1].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) wantStatus.StagesStatus[1].Conditions = append(wantStatus.StagesStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) + wantStatus.Conditions[1] = generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) for i := range wantStatus.DeletionStageStatus.Clusters { @@ -487,6 +536,9 @@ var _ = Describe("UpdateRun execution tests", func() { } return condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.Generation), nil }, timeout, interval).Should(BeTrue(), "failed to validate the approvalRequest approval accepted") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should delete all the clusterResourceBindings in the delete stage and complete the update run", func() { @@ -516,6 +568,9 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateWaitingMetric(updateRun), generateProgressingMetric(updateRun), generateSucceededMetric(updateRun)) }) }) @@ -533,6 +588,9 @@ var _ = Describe("UpdateRun execution tests", func() { initialized := generateSucceededInitializationStatus(crp, updateRun, policySnapshot, updateStrategy, clusterResourceOverride) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun)) }) It("Should mark the 1st cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -551,6 +609,9 @@ var _ = Describe("UpdateRun execution tests", func() { By("Validating the 1st stage has startTime set") Expect(updateRun.Status.StagesStatus[0].StartTime).ShouldNot(BeNil()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun)) }) It("Should mark the 2nd cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -566,6 +627,9 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.StagesStatus[0].Clusters[1].Conditions = append(wantStatus.StagesStatus[0].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun)) }) It("Should mark the 3rd cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -581,6 +645,9 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[0].Clusters[3].Conditions = append(wantStatus.StagesStatus[0].Clusters[3].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun)) }) It("Should mark the 4th cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -596,6 +663,9 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.StagesStatus[0].Clusters[3].Conditions = append(wantStatus.StagesStatus[0].Clusters[3].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[0].Clusters[4].Conditions = append(wantStatus.StagesStatus[0].Clusters[4].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun)) }) It("Should mark the 5th cluster in the 1st stage as succeeded after marking the binding available", func() { @@ -610,7 +680,11 @@ var _ = Describe("UpdateRun execution tests", func() { By("Validating the 5th cluster has succeeded and stage waiting for AfterStageTasks") wantStatus.StagesStatus[0].Clusters[4].Conditions = append(wantStatus.StagesStatus[0].Clusters[4].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[0].Conditions[0] = generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) // The progressing condition now becomes false with waiting reason. + wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun), generateWaitingMetric(updateRun)) }) It("Should complete the 1st stage after wait time passed and move on to the 2nd stage", func() { @@ -625,6 +699,7 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.StagesStatus[1].Conditions = append(wantStatus.StagesStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) // 1st cluster in 2nd stage started. wantStatus.StagesStatus[1].Clusters[0].Conditions = append(wantStatus.StagesStatus[1].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + wantStatus.Conditions[1] = generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Validating the 1st stage has endTime set") @@ -635,6 +710,9 @@ var _ = Describe("UpdateRun execution tests", func() { waitEndTime := meta.FindStatusCondition(updateRun.Status.StagesStatus[0].AfterStageTaskStatus[0].Conditions, string(placementv1beta1.AfterStageTaskConditionWaitTimeElapsed)).LastTransitionTime.Time Expect(waitStartTime.Add(updateStrategy.Spec.Stages[0].AfterStageTasks[0].WaitTime.Duration).After(waitEndTime)).Should(BeFalse(), fmt.Sprintf("waitEndTime %v did not pass waitStartTime %v long enough, want at least %v", waitEndTime, waitStartTime, updateStrategy.Spec.Stages[0].AfterStageTasks[0].WaitTime.Duration)) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 1st cluster in the 2nd stage as succeeded after marking the binding available", func() { @@ -653,6 +731,9 @@ var _ = Describe("UpdateRun execution tests", func() { By("Validating the 2nd stage has startTime set") Expect(updateRun.Status.StagesStatus[0].StartTime).ShouldNot(BeNil()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 2nd cluster in the 2nd stage as succeeded after marking the binding available", func() { @@ -668,6 +749,9 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.StagesStatus[1].Clusters[1].Conditions = append(wantStatus.StagesStatus[1].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[1].Clusters[2].Conditions = append(wantStatus.StagesStatus[1].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 3rd cluster in the 2nd stage as succeeded after marking the binding available", func() { @@ -683,6 +767,9 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.StagesStatus[1].Clusters[2].Conditions = append(wantStatus.StagesStatus[1].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[1].Clusters[3].Conditions = append(wantStatus.StagesStatus[1].Clusters[3].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 4th cluster in the 2nd stage as succeeded after marking the binding available", func() { @@ -698,6 +785,9 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.StagesStatus[1].Clusters[3].Conditions = append(wantStatus.StagesStatus[1].Clusters[3].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) wantStatus.StagesStatus[1].Clusters[4].Conditions = append(wantStatus.StagesStatus[1].Clusters[4].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should mark the 5th cluster in the 2nd stage as succeeded after marking the binding available", func() { @@ -714,7 +804,11 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.StagesStatus[1].Conditions[0] = generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing) // The progressing condition now becomes false with waiting reason. wantStatus.StagesStatus[1].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[1].AfterStageTaskStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.AfterStageTaskConditionApprovalRequestCreated)) + wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun), generateWaitingMetric(updateRun)) }) It("Should complete the 2nd stage after the after stage task is completed and move on to the delete stage", func() { @@ -761,6 +855,7 @@ var _ = Describe("UpdateRun execution tests", func() { for i := range wantStatus.DeletionStageStatus.Clusters { wantStatus.DeletionStageStatus.Clusters[i].Conditions = append(wantStatus.DeletionStageStatus.Clusters[i].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) } + wantStatus.Conditions[1] = generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") By("Validating the 2nd stage has endTime set") @@ -773,6 +868,9 @@ var _ = Describe("UpdateRun execution tests", func() { } return condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.Generation), nil }, timeout, interval).Should(BeTrue(), "failed to validate the approvalRequest approval accepted") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateWaitingMetric(updateRun), generateProgressingMetric(updateRun)) }) It("Should delete all the clusterResourceBindings in the delete stage and complete the update run", func() { @@ -800,11 +898,19 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateWaitingMetric(updateRun), generateProgressingMetric(updateRun), generateSucceededMetric(updateRun)) }) }) Context("Cluster staged update run should abort the execution within a failed updating stage", Ordered, func() { + var oldUpdateRunStuckThreshold time.Duration BeforeAll(func() { + // Set the updateRunStuckThreshold to 1 second for this test. + oldUpdateRunStuckThreshold = updateRunStuckThreshold + updateRunStuckThreshold = 1 * time.Second + By("Creating a new clusterStagedUpdateRun") Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) @@ -812,6 +918,14 @@ var _ = Describe("UpdateRun execution tests", func() { initialized := generateSucceededInitializationStatus(crp, updateRun, policySnapshot, updateStrategy, clusterResourceOverride) wantStatus = generateExecutionStartedStatus(updateRun, initialized) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun)) + }) + + AfterAll(func() { + // Restore the updateRunStuckThreshold to the original value. + updateRunStuckThreshold = oldUpdateRunStuckThreshold }) It("Should keep waiting for the 1st cluster while it's not available", func() { @@ -824,10 +938,17 @@ var _ = Describe("UpdateRun execution tests", func() { Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") By("Validating the updateRun is stuck in the 1st cluster of the 1st stage") + wantStatus.Conditions[1] = generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) + wantStatus.Conditions[1].Reason = condition.UpdateRunStuckReason validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, wantStatus, "") }) + It("Should emit stuck status metrics after time waiting for the 1st cluster reaches threshold", func() { + By("Checking update run stuck metrics is emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun), generateStuckMetric(updateRun)) + }) + It("Should abort the execution if the binding has unexpected state", func() { By("Validating the 1st clusterResourceBinding is updated to Bound") binding := resourceBindings[numTargetClusters-1] // cluster-9 @@ -844,6 +965,9 @@ var _ = Describe("UpdateRun execution tests", func() { wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, false) wantStatus.Conditions = append(wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun), generateStuckMetric(updateRun), generateFailedMetric(updateRun)) }) }) }) diff --git a/pkg/controllers/updaterun/initialization_integration_test.go b/pkg/controllers/updaterun/initialization_integration_test.go index 09c677bc7..825531f6b 100644 --- a/pkg/controllers/updaterun/initialization_integration_test.go +++ b/pkg/controllers/updaterun/initialization_integration_test.go @@ -28,6 +28,7 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -62,6 +63,7 @@ var _ = Describe("Updaterun initialization tests", func() { var unscheduledCluster []*clusterv1beta1.MemberCluster var resourceSnapshot *placementv1beta1.ClusterResourceSnapshot var clusterResourceOverride *placementv1alpha1.ClusterResourceOverrideSnapshot + var customRegistry *prometheus.Registry BeforeEach(func() { testUpdateRunName = "updaterun-" + utils.RandStr() @@ -116,6 +118,8 @@ var _ = Describe("Updaterun initialization tests", func() { // Set smaller wait time for testing stageUpdatingWaitTime = time.Second * 3 clusterUpdatingWaitTime = time.Second * 2 + + customRegistry = initializeUpdateRunMetricsRegistry() }) AfterEach(func() { @@ -157,9 +161,19 @@ var _ = Describe("Updaterun initialization tests", func() { By("Deleting the clusterResourceOverride") Expect(k8sClient.Delete(ctx, clusterResourceOverride)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) clusterResourceOverride = nil + + By("Checking the update run status metrics are removed") + // No metrics are emitted as all are removed after updateRun is deleted. + validateUpdateRunMetricsEmitted(customRegistry) + unregisterUpdateRunMetrics(customRegistry) }) Context("Test validateCRP", func() { + AfterEach(func() { + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateInitializationFailedMetric(updateRun)) + }) + It("Should fail to initialize if CRP is not found", func() { By("Creating a new clusterStagedUpdateRun") Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) @@ -207,6 +221,11 @@ var _ = Describe("Updaterun initialization tests", func() { Expect(k8sClient.Create(ctx, crp)).To(Succeed()) }) + AfterEach(func() { + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateInitializationFailedMetric(updateRun)) + }) + It("Should fail to initialize if the latest policy snapshot is not found", func() { By("Creating a new clusterStagedUpdateRun") Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) @@ -381,6 +400,11 @@ var _ = Describe("Updaterun initialization tests", func() { Expect(k8sClient.Status().Update(ctx, policySnapshot)).Should(Succeed(), "failed to update the policy snapshot condition") }) + AfterEach(func() { + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateInitializationFailedMetric(updateRun)) + }) + It("Should fail to initialize if there is no selected or to-be-deleted cluster", func() { By("Creating a new clusterStagedUpdateRun") Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) @@ -512,6 +536,11 @@ var _ = Describe("Updaterun initialization tests", func() { } }) + AfterEach(func() { + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateInitializationFailedMetric(updateRun)) + }) + It("fail the initialization if the clusterStagedUpdateStrategy is not found", func() { By("Creating a new clusterStagedUpdateRun") Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) @@ -664,6 +693,9 @@ var _ = Describe("Updaterun initialization tests", func() { By("Validating the initialization failed") validateFailedInitCondition(ctx, updateRun, "invalid resource snapshot index `invalid-index` provided") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateInitializationFailedMetric(updateRun)) }) It("Should fail to initialize if the specified resource snapshot index is invalid - negative integer", func() { @@ -673,6 +705,9 @@ var _ = Describe("Updaterun initialization tests", func() { By("Validating the initialization failed") validateFailedInitCondition(ctx, updateRun, "invalid resource snapshot index `-1` provided") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateInitializationFailedMetric(updateRun)) }) It("Should fail to initialize if the specified resource snapshot is not found - no resourceSnapshots at all", func() { @@ -681,6 +716,9 @@ var _ = Describe("Updaterun initialization tests", func() { By("Validating the initialization failed") validateFailedInitCondition(ctx, updateRun, "no clusterResourceSnapshots with index `0` found") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateInitializationFailedMetric(updateRun)) }) It("Should fail to initialize if the specified resource snapshot is not found - no CRP label found", func() { @@ -693,6 +731,9 @@ var _ = Describe("Updaterun initialization tests", func() { By("Validating the initialization failed") validateFailedInitCondition(ctx, updateRun, "no clusterResourceSnapshots with index `0` found") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateInitializationFailedMetric(updateRun)) }) It("Should fail to initialize if the specified resource snapshot is not found - no resource index label found", func() { @@ -705,6 +746,9 @@ var _ = Describe("Updaterun initialization tests", func() { By("Validating the initialization failed") validateFailedInitCondition(ctx, updateRun, "no clusterResourceSnapshots with index `0` found") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateInitializationFailedMetric(updateRun)) }) It("Should fail to initialize if the specified resource snapshot is not master snapshot", func() { @@ -717,6 +761,9 @@ var _ = Describe("Updaterun initialization tests", func() { By("Validating the initialization failed") validateFailedInitCondition(ctx, updateRun, "no master clusterResourceSnapshot found for clusterResourcePlacement") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateInitializationFailedMetric(updateRun)) }) It("Should put related ClusterResourceOverrides in the status", func() { @@ -736,6 +783,9 @@ var _ = Describe("Updaterun initialization tests", func() { By("Validating the clusterStagedUpdateRun initialized consistently") validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, want, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun)) }) }) }) diff --git a/pkg/controllers/updaterun/validation_integration_test.go b/pkg/controllers/updaterun/validation_integration_test.go index 8e1138712..80938583d 100644 --- a/pkg/controllers/updaterun/validation_integration_test.go +++ b/pkg/controllers/updaterun/validation_integration_test.go @@ -27,6 +27,7 @@ import ( "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -49,6 +50,7 @@ var _ = Describe("UpdateRun validation tests", func() { var resourceSnapshot *placementv1beta1.ClusterResourceSnapshot var clusterResourceOverride *placementv1alpha1.ClusterResourceOverrideSnapshot var wantStatus *placementv1beta1.StagedUpdateRunStatus + var customRegistry *prometheus.Registry BeforeEach(func() { testUpdateRunName = "updaterun-" + utils.RandStr() @@ -104,6 +106,8 @@ var _ = Describe("UpdateRun validation tests", func() { stageUpdatingWaitTime = time.Second * 3 clusterUpdatingWaitTime = time.Second * 2 + customRegistry = initializeUpdateRunMetricsRegistry() + By("Creating a new clusterResourcePlacement") Expect(k8sClient.Create(ctx, crp)).To(Succeed()) @@ -189,9 +193,19 @@ var _ = Describe("UpdateRun validation tests", func() { By("Deleting the clusterResourceOverride") Expect(k8sClient.Delete(ctx, clusterResourceOverride)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{})) clusterResourceOverride = nil + + By("Checking the update run status metrics are removed") + // No metrics are emitted as all are removed after updateRun is deleted. + validateUpdateRunMetricsEmitted(customRegistry) + unregisterUpdateRunMetrics(customRegistry) }) Context("Test validateCRP", func() { + AfterEach(func() { + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun), generateFailedMetric(updateRun)) + }) + It("Should fail to validate if the CRP is not found", func() { By("Deleting the clusterResourcePlacement") Expect(k8sClient.Delete(ctx, crp)).Should(Succeed()) @@ -231,6 +245,9 @@ var _ = Describe("UpdateRun validation tests", func() { By("Validating the validation failed") wantStatus = generateFailedValidationStatus(updateRun, wantStatus) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "no latest policy snapshot associated") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun), generateFailedMetric(updateRun)) }) It("Should fail to validate if the latest policySnapshot has changed", func() { @@ -257,6 +274,9 @@ var _ = Describe("UpdateRun validation tests", func() { By("Deleting the new policySnapshot") Expect(k8sClient.Delete(ctx, newPolicySnapshot)).Should(Succeed()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun), generateFailedMetric(updateRun)) }) It("Should fail to validate if the cluster count has changed", func() { @@ -268,6 +288,9 @@ var _ = Describe("UpdateRun validation tests", func() { wantStatus = generateFailedValidationStatus(updateRun, wantStatus) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "the cluster count initialized in the clusterStagedUpdateRun is outdated") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun), generateFailedMetric(updateRun)) }) It("Should not fail due to different cluster count if it's pickAll policy", func() { @@ -286,10 +309,18 @@ var _ = Describe("UpdateRun validation tests", func() { By("Validating the validation does not fail") validateClusterStagedUpdateRunStatusConsistently(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun)) }) }) Context("Test validateStagesStatus", func() { + AfterEach(func() { + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(customRegistry, generateProgressingMetric(updateRun), generateFailedMetric(updateRun)) + }) + It("Should fail to validate if the StagedUpdateStrategySnapshot is nil", func() { By("Updating the status.StagedUpdateStrategySnapshot to nil") updateRun.Status.StagedUpdateStrategySnapshot = nil diff --git a/pkg/utils/condition/condition.go b/pkg/utils/condition/condition.go index 2ccc54262..3a34f13c8 100644 --- a/pkg/utils/condition/condition.go +++ b/pkg/utils/condition/condition.go @@ -150,6 +150,12 @@ const ( // UpdateRunFailedReason is the reason string of condition if the staged update run failed. UpdateRunFailedReason = "UpdateRunFailed" + // UpdateRunStuckReason is the reason string of condition if the staged update run is stuck waiting for a cluster to be updated. + UpdateRunStuckReason = "UpdateRunStuck" + + // UpdateRunWaitingReason is the reason string of condition if the staged update run is waiting for an after-stage task to complete. + UpdateRunWaitingReason = "UpdateRunWaiting" + // UpdateRunSucceededReason is the reason string of condition if the staged update run succeeded. UpdateRunSucceededReason = "UpdateRunSucceeded" diff --git a/pkg/utils/controller/metrics/metrics.go b/pkg/utils/controller/metrics/metrics.go index b6fa9187c..327f3589a 100644 --- a/pkg/utils/controller/metrics/metrics.go +++ b/pkg/utils/controller/metrics/metrics.go @@ -73,6 +73,12 @@ var ( Name: "fleet_workload_eviction_complete", Help: "Eviction complete status ", }, []string{"name", "isCompleted", "isValid"}) + // FleetUpdateRunStatusLastTimestampSeconds is a prometheus metric which holds the + // last update timestamp of update run status in seconds. + FleetUpdateRunStatusLastTimestampSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "fleet_workload_update_run_status_last_timestamp_seconds", + Help: "Last update timestamp of update run status in seconds", + }, []string{"name", "generation", "condition", "status", "reason"}) ) func init() { @@ -84,5 +90,6 @@ func init() { FleetActiveWorkers, FleetPlacementStatusLastTimeStampSeconds, FleetEvictionStatus, + FleetUpdateRunStatusLastTimestampSeconds, ) } From 00ca14e5c8ed158488e3cdf16d3ccfef74b3f08c Mon Sep 17 00:00:00 2001 From: Wantong Jiang Date: Wed, 23 Apr 2025 04:11:17 +0000 Subject: [PATCH 2/2] fix comments Signed-off-by: Wantong Jiang --- .../updaterun/controller_integration_test.go | 4 +- pkg/controllers/updaterun/execution.go | 50 ++++++++----------- pkg/utils/condition/condition.go | 4 +- 3 files changed, 24 insertions(+), 34 deletions(-) diff --git a/pkg/controllers/updaterun/controller_integration_test.go b/pkg/controllers/updaterun/controller_integration_test.go index fe8f892ce..92b3855d9 100644 --- a/pkg/controllers/updaterun/controller_integration_test.go +++ b/pkg/controllers/updaterun/controller_integration_test.go @@ -300,7 +300,7 @@ func generateInitializationFailedMetric(updateRun *placementv1beta1.ClusterStage func generateProgressingMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { return &prometheusclientmodel.Metric{ Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionProgressing), - string(metav1.ConditionTrue), condition.UpdateRunStartedReason), + string(metav1.ConditionTrue), condition.UpdateRunProgressingReason), Gauge: &prometheusclientmodel.Gauge{ Value: ptr.To(float64(time.Now().UnixNano()) / 1e9), }, @@ -627,7 +627,7 @@ func generateTrueCondition(obj client.Object, condType any) metav1.Condition { case placementv1beta1.StagedUpdateRunConditionInitialized: reason = condition.UpdateRunInitializeSucceededReason case placementv1beta1.StagedUpdateRunConditionProgressing: - reason = condition.UpdateRunStartedReason + reason = condition.UpdateRunProgressingReason case placementv1beta1.StagedUpdateRunConditionSucceeded: reason = condition.UpdateRunSucceededReason } diff --git a/pkg/controllers/updaterun/execution.go b/pkg/controllers/updaterun/execution.go index 151193f46..00a27c75c 100644 --- a/pkg/controllers/updaterun/execution.go +++ b/pkg/controllers/updaterun/execution.go @@ -59,9 +59,10 @@ func (r *Reconciler) execute( updatingStageIndex int, toBeUpdatedBindings, toBeDeletedBindings []*placementv1beta1.ClusterResourceBinding, ) (bool, time.Duration, error) { - // Before the execute returns, mark the update run as started in memory if it's not marked as waiting or stuck. - // If the update run actually succeeds or fails, it will be remarked later in controller.go. - defer markUpdateRunStartedIfNotWaitingOrStuck(updateRun) + // Mark updateRun as progressing if it's not already marked as waiting or stuck. + // This avoids triggering an unnecessary in-memory transition from stuck (waiting) -> progressing -> stuck (waiting), + // which would update the lastTransitionTime even though the status hasn't effectively changed. + markUpdateRunProgressingIfNotWaitingOrStuck(updateRun) if updatingStageIndex < len(updateRun.Status.StagesStatus) { updatingStage := &updateRun.Status.StagesStatus[updatingStageIndex] @@ -181,7 +182,7 @@ func (r *Reconciler) executeUpdatingStage( finished, updateErr := checkClusterUpdateResult(binding, clusterStatus, updatingStageStatus, updateRun) if finished { finishedClusterCount++ - unmarkUpdateRunStuck(updateRun) + markUpdateRunProgressing(updateRun) continue } else { // If cluster update has been running for more than "updateRunStuckThreshold", mark the update run as stuck. @@ -206,7 +207,7 @@ func (r *Reconciler) executeUpdatingStage( return 0, err } if approved { - unmarkUpdateRunWaiting(updateRun) + markUpdateRunProgressing(updateRun) markStageUpdatingSucceeded(updatingStageStatus, updateRun.Generation) // No need to wait to get to the next stage. return 0, nil @@ -464,21 +465,26 @@ func checkClusterUpdateResult( return false, nil } -// markUpdateRunStartedIfNotWaitingOrStuck marks the update run as started in memory if it's not marked as waiting or stuck already. -func markUpdateRunStartedIfNotWaitingOrStuck(updateRun *placementv1beta1.ClusterStagedUpdateRun) { +// markUpdateRunProgressing marks the update run as progressing in memory. +func markUpdateRunProgressing(updateRun *placementv1beta1.ClusterStagedUpdateRun) { + meta.SetStatusCondition(&updateRun.Status.Conditions, metav1.Condition{ + Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), + Status: metav1.ConditionTrue, + ObservedGeneration: updateRun.Generation, + Reason: condition.UpdateRunProgressingReason, + Message: "The update run is making progress", + }) +} + +// markUpdateRunProgressingIfNotWaitingOrStuck marks the update run as proegressing in memory if it's not marked as waiting or stuck already. +func markUpdateRunProgressingIfNotWaitingOrStuck(updateRun *placementv1beta1.ClusterStagedUpdateRun) { progressingCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) if condition.IsConditionStatusFalse(progressingCond, updateRun.Generation) && (progressingCond.Reason == condition.UpdateRunWaitingReason || progressingCond.Reason == condition.UpdateRunStuckReason) { // The updateRun is waiting or stuck, no need to mark it as started. return } - meta.SetStatusCondition(&updateRun.Status.Conditions, metav1.Condition{ - Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), - Status: metav1.ConditionTrue, - ObservedGeneration: updateRun.Generation, - Reason: condition.UpdateRunStartedReason, - Message: "The stages started updating", - }) + markUpdateRunProgressing(updateRun) } // markUpdateRunStuck marks the updateRun as stuck in memory. @@ -492,14 +498,6 @@ func markUpdateRunStuck(updateRun *placementv1beta1.ClusterStagedUpdateRun, stag }) } -// unmarkUpdateRunStuck unmarks the updateRun as stuck in memory. -func unmarkUpdateRunStuck(updateRun *placementv1beta1.ClusterStagedUpdateRun) { - progressingCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) - if condition.IsConditionStatusFalse(progressingCond, updateRun.Generation) && progressingCond.Reason == condition.UpdateRunStuckReason { - meta.RemoveStatusCondition(&updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) - } -} - // markUpdateRunWaiting marks the updateRun as waiting in memory. func markUpdateRunWaiting(updateRun *placementv1beta1.ClusterStagedUpdateRun, stageName string) { meta.SetStatusCondition(&updateRun.Status.Conditions, metav1.Condition{ @@ -511,14 +509,6 @@ func markUpdateRunWaiting(updateRun *placementv1beta1.ClusterStagedUpdateRun, st }) } -// unmarkUpdateRunWaiting unmarks the updateRun as waiting in memory. -func unmarkUpdateRunWaiting(updateRun *placementv1beta1.ClusterStagedUpdateRun) { - progressingCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) - if condition.IsConditionStatusFalse(progressingCond, updateRun.Generation) && progressingCond.Reason == condition.UpdateRunWaitingReason { - meta.RemoveStatusCondition(&updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) - } -} - // markStageUpdatingStarted marks the stage updating status as started in memory. func markStageUpdatingStarted(stageUpdatingStatus *placementv1beta1.StageUpdatingStatus, generation int64) { if stageUpdatingStatus.StartTime == nil { diff --git a/pkg/utils/condition/condition.go b/pkg/utils/condition/condition.go index 3a34f13c8..b566a009c 100644 --- a/pkg/utils/condition/condition.go +++ b/pkg/utils/condition/condition.go @@ -144,8 +144,8 @@ const ( // UpdateRunInitializeFailedReason is the reason string of condition if the update run is failed to initialize. UpdateRunInitializeFailedReason = "UpdateRunInitializedFailed" - // UpdateRunStartedReason is the reason string of condition if the staged update run has started. - UpdateRunStartedReason = "UpdateRunStarted" + // UpdateRunProgressingReason is the reason string of condition if the staged update run is progressing. + UpdateRunProgressingReason = "UpdateRunProgressing" // UpdateRunFailedReason is the reason string of condition if the staged update run failed. UpdateRunFailedReason = "UpdateRunFailed"