From def75625872bb6afd0aa0b323bb103335034a45c Mon Sep 17 00:00:00 2001 From: Wantong Jiang Date: Fri, 25 Apr 2025 22:01:02 +0000 Subject: [PATCH 1/5] interface: CRP API change to support multiple resourceSnapshot versions across clusters Signed-off-by: Wantong Jiang --- .../v1beta1/clusterresourceplacement_types.go | 10 ++++++++++ ....kubernetes-fleet.io_clusterresourceplacements.yaml | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/apis/placement/v1beta1/clusterresourceplacement_types.go b/apis/placement/v1beta1/clusterresourceplacement_types.go index 9b07f5ef5..2cefc94f5 100644 --- a/apis/placement/v1beta1/clusterresourceplacement_types.go +++ b/apis/placement/v1beta1/clusterresourceplacement_types.go @@ -822,6 +822,9 @@ type ClusterResourcePlacementStatus struct { // One resource snapshot can contain multiple clusterResourceSnapshots CRs in order to store large amount of resources. // To get clusterResourceSnapshot of a given resource index, use the following command: // `kubectl get ClusterResourceSnapshot --selector=kubernetes-fleet.io/resource-index=$ObservedResourceIndex ` + // During rollout, clusters may run different versions of the resource snapshot concurrently. + // In this case, ObservedResourceIndex represents the index of the latest resource snapshot installed across all clusters. + // Note that this may differ from the index of the latest resource snapshot available in the hub cluster depending on the rollout strategy. // ObservedResourceIndex is the resource index that the conditions in the ClusterResourcePlacementStatus observe. // For example, a condition of `ClusterResourcePlacementWorkSynchronized` type // is observing the synchronization status of the resource snapshot with the resource index $ObservedResourceIndex. @@ -954,6 +957,13 @@ type ResourcePlacementStatus struct { // +kubebuilder:validation:MaxItems=100 DiffedPlacements []DiffedResourcePlacement `json:"diffedPlacements,omitempty"` + // ObservedResourceIndex is the index of the resource snapshot that is currently rolled out on the given cluster. + // During rollout, depending on the rollout strategy, clusters may observe different resource indices. + // ObservedResourceIndex is the resource snapshot index observed by the conditions in the ResourcePlacementStatus. + // This field is only meaningful if the `ClusterName` is not empty. + // +kubebuilder:validation:Optional + ObservedResourceIndex string `json:"observedResourceIndex,omitempty"` + // Conditions is an array of current observed conditions for ResourcePlacementStatus. // +kubebuilder:validation:Optional Conditions []metav1.Condition `json:"conditions,omitempty"` diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml index 195c11d65..ad5877566 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml @@ -2044,6 +2044,9 @@ spec: One resource snapshot can contain multiple clusterResourceSnapshots CRs in order to store large amount of resources. To get clusterResourceSnapshot of a given resource index, use the following command: `kubectl get ClusterResourceSnapshot --selector=kubernetes-fleet.io/resource-index=$ObservedResourceIndex ` + During rollout, clusters may run different versions of the resource snapshot concurrently. + In this case, ObservedResourceIndex represents the index of the latest resource snapshot installed across all clusters. + Note that this may differ from the index of the latest resource snapshot available in the hub cluster depending on the rollout strategy. ObservedResourceIndex is the resource index that the conditions in the ClusterResourcePlacementStatus observe. For example, a condition of `ClusterResourcePlacementWorkSynchronized` type is observing the synchronization status of the resource snapshot with the resource index $ObservedResourceIndex. @@ -2500,6 +2503,13 @@ spec: type: object maxItems: 100 type: array + observedResourceIndex: + description: |- + ObservedResourceIndex is the index of the resource snapshot that is currently rolled out on the given cluster. + During rollout, depending on the rollout strategy, clusters may observe different resource indices. + ObservedResourceIndex is the resource snapshot index observed by the conditions in the ResourcePlacementStatus. + This field is only meaningful if the `ClusterName` is not empty. + type: string type: object type: array selectedResources: From 80abd226bdce48bd72c01fa4e4635b34f56039bd Mon Sep 17 00:00:00 2001 From: Wantong Jiang Date: Tue, 29 Apr 2025 01:26:59 +0000 Subject: [PATCH 2/5] revise API Signed-off-by: Wantong Jiang --- .../v1beta1/clusterresourceplacement_types.go | 10 +++++----- ...ernetes-fleet.io_clusterresourceplacements.yaml | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/apis/placement/v1beta1/clusterresourceplacement_types.go b/apis/placement/v1beta1/clusterresourceplacement_types.go index 2cefc94f5..36901e441 100644 --- a/apis/placement/v1beta1/clusterresourceplacement_types.go +++ b/apis/placement/v1beta1/clusterresourceplacement_types.go @@ -813,6 +813,7 @@ type RollingUpdateConfig struct { // ClusterResourcePlacementStatus defines the observed state of the ClusterResourcePlacement object. type ClusterResourcePlacementStatus struct { // SelectedResources contains a list of resources selected by ResourceSelectors. + // This field is only meaningful if the `ObservedResourceIndex` is not empty. // +kubebuilder:validation:Optional SelectedResources []ResourceIdentifier `json:"selectedResources,omitempty"` @@ -822,17 +823,16 @@ type ClusterResourcePlacementStatus struct { // One resource snapshot can contain multiple clusterResourceSnapshots CRs in order to store large amount of resources. // To get clusterResourceSnapshot of a given resource index, use the following command: // `kubectl get ClusterResourceSnapshot --selector=kubernetes-fleet.io/resource-index=$ObservedResourceIndex ` - // During rollout, clusters may run different versions of the resource snapshot concurrently. - // In this case, ObservedResourceIndex represents the index of the latest resource snapshot installed across all clusters. - // Note that this may differ from the index of the latest resource snapshot available in the hub cluster depending on the rollout strategy. // ObservedResourceIndex is the resource index that the conditions in the ClusterResourcePlacementStatus observe. // For example, a condition of `ClusterResourcePlacementWorkSynchronized` type // is observing the synchronization status of the resource snapshot with the resource index $ObservedResourceIndex. + // If the rollout strategy type is `RollingUpdate`, `ObservedResourceIndex` is the default-latest resource snapshot index. + // If the rollout strategy type is `External`, rollout and version control are managed by an external controller, and this field remains empty. // +kubebuilder:validation:Optional ObservedResourceIndex string `json:"observedResourceIndex,omitempty"` // PlacementStatuses contains a list of placement status on the clusters that are selected by PlacementPolicy. - // Each selected cluster according to the latest resource placement is guaranteed to have a corresponding placementStatuses. + // Each selected cluster according to the observed resource placement is guaranteed to have a corresponding placementStatuses. // In the pickN case, there are N placement statuses where N = NumberOfClusters; Or in the pickFixed case, there are // N placement statuses where N = ClusterNames. // In these cases, some of them may not have assigned clusters when we cannot fill the required number of clusters. @@ -957,7 +957,7 @@ type ResourcePlacementStatus struct { // +kubebuilder:validation:MaxItems=100 DiffedPlacements []DiffedResourcePlacement `json:"diffedPlacements,omitempty"` - // ObservedResourceIndex is the index of the resource snapshot that is currently rolled out on the given cluster. + // ObservedResourceIndex is the index of the resource snapshot that is currently being rolled out to the given cluster. // During rollout, depending on the rollout strategy, clusters may observe different resource indices. // ObservedResourceIndex is the resource snapshot index observed by the conditions in the ResourcePlacementStatus. // This field is only meaningful if the `ClusterName` is not empty. diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml index ad5877566..b27c82a82 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml @@ -2044,17 +2044,16 @@ spec: One resource snapshot can contain multiple clusterResourceSnapshots CRs in order to store large amount of resources. To get clusterResourceSnapshot of a given resource index, use the following command: `kubectl get ClusterResourceSnapshot --selector=kubernetes-fleet.io/resource-index=$ObservedResourceIndex ` - During rollout, clusters may run different versions of the resource snapshot concurrently. - In this case, ObservedResourceIndex represents the index of the latest resource snapshot installed across all clusters. - Note that this may differ from the index of the latest resource snapshot available in the hub cluster depending on the rollout strategy. ObservedResourceIndex is the resource index that the conditions in the ClusterResourcePlacementStatus observe. For example, a condition of `ClusterResourcePlacementWorkSynchronized` type is observing the synchronization status of the resource snapshot with the resource index $ObservedResourceIndex. + If the rollout strategy type is `RollingUpdate`, `ObservedResourceIndex` is the default-latest resource snapshot index. + If the rollout strategy type is `External`, rollout and version control are managed by an external controller, and this field remains empty. type: string placementStatuses: description: |- PlacementStatuses contains a list of placement status on the clusters that are selected by PlacementPolicy. - Each selected cluster according to the latest resource placement is guaranteed to have a corresponding placementStatuses. + Each selected cluster according to the observed resource placement is guaranteed to have a corresponding placementStatuses. In the pickN case, there are N placement statuses where N = NumberOfClusters; Or in the pickFixed case, there are N placement statuses where N = ClusterNames. In these cases, some of them may not have assigned clusters when we cannot fill the required number of clusters. @@ -2505,7 +2504,7 @@ spec: type: array observedResourceIndex: description: |- - ObservedResourceIndex is the index of the resource snapshot that is currently rolled out on the given cluster. + ObservedResourceIndex is the index of the resource snapshot that is currently being rolled out to the given cluster. During rollout, depending on the rollout strategy, clusters may observe different resource indices. ObservedResourceIndex is the resource snapshot index observed by the conditions in the ResourcePlacementStatus. This field is only meaningful if the `ClusterName` is not empty. @@ -2513,8 +2512,9 @@ spec: type: object type: array selectedResources: - description: SelectedResources contains a list of resources selected - by ResourceSelectors. + description: |- + SelectedResources contains a list of resources selected by ResourceSelectors. + This field is only meaningful if the `ObservedResourceIndex` is not empty. items: description: ResourceIdentifier identifies one Kubernetes resource. properties: From 1b8449c383673b98a14552944ec10ef1d1d1b355 Mon Sep 17 00:00:00 2001 From: Wantong Jiang Date: Tue, 29 Apr 2025 18:43:11 +0000 Subject: [PATCH 3/5] minor fix Signed-off-by: Wantong Jiang --- apis/placement/v1beta1/clusterresourceplacement_types.go | 2 +- ...placement.kubernetes-fleet.io_clusterresourceplacements.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apis/placement/v1beta1/clusterresourceplacement_types.go b/apis/placement/v1beta1/clusterresourceplacement_types.go index 36901e441..2c9893952 100644 --- a/apis/placement/v1beta1/clusterresourceplacement_types.go +++ b/apis/placement/v1beta1/clusterresourceplacement_types.go @@ -958,7 +958,7 @@ type ResourcePlacementStatus struct { DiffedPlacements []DiffedResourcePlacement `json:"diffedPlacements,omitempty"` // ObservedResourceIndex is the index of the resource snapshot that is currently being rolled out to the given cluster. - // During rollout, depending on the rollout strategy, clusters may observe different resource indices. + // During rollout, depending on the rollout strategy, clusters may observe different resource snapshot indices. // ObservedResourceIndex is the resource snapshot index observed by the conditions in the ResourcePlacementStatus. // This field is only meaningful if the `ClusterName` is not empty. // +kubebuilder:validation:Optional diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml index b27c82a82..fed16bc9c 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml @@ -2505,7 +2505,7 @@ spec: observedResourceIndex: description: |- ObservedResourceIndex is the index of the resource snapshot that is currently being rolled out to the given cluster. - During rollout, depending on the rollout strategy, clusters may observe different resource indices. + During rollout, depending on the rollout strategy, clusters may observe different resource snapshot indices. ObservedResourceIndex is the resource snapshot index observed by the conditions in the ResourcePlacementStatus. This field is only meaningful if the `ClusterName` is not empty. type: string From 1890ad9c4874ad132607f922ed0f8f5944c2926f Mon Sep 17 00:00:00 2001 From: Wantong Jiang Date: Wed, 30 Apr 2025 01:07:43 +0000 Subject: [PATCH 4/5] revise Signed-off-by: Wantong Jiang --- .../v1beta1/clusterresourceplacement_types.go | 19 ++++++++------ ...es-fleet.io_clusterresourceplacements.yaml | 25 +++++++++++-------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/apis/placement/v1beta1/clusterresourceplacement_types.go b/apis/placement/v1beta1/clusterresourceplacement_types.go index 2c9893952..50525f737 100644 --- a/apis/placement/v1beta1/clusterresourceplacement_types.go +++ b/apis/placement/v1beta1/clusterresourceplacement_types.go @@ -822,12 +822,10 @@ type ClusterResourcePlacementStatus struct { // Each snapshot has a different resource index. // One resource snapshot can contain multiple clusterResourceSnapshots CRs in order to store large amount of resources. // To get clusterResourceSnapshot of a given resource index, use the following command: - // `kubectl get ClusterResourceSnapshot --selector=kubernetes-fleet.io/resource-index=$ObservedResourceIndex ` - // ObservedResourceIndex is the resource index that the conditions in the ClusterResourcePlacementStatus observe. - // For example, a condition of `ClusterResourcePlacementWorkSynchronized` type - // is observing the synchronization status of the resource snapshot with the resource index $ObservedResourceIndex. + // `kubectl get ClusterResourceSnapshot --selector=kubernetes-fleet.io/resource-index=$ObservedResourceIndex` // If the rollout strategy type is `RollingUpdate`, `ObservedResourceIndex` is the default-latest resource snapshot index. - // If the rollout strategy type is `External`, rollout and version control are managed by an external controller, and this field remains empty. + // If the rollout strategy type is `External`, rollout and version control are managed by an external controller, + // and this field is not empty only if all targeted clusters observe the same resource index in `PlacementStatuses`. // +kubebuilder:validation:Optional ObservedResourceIndex string `json:"observedResourceIndex,omitempty"` @@ -846,6 +844,11 @@ type ClusterResourcePlacementStatus struct { // +listMapKey=type // Conditions is an array of current observed conditions for ClusterResourcePlacement. + // All conditions except `ClusterResourcePlacementScheduled` correspond to the resource snapshot at the index specified by `ObservedResourceIndex`. + // For example, a condition of `ClusterResourcePlacementWorkSynchronized` type + // is observing the synchronization status of the resource snapshot with index `ObservedResourceIndex`. + // If the rollout strategy type is `External`, and `ObservedResourceIndex` is unset due to clusters reporting different resource indices, + // conditions except `ClusterResourcePlacementScheduled` will be empty or set to Unknown. // +kubebuilder:validation:Optional Conditions []metav1.Condition `json:"conditions,omitempty"` } @@ -958,13 +961,13 @@ type ResourcePlacementStatus struct { DiffedPlacements []DiffedResourcePlacement `json:"diffedPlacements,omitempty"` // ObservedResourceIndex is the index of the resource snapshot that is currently being rolled out to the given cluster. - // During rollout, depending on the rollout strategy, clusters may observe different resource snapshot indices. - // ObservedResourceIndex is the resource snapshot index observed by the conditions in the ResourcePlacementStatus. // This field is only meaningful if the `ClusterName` is not empty. // +kubebuilder:validation:Optional ObservedResourceIndex string `json:"observedResourceIndex,omitempty"` - // Conditions is an array of current observed conditions for ResourcePlacementStatus. + // Conditions is an array of current observed conditions on the cluster. + // Each condition corresponds to the resource snapshot at the index specified by `ObservedResourceIndex`. + // For example, the condition of type `RolloutStarted` is observing the rollout status of the resource snapshot with index `ObservedResourceIndex`. // +kubebuilder:validation:Optional Conditions []metav1.Condition `json:"conditions,omitempty"` } diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml index fed16bc9c..a8a0933f9 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml @@ -1976,8 +1976,13 @@ spec: description: The observed status of ClusterResourcePlacement. properties: conditions: - description: Conditions is an array of current observed conditions - for ClusterResourcePlacement. + description: |- + Conditions is an array of current observed conditions for ClusterResourcePlacement. + All conditions except `ClusterResourcePlacementScheduled` correspond to the resource snapshot at the index specified by `ObservedResourceIndex`. + For example, a condition of `ClusterResourcePlacementWorkSynchronized` type + is observing the synchronization status of the resource snapshot with index `ObservedResourceIndex`. + If the rollout strategy type is `External`, and `ObservedResourceIndex` is unset due to clusters reporting different resource indices, + conditions except `ClusterResourcePlacementScheduled` will be empty or set to Unknown. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -2043,12 +2048,10 @@ spec: Each snapshot has a different resource index. One resource snapshot can contain multiple clusterResourceSnapshots CRs in order to store large amount of resources. To get clusterResourceSnapshot of a given resource index, use the following command: - `kubectl get ClusterResourceSnapshot --selector=kubernetes-fleet.io/resource-index=$ObservedResourceIndex ` - ObservedResourceIndex is the resource index that the conditions in the ClusterResourcePlacementStatus observe. - For example, a condition of `ClusterResourcePlacementWorkSynchronized` type - is observing the synchronization status of the resource snapshot with the resource index $ObservedResourceIndex. + `kubectl get ClusterResourceSnapshot --selector=kubernetes-fleet.io/resource-index=$ObservedResourceIndex` If the rollout strategy type is `RollingUpdate`, `ObservedResourceIndex` is the default-latest resource snapshot index. - If the rollout strategy type is `External`, rollout and version control are managed by an external controller, and this field remains empty. + If the rollout strategy type is `External`, rollout and version control are managed by an external controller, + and this field is not empty only if all targeted clusters observe the same resource index in `PlacementStatuses`. type: string placementStatuses: description: |- @@ -2099,8 +2102,10 @@ spec: If it is not empty, its value should be unique cross all placement decisions for the Placement. type: string conditions: - description: Conditions is an array of current observed conditions - for ResourcePlacementStatus. + description: |- + Conditions is an array of current observed conditions on the cluster. + Each condition corresponds to the resource snapshot at the index specified by `ObservedResourceIndex`. + For example, the condition of type `RolloutStarted` is observing the rollout status of the resource snapshot with index `ObservedResourceIndex`. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -2505,8 +2510,6 @@ spec: observedResourceIndex: description: |- ObservedResourceIndex is the index of the resource snapshot that is currently being rolled out to the given cluster. - During rollout, depending on the rollout strategy, clusters may observe different resource snapshot indices. - ObservedResourceIndex is the resource snapshot index observed by the conditions in the ResourcePlacementStatus. This field is only meaningful if the `ClusterName` is not empty. type: string type: object From 48993ecec314a0d6c25110dcec25104b0ebac3bf Mon Sep 17 00:00:00 2001 From: Wantong Jiang Date: Wed, 30 Apr 2025 17:34:50 +0000 Subject: [PATCH 5/5] fix comments Signed-off-by: Wantong Jiang --- .../v1beta1/clusterresourceplacement_types.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apis/placement/v1beta1/clusterresourceplacement_types.go b/apis/placement/v1beta1/clusterresourceplacement_types.go index 50525f737..896c90ddd 100644 --- a/apis/placement/v1beta1/clusterresourceplacement_types.go +++ b/apis/placement/v1beta1/clusterresourceplacement_types.go @@ -913,6 +913,11 @@ type ResourcePlacementStatus struct { // +kubebuilder:validation:Optional ClusterName string `json:"clusterName,omitempty"` + // ObservedResourceIndex is the index of the resource snapshot that is currently being rolled out to the given cluster. + // This field is only meaningful if the `ClusterName` is not empty. + // +kubebuilder:validation:Optional + ObservedResourceIndex string `json:"observedResourceIndex,omitempty"` + // ApplicableResourceOverrides contains a list of applicable ResourceOverride snapshots associated with the selected // resources. // @@ -960,11 +965,6 @@ type ResourcePlacementStatus struct { // +kubebuilder:validation:MaxItems=100 DiffedPlacements []DiffedResourcePlacement `json:"diffedPlacements,omitempty"` - // ObservedResourceIndex is the index of the resource snapshot that is currently being rolled out to the given cluster. - // This field is only meaningful if the `ClusterName` is not empty. - // +kubebuilder:validation:Optional - ObservedResourceIndex string `json:"observedResourceIndex,omitempty"` - // Conditions is an array of current observed conditions on the cluster. // Each condition corresponds to the resource snapshot at the index specified by `ObservedResourceIndex`. // For example, the condition of type `RolloutStarted` is observing the rollout status of the resource snapshot with index `ObservedResourceIndex`.