feat: make hub statefulset work by stripping some properties from generated PVCs

pkg/controllers/placement/resource_selector.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	pvutil "k8s.io/component-helpers/storage/volume"
 	"k8s.io/klog/v2"
 	"k8s.io/kubectl/pkg/util/deployment"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -444,6 +445,18 @@ func generateRawContent(object *unstructured.Unstructured) ([]byte, error) {
 		delete(annots, corev1.LastAppliedConfigAnnotation)
 		// Remove the revision annotation set by deployment controller.
 		delete(annots, deployment.RevisionAnnotation)
+		// Remove node-specific and provisioning-related annotations from PVCs that would break when propagated to member clusters
+		// These annotations reference specific nodes/provisioners from the hub cluster which don't exist on member clusters
+		// The member cluster's storage provisioner will set appropriate values for its own environment
+		// All annotations below are listed in well-known labels, annotations and taints document:
+		// https://kubernetes.io/docs/reference/labels-annotations-taints/
+		delete(annots, pvutil.AnnSelectedNode)           // Node selected for volume binding
+		delete(annots, pvutil.AnnBindCompleted)          // Binding completion status
+		delete(annots, pvutil.AnnBoundByController)      // Controller binding status
+		delete(annots, pvutil.AnnStorageProvisioner)     // Storage provisioner annotation
+		delete(annots, pvutil.AnnBetaStorageProvisioner) // Beta storage provisioner annotation
+		delete(annots, pvutil.AnnDynamicallyProvisioned) // Dynamically provisioned by annotation
+		delete(annots, pvutil.AnnMigratedTo)             // CSI migration annotation
 		if len(annots) == 0 {
 			object.SetAnnotations(nil)
 		} else {
@@ -491,6 +504,10 @@ func generateRawContent(object *unstructured.Unstructured) ([]byte, error) {
 			unstructured.RemoveNestedField(object.Object, "spec", "template", "metadata", "labels", "controller-uid")
 			unstructured.RemoveNestedField(object.Object, "spec", "template", "metadata", "labels", "batch.kubernetes.io/controller-uid")
 		}
+	} else if object.GetKind() == "PersistentVolumeClaim" && object.GetAPIVersion() == "v1" {
+		// Remove volumeName which references a specific PV from the hub cluster that won't exist on member clusters.
+		// The member cluster's storage provisioner will create and bind a new PV.
+		unstructured.RemoveNestedField(object.Object, "spec", "volumeName")
 	}
 
 	rawContent, err := object.MarshalJSON()

pkg/controllers/placement/resource_selector_test.go

@@ -27,12 +27,14 @@ import (
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	utilrand "k8s.io/apimachinery/pkg/util/rand"
+	pvutil "k8s.io/component-helpers/storage/volume"
 	"k8s.io/kubectl/pkg/util/deployment"
 	"k8s.io/utils/ptr"
 
@@ -243,6 +245,65 @@ func TestGenerateResourceContent(t *testing.T) {
 				},
 			},
 		},
+		"PersistentVolumeClaim with node-specific annotations": {
+			resource: corev1.PersistentVolumeClaim{
+				TypeMeta: metav1.TypeMeta{
+					APIVersion: "v1",
+					Kind:       "PersistentVolumeClaim",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-pvc",
+					Namespace: "test-namespace",
+					Annotations: map[string]string{
+						pvutil.AnnSelectedNode:           "hub-control-plane",
+						pvutil.AnnBindCompleted:          "yes",
+						pvutil.AnnBoundByController:      "yes",
+						pvutil.AnnStorageProvisioner:     "kubernetes.io/aws-ebs",
+						pvutil.AnnBetaStorageProvisioner: "kubernetes.io/no-provisioner",
+						pvutil.AnnDynamicallyProvisioned: "kubernetes.io/aws-ebs",
+						pvutil.AnnMigratedTo:             "ebs.csi.aws.com",
+						"custom-annotation":              "should-remain",
+					},
+				},
+				Spec: corev1.PersistentVolumeClaimSpec{
+					AccessModes: []corev1.PersistentVolumeAccessMode{
+						corev1.ReadWriteOnce,
+					},
+					Resources: corev1.VolumeResourceRequirements{
+						Requests: corev1.ResourceList{
+							corev1.ResourceStorage: resource.MustParse("1Gi"),
+						},
+					},
+					StorageClassName: ptr.To("standard"),
+					VolumeName:       "pvc-12345-from-hub-cluster",
+				},
+			},
+			wantResource: corev1.PersistentVolumeClaim{
+				TypeMeta: metav1.TypeMeta{
+					APIVersion: "v1",
+					Kind:       "PersistentVolumeClaim",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-pvc",
+					Namespace: "test-namespace",
+					Annotations: map[string]string{
+						"custom-annotation": "should-remain",
+					},
+				},
+				Spec: corev1.PersistentVolumeClaimSpec{
+					AccessModes: []corev1.PersistentVolumeAccessMode{
+						corev1.ReadWriteOnce,
+					},
+					Resources: corev1.VolumeResourceRequirements{
+						Requests: corev1.ResourceList{
+							corev1.ResourceStorage: resource.MustParse("1Gi"),
+						},
+					},
+					StorageClassName: ptr.To("standard"),
+					// VolumeName should be removed
+				},
+			},
+		},
 	}
 
 	for testName, tt := range tests {

pkg/controllers/workapplier/availability_tracker.go

@@ -104,6 +104,8 @@ func trackInMemberClusterObjAvailabilityByGVR(
 		return trackCRDAvailability(inMemberClusterObj)
 	case utils.PodDisruptionBudgetGVR:
 		return trackPDBAvailability(inMemberClusterObj)
+	case utils.PersistentVolumeClaimGVR:
+		return trackPVCAvailability(inMemberClusterObj)
 	default:
 		if isDataResource(*gvr) {
 			klog.V(2).InfoS("The object from the member cluster is a data object, consider it to be immediately available",
@@ -269,6 +271,28 @@ func trackPDBAvailability(curObj *unstructured.Unstructured) (ManifestProcessing
 	return AvailabilityResultTypeNotYetAvailable, nil
 }
 
+// trackPVCAvailability tracks the availability of a persistent volume claim in the member cluster.
+func trackPVCAvailability(inMemberClusterObj *unstructured.Unstructured) (ManifestProcessingAvailabilityResultType, error) {
+	var pvc corev1.PersistentVolumeClaim
+	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(inMemberClusterObj.Object, &pvc); err != nil {
+		wrappedErr := fmt.Errorf("failed to convert the unstructured object to a persistent volume claim: %w", err)
+		_ = controller.NewUnexpectedBehaviorError(wrappedErr)
+		return AvailabilityResultTypeFailed, wrappedErr
+	}
+
+	// Check if the PVC is bound.
+	// A PVC is considered available when it's in the Bound phase, meaning it has been
+	// successfully bound to a PersistentVolume and is ready to be used by pods.
+	if pvc.Status.Phase == corev1.ClaimBound {
+		klog.V(2).InfoS("PersistentVolumeClaim is available", "pvc", klog.KObj(inMemberClusterObj))
+		return AvailabilityResultTypeAvailable, nil
+	}
+
+	klog.V(2).InfoS("PersistentVolumeClaim is not ready yet, will check later to see if it becomes available",
+		"pvc", klog.KObj(inMemberClusterObj), "phase", pvc.Status.Phase)
+	return AvailabilityResultTypeNotYetAvailable, nil
+}
+
 // isDataResource checks if the resource is a data resource; such resources are
 // available immediately after creation.
 func isDataResource(gvr schema.GroupVersionResource) bool {

pkg/controllers/workapplier/availability_tracker_test.go

@@ -802,6 +802,100 @@ func TestTrackPDBAvailability(t *testing.T) {
 	}
 }
 
+// TestTrackPVCAvailability tests the trackPVCAvailability function.
+func TestTrackPVCAvailability(t *testing.T) {
+	boundPVC := &corev1.PersistentVolumeClaim{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "PersistentVolumeClaim",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-pvc-bound",
+			Namespace: nsName,
+		},
+		Spec: corev1.PersistentVolumeClaimSpec{
+			AccessModes: []corev1.PersistentVolumeAccessMode{
+				corev1.ReadWriteOnce,
+			},
+		},
+		Status: corev1.PersistentVolumeClaimStatus{
+			Phase: corev1.ClaimBound,
+		},
+	}
+
+	pendingPVC := &corev1.PersistentVolumeClaim{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "PersistentVolumeClaim",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-pvc-pending",
+			Namespace: nsName,
+		},
+		Spec: corev1.PersistentVolumeClaimSpec{
+			AccessModes: []corev1.PersistentVolumeAccessMode{
+				corev1.ReadWriteOnce,
+			},
+		},
+		Status: corev1.PersistentVolumeClaimStatus{
+			Phase: corev1.ClaimPending,
+		},
+	}
+
+	lostPVC := &corev1.PersistentVolumeClaim{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "PersistentVolumeClaim",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-pvc-lost",
+			Namespace: nsName,
+		},
+		Spec: corev1.PersistentVolumeClaimSpec{
+			AccessModes: []corev1.PersistentVolumeAccessMode{
+				corev1.ReadWriteOnce,
+			},
+		},
+		Status: corev1.PersistentVolumeClaimStatus{
+			Phase: corev1.ClaimLost,
+		},
+	}
+
+	testCases := []struct {
+		name                       string
+		pvc                        *corev1.PersistentVolumeClaim
+		wantAvailabilityResultType ManifestProcessingAvailabilityResultType
+	}{
+		{
+			name:                       "available PVC (bound)",
+			pvc:                        boundPVC,
+			wantAvailabilityResultType: AvailabilityResultTypeAvailable,
+		},
+		{
+			name:                       "unavailable PVC (pending)",
+			pvc:                        pendingPVC,
+			wantAvailabilityResultType: AvailabilityResultTypeNotYetAvailable,
+		},
+		{
+			name:                       "unavailable PVC (lost)",
+			pvc:                        lostPVC,
+			wantAvailabilityResultType: AvailabilityResultTypeNotYetAvailable,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			gotResTyp, err := trackPVCAvailability(toUnstructured(t, tc.pvc))
+			if err != nil {
+				t.Fatalf("trackPVCAvailability() = %v, want no error", err)
+			}
+			if gotResTyp != tc.wantAvailabilityResultType {
+				t.Errorf("manifestProcessingAvailabilityResultType = %v, want %v", gotResTyp, tc.wantAvailabilityResultType)
+			}
+		})
+	}
+}
+
 // TestTrackInMemberClusterObjAvailabilityByGVR tests the trackInMemberClusterObjAvailabilityByGVR function.
 func TestTrackInMemberClusterObjAvailabilityByGVR(t *testing.T) {
 	availableDeploy := deploy.DeepCopy()
@@ -875,6 +969,34 @@ func TestTrackInMemberClusterObjAvailabilityByGVR(t *testing.T) {
 		},
 	}
 
+	availablePVC := &corev1.PersistentVolumeClaim{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "PersistentVolumeClaim",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-pvc",
+			Namespace: nsName,
+		},
+		Status: corev1.PersistentVolumeClaimStatus{
+			Phase: corev1.ClaimBound,
+		},
+	}
+
+	unavailablePVC := &corev1.PersistentVolumeClaim{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "PersistentVolumeClaim",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-pvc-pending",
+			Namespace: nsName,
+		},
+		Status: corev1.PersistentVolumeClaimStatus{
+			Phase: corev1.ClaimPending,
+		},
+	}
+
 	testCases := []struct {
 		name                       string
 		gvr                        schema.GroupVersionResource
@@ -995,6 +1117,18 @@ func TestTrackInMemberClusterObjAvailabilityByGVR(t *testing.T) {
 			inMemberClusterObj:         toUnstructured(t, &schedulingv1.PriorityClass{}),
 			wantAvailabilityResultType: AvailabilityResultTypeAvailable,
 		},
+		{
+			name:                       "available persistent volume claim (bound)",
+			gvr:                        utils.PersistentVolumeClaimGVR,
+			inMemberClusterObj:         toUnstructured(t, availablePVC),
+			wantAvailabilityResultType: AvailabilityResultTypeAvailable,
+		},
+		{
+			name:                       "unavailable persistent volume claim (pending)",
+			gvr:                        utils.PersistentVolumeClaimGVR,
+			inMemberClusterObj:         toUnstructured(t, unavailablePVC),
+			wantAvailabilityResultType: AvailabilityResultTypeNotYetAvailable,
+		},
 	}
 
 	for _, tc := range testCases {

pkg/utils/common.go

@@ -438,6 +438,12 @@ var (
 		Resource: "clusterrolebindings",
 	}
 
+	PersistentVolumeClaimGVR = schema.GroupVersionResource{
+		Group:    corev1.GroupName,
+		Version:  corev1.SchemeGroupVersion.Version,
+		Resource: "persistentvolumeclaims",
+	}
+
 	PersistentVolumeClaimGVK = schema.GroupVersionKind{
 		Group:   corev1.GroupName,
 		Version: corev1.SchemeGroupVersion.Version,

test/e2e/enveloped_object_placement_test.go

@@ -226,7 +226,7 @@ var _ = Describe("placing wrapped resources using a CRP", func() {
 			// read the test resources.
 			readDeploymentTestManifest(&testDeployment)
 			readDaemonSetTestManifest(&testDaemonSet)
-			readStatefulSetTestManifest(&testStatefulSet, true)
+			readStatefulSetTestManifest(&testStatefulSet, StatefulSetInvalidStorage)
 			readEnvelopeResourceTestManifest(&testResourceEnvelope)
 		})