Skip to content

Commit fbc2483

Browse files
weng271190436Wei Weng
andauthored
feat: allow pod and replica sets to be created in hub cluster (#334)
* remove pod handler Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * remove replicaset webhook as well Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * update guard rail Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * add e2e tests Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * rename Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * do not propagate rs and pod Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * fix unit test Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * add guard rail test for pod and rs Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * only block propagation for replica set and pod Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * Revert "remove pod handler" This reverts commit 562b151. Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * Revert "remove replicaset webhook as well" This reverts commit ee9b425. Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * revert Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * add options to disable rs and pod validating webhooks Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * guard rail pod and replica set if validating webhooks disabled Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * remove some test because validating webhooks are on by default Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * remove unintentional change Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * remove empty line Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * remove one unintentional change and remove owner ref check for other resources Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * fix readme Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * combine into single flag Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * add guard rail test for pod and rs Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * verify workload running in hub Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * rename flag Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * update readme Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * fix tests Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * do not propagate controller revision Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * fix test Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * ignore some pvc annotations Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * enum for stateful set test yaml Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * strip volume name from PVC Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * fix test Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * Revert "ignore some pvc annotations" This reverts commit 9ba69dd. Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * test job Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * address comment Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> * fix unit test Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> --------- Signed-off-by: Wei Weng <Wei.Weng@microsoft.com> Co-authored-by: Wei Weng <Wei.Weng@microsoft.com>
1 parent 2ca743d commit fbc2483

File tree

13 files changed

+800
-272
lines changed

13 files changed

+800
-272
lines changed

charts/hub-agent/README.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,5 @@ _See [helm install](https://helm.sh/docs/helm/helm_install/) for command documen
4040
| `logFileMaxSize` | Max log file size before rotation | `1000000` |
4141
| `MaxFleetSizeSupported` | Max number of member clusters supported | `100` |
4242
| `resourceSnapshotCreationMinimumInterval` | The minimum interval at which resource snapshots could be created. | `30s` |
43-
| `resourceChangesCollectionDuration` | The duration for collecting resource changes into one snapshot. | `15s` |
43+
| `resourceChangesCollectionDuration` | The duration for collecting resource changes into one snapshot. | `15s` |
44+
| `enableWorkload` | Enable kubernetes builtin workload to run in hub cluster. | `false` |

charts/hub-agent/templates/deployment.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ spec:
2424
- --enable-webhook={{ .Values.enableWebhook }}
2525
- --webhook-service-name={{ .Values.webhookServiceName }}
2626
- --enable-guard-rail={{ .Values.enableGuardRail }}
27+
- --enable-workload={{ .Values.enableWorkload }}
2728
- --whitelisted-users=system:serviceaccount:fleet-system:hub-agent-sa
2829
- --webhook-client-connection-type={{.Values.webhookClientConnectionType}}
2930
- --v={{ .Values.logVerbosity }}

charts/hub-agent/values.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ enableWebhook: true
1616
webhookServiceName: fleetwebhook
1717
enableGuardRail: true
1818
webhookClientConnectionType: service
19+
enableWorkload: false
1920
forceDeleteWaitTime: 15m0s
2021
clusterUnhealthyThreshold: 3m0s
2122
resourceSnapshotCreationMinimumInterval: 30s

cmd/hubagent/main.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ func main() {
156156

157157
if opts.EnableWebhook {
158158
whiteListedUsers := strings.Split(opts.WhiteListedUsers, ",")
159-
if err := SetupWebhook(mgr, options.WebhookClientConnectionType(opts.WebhookClientConnectionType), opts.WebhookServiceName, whiteListedUsers, opts.EnableGuardRail, opts.EnableV1Beta1APIs, opts.DenyModifyMemberClusterLabels); err != nil {
159+
if err := SetupWebhook(mgr, options.WebhookClientConnectionType(opts.WebhookClientConnectionType), opts.WebhookServiceName, whiteListedUsers, opts.EnableGuardRail, opts.EnableV1Beta1APIs, opts.DenyModifyMemberClusterLabels, opts.EnableWorkload); err != nil {
160160
klog.ErrorS(err, "unable to set up webhook")
161161
exitWithErrorFunc()
162162
}
@@ -188,9 +188,9 @@ func main() {
188188
}
189189

190190
// SetupWebhook generates the webhook cert and then set up the webhook configurator.
191-
func SetupWebhook(mgr manager.Manager, webhookClientConnectionType options.WebhookClientConnectionType, webhookServiceName string, whiteListedUsers []string, enableGuardRail, isFleetV1Beta1API bool, denyModifyMemberClusterLabels bool) error {
191+
func SetupWebhook(mgr manager.Manager, webhookClientConnectionType options.WebhookClientConnectionType, webhookServiceName string, whiteListedUsers []string, enableGuardRail, isFleetV1Beta1API bool, denyModifyMemberClusterLabels bool, enableWorkload bool) error {
192192
// Generate self-signed key and crt files in FleetWebhookCertDir for the webhook server to start.
193-
w, err := webhook.NewWebhookConfig(mgr, webhookServiceName, FleetWebhookPort, &webhookClientConnectionType, FleetWebhookCertDir, enableGuardRail, denyModifyMemberClusterLabels)
193+
w, err := webhook.NewWebhookConfig(mgr, webhookServiceName, FleetWebhookPort, &webhookClientConnectionType, FleetWebhookCertDir, enableGuardRail, denyModifyMemberClusterLabels, enableWorkload)
194194
if err != nil {
195195
klog.ErrorS(err, "fail to generate WebhookConfig")
196196
return err

cmd/hubagent/options/options.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,9 @@ type Options struct {
107107
PprofPort int
108108
// DenyModifyMemberClusterLabels indicates if the member cluster labels cannot be modified by groups (excluding system:masters)
109109
DenyModifyMemberClusterLabels bool
110+
// EnableWorkload enables workload resources (pods and replicasets) to be created in the hub cluster.
111+
// When set to true, the pod and replicaset validating webhooks are disabled.
112+
EnableWorkload bool
110113
// ResourceSnapshotCreationMinimumInterval is the minimum interval at which resource snapshots could be created.
111114
// Whether the resource snapshot is created or not depends on the both ResourceSnapshotCreationMinimumInterval and ResourceChangesCollectionDuration.
112115
ResourceSnapshotCreationMinimumInterval time.Duration
@@ -181,6 +184,7 @@ func (o *Options) AddFlags(flags *flag.FlagSet) {
181184
flags.BoolVar(&o.EnablePprof, "enable-pprof", false, "If set, the pprof profiling is enabled.")
182185
flags.IntVar(&o.PprofPort, "pprof-port", 6065, "The port for pprof profiling.")
183186
flags.BoolVar(&o.DenyModifyMemberClusterLabels, "deny-modify-member-cluster-labels", false, "If set, users not in the system:masters cannot modify member cluster labels.")
187+
flags.BoolVar(&o.EnableWorkload, "enable-workload", false, "If set, workloads (pods and replicasets) can be created in the hub cluster. This disables the pod and replicaset validating webhooks.")
184188
flags.DurationVar(&o.ResourceSnapshotCreationMinimumInterval, "resource-snapshot-creation-minimum-interval", 30*time.Second, "The minimum interval at which resource snapshots could be created.")
185189
flags.DurationVar(&o.ResourceChangesCollectionDuration, "resource-changes-collection-duration", 15*time.Second,
186190
"The duration for collecting resource changes into one snapshot. The default is 15 seconds, which means that the controller will collect resource changes for 15 seconds before creating a resource snapshot.")

pkg/utils/common.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@ const (
7979
ServiceKind = "Service"
8080
NamespaceKind = "Namespace"
8181
JobKind = "Job"
82+
ReplicaSetKind = "ReplicaSet"
83+
PodKind = "Pod"
8284
)
8385

8486
const (
@@ -507,6 +509,18 @@ func CheckCRDInstalled(discoveryClient discovery.DiscoveryInterface, gvk schema.
507509
func ShouldPropagateObj(informerManager informer.Manager, uObj *unstructured.Unstructured) (bool, error) {
508510
// TODO: add more special handling for different resource kind
509511
switch uObj.GroupVersionKind() {
512+
case appv1.SchemeGroupVersion.WithKind(ReplicaSetKind):
513+
// Skip ReplicaSets if they are managed by Deployments (have owner references)
514+
// Standalone ReplicaSets (without owners) can be propagated
515+
if len(uObj.GetOwnerReferences()) > 0 {
516+
return false, nil
517+
}
518+
case appv1.SchemeGroupVersion.WithKind("ControllerRevision"):
519+
// Skip ControllerRevisions if they are managed by DaemonSets/StatefulSets (have owner references)
520+
// These are automatically created by controllers and will be recreated on member clusters
521+
if len(uObj.GetOwnerReferences()) > 0 {
522+
return false, nil
523+
}
510524
case corev1.SchemeGroupVersion.WithKind(ConfigMapKind):
511525
// Skip the built-in custom CA certificate created in the namespace
512526
if uObj.GetName() == "kube-root-ca.crt" {

pkg/utils/common_test.go

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66

77
"github.com/google/go-cmp/cmp"
88
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
9+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
910
"k8s.io/utils/ptr"
1011

1112
fleetv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1"
@@ -1189,3 +1190,150 @@ func TestIsDiffedResourcePlacementEqual(t *testing.T) {
11891190
})
11901191
}
11911192
}
1193+
1194+
func TestShouldPropagateObj_PodAndReplicaSet(t *testing.T) {
1195+
tests := []struct {
1196+
name string
1197+
obj map[string]interface{}
1198+
ownerReferences []metav1.OwnerReference
1199+
want bool
1200+
}{
1201+
{
1202+
name: "standalone replicaset without ownerReferences should propagate",
1203+
obj: map[string]interface{}{
1204+
"apiVersion": "apps/v1",
1205+
"kind": "ReplicaSet",
1206+
"metadata": map[string]interface{}{
1207+
"name": "standalone-rs",
1208+
"namespace": "default",
1209+
},
1210+
},
1211+
ownerReferences: nil,
1212+
want: true,
1213+
},
1214+
{
1215+
name: "standalone pod without ownerReferences should propagate",
1216+
obj: map[string]interface{}{
1217+
"apiVersion": "v1",
1218+
"kind": "Pod",
1219+
"metadata": map[string]interface{}{
1220+
"name": "standalone-pod",
1221+
"namespace": "default",
1222+
},
1223+
},
1224+
ownerReferences: nil,
1225+
want: true,
1226+
},
1227+
{
1228+
name: "replicaset with deployment owner should NOT propagate",
1229+
obj: map[string]interface{}{
1230+
"apiVersion": "apps/v1",
1231+
"kind": "ReplicaSet",
1232+
"metadata": map[string]interface{}{
1233+
"name": "test-deploy-abc123",
1234+
"namespace": "default",
1235+
},
1236+
},
1237+
ownerReferences: []metav1.OwnerReference{
1238+
{
1239+
APIVersion: "apps/v1",
1240+
Kind: "Deployment",
1241+
Name: "test-deploy",
1242+
UID: "12345",
1243+
},
1244+
},
1245+
want: false,
1246+
},
1247+
{
1248+
name: "pod owned by replicaset - passes ShouldPropagateObj but filtered by resource config",
1249+
obj: map[string]interface{}{
1250+
"apiVersion": "v1",
1251+
"kind": "Pod",
1252+
"metadata": map[string]interface{}{
1253+
"name": "test-deploy-abc123-xyz",
1254+
"namespace": "default",
1255+
},
1256+
},
1257+
ownerReferences: []metav1.OwnerReference{
1258+
{
1259+
APIVersion: "apps/v1",
1260+
Kind: "ReplicaSet",
1261+
Name: "test-deploy-abc123",
1262+
UID: "67890",
1263+
},
1264+
},
1265+
want: true, // ShouldPropagateObj doesn't filter Pods - they're filtered by NewResourceConfig
1266+
},
1267+
{
1268+
name: "controllerrevision owned by daemonset should NOT propagate",
1269+
obj: map[string]interface{}{
1270+
"apiVersion": "apps/v1",
1271+
"kind": "ControllerRevision",
1272+
"metadata": map[string]interface{}{
1273+
"name": "test-ds-7b9848797f",
1274+
"namespace": "default",
1275+
},
1276+
},
1277+
ownerReferences: []metav1.OwnerReference{
1278+
{
1279+
APIVersion: "apps/v1",
1280+
Kind: "DaemonSet",
1281+
Name: "test-ds",
1282+
UID: "abcdef",
1283+
},
1284+
},
1285+
want: false,
1286+
},
1287+
{
1288+
name: "controllerrevision owned by statefulset should NOT propagate",
1289+
obj: map[string]interface{}{
1290+
"apiVersion": "apps/v1",
1291+
"kind": "ControllerRevision",
1292+
"metadata": map[string]interface{}{
1293+
"name": "test-ss-7878b4b446",
1294+
"namespace": "default",
1295+
},
1296+
},
1297+
ownerReferences: []metav1.OwnerReference{
1298+
{
1299+
APIVersion: "apps/v1",
1300+
Kind: "StatefulSet",
1301+
Name: "test-ss",
1302+
UID: "fedcba",
1303+
},
1304+
},
1305+
want: false,
1306+
},
1307+
{
1308+
name: "standalone controllerrevision without owner should propagate",
1309+
obj: map[string]interface{}{
1310+
"apiVersion": "apps/v1",
1311+
"kind": "ControllerRevision",
1312+
"metadata": map[string]interface{}{
1313+
"name": "custom-revision",
1314+
"namespace": "default",
1315+
},
1316+
},
1317+
ownerReferences: nil,
1318+
want: true,
1319+
},
1320+
}
1321+
1322+
for _, tt := range tests {
1323+
t.Run(tt.name, func(t *testing.T) {
1324+
uObj := &unstructured.Unstructured{Object: tt.obj}
1325+
if tt.ownerReferences != nil {
1326+
uObj.SetOwnerReferences(tt.ownerReferences)
1327+
}
1328+
1329+
got, err := ShouldPropagateObj(nil, uObj)
1330+
if err != nil {
1331+
t.Errorf("ShouldPropagateObj() error = %v", err)
1332+
return
1333+
}
1334+
if got != tt.want {
1335+
t.Errorf("ShouldPropagateObj() = %v, want %v", got, tt.want)
1336+
}
1337+
})
1338+
}
1339+
}

0 commit comments

Comments
 (0)