Skip to content

Commit de31cb0

Browse files
britaniarbritaniar
authored
fix flaky test & refactor (#24)
1 parent f27029f commit de31cb0

File tree

1 file changed

+158
-96
lines changed

1 file changed

+158
-96
lines changed

‎pkg/controllers/clusterresourceplacement/controller_integration_test.go‎

Lines changed: 158 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -226,8 +226,19 @@ func createSynchronizedClusterResourceBinding(cluster string, policySnapshot *pl
226226
return binding
227227
}
228228

229-
func createAvailableClusterResourceBinding(cluster string, policySnapshot *placementv1beta1.ClusterSchedulingPolicySnapshot, resourceSnapshot *placementv1beta1.ClusterResourceSnapshot) *placementv1beta1.ClusterResourceBinding {
230-
binding := createSynchronizedClusterResourceBinding(cluster, policySnapshot, resourceSnapshot)
229+
func updateClusterResourceBindingWithSynchronized(binding *placementv1beta1.ClusterResourceBinding) *placementv1beta1.ClusterResourceBinding {
230+
cond := metav1.Condition{
231+
Status: metav1.ConditionTrue,
232+
Type: string(placementv1beta1.ResourceBindingWorkSynchronized),
233+
Reason: condition.WorkSynchronizedReason,
234+
ObservedGeneration: binding.Generation,
235+
}
236+
meta.SetStatusCondition(&binding.Status.Conditions, cond)
237+
Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "Failed to update the binding status")
238+
return binding
239+
}
240+
241+
func updateClusterResourceBindingWithAvailable(binding *placementv1beta1.ClusterResourceBinding) *placementv1beta1.ClusterResourceBinding {
231242
cond := metav1.Condition{
232243
Status: metav1.ConditionTrue,
233244
Type: string(placementv1beta1.ResourceBindingApplied),
@@ -821,7 +832,7 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() {
821832
}
822833
gotCRP = retrieveAndValidateClusterResourcePlacement(testCRPName, wantCRP)
823834

824-
By("Ensure placement status metric was emitted")
835+
By("Ensure placement status metric was emitted for 1st generation")
825836
wantMetrics := []*prometheusclientmodel.Metric{
826837
{
827838
Label: []*prometheusclientmodel.LabelPair{
@@ -851,6 +862,7 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() {
851862
By("Create a synchronized clusterResourceBinding on member-2")
852863
member2Binding = createSynchronizedClusterResourceBinding(member2Name, gotPolicySnapshot, gotResourceSnapshot)
853864

865+
By("Validate the CRP status with updated binding")
854866
wantCRP.Status.Conditions = []metav1.Condition{
855867
{
856868
Status: metav1.ConditionTrue,
@@ -932,7 +944,7 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() {
932944
}
933945
gotCRP = retrieveAndValidateClusterResourcePlacement(testCRPName, wantCRP)
934946

935-
By("Ensure placement status metric was emitted")
947+
By("Ensure placement status metric was emitted for 1st generation")
936948
wantMetrics = append(wantMetrics, &prometheusclientmodel.Metric{
937949
Label: []*prometheusclientmodel.LabelPair{
938950
{Name: ptr.To("name"), Value: ptr.To(gotCRP.Name)},
@@ -1131,12 +1143,12 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() {
11311143
updateClusterSchedulingPolicySnapshotStatus(metav1.ConditionTrue, true)
11321144

11331145
By("Create a synchronized clusterResourceBinding on member-1")
1134-
member1Binding = createAvailableClusterResourceBinding(member1Name, gotPolicySnapshot, gotResourceSnapshot)
1146+
member1Binding = createSynchronizedClusterResourceBinding(member1Name, gotPolicySnapshot, gotResourceSnapshot)
11351147

1136-
By("Create synchronized clusterResourceBinding on member-2")
1137-
member2Binding = createAvailableClusterResourceBinding(member2Name, gotPolicySnapshot, gotResourceSnapshot)
1148+
By("Create an overridden clusterResourceBinding on member-2")
1149+
member2Binding = createOverriddenClusterResourceBinding(member2Name, gotPolicySnapshot, gotResourceSnapshot)
11381150

1139-
By("Validate the CRP status is Available")
1151+
By("Validate the CRP status")
11401152
wantCRP := &placementv1beta1.ClusterResourcePlacement{
11411153
ObjectMeta: metav1.ObjectMeta{
11421154
Name: testCRPName,
@@ -1162,19 +1174,9 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() {
11621174
Reason: condition.OverrideNotSpecifiedReason,
11631175
},
11641176
{
1165-
Status: metav1.ConditionTrue,
1177+
Status: metav1.ConditionUnknown,
11661178
Type: string(placementv1beta1.ClusterResourcePlacementWorkSynchronizedConditionType),
1167-
Reason: condition.WorkSynchronizedReason,
1168-
},
1169-
{
1170-
Status: metav1.ConditionTrue,
1171-
Type: string(placementv1beta1.ClusterResourcePlacementAppliedConditionType),
1172-
Reason: condition.ApplySucceededReason,
1173-
},
1174-
{
1175-
Status: metav1.ConditionTrue,
1176-
Type: string(placementv1beta1.ClusterResourcePlacementAvailableConditionType),
1177-
Reason: condition.AvailableReason,
1179+
Reason: condition.WorkSynchronizedUnknownReason,
11781180
},
11791181
},
11801182
PlacementStatuses: []placementv1beta1.ResourcePlacementStatus{
@@ -1202,14 +1204,9 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() {
12021204
Reason: condition.WorkSynchronizedReason,
12031205
},
12041206
{
1205-
Status: metav1.ConditionTrue,
1206-
Type: string(placementv1beta1.ResourcesAppliedConditionType),
1207-
Reason: condition.ApplySucceededReason,
1208-
},
1209-
{
1210-
Status: metav1.ConditionTrue,
1211-
Type: string(placementv1beta1.ResourcesAvailableConditionType),
1212-
Reason: condition.AvailableReason,
1207+
Status: metav1.ConditionUnknown,
1208+
Type: string(placementv1beta1.ResourceBindingApplied),
1209+
Reason: condition.ApplyPendingReason,
12131210
},
12141211
},
12151212
},
@@ -1232,19 +1229,9 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() {
12321229
Reason: condition.OverriddenSucceededReason,
12331230
},
12341231
{
1235-
Status: metav1.ConditionTrue,
1232+
Status: metav1.ConditionUnknown,
12361233
Type: string(placementv1beta1.ResourceWorkSynchronizedConditionType),
1237-
Reason: condition.WorkSynchronizedReason,
1238-
},
1239-
{
1240-
Status: metav1.ConditionTrue,
1241-
Type: string(placementv1beta1.ResourcesAppliedConditionType),
1242-
Reason: condition.ApplySucceededReason,
1243-
},
1244-
{
1245-
Status: metav1.ConditionTrue,
1246-
Type: string(placementv1beta1.ResourcesAvailableConditionType),
1247-
Reason: condition.AvailableReason,
1234+
Reason: condition.WorkSynchronizedUnknownReason,
12481235
},
12491236
},
12501237
},
@@ -1288,29 +1275,89 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() {
12881275
Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
12891276
},
12901277
},
1291-
{
1292-
Label: []*prometheusclientmodel.LabelPair{
1293-
{Name: ptr.To("name"), Value: ptr.To(crp.Name)},
1294-
{Name: ptr.To("generation"), Value: ptr.To(strconv.FormatInt(gotCRP.Generation, 10))},
1295-
{Name: ptr.To("conditionType"), Value: ptr.To(string(placementv1beta1.ClusterResourcePlacementAppliedConditionType))},
1296-
{Name: ptr.To("status"), Value: ptr.To(string(corev1.ConditionUnknown))},
1297-
},
1298-
Gauge: &prometheusclientmodel.Gauge{
1299-
Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
1300-
},
1278+
}
1279+
checkPlacementStatusMetric(customRegistry, wantMetrics)
1280+
1281+
By("Update to a synchronized clusterResourceBinding on member-1")
1282+
member1Binding = updateClusterResourceBindingWithSynchronized(member1Binding)
1283+
1284+
By("Update to a synchronized clusterResourceBinding on member-2")
1285+
member2Binding = updateClusterResourceBindingWithSynchronized(member2Binding)
1286+
1287+
By("Validate CRP status with apply pending condition")
1288+
wantCondition := metav1.Condition{
1289+
Status: metav1.ConditionTrue,
1290+
Type: string(placementv1beta1.ClusterResourcePlacementWorkSynchronizedConditionType),
1291+
Reason: condition.WorkSynchronizedReason,
1292+
}
1293+
meta.SetStatusCondition(&wantCRP.Status.Conditions, wantCondition)
1294+
wantCondition.Type = string(placementv1beta1.ResourceBindingWorkSynchronized)
1295+
meta.SetStatusCondition(&wantCRP.Status.PlacementStatuses[0].Conditions, wantCondition)
1296+
meta.SetStatusCondition(&wantCRP.Status.PlacementStatuses[1].Conditions, wantCondition)
1297+
1298+
wantCondition = metav1.Condition{
1299+
Status: metav1.ConditionUnknown,
1300+
Type: string(placementv1beta1.ClusterResourcePlacementAppliedConditionType),
1301+
Reason: condition.ApplyPendingReason,
1302+
}
1303+
meta.SetStatusCondition(&wantCRP.Status.Conditions, wantCondition)
1304+
wantCondition.Type = string(placementv1beta1.ResourceBindingApplied)
1305+
meta.SetStatusCondition(&wantCRP.Status.PlacementStatuses[0].Conditions, wantCondition)
1306+
meta.SetStatusCondition(&wantCRP.Status.PlacementStatuses[1].Conditions, wantCondition)
1307+
1308+
By("Ensure placement status applied metric was emitted")
1309+
wantMetrics = append(wantMetrics, &prometheusclientmodel.Metric{
1310+
Label: []*prometheusclientmodel.LabelPair{
1311+
{Name: ptr.To("name"), Value: ptr.To(crp.Name)},
1312+
{Name: ptr.To("generation"), Value: ptr.To(strconv.FormatInt(gotCRP.Generation, 10))},
1313+
{Name: ptr.To("conditionType"), Value: ptr.To(string(placementv1beta1.ClusterResourcePlacementAppliedConditionType))},
1314+
{Name: ptr.To("status"), Value: ptr.To(string(corev1.ConditionUnknown))},
13011315
},
1302-
{
1303-
Label: []*prometheusclientmodel.LabelPair{
1304-
{Name: ptr.To("name"), Value: ptr.To(gotCRP.Name)},
1305-
{Name: ptr.To("generation"), Value: ptr.To(strconv.FormatInt(gotCRP.Generation, 10))},
1306-
{Name: ptr.To("conditionType"), Value: ptr.To("Completed")},
1307-
{Name: ptr.To("status"), Value: ptr.To(string(corev1.ConditionTrue))},
1308-
},
1309-
Gauge: &prometheusclientmodel.Gauge{
1310-
Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
1311-
},
1316+
Gauge: &prometheusclientmodel.Gauge{
1317+
Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
13121318
},
1319+
})
1320+
1321+
By("Update to an available clusterResourceBinding on member-1")
1322+
member1Binding = updateClusterResourceBindingWithAvailable(member1Binding)
1323+
1324+
By("Update to an available clusterResourceBinding on member-2")
1325+
member2Binding = updateClusterResourceBindingWithAvailable(member2Binding)
1326+
1327+
By("Validate CRP status with all true conditions")
1328+
wantCondition = metav1.Condition{
1329+
Status: metav1.ConditionTrue,
1330+
Type: string(placementv1beta1.ClusterResourcePlacementAppliedConditionType),
1331+
Reason: condition.ApplySucceededReason,
1332+
}
1333+
meta.SetStatusCondition(&wantCRP.Status.Conditions, wantCondition)
1334+
wantCondition.Type = string(placementv1beta1.ResourceBindingApplied)
1335+
meta.SetStatusCondition(&wantCRP.Status.PlacementStatuses[0].Conditions, wantCondition)
1336+
meta.SetStatusCondition(&wantCRP.Status.PlacementStatuses[1].Conditions, wantCondition)
1337+
1338+
wantCondition = metav1.Condition{
1339+
Status: metav1.ConditionTrue,
1340+
Type: string(placementv1beta1.ClusterResourcePlacementAvailableConditionType),
1341+
Reason: condition.AvailableReason,
13131342
}
1343+
meta.SetStatusCondition(&wantCRP.Status.Conditions, wantCondition)
1344+
wantCondition.Type = string(placementv1beta1.ResourceBindingAvailable)
1345+
meta.SetStatusCondition(&wantCRP.Status.PlacementStatuses[0].Conditions, wantCondition)
1346+
meta.SetStatusCondition(&wantCRP.Status.PlacementStatuses[1].Conditions, wantCondition)
1347+
gotCRP = retrieveAndValidateClusterResourcePlacement(testCRPName, wantCRP)
1348+
1349+
By("Ensure placement status completed metric was emitted")
1350+
wantMetrics = append(wantMetrics, &prometheusclientmodel.Metric{
1351+
Label: []*prometheusclientmodel.LabelPair{
1352+
{Name: ptr.To("name"), Value: ptr.To(gotCRP.Name)},
1353+
{Name: ptr.To("generation"), Value: ptr.To(strconv.FormatInt(gotCRP.Generation, 10))},
1354+
{Name: ptr.To("conditionType"), Value: ptr.To("Completed")},
1355+
{Name: ptr.To("status"), Value: ptr.To(string(corev1.ConditionTrue))},
1356+
},
1357+
Gauge: &prometheusclientmodel.Gauge{
1358+
Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
1359+
},
1360+
})
13141361
checkPlacementStatusMetric(customRegistry, wantMetrics)
13151362
})
13161363
})
@@ -1389,11 +1436,11 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() {
13891436
By("Update clusterSchedulingPolicySnapshot status to schedule success")
13901437
updateClusterSchedulingPolicySnapshotStatus(metav1.ConditionTrue, true)
13911438

1392-
By("Create a synchronized clusterResourceBinding on member-1")
1393-
member1Binding = createSynchronizedClusterResourceBinding(member1Name, gotPolicySnapshot, gotResourceSnapshot)
1439+
By("Create an overridden clusterResourceBinding on member-1")
1440+
member1Binding = createOverriddenClusterResourceBinding(member1Name, gotPolicySnapshot, gotResourceSnapshot)
13941441

1395-
By("Create a synchronized clusterResourceBinding on member-2")
1396-
member2Binding = createSynchronizedClusterResourceBinding(member2Name, gotPolicySnapshot, gotResourceSnapshot)
1442+
By("Create an overridden clusterResourceBinding on member-2")
1443+
member2Binding = createOverriddenClusterResourceBinding(member2Name, gotPolicySnapshot, gotResourceSnapshot)
13971444

13981445
By("Validate CRP status")
13991446
wantCRP := &placementv1beta1.ClusterResourcePlacement{
@@ -1420,15 +1467,10 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() {
14201467
Type: string(placementv1beta1.ClusterResourcePlacementOverriddenConditionType),
14211468
Reason: condition.OverrideNotSpecifiedReason,
14221469
},
1423-
{
1424-
Status: metav1.ConditionTrue,
1425-
Type: string(placementv1beta1.ClusterResourcePlacementWorkSynchronizedConditionType),
1426-
Reason: condition.WorkSynchronizedReason,
1427-
},
14281470
{
14291471
Status: metav1.ConditionUnknown,
1430-
Type: string(placementv1beta1.ClusterResourcePlacementDiffReportedConditionType),
1431-
Reason: condition.DiffReportedStatusUnknownReason,
1472+
Type: string(placementv1beta1.ClusterResourcePlacementWorkSynchronizedConditionType),
1473+
Reason: condition.WorkSynchronizedUnknownReason,
14321474
},
14331475
},
14341476
PlacementStatuses: []placementv1beta1.ResourcePlacementStatus{
@@ -1450,15 +1492,10 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() {
14501492
Type: string(placementv1beta1.ResourceOverriddenConditionType),
14511493
Reason: condition.OverriddenSucceededReason,
14521494
},
1453-
{
1454-
Status: metav1.ConditionTrue,
1455-
Type: string(placementv1beta1.ResourceWorkSynchronizedConditionType),
1456-
Reason: condition.WorkSynchronizedReason,
1457-
},
14581495
{
14591496
Status: metav1.ConditionUnknown,
1460-
Type: string(placementv1beta1.ResourcesDiffReportedConditionType),
1461-
Reason: condition.DiffReportedStatusUnknownReason,
1497+
Type: string(placementv1beta1.ResourceWorkSynchronizedConditionType),
1498+
Reason: condition.WorkSynchronizedUnknownReason,
14621499
},
14631500
},
14641501
},
@@ -1480,15 +1517,10 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() {
14801517
Type: string(placementv1beta1.ResourceOverriddenConditionType),
14811518
Reason: condition.OverriddenSucceededReason,
14821519
},
1483-
{
1484-
Status: metav1.ConditionTrue,
1485-
Type: string(placementv1beta1.ResourceWorkSynchronizedConditionType),
1486-
Reason: condition.WorkSynchronizedReason,
1487-
},
14881520
{
14891521
Status: metav1.ConditionUnknown,
1490-
Type: string(placementv1beta1.ResourcesDiffReportedConditionType),
1491-
Reason: condition.DiffReportedStatusUnknownReason,
1522+
Type: string(placementv1beta1.ResourceWorkSynchronizedConditionType),
1523+
Reason: condition.WorkSynchronizedUnknownReason,
14921524
},
14931525
},
14941526
},
@@ -1532,19 +1564,49 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() {
15321564
Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
15331565
},
15341566
},
1535-
{
1536-
Label: []*prometheusclientmodel.LabelPair{
1537-
{Name: ptr.To("name"), Value: ptr.To(gotCRP.Name)},
1538-
{Name: ptr.To("generation"), Value: ptr.To(strconv.FormatInt(gotCRP.Generation, 10))},
1539-
{Name: ptr.To("conditionType"), Value: ptr.To(string(placementv1beta1.ClusterResourcePlacementDiffReportedConditionType))},
1540-
{Name: ptr.To("status"), Value: ptr.To(string(corev1.ConditionUnknown))},
1541-
},
1542-
Gauge: &prometheusclientmodel.Gauge{
1543-
Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
1544-
},
1545-
},
15461567
}
15471568
checkPlacementStatusMetric(customRegistry, wantMetrics)
1569+
1570+
By("Update to a synchronized clusterResourceBinding on member-1")
1571+
member1Binding = updateClusterResourceBindingWithSynchronized(member1Binding)
1572+
1573+
By("Update to a synchronized clusterResourceBinding on member-2")
1574+
member2Binding = updateClusterResourceBindingWithSynchronized(member2Binding)
1575+
1576+
By("Validate CRP status")
1577+
wantCondition := metav1.Condition{
1578+
Status: metav1.ConditionTrue,
1579+
Type: string(placementv1beta1.ClusterResourcePlacementWorkSynchronizedConditionType),
1580+
Reason: condition.WorkSynchronizedReason,
1581+
}
1582+
meta.SetStatusCondition(&wantCRP.Status.Conditions, wantCondition)
1583+
wantCondition.Type = string(placementv1beta1.ResourceWorkSynchronizedConditionType)
1584+
meta.SetStatusCondition(&wantCRP.Status.PlacementStatuses[0].Conditions, wantCondition)
1585+
meta.SetStatusCondition(&wantCRP.Status.PlacementStatuses[1].Conditions, wantCondition)
1586+
wantCondition = metav1.Condition{
1587+
Status: metav1.ConditionUnknown,
1588+
Type: string(placementv1beta1.ClusterResourcePlacementDiffReportedConditionType),
1589+
Reason: condition.DiffReportedStatusUnknownReason,
1590+
}
1591+
meta.SetStatusCondition(&wantCRP.Status.Conditions, wantCondition)
1592+
wantCondition.Type = string(placementv1beta1.ResourcesDiffReportedConditionType)
1593+
meta.SetStatusCondition(&wantCRP.Status.PlacementStatuses[0].Conditions, wantCondition)
1594+
meta.SetStatusCondition(&wantCRP.Status.PlacementStatuses[1].Conditions, wantCondition)
1595+
gotCRP = retrieveAndValidateClusterResourcePlacement(testCRPName, wantCRP)
1596+
1597+
By("Ensure placement status metric for reportDiff was emitted")
1598+
wantMetrics = append(wantMetrics, &prometheusclientmodel.Metric{
1599+
Label: []*prometheusclientmodel.LabelPair{
1600+
{Name: ptr.To("name"), Value: ptr.To(gotCRP.Name)},
1601+
{Name: ptr.To("generation"), Value: ptr.To(strconv.FormatInt(gotCRP.Generation, 10))},
1602+
{Name: ptr.To("conditionType"), Value: ptr.To(string(placementv1beta1.ClusterResourcePlacementDiffReportedConditionType))},
1603+
{Name: ptr.To("status"), Value: ptr.To(string(corev1.ConditionUnknown))},
1604+
},
1605+
Gauge: &prometheusclientmodel.Gauge{
1606+
Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
1607+
},
1608+
})
1609+
checkPlacementStatusMetric(customRegistry, wantMetrics)
15481610
})
15491611

15501612
It("Emit metrics for ReportDiff Complete CRP", func() {

0 commit comments

Comments
 (0)