Skip to content

Commit 6902648

Browse files
Vacant2333gitautomator[bot]
Vacant2333
authored and
gitautomator[bot]
committed
fix isUnavailable
Signed-off-by: Vacant2333 <vacant2333@gmail.com>
1 parent 0ae164e commit 6902648

File tree

6 files changed

+5975
-671
lines changed

6 files changed

+5975
-671
lines changed

‎pkg/cache/unavailableofferings.go‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,6 @@ import (
2222

2323
"github.com/patrickmn/go-cache"
2424
"knative.dev/pkg/logging"
25-
karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
26-
)
27-
28-
var (
29-
spotKey = key("", "", karpv1.CapacityTypeSpot)
3025
)
3126

3227
// UnavailableOfferings stores any offerings that return ICE (insufficient capacity errors) when
@@ -56,11 +51,6 @@ func NewUnavailableOfferings() *UnavailableOfferings {
5651

5752
// IsUnavailable returns true if the offering appears in the cache
5853
func (u *UnavailableOfferings) IsUnavailable(instanceType, zone, capacityType string) bool {
59-
if capacityType == karpv1.CapacityTypeSpot {
60-
if _, found := u.cache.Get(spotKey); found {
61-
return true
62-
}
63-
}
6454
_, found := u.cache.Get(key(instanceType, zone, capacityType))
6555
return found
6656
}

‎pkg/controllers/interruption/controller.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ func (c *Controller) Reconcile(ctx context.Context, node *corev1.Node) (reconcil
8585
return reconcile.Result{}, nil
8686
}
8787

88-
func (c *Controller) Register(ctx context.Context, m manager.Manager) error {
88+
func (c *Controller) Register(_ context.Context, m manager.Manager) error {
8989
return controllerruntime.NewControllerManagedBy(m).
9090
Named("interruption").
9191
For(&corev1.Node{}).

‎pkg/controllers/nodeclaim/garbagecollection/controller.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,10 @@ import (
3939
)
4040

4141
type Controller struct {
42-
kubeClient client.Client
43-
cloudProvider cloudprovider.CloudProvider
44-
successfulCount uint64 // keeps track of successful reconciles for more aggressive requeueing near the start of the controller
42+
kubeClient client.Client
43+
cloudProvider cloudprovider.CloudProvider
44+
// Keeps track of successful reconciles for more aggressive requeueing near the start of the controller.
45+
successfulCount uint64
4546
}
4647

4748
func NewController(kubeClient client.Client, cloudProvider cloudprovider.CloudProvider) *Controller {

‎pkg/providers/instancetype/instancetype.go‎

Lines changed: 73 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,9 @@ type DefaultProvider struct {
6666

6767
instanceTypesInfo []*ecsclient.DescribeInstanceTypesResponseBodyInstanceTypesInstanceType
6868

69-
muInstanceTypesOfferings sync.RWMutex
70-
instanceTypesOfferings map[string]sets.Set[string]
69+
muInstanceTypesOfferings sync.RWMutex
70+
instanceTypesOfferings map[string]sets.Set[string]
71+
spotInstanceTypesOfferings map[string]sets.Set[string]
7172

7273
instanceTypesCache *cache.Cache
7374

@@ -83,17 +84,18 @@ func NewDefaultProvider(region string, kubeClient client.Client, ecsClient *ecsc
8384
instanceTypesCache *cache.Cache, unavailableOfferingsCache *kcache.UnavailableOfferings,
8485
pricingProvider pricing.Provider, ackProvider ack.Provider) *DefaultProvider {
8586
return &DefaultProvider{
86-
kubeClient: kubeClient,
87-
ecsClient: ecsClient,
88-
region: region,
89-
pricingProvider: pricingProvider,
90-
ackProvider: ackProvider,
91-
instanceTypesInfo: []*ecsclient.DescribeInstanceTypesResponseBodyInstanceTypesInstanceType{},
92-
instanceTypesOfferings: map[string]sets.Set[string]{},
93-
instanceTypesCache: instanceTypesCache,
94-
unavailableOfferings: unavailableOfferingsCache,
95-
cm: pretty.NewChangeMonitor(),
96-
instanceTypesSeqNum: 0,
87+
kubeClient: kubeClient,
88+
ecsClient: ecsClient,
89+
region: region,
90+
pricingProvider: pricingProvider,
91+
ackProvider: ackProvider,
92+
instanceTypesInfo: []*ecsclient.DescribeInstanceTypesResponseBodyInstanceTypesInstanceType{},
93+
instanceTypesOfferings: map[string]sets.Set[string]{},
94+
spotInstanceTypesOfferings: map[string]sets.Set[string]{},
95+
instanceTypesCache: instanceTypesCache,
96+
unavailableOfferings: unavailableOfferingsCache,
97+
cm: pretty.NewChangeMonitor(),
98+
instanceTypesSeqNum: 0,
9799
}
98100
}
99101

@@ -111,6 +113,9 @@ func (p *DefaultProvider) validateState(nodeClass *v1alpha1.ECSNodeClass) error
111113
if len(p.instanceTypesOfferings) == 0 {
112114
return errors.New("no instance types offerings found")
113115
}
116+
if len(p.spotInstanceTypesOfferings) == 0 {
117+
return errors.New("no spot instance types offerings found")
118+
}
114119
if len(nodeClass.Status.VSwitches) == 0 {
115120
return errors.New("no vswitches found")
116121
}
@@ -182,13 +187,15 @@ func (p *DefaultProvider) List(ctx context.Context, kc *v1alpha1.KubeletConfigur
182187
zoneData := lo.Map(allZones.UnsortedList(), func(zoneID string, _ int) ZoneData {
183188
if !p.instanceTypesOfferings[lo.FromPtr(i.InstanceTypeId)].Has(zoneID) || !vSwitchsZones.Has(zoneID) {
184189
return ZoneData{
185-
ID: zoneID,
186-
Available: false,
190+
ID: zoneID,
191+
Available: false,
192+
SpotAvailable: false,
187193
}
188194
}
189195
return ZoneData{
190-
ID: zoneID,
191-
Available: true,
196+
ID: zoneID,
197+
Available: true,
198+
SpotAvailable: p.spotInstanceTypesOfferings[lo.FromPtr(i.InstanceTypeId)].Has(zoneID),
192199
}
193200
})
194201

@@ -271,7 +278,40 @@ func (p *DefaultProvider) UpdateInstanceTypeOfferings(ctx context.Context) error
271278
log.FromContext(ctx).Error(err, "failed to get instance type offerings")
272279
return err
273280
}
281+
if err := processAvailableResourcesResponse(resp, instanceTypesOfferings); err != nil {
282+
log.FromContext(ctx).Error(err, "failed to process available resource response")
283+
return err
284+
}
274285

286+
if p.cm.HasChanged("instance-type-offering", instanceTypesOfferings) {
287+
// Only update instanceTypesSeqNun with the instance type offerings have been changed
288+
// This is to not create new keys with duplicate instance type offerings option
289+
atomic.AddUint64(&p.instanceTypesOfferingsSeqNum, 1)
290+
log.FromContext(ctx).WithValues("instance-type-count", len(instanceTypesOfferings)).V(1).Info("discovered offerings for instance types")
291+
}
292+
p.instanceTypesOfferings = instanceTypesOfferings
293+
294+
spotInstanceTypesOfferings := map[string]sets.Set[string]{}
295+
describeAvailableResourceRequest = &ecsclient.DescribeAvailableResourceRequest{
296+
RegionId: tea.String(p.region),
297+
DestinationResource: tea.String("InstanceType"),
298+
SpotStrategy: tea.String("SpotAsPriceGo"),
299+
}
300+
resp, err = p.ecsClient.DescribeAvailableResourceWithOptions(
301+
describeAvailableResourceRequest, &util.RuntimeOptions{})
302+
if err != nil {
303+
log.FromContext(ctx).Error(err, "failed to get spot instance type offerings")
304+
return err
305+
}
306+
if err := processAvailableResourcesResponse(resp, spotInstanceTypesOfferings); err != nil {
307+
log.FromContext(ctx).Error(err, "failed to process spot instance type offerings")
308+
return err
309+
}
310+
p.spotInstanceTypesOfferings = spotInstanceTypesOfferings
311+
return nil
312+
}
313+
314+
func processAvailableResourcesResponse(resp *ecsclient.DescribeAvailableResourceResponse, offerings map[string]sets.Set[string]) error {
275315
if resp == nil || resp.Body == nil {
276316
return errors.New("DescribeAvailableResourceWithOptions failed to return any instance types")
277317
} else if resp.Body.AvailableZones == nil || len(resp.Body.AvailableZones.AvailableZone) == 0 {
@@ -280,18 +320,12 @@ func (p *DefaultProvider) UpdateInstanceTypeOfferings(ctx context.Context) error
280320

281321
for _, az := range resp.Body.AvailableZones.AvailableZone {
282322
// TODO: Later, `ClosedWithStock` will be tested to determine if `ClosedWithStock` should be added.
283-
if *az.StatusCategory == "WithStock" { // WithStock, ClosedWithStock, WithoutStock, ClosedWithoutStock
284-
processAvailableResources(az, instanceTypesOfferings)
323+
// WithStock, ClosedWithStock, WithoutStock, ClosedWithoutStock
324+
if *az.StatusCategory != "WithStock" {
325+
continue
285326
}
327+
processAvailableResources(az, offerings)
286328
}
287-
288-
if p.cm.HasChanged("instance-type-offering", instanceTypesOfferings) {
289-
// Only update instanceTypesSeqNun with the instance type offerings have been changed
290-
// This is to not create new keys with duplicate instance type offerings option
291-
atomic.AddUint64(&p.instanceTypesOfferingsSeqNum, 1)
292-
log.FromContext(ctx).WithValues("instance-type-count", len(instanceTypesOfferings)).V(1).Info("discovered offerings for instance types")
293-
}
294-
p.instanceTypesOfferings = instanceTypesOfferings
295329
return nil
296330
}
297331

@@ -307,12 +341,14 @@ func processAvailableResources(az *ecsclient.DescribeAvailableResourceResponseBo
307341

308342
for _, sr := range ar.SupportedResources.SupportedResource {
309343
// TODO: Later, `ClosedWithStock` will be tested to determine if `ClosedWithStock` should be added.
310-
if *sr.StatusCategory == "WithStock" { // WithStock, ClosedWithStock, WithoutStock, ClosedWithoutStock
311-
if _, ok := instanceTypesOfferings[*sr.Value]; !ok {
312-
instanceTypesOfferings[*sr.Value] = sets.New[string]()
313-
}
314-
instanceTypesOfferings[*sr.Value].Insert(*az.ZoneId)
344+
// WithStock, ClosedWithStock, WithoutStock, ClosedWithoutStock
345+
if *sr.StatusCategory != "WithStock" {
346+
continue
315347
}
348+
if _, ok := instanceTypesOfferings[*sr.Value]; !ok {
349+
instanceTypesOfferings[*sr.Value] = sets.New[string]()
350+
}
351+
instanceTypesOfferings[*sr.Value].Insert(*az.ZoneId)
316352
}
317353
}
318354
}
@@ -365,6 +401,10 @@ func getAllInstanceTypes(client *ecsclient.Client) ([]*ecsclient.DescribeInstanc
365401
func (p *DefaultProvider) createOfferings(_ context.Context, instanceType string, zones []ZoneData) []cloudprovider.Offering {
366402
var offerings []cloudprovider.Offering
367403
for _, zone := range zones {
404+
if !zone.Available {
405+
continue
406+
}
407+
368408
odPrice, odOK := p.pricingProvider.OnDemandPrice(instanceType)
369409
spotPrice, spotOK := p.pricingProvider.SpotPrice(instanceType, zone.ID)
370410

@@ -375,7 +415,7 @@ func (p *DefaultProvider) createOfferings(_ context.Context, instanceType string
375415
offerings = append(offerings, p.createOffering(zone.ID, karpv1.CapacityTypeOnDemand, odPrice, offeringAvailable))
376416
}
377417

378-
if spotOK {
418+
if spotOK && zone.SpotAvailable {
379419
isUnavailable := p.unavailableOfferings.IsUnavailable(instanceType, zone.ID, karpv1.CapacityTypeSpot)
380420
offeringAvailable := !isUnavailable && zone.Available
381421

‎pkg/providers/instancetype/types.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,9 @@ const (
5353
)
5454

5555
type ZoneData struct {
56-
ID string
57-
Available bool
56+
ID string
57+
Available bool
58+
SpotAvailable bool
5859
}
5960

6061
func calculateResourceOverhead(pods, cpuM, memoryMi int64) corev1.ResourceList {
@@ -79,7 +80,7 @@ var thresholds = [...]struct {
7980
overhead float64
8081
}{
8182
{1000, 0.06},
82-
{3000, 0.01},
83+
{2000, 0.01},
8384
{3000, 0.005},
8485
{4000, 0.005},
8586
}

0 commit comments

Comments
 (0)