Skip to content

Commit fb4c8bc

Browse files
authored
Merge pull request #110 from kubernetes-sigs/permit
Support permit extension point
2 parents 45428bb + 0bb5a81 commit fb4c8bc

File tree

19 files changed

+366
-18
lines changed

19 files changed

+366
-18
lines changed

Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ examples/advanced/main.wasm: examples/advanced/main.go
1010

1111
.PHONY: build-tinygo
1212
build-tinygo: examples/nodenumber/main.wasm examples/advanced/main.wasm guest/testdata/cyclestate/main.wasm guest/testdata/filter/main.wasm guest/testdata/score/main.wasm \
13-
guest/testdata/bind/main.wasm guest/testdata/reserve/main.wasm guest/testdata/handle/main.wasm
13+
guest/testdata/bind/main.wasm guest/testdata/reserve/main.wasm guest/testdata/handle/main.wasm guest/testdata/permit/main.wasm
1414

1515
%/main-debug.wasm: %/main.go
1616
@(cd $(@D); tinygo build -o main-debug.wasm -gc=custom -tags=custommalloc -scheduler=none -target=wasi .)

RATIONALE.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,17 @@ result, not a parameter. The only impact would be to new plugins or those
223223
ported to WebAssembly. We do not expect limiting the scores to two billion
224224
above the valid range to be a practical concern for these authors.
225225

226+
### Why does the Permit function return a uint32 representing milliseconds for the timeout, not `time.Duration`?
227+
228+
`framework.PermitPlugin` returns `time.Duration` to represent the timeout.
229+
`time.Duration` is int64 underneath and 1 time.Duration represents 1 nanosecond.
230+
231+
Given the scheduling throughput in the upstream kube-scheduler is around 300 pods/s,
232+
that is, 3+ milliseconds per pod,
233+
we consider a millisecond-level timeout with uint32 to be sufficiently fine-grained.
234+
235+
Also, tha maximum timeout is around 24 days, which also should be large enough.
236+
226237
## Why do we return a non-status, second numeric result as an i32?
227238

228239
Most compilers that target WebAssembly Core Specification 1.0, the only REC

guest/api/types.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@
1616

1717
package api
1818

19-
import "sigs.k8s.io/kube-scheduler-wasm-extension/guest/api/proto"
19+
import (
20+
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/api/proto"
21+
)
2022

2123
// CycleState is a WebAssembly implementation of framework.CycleState.
2224
//
@@ -122,6 +124,14 @@ type ReservePlugin interface {
122124
Unreserve(state CycleState, p proto.Pod, nodeName string)
123125
}
124126

127+
// PermitPlugin is a WebAssembly implementation of framework.PermitPlugin.
128+
type PermitPlugin interface {
129+
Plugin
130+
131+
// Note: This is uint32, not time.Duration. See /RATIONALE.md for why.
132+
Permit(state CycleState, p proto.Pod, nodeName string) (status *Status, timeoutMilliSeconds uint32)
133+
}
134+
125135
// PreBindPlugin is a WebAssembly implementation of framework.PreBindPlugin.
126136
type PreBindPlugin interface {
127137
Plugin

guest/permit/permit.go

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
Copyright 2023 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// Package permit exports an api.PermitPlugin to the host.
18+
package permit
19+
20+
import (
21+
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/api"
22+
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/internal/cyclestate"
23+
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/internal/imports"
24+
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/internal/plugin"
25+
)
26+
27+
// permit is the current plugin assigned with SetPlugin.
28+
var permit api.PermitPlugin
29+
30+
// SetPlugin should be called in `main` to assign an api.PermitPlugin
31+
// instance.
32+
//
33+
// For example:
34+
//
35+
// func main() {
36+
// plugin := permitPlugin{}
37+
// permit.SetPlugin(plugin)
38+
// }
39+
//
40+
// type permitPlugin struct{}
41+
//
42+
// func (permitPlugin) Permit(state api.CycleState, p proto.Pod, nodeName string) (status *api.Status, timeout uint32)
43+
// // Write state you need on Permit
44+
// }
45+
func SetPlugin(permitPlugin api.PermitPlugin) {
46+
if permitPlugin == nil {
47+
panic("nil permitPlugin")
48+
}
49+
permit = permitPlugin
50+
plugin.MustSet(permit)
51+
}
52+
53+
// prevent unused lint errors (lint is run with normal go).
54+
var _ func() uint64 = _permit
55+
56+
// _permit is only exported to the host.
57+
//
58+
//export permit
59+
func _permit() uint64 {
60+
if permit == nil { // Then, the user didn't define one.
61+
// Unlike most plugins we always export permit so that we can reset
62+
// the cycle state: return success to avoid no-op overhead.
63+
return 0
64+
}
65+
66+
pod := cyclestate.Pod
67+
nodeName := imports.NodeName()
68+
status, timeout := permit.Permit(cyclestate.Values, pod, nodeName)
69+
70+
// Pack the score and status code into a single WebAssembly 1.0 compatible
71+
// result
72+
return (uint64(imports.StatusToCode(status)) << uint64(32)) | uint64(timeout)
73+
}

guest/plugin/plugin.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/enqueue"
2323
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/filter"
2424
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/internal/prefilter"
25+
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/permit"
2526
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/postbind"
2627
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/postfilter"
2728
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/prebind"
@@ -72,6 +73,9 @@ func Set(plugin api.Plugin) {
7273
if plugin, ok := plugin.(api.ReservePlugin); ok {
7374
reserve.SetPlugin(plugin)
7475
}
76+
if plugin, ok := plugin.(api.PermitPlugin); ok {
77+
permit.SetPlugin(plugin)
78+
}
7579
if plugin, ok := plugin.(api.PreBindPlugin); ok {
7680
prebind.SetPlugin(plugin)
7781
}

guest/testdata/cyclestate/main.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/bind"
2828
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/enqueue"
2929
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/filter"
30+
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/permit"
3031
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/postbind"
3132
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/postfilter"
3233
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/prebind"
@@ -67,6 +68,7 @@ func main() {
6768
score.SetPlugin(plugin)
6869
scoreextensions.SetPlugin(plugin)
6970
reserve.SetPlugin(plugin)
71+
permit.SetPlugin(plugin)
7072
prebind.SetPlugin(plugin)
7173
bind.SetPlugin(plugin)
7274
postbind.SetPlugin(plugin)
@@ -187,6 +189,11 @@ func (statePlugin) Unreserve(state api.CycleState, pod proto.Pod, nodeName strin
187189
mustFilterState(state)
188190
}
189191

192+
func (statePlugin) Permit(state api.CycleState, pod proto.Pod, nodeName string) (status *api.Status, timeout uint32) {
193+
mustFilterState(state)
194+
return
195+
}
196+
190197
func (statePlugin) PreBind(state api.CycleState, pod proto.Pod, _ string) (status *api.Status) {
191198
if unsafe.Pointer(pod.Spec()) != unsafe.Pointer(podSpec) {
192199
panic("didn't cache pod from pre-filter")
9.06 KB
Binary file not shown.

guest/testdata/permit/main.go

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
Copyright 2023 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package main
18+
19+
import (
20+
"time"
21+
22+
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/api"
23+
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/api/proto"
24+
"sigs.k8s.io/kube-scheduler-wasm-extension/guest/permit"
25+
)
26+
27+
type extensionPoints interface {
28+
api.PermitPlugin
29+
}
30+
31+
func main() {
32+
var plugin extensionPoints = permitPlugin{}
33+
permit.SetPlugin(plugin)
34+
}
35+
36+
type permitPlugin struct{}
37+
38+
func (permitPlugin) Permit(state api.CycleState, pod proto.Pod, nodeName string) (*api.Status, uint32) {
39+
status, timeout := api.StatusCodeSuccess, time.Duration(0)
40+
if nodeName == "bad" {
41+
status = api.StatusCodeError
42+
} else if nodeName == "wait" {
43+
status = api.StatusCodeWait
44+
timeout = 10 * time.Second
45+
}
46+
return &api.Status{Code: status, Reason: "name is " + nodeName}, uint32(timeout.Milliseconds())
47+
}

guest/testdata/permit/main.wasm

545 KB
Binary file not shown.

internal/e2e/e2e.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,11 @@ func RunAll(ctx context.Context, t Testing, plugin framework.Plugin, pod *v1.Pod
4949
reserveP.Unreserve(ctx, nil, pod, ni.Node().Name)
5050
}
5151

52+
if permitP, ok := plugin.(framework.PermitPlugin); ok {
53+
s, _ = permitP.Permit(ctx, nil, pod, ni.Node().Name)
54+
RequireSuccess(t, s)
55+
}
56+
5257
if prebindP, ok := plugin.(framework.PreBindPlugin); ok {
5358
s = prebindP.PreBind(ctx, nil, pod, "")
5459
RequireSuccess(t, s)

0 commit comments

Comments
 (0)