Skip to content

Commit 199361a

Browse files
chriskerychriskery
authored
Merge pull request #7 from chriskery/hadoop-job-submit
support hadoop job submit
2 parents 4568ab5 + 2f5fb7a commit 199361a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+4223
-101
lines changed

‎.github/workflows/unittest.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@ jobs:
4040
if: failure()
4141
with:
4242
detached: false
43-
4443
- name: Coveralls report
4544
uses: shogo82148/actions-goveralls@v1
4645
with:

‎Makefile‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,9 @@ vet: ## Run go vet against code.
6868
go vet ./...
6969

7070
.PHONY: test
71-
test: manifests generate fmt vet envtest ## Run tests.
72-
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out
71+
test: ## Run tests.
72+
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -v ./pkg/util -coverprofile cover.out
73+
cat cover.out
7374

7475
##@ Build
7576

‎README.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,9 @@ Please refer to the [quick-start.md](docs/quick-start.md) for more information.
3636
## Features
3737

3838
- [x] [Create and destroy a hadoop-cluster](https://github.com/chriskery/charts/tree/master/charts/hadoop-cluster#deploying-hadoop-cluster)
39-
- [x] [Hdfs dataNode HPA](https://github.com/chriskery/charts/tree/master/charts/hadoop-cluster#deploying-hadoop-cluster)
40-
- [x] [Yarn nodeManager HPA](https://github.com/chriskery/charts/tree/master/charts/hadoop-cluster#deploying-hadoop-cluster)
39+
- [x] [Hdfs DataNode HPA](https://github.com/chriskery/charts/tree/master/charts/hadoop-cluster#deploying-hadoop-cluster)
40+
- [x] [Yarn NodeManager HPA](https://github.com/chriskery/charts/tree/master/charts/hadoop-cluster#deploying-hadoop-cluster)
41+
- [x] [Submit Hadoop Job](https://github.com/chriskery/charts/tree/master/charts/hadoop-cluster#deploying-hadoop-cluster)
4142

4243
## Contributing and Community
4344

‎docs/api/kubecluster.org_v1alpha1_generated.asciidoc‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ Package v1alpha1 contains API Schema definitions for the kubecluster.org v1alpha
1616
.Resource Types
1717
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopcluster[$$HadoopCluster$$]
1818
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopclusterlist[$$HadoopClusterList$$]
19+
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjob[$$HadoopJob$$]
20+
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjoblist[$$HadoopJobList$$]
1921

2022

2123
=== Definitions
@@ -192,6 +194,86 @@ HadoopClusterStatus defines the observed state of HadoopCluster
192194
|===
193195

194196

197+
[id="{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjob"]
198+
==== HadoopJob
199+
200+
HadoopJob is the Schema for the hadoopjobs API
201+
202+
.Appears In:
203+
****
204+
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjoblist[$$HadoopJobList$$]
205+
****
206+
207+
[cols="25a,75a", options="header"]
208+
|===
209+
| Field | Description
210+
| *`apiVersion`* __string__ | `kubecluster.org/v1alpha1`
211+
| *`kind`* __string__ | `HadoopJob`
212+
| *`TypeMeta`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#typemeta-v1-meta[$$TypeMeta$$]__ |
213+
| *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#objectmeta-v1-meta[$$ObjectMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`.
214+
215+
| *`spec`* __xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjobspec[$$HadoopJobSpec$$]__ |
216+
| *`status`* __xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjobstatus[$$HadoopJobStatus$$]__ |
217+
|===
218+
219+
220+
[id="{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjoblist"]
221+
==== HadoopJobList
222+
223+
HadoopJobList contains a list of HadoopJob
224+
225+
226+
227+
[cols="25a,75a", options="header"]
228+
|===
229+
| Field | Description
230+
| *`apiVersion`* __string__ | `kubecluster.org/v1alpha1`
231+
| *`kind`* __string__ | `HadoopJobList`
232+
| *`TypeMeta`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#typemeta-v1-meta[$$TypeMeta$$]__ |
233+
| *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#listmeta-v1-meta[$$ListMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`.
234+
235+
| *`items`* __xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjob[$$HadoopJob$$] array__ |
236+
|===
237+
238+
239+
[id="{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjobspec"]
240+
==== HadoopJobSpec
241+
242+
HadoopJobSpec defines the desired state of HadoopJob NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.
243+
244+
.Appears In:
245+
****
246+
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjob[$$HadoopJob$$]
247+
****
248+
249+
[cols="25a,75a", options="header"]
250+
|===
251+
| Field | Description
252+
| *`mainApplicationFile`* __string__ | MainFile is the path to a bundled JAR, Python, or R file of the application.
253+
| *`arguments`* __string array__ | Arguments is a list of arguments to be passed to the application.
254+
| *`executorSpec`* __xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopnodespec[$$HadoopNodeSpec$$]__ |
255+
|===
256+
257+
258+
[id="{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjobstatus"]
259+
==== HadoopJobStatus
260+
261+
HadoopJobStatus defines the observed state of HadoopJob
262+
263+
.Appears In:
264+
****
265+
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjob[$$HadoopJob$$]
266+
****
267+
268+
[cols="25a,75a", options="header"]
269+
|===
270+
| Field | Description
271+
| *`conditions`* __xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-jobcondition[$$JobCondition$$] array__ | INSERT ADDITIONAL STATUS FIELD - define observed state of cluster Important: Run "make" to regenerate code after modifying this file Conditions is an array of current observed job conditions.
272+
| *`startTime`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#time-v1-meta[$$Time$$]__ | Represents time when the job was acknowledged by the job controller. It is not guaranteed to be set in happens-before order across separate operations. It is represented in RFC3339 form and is in UTC.
273+
| *`completionTime`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#time-v1-meta[$$Time$$]__ | Represents time when the job was completed. It is not guaranteed to be set in happens-before order across separate operations. It is represented in RFC3339 form and is in UTC.
274+
|===
275+
276+
195277
[id="{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopnodespec"]
196278
==== HadoopNodeSpec
197279

@@ -201,6 +283,7 @@ HadoopClusterStatus defines the observed state of HadoopCluster
201283
****
202284
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hdfsdatanodespectemplate[$$HDFSDataNodeSpecTemplate$$]
203285
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hdfsnamenodespectemplate[$$HDFSNameNodeSpecTemplate$$]
286+
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjobspec[$$HadoopJobSpec$$]
204287
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-yarnnodemanagerspectemplate[$$YarnNodeManagerSpecTemplate$$]
205288
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-yarnresourcemanagerspectemplate[$$YarnResourceManagerSpecTemplate$$]
206289
****
@@ -220,6 +303,40 @@ HadoopClusterStatus defines the observed state of HadoopCluster
220303
|===
221304

222305

306+
[id="{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-jobcondition"]
307+
==== JobCondition
308+
309+
JobCondition describes current state of a cluster
310+
311+
.Appears In:
312+
****
313+
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjobstatus[$$HadoopJobStatus$$]
314+
****
315+
316+
[cols="25a,75a", options="header"]
317+
|===
318+
| Field | Description
319+
| *`type`* __xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-jobconditiontype[$$JobConditionType$$]__ | Type of job condition.
320+
| *`status`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#conditionstatus-v1-core[$$ConditionStatus$$]__ | Status of the condition, one of True, False, Unknown.
321+
| *`reason`* __string__ | The reason for the condition's last transition.
322+
| *`message`* __string__ | A human readable message indicating details about the transition.
323+
| *`lastUpdateTime`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#time-v1-meta[$$Time$$]__ | The last time this condition was updated.
324+
| *`lastTransitionTime`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#time-v1-meta[$$Time$$]__ | Last time the condition transitioned from one status to another.
325+
|===
326+
327+
328+
[id="{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-jobconditiontype"]
329+
==== JobConditionType (string)
330+
331+
332+
333+
.Appears In:
334+
****
335+
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-jobcondition[$$JobCondition$$]
336+
****
337+
338+
339+
223340
[id="{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-replicastatus"]
224341
==== ReplicaStatus
225342

‎hack/violation_exception.list‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
API rule violation: list_type_missing,github.com/chriskery/hadoop-cluster-operator/pkg/apis/kubecluster.org/v1alpha1,HadoopClusterStatus,Conditions
2+
API rule violation: list_type_missing,github.com/chriskery/hadoop-cluster-operator/pkg/apis/kubecluster.org/v1alpha1,HadoopJobStatus,Conditions

‎main.go‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,10 @@ package main
1818

1919
import (
2020
"flag"
21-
"github.com/chriskery/hadoop-cluster-operator/pkg/config"
21+
"fmt"
2222
"os"
23+
24+
"github.com/chriskery/hadoop-cluster-operator/pkg/config"
2325
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
2426
"sigs.k8s.io/controller-runtime/pkg/webhook"
2527

@@ -55,6 +57,7 @@ func main() {
5557
var enableLeaderElection bool
5658
var probeAddr string
5759
var certDir string
60+
var enabledSchemes controllers.EnabledSchemes
5861

5962
flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
6063
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
@@ -66,6 +69,8 @@ func main() {
6669
config.HadoopInitContainerImageDefault, "The image for hadoop init container")
6770
flag.StringVar(&config.Config.HadoopInitContainerTemplateFile, "hadoop-init-container-template-file",
6871
config.HadoopInitContainerTemplateFileDefault, "The template file for hadoop init container")
72+
flag.Var(&enabledSchemes, "enable-scheme", "Enable scheme(s) as --enable-scheme=tfjob --enable-scheme=pytorchjob, case insensitive."+
73+
" Now supporting TFJob, PyTorchJob, MXNetJob, XGBoostJob, PaddleJob. By default, all supported schemes will be enabled.")
6974

7075
opts := zap.Options{
7176
Development: true,
@@ -99,13 +104,22 @@ func main() {
99104
os.Exit(1)
100105
}
101106

102-
if err = controllers.NewReconciler(mgr).SetupWithManager(mgr); err != nil {
103-
setupLog.Error(err, "unable to create controller", "controller", "hadoopCluster")
104-
os.Exit(1)
107+
// TODO: We need a general manager. all rest reconciler addsToManager
108+
// Based on the user configuration, we start different controllers
109+
if enabledSchemes.Empty() {
110+
enabledSchemes.FillAll()
105111
}
106-
if err = (&hadoopclusterorgv1alpha1.HadoopCluster{}).SetupWebhookWithManager(mgr); err != nil {
107-
setupLog.Error(err, "unable to create webhook", "webhook", "hadoopCluster")
108-
os.Exit(1)
112+
for _, s := range enabledSchemes {
113+
setupFunc, supported := controllers.SupportedSchemeReconciler[s]
114+
if !supported {
115+
setupLog.Error(fmt.Errorf("cannot find %s in supportedSchemeReconciler", s),
116+
"scheme not supported", "scheme", s)
117+
os.Exit(1)
118+
}
119+
if err = setupFunc(mgr); err != nil {
120+
setupLog.Error(err, "unable to create controller", "controller", s)
121+
os.Exit(1)
122+
}
109123
}
110124
//+kubebuilder:scaffold:builder
111125

0 commit comments

Comments
 (0)