Skip to content

Commit 2f5fb7a

Browse files
chriskerychriskery
committed
1. support submit hadoop job
2. add examples for hadoop job
1 parent f895c53 commit 2f5fb7a

40 files changed

+2316
-45
lines changed

‎README.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,9 @@ Please refer to the [quick-start.md](docs/quick-start.md) for more information.
3636
## Features
3737

3838
- [x] [Create and destroy a hadoop-cluster](https://github.com/chriskery/charts/tree/master/charts/hadoop-cluster#deploying-hadoop-cluster)
39-
- [x] [Hdfs dataNode HPA](https://github.com/chriskery/charts/tree/master/charts/hadoop-cluster#deploying-hadoop-cluster)
40-
- [x] [Yarn nodeManager HPA](https://github.com/chriskery/charts/tree/master/charts/hadoop-cluster#deploying-hadoop-cluster)
39+
- [x] [Hdfs DataNode HPA](https://github.com/chriskery/charts/tree/master/charts/hadoop-cluster#deploying-hadoop-cluster)
40+
- [x] [Yarn NodeManager HPA](https://github.com/chriskery/charts/tree/master/charts/hadoop-cluster#deploying-hadoop-cluster)
41+
- [x] [Submit Hadoop Job](https://github.com/chriskery/charts/tree/master/charts/hadoop-cluster#deploying-hadoop-cluster)
4142

4243
## Contributing and Community
4344

‎docs/api/kubecluster.org_v1alpha1_generated.asciidoc‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ Package v1alpha1 contains API Schema definitions for the kubecluster.org v1alpha
1616
.Resource Types
1717
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopcluster[$$HadoopCluster$$]
1818
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopclusterlist[$$HadoopClusterList$$]
19+
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjob[$$HadoopJob$$]
20+
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjoblist[$$HadoopJobList$$]
1921

2022

2123
=== Definitions
@@ -192,6 +194,86 @@ HadoopClusterStatus defines the observed state of HadoopCluster
192194
|===
193195

194196

197+
[id="{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjob"]
198+
==== HadoopJob
199+
200+
HadoopJob is the Schema for the hadoopjobs API
201+
202+
.Appears In:
203+
****
204+
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjoblist[$$HadoopJobList$$]
205+
****
206+
207+
[cols="25a,75a", options="header"]
208+
|===
209+
| Field | Description
210+
| *`apiVersion`* __string__ | `kubecluster.org/v1alpha1`
211+
| *`kind`* __string__ | `HadoopJob`
212+
| *`TypeMeta`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#typemeta-v1-meta[$$TypeMeta$$]__ |
213+
| *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#objectmeta-v1-meta[$$ObjectMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`.
214+
215+
| *`spec`* __xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjobspec[$$HadoopJobSpec$$]__ |
216+
| *`status`* __xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjobstatus[$$HadoopJobStatus$$]__ |
217+
|===
218+
219+
220+
[id="{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjoblist"]
221+
==== HadoopJobList
222+
223+
HadoopJobList contains a list of HadoopJob
224+
225+
226+
227+
[cols="25a,75a", options="header"]
228+
|===
229+
| Field | Description
230+
| *`apiVersion`* __string__ | `kubecluster.org/v1alpha1`
231+
| *`kind`* __string__ | `HadoopJobList`
232+
| *`TypeMeta`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#typemeta-v1-meta[$$TypeMeta$$]__ |
233+
| *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#listmeta-v1-meta[$$ListMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`.
234+
235+
| *`items`* __xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjob[$$HadoopJob$$] array__ |
236+
|===
237+
238+
239+
[id="{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjobspec"]
240+
==== HadoopJobSpec
241+
242+
HadoopJobSpec defines the desired state of HadoopJob NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.
243+
244+
.Appears In:
245+
****
246+
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjob[$$HadoopJob$$]
247+
****
248+
249+
[cols="25a,75a", options="header"]
250+
|===
251+
| Field | Description
252+
| *`mainApplicationFile`* __string__ | MainFile is the path to a bundled JAR, Python, or R file of the application.
253+
| *`arguments`* __string array__ | Arguments is a list of arguments to be passed to the application.
254+
| *`executorSpec`* __xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopnodespec[$$HadoopNodeSpec$$]__ |
255+
|===
256+
257+
258+
[id="{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjobstatus"]
259+
==== HadoopJobStatus
260+
261+
HadoopJobStatus defines the observed state of HadoopJob
262+
263+
.Appears In:
264+
****
265+
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjob[$$HadoopJob$$]
266+
****
267+
268+
[cols="25a,75a", options="header"]
269+
|===
270+
| Field | Description
271+
| *`conditions`* __xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-jobcondition[$$JobCondition$$] array__ | INSERT ADDITIONAL STATUS FIELD - define observed state of cluster Important: Run "make" to regenerate code after modifying this file Conditions is an array of current observed job conditions.
272+
| *`startTime`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#time-v1-meta[$$Time$$]__ | Represents time when the job was acknowledged by the job controller. It is not guaranteed to be set in happens-before order across separate operations. It is represented in RFC3339 form and is in UTC.
273+
| *`completionTime`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#time-v1-meta[$$Time$$]__ | Represents time when the job was completed. It is not guaranteed to be set in happens-before order across separate operations. It is represented in RFC3339 form and is in UTC.
274+
|===
275+
276+
195277
[id="{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopnodespec"]
196278
==== HadoopNodeSpec
197279

@@ -201,6 +283,7 @@ HadoopClusterStatus defines the observed state of HadoopCluster
201283
****
202284
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hdfsdatanodespectemplate[$$HDFSDataNodeSpecTemplate$$]
203285
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hdfsnamenodespectemplate[$$HDFSNameNodeSpecTemplate$$]
286+
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjobspec[$$HadoopJobSpec$$]
204287
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-yarnnodemanagerspectemplate[$$YarnNodeManagerSpecTemplate$$]
205288
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-yarnresourcemanagerspectemplate[$$YarnResourceManagerSpecTemplate$$]
206289
****
@@ -220,6 +303,40 @@ HadoopClusterStatus defines the observed state of HadoopCluster
220303
|===
221304

222305

306+
[id="{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-jobcondition"]
307+
==== JobCondition
308+
309+
JobCondition describes current state of a cluster
310+
311+
.Appears In:
312+
****
313+
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-hadoopjobstatus[$$HadoopJobStatus$$]
314+
****
315+
316+
[cols="25a,75a", options="header"]
317+
|===
318+
| Field | Description
319+
| *`type`* __xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-jobconditiontype[$$JobConditionType$$]__ | Type of job condition.
320+
| *`status`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#conditionstatus-v1-core[$$ConditionStatus$$]__ | Status of the condition, one of True, False, Unknown.
321+
| *`reason`* __string__ | The reason for the condition's last transition.
322+
| *`message`* __string__ | A human readable message indicating details about the transition.
323+
| *`lastUpdateTime`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#time-v1-meta[$$Time$$]__ | The last time this condition was updated.
324+
| *`lastTransitionTime`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#time-v1-meta[$$Time$$]__ | Last time the condition transitioned from one status to another.
325+
|===
326+
327+
328+
[id="{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-jobconditiontype"]
329+
==== JobConditionType (string)
330+
331+
332+
333+
.Appears In:
334+
****
335+
- xref:{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-jobcondition[$$JobCondition$$]
336+
****
337+
338+
339+
223340
[id="{anchor_prefix}-github-com-chriskery-hadoop-cluster-operator-pkg-apis-kubecluster-org-v1alpha1-replicastatus"]
224341
==== ReplicaStatus
225342

‎hack/violation_exception.list‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
API rule violation: list_type_missing,github.com/chriskery/hadoop-cluster-operator/pkg/apis/kubecluster.org/v1alpha1,HadoopClusterStatus,Conditions
2+
API rule violation: list_type_missing,github.com/chriskery/hadoop-cluster-operator/pkg/apis/kubecluster.org/v1alpha1,HadoopJobStatus,Conditions

‎pkg/.DS_Store‎

-6 KB
Binary file not shown.

‎pkg/builder/configmap.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -291,7 +291,7 @@ func (h *ConfigMapBuilder) Build(obj interface{}, _ interface{}) error {
291291
if err != nil {
292292
return err
293293
}
294-
ownerRef := util.GenOwnerReference(cluster)
294+
ownerRef := util.GenOwnerReference(cluster, hadoopclusterorgv1alpha1.GroupVersion.WithKind(hadoopclusterorgv1alpha1.HadoopClusterKind).Kind)
295295
return h.ConfigMapControl.CreateConfigMapWithControllerRef(cluster.GetNamespace(), configMap, cluster, ownerRef)
296296
}
297297

‎pkg/builder/driver.go‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ func (h *DriverBuilder) Build(obj interface{}, objStatus interface{}) error {
4444
}
4545

4646
podTemplateSpec := h.genDriverPodSpec(job, driverPodName)
47-
ownerRef := util.GenOwnerReference(job)
47+
ownerRef := util.GenOwnerReference(job, v1alpha1.GroupVersion.WithKind(v1alpha1.HadoopJobKind).Kind)
4848
if err = h.PodControl.CreatePodsWithControllerRef(job.GetNamespace(), podTemplateSpec, job, ownerRef); err != nil {
4949
return err
5050
}
@@ -64,12 +64,15 @@ func (h *DriverBuilder) genDriverPodSpec(job *v1alpha1.HadoopJob, driverPodName
6464
driverPodSpec := job.Spec.ExecutorSpec
6565
podTemplateSpec := &corev1.PodTemplateSpec{
6666
ObjectMeta: metav1.ObjectMeta{
67-
Labels: map[string]string{v1alpha1.JobNameLabel: job.GetName()},
68-
Name: driverPodName,
67+
Labels: map[string]string{
68+
v1alpha1.JobNameLabel: job.GetName(),
69+
v1alpha1.ReplicaTypeLabel: string(v1alpha1.ReplicaTypeDriver),
70+
},
71+
Name: driverPodName,
6972
},
7073
Spec: corev1.PodSpec{
7174
Volumes: driverPodSpec.Volumes,
72-
RestartPolicy: corev1.RestartPolicyAlways,
75+
RestartPolicy: corev1.RestartPolicyNever,
7376
DNSPolicy: corev1.DNSClusterFirstWithHostNet,
7477
},
7578
}

‎pkg/builder/hdfs.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ func (h *HdfsBuilder) buildNameNodeService(cluster *hadoopclusterorgv1alpha1.Had
138138
},
139139
}
140140

141-
ownerRef := util.GenOwnerReference(cluster)
141+
ownerRef := util.GenOwnerReference(cluster, hadoopclusterorgv1alpha1.GroupVersion.WithKind(hadoopclusterorgv1alpha1.HadoopClusterKind).Kind)
142142
if err := h.ServiceControl.CreateServicesWithControllerRef(cluster.GetNamespace(), nameNodeService, cluster, ownerRef); err != nil {
143143
return err
144144
}
@@ -151,7 +151,7 @@ func (h *HdfsBuilder) buildNameNodePod(cluster *hadoopclusterorgv1alpha1.HadoopC
151151
return err
152152
}
153153

154-
ownerRef := util.GenOwnerReference(cluster)
154+
ownerRef := util.GenOwnerReference(cluster, hadoopclusterorgv1alpha1.GroupVersion.WithKind(hadoopclusterorgv1alpha1.HadoopClusterKind).Kind)
155155
if err = h.PodControl.CreatePodsWithControllerRef(cluster.GetNamespace(), podSpec, cluster, ownerRef); err != nil {
156156
return err
157157
}
@@ -241,7 +241,7 @@ func (h *HdfsBuilder) buildDataNodeStatefulSet(cluster *hadoopclusterorgv1alpha1
241241
}
242242
dataNodeStatulSet.Spec = *statefulSetSpec
243243

244-
ownerRef := util.GenOwnerReference(cluster)
244+
ownerRef := util.GenOwnerReference(cluster, hadoopclusterorgv1alpha1.GroupVersion.WithKind(hadoopclusterorgv1alpha1.HadoopClusterKind).Kind)
245245
if err = h.StatefulSetControl.CreateStatefulSetsWithControllerRef(cluster.GetNamespace(), dataNodeStatulSet, cluster, ownerRef); err != nil {
246246
return err
247247
}
@@ -392,7 +392,7 @@ func (h *HdfsBuilder) buildDataNodeService(cluster *hadoopclusterorgv1alpha1.Had
392392
},
393393
}
394394

395-
ownerRef := util.GenOwnerReference(cluster)
395+
ownerRef := util.GenOwnerReference(cluster, hadoopclusterorgv1alpha1.GroupVersion.WithKind(hadoopclusterorgv1alpha1.HadoopClusterKind).Kind)
396396
if err = h.ServiceControl.CreateServicesWithControllerRef(cluster.GetNamespace(), nameNodeService, cluster, ownerRef); err != nil {
397397
return err
398398
}
@@ -475,7 +475,7 @@ func (h *HdfsBuilder) buildNameNodeNodePortService(
475475
},
476476
}
477477

478-
ownerRef := util.GenOwnerReference(cluster)
478+
ownerRef := util.GenOwnerReference(cluster, hadoopclusterorgv1alpha1.GroupVersion.WithKind(hadoopclusterorgv1alpha1.HadoopClusterKind).Kind)
479479
if err := h.ServiceControl.CreateServicesWithControllerRef(cluster.GetNamespace(), resourceManagerService, cluster, ownerRef); err != nil {
480480
return err
481481
}

‎pkg/builder/yarn.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ func (h *YarnBuilder) buildResourceManagerService(cluster *hadoopclusterorgv1alp
135135
},
136136
}
137137

138-
ownerRef := util.GenOwnerReference(cluster)
138+
ownerRef := util.GenOwnerReference(cluster, hadoopclusterorgv1alpha1.GroupVersion.WithKind(hadoopclusterorgv1alpha1.HadoopClusterKind).Kind)
139139
if err := h.ServiceControl.CreateServicesWithControllerRef(cluster.GetNamespace(), resourceManagerService, cluster, ownerRef); err != nil {
140140
return err
141141
}
@@ -149,7 +149,7 @@ func (h *YarnBuilder) buildResourceManagerPod(cluster *hadoopclusterorgv1alpha1.
149149
return err
150150
}
151151

152-
ownerRef := util.GenOwnerReference(cluster)
152+
ownerRef := util.GenOwnerReference(cluster, hadoopclusterorgv1alpha1.GroupVersion.WithKind(hadoopclusterorgv1alpha1.HadoopClusterKind).Kind)
153153
if err = h.PodControl.CreatePodsWithControllerRef(cluster.GetNamespace(), podTemplate, cluster, ownerRef); err != nil {
154154
return err
155155
}
@@ -202,7 +202,7 @@ func (h *YarnBuilder) buildNodeManagerService(cluster *hadoopclusterorgv1alpha1.
202202
},
203203
}
204204

205-
ownerRef := util.GenOwnerReference(cluster)
205+
ownerRef := util.GenOwnerReference(cluster, hadoopclusterorgv1alpha1.GroupVersion.WithKind(hadoopclusterorgv1alpha1.HadoopClusterKind).Kind)
206206
if err := h.ServiceControl.CreateServicesWithControllerRef(cluster.GetNamespace(), resourceManagerService, cluster, ownerRef); err != nil {
207207
return err
208208
}
@@ -223,7 +223,7 @@ func (h *YarnBuilder) buildNodeManagerStatefulSet(cluster *hadoopclusterorgv1alp
223223
}
224224
nodeManagerStatefulSet.Spec = *deploySpec
225225

226-
ownerRef := util.GenOwnerReference(cluster)
226+
ownerRef := util.GenOwnerReference(cluster, hadoopclusterorgv1alpha1.GroupVersion.WithKind(hadoopclusterorgv1alpha1.HadoopClusterKind).Kind)
227227
if err = h.StatefulSetControl.CreateStatefulSetsWithControllerRef(cluster.GetNamespace(), nodeManagerStatefulSet, cluster, ownerRef); err != nil {
228228
return err
229229
}
@@ -387,7 +387,7 @@ func (h *YarnBuilder) buildResourceManageNodePortService(
387387
},
388388
}
389389

390-
ownerRef := util.GenOwnerReference(cluster)
390+
ownerRef := util.GenOwnerReference(cluster, hadoopclusterorgv1alpha1.GroupVersion.WithKind(hadoopclusterorgv1alpha1.HadoopClusterKind).Kind)
391391
if err := h.ServiceControl.CreateServicesWithControllerRef(cluster.GetNamespace(), resourceManagerService, cluster, ownerRef); err != nil {
392392
return err
393393
}

0 commit comments

Comments
 (0)