Features/add demos (#10)

1.Fix for last commit we introduced some configurations incorrectly,
causing the reduce process to fail
2. Support dataloader if need upload some data to hdfs before
application run

Author: chriskery

demos/mapreduce/pom.xml

@@ -0,0 +1,54 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>kubecluster.org</groupId>
+  <artifactId>mapreduce</artifactId>
+  <version>1.0-SNAPSHOT</version>
+  <packaging>jar</packaging>
+
+  <name>mapreduce</name>
+  <url>http://maven.apache.org</url>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-client</artifactId>
+      <version>3.3.6</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <configuration>
+          <archive>
+            <manifest>
+              <mainClass>kubecluster.org.WordCountDemo</mainClass>
+            </manifest>
+            <manifestEntries>
+              <Class-Path>.</Class-Path>
+            </manifestEntries>
+          </archive>
+          <descriptorRefs>
+            <descriptorRef>jar-with-dependencies</descriptorRef>
+          </descriptorRefs>
+        </configuration>
+        <executions>
+          <execution>
+            <id>make-assembly</id>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>

demos/mapreduce/src/main/java/kubecluster/org/WordCountDemo.java

@@ -0,0 +1,34 @@
+package kubecluster.org;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.GenericOptionsParser;
+
+public class WordCountDemo {
+
+    public static void main(String[] args) throws Exception {
+        Configuration conf = new Configuration();
+        String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+        if (otherArgs.length < 2) {
+            System.err.println("Usage: wordcount <in> [<in>...] <out>");
+            System.exit(2);
+        }
+        Application application = Application.getInstance(conf, "word count");
+        application.setJarByClass(WordCountDemo.class);
+        application.setMapperClass(WordCountMapper.class);
+        application.setCombinerClass(WordCountReducer.class);
+        application.setReducerClass(WordCountReducer.class);
+        application.setOutputKeyClass(Text.class);
+        application.setOutputValueClass(IntWritable.class);
+        for (int i = 0; i < otherArgs.length - 1; ++i) {
+            FileInputFormat.addInputPath(application, new Path(otherArgs[i]));
+        }
+        FileOutputFormat.setOutputPath(application, new Path(otherArgs[otherArgs.length - 1]));
+        System.exit(application.waitForCompletion(true) ? 0 : 1);
+    }
+}

demos/mapreduce/src/main/java/kubecluster/org/WordCountMapper.java

@@ -0,0 +1,21 @@
+package kubecluster.org;
+import java.io.IOException;
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Mapper;
+
+public class WordCountMapper extends Mapper<Object, Text, Text, IntWritable> {
+
+    private final static IntWritable one = new IntWritable(1);
+    private Text word = new Text();
+
+    public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
+        StringTokenizer itr = new StringTokenizer(value.toString());
+        while (itr.hasMoreTokens()) {
+            word.set(itr.nextToken());
+            context.write(word, one);
+        }
+    }
+}

demos/mapreduce/src/main/java/kubecluster/org/WordCountReducer.java

@@ -0,0 +1,21 @@
+package kubecluster.org;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Reducer;
+
+public class WordCountReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
+
+    private IntWritable result = new IntWritable();
+
+    public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
+        int sum = 0;
+        for (IntWritable val : values) {
+            sum += val.get();
+        }
+        result.set(sum);
+        context.write(key, result);
+    }
+}

docs/api/kubecluster.org_v1alpha1_generated.asciidoc

@@ -50,7 +50,7 @@ hadoopcluster-sample-resourcemanager   1/1     Running   0          6s      10.2
 
 As you can see, there is a Pod that simulates **NameNode**, DataNode, NodeManager, and ResourceManager. If you need more Pods to run more tasks, you can adjust the number of associated Datanodes and Nodemanagers
 
-Now let's try logging in to a node and running a MapReduce job:
+Now let's try logging in to a node and running a MapReduce application:
 ```bash
 > kubectl exec -it hadoopcluster-sample-resourcemanager /bin/bash
 kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
@@ -59,8 +59,8 @@ LICENSE-binary  LICENSE.txt  NOTICE-binary  NOTICE.txt  README.txt  bin  etc  in
 bash-4.2$ cd share/hadoop/mapreduce/
 bash-4.2$ ls         
 hadoop-mapreduce-client-app-3.3.1.jar     hadoop-mapreduce-client-hs-plugins-3.3.1.jar       hadoop-mapreduce-client-shuffle-3.3.1.jar   lib-examples
-hadoop-mapreduce-client-common-3.3.1.jar  hadoop-mapreduce-client-jobclient-3.3.1-tests.jar  hadoop-mapreduce-client-uploader-3.3.1.jar  sources
-hadoop-mapreduce-client-core-3.3.1.jar    hadoop-mapreduce-client-jobclient-3.3.1.jar        hadoop-mapreduce-examples-3.3.1.jar
+hadoop-mapreduce-client-common-3.3.1.jar  hadoop-mapreduce-client-applicationclient-3.3.1-tests.jar  hadoop-mapreduce-client-uploader-3.3.1.jar  sources
+hadoop-mapreduce-client-core-3.3.1.jar    hadoop-mapreduce-client-applicationclient-3.3.1.jar        hadoop-mapreduce-examples-3.3.1.jar
 hadoop-mapreduce-client-hs-3.3.1.jar      hadoop-mapreduce-client-nativetask-3.3.1.jar       jdiff
 bash-4.2$ hadoop jar hadoop-mapreduce-examples-3.3.1.jar pi 8 1000
 Number of Maps  = 8
@@ -73,16 +73,16 @@ Wrote input for Map #4
 Wrote input for Map #5
 Wrote input for Map #6
 Wrote input for Map #7
-Starting Job
+Starting Application
 2024-01-11 08:28:41 INFO  DefaultNoHARMFailoverProxyProvider:64 - Connecting to ResourceManager at hadoopcluster-sample-resourcemanager/10.244.0.101:8032
-2024-01-11 08:28:42 INFO  JobResourceUploader:906 - Disabling Erasure Coding for path: /tmp/hadoop-yarn/staging/hadoop/.staging/job_1704961336749_0001
+2024-01-11 08:28:42 INFO  ApplicationResourceUploader:906 - Disabling Erasure Coding for path: /tmp/hadoop-yarn/staging/hadoop/.staging/application_1704961336749_0001
 2024-01-11 08:28:42 INFO  FileInputFormat:300 - Total input files to process : 8
-2024-01-11 08:28:43 INFO  JobSubmitter:202 - number of splits:8
-2024-01-11 08:28:43 INFO  JobSubmitter:298 - Submitting tokens for job: job_1704961336749_0001
-2024-01-11 08:28:43 INFO  JobSubmitter:299 - Executing with tokens: []
+2024-01-11 08:28:43 INFO  ApplicationSubmitter:202 - number of splits:8
+2024-01-11 08:28:43 INFO  ApplicationSubmitter:298 - Submitting tokens for application: application_1704961336749_0001
+2024-01-11 08:28:43 INFO  ApplicationSubmitter:299 - Executing with tokens: []
 ......
 ......
-Job Finished in 26.166 seconds
+Application Finished in 26.166 seconds
 Estimated value of Pi is 3.14100000000000000000
 bash-4.2$ 
 ```