++

Author: FroCode

docker/Dockerfile

@@ -1,20 +1,29 @@
+# Use the Bitnami Spark image as the base
 FROM bitnami/spark:latest
 
-# Install Python and pip (if not already included in the base image)
+# Switch to root to install dependencies
 USER root
+
+# Install Python and pip
 RUN apt-get update && apt-get install -y python3 python3-pip
 
-# Install additional dependencies (e.g., Python libraries)
+# Set working directory
+WORKDIR /opt/bitnami/spark
+
+# Copy dependencies
 COPY requirements.txt .
 
+# Install required Python packages
 RUN pip3 install --no-cache-dir -r requirements.txt
 
-# Copy any additional scripts or files into the container
-COPY ./jobs/* /opt/bitnami/spark/jobs
-COPY ./input/* /opt/bitnami/spark/input
+# Copy Spark job scripts
+COPY jobs /opt/bitnami/spark/jobs/
 
-# Set the working directory
-WORKDIR /opt/bitnami/spark
+# Copy input files
+COPY input /opt/bitnami/spark/input/
+
+# Ensure correct permissions
+RUN chmod -R 777 /opt/bitnami/spark
 
-# Optionally run the script automatically when the container starts
+# Default command to submit the Spark job
 CMD ["spark-submit", "--master", "spark://spark-master:7077", "/opt/bitnami/spark/jobs/main.py"]

jenkins/pipeline.jenkins

@@ -24,9 +24,10 @@ pipeline {
         stage('Build Spark Job Docker Image') {
             steps {
                 script {
-                    // Copy requirements.txt to the Docker directory
+                    // Copy necessary files into the Docker directory
                     sh '''
                     cp Realtime_Streaming_Unstructured-Data/requirements.txt Realtime_Streaming_Unstructured-Data/docker/
+                    cp -r Realtime_Streaming_Unstructured-Data/input Realtime_Streaming_Unstructured-Data/docker/
                     cd Realtime_Streaming_Unstructured-Data/docker
                     sudo docker build -t spark-job -f Dockerfile .
                     '''
@@ -49,8 +50,9 @@ pipeline {
         stage('Copy Input Files') {
             steps {
                 script {
-                    // Copy any necessary input files to the Spark input directory
+                    // Ensure Spark master is running before copying files
                     sh '''
+                    until sudo docker ps | grep -q spark-master; do sleep 5; done
                     sudo docker cp Realtime_Streaming_Unstructured-Data/input/input_file.txt spark-master:/opt/bitnami/spark/input/
                     '''
                 }
@@ -62,7 +64,7 @@ pipeline {
                 script {
                     // Submit the Spark job
                     sh '''
-                    sudo docker exec spark-job spark-submit --master spark://spark-master:7077 /opt/bitnami/spark/jobs/main.py
+                    sudo docker exec spark-master spark-submit --master spark://spark-master:7077 /opt/bitnami/spark/jobs/main.py
                     '''
                 }
             }
@@ -71,7 +73,6 @@ pipeline {
 
     post {
         always {
-            // Optionally stop and remove containers after the build
             script {
                 sh '''
                 cd Realtime_Streaming_Unstructured-Data/docker