++

Author: FroCode

docker/Dockerfile

@@ -16,10 +16,10 @@ COPY requirements.txt .
 # Install required Python packages
 RUN pip3 install --no-cache-dir -r requirements.txt
 
-# Copy Spark job scripts
+# Copy Spark job scripts from the jobs folder
 COPY jobs /opt/bitnami/spark/jobs/
 
-# Copy input files
+# Copy input files from the input folder
 COPY input /opt/bitnami/spark/input/
 
 # Ensure correct permissions

jobs/kafka_str.py

@@ -0,0 +1,29 @@
+from pyspark.sql import SparkSession
+from pyspark.sql.functions import col
+
+# Create Spark Session
+spark = SparkSession.builder \
+    .appName("KafkaSparkStreaming") \
+    .config("spark.jars.packages", 
+            "org.apache.spark:spark-sql-kafka-0-10_2.12:3.3.1") \
+    .getOrCreate()
+
+# Read Stream from Kafka
+df = spark \
+    .readStream \
+    .format("kafka") \
+    .option("kafka.bootstrap.servers", "localhost:9092") \
+    .option("subscribe", "streaming-topic") \
+    .option("startingOffsets", "earliest") \
+    .load()
+
+# Convert Kafka message from binary to string
+df = df.selectExpr("CAST(value AS STRING)")
+
+# Process Data (Modify as needed)
+query = df.writeStream \
+    .outputMode("append") \
+    .format("console") \
+    .start()
+
+query.awaitTermination()

promothues/docker-compose.yaml

@@ -0,0 +1,46 @@
+version: '3.7'
+
+services:
+  prometheus:
+    image: prom/prometheus
+    container_name: prometheus
+    volumes:
+      - /home/frocode/githubRepos/Realtime_Streaming_Unstructured-Data/promothues/promo/prometheus.yaml:/etc/prometheus/prometheus.yml
+    ports:
+      - "9090:9090"
+    networks:
+      - monitoring
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana
+    container_name: grafana
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+    ports:
+      - "3000:3000"
+    volumes:
+      - grafana-storage:/var/lib/grafana
+    depends_on:
+      - prometheus
+    networks:
+      - monitoring
+    restart: unless-stopped
+    
+  node_exporter:
+    image: quay.io/prometheus/node-exporter:v1.8.2
+    container_name: node_exporter
+    command: "--path.procfs=/proc --path.sysfs=/sys"
+    restart: unless-stopped
+    ports:
+      - "9100:9100"
+    networks:
+      - monitoring
+
+networks:
+  monitoring:
+    driver: bridge
+
+volumes:
+  grafana-storage:
+    driver: local

promothues/promo/prometheus.yaml

@@ -0,0 +1,16 @@
+global:
+  scrape_interval: 5s  # Adjust as needed
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'node_exporter'
+    static_configs:
+      - targets: ['node_exporter:9100']
+
+  - job_name: 'docker'
+    static_configs:
+      - targets: ['host.docker.internal:9323']  # Use this for Mac/Windows
+      # - targets: ['localhost:9323']  # Use this for Linux