Revert back to Spark 3.3.2

Author: syedhassaanahmed

.devcontainer/devcontainer.json

@@ -35,7 +35,7 @@
         "ghcr.io/devcontainers/features/azure-cli:1": {},
         "ghcr.io/devcontainers/features/docker-in-docker:2": {},
         "ghcr.io/devcontainers/features/terraform:1": {},
-        "ghcr.io/devcontainers-contrib/features/spark-sdkman:2": { "version": "3.4.0"}
+        "ghcr.io/devcontainers-contrib/features/spark-sdkman:2": { "version": "3.3.2"}
     },
     "postCreateCommand": "sudo pip3 install -r src/requirements.txt"
 }

src/databricks.tf

@@ -11,8 +11,9 @@ provider "databricks" {
   azure_workspace_resource_id = azurerm_databricks_workspace.dbw.id
 }
 
-data "databricks_spark_version" "latest" {
-  depends_on = [azurerm_databricks_workspace.dbw]
+data "databricks_spark_version" "v332" {
+  spark_version = "3.3.2"
+  depends_on    = [azurerm_databricks_workspace.dbw]
 }
 
 data "databricks_node_type" "smallest" {
@@ -68,7 +69,7 @@ resource "databricks_secret" "dbpassword" {
 
 resource "databricks_cluster" "this" {
   cluster_name            = "Exploration Cluster"
-  spark_version           = data.databricks_spark_version.latest.id
+  spark_version           = data.databricks_spark_version.v332.id
   node_type_id            = data.databricks_node_type.smallest.id
   autotermination_minutes = 20
   autoscale {
@@ -96,7 +97,7 @@ resource "databricks_library" "wheel" {
 resource "databricks_library" "kafka" {
   cluster_id = databricks_cluster.this.id
   maven {
-    coordinates = "org.apache.spark:spark-sql-kafka-0-10_2.12:3.4.0"
+    coordinates = "org.apache.spark:spark-sql-kafka-0-10_2.12:3.3.2"
   }
 }
 

src/docker-compose.yml

@@ -50,7 +50,7 @@ services:
     # Explicitly delete Wheel before building in order to avoid picking up existing version from mount
     entrypoint: ["bash", "-c", "rm -rf /common_lib/dist/*.whl && pip install build && python -m build /common_lib"]
   spark:
-    image: apache/spark-py:v3.4.0
+    image: apache/spark-py:v3.3.2
     ports:
       - 4040:4040
     volumes:
@@ -74,7 +74,7 @@ services:
     # The mssql-jdbc package must be explicitly added https://stackoverflow.com/questions/66903523/apache-spark-connector-for-sql-server-and-azure-sql
     # The default Ivy Cache location is not writable in the container, hence we have to override it to /tmp https://stackoverflow.com/a/69559038
     command: ["/opt/spark/bin/spark-submit", "--packages",
-      "org.apache.spark:spark-sql-kafka-0-10_2.12:3.4.0,com.microsoft.azure:spark-mssql-connector_2.12:1.3.0-BETA,com.microsoft.sqlserver:mssql-jdbc:12.4.0.jre8",
+      "org.apache.spark:spark-sql-kafka-0-10_2.12:3.3.2,com.microsoft.azure:spark-mssql-connector_2.12:1.3.0-BETA,com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre8",
       "--conf", "spark.driver.extraJavaOptions=-Divy.cache.dir=/tmp -Divy.home=/tmp",
       "--conf", "spark.ui.prometheus.enabled=true",
       "--conf", "spark.executor.processTreeMetrics.enabled=true",

src/grafana/dashboards/spark_performance.json

@@ -275,7 +275,7 @@
         "sortBy": [
           {
             "desc": false,
-            "displayName": "{__name__=\"spark_info\", instance=\"spark:4040\", job=\"Spark Executors Metrics\", revision=\"87a5442f7ed96b11051d8a9333476d080054e5a0\", version=\"3.4.0\"}"
+            "displayName": "{__name__=\"spark_info\", instance=\"spark:4040\", job=\"Spark Executors Metrics\", revision=\"5103e00c4ce5fcc4264ca9c4df12295d42557af6\", version=\"3.3.2\"}"
           }
         ]
       },

src/requirements.txt

@@ -1,6 +1,6 @@
 build
 flake8
-pyspark==3.4.0
+pyspark==3.3.2
 pytest
 pytest-github-report
 setuptools