add OpenLineage Spark dependency

Author: jaehyeon-kim

compose-flex.yml

@@ -245,9 +245,10 @@ services:
       - ./resources/spark/log4j2.properties:/opt/spark/conf/log4j2.properties:ro
       - ./resources/spark/hive-site.xml:/opt/spark/conf/hive-site.xml
       - ./resources/spark/core-site.xml:/opt/spark/conf/core-site.xml
-      # Iceberg & Hadoop dependencies
+      # Iceberg, Hadoop & OpenLineage dependencies
       - ./resources/deps/spark/iceberg/iceberg-spark-runtime-3.5_2.12-1.8.1.jar:/opt/spark/jars/iceberg-spark-runtime-3.5_2.12-1.8.1.jar
       - ./resources/deps/spark/iceberg/iceberg-aws-bundle-1.8.1.jar:/opt/spark/jars/iceberg-aws-bundle-1.8.1.jar
+      - ./resources/deps/spark/lineage/openlineage-spark_2.12-1.37.0.jar:/opt/spark/jars/openlineage-spark_2.12-1.37.0.jar
       - ./resources/deps/hadoop/hadoop-aws-3.3.6.jar:/opt/spark/jars/hadoop-aws-3.3.6.jar
       - ./resources/deps/hadoop/aws-java-sdk-bundle-1.11.1026.jar:/opt/spark/jars/aws-java-sdk-bundle-1.11.1026.jar
       - ./resources/deps/hadoop/hadoop-common-3.3.6.jar:/opt/spark/jars/hadoop-common-3.3.6.jar

resources/setup-env.sh

@@ -26,8 +26,8 @@ JAR_PATH=$SCRIPT_PATH/deps
 rm -rf $JAR_PATH \
   && mkdir -p $JAR_PATH/flink/connector -p $JAR_PATH/flink/hive \
     -p $JAR_PATH/flink/iceberg -p $JAR_PATH/flink/parquet \
-    -p $JAR_PATH/hadoop -p $JAR_PATH/hms \
-    -p $JAR_PATH/kafka/connector -p $JAR_PATH/spark/iceberg
+    -p $JAR_PATH/hadoop -p $JAR_PATH/hms -p $JAR_PATH/kafka/connector \
+    -p $JAR_PATH/spark/iceberg -p $JAR_PATH/spark/lineage
 
 ####
 #### Kafka connectors
@@ -214,13 +214,13 @@ curl --silent -o $HMS_PATH/postgresql-42.7.3.jar \
 flag_time_taken
 
 ####
-#### Spark Iceberg dependencies
+#### Spark Iceberg/OpenLineage dependencies
 ####
 echo ""
-echo "▶️  Downloading Spark Iceberg dependencies..."
+echo "▶️  Downloading Spark Iceberg/OpenLineage dependencies..."
 
 START_TIME=$(date +%s)
-TOTAL_STEPS=2
+TOTAL_STEPS=3
 CURRENT_STEP=0
 
 SPARK_ICEBERG_PATH=$JAR_PATH/spark/iceberg
@@ -233,6 +233,12 @@ curl --silent -o $SPARK_ICEBERG_PATH/iceberg-aws-bundle-1.8.1.jar \
   https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/1.8.1/iceberg-aws-bundle-1.8.1.jar
 ((CURRENT_STEP++)); progress_bar
 
+SPARK_LINEAGE_PATH=$JAR_PATH/spark/lineage
+
+curl --silent -o $SPARK_LINEAGE_PATH/openlineage-spark_2.12-1.37.0.jar \
+  https://repo1.maven.org/maven2/io/openlineage/openlineage-spark_2.12/1.37.0/openlineage-spark_2.12-1.37.0.jar
+((CURRENT_STEP++)); progress_bar
+
 flag_time_taken
 
 echo ""

resources/spark/log4j2.properties

@@ -75,4 +75,8 @@ appender.console.filter.1.onMismatch = neutral
 # logger.metastore.name = org.apache.hadoop.hive.metastore
 # logger.metastore.level = debug
 # logger.hiveexec.name = org.apache.hadoop.hive.ql.exec
-# logger.hiveexec.level = debug
+# logger.hiveexec.level = debug
+
+# # OpenLineage
+# logger.openlineage.name = io.openlineage
+# logger.openlineage.level = DEBUG