Merge pull request #17 from factorhouse/feature/add-jdbc-connector

Add flink jdbc sql connector

Author: jaehyeon-kim

README.md

@@ -514,8 +514,9 @@ Core services like Flink, Spark, and Kafka Connect are designed to be modular an
 #### Flink Connectors
 
 - **Kafka SQL Connector:** Enables Flink to read from and write to Kafka topics using SQL.
-- **Avro Confluent Registry:** Allows Flink to work with Avro schemas stored in Confluent Schema Registry.
-- **Flink Faker:** A connector for generating fake data streams within Flink, useful for development and testing.
+  - **Avro Confluent Registry:** Allows Flink to work with Avro schemas stored in Confluent Schema Registry.
+- **JDBC SQL Connector:** A connector for reading data from and writing data into any relational databases with a JDBC driver.
+  - **PostgreSQL JDBC Driver:** A driver to connect to a PostgreSQL database.
 
 #### Flink Hive Dependencies
 

compose-flex.yml

@@ -26,6 +26,7 @@ x-common-flink-volumes: &flink_common_volumes
   - ./resources/deps/flink/iceberg:/tmp/iceberg
   - ./resources/deps/flink/parquet:/tmp/parquet
   - ./resources/deps/flink/connector:/tmp/connector
+  - ./resources/deps/postgres:/tmp/postgres
 
 services:
   flex:
@@ -210,7 +211,7 @@ services:
     volumes:
       - ./resources/hms/hive-site.xml:/opt/hive/conf/hive-site.xml
       - ./resources/hms/core-site.xml:/opt/hive/conf/core-site.xml
-      - ./resources/deps/hms/postgresql-42.7.3.jar:/opt/hive/lib/postgresql-42.7.3.jar
+      - ./resources/deps/postgres/postgresql-42.7.3.jar:/opt/hive/lib/postgresql-42.7.3.jar
       - ./resources/deps/hadoop/hadoop-common-3.3.6.jar:/opt/hive/lib/hadoop-common-3.3.6.jar
       - ./resources/deps/hadoop/hadoop-auth-3.3.6.jar:/opt/hive/lib/hadoop-auth-3.3.6.jar
       - ./resources/deps/hadoop/hadoop-aws-3.3.6.jar:/opt/hive/lib/hadoop-aws-3.3.6.jar

resources/setup-env.sh

@@ -26,7 +26,7 @@ JAR_PATH=$SCRIPT_PATH/deps
 rm -rf $JAR_PATH \
   && mkdir -p $JAR_PATH/flink/connector -p $JAR_PATH/flink/hive \
     -p $JAR_PATH/flink/iceberg -p $JAR_PATH/flink/parquet \
-    -p $JAR_PATH/hadoop -p $JAR_PATH/hms -p $JAR_PATH/kafka/connector \
+    -p $JAR_PATH/hadoop -p $JAR_PATH/postgres -p $JAR_PATH/kafka/connector \
     -p $JAR_PATH/spark/iceberg -p $JAR_PATH/spark/lineage
 
 ####
@@ -76,7 +76,7 @@ echo ""
 echo "▶️  Downloading Flink connectors..."
 
 START_TIME=$(date +%s)
-TOTAL_STEPS=3
+TOTAL_STEPS=4
 CURRENT_STEP=0
 
 FLINK_CONNECTOR_PATH=$JAR_PATH/flink/connector
@@ -89,8 +89,14 @@ curl --silent -o $FLINK_CONNECTOR_PATH/flink-sql-avro-confluent-registry-1.20.1.
   https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-avro-confluent-registry/1.20.1/flink-sql-avro-confluent-registry-1.20.1.jar
 ((CURRENT_STEP++)); progress_bar
 
-curl --silent -L -o $FLINK_CONNECTOR_PATH/flink-faker-0.5.3.jar \
-  https://github.com/knaufk/flink-faker/releases/download/v0.5.3/flink-faker-0.5.3.jar
+curl --silent -L -o $FLINK_CONNECTOR_PATH/flink-connector-jdbc-3.3.0-1.20.jar \
+  https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc/3.3.0-1.20/flink-connector-jdbc-3.3.0-1.20.jar
+((CURRENT_STEP++)); progress_bar
+
+POSTGRES_PATH=$JAR_PATH/postgres
+
+curl --silent -o $POSTGRES_PATH/postgresql-42.7.3.jar \
+  https://repo1.maven.org/maven2/org/postgresql/postgresql/42.7.3/postgresql-42.7.3.jar
 ((CURRENT_STEP++)); progress_bar
 
 flag_time_taken
@@ -160,7 +166,7 @@ echo ""
 echo "▶️  Downloading Hadoop/Hive Metastore dependencies..."
 
 START_TIME=$(date +%s)
-TOTAL_STEPS=11
+TOTAL_STEPS=10
 CURRENT_STEP=0
 
 HADOOP_PATH=$JAR_PATH/hadoop
@@ -205,12 +211,6 @@ curl --silent -o $HADOOP_PATH/woodstox-core-6.5.1.jar \
   https://repo1.maven.org/maven2/com/fasterxml/woodstox/woodstox-core/6.5.1/woodstox-core-6.5.1.jar
 ((CURRENT_STEP++)); progress_bar
 
-HMS_PATH=$JAR_PATH/hms
-
-curl --silent -o $HMS_PATH/postgresql-42.7.3.jar \
-  https://repo1.maven.org/maven2/org/postgresql/postgresql/42.7.3/postgresql-42.7.3.jar
-((CURRENT_STEP++)); progress_bar
-
 flag_time_taken
 
 ####