Add Flink SQL module and data generators for airlines and airports

Author: gAmUssA

Makefile

@@ -40,10 +40,15 @@ help:
 	@echo "${BLUE}${INFO} 🚀 run-sql${RESET}            - Run the Flink Table API application"
 	@echo "${BLUE}${INFO} 🧪 ci-checks${RESET}          - Run CI checks locally"
 	@echo ""
-	@echo "${YELLOW}${STAR} Data Generator:${RESET}"
+	@echo "${YELLOW}${STAR} Data Generators:${RESET}"
+	@echo "${BLUE}${INFO} 🏗️ build-data-generator${RESET} - Build the Flink Data Generator module"
+	@echo "${BLUE}${INFO} 🏗️ build-ref-generator${RESET}  - Build the Reference Data Generator module"
 	@echo "${BLUE}${INFO} 🚀 run-data-generator-local${RESET} - Run the Flink Data Generator in local environment"
 	@echo "${BLUE}${INFO} ☁️ run-data-generator-cloud${RESET} - Run the Flink Data Generator in cloud environment"
 	@echo "${BLUE}${INFO} 🚀 run-data-generator-with-props${RESET} - Run with custom properties (PROPS=path/to/properties)"
+	@echo "${BLUE}${INFO} 🚀 run-ref-generator-local${RESET} - Run the Reference Data Generator in local environment"
+	@echo "${BLUE}${INFO} ☁️ run-ref-generator-cloud${RESET} - Run the Reference Data Generator in cloud environment"
+	@echo "${BLUE}${INFO} 🚀 run-ref-generator-with-props${RESET} - Run the Reference Data Generator with custom properties (PROPS=path/to/properties)"
 	@echo ""
 	@echo "${YELLOW}${STAR} Flink Table API:${RESET}"
 	@echo "${BLUE}${INFO} 🚀 run-sql-status-local${RESET}  - Run Flight Status Dashboard locally"
@@ -55,6 +60,11 @@ help:
 	@echo "${BLUE}${INFO} ☁️ run-sql-delays-cloud${RESET}  - Run Airline Delay Analytics on cloud"
 	@echo "${BLUE}${INFO} ☁️ run-sql-all-cloud${RESET}     - Run all SQL use cases on cloud"
 	@echo ""
+	@echo "${YELLOW}${STAR} Flink SQL:${RESET}"
+	@echo "${BLUE}${INFO} 🚀 flink-sql-client${RESET}      - Start interactive Flink SQL client"
+	@echo "${BLUE}${INFO} 🚀 flink-sql-execute${RESET}     - Execute a specific SQL file (SQL_FILE=path/to/file.sql)"
+	@echo "${BLUE}${INFO} 🚀 flink-sql-build${RESET}       - Build Flink SQL client image"
+	@echo ""
 	@echo "${YELLOW}${STAR} Docker Management:${RESET}"
 	@echo "${BLUE}${INFO} 🐳 docker-up${RESET}          - Start all containers"
 	@echo "${BLUE}${INFO} 🐳 docker-down${RESET}        - Stop and remove all containers"
@@ -297,7 +307,7 @@ terraform-org-id:
 		exit 1; \
 	fi
 	@echo "${BLUE}${INFO} Exporting organization ID to TF_VAR_org_id...${RESET}"
-	@export TF_VAR_org_id=$$(confluent organization list -o json | jq -c -r '.[] | select(.is_current)' | jq '.id'); \
+	@export TF_VAR_org_id=$$(confluent organization list -o json | jq -c -r '.[] | select(.is_current)' | jq -r '.id'); \
 	echo "TF_VAR_org_id=$$TF_VAR_org_id"; \
 	echo "export TF_VAR_org_id=$$TF_VAR_org_id" >> .env; \
 	echo "${GREEN}${CHECK} Organization ID exported to TF_VAR_org_id and saved to .env file!${RESET}"
@@ -460,7 +470,8 @@ docker-logs:
 
 docker-restart:
 	@echo "${BLUE}${ROCKET} Restarting Docker containers...${RESET}"
-	docker compose restart
+	docker compose down
+	docker compose up -d
 	@echo "${GREEN}${CHECK} Docker containers restarted successfully!${RESET}"
 
 # Run CI checks locally
@@ -492,26 +503,45 @@ docker-build:
 	@echo "${GREEN}${CHECK} Docker image built successfully!${RESET}"
 	@echo "${YELLOW}${INFO} Run with: docker run -it flink-for-java-workshop:local${RESET}"
 
+# Flink SQL related targets
+.PHONY: flink-sql-client flink-sql-execute flink-sql-build
+
+flink-sql-build:
+	@echo "${BLUE}${ROCKET} Building Flink SQL client image...${RESET}"
+	docker compose build jobmanager
+	docker compose build taskmanager
+
+flink-sql-client:
+	@echo "${BLUE}${ROCKET} Starting interactive Flink SQL client...${RESET}"
+	docker compose run --rm sql-client
+
+flink-sql-execute:
+	@echo "${BLUE}${ROCKET} Executing SQL file...${RESET}"
+	@if [ -z "$(SQL_FILE)" ]; then \
+		echo "${RED}${ERROR} SQL_FILE parameter is required${RESET}"; \
+		echo "Example: make flink-sql-execute SQL_FILE=usecases/airline_delays.sql"; \
+		exit 1; \
+	fi
+	docker compose run --rm -e SQL_FILE=$(SQL_FILE) sql-client
+
 # Flink Data Generator targets
-.PHONY: build-data-generator
+.PHONY: build-data-generator run-data-generator-local run-data-generator-cloud run-data-generator-with-props
+
 build-data-generator:
 	@echo "${BLUE}${ROCKET} Building Flink Data Generator...${RESET}"
 	./gradlew :flink-data-generator:build
 	@echo "${GREEN}${CHECK} Flink Data Generator built successfully!${RESET}"
 
-.PHONY: run-data-generator-local
 run-data-generator-local:
 	@echo "${BLUE}${ROCKET} Running Flink Data Generator in local environment...${RESET}"
 	./gradlew :flink-data-generator:run --args="--env local"
 	@echo "${GREEN}${CHECK} Flink Data Generator completed!${RESET}"
 
-.PHONY: run-data-generator-cloud
 run-data-generator-cloud:
 	@echo "${BLUE}${CLOUD} Running Flink Data Generator in cloud environment...${RESET}"
 	./gradlew :flink-data-generator:run --args="--env cloud"
 	@echo "${GREEN}${CHECK} Flink Data Generator completed!${RESET}"
 
-.PHONY: run-data-generator-with-props
 run-data-generator-with-props:
 	@echo "${BLUE}${ROCKET} Running Flink Data Generator with custom properties...${RESET}"
 	@if [ -z "$(PROPS)" ]; then \
@@ -521,6 +551,33 @@ run-data-generator-with-props:
 	./gradlew :flink-data-generator:run --args="--properties $(PROPS)"
 	@echo "${GREEN}${CHECK} Flink Data Generator completed!${RESET}"
 
+# Reference Data Generator targets
+.PHONY: build-ref-generator run-ref-generator-local run-ref-generator-cloud run-ref-generator-with-props
+
+build-ref-generator:
+	@echo "${BLUE}${ROCKET} Building Reference Data Generator...${RESET}"
+	./gradlew :data-generator:build
+	@echo "${GREEN}${CHECK} Reference Data Generator built successfully!${RESET}"
+
+run-ref-generator-local:
+	@echo "${BLUE}${ROCKET} Running Reference Data Generator in local environment...${RESET}"
+	./gradlew :data-generator:run --args="--env local"
+	@echo "${GREEN}${CHECK} Reference Data Generator completed!${RESET}"
+
+run-ref-generator-cloud:
+	@echo "${BLUE}${CLOUD} Running Reference Data Generator in cloud environment...${RESET}"
+	./gradlew :data-generator:run --args="--env cloud"
+	@echo "${GREEN}${CHECK} Reference Data Generator completed!${RESET}"
+
+run-ref-generator-with-props:
+	@echo "${BLUE}${ROCKET} Running Reference Data Generator with custom properties...${RESET}"
+	@if [ -z "$(PROPS)" ]; then \
+		echo "${RED}${ERROR} Please specify properties file with PROPS=path/to/properties.${RESET}"; \
+		exit 1; \
+	fi
+	./gradlew :data-generator:run --args="--properties $(PROPS)"
+	@echo "${GREEN}${CHECK} Reference Data Generator completed!${RESET}"
+
 # Clean up
 .PHONY: clean
 clean:

common/models/src/main/avro/airline.avsc

@@ -0,0 +1,23 @@
+{
+  "type": "record",
+  "name": "Airline",
+  "namespace": "io.confluent.developer.models.reference",
+  "doc": "Schema for airline reference data",
+  "fields": [
+    {
+      "name": "airlineCode",
+      "type": "string",
+      "doc": "Unique identifier for the airline"
+    },
+    {
+      "name": "airlineName",
+      "type": "string",
+      "doc": "Full name of the airline"
+    },
+    {
+      "name": "country",
+      "type": "string",
+      "doc": "Country where the airline is based"
+    }
+  ]
+}

common/models/src/main/avro/airport.avsc

@@ -0,0 +1,38 @@
+{
+  "type": "record",
+  "name": "Airport",
+  "namespace": "io.confluent.developer.models.reference",
+  "doc": "Schema for airport reference data",
+  "fields": [
+    {
+      "name": "airportCode",
+      "type": "string",
+      "doc": "IATA airport code"
+    },
+    {
+      "name": "airportName",
+      "type": "string",
+      "doc": "Full name of the airport"
+    },
+    {
+      "name": "city",
+      "type": "string",
+      "doc": "City where the airport is located"
+    },
+    {
+      "name": "country",
+      "type": "string",
+      "doc": "Country where the airport is located"
+    },
+    {
+      "name": "latitude",
+      "type": "double",
+      "doc": "Latitude coordinate of the airport"
+    },
+    {
+      "name": "longitude",
+      "type": "double",
+      "doc": "Longitude coordinate of the airport"
+    }
+  ]
+}

data-generator/README.adoc

@@ -0,0 +1,133 @@
+= Reference Data Generator 🏢
+:toc:
+:icons: font
+:sectnums:
+
+This module generates reference data for airlines and airports and sends it to Kafka topics. 
+The data is used by the Flink SQL module for enriching flight data.
+
+== Features
+
+* Generates realistic airline data with airline codes, names, and countries
+* Generates realistic airport data with airport codes, names, cities, countries, and locations
+* Sends data to Kafka topics in JSON format
+* Configurable via properties files or command-line arguments
+* Supports both local Kafka and Confluent Cloud
+
+== Configuration
+
+The generator can be configured using properties files or command-line arguments.
+
+=== Properties Files
+
+Two property files are included:
+
+* `local.properties` - Configuration for local Kafka
+* `cloud-template.properties` - Template for Confluent Cloud configuration
+
+You can also create your own properties file and specify it with the `--properties` option.
+
+#### Local Properties
+
+```properties
+# Kafka Configuration for local environment
+bootstrap.servers=localhost:29092
+
+# Topic Configuration
+topic.airlines=airlines
+topic.airports=airports
+
+# Generator Configuration
+generator.airlines.count=20
+generator.airports.count=50
+
+# Environment
+environment=local
+```
+
+#### Cloud Properties
+
+```properties
+# Kafka Configuration for Confluent Cloud
+bootstrap.servers=${BOOTSTRAP_SERVERS}
+
+# Topic Configuration
+topic.airlines=airlines
+topic.airports=airports
+
+# Generator Configuration
+generator.airlines.count=20
+generator.airports.count=50
+
+# Security settings
+security.protocol=SASL_SSL
+sasl.mechanism=PLAIN
+sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="${CONFLUENT_API_KEY}" password="${CONFLUENT_API_SECRET}";
+
+# Environment
+environment=cloud
+```
+
+=== Command-Line Options
+
+The generator supports the following command-line options:
+
+* `-e, --env <environment>` - Environment (local or cloud), default: local
+* `-p, --properties <path>` - Path to properties file
+* `--cloud-key <key>` - Confluent Cloud API Key (overrides properties)
+* `--cloud-secret <secret>` - Confluent Cloud API Secret (overrides properties)
+* `-b, --bootstrap-servers <servers>` - Kafka bootstrap servers (overrides properties)
+* `-h, --help` - Show help message
+
+== Running the Generator
+
+=== Using Makefile
+
+The easiest way to run the generator is using the Makefile targets:
+
+```bash
+# Run with local Kafka
+make run-ref-data-generator
+
+# Run with Confluent Cloud
+make run-ref-data-generator-cloud CLOUD_KEY=<key> CLOUD_SECRET=<secret> BOOTSTRAP_SERVERS=<servers>
+
+# Run with custom properties file
+make run-ref-data-generator-with-props PROPS=path/to/properties
+```
+
+=== Using Gradle
+
+You can also run the generator directly with Gradle:
+
+```bash
+# Run with local Kafka
+./gradlew :data-generator:run --args="--env local"
+
+# Run with Confluent Cloud
+./gradlew :data-generator:run --args="--env cloud --cloud-key <key> --cloud-secret <secret> --bootstrap-servers <servers>"
+
+# Run with custom properties file
+./gradlew :data-generator:run --args="--properties path/to/properties"
+```
+
+== Data Models
+
+=== Airline
+
+The airline model includes:
+
+* `airlineCode` - IATA airline code (e.g., "AA")
+* `airlineName` - Airline name (e.g., "American Airlines")
+* `country` - Country of the airline (e.g., "United States")
+
+=== Airport
+
+The airport model includes:
+
+* `airportCode` - IATA airport code (e.g., "JFK")
+* `airportName` - Airport name (e.g., "John F. Kennedy International Airport")
+* `city` - City (e.g., "New York")
+* `country` - Country (e.g., "United States")
+* `latitude` - Latitude coordinate
+* `longitude` - Longitude coordinate