Merge branch 'master' of github.com:simplesteph/kafka-stack-docker-compose

Author: simplesteph

.travis.yml

@@ -25,4 +25,4 @@ script:
 - ./test.sh zk-multiple-kafka-single.yml 4
 - ./test.sh zk-single-kafka-multiple.yml 4
 - ./test.sh zk-multiple-kafka-multiple.yml 6
-- ./test.sh full-stack.yml 10
+- ./test.sh full-stack.yml 10

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2019 Stephane Maarek
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -4,19 +4,20 @@
 
 This replicates as well as possible real deployment configurations, where you have your zookeeper servers and kafka servers actually all distinct from each other. This solves all the networking hurdles that comes with Docker and docker-compose, and is compatible cross platform.
 
-[UPDATE]: No /etc/hosts file changes are necessary anymore. Explanations at: https://rmoff.net/2018/08/02/kafka-listeners-explained/
+**UPDATE**: No /etc/hosts file changes are necessary anymore. Explanations at: https://rmoff.net/2018/08/02/kafka-listeners-explained/
 
 ## Stack version
 
   - Zookeeper version: 3.4.9
-  - Kafka version: 2.1.0 (Confluent 5.1.0)
-  - Kafka Schema Registry: Confluent 5.1.0
+  - Kafka version: 2.4.0 (Confluent 5.4.1)
+  - Kafka Schema Registry: Confluent 5.4.1
   - Kafka Schema Registry UI: 0.9.4
-  - Kafka Rest Proxy: Confluent 5.1.0
+  - Kafka Rest Proxy: Confluent 5.4.1
   - Kafka Topics UI: 0.9.4
-  - Kafka Connect: Confluent 5.1.0
+  - Kafka Connect: Confluent 5.4.1
   - Kafka Connect UI: 0.9.4
-  - Zoonavigator: 0.5.1
+  - KSQL Server: Confluent 5.4.1
+  - Zoonavigator: 0.8.0
 
 
 ## Optional: Kafka Desktop Application
@@ -59,7 +60,7 @@ docker-compose -f zk-single-kafka-single.yml down
 
 ## Single Zookeeper / Multiple Kafka
 
-If you want to have two brokers and experiment with replication / fault-tolerance.
+If you want to have three brokers and experiment with kafka replication / fault-tolerance.
 
 - Zookeeper will be available at `$DOCKER_HOST_IP:2181`
 - Kafka will be available at `$DOCKER_HOST_IP:9092,$DOCKER_HOST_IP:9093,$DOCKER_HOST_IP:9094`
@@ -73,7 +74,7 @@ docker-compose -f zk-single-kafka-multiple.yml down
 
 ## Multiple Zookeeper / Single Kafka
 
-If you want to have three zookeeper and experiment with zookeeper fault-tolerance.
+If you want to have three zookeeper nodes and experiment with zookeeper fault-tolerance.
 
 - Zookeeper will be available at `$DOCKER_HOST_IP:2181,$DOCKER_HOST_IP:2182,$DOCKER_HOST_IP:2183`
 - Kafka will be available at `$DOCKER_HOST_IP:9092`
@@ -87,7 +88,7 @@ docker-compose -f zk-multiple-kafka-single.yml down
 
 ## Multiple Zookeeper / Multiple Kafka
 
-If you want to have three zookeeper and two kafka brokers to experiment with production setup.
+If you want to have three zookeeper nodes and three kafka brokers to experiment with production setup.
 
 - Zookeeper will be available at `$DOCKER_HOST_IP:2181,$DOCKER_HOST_IP:2182,$DOCKER_HOST_IP:2183`
 - Kafka will be available at `$DOCKER_HOST_IP:9092,$DOCKER_HOST_IP:9093,$DOCKER_HOST_IP:9094`
@@ -109,6 +110,7 @@ docker-compose -f zk-multiple-kafka-multiple.yml down
  - Kafka Topics UI: `$DOCKER_HOST_IP:8000`
  - Kafka Connect: `$DOCKER_HOST_IP:8083`
  - Kafka Connect UI: `$DOCKER_HOST_IP:8003`
+ - KSQL Server: `$DOCKER_HOST_IP:8088`
  - Zoonavigator Web: `$DOCKER_HOST_IP:8004`
 
 
@@ -150,7 +152,7 @@ A: yes. Say you want to change `zoo1` port to `12181` (only relevant lines are s
 A: yes. Say you want to change `kafka1` port to `12345` (only relevant lines are shown). Note only `LISTENER_DOCKER_EXTERNAL` changes:
 ```
   kafka1:
-    image: confluentinc/cp-kafka:5.1.0
+    image: confluentinc/cp-kafka:5.4.1
     hostname: kafka1
     ports:
       - "12345:12345"

full-stack.yml

@@ -17,7 +17,7 @@ services:
 
 
   kafka1:
-    image: confluentinc/cp-kafka:5.1.0
+    image: confluentinc/cp-kafka:5.4.1
     hostname: kafka1
     ports:
       - "9092:9092"
@@ -35,7 +35,7 @@ services:
       - zoo1
 
   kafka-schema-registry:
-    image: confluentinc/cp-schema-registry:5.1.0
+    image: confluentinc/cp-schema-registry:5.4.1
     hostname: kafka-schema-registry
     ports:
       - "8081:8081"
@@ -59,7 +59,7 @@ services:
       - kafka-schema-registry
 
   kafka-rest-proxy:
-    image: confluentinc/cp-kafka-rest:5.1.0
+    image: confluentinc/cp-kafka-rest:5.4.1
     hostname: kafka-rest-proxy
     ports:
       - "8082:8082"
@@ -89,7 +89,7 @@ services:
       - kafka-rest-proxy
 
   kafka-connect:
-    image: confluentinc/cp-kafka-connect:5.1.0
+    image: confluentinc/cp-kafka-connect:5.4.1
     hostname: kafka-connect
     ports:
       - "8083:8083"
@@ -112,6 +112,9 @@ services:
       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
+      CONNECT_PLUGIN_PATH: '/usr/share/java,/etc/kafka-connect/jars'
+    volumes:
+      - ./connectors:/etc/kafka-connect/jars/
     depends_on:
       - zoo1
       - kafka1
@@ -129,21 +132,23 @@ services:
     depends_on:
       - kafka-connect
 
-  zoonavigator-web:
-    image: elkozmon/zoonavigator-web:0.5.1
+  ksql-server:
+    image: confluentinc/cp-ksql-server:5.4.1
+    hostname: ksql-server
     ports:
-     - "8004:8000"
+      - "8088:8088"
     environment:
-      API_HOST: "zoonavigator-api"
-      API_PORT: 9000
-    links:
-     - zoonavigator-api
+      KSQL_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092
+      KSQL_LISTENERS: http://0.0.0.0:8088/
+      KSQL_KSQL_SERVICE_ID: ksql-server_
     depends_on:
-     - zoonavigator-api
+      - zoo1
+      - kafka1
 
-  zoonavigator-api:
-    image: elkozmon/zoonavigator-api:0.5.1
+  zoonavigator:
+    image: elkozmon/zoonavigator:0.8.0
+    ports:
+      - "8004:8000"
     environment:
-      SERVER_HTTP_PORT: 9000
-    depends_on:
-      - zoo1
+      HTTP_PORT: 8000
+      AUTO_CONNECT_CONNECTION_STRING: zoo1:2181

zk-multiple-kafka-multiple.yml

@@ -42,7 +42,7 @@ services:
 
 
   kafka1:
-    image: confluentinc/cp-kafka:5.1.0
+    image: confluentinc/cp-kafka:5.4.1
     hostname: kafka1
     ports:
       - "9092:9092"
@@ -61,7 +61,7 @@ services:
       - zoo3
 
   kafka2:
-    image: confluentinc/cp-kafka:5.1.0
+    image: confluentinc/cp-kafka:5.4.1
     hostname: kafka2
     ports:
       - "9093:9093"
@@ -80,7 +80,7 @@ services:
       - zoo3
 
   kafka3:
-    image: confluentinc/cp-kafka:5.1.0
+    image: confluentinc/cp-kafka:5.4.1
     hostname: kafka3
     ports:
       - "9094:9094"

zk-multiple-kafka-single.yml

@@ -42,15 +42,15 @@ services:
 
 
   kafka1:
-    image: confluentinc/cp-kafka:5.1.0
+    image: confluentinc/cp-kafka:5.4.1
     hostname: kafka1
     ports:
       - "9092:9092"
     environment:
       KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
-      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
+      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181,zoo2:2182,zoo3:2183"
       KAFKA_BROKER_ID: 1
       KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1

zk-single-kafka-multiple.yml

@@ -15,7 +15,7 @@ services:
       - ./zk-single-kafka-multiple/zoo1/datalog:/datalog
 
   kafka1:
-    image: confluentinc/cp-kafka:5.1.0
+    image: confluentinc/cp-kafka:5.4.1
     hostname: kafka1
     ports:
       - "9092:9092"
@@ -32,7 +32,7 @@ services:
       - zoo1
 
   kafka2:
-    image: confluentinc/cp-kafka:5.1.0
+    image: confluentinc/cp-kafka:5.4.1
     hostname: kafka2
     ports:
       - "9093:9093"
@@ -50,7 +50,7 @@ services:
 
 
   kafka3:
-    image: confluentinc/cp-kafka:5.1.0
+    image: confluentinc/cp-kafka:5.4.1
     hostname: kafka3
     ports:
       - "9094:9094"

zk-single-kafka-single.yml

@@ -15,7 +15,7 @@ services:
       - ./zk-single-kafka-single/zoo1/datalog:/datalog
 
   kafka1:
-    image: confluentinc/cp-kafka:5.1.0
+    image: confluentinc/cp-kafka:5.4.1
     hostname: kafka1
     ports:
       - "9092:9092"