Added docker image with 2-node Kafka cluster

README.md

@@ -52,6 +52,19 @@ docker run -p 2181:2181 -p 9092:9092 \
     spotify/kafkaproxy
 ```
 
+Running 2-node cluster
+----------------------
+
+Both brokers are running in the same container on ports `9092` and `9093`.
+
+```bash
+docker build -t kafkacluster kafkacluster/
+docker run -p 2181:2181 -p 9092:9092 -p 9093:9093 --env ADVERTISED_HOST=`boot2docker ip` kafkacluster
+```
+
+Note that you must provide proper `ADVERTISED_HOST` env variable. However avoid
+setting `ADVERTISED_PORT`, since it may not play well with current 2-node setup.
+
 In the box
 ---
 * **spotify/kafka**
@@ -64,6 +77,10 @@ In the box
   The docker image with Kafka, Zookeeper and a Kafka 7 proxy that can be
   configured with a set of topics to mirror.
 
+  * **spotify/kafkacluster**
+
+    The docker image with cluster of 2 Kafka brokers and Zookeeper.
+
 Public Builds
 ---
 
@@ -76,10 +93,10 @@ Build from Source
 
     docker build -t spotify/kafka kafka/
     docker build -t spotify/kafkaproxy kafkaproxy/
+    docker build -t spotify/kafkacluster kafkacluster/
 
 Todo
 ---
 
 * Not particularily optimzed for startup time.
 * Better docs
-

kafkacluster/Dockerfile

@@ -0,0 +1,16 @@
+# Two-node Kafka Cluster and Zookeeper
+
+FROM spotify/kafka
+
+RUN ["rm", "-f", "/usr/bin/start-kafka.sh"]
+ADD scripts/start-kafka.sh /usr/bin/start-kafka.sh
+RUN ["chmod", "+x", "/usr/bin/start-kafka.sh"]
+
+# Supervisor config
+RUN ["rm", "-f", "/etc/supervisor/conf.d/kafka.conf"]
+ADD supervisor/kafka1.conf supervisor/kafka2.conf /etc/supervisor/conf.d/
+
+# 2181 is zookeeper, 9092, 9093 is two kafka brokers
+EXPOSE 2181 9092 9093
+
+CMD ["supervisord", "-n"]

kafkacluster/scripts/start-kafka.sh

@@ -0,0 +1,73 @@
+#!/bin/sh
+
+# Optional ENV variables:
+# * ADVERTISED_HOST: the external ip for the container, e.g. `boot2docker ip`
+# * ADVERTISED_PORT: the external port for Kafka, e.g. 9092
+# * ZK_CHROOT: the zookeeper chroot that's used by Kafka (without / prefix), e.g. "kafka"
+# * LOG_RETENTION_HOURS: the minimum age of a log file in hours to be eligible for deletion (default is 168, for 1 week)
+# * LOG_RETENTION_BYTES: configure the size at which segments are pruned from the log, (default is 1073741824, for 1GB)
+# * NUM_PARTITIONS: configure the default number of log partitions per topic
+
+BROKER_ID=$1
+BROKER_PORT=$2
+
+cp $KAFKA_HOME/config/server.properties $KAFKA_HOME/config/server$BROKER_ID.properties
+
+# Configure advertised host/port if we run in helios
+if [ ! -z "$HELIOS_PORT_kafka" ]; then
+    ADVERTISED_HOST=`echo $HELIOS_PORT_kafka | cut -d':' -f 1 | xargs -n 1 dig +short | tail -n 1`
+    ADVERTISED_PORT=`echo $HELIOS_PORT_kafka | cut -d':' -f 2`
+fi
+
+# Set the external host and port
+if [ ! -z "$ADVERTISED_HOST" ]; then
+    echo "advertised host: $ADVERTISED_HOST"
+    sed -r -i "s/#(advertised.host.name)=(.*)/\1=$ADVERTISED_HOST/g" $KAFKA_HOME/config/server$BROKER_ID.properties
+fi
+if [ ! -z "$ADVERTISED_PORT" ]; then
+    echo "advertised port: $ADVERTISED_PORT"
+    sed -r -i "s/#(advertised.port)=(.*)/\1=$ADVERTISED_PORT/g" $KAFKA_HOME/config/server$BROKER_ID.properties
+fi
+
+# Set the zookeeper chroot
+if [ ! -z "$ZK_CHROOT" ]; then
+    # wait for zookeeper to start up
+    until /usr/share/zookeeper/bin/zkServer.sh status; do
+      sleep 0.1
+    done
+
+    # create the chroot node
+    echo "create /$ZK_CHROOT \"\"" | /usr/share/zookeeper/bin/zkCli.sh || {
+        echo "can't create chroot in zookeeper, exit"
+        exit 1
+    }
+
+    # configure kafka
+    sed -r -i "s/(zookeeper.connect)=(.*)/\1=localhost:2181\/$ZK_CHROOT/g" $KAFKA_HOME/config/server$BROKER_ID.properties
+fi
+
+# Allow specification of log retention policies
+if [ ! -z "$LOG_RETENTION_HOURS" ]; then
+    echo "log retention hours: $LOG_RETENTION_HOURS"
+    sed -r -i "s/(log.retention.hours)=(.*)/\1=$LOG_RETENTION_HOURS/g" $KAFKA_HOME/config/server$BROKER_ID.properties
+fi
+if [ ! -z "$LOG_RETENTION_BYTES" ]; then
+    echo "log retention bytes: $LOG_RETENTION_BYTES"
+    sed -r -i "s/#(log.retention.bytes)=(.*)/\1=$LOG_RETENTION_BYTES/g" $KAFKA_HOME/config/server$BROKER_ID.properties
+fi
+
+# Configure the default number of log partitions per topic
+if [ ! -z "$NUM_PARTITIONS" ]; then
+    echo "default number of partition: $NUM_PARTITIONS"
+    sed -r -i "s/(num.partitions)=(.*)/\1=$NUM_PARTITIONS/g" $KAFKA_HOME/config/server$BROKER_ID.properties
+fi
+
+# Configure broker ID
+sed -r -i "s/^(broker\.id)=(.*)/\1=$BROKER_ID/g" $KAFKA_HOME/config/server$BROKER_ID.properties
+# Configure log directory
+sed -r -i "s/(log.dirs)=(.*)/\1=\/tmp\/kafka-logs-$BROKER_ID/g" $KAFKA_HOME/config/server$BROKER_ID.properties
+# Configure port
+sed -r -i "s/^(port)=(.*)/\1=$BROKER_PORT/g" $KAFKA_HOME/config/server$BROKER_ID.properties
+
+# Run Kafka
+$KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server$BROKER_ID.properties

kafkacluster/supervisor/kafka1.conf

@@ -0,0 +1,4 @@
+[program:kafka1]
+command=/usr/bin/start-kafka.sh 1 9092
+autostart=true
+autorestart=true

kafkacluster/supervisor/kafka2.conf

@@ -0,0 +1,4 @@
+[program:kafka2]
+command=/usr/bin/start-kafka.sh 2 9093
+autostart=true
+autorestart=true

kafkacluster/supervisor/zookeeper.conf

@@ -0,0 +1,4 @@
+[program:zookeeper]
+command=/usr/share/zookeeper/bin/zkServer.sh start-foreground
+autostart=true
+autorestart=true