diff --git a/README.md b/README.md index 96026a3..94cdf54 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,19 @@ docker run -p 2181:2181 -p 9092:9092 \ spotify/kafkaproxy ``` +Running 2-node cluster +---------------------- + +Both brokers are running in the same container on ports `9092` and `9093`. + +```bash +docker build -t kafkacluster kafkacluster/ +docker run -p 2181:2181 -p 9092:9092 -p 9093:9093 --env ADVERTISED_HOST=`boot2docker ip` kafkacluster +``` + +Note that you must provide proper `ADVERTISED_HOST` env variable. However avoid +setting `ADVERTISED_PORT`, since it may not play well with current 2-node setup. + In the box --- * **spotify/kafka** @@ -64,6 +77,10 @@ In the box The docker image with Kafka, Zookeeper and a Kafka 7 proxy that can be configured with a set of topics to mirror. + * **spotify/kafkacluster** + + The docker image with cluster of 2 Kafka brokers and Zookeeper. + Public Builds --- @@ -76,10 +93,10 @@ Build from Source docker build -t spotify/kafka kafka/ docker build -t spotify/kafkaproxy kafkaproxy/ + docker build -t spotify/kafkacluster kafkacluster/ Todo --- * Not particularily optimzed for startup time. * Better docs - diff --git a/kafkacluster/Dockerfile b/kafkacluster/Dockerfile new file mode 100644 index 0000000..9ac64e9 --- /dev/null +++ b/kafkacluster/Dockerfile @@ -0,0 +1,16 @@ +# Two-node Kafka Cluster and Zookeeper + +FROM spotify/kafka + +RUN ["rm", "-f", "/usr/bin/start-kafka.sh"] +ADD scripts/start-kafka.sh /usr/bin/start-kafka.sh +RUN ["chmod", "+x", "/usr/bin/start-kafka.sh"] + +# Supervisor config +RUN ["rm", "-f", "/etc/supervisor/conf.d/kafka.conf"] +ADD supervisor/kafka1.conf supervisor/kafka2.conf /etc/supervisor/conf.d/ + +# 2181 is zookeeper, 9092, 9093 is two kafka brokers +EXPOSE 2181 9092 9093 + +CMD ["supervisord", "-n"] diff --git a/kafkacluster/scripts/start-kafka.sh b/kafkacluster/scripts/start-kafka.sh new file mode 100644 index 0000000..30b6633 --- /dev/null +++ b/kafkacluster/scripts/start-kafka.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +# Optional ENV variables: +# * ADVERTISED_HOST: the external ip for the container, e.g. `boot2docker ip` +# * ADVERTISED_PORT: the external port for Kafka, e.g. 9092 +# * ZK_CHROOT: the zookeeper chroot that's used by Kafka (without / prefix), e.g. "kafka" +# * LOG_RETENTION_HOURS: the minimum age of a log file in hours to be eligible for deletion (default is 168, for 1 week) +# * LOG_RETENTION_BYTES: configure the size at which segments are pruned from the log, (default is 1073741824, for 1GB) +# * NUM_PARTITIONS: configure the default number of log partitions per topic + +BROKER_ID=$1 +BROKER_PORT=$2 + +cp $KAFKA_HOME/config/server.properties $KAFKA_HOME/config/server$BROKER_ID.properties + +# Configure advertised host/port if we run in helios +if [ ! -z "$HELIOS_PORT_kafka" ]; then + ADVERTISED_HOST=`echo $HELIOS_PORT_kafka | cut -d':' -f 1 | xargs -n 1 dig +short | tail -n 1` + ADVERTISED_PORT=`echo $HELIOS_PORT_kafka | cut -d':' -f 2` +fi + +# Set the external host and port +if [ ! -z "$ADVERTISED_HOST" ]; then + echo "advertised host: $ADVERTISED_HOST" + sed -r -i "s/#(advertised.host.name)=(.*)/\1=$ADVERTISED_HOST/g" $KAFKA_HOME/config/server$BROKER_ID.properties +fi +if [ ! -z "$ADVERTISED_PORT" ]; then + echo "advertised port: $ADVERTISED_PORT" + sed -r -i "s/#(advertised.port)=(.*)/\1=$ADVERTISED_PORT/g" $KAFKA_HOME/config/server$BROKER_ID.properties +fi + +# Set the zookeeper chroot +if [ ! -z "$ZK_CHROOT" ]; then + # wait for zookeeper to start up + until /usr/share/zookeeper/bin/zkServer.sh status; do + sleep 0.1 + done + + # create the chroot node + echo "create /$ZK_CHROOT \"\"" | /usr/share/zookeeper/bin/zkCli.sh || { + echo "can't create chroot in zookeeper, exit" + exit 1 + } + + # configure kafka + sed -r -i "s/(zookeeper.connect)=(.*)/\1=localhost:2181\/$ZK_CHROOT/g" $KAFKA_HOME/config/server$BROKER_ID.properties +fi + +# Allow specification of log retention policies +if [ ! -z "$LOG_RETENTION_HOURS" ]; then + echo "log retention hours: $LOG_RETENTION_HOURS" + sed -r -i "s/(log.retention.hours)=(.*)/\1=$LOG_RETENTION_HOURS/g" $KAFKA_HOME/config/server$BROKER_ID.properties +fi +if [ ! -z "$LOG_RETENTION_BYTES" ]; then + echo "log retention bytes: $LOG_RETENTION_BYTES" + sed -r -i "s/#(log.retention.bytes)=(.*)/\1=$LOG_RETENTION_BYTES/g" $KAFKA_HOME/config/server$BROKER_ID.properties +fi + +# Configure the default number of log partitions per topic +if [ ! -z "$NUM_PARTITIONS" ]; then + echo "default number of partition: $NUM_PARTITIONS" + sed -r -i "s/(num.partitions)=(.*)/\1=$NUM_PARTITIONS/g" $KAFKA_HOME/config/server$BROKER_ID.properties +fi + +# Configure broker ID +sed -r -i "s/^(broker\.id)=(.*)/\1=$BROKER_ID/g" $KAFKA_HOME/config/server$BROKER_ID.properties +# Configure log directory +sed -r -i "s/(log.dirs)=(.*)/\1=\/tmp\/kafka-logs-$BROKER_ID/g" $KAFKA_HOME/config/server$BROKER_ID.properties +# Configure port +sed -r -i "s/^(port)=(.*)/\1=$BROKER_PORT/g" $KAFKA_HOME/config/server$BROKER_ID.properties + +# Run Kafka +$KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server$BROKER_ID.properties diff --git a/kafkacluster/supervisor/kafka1.conf b/kafkacluster/supervisor/kafka1.conf new file mode 100644 index 0000000..6aa313c --- /dev/null +++ b/kafkacluster/supervisor/kafka1.conf @@ -0,0 +1,4 @@ +[program:kafka1] +command=/usr/bin/start-kafka.sh 1 9092 +autostart=true +autorestart=true diff --git a/kafkacluster/supervisor/kafka2.conf b/kafkacluster/supervisor/kafka2.conf new file mode 100644 index 0000000..4164874 --- /dev/null +++ b/kafkacluster/supervisor/kafka2.conf @@ -0,0 +1,4 @@ +[program:kafka2] +command=/usr/bin/start-kafka.sh 2 9093 +autostart=true +autorestart=true diff --git a/kafkacluster/supervisor/zookeeper.conf b/kafkacluster/supervisor/zookeeper.conf new file mode 100644 index 0000000..5650d95 --- /dev/null +++ b/kafkacluster/supervisor/zookeeper.conf @@ -0,0 +1,4 @@ +[program:zookeeper] +command=/usr/share/zookeeper/bin/zkServer.sh start-foreground +autostart=true +autorestart=true \ No newline at end of file