one of many

Signed-off-by: Hermann Mayer <[email protected]>

test/0.0/test.broker-list.kafka.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+
+testBrokerList() {
+	# Need to get the proxied ports for kafka
+	PORT1=$(docker inspect -f '{{ index .NetworkSettings.Ports "9092/tcp" 0 "HostPort" }}' test_kafka_1)
+	PORT2=$(docker inspect -f '{{ index .NetworkSettings.Ports "9092/tcp" 0 "HostPort" }}' test_kafka_2)
+
+	RESULT=$(HOST_IP=1.2.3.4 broker-list.sh)
+
+	echo "$RESULT"
+
+	if [[ "$RESULT" == "1.2.3.4:$PORT1,1.2.3.4:$PORT2" || "$RESULT" == "1.2.3.4:$PORT2,1.2.3.4:$PORT1" ]]; then
+		return 0
+	else
+		return 1
+	fi
+}
+
+testBrokerList

test/0.0/test.create-topics-custom-separator.kafka.sh

@@ -0,0 +1,55 @@
+#!/bin/bash -e
+
+# NOTE: create-topics.sh requires KAFKA_PORT and KAFKA_ZOOKEEPER_CONNECT to be set (see docker-compose.yml)
+testCreateTopicsCustomSeparator() {
+	NOW=$(date +%s)
+
+	# TOPICS array contains the topic name to create / validate
+	TOPICS[0]="one-$NOW"
+	TOPICS[1]="two-$NOW"
+	TOPICS[2]="three-$NOW"
+
+	export KAFKA_CREATE_TOPICS_SEPARATOR=$'\n'
+	KAFKA_CREATE_TOPICS=$(cat <<-EOF
+		${TOPICS[0]}:1:1
+		${TOPICS[1]}:1:1
+		${TOPICS[2]}:1:1
+	EOF
+	)
+	export KAFKA_CREATE_TOPICS
+
+	create-topics.sh
+
+	# shellcheck disable=SC1091
+	source "/usr/bin/versions.sh"
+
+	# since 3.0.0 there is no --zookeeper option anymore, so we have to use the
+	# --bootstrap-server option with a random broker
+	if [[ "$MAJOR_VERSION" -ge "3" ]]; then
+		CONNECT_OPTS="--bootstrap-server $(echo "${BROKER_LIST}" | cut -d ',' -f1)"
+	else
+		CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}"
+	fi
+
+	# Loop through each array, validate that topic exists
+	for i in "${!TOPICS[@]}"; do
+		TOPIC=${TOPICS[i]}
+
+		echo "Validating topic '$TOPIC'"
+
+		# shellcheck disable=SC2086
+		EXISTS=$(/opt/kafka/bin/kafka-topics.sh ${CONNECT_OPTS} --list --topic "$TOPIC")
+		if [[ "$EXISTS" != "$TOPIC" ]]; then
+			echo "$TOPIC topic not created"
+			return 1
+		fi
+	done
+
+	return 0
+}
+
+# mock the netstat call as made by the create-topics.sh script
+function netstat() { echo "1 2 3 :$KAFKA_PORT"; }
+export -f netstat
+
+testCreateTopicsCustomSeparator

test/0.0/test.path.kafka.sh

@@ -0,0 +1,15 @@
+#!/bin/bash -e
+
+# NOTE: this tests to see if the /opt/kafka/bin is existing in the path within the docker container
+
+testPath() {
+	echo "Checking PATH '$PATH'"
+	if [[ ! "$PATH" =~ "/opt/kafka/bin" ]]; then
+		echo "path is not set correctly: $PATH"
+		return 1
+	fi
+
+	return 0
+}
+
+testPath

test/0.0/test.read-write.kafkacat.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -e
+
+source version.functions
+
+testReadWrite() {
+	echo 'foo,bar' | eval "kafkacat -b $BROKER_LIST $KAFKACAT_OPTS -P -D, -t readwrite"
+	eval "kafkacat -b $BROKER_LIST $KAFKACAT_OPTS -C -e -t readwrite"
+}
+
+testReadWrite

test/0.0/test.start-kafka-advertised-host.kafka.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -e
+
+source test.functions
+
+testAdvertisedHost() {
+	# Given a hostname is provided
+	export KAFKA_ADVERTISED_HOST_NAME=monkey
+	export KAFKA_ADVERTISED_PORT=8888
+
+	# When the script is invoked
+	source "$START_KAFKA"
+
+	# Then the configuration file is correct
+	assertExpectedConfig "advertised.host.name=monkey"
+	assertExpectedConfig "advertised.port=8888"
+	assertAbsent 'advertised.listeners'
+	assertAbsent 'listeners'
+}
+
+testAdvertisedHost

test/0.0/test.start-kafka-broker-id.kafka.sh

@@ -0,0 +1,51 @@
+#!/bin/bash -e
+
+source test.functions
+
+testManualBrokerId() {
+	echo "testManualBrokerId"
+
+	# Given a Broker Id is provided
+	export KAFKA_LISTENERS=PLAINTEXT://:9092
+	export KAFKA_BROKER_ID=57
+
+	# When the script is invoked
+	source "$START_KAFKA"
+
+	# Then the broker Id is set
+	assertExpectedConfig 'broker.id=57'
+}
+
+testAutomaticBrokerId() {
+	echo "testAutomaticBrokerId"
+
+	# Given no Broker Id is provided
+	export KAFKA_LISTENERS=PLAINTEXT://:9092
+	unset KAFKA_BROKER_ID
+
+	# When the script is invoked
+	source "$START_KAFKA"
+
+	# Then the broker Id is configured to automatic
+	assertExpectedConfig 'broker.id=-1'
+}
+
+testBrokerIdCommand() {
+	echo "testBrokerIdCommand"
+
+	# Given a Broker Id command is provided
+	export KAFKA_LISTENERS=PLAINTEXT://:9092
+	unset KAFKA_BROKER_ID
+	export BROKER_ID_COMMAND='f() { echo "23"; }; f'
+
+	# When the script is invoked
+	source "$START_KAFKA"
+
+	# Then the broker Id is the result of the command
+	assertExpectedConfig 'broker.id=23'
+}
+
+
+testManualBrokerId \
+ && testAutomaticBrokerId \
+ && testBrokerIdCommand

test/0.0/test.start-kafka-bug-312-kafka-env.kafka.sh

@@ -0,0 +1,25 @@
+#!/bin/bash -e
+
+source test.functions
+
+testKafkaEnv() {
+    # Given required settings are provided
+    export KAFKA_ADVERTISED_HOST_NAME="testhost"
+    export KAFKA_OPTS="-Djava.security.auth.login.config=/kafka_server_jaas.conf"
+
+    # When the script is invoked
+    source "$START_KAFKA"
+
+    # Then env should remain untouched
+    if [[ ! "$KAFKA_OPTS" == "-Djava.security.auth.login.config=/kafka_server_jaas.conf" ]]; then
+        echo "KAFKA_OPTS not set to expected value. $KAFKA_OPTS"
+        exit 1
+    fi
+
+    # And the broker config should not be set
+    assertAbsent 'opts'
+
+    echo " > Set KAFKA_OPTS=$KAFKA_OPTS"
+}
+
+testKafkaEnv

test/0.0/test.start-kafka-bug-313-kafka-opts.kafka.sh

@@ -0,0 +1,23 @@
+#!/bin/bash -e
+
+source test.functions
+
+testKafkaOpts() {
+    # Given required settings are provided
+    export KAFKA_ADVERTISED_HOST_NAME="testhost"
+    # .. and a CUSTOM_INIT_SCRIPT with spaces
+    export CUSTOM_INIT_SCRIPT="export KAFKA_OPTS=-Djava.security.auth.login.config=/kafka_server_jaas.conf"
+
+    # When the script is invoked
+    source "$START_KAFKA"
+
+    # Then the custom init script should be evaluated
+    if [[ ! "$KAFKA_OPTS" == "-Djava.security.auth.login.config=/kafka_server_jaas.conf" ]]; then
+        echo "KAFKA_OPTS not set to expected value. $KAFKA_OPTS"
+        exit 1
+    fi
+
+    echo "  > Set KAFKA_OPTS=$KAFKA_OPTS"
+}
+
+testKafkaOpts

test/0.0/test.start-kafka-host-name.kafka.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+
+source test.functions
+
+testHostnameCommand() {
+	# Given a hostname command is provided
+	export HOSTNAME_COMMAND='f() { echo "my-host"; }; f'
+
+	# When the script is invoked
+	source "$START_KAFKA"
+
+	# Then the configuration uses the value from the command
+	assertExpectedConfig 'advertised.host.name=my-host'
+	assertAbsent 'advertised.listeners'
+	assertAbsent 'listeners'
+}
+
+testHostnameCommand

test/0.0/test.start-kafka-log4j-config.kafka.sh

@@ -0,0 +1,17 @@
+#!/bin/bash -e
+
+source test.functions
+
+testLog4jConfig() {
+	# Given Log4j overrides are provided
+	export KAFKA_ADVERTISED_HOST_NAME="testhost"
+	export LOG4J_LOGGER_KAFKA=DEBUG
+
+	# When the script is invoked
+	source "$START_KAFKA"
+
+	# Then the configuration file is correct
+	assertExpectedLog4jConfig "log4j.logger.kafka=DEBUG"
+}
+
+testLog4jConfig

test/0.0/test.start-kafka-port-command.kafka.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -e
+
+source test.functions
+
+testPortCommand() {
+	# Given a port command is provided
+	export PORT_COMMAND='f() { echo "12345"; }; f'
+	export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://1.2.3.4:_{PORT_COMMAND}"
+	export KAFKA_LISTENERS="PLAINTEXT://:9092"
+
+	# When the script is invoked
+	source "$START_KAFKA"
+
+	# Then the configuration uses the value from the command
+	assertExpectedConfig 'advertised.listeners=PLAINTEXT://1.2.3.4:12345'
+	assertExpectedConfig 'listeners=PLAINTEXT://:9092'
+	assertAbsent 'advertised.host.name'
+	assertAbsent 'advertised.port'
+}
+
+testPortCommand

test/0.0/test.start-kafka-restart.kafka.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+
+source test.functions
+
+testRestart() {
+	# Given a hostname is provided
+	export KAFKA_ADVERTISED_HOST_NAME="testhost"
+
+	# When the container is restarted (Script invoked multiple times)
+	source "$START_KAFKA"
+	source "$START_KAFKA"
+
+	# Then the configuration file only has one instance of the config
+	assertExpectedConfig 'advertised.host.name=testhost'
+	assertAbsent 'listeners'
+}
+
+testRestart

test/0.10/test.create-topics.kafka.sh

@@ -8,10 +8,10 @@ testCreateTopics() {
 	# TOPICS array contains the topic name to create / validate
 	# CLEANUP array contains the expected cleanup policy configuration for the topic
 	TOPICS[0]="default-$NOW"
-	CLEANUP[0]=""
+	CONFIG[0]=""
 
 	TOPICS[1]="compact-$NOW"
-	CLEANUP[1]="compact,compression.type=snappy"
+	CONFIG[1]="cleanup.policy=compact,compression.type=snappy"
 
 	KAFKA_CREATE_TOPICS="${TOPICS[0]}:1:1,${TOPICS[1]}:2:1:compact --config=compression.type=snappy" create-topics.sh
 
@@ -26,27 +26,29 @@ testCreateTopics() {
 		CONNECT_OPTS="--zookeeper ${KAFKA_ZOOKEEPER_CONNECT}"
 	fi
 
-	# Loop through each array, validate that topic exists, and correct cleanup policy is set
+	# Loop through each array, validate that topic exists, and correct configuration is set
 	for i in "${!TOPICS[@]}"; do
 		TOPIC=${TOPICS[i]}
 
 		echo "Validating topic '$TOPIC'"
 
 		# shellcheck disable=SC2086
 		EXISTS=$(/opt/kafka/bin/kafka-topics.sh ${CONNECT_OPTS} --list --topic "$TOPIC")
-
-        echo '-----debug'
-        /opt/kafka/bin/kafka-configs.sh ${CONNECT_OPTS} --entity-type topics --entity-name "$TOPIC" --describe
-        echo '-----debug'
-
 		# shellcheck disable=SC2086
-		POLICY=$(/opt/kafka/bin/kafka-configs.sh ${CONNECT_OPTS} --entity-type topics --entity-name "$TOPIC" --describe | grep 'Configs for topic' | awk -F'cleanup.policy=' '{print $2}')
+		ACTUAL_CONFIG=$(/opt/kafka/bin/kafka-configs.sh ${CONNECT_OPTS} --entity-type topics --entity-name "$TOPIC" --describe \
+			| cut -d'{' -f1 \
+			| grep -oE '(compression.type|cleanup.policy)=([^ ,]+)' \
+			| sort \
+			| tr '\n' ',' \
+			| sed 's/,$//')
 
-		RESULT="$EXISTS:$POLICY"
-		EXPECTED="$TOPIC:${CLEANUP[i]}"
+		RESULT="$EXISTS:$ACTUAL_CONFIG"
+		EXPECTED="$TOPIC:${CONFIG[i]}"
 
 		if [[ "$RESULT" != "$EXPECTED" ]]; then
-			echo "$TOPIC topic not configured correctly: '$RESULT'"
+			echo "$TOPIC topic not configured correctly:"
+			echo "    Actual: '$RESULT'"
+			echo "  Expected: '$EXPECTED'"
 			return 1
 		fi
 	done

test/0.9/test.snappy.kafkacat.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -e
+
+source version.functions
+
+testSnappy() {
+	echo 'foo,bar' | eval "kafkacat -X compression.codec=snappy -b $BROKER_LIST $KAFKACAT_OPTS -P -D, -t snappy"
+	eval "kafkacat -X compression.codec=snappy -b $BROKER_LIST $KAFKACAT_OPTS -C -e -t snappy"
+}
+
+testSnappy

test/0.9/test.start-kafka-advertised-listeners.kafka.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+
+source test.functions
+
+testAdvertisedListeners() {
+	# Given a hostname is provided
+	export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://my.domain.com:9040"
+	export KAFKA_LISTENERS="PLAINTEXT://:9092"
+
+	# When the script is invoked
+	source "$START_KAFKA"
+
+	# Then the configuration file is correct
+	assertExpectedConfig 'advertised.listeners=PLAINTEXT://my.domain.com:9040'
+	assertExpectedConfig 'listeners=PLAINTEXT://:9092'
+}
+
+testAdvertisedListeners