From 4ccb27fb166ab2ad07588d1848010c8aa23c8bf9 Mon Sep 17 00:00:00 2001 From: hifly81 Date: Thu, 12 Dec 2024 15:45:58 +0100 Subject: [PATCH] added profile clusterlinking in devtoolkit - moved away from cp-demo - #261 --- README.md | 1 + .../docker-compose.clusterlinking.yaml | 38 +++++++++ dev-toolkit/start.sh | 40 ++++++++++ dev-toolkit/stop.sh | 1 + jmxexporter-prometheus-grafana/README.md | 26 ++---- .../docker-compose.override.yaml | 57 ------------- .../cluster-linking/start.sh | 79 ------------------- .../cluster-linking/stop.sh | 17 ---- utils/setup_cluster-linking_demo.sh | 24 ------ 9 files changed, 85 insertions(+), 198 deletions(-) create mode 100644 dev-toolkit/docker-compose.clusterlinking.yaml delete mode 100644 jmxexporter-prometheus-grafana/cluster-linking/docker-compose.override.yaml delete mode 100755 jmxexporter-prometheus-grafana/cluster-linking/start.sh delete mode 100755 jmxexporter-prometheus-grafana/cluster-linking/stop.sh delete mode 100644 utils/setup_cluster-linking_demo.sh diff --git a/README.md b/README.md index 588eb237..eb0d2b6c 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,7 @@ Currently supported profiles: - _consumer_: it will add a demo application implemented with Spring with full client metrics - _consumer-minimal_: it will add a demo application implemented with Spring with a limited number of client metrics - _jr_: it will add [JR](https://jrnd.io/) to generate random traffic for kafka. +- _clusterlinking_: add Cluster Linking between _kafka1-kafka2-kafka3-kafka4_ and a new cluster with 1 broker _broker-dest_ ## DEV-toolkit FAQ diff --git a/dev-toolkit/docker-compose.clusterlinking.yaml b/dev-toolkit/docker-compose.clusterlinking.yaml new file mode 100644 index 00000000..99038e83 --- /dev/null +++ b/dev-toolkit/docker-compose.clusterlinking.yaml @@ -0,0 +1,38 @@ +services: + broker-replicator-dst: + image: confluentinc/cp-server:${CFLT_TAG} + hostname: broker-replicator-dst + container_name: broker-replicator-dst + profiles: + - clusterlinking + environment: + KAFKA_NODE_ID: 1 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://broker-replicator-dst:29092" + KAFKA_DEFAULT_REPLICATION_FACTOR: 1 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_CONFLUENT_METADATA_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_CONFLUENT_SECURITY_EVENT_LOGGER_EXPORTER_KAFKA_TOPIC_REPLICAS: 1 + KAFKA_CONFLUENT_CLUSTER_LINK_METADATA_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_CONFLUENT_REPORTER_RESULT_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_CONFLUENT_BALANCER_ENABLE: false + KAFKA_AUTO_CREATE_TOPICS: true + KAFKA_PROCESS_ROLES: "broker,controller" + KAFKA_CONTROLLER_QUORUM_VOTERS: "1@broker-replicator-dst:29093" + KAFKA_LISTENERS: "PLAINTEXT://broker-replicator-dst:29092,CONTROLLER://broker-replicator-dst:29093" + KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT" + KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER" + KAFKA_LOG_DIRS: "/tmp/kraft-combined-logs" + KAFKA_LOG4J_LOGGERS: "org.apache.kafka.image.loader.MetadataLoader=WARN" + CLUSTER_ID: "N3Ept9vxTMijB5S74gkfPA" + KAFKA_CONFLUENT_HTTP_SERVER_LISTENERS: http://0.0.0.0:8090 + KAFKA_REST_BOOTSTRAP_SERVERS: "broker-replicator-dst:29092" + KAFKA_OPTS: "-javaagent:/tmp/jmx_prometheus_javaagent-1.1.0.jar=1234:/tmp/kafka_config.yml" + volumes: + - $PWD/jmx-exporter/jmx_prometheus_javaagent-1.1.0.jar:/tmp/jmx_prometheus_javaagent-1.1.0.jar + - $PWD/jmx-exporter/kafka_broker.yml:/tmp/kafka_config.yml diff --git a/dev-toolkit/start.sh b/dev-toolkit/start.sh index f493e9a9..b4818e45 100755 --- a/dev-toolkit/start.sh +++ b/dev-toolkit/start.sh @@ -99,6 +99,23 @@ cat <>assets/prometheus/prometheus-config/prometheus.yml replacement: '${1}' EOF +# ADD Brokers dest cluster monitoring to prometheus config +cat <>assets/prometheus/prometheus-config/prometheus.yml + + - job_name: "kafka-broker-dest" + static_configs: + - targets: + - "broker-replicator-dst:1234" + labels: + env: "dev" + job: "kafka-broker" + relabel_configs: + - source_labels: [__address__] + target_label: hostname + regex: '([^:]+)(:[0-9]+)?' + replacement: '${1}' +EOF + # ADD Schema Registry monitoring to prometheus config (default was for 1 SR only) cat <>assets/prometheus/prometheus-config/prometheus.yml @@ -128,6 +145,7 @@ $DOCKER_COMPOSE_CMD ${docker_args[@]} \ -f docker-compose.consumer-minimal.yaml \ -f docker-compose.schema-registry-primary-secondary.yaml \ -f docker-compose.jr.yaml \ + -f docker-compose.clusterlinking.yaml \ up -d # if docker_args contains replicator, then start the replicator @@ -168,6 +186,28 @@ if [[ " ${docker_args[@]} " =~ " ksqldb " ]]; then fi +# if docker_args contains clusterlinking, then start the clusterlinking +if [[ " ${docker_args[@]} " =~ " clusterlinking " ]]; then + + echo -e "\nWaiting 45 seconds before starting clusterlinking..." + sleep 45 + + docker exec kafka1 bash -c "KAFKA_OPTS= kafka-topics --bootstrap-server kafka1:29092 --create --topic product --replication-factor 1 --partitions 1" + + echo -e "Create link main-to-disaster-cl" + + docker exec broker-replicator-dst bash -c '\ + echo "\ + bootstrap.servers=kafka1:29092 + " > /home/appuser/cl.properties' + + docker exec broker-replicator-dst bash -c "KAFKA_OPTS= kafka-cluster-links --bootstrap-server broker-replicator-dst:29092 --create --link main-to-disaster-cl --config-file /home/appuser/cl.properties" + + docker exec broker-replicator-dst bash -c "KAFKA_OPTS= kafka-mirrors --create --source-topic product --mirror-topic product --link main-to-disaster-cl --bootstrap-server broker-replicator-dst:29092" + + +fi + echo -e "\ndev-toolkit started!" # Look at Prometheus metrics diff --git a/dev-toolkit/stop.sh b/dev-toolkit/stop.sh index b2d78d93..e33c9a2c 100755 --- a/dev-toolkit/stop.sh +++ b/dev-toolkit/stop.sh @@ -25,6 +25,7 @@ $DOCKER_COMPOSE_CMD \ -f docker-compose.consumer-minimal.yaml \ -f docker-compose.schema-registry-primary-secondary.yaml \ -f docker-compose.jr.yaml \ + -f docker-compose.clusterlinking.yaml \ down -v rm -rf jmx-exporter rm -rf assets diff --git a/jmxexporter-prometheus-grafana/README.md b/jmxexporter-prometheus-grafana/README.md index 276de5dd..2dc1c7c1 100644 --- a/jmxexporter-prometheus-grafana/README.md +++ b/jmxexporter-prometheus-grafana/README.md @@ -174,29 +174,13 @@ To test: ### Cluster Linking -Demo is based on https://github.com/confluentinc/demo-scene/tree/master/cluster-linking-disaster-recovery +To test use dev-toolkit with _clusterlinking_ profile: -To test follow the next steps: - -1. Set env: - ```bash -MONITORING_STACK=jmxexporter-prometheus-grafana - ``` -2. Clone demo cluster linking disaster recovery from confluentinc/demo-scene: -```bash - [[ -d "clink-demo" ]] || git clone git@github.com:confluentinc/demo-scene.git clink-demo - (cd clink-demo && git fetch && git pull) - ``` -3. Start the monitoring solution with the STACK selected. This command also starts clink-demo, you do not need to start clink-demo separately. - -```bash -${MONITORING_STACK}/cluster-linking/start.sh -``` - -4. Stop the monitoring solution. This command also stops clink-demo, you do not need to stop clink-demo separately. +1. Start dev-toolkit with ```bash -${MONITORING_STACK}/cluster-linking/stop.sh +$ cd dev-toolkit +$ start.sh --profile clusterlinking ``` ![clusterlinking](img/clusterlinking.png) @@ -208,7 +192,7 @@ ${MONITORING_STACK}/cluster-linking/stop.sh ### KRaft -To test use dev-toolkit with _Default_ profile: +To test use dev-toolkit with _default_ profile: 1. Start dev-toolkit with diff --git a/jmxexporter-prometheus-grafana/cluster-linking/docker-compose.override.yaml b/jmxexporter-prometheus-grafana/cluster-linking/docker-compose.override.yaml deleted file mode 100644 index fe3ce43a..00000000 --- a/jmxexporter-prometheus-grafana/cluster-linking/docker-compose.override.yaml +++ /dev/null @@ -1,57 +0,0 @@ -services: - mainZookeeper: - volumes: - - $MONITORING_STACK/../../shared-assets/jmx-exporter/:/usr/share/jmx-exporter - environment: - EXTRA_ARGS: -javaagent:/usr/share/jmx-exporter/jmx_prometheus_javaagent-1.1.0.jar=1234:/usr/share/jmx-exporter/zookeeper.yml - - mainKafka: - volumes: - - $MONITORING_STACK/../../shared-assets/jmx-exporter/:/usr/share/jmx-exporter - environment: - EXTRA_ARGS: -javaagent:/usr/share/jmx-exporter/jmx_prometheus_javaagent-1.1.0.jar=1234:/usr/share/jmx-exporter/kafka_broker.yml - - disasterZookeeper: - volumes: - - $MONITORING_STACK/../../shared-assets/jmx-exporter/:/usr/share/jmx-exporter - environment: - EXTRA_ARGS: -javaagent:/usr/share/jmx-exporter/jmx_prometheus_javaagent-1.1.0.jar=1234:/usr/share/jmx-exporter/zookeeper.yml - - disasterKafka: - volumes: - - $MONITORING_STACK/../../shared-assets/jmx-exporter/:/usr/share/jmx-exporter - environment: - EXTRA_ARGS: -javaagent:/usr/share/jmx-exporter/jmx_prometheus_javaagent-1.1.0.jar=1234:/usr/share/jmx-exporter/kafka_broker.yml - - prometheus: - image: prom/prometheus:v3.0.0 - container_name: prometheus - ports: - - 9090:9090 - volumes: - - $MONITORING_STACK/../assets/prometheus/prometheus-config-cluster-linking/:/etc/prometheus - - node-exporter: - image: prom/node-exporter:v1.2.2 - container_name: node-exporter - volumes: - - /proc:/host/proc:ro - - /sys:/host/sys:ro - - /:/rootfs:ro - command: - - "--path.procfs=/host/proc" - - "--path.sysfs=/host/sys" - - "--collector.filesystem.ignored-mount-points" - - "^(aufs|proc|nsfs|shm|cgroup|tmpfs|binfmt_misc|debugfs|devpts|fusectl|hugetlbfs|fuse.lxcfs|mqueue|pstore|securityfs|sysfs|autofs|devtmpfs|configfs)" - - grafana: - image: grafana/grafana:10.2.0 - container_name: grafana - environment: - - "GF_SECURITY_ADMIN_USER=admin" - - "GF_SECURITY_ADMIN_PASSWORD=password" - - "GF_USERS_ALLOW_SIGN_UP=false" - ports: - - 3000:3000 - volumes: - - $MONITORING_STACK/../assets/grafana/provisioning/import:/etc/grafana/provisioning diff --git a/jmxexporter-prometheus-grafana/cluster-linking/start.sh b/jmxexporter-prometheus-grafana/cluster-linking/start.sh deleted file mode 100755 index ccf3df5a..00000000 --- a/jmxexporter-prometheus-grafana/cluster-linking/start.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash - -######################################## -# Start demo-scene - cluster linking-disaster-recovery -######################################## - -export MONITORING_STACK="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -export OVERRIDE_PATH="${MONITORING_STACK}/docker-compose.override.yaml" - - -echo "Using ${OVERRIDE_PATH} for docker-compose override" - -. $MONITORING_STACK/../../utils/setup_cluster-linking_demo.sh - -echo -e "Launch clink-demo in CLINK_DEMO_HOME monitoring stack in $MONITORING_STACK" -(cd $CLINK_DEMO_HOME && docker-compose up -d mainZookeeper mainKafka disasterZookeeper disasterKafka) - -echo -e "Wait 30 seconds to start up..." - -######################################## -# Create resources -######################################## - -sleep 30 - -docker exec mainKafka kafka-topics --bootstrap-server mainKafka:19092 --topic product --create --partitions 1 --replication-factor 1 - -echo -e "Create link main-to-disaster-cl" - -docker exec disasterKafka bash -c '\ -echo "\ -bootstrap.servers=mainKafka:19092 -" > /home/appuser/cl.properties' - -docker exec disasterKafka kafka-cluster-links --bootstrap-server disasterKafka:29092 --create --link main-to-disaster-cl --config-file /home/appuser/cl.properties - -docker exec disasterKafka kafka-mirrors --create --source-topic product --mirror-topic product --link main-to-disaster-cl --bootstrap-server disasterKafka:29092 - -echo -e "Verify link main-to-disaster-cl" - -docker exec disasterKafka kafka-cluster-links --bootstrap-server disasterKafka:29092 --link main-to-disaster-cl --list - -echo -e "Create dashboards for auto import in grafana" -GRAFANA_IMPORT_FOLDER=${MONITORING_STACK}/../assets/grafana/provisioning/import -mkdir -p ${GRAFANA_IMPORT_FOLDER}/dashboards ${GRAFANA_IMPORT_FOLDER}/datasources -cp -rf ${MONITORING_STACK}/../assets/grafana/provisioning/datasources/datasource.yml ${GRAFANA_IMPORT_FOLDER}/datasources/datasource.yml -cp -rf ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/dashboard.yml ${GRAFANA_IMPORT_FOLDER}/dashboards/dashboard.yml -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/cluster-linking.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/cluster-linking.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/confluent-oracle-cdc.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/confluent-oracle-cdc.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/confluent-platform.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/confluent-platform.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/confluent-rbac.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/confluent-rbac.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/rest-proxy.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/rest-proxy.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/kafka-cluster.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/kafka-cluster.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/kafka-connect-cluster.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/kafka-connect-cluster.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/kafka-consumer.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/kafka-consumer.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/kafka-lag-exporter.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/kafka-lag-exporter.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/kafka-producer.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/kafka-producer.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/kafka-quotas.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/kafka-quotas.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/kafka-stream.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/kafka-stream.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/kafka-streams-rocksdb.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/kafka-streams-rocksdb.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/kafka-topics.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/kafka-topics.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/kafka-transaction-coordinator.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/kafka-transaction-coordinator.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/kraft.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/kraft.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/ksqldb-cluster.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/ksqldb-cluster.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/replicator.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/replicator.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/schema-registry-cluster.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/schema-registry-cluster.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/tiered-storage.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/tiered-storage.json -sed 's/${Prometheus}/Prometheus/g' ${MONITORING_STACK}/../assets/grafana/provisioning/dashboards/zookeeper-cluster.json > ${GRAFANA_IMPORT_FOLDER}/dashboards/zookeeper-cluster.json - - -######################################## -# Start monitoring solution -######################################## - -echo -e "Launch $MONITORING_STACK" -docker-compose up -d prometheus node-exporter grafana - -echo -e "\nView Grafana dashboards at (admin/password) ->" -echo -e "http://localhost:3000" diff --git a/jmxexporter-prometheus-grafana/cluster-linking/stop.sh b/jmxexporter-prometheus-grafana/cluster-linking/stop.sh deleted file mode 100755 index e64f0c63..00000000 --- a/jmxexporter-prometheus-grafana/cluster-linking/stop.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -######################################## -# Stop demo-scene - cluster linking-disaster-recovery -######################################## - -export MONITORING_STACK="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -. $MONITORING_STACK/../../utils/setup_cluster-linking_demo.sh - - -echo -e "Stop clink-demo in $CLINK_DEMO_HOME and monitoring stack in $MONITORING_STACK" - -(cd $CLINK_DEMO_HOME && docker-compose down --volumes) - -GRAFANA_IMPORT_FOLDER=${MONITORING_STACK}/../assets/grafana/provisioning/import -rm -rf ${GRAFANA_IMPORT_FOLDER} \ No newline at end of file diff --git a/utils/setup_cluster-linking_demo.sh b/utils/setup_cluster-linking_demo.sh deleted file mode 100644 index 2ab935a3..00000000 --- a/utils/setup_cluster-linking_demo.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# This method does a job similar to realpath, but avoid that extra dependency. -# It returns fullpath for a given parameter to avoind using relative paths. -fullpath () -{ - fullpath=$(cd $1 && pwd -P) - echo $fullpath - cd $OLDPWD -} - -DEFAULT_CLINK_DEMO_HOME=$(fullpath ${MONITORING_STACK}/../../../clink-demo) -CLINK_DEMO_HOME=${DEFAULT_CLINK_DEMO_HOME}/cluster-linking-disaster-recovery - -[ -d "${CLINK_DEMO_HOME}" ] || { - echo "ERROR: ${CLINK_DEMO_HOME} does not exist. Have you cloned https://github.com/confluentinc/demo-scene? If demo-scene is not in ${CLINK_DEMO_HOME}, you can set CLINK_DEMO_HOME and try again." - exit 1 -} - -if [[ -z "${OVERRIDE_PATH}" ]]; then - OVERRIDE_PATH="${MONITORING_STACK}/docker-compose.override.yaml" -fi - -export COMPOSE_FILE="${CLINK_DEMO_HOME}/docker-compose.yml:${OVERRIDE_PATH}"