Kafka and arangodb components update

docker-compose/arangodb/.env

@@ -1,4 +1,4 @@
-ARANGO_ROOT_PASSWORD=iJJ9fWxb9Z6CS1aPagoW
+ARANGO_ROOT_PASSWORD=CH@NGE-ME!
 NETWORK_TYPE=placeholder
 INSTANCE_NAME=first
 ARANGO_SERVICE_SERVICE_HOST=arangodb

docker-compose/arangodb/build/Dockerfile

@@ -1,4 +1,4 @@
-FROM arangodb:3.7.15
+FROM arangodb:3.10.5
 
 RUN apk add bash
 RUN rm /entrypoint.sh

docker-compose/kafka/build_check/Dockerfile

@@ -1,9 +1,8 @@
-FROM confluentinc/cp-kafka-connect:5.3.6
-RUN apt update && apt -y --force-yes install jq bsdmainutils
+FROM confluentinc/cp-kafka-connect:7.3.3
+USER root
+RUN microdnf update
+RUN microdnf install jq findutils
 COPY check.sh /work/check.sh
-COPY create_topics.sh /work/create_topics.sh
-COPY create_connectors.sh /work/create_connectors.sh
-COPY connect-cli /usr/bin/connect-cli
 COPY /connectors/ /work/connectors/
 COPY entrypoint.sh /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]

docker-compose/kafka/build_check/connectors/arangodb-accounts-sink.json

@@ -0,0 +1,20 @@
+{
+    "name": "arangodb-accounts-sink",
+    "config": {
+        "connector.class": "io.github.jaredpetersen.kafkaconnectarangodb.sink.ArangoDbSinkConnector",
+        "tasks.max": "1",
+        "topics": "accounts",
+        "arangodb.host":"arangodb",
+        "arangodb.user":"root",
+        "arangodb.password":"CH@NGE-ME!",
+        "arangodb.port": "8529",
+        "arangodb.database.name": "blockchain",
+        "transforms": "KeyTransform",
+        "transforms.KeyTransform.field": "id",
+        "transforms.KeyTransform.type": "org.apache.kafka.connect.transforms.HoistField$Key",
+        "key.converter.schemas.enable": "false",
+        "key.converter": "org.apache.kafka.connect.json.JsonConverter",
+        "value.converter.schemas.enable": "false",
+        "value.converter": "org.apache.kafka.connect.json.JsonConverter"
+    }
+}

docker-compose/kafka/build_check/connectors/arangodb-blocks-signatures-sink.json

@@ -0,0 +1,20 @@
+{
+    "name": "arangodb-blocks-signatures-sink",
+    "config": {
+        "connector.class": "io.github.jaredpetersen.kafkaconnectarangodb.sink.ArangoDbSinkConnector",
+        "tasks.max": "1",
+        "topics": "blocks_signatures",
+        "arangodb.host":"arangodb",
+        "arangodb.user":"root",
+        "arangodb.password":"CH@NGE-ME!",
+        "arangodb.port": "8529",
+        "arangodb.database.name": "blockchain",
+        "transforms": "KeyTransform",
+        "transforms.KeyTransform.field": "id",
+        "transforms.KeyTransform.type": "org.apache.kafka.connect.transforms.HoistField$Key",
+        "key.converter.schemas.enable": "false",
+        "key.converter": "org.apache.kafka.connect.json.JsonConverter",
+        "value.converter.schemas.enable": "false",
+        "value.converter": "org.apache.kafka.connect.json.JsonConverter"
+    }
+}

docker-compose/kafka/build_check/connectors/arangodb-blocks-sink.json

@@ -0,0 +1,20 @@
+{
+    "name": "arangodb-blocks-sink",
+    "config": {
+        "connector.class": "io.github.jaredpetersen.kafkaconnectarangodb.sink.ArangoDbSinkConnector",
+        "tasks.max": "1",
+        "topics": "blocks",
+        "arangodb.host":"arangodb",
+        "arangodb.user":"root",
+        "arangodb.password":"CH@NGE-ME!",
+        "arangodb.port": "8529",
+        "arangodb.database.name": "blockchain",
+        "transforms": "KeyTransform",
+        "transforms.KeyTransform.field": "id",
+        "transforms.KeyTransform.type": "org.apache.kafka.connect.transforms.HoistField$Key",
+        "key.converter.schemas.enable": "false",
+        "key.converter": "org.apache.kafka.connect.json.JsonConverter",
+        "value.converter.schemas.enable": "false",
+        "value.converter": "org.apache.kafka.connect.json.JsonConverter"
+    }
+}

docker-compose/kafka/build_check/connectors/arangodb-messages-sink.json

@@ -0,0 +1,21 @@
+{
+    "name": "arangodb-messages-sink",
+    "config": {
+        "connector.class": "io.github.jaredpetersen.kafkaconnectarangodb.sink.ArangoDbSinkConnector",
+        "tasks.max": "1",
+        "topics": "messages",
+        "arangodb.host":"arangodb",
+        "arangodb.user":"root",
+        "arangodb.password":"CH@NGE-ME!",
+        "arangodb.port": "8529",
+        "arangodb.database.name": "blockchain",
+        "arangodb.insert.overwritemode": "update",
+        "transforms": "KeyTransform",
+        "transforms.KeyTransform.field": "id",
+        "transforms.KeyTransform.type": "org.apache.kafka.connect.transforms.HoistField$Key",
+        "key.converter.schemas.enable": "false",
+        "key.converter": "org.apache.kafka.connect.json.JsonConverter",
+        "value.converter.schemas.enable": "false",
+        "value.converter": "org.apache.kafka.connect.json.JsonConverter"
+    }
+}

docker-compose/kafka/build_check/connectors/arangodb-transactions-sink.json

@@ -0,0 +1,20 @@
+{
+    "name": "arangodb-transactions-sink",
+    "config": {
+        "connector.class": "io.github.jaredpetersen.kafkaconnectarangodb.sink.ArangoDbSinkConnector",
+        "tasks.max": "1",
+        "topics": "transactions",
+        "arangodb.host":"arangodb",
+        "arangodb.user":"root",
+        "arangodb.password":"CH@NGE-ME!",
+        "arangodb.port": "8529",
+        "arangodb.database.name": "blockchain",
+        "transforms": "KeyTransform",
+        "transforms.KeyTransform.field": "id",
+        "transforms.KeyTransform.type": "org.apache.kafka.connect.transforms.HoistField$Key",
+        "key.converter.schemas.enable": "false",
+        "key.converter": "org.apache.kafka.connect.json.JsonConverter",
+        "value.converter.schemas.enable": "false",
+        "value.converter": "org.apache.kafka.connect.json.JsonConverter"
+    }
+}

docker-compose/kafka/build_check/entrypoint.sh

@@ -1,7 +1,35 @@
 #!/bin/bash
 
-/work/create_topics.sh
-/work/create_connectors.sh
+kafka-topics --create --bootstrap-server kafka:29092 --topic accounts  --replication-factor 1 --partitions 20 --if-not-exists
+sleep 2
+kafka-topics --create --bootstrap-server kafka:29092 --topic transactions  --replication-factor 1 --partitions 40 --if-not-exists
+sleep 2
+kafka-topics --create --bootstrap-server kafka:29092 --topic blocks  --replication-factor 1 --partitions 5 --if-not-exists
+sleep 2
+kafka-topics --create --bootstrap-server kafka:29092 --topic messages  --replication-factor 1 --partitions 60 --if-not-exists
+sleep 2
+kafka-topics --create --bootstrap-server kafka:29092 --topic requests  --replication-factor 1 --partitions 30 --if-not-exists
+sleep 2
+kafka-topics --create --bootstrap-server kafka:29092 --topic blocks_signatures  --replication-factor 1 --partitions 5 --if-not-exists
+sleep 2
+kafka-topics --alter --bootstrap-server kafka:29092 --topic accounts --partitions 20 --if-exists
+sleep 2
+kafka-topics --alter --bootstrap-server kafka:29092 --topic transactions --partitions 40 --if-exists
+sleep 2
+kafka-topics --alter --bootstrap-server kafka:29092 --topic blocks --partitions 5 --if-exists
+sleep 2
+kafka-topics --alter --bootstrap-server kafka:29092 --topic messages --partitions 60 --if-exists
+sleep 2
+kafka-topics --alter --bootstrap-server kafka:29092 --topic requests --partitions 30 --if-exists
+sleep 2
+kafka-topics --alter --bootstrap-server kafka:29092 --topic blocks_signatures --partitions 5 --if-exists
+
+
+for connector in $(ls /work/connectors/*.json)
+do
+    curl --max-time 5 -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" connect:8083/connectors/ -d @/${connector}
+    sleep 5s
+done
 
 exec tail -f /dev/null