-
Notifications
You must be signed in to change notification settings - Fork 0
/
Makefile
173 lines (136 loc) · 6.81 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
#________________________________________________________________________________
# DEFAULT
#________________________________________________________________________________
USER = $(shell whoami)
USER_ID = $(shell id -u ${USER})
DIR = ${CURDIR}
RND ?= 12345678
NETWORK ?= rocket-bridge-${RND}
KAFKA_IP ?= 192.168.0.101
RABBIT_PORT ?= -p 5672:5672 -p 15672:15672
KAFKA_PORT ?= -p 9092:9092
ZOOKEEPER_PORT ?= -p 2181:2181 -p 2888:2888 -p 3888:3888 -p 8091:8080
ZOOKEEPER_UI_PORT ?= -p 9090:9090
default:
@echo ${NETWORK}
@echo ${CURDIR}
@echo ${USER}
@echo ${USER_ID}
@echo ${RND}
#________________________________________________________________________________
# DEVELOPMENT
#________________________________________________________________________________
.PHONY: unit now dev clean package
clean:
./gradlew clean
run:
./gradlew run
unit:
./gradlew test --tests *UnitTest*
int:
./gradlew test --tests *IntTest*
test:
./gradlew test
build:
./gradlew shadowJar
#________________________________________________________________________________
# PRODUCTION
#________________________________________________________________________________
frontend:
@echo "frontend"
cd webapp ; npm install
cd webapp ; npm run build
rm -rf flink-monitor/src/main/resources/static/*
cp -r webapp/dist/* flink-monitor/src/main/resources/static
package:
./gradlew assemble -x test -Pprod -PVERSION=${VERSION} --no-daemon
#________________________________________________________________________________
# Docker Start
#________________________________________________________________________________
.PHONY: docker-start network-start zookeeper-start kafka-start rabbit-start
.PHONY: docker-stop rabbit-stop kafka-stop zookeeper-stop network-stop
docker-start: network-start zookeeper-start kafka-start rabbit-start
docker-stop: rabbit-stop kafka-stop zookeeper-stop network-stop
#________________________________________________________________________________
# DOCKER NETWORK
# SUBNET: "172.18.0.0/16"
# GATEWAY: "172.18.0.1"
#________________________________________________________________________________
network-start:
-docker network create -d bridge ${NETWORK}
network-stop:
-docker network rm ${NETWORK}
#________________________________________________________________________________
# RUN TEST IN CONTAINER
#________________________________________________________________________________
test:
docker run --rm -it --name test-${RND} --network ${NETWORK} \
-u id -u ${USER}):$(id -g ${USER})
-e GRADLE_USER_HOME=.gradle \
-v ${DIR}:/project \
-v ~/.gradle/gradle.properties:/project/.gradle/gradle.properties \
-w /project \
openjdk:8 bash
docker stop test-${RND}
#________________________________________________________________________________
# Zookepper
# Public port 2181 2888 3888 8091 (2181 is cconnection port)
# Public port
# IP: 172.18.0.6
#________________________________________________________________________________
zookeeper-start:
-docker run -d --rm --name zookeeper-${RND} --network=${NETWORK} --network-alias=zooKeeperNet ${ZOOKEEPER_PORT} \
-e "ZOO_4LW_COMMANDS_WHITELIST=*" \
zookeeper:3.5.5
zookeeper-stop:
-docker stop zookeeper-${RND}
zoo-ui-start:
docker run -d --rm --name zkui ${ZOOKEEPER_UI_PORT} --network=${NETWORK} -e ZK_SERVER=zooKeeperNet:2181 qnib/plain-zkui
zoo-ui-stop:
docker stop zkui
#________________________________________________________________________________
# RabbitMQ
# # Public port 5672 15672
# IP: 172.18.0.7
#________________________________________________________________________________
rabbit-start:
-docker run -d --rm --name rabbitmq-${RND} --network=${NETWORK} ${RABBIT_PORT} --network-alias=rabbitNet rabbitmq:3.7.2-management
rabbit-stop:
-docker stop rabbitmq-${RND}
#________________________________________________________________________________
# Kafka (version = 2.12-2.4.0)
# Public port 9092 (bootstrapServer)
# IP: 172.18.0.8
#________________________________________________________________________________
kafka-start:
-docker run -d --rm --name kafka-${RND} --network=${NETWORK} ${KAFKA_PORT} --network-alias=kafkaNet \
-e "ADVERTISED_HOST_NAME=${KAFKA_IP}" \
-e "ZOOKEEPER_CONNECT=zookeeper-${RND}:2181" \
debezium/kafka:1.0
kafka-stop:
-docker stop kafka-${RND}
# onnect-distributed.sh kafka-configs.sh kafka-delegation-tokens.sh kafka-mirror-maker.sh kafka-run-class.sh kafka-verifiable-consumer.sh zookeeper-server-start.sh
# connect-mirror-maker.sh kafka-console-consumer.sh kafka-delete-records.sh kafka-preferred-replica-election.sh kafka-server-start.sh kafka-verifiable-producer.sh zookeeper-server-stop.sh
# connect-standalone.sh kafka-console-producer.sh kafka-dump-log.sh kafka-producer-perf-test.sh kafka-server-stop.sh trogdor.sh zookeeper-shell.sh
# kafka-acls.sh kafka-consumer-groups.sh kafka-leader-election.sh kafka-reassign-partitions.sh kafka-streams-application-reset.sh windows
# kafka-broker-api-versions.sh kafka-consumer-perf-test.sh kafka-log-dirs.sh kafka-replica-verification.sh kafka-topics.sh zookeeper-security-migration.sh
# current-offset is the last committed offset of the consumer instance
# log-end-offset is the highest offset of the partition (hence, summing this column gives you the total number of messages for the topic)
# lag is the difference between the current consumer offset and the highest offset, hence how far behind the consumer is,
# owner is the client.id of the consumer (if not specified, a default one is displayed).
test:
# docker exec -it kafka-${RND} bin/kafka-consumer-groups.sh --bootstrap-server kafkaNet:9092 --list
# docker exec -it kafka-${RND} bin/kafka-consumer-groups.sh --bootstrap-server kafkaNet:9092 --describe --group octopus
# docker exec -it kafka-${RND} bin/kafka-topics.sh --zookeeper zooKeeperNet:2181 --list
# docker exec -it kafka-${RND} bin/kafka-topics.sh --zookeeper zooKeeperNet:2181 --describe --topic test-collect_mm
# docker exec -it kafka-${RND} bin/kafka-log-dirs.sh --bootstrap-server kafkaNet:9092 --describe --topic-list test-collect_mm
docker exec -it kafka-${RND} bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list kafkaNet:9092 --topic mapper-topic
# docker exec -it kafka-${RND} bin/kafka-console-consumer.sh --bootstrap-server kafkaNet:9092 --topic mapper-topic --offset 0 --partition 0
prod:
docker exec -it kafka-${RND} bin/kafka-console-producer.sh --broker-list kafkaNet:9092 --topic my-topic
kafka-ui:
docker run -d --rm --network=${NETWORK} -p 9000:9000 \
-e KAFKA_BROKERCONNECT=kafka-${RND}:9092 \
-e JVM_OPTS="-Xms32M -Xmx64M" \
-e SERVER_SERVLET_CONTEXTPATH="/" \
obsidiandynamics/kafdrop