Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Monorepo] Update manifest using the monorepo components #4082

Merged
merged 5 commits into from
Sep 26, 2023
Merged

[Monorepo] Update manifest using the monorepo components #4082

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
bae3ff6
Use latest tag in script/release.sh
eapolinario Sep 26, 2023
f19c6e3
Use a more idiomatic way of consuming grep results in call to kustomi…
eapolinario Sep 26, 2023
eb98561
Rearrange how config docs are created
eapolinario Sep 26, 2023
0ca961b
Add compile_flytepropeller make target
eapolinario Sep 26, 2023
8a97a37
Move the binaries instead of copying them
eapolinario Sep 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion datacatalog/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ update_boilerplate:
.PHONY: compile
compile:
mkdir -p ./bin
go build -o datacatalog ./cmd/main.go && mv ./datacatalog ./bin
go build -o bin/datacatalog ./cmd/main.go

.PHONY: linux_compile
linux_compile: export CGO_ENABLED ?= 0
Expand Down
6 changes: 4 additions & 2 deletions flyteadmin/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ k8s_integration_execute:

.PHONY: compile
compile:
go build -o flyteadmin -ldflags=$(LD_FLAGS) ./cmd/ && mv ./flyteadmin ${GOPATH}/bin
mkdir -p ./bin
go build -o bin/flyteadmin -ldflags=$(LD_FLAGS) ./cmd/

.PHONY: compile_debug
compile_debug:
Expand All @@ -46,7 +47,8 @@ compile_debug:

.PHONY: compile_scheduler
compile_scheduler:
go build -o flytescheduler -ldflags=$(LD_FLAGS) ./cmd/scheduler/ && mv ./flytescheduler ${GOPATH}/bin
mkdir -p ./bin
go build -o bin/flytescheduler -ldflags=$(LD_FLAGS) ./cmd/scheduler/

.PHONY: compile_scheduler_debug
compile_scheduler_debug:
Expand Down
7 changes: 6 additions & 1 deletion flytepropeller/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,12 @@ compile:
mkdir -p ./bin
go build -o bin/flytepropeller ./cmd/controller/main.go
go build -o bin/flytepropeller-manager ./cmd/manager/main.go
go build -o bin/kubectl-flyte ./cmd/kubectl-flyte/main.go && cp bin/kubectl-flyte ${GOPATH}/bin
go build -o bin/kubectl-flyte ./cmd/kubectl-flyte/main.go

.PHONY: compile_flytepropeller
compile_flytepropeller:
mkdir -p ./bin
go build -o bin/flytepropeller ./cmd/controller/main.go

cross_compile:
@glide install
Expand Down
22 changes: 8 additions & 14 deletions script/generate_config_docs.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,14 @@ ROOT_DIR=${CUR_DIR}/..
OUTPUT_DIR="${ROOT_DIR}"/rsts/deployment/configuration/generated
GOBIN=${GOPATH:-~/go}/bin

# The version should same as https://github.com/flyteorg/flyte/blob/6b92b72f508d0603fa44153a4e30cf81be76adfd/script/release.sh#L5-L8
FLYTEADMIN_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flyteadmin/releases/latest" | jq -r .tag_name)
FLYTEPROPELLER_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flytepropeller/releases/latest" | jq -r .tag_name)
DATACATALOG_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/datacatalog/releases/latest" | jq -r .tag_name)

pushd "$(mktemp -d)"
curl https://github.com/flyteorg/flyteadmin/releases/download/${FLYTEADMIN_TAG}/flyteadmin_${FLYTEADMIN_TAG}_linux_x86_64.tar.gz --output flyteadmin.tar.gz -s -L && tar -xvf flyteadmin.tar.gz
mv flyteadmin $GOBIN/flyteadmin
mv flytescheduler $GOBIN/scheduler
curl https://github.com/flyteorg/flytepropeller/releases/download/${FLYTEPROPELLER_TAG}/flytepropeller_${FLYTEPROPELLER_TAG}_linux_x86_64.tar.gz --output flytepropeller.tar.gz -s -L && tar -xvf flytepropeller.tar.gz
mv flytepropeller $GOBIN/flytepropeller
curl https://github.com/flyteorg/datacatalog/releases/download/${DATACATALOG_TAG}/datacatalog_${DATACATALOG_TAG}_linux_x86_64.tar.gz --output datacatalog.tar.gz -s -L && tar -xvf datacatalog.tar.gz
mv datacatalog $GOBIN/datacatalog
popd
make -C datacatalog compile
mv datacatalog/bin/datacatalog ${GOBIN}/datacatalog
make -C flyteadmin compile
mv flyteadmin/bin/flyteadmin ${GOBIN}/flyteadmin
make -C flyteadmin compile_scheduler
mv flyteadmin/bin/flytescheduler ${GOBIN}/scheduler
make -C flytepropeller compile_flytepropeller
mv flytepropeller/bin/flytepropeller ${GOBIN}/flytepropeller

output_config () {
CONFIG_NAME=$1
Expand Down
49 changes: 25 additions & 24 deletions script/release.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,41 +2,42 @@

set -ex

FLYTEADMIN_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flyteadmin/releases/latest" | jq -r .tag_name)
DATACATALOG_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/datacatalog/releases/latest" | jq -r .tag_name)
# Get latest release tag (minus the component prefix)
# TODO(monorepo): This only works if we have at least one component tag per release.
# In other words, if we have two consecutive releases the latest tag in the second release is going to point to an invalid
# tag (because there will not be images tagged with the previous release tag).
LATEST_TAG=$(git tag | sed 's#[^/]*/##' | sort | tail -n 1)

FLYTECONSOLE_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flyteconsole/releases/latest" | jq -r .tag_name)
FLYTEPROPELLER_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flytepropeller/releases/latest" | jq -r .tag_name)
FLYTECOPILOT_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flytecopilot/releases/latest" | jq -r .tag_name)
FLYTESTDLIB_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flytestdlib/releases/latest" | jq -r .tag_name)

# bump latest release of flyte component in kustomize
grep -rlZ "newTag:[^P]*# FLYTEADMIN_TAG" ./kustomize/overlays | xargs -0 sed -i "s/newTag:[^P]*# FLYTEADMIN_TAG/newTag: ${FLYTEADMIN_TAG} # FLYTEADMIN_TAG/g"
grep -rlZ "newTag:[^P]*# DATACATALOG_TAG" ./kustomize/overlays | xargs -0 sed -i "s/newTag:[^P]*# DATACATALOG_TAG/newTag: ${DATACATALOG_TAG} # DATACATALOG_TAG/g"
grep -rlZ "newTag:[^P]*# FLYTECONSOLE_TAG" ./kustomize/overlays | xargs -0 sed -i "s/newTag:[^P]*# FLYTECONSOLE_TAG/newTag: ${FLYTECONSOLE_TAG} # FLYTECONSOLE_TAG/g"
grep -rlZ "newTag:[^P]*# FLYTEPROPELLER_TAG" ./kustomize/overlays | xargs -0 sed -i "s/newTag:[^P]*# FLYTEPROPELLER_TAG/newTag: ${FLYTEPROPELLER_TAG} # FLYTEPROPELLER_TAG/g"
grep -rlZ "newTag:[^P]*# FLYTEADMIN_TAG" ./kustomize/overlays | xargs -I {} sed -i "s/newTag:[^P]*# FLYTEADMIN_TAG/newTag: ${LATEST_TAG} # FLYTEADMIN_TAG/g" {}
grep -rlZ "newTag:[^P]*# DATACATALOG_TAG" ./kustomize/overlays | xargs -I {} sed -i "s/newTag:[^P]*# DATACATALOG_TAG/newTag: ${LATEST_TAG} # DATACATALOG_TAG/g" {}
grep -rlZ "newTag:[^P]*# FLYTECONSOLE_TAG" ./kustomize/overlays | xargs -I {} sed -i "s/newTag:[^P]*# FLYTECONSOLE_TAG/newTag: ${FLYTECONSOLE_TAG} # FLYTECONSOLE_TAG/g" {}
grep -rlZ "newTag:[^P]*# FLYTEPROPELLER_TAG" ./kustomize/overlays | xargs -I {} sed -i "s/newTag:[^P]*# FLYTEPROPELLER_TAG/newTag: ${LATEST_TAG} # FLYTEPROPELLER_TAG/g" {}

# bump latest release of flyte component in helm
sed -i "s,tag:[^P]*# FLYTEADMIN_TAG,tag: ${FLYTEADMIN_TAG} # FLYTEADMIN_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# FLYTEADMIN_TAG,tag: ${FLYTEADMIN_TAG} # FLYTEADMIN_TAG," ./charts/flyte-core/values.yaml
sed -i "s,tag:[^P]*# FLYTEADMIN_TAG,tag: ${LATEST_TAG} # FLYTEADMIN_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# FLYTEADMIN_TAG,tag: ${LATEST_TAG} # FLYTEADMIN_TAG," ./charts/flyte-core/values.yaml

sed -i "s,tag:[^P]*# FLYTESCHEDULER_TAG,tag: ${FLYTEADMIN_TAG} # FLYTESCHEDULER_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# FLYTESCHEDULER_TAG,tag: ${FLYTEADMIN_TAG} # FLYTESCHEDULER_TAG," ./charts/flyte-core/values.yaml
sed -i "s,tag:[^P]*# FLYTESCHEDULER_TAG,tag: ${LATEST_TAG} # FLYTESCHEDULER_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# FLYTESCHEDULER_TAG,tag: ${LATEST_TAG} # FLYTESCHEDULER_TAG," ./charts/flyte-core/values.yaml

sed -i "s,tag:[^P]*# DATACATALOG_TAG,tag: ${DATACATALOG_TAG} # DATACATALOG_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# DATACATALOG_TAG,tag: ${DATACATALOG_TAG} # DATACATALOG_TAG," ./charts/flyte-core/values.yaml
sed -i "s,tag:[^P]*# DATACATALOG_TAG,tag: ${LATEST_TAG} # DATACATALOG_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# DATACATALOG_TAG,tag: ${LATEST_TAG} # DATACATALOG_TAG," ./charts/flyte-core/values.yaml

sed -i "s,tag:[^P]*# FLYTECONSOLE_TAG,tag: ${FLYTECONSOLE_TAG} # FLYTECONSOLE_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# FLYTECONSOLE_TAG,tag: ${FLYTECONSOLE_TAG} # FLYTECONSOLE_TAG," ./charts/flyte-core/values.yaml

sed -i "s,tag:[^P]*# FLYTEPROPELLER_TAG,tag: ${FLYTEPROPELLER_TAG} # FLYTEPROPELLER_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# FLYTEPROPELLER_TAG,tag: ${FLYTEPROPELLER_TAG} # FLYTEPROPELLER_TAG," ./charts/flyte-core/values.yaml
sed -i "s,tag:[^P]*# FLYTEPROPELLER_TAG,tag: ${LATEST_TAG} # FLYTEPROPELLER_TAG," ./charts/flyte/values.yaml
sed -i "s,tag:[^P]*# FLYTEPROPELLER_TAG,tag: ${LATEST_TAG} # FLYTEPROPELLER_TAG," ./charts/flyte-core/values.yaml

sed -i "s,image:[^P]*# FLYTECOPILOT_IMAGE,image: cr.flyte.org/flyteorg/flytecopilot:${FLYTECOPILOT_TAG} # FLYTECOPILOT_IMAGE," ./charts/flyte/values.yaml
sed -i "s,image:[^P]*# FLYTECOPILOT_IMAGE,image: cr.flyte.org/flyteorg/flytecopilot:${FLYTECOPILOT_TAG} # FLYTECOPILOT_IMAGE," ./charts/flyte-core/values.yaml
sed -i "s,tag:[^P]*# FLYTECOPILOT_TAG,tag: ${FLYTECOPILOT_TAG} # FLYTECOPILOT_TAG," ./charts/flyte-binary/values.yaml
sed -i "s,image:[^P]*# FLYTECOPILOT_IMAGE,image: cr.flyte.org/flyteorg/flytecopilot:${LATEST_TAG} # FLYTECOPILOT_IMAGE," ./charts/flyte/values.yaml
sed -i "s,image:[^P]*# FLYTECOPILOT_IMAGE,image: cr.flyte.org/flyteorg/flytecopilot:${LATEST_TAG} # FLYTECOPILOT_IMAGE," ./charts/flyte-core/values.yaml
sed -i "s,tag:[^P]*# FLYTECOPILOT_TAG,tag: ${LATEST_TAG} # FLYTECOPILOT_TAG," ./charts/flyte-binary/values.yaml

go get github.com/flyteorg/flyteadmin@${FLYTEADMIN_TAG}
go get github.com/flyteorg/flytepropeller@${FLYTEPROPELLER_TAG}
go get github.com/flyteorg/datacatalog@${DATACATALOG_TAG}
go get github.com/flyteorg/flytestdlib@${FLYTESTDLIB_TAG}
go get github.com/flyteorg/flyte/flyteadmin@${LATEST_TAG}
go get github.com/flyteorg/flyte/flytepropeller@${LATEST_TAG}
go get github.com/flyteorg/flyte/datacatalog@${LATEST_TAG}
go get github.com/flyteorg/flyte/flytestdlib@${LATEST_TAG}
go mod tidy
Loading