Skip to content

Commit

Permalink
Headless release: wire everything up
Browse files Browse the repository at this point in the history
Signed-off-by: Tom Sellman <[email protected]>
  • Loading branch information
tom-seqera committed Dec 2, 2024
1 parent 03f70a2 commit 8c67025
Show file tree
Hide file tree
Showing 8 changed files with 233 additions and 66 deletions.
22 changes: 22 additions & 0 deletions .github/scripts/deploy-to-github.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/usr/bin/env bash
set -e

# change to the project root
cd "$(dirname "$0")/../.."

# read the nextflow version
read -r NF_VERSION<VERSION

echo "Publishing nextflow release to github"

# create a github (pre)release and attach launcher and dist files
# use --verify-tag to fail if tag doesn't exist
gh release create \
--prerelease \
--title "Version $NF_VERSION" \
--verify-tag \
"v$NF_VERSION" \
nextflow \
"build/releases/nextflow-$NF_VERSION-dist"

echo "Done"
35 changes: 35 additions & 0 deletions .github/scripts/publish-release.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
#!/usr/bin/env bash
set -e

# change to the project root
cd "$(dirname "$0")/../.."

# read the nextflow version
read -r NF_VERSION<VERSION

# determine publish location
S3_RELEASE_BUCKET=${S3_RELEASE_BUCKET:-'www2.nextflow.io'}
if [[ "$NF_VERSION" =~ /^.+(-edge|-EDGE)$/ ]]; then
S3_RELEASE_DIR="releases/edge"
else
S3_RELEASE_DIR="releases/latest"
fi


#publish nextflow script as latest
files+=(
'nextflow'
'nextflow.sha1'
'nextflow.sha256'
'nextflow.md5'
'VERSION'
)

for file in "${files[@]}"; do
filename=$(echo "$file" | tr '[:upper:]' '[:lower:]')
aws s3 cp "$file" "s3://$S3_RELEASE_BUCKET/$S3_RELEASE_DIR/$filename" \
--no-progress \
--storage-class STANDARD \
--region eu-west-1 \
--acl public-read
done
14 changes: 14 additions & 0 deletions .github/scripts/tag-release.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -e

# change to the project root
cd "$(dirname "$0")/../.."

# read the nextflow version
read -r NF_VERSION<VERSION

# create & push an annotated git tag
git tag "v$NF_VERSION"
git push origin "v$NF_VERSION"

echo "Done"
18 changes: 18 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -243,3 +243,21 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.AUTOMATION_GITHUB_TOKEN }}
GRADLE_OPTS: '-Dorg.gradle.daemon=false'

release:
name: Release
if: ${{ contains(needs.build.outputs.commit_message,'[release]') }}
runs-on: ubuntu-latest
needs: build
timeout-minutes: 10
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
submodules: true

- name: Release
run: gh workflow run release.yml --ref ${{ github.ref }}
env:
GH_TOKEN: ${{ secrets.AUTOMATION_GITHUB_TOKEN }}
96 changes: 90 additions & 6 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
name: Release
run-name: Release ${{ github.ref_name }}

# TODO:tom real workflow trigger
on:
workflow_dispatch:

Expand Down Expand Up @@ -61,13 +59,35 @@ jobs:
plugins/build/libs/
plugins/*/build/libs/
# --------------------------------------------------
# job: tag
# --------------------------------------------------
tag:
name: Tag
runs-on: ubuntu-latest
timeout-minutes: 15
needs: assemble
permissions:
contents: write
steps:
# setup steps
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
submodules: true

# execute steps
- name: Tag
run: bash .github/scripts/tag-release.sh

# --------------------------------------------------
# job: deploy-maven
# --------------------------------------------------
deploy-maven:
name: Deploy to Maven
runs-on: ubuntu-latest
needs: assemble
needs: tag
timeout-minutes: 15
steps:
# setup steps
Expand Down Expand Up @@ -105,7 +125,7 @@ jobs:
deploy-s3:
name: Deploy to S3
runs-on: ubuntu-latest
needs: assemble
needs: tag
timeout-minutes: 15
steps:
# setup steps
Expand Down Expand Up @@ -140,7 +160,9 @@ jobs:
deploy-docker:
name: Deploy to Docker
runs-on: ubuntu-latest
needs: assemble
needs:
- tag
- deploy-s3
timeout-minutes: 15
steps:
# setup steps
Expand Down Expand Up @@ -169,13 +191,41 @@ jobs:
env:
SEQERA_REGISTRY: ${{ vars.SEQERA_PUBLIC_CR_URL }}

# --------------------------------------------------
# job: deploy-github
# --------------------------------------------------
deploy-github:
name: Deploy to Github
runs-on: ubuntu-latest
needs: tag
timeout-minutes: 15
steps:
# setup steps
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
submodules: true

- name: Download artifacts (distribution)
uses: actions/download-artifact@v4
with:
name: distribution
path: build/releases

# deploy steps
- name: Create github release
run: bash .github/scripts/deploy-to-github.sh
env:
GH_TOKEN: ${{ secrets.DEPLOY_GITHUB_TOKEN }}

# --------------------------------------------------
# job: deploy-plugins
# --------------------------------------------------
deploy-plugins:
name: Deploy Plugins
runs-on: ubuntu-latest
needs: assemble
needs: tag
timeout-minutes: 15
steps:
# setup steps
Expand Down Expand Up @@ -221,3 +271,37 @@ jobs:
GH_USER_EMAIL: ${{ vars.DEPLOY_GITHUB_EMAIL }}
GH_TOKEN: ${{ secrets.DEPLOY_GITHUB_TOKEN }}
PLUGINS_INDEX_JSON: ${{ vars.PLUGINS_INDEX_JSON }}


# --------------------------------------------------
# job: publish
# --------------------------------------------------
publish:
name: Publish release
runs-on: ubuntu-latest
needs:
- deploy-s3
- deploy-docker
- deploy-maven
- deploy-github
timeout-minutes: 15
steps:
# setup steps
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
submodules: true

- name: Setup AWS
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: eu-west-1
aws-access-key-id: ${{ secrets.AWS_DEPLOY_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_DEPLOY_SECRET_ACCESS_KEY }}

# deploy steps
- name: Publish release
run: bash .github/scripts/publish-release.sh
env:
S3_RELEASE_BUCKET: ${{ vars.S3_RELEASE_BUCKET }}
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ pack:
# Upload final package to GitHub
#
release:
BUILD_PACK=1 ./gradlew release
./make-release.sh

#
# Create local docker image
Expand Down
53 changes: 53 additions & 0 deletions make-release.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
#!/usr/bin/env bash
set -e

cd "$(dirname "$0")"

# read the nextflow version
read -r NF_VERSION<VERSION

# prompt user to check version is correct
# TODO:tom display plugin versions too
echo "
------------------------
This will release:
nextflow: $NF_VERSION
------------------------
If these are not the versions you want to release, type 'no' and update the below
files (without committing), then run this script again:
- VERSION
- nextflow
- changelog.txt
"

echo -n "Type 'yes' to proceed: "
read -r proceed
if [[ "$proceed" != "yes" ]]; then
echo "Aborting"
exit
fi

# update the digest files before committing anything
./gradlew makeDigest

# create the release commit, including text '[release]' to trigger the github action
# the github action workflow will perform the following tasks
# - tag the release
# - build the release artifacts
# - deploy the release artifacts to maven/docker/S3/etc
# - create a (pre-release) github release
# - deploy the plugins, and update the plugin index
git commit -s -am "Release $NF_VERSION [release]"
# push the current branch is to the remote origin
git push origin HEAD

echo "
------------------------------------------------------------
Release commit pushed.
This should trigger a github release workflow.
Once the workflow is complete, you should add the changelog
to the github release.
------------------------------------------------------------
"
59 changes: 0 additions & 59 deletions packing.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ ext.mainClassName = 'nextflow.cli.Launcher'
ext.homeDir = System.properties['user.home']
ext.nextflowDir = "$homeDir/.nextflow/framework/$version"
ext.releaseDir = "$buildDir/releases"
ext.s3CmdOpts="--acl public-read --storage-class STANDARD --region eu-west-1"

protected resolveDeps( String configName, String... x ) {

Expand Down Expand Up @@ -161,61 +160,3 @@ task dockerPack(type: Exec, dependsOn: ['packOne']) {

commandLine 'bash', '-e', temp.absolutePath
}

/*
* Tag and upload the release
*
* https://github.com/aktau/github-release/
*/
task release(type: Exec, dependsOn: [pack]) {

def launcherFile = file('nextflow').absoluteFile
def launcherSha1 = file('nextflow.sha1').absoluteFile
def launcherSha256 = file('nextflow.sha256').absoluteFile
def nextflowAllFile = file("$releaseDir/nextflow-${version}-dist")
def versionFile = file('VERSION').absoluteFile

def snapshot = version ==~ /^.+(-RC\d+|-SNAPSHOT)$/
def edge = version ==~ /^.+(-edge|-EDGE)$/
def isLatest = !snapshot && !edge
def cmd = """\
# tag the release
git push || exit \$?
(git tag -a v$version -m 'Tagging version $version [release]' -f && git push origin v$version -f) || exit \$?
sleep 1
gh release create --repo nextflow-io/nextflow --prerelease --title "Version $version" v$version
sleep 1
gh release upload --repo nextflow-io/nextflow v$version ${launcherFile} ${nextflowAllFile}
""".stripIndent()

if( edge )
cmd += """
# publish the script as the latest
export AWS_ACCESS_KEY_ID=${System.env.NXF_AWS_ACCESS}
export AWS_SECRET_ACCESS_KEY=${System.env.NXF_AWS_SECRET}
aws s3 cp $launcherFile s3://www2.nextflow.io/releases/edge/nextflow $s3CmdOpts
aws s3 cp $launcherSha1 s3://www2.nextflow.io/releases/edge/nextflow.sha1 $s3CmdOpts
aws s3 cp $launcherSha256 s3://www2.nextflow.io/releases/edge/nextflow.sha256 $s3CmdOpts
aws s3 cp $launcherSha256 s3://www2.nextflow.io/releases/edge/nextflow.md5 $s3CmdOpts
aws s3 cp $versionFile s3://www2.nextflow.io/releases/edge/version $s3CmdOpts
""".stripIndent()

else if( isLatest )
cmd += """
# publish the script as the latest
export AWS_ACCESS_KEY_ID=${System.env.NXF_AWS_ACCESS}
export AWS_SECRET_ACCESS_KEY=${System.env.NXF_AWS_SECRET}
aws s3 cp $launcherFile s3://www2.nextflow.io/releases/latest/nextflow $s3CmdOpts
aws s3 cp $launcherSha1 s3://www2.nextflow.io/releases/latest/nextflow.sha1 $s3CmdOpts
aws s3 cp $launcherSha256 s3://www2.nextflow.io/releases/latest/nextflow.sha256 $s3CmdOpts
aws s3 cp $launcherSha256 s3://www2.nextflow.io/releases/latest/nextflow.md5 $s3CmdOpts
aws s3 cp $versionFile s3://www2.nextflow.io/releases/latest/version $s3CmdOpts
""".stripIndent()

def temp = File.createTempFile('upload',null)
temp.deleteOnExit()
temp.text = cmd
commandLine 'bash', '-e', temp.absolutePath
}


0 comments on commit 8c67025

Please sign in to comment.