Headless release

.github/workflows/build.yml

@@ -243,3 +243,89 @@ jobs:
             env:
                 GITHUB_TOKEN: ${{ secrets.AUTOMATION_GITHUB_TOKEN }}
                 GRADLE_OPTS: '-Dorg.gradle.daemon=false'
+
+  # --------------------------------------------------
+  # job: release
+  # --------------------------------------------------
+  release:
+    name: Release
+    if: ${{ contains(needs.build.outputs.commit_message,'[release]') }}
+    runs-on: ubuntu-latest
+    needs: build
+    permissions:
+        contents: write
+    timeout-minutes: 10
+    steps:
+      # setup steps
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+          submodules: true
+
+      - name: Setup Java
+        uses: actions/setup-java@v4
+        with:
+          java-version: 17
+          distribution: 'temurin'
+          architecture: x64
+
+      - name: Setup AWS
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: eu-west-1
+          aws-access-key-id: ${{ secrets.AWS_DEPLOY_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_DEPLOY_SECRET_ACCESS_KEY }}
+
+      - name: Login to Docker hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_ID }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+      - name: Login to Seqera registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ vars.SEQERA_PUBLIC_CR_URL }}
+          username: ${{ secrets.SEQERA_PUBLIC_CR_USER }}
+          password: ${{ secrets.SEQERA_PUBLIC_CR_PASSWORD }}
+
+      # release step
+      - name: Release
+        run: bash release/main.sh
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DEPLOY_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_DEPLOY_SECRET_ACCESS_KEY }}
+          GH_ORG: ${{ vars.PLUGINS_GITHUB_ORG }}
+          GH_USER: ${{ vars.DEPLOY_GITHUB_USER }}
+          GH_USER_EMAIL: ${{ vars.DEPLOY_GITHUB_EMAIL }}
+          GH_TOKEN: ${{ secrets.DEPLOY_GITHUB_TOKEN }}
+          MAVEN_PUBLISH_URL: ${{ vars.MAVEN_PLUGINS_PUBLISH_URL }}
+          PLUGINS_INDEX_JSON: ${{ vars.PLUGINS_INDEX_JSON }}
+          S3_RELEASE_BUCKET: ${{ vars.S3_RELEASE_BUCKET }}
+          SEQERA_CONTAINER_REGISTRY: ${{ vars.SEQERA_PUBLIC_CR_URL }}
+
+      # upload steps
+      - name: Upload artifacts (libs)
+        uses: actions/upload-artifact@v4
+        with:
+          retention-days: 3
+          name: libs
+          path: modules/*/build/libs/
+
+      - name: Upload artifacts (distribution)
+        uses: actions/upload-artifact@v4
+        with:
+          retention-days: 3
+          name: distribution
+          path: build/releases/
+
+      - name: Upload artifacts (plugins)
+        uses: actions/upload-artifact@v4
+        with:
+          retention-days: 3
+          compression-level: 0
+          name: plugins
+          path: |
+            plugins/build/libs/
+            plugins/*/build/libs/

Makefile

@@ -86,10 +86,10 @@ smoke:
 	NXF_SMOKE=1 ./gradlew ${mm}test
 
 #
-# Upload JAR artifacts to Maven Central
+# Generate all the jars required to create a release
 #
-upload:
-	./gradlew upload
+distribution:
+	BUILD_PACK=1 ./gradlew buildInfo compile assemble pack javadocJar sourcesJar testFixturesJar
 
 #
 # Create self-contained distribution package
@@ -98,38 +98,14 @@ pack:
 	BUILD_PACK=1 ./gradlew pack
 
 #
-# Upload NF launcher to nextflow.io web site
-#
-deploy:
-	BUILD_PACK=1 ./gradlew deploy
-
-#
-# Close artifacts uploaded to Maven central
-#
-close:
-	./gradlew closeAndReleaseRepository
-
-#
-# Upload final package to GitHub
+# Initiate the nextflow release process
 #
+.PHONY: release
 release:
-	BUILD_PACK=1 ./gradlew release
-
-#
-# Create and upload docker image distribution
-#
-dockerImage:
-	BUILD_PACK=1 ./gradlew dockerImage
+	./make-release.sh
 
 #
 # Create local docker image
 #
 dockerPack:
 	BUILD_PACK=1 ./gradlew publishToMavenLocal dockerPack -Dmaven.repo.local=${PWD}/build/docker/.nextflow/capsule/deps/
-
-
-upload-plugins:
-	./gradlew plugins:upload
-
-publish-index:
-	./gradlew plugins:publishIndex

build.gradle

@@ -15,7 +15,6 @@
  */
 
 plugins {
-    id "io.codearte.nexus-staging" version "0.30.0"
     id 'java'
     id 'idea'
 }
@@ -274,29 +273,21 @@ task exportClasspath {
     }
 }
 
-ext.nexusUsername = project.findProperty('nexusUsername')
-ext.nexusPassword = project.findProperty('nexusPassword')
-ext.nexusFullName = project.findProperty('nexusFullName')
-ext.nexusEmail = project.findProperty('nexusEmail')
-
 // `signing.keyId` property needs to be defined in the `gradle.properties` file
 ext.enableSignArchives = project.findProperty('signing.keyId')
 
 ext.coreProjects = projects( ':nextflow', ':nf-commons', ':nf-httpfs' )
 
-configure(coreProjects) {
-    group = 'io.nextflow'
-    version = rootProject.file('VERSION').text.trim()
-}
-
 /*
- * Maven central deployment
- * http://central.sonatype.org/pages/gradle.html
+ * Maven deployment
  */
 configure(coreProjects) {
     apply plugin: 'maven-publish'
     apply plugin: 'signing'
 
+    group = 'io.nextflow'
+    version = rootProject.file('VERSION').text.trim()
+
     task javadocJar(type: Jar) {
         archiveClassifier = 'javadoc'
         from configurations.groovyDoc
@@ -333,9 +324,8 @@ configure(coreProjects) {
                     }
                     developers {
                         developer {
-                            id = nexusUsername
-                            name = nexusFullName
-                            email = nexusEmail
+                            name = 'Paolo Di Tommaso'
+                            email = '[email protected]'
                         }
                     }
                     scm {
@@ -352,13 +342,12 @@ configure(coreProjects) {
 
         repositories {
             maven {
-                // change URLs to point to your repos, e.g. http://my.org/repo
-                def releasesRepoUrl = "https://oss.sonatype.org/service/local/staging/deploy/maven2/"
-                def snapshotsRepoUrl = "https://oss.sonatype.org/content/repositories/snapshots/"
-                url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl
-                credentials(PasswordCredentials) {
-                    username nexusUsername
-                    password nexusPassword
+                url = System.getenv('MAVEN_PUBLISH_URL') ?: 's3://maven.seqera.io'
+                url = version.endsWith('-SNAPSHOT') ? "$url/snapshots" : "$url/releases"
+
+                credentials(AwsCredentials) {
+                    accessKey = System.getenv('AWS_ACCESS_KEY_ID') ?: findProperty('aws_access_key_id')
+                    secretKey = System.getenv('AWS_SECRET_ACCESS_KEY') ?: findProperty('aws_secret_access_key')
                 }
             }
         }
@@ -368,49 +357,26 @@ configure(coreProjects) {
         required { enableSignArchives }
         sign publishing.publications.mavenJava
     }
-
 }
 
-
-String bytesToHex(byte[] bytes) {
-    StringBuffer result = new StringBuffer();
-    for (byte byt : bytes) result.append(Integer.toString((byt & 0xff) + 0x100, 16).substring(1));
-    return result.toString();
+task publishToMaven {
+    dependsOn coreProjects.publish
 }
 
 task makeDigest { doLast {
     byte[] digest
     String  str = file('nextflow').text
     // create sha1
     digest = java.security.MessageDigest.getInstance("SHA1").digest(str.getBytes())
-    file('nextflow.sha1').text = new BigInteger(1, digest).toString(16) + '\n'
+    file('nextflow.sha1').text = digest.encodeHex().toString() + '\n'
     // create sha-256
     digest = java.security.MessageDigest.getInstance("SHA-256").digest(str.getBytes())
-    file('nextflow.sha256').text = bytesToHex(digest) + '\n'
+    file('nextflow.sha256').text = digest.encodeHex().toString() + '\n'
     // create md5
     digest = java.security.MessageDigest.getInstance("MD5").digest(str.getBytes())
-    file('nextflow.md5').text = bytesToHex(digest) + '\n'
+    file('nextflow.md5').text = digest.encodeHex().toString() + '\n'
 }}
 
-
-task upload {
-    dependsOn compile
-    dependsOn makeDigest
-    dependsOn coreProjects.publish
-}
-
-/*
- * Configure Nextflow staging plugin -- https://github.com/Codearte/gradle-nexus-staging-plugin
- * It adds the tasks
- * - closeRepository
- * - releaseRepository
- * - closeAndReleaseRepository
- */
-nexusStaging {
-    packageGroup = 'io.nextflow'
-    delayBetweenRetriesInMillis = 10_000
-}
-
 if( System.env.BUILD_PACK ) {
     apply from: 'packing.gradle'
 }

buildSrc/src/main/groovy/io/nextflow/gradle/tasks/GithubRepositoryPublisher.groovy

@@ -61,6 +61,7 @@ class GithubRepositoryPublisher extends DefaultTask {
 
 
     String mergeIndex(List<PluginMeta> mainIndex, Map<String,List<PluginRelease>> pluginsToPublish) {
+        boolean modified = false
 
         for( Map.Entry<String,List<PluginRelease>> item : pluginsToPublish ) {
             final pluginId = item.key
@@ -69,6 +70,7 @@ class GithubRepositoryPublisher extends DefaultTask {
 
             if (!indexEntry) {
                 mainIndex.add(new PluginMeta(id: pluginId, releases: pluginReleases))
+                modified = true
             }
             else {
                 for (PluginRelease rel : pluginReleases) {
@@ -79,11 +81,13 @@ class GithubRepositoryPublisher extends DefaultTask {
                     // if not, add to the index
                     if( !indexRel ) {
                         indexEntry.releases << rel
+                        modified = true
                     }
                     // otherwise, verify the checksum matches
                     else if( indexRel.sha512sum != rel.sha512sum ) {
                         if( overwrite ) {
                             indexEntry.releases[index] = rel
+                            modified = true
                         }
                         else {
                             def msg = "Plugin $pluginId@${rel.version} invalid checksum:\n"
@@ -97,11 +101,15 @@ class GithubRepositoryPublisher extends DefaultTask {
             }
         }
 
-        new GsonBuilder()
+        if ( modified ) {
+            new GsonBuilder()
                 .setPrettyPrinting()
                 .disableHtmlEscaping()
                 .create()
                 .toJson(mainIndex)
+        } else {
+            null
+        }
     }
 
     List<PluginMeta> parseMainIndex(GithubClient github, String path) {
@@ -188,10 +196,13 @@ class GithubRepositoryPublisher extends DefaultTask {
         logger.quiet("Merging index")
         final result = mergeIndex(mainIndex, pluginsToPublish)
 
-        // push to github
-        logger.quiet("Publish merged index to $indexUrl")
-
-        github.pushChange(targetFileName, result.toString() + '\n', "Nextflow plugins update")
+        if ( result ) {
+            // push to github
+            logger.quiet("Publish merged index to $indexUrl")
+            github.pushChange(targetFileName, result.toString() + '\n', "Nextflow plugins update")
+        } else {
+            logger.quiet("No changes to index")
+        }
     }
 
 }

docker/Makefile

@@ -23,12 +23,6 @@ build-arm: dist/docker/arm64
 	cp ../nextflow .
 	docker buildx build --platform linux/arm64 --output=type=docker --progress=plain --tag nextflow/nextflow:${version} --build-arg TARGETPLATFORM=linux/arm64 .
 
-release: build
-	docker push nextflow/nextflow:${version}
-	#
-	docker tag nextflow/nextflow:${version} public.cr.seqera.io/nextflow/nextflow:${version}
-	docker push public.cr.seqera.io/nextflow/nextflow:${version}
-
 #Static builds can now be found at:
 #
 #  Linux:   https://download.docker.com/linux/static