Adapt client release pipeline to use Koji or Staging repositories

theforeman · Sep 12, 2023 · 06c7293 · 06c7293
1 parent a44d192
commit 06c7293
Show file tree

Hide file tree

Showing 17 changed files with 159 additions and 37 deletions.
diff --git a/theforeman.org/pipelines/lib/obal.groovy b/theforeman.org/pipelines/lib/obal.groovy
@@ -7,7 +7,13 @@ def obal(args) {
         packages = packages.join(' ')
     }
 
-    def cmd = "python -m obal ${args.action} ${packages}"
+    if (env.NODE_LABELS.contains('el8')) {
+        python = 'python3.11'
+    } else {
+        python = 'python'
+    }
+
+    def cmd = "${python} -m obal ${args.action} ${packages}"
 
     if (extra_vars) {
         extra_vars_file = writeExtraVars(extraVars: extra_vars)

diff --git a/theforeman.org/pipelines/lib/packaging.groovy b/theforeman.org/pipelines/lib/packaging.groovy
@@ -463,3 +463,23 @@ def rsync_debian(user, ssh_key, suite, component, deb_paths) {
         """
     }
 }
+
+def rsync_to_yum_stage(collection, target, version) {
+    def user = 'yumrepostage'
+    def ssh_key = '/home/jenkins/workspace/staging_key/rsync_yumrepostage_key'
+
+    rsync_yum(user, ssh_key, collection, target, version)
+}
+
+def rsync_yum(user, ssh_key, collection, target, version) {
+    def hosts = ["web01.osuosl.theforeman.org"]
+
+    for(host in hosts) {
+        def target_path = "${user}@${host}:rsync_cache/${target}"
+
+        sh """
+            export RSYNC_RSH="ssh -i ${ssh_key}"
+            /usr/bin/rsync --archive --verbose --partial --one-file-system --delete-after ${collection}/${version} ${target_path}
+        """
+    }
+}
diff --git a/theforeman.org/pipelines/lib/release.groovy b/theforeman.org/pipelines/lib/release.groovy
@@ -3,13 +3,13 @@ void push_foreman_rpms(repo_type, version, distro) {
     push_rpms("foreman-${repo_type}-${version}", repo_type, version, distro)
 }
 
-void push_rpms(repo_src, repo_dest, version, distro, keep_old_files = false) {
-    push_rpms_direct("${repo_src}/${distro}", "${repo_dest}/${version}/${distro}", !keep_old_files, keep_old_files)
+void push_rpms(repo_src, repo_dest, version, distro, keep_old_files = false, staging = false) {
+    push_rpms_direct("${repo_src}/${distro}", "${repo_dest}/${version}/${distro}", !keep_old_files, keep_old_files, staging)
 }
 
-void push_rpms_direct(repo_source, repo_dest, overwrite = true, merge = false) {
+void push_rpms_direct(repo_source, repo_dest, overwrite = true, merge = false, staging = false) {
     sshagent(['repo-sync']) {
-        sh "ssh [email protected] ${repo_source} ${repo_dest} ${overwrite} ${merge}"
+        sh "ssh [email protected] ${repo_source} ${repo_dest} ${overwrite} ${merge} ${staging}"
     }
 }
 
@@ -40,3 +40,13 @@ void mash(collection, version) {
         sh "ssh -o 'BatchMode yes' [email protected] collection-mash-split.py ${collection} ${version}"
     }
 }
+
+void push_staging_rpms(repo_src, repo_dest, version, distro, keep_old_files = false) {
+    push_rpms_direct("${repo_src}/${distro}", "${repo_dest}/${version}/${distro}", !keep_old_files, keep_old_files, true)
+}
+
+void push_foreman_staging_rpms(repo_type, version, distro) {
+    version = version == 'develop' ? 'nightly' : version
+    push_staging_rpms("${repo_type}/${version}", repo_type, version, distro)
+}
+
diff --git a/theforeman.org/pipelines/release/pipelines/client.groovy b/theforeman.org/pipelines/release/pipelines/client.groovy
@@ -9,43 +9,110 @@ pipeline {
     }
 
     stages {
-        stage('Mash Koji Repositories') {
-            agent { label 'sshkey' }
-
-            steps {
-                mash('foreman-client', foreman_version)
+        stage('koji') {
+            when {
+                expression { stage_source == 'koji' }
             }
-        }
-        stage('Repoclosure') {
-            agent { label 'el' }
+            stages {
+                stage('koji-mash-repositories') {
+                    agent { label 'sshkey' }
 
-            steps {
-                script {
-                    def parallelStagesMap = [:]
-                    def name = 'foreman-client'
-                    foreman_client_distros.each { distro ->
-                        if (distro.startsWith('el')) {
-                            parallelStagesMap[distro] = { repoclosure(name, distro, foreman_version) }
-                        } else if (distro.startsWith('fc')) {
-                            parallelStagesMap[distro] = { repoclosure(name, distro.replace('fc', 'f'), foreman_version) }
-                        }
-                    }
-                    parallel parallelStagesMap
+                    steps {
+                        mash('foreman-client', foreman_version)
+                    }
                 }
-            }
-            post {
-                always {
-                    deleteDir()
+                stage('koji-repoclosure') {
+                    agent { label 'el' }
+
+                    steps {
+                        script {
+                            def parallelStagesMap = [:]
+                            def name = 'foreman-client'
+                            foreman_client_distros.each { distro ->
+                                if (distro.startsWith('el')) {
+                                    parallelStagesMap[distro] = { repoclosure(name, distro, foreman_version) }
+                                } else if (distro.startsWith('fc')) {
+                                    parallelStagesMap[distro] = { repoclosure(name, distro.replace('fc', 'f'), foreman_version) }
+                                }
+                            }
+                            parallel parallelStagesMap
+                        }
+                    }
+                    post {
+                        always {
+                            deleteDir()
+                        }
+                    }
+                }
+                stage('koji-push-rpms') {
+                    agent { label 'admin && sshkey' }
+
+                    steps {
+                        script {
+                            foreman_client_distros.each { distro ->
+                                push_foreman_rpms('client', foreman_version, distro)
+                            }
+                        }
+                    }
                 }
             }
         }
-        stage('Push RPMs') {
-            agent { label 'admin && sshkey' }
+        stage('staging') {
+            agent { label 'el8' }
+            when {
+                expression { stage_source == 'stagingyum' }
+            }
+            stages {
+                stage('staging-build-repository') {
+                    steps {
+                        git url: "https://github.com/theforeman/theforeman-rel-eng", poll: false
 
-            steps {
-                script {
-                    foreman_client_distros.each { distro ->
-                        push_foreman_rpms('client', foreman_version, distro)
+                        script {
+                            foreman_client_distros.each { distro ->
+                                sh "./build_stage_repository foreman-client ${foreman_version} ${distro}"
+                            }
+                        }
+                    }
+                }
+                stage('staging-copy-repository') {
+                    steps {
+                        script {
+                            dir('tmp') {
+                                rsync_to_yum_stage('foreman-client', 'client', foreman_version)
+                            }
+                        }
+                    }
+                }
+                stage('staging-repoclosure') {
+                    steps {
+                        script {
+                            def parallelStagesMap = [:]
+                            def name = 'foreman-client-staging'
+                            foreman_client_distros.each { distro ->
+                                if (distro.startsWith('el')) {
+                                    parallelStagesMap[distro] = { repoclosure(name, distro, foreman_version) }
+                                } else if (distro.startsWith('fc')) {
+                                    parallelStagesMap[distro] = { repoclosure(name, distro.replace('fc', 'f'), foreman_version) }
+                                }
+                            }
+                            parallel parallelStagesMap
+                        }
+                    }
+                    post {
+                        always {
+                            deleteDir()
+                        }
+                    }
+                }
+                stage('staging-push-rpms') {
+                    agent { label 'admin && sshkey' }
+
+                    steps {
+                        script {
+                            foreman_client_distros.each { distro ->
+                                push_foreman_staging_rpms('client', foreman_version, distro)
+                            }
+                        }
                     }
                 }
             }

diff --git a/theforeman.org/pipelines/vars/foreman/3.4.groovy b/theforeman.org/pipelines/vars/foreman/3.4.groovy
@@ -1,5 +1,7 @@
 def foreman_version = '3.4'
 def git_branch = "${foreman_version}-stable"
+def stage_source = 'koji'
+
 def foreman_client_distros = [
     'el9',
     'el8',

diff --git a/theforeman.org/pipelines/vars/foreman/3.5.groovy b/theforeman.org/pipelines/vars/foreman/3.5.groovy
@@ -1,5 +1,7 @@
 def foreman_version = '3.5'
 def git_branch = "${foreman_version}-stable"
+def stage_source = 'koji'
+
 def foreman_client_distros = [
     'el9',
     'el8',

diff --git a/theforeman.org/pipelines/vars/foreman/3.6.groovy b/theforeman.org/pipelines/vars/foreman/3.6.groovy
@@ -1,5 +1,7 @@
 def foreman_version = '3.6'
 def git_branch = "${foreman_version}-stable"
+def stage_source = 'koji'
+
 def foreman_client_distros = [
     'el9',
     'el8',

diff --git a/theforeman.org/pipelines/vars/foreman/3.7.groovy b/theforeman.org/pipelines/vars/foreman/3.7.groovy
@@ -1,5 +1,7 @@
 def foreman_version = '3.7'
 def git_branch = "${foreman_version}-stable"
+def stage_source = 'koji'
+
 def foreman_client_distros = [
     'el9',
     'el8',

diff --git a/theforeman.org/pipelines/vars/foreman/3.8.groovy b/theforeman.org/pipelines/vars/foreman/3.8.groovy
@@ -1,5 +1,7 @@
 def foreman_version = '3.8'
 def git_branch = "${foreman_version}-stable"
+def stage_source = 'koji'
+
 def foreman_client_distros = [
     'el9',
     'el8',

diff --git a/theforeman.org/pipelines/vars/foreman/nightly.groovy b/theforeman.org/pipelines/vars/foreman/nightly.groovy
@@ -1,10 +1,11 @@
 def foreman_version = 'nightly'
 def git_branch = "develop"
+def stage_source = 'koji'
+
 def foreman_client_distros = [
     'el9',
     'el8',
-    'el7',
-    'sles12'
+    'el7'
 ]
 def foreman_el_releases = [
     'el8'

diff --git a/theforeman.org/pipelines/vars/katello/4.10.groovy b/theforeman.org/pipelines/vars/katello/4.10.groovy
@@ -1,5 +1,6 @@
 def foreman_version = '3.8'
 def katello_version = '4.10'
+def stage_source = 'koji'
 def foreman_el_releases = [
     'el8'
 ]

diff --git a/theforeman.org/pipelines/vars/katello/4.6.groovy b/theforeman.org/pipelines/vars/katello/4.6.groovy
@@ -1,5 +1,6 @@
 def foreman_version = '3.4'
 def katello_version = '4.6'
+def stage_source = 'koji'
 def foreman_el_releases = [
     'el8'
 ]

diff --git a/theforeman.org/pipelines/vars/katello/4.7.groovy b/theforeman.org/pipelines/vars/katello/4.7.groovy
@@ -1,5 +1,6 @@
 def foreman_version = '3.5'
 def katello_version = '4.7'
+def stage_source = 'koji'
 def foreman_el_releases = [
     'el8'
 ]

diff --git a/theforeman.org/pipelines/vars/katello/4.8.groovy b/theforeman.org/pipelines/vars/katello/4.8.groovy
@@ -1,5 +1,6 @@
 def foreman_version = '3.6'
 def katello_version = '4.8'
+def stage_source = 'koji'
 def foreman_el_releases = [
     'el8'
 ]

diff --git a/theforeman.org/pipelines/vars/katello/4.9.groovy b/theforeman.org/pipelines/vars/katello/4.9.groovy
@@ -1,5 +1,6 @@
 def foreman_version = '3.7'
 def katello_version = '4.9'
+def stage_source = 'koji'
 def foreman_el_releases = [
     'el8'
 ]

diff --git a/theforeman.org/pipelines/vars/katello/nightly.groovy b/theforeman.org/pipelines/vars/katello/nightly.groovy
@@ -1,5 +1,6 @@
 def foreman_version = 'nightly'
 def katello_version = 'nightly'
+def stage_source = 'koji'
 def foreman_el_releases = [
     'el8'
 ]

diff --git a/theforeman.org/yaml/jobs/pipeline/foreman-release-pipelines.yaml b/theforeman.org/yaml/jobs/pipeline/foreman-release-pipelines.yaml
@@ -24,6 +24,8 @@
     sandbox: true
     triggers:
       - timed: 'H 21 * * *'
+    wrappers:
+      - ansicolor
     dsl:
       !include-raw:
         - 'pipelines/vars/foreman/{version}.groovy'