diff --git a/theforeman.org/pipelines/lib/obal.groovy b/theforeman.org/pipelines/lib/obal.groovy index 0a55d448..2a7157b0 100644 --- a/theforeman.org/pipelines/lib/obal.groovy +++ b/theforeman.org/pipelines/lib/obal.groovy @@ -7,7 +7,13 @@ def obal(args) { packages = packages.join(' ') } - def cmd = "python -m obal ${args.action} ${packages}" + if (env.NODE_LABELS.contains('el8')) { + python = 'python3.11' + } else { + python = 'python' + } + + def cmd = "${python} -m obal ${args.action} ${packages}" if (extra_vars) { extra_vars_file = writeExtraVars(extraVars: extra_vars) diff --git a/theforeman.org/pipelines/lib/packaging.groovy b/theforeman.org/pipelines/lib/packaging.groovy index baca5a74..0a21acde 100644 --- a/theforeman.org/pipelines/lib/packaging.groovy +++ b/theforeman.org/pipelines/lib/packaging.groovy @@ -463,3 +463,23 @@ def rsync_debian(user, ssh_key, suite, component, deb_paths) { """ } } + +def rsync_to_yum_stage(collection, target, version) { + def user = 'yumrepostage' + def ssh_key = '/home/jenkins/workspace/staging_key/rsync_yumrepostage_key' + + rsync_yum(user, ssh_key, collection, target, version) +} + +def rsync_yum(user, ssh_key, collection, target, version) { + def hosts = ["web01.osuosl.theforeman.org"] + + for(host in hosts) { + def target_path = "${user}@${host}:rsync_cache/${target}" + + sh """ + export RSYNC_RSH="ssh -i ${ssh_key}" + /usr/bin/rsync --archive --verbose --partial --one-file-system --delete-after ${collection}/${version} ${target_path} + """ + } +} diff --git a/theforeman.org/pipelines/lib/release.groovy b/theforeman.org/pipelines/lib/release.groovy index d32946b1..bd2eb7e6 100644 --- a/theforeman.org/pipelines/lib/release.groovy +++ b/theforeman.org/pipelines/lib/release.groovy @@ -3,13 +3,13 @@ void push_foreman_rpms(repo_type, version, distro) { push_rpms("foreman-${repo_type}-${version}", repo_type, version, distro) } -void push_rpms(repo_src, repo_dest, version, distro, keep_old_files = false) { - push_rpms_direct("${repo_src}/${distro}", "${repo_dest}/${version}/${distro}", !keep_old_files, keep_old_files) +void push_rpms(repo_src, repo_dest, version, distro, keep_old_files = false, staging = false) { + push_rpms_direct("${repo_src}/${distro}", "${repo_dest}/${version}/${distro}", !keep_old_files, keep_old_files, staging) } -void push_rpms_direct(repo_source, repo_dest, overwrite = true, merge = false) { +void push_rpms_direct(repo_source, repo_dest, overwrite = true, merge = false, staging = false) { sshagent(['repo-sync']) { - sh "ssh yumrepo@web01.osuosl.theforeman.org ${repo_source} ${repo_dest} ${overwrite} ${merge}" + sh "ssh yumrepo@web01.osuosl.theforeman.org ${repo_source} ${repo_dest} ${overwrite} ${merge} ${staging}" } } @@ -40,3 +40,13 @@ void mash(collection, version) { sh "ssh -o 'BatchMode yes' root@koji.katello.org collection-mash-split.py ${collection} ${version}" } } + +void push_staging_rpms(repo_src, repo_dest, version, distro, keep_old_files = false) { + push_rpms_direct("${repo_src}/${distro}", "${repo_dest}/${version}/${distro}", !keep_old_files, keep_old_files, true) +} + +void push_foreman_staging_rpms(repo_type, version, distro) { + version = version == 'develop' ? 'nightly' : version + push_staging_rpms("${repo_type}/${version}", repo_type, version, distro) +} + diff --git a/theforeman.org/pipelines/release/pipelines/client.groovy b/theforeman.org/pipelines/release/pipelines/client.groovy index 8093a96d..7d15679f 100644 --- a/theforeman.org/pipelines/release/pipelines/client.groovy +++ b/theforeman.org/pipelines/release/pipelines/client.groovy @@ -9,43 +9,110 @@ pipeline { } stages { - stage('Mash Koji Repositories') { - agent { label 'sshkey' } - - steps { - mash('foreman-client', foreman_version) + stage('koji') { + when { + expression { stage_source == 'koji' } } - } - stage('Repoclosure') { - agent { label 'el' } + stages { + stage('koji-mash-repositories') { + agent { label 'sshkey' } - steps { - script { - def parallelStagesMap = [:] - def name = 'foreman-client' - foreman_client_distros.each { distro -> - if (distro.startsWith('el')) { - parallelStagesMap[distro] = { repoclosure(name, distro, foreman_version) } - } else if (distro.startsWith('fc')) { - parallelStagesMap[distro] = { repoclosure(name, distro.replace('fc', 'f'), foreman_version) } - } - } - parallel parallelStagesMap + steps { + mash('foreman-client', foreman_version) + } } - } - post { - always { - deleteDir() + stage('koji-repoclosure') { + agent { label 'el' } + + steps { + script { + def parallelStagesMap = [:] + def name = 'foreman-client' + foreman_client_distros.each { distro -> + if (distro.startsWith('el')) { + parallelStagesMap[distro] = { repoclosure(name, distro, foreman_version) } + } else if (distro.startsWith('fc')) { + parallelStagesMap[distro] = { repoclosure(name, distro.replace('fc', 'f'), foreman_version) } + } + } + parallel parallelStagesMap + } + } + post { + always { + deleteDir() + } + } + } + stage('koji-push-rpms') { + agent { label 'admin && sshkey' } + + steps { + script { + foreman_client_distros.each { distro -> + push_foreman_rpms('client', foreman_version, distro) + } + } + } } } } - stage('Push RPMs') { - agent { label 'admin && sshkey' } + stage('staging') { + agent { label 'el8' } + when { + expression { stage_source == 'stagingyum' } + } + stages { + stage('staging-build-repository') { + steps { + git url: "https://github.com/theforeman/theforeman-rel-eng", poll: false - steps { - script { - foreman_client_distros.each { distro -> - push_foreman_rpms('client', foreman_version, distro) + script { + foreman_client_distros.each { distro -> + sh "./build_stage_repository foreman-client ${foreman_version} ${distro}" + } + } + } + } + stage('staging-copy-repository') { + steps { + script { + dir('tmp') { + rsync_to_yum_stage('foreman-client', 'client', foreman_version) + } + } + } + } + stage('staging-repoclosure') { + steps { + script { + def parallelStagesMap = [:] + def name = 'foreman-client-staging' + foreman_client_distros.each { distro -> + if (distro.startsWith('el')) { + parallelStagesMap[distro] = { repoclosure(name, distro, foreman_version) } + } else if (distro.startsWith('fc')) { + parallelStagesMap[distro] = { repoclosure(name, distro.replace('fc', 'f'), foreman_version) } + } + } + parallel parallelStagesMap + } + } + post { + always { + deleteDir() + } + } + } + stage('staging-push-rpms') { + agent { label 'admin && sshkey' } + + steps { + script { + foreman_client_distros.each { distro -> + push_foreman_staging_rpms('client', foreman_version, distro) + } + } } } } diff --git a/theforeman.org/pipelines/vars/foreman/3.4.groovy b/theforeman.org/pipelines/vars/foreman/3.4.groovy index 4d042f34..4310f4e3 100644 --- a/theforeman.org/pipelines/vars/foreman/3.4.groovy +++ b/theforeman.org/pipelines/vars/foreman/3.4.groovy @@ -1,5 +1,7 @@ def foreman_version = '3.4' def git_branch = "${foreman_version}-stable" +def stage_source = 'koji' + def foreman_client_distros = [ 'el9', 'el8', diff --git a/theforeman.org/pipelines/vars/foreman/3.5.groovy b/theforeman.org/pipelines/vars/foreman/3.5.groovy index f13ff717..189a9dc4 100644 --- a/theforeman.org/pipelines/vars/foreman/3.5.groovy +++ b/theforeman.org/pipelines/vars/foreman/3.5.groovy @@ -1,5 +1,7 @@ def foreman_version = '3.5' def git_branch = "${foreman_version}-stable" +def stage_source = 'koji' + def foreman_client_distros = [ 'el9', 'el8', diff --git a/theforeman.org/pipelines/vars/foreman/3.6.groovy b/theforeman.org/pipelines/vars/foreman/3.6.groovy index c1a57769..7c1de6b4 100644 --- a/theforeman.org/pipelines/vars/foreman/3.6.groovy +++ b/theforeman.org/pipelines/vars/foreman/3.6.groovy @@ -1,5 +1,7 @@ def foreman_version = '3.6' def git_branch = "${foreman_version}-stable" +def stage_source = 'koji' + def foreman_client_distros = [ 'el9', 'el8', diff --git a/theforeman.org/pipelines/vars/foreman/3.7.groovy b/theforeman.org/pipelines/vars/foreman/3.7.groovy index 2c114921..4194625f 100644 --- a/theforeman.org/pipelines/vars/foreman/3.7.groovy +++ b/theforeman.org/pipelines/vars/foreman/3.7.groovy @@ -1,5 +1,7 @@ def foreman_version = '3.7' def git_branch = "${foreman_version}-stable" +def stage_source = 'koji' + def foreman_client_distros = [ 'el9', 'el8', diff --git a/theforeman.org/pipelines/vars/foreman/3.8.groovy b/theforeman.org/pipelines/vars/foreman/3.8.groovy index a6e158d3..524b65fc 100644 --- a/theforeman.org/pipelines/vars/foreman/3.8.groovy +++ b/theforeman.org/pipelines/vars/foreman/3.8.groovy @@ -1,5 +1,7 @@ def foreman_version = '3.8' def git_branch = "${foreman_version}-stable" +def stage_source = 'koji' + def foreman_client_distros = [ 'el9', 'el8', diff --git a/theforeman.org/pipelines/vars/foreman/nightly.groovy b/theforeman.org/pipelines/vars/foreman/nightly.groovy index 5efb0f3d..59c47277 100644 --- a/theforeman.org/pipelines/vars/foreman/nightly.groovy +++ b/theforeman.org/pipelines/vars/foreman/nightly.groovy @@ -1,10 +1,11 @@ def foreman_version = 'nightly' def git_branch = "develop" +def stage_source = 'koji' + def foreman_client_distros = [ 'el9', 'el8', - 'el7', - 'sles12' + 'el7' ] def foreman_el_releases = [ 'el8' diff --git a/theforeman.org/pipelines/vars/katello/4.10.groovy b/theforeman.org/pipelines/vars/katello/4.10.groovy index 0743bdbe..3daf6a9b 100644 --- a/theforeman.org/pipelines/vars/katello/4.10.groovy +++ b/theforeman.org/pipelines/vars/katello/4.10.groovy @@ -1,5 +1,6 @@ def foreman_version = '3.8' def katello_version = '4.10' +def stage_source = 'koji' def foreman_el_releases = [ 'el8' ] diff --git a/theforeman.org/pipelines/vars/katello/4.6.groovy b/theforeman.org/pipelines/vars/katello/4.6.groovy index b357d063..1dbdcc4a 100644 --- a/theforeman.org/pipelines/vars/katello/4.6.groovy +++ b/theforeman.org/pipelines/vars/katello/4.6.groovy @@ -1,5 +1,6 @@ def foreman_version = '3.4' def katello_version = '4.6' +def stage_source = 'koji' def foreman_el_releases = [ 'el8' ] diff --git a/theforeman.org/pipelines/vars/katello/4.7.groovy b/theforeman.org/pipelines/vars/katello/4.7.groovy index c2547398..f7f8a44c 100644 --- a/theforeman.org/pipelines/vars/katello/4.7.groovy +++ b/theforeman.org/pipelines/vars/katello/4.7.groovy @@ -1,5 +1,6 @@ def foreman_version = '3.5' def katello_version = '4.7' +def stage_source = 'koji' def foreman_el_releases = [ 'el8' ] diff --git a/theforeman.org/pipelines/vars/katello/4.8.groovy b/theforeman.org/pipelines/vars/katello/4.8.groovy index a41de10e..f68e11b2 100644 --- a/theforeman.org/pipelines/vars/katello/4.8.groovy +++ b/theforeman.org/pipelines/vars/katello/4.8.groovy @@ -1,5 +1,6 @@ def foreman_version = '3.6' def katello_version = '4.8' +def stage_source = 'koji' def foreman_el_releases = [ 'el8' ] diff --git a/theforeman.org/pipelines/vars/katello/4.9.groovy b/theforeman.org/pipelines/vars/katello/4.9.groovy index 3fb6cfb0..23919ad0 100644 --- a/theforeman.org/pipelines/vars/katello/4.9.groovy +++ b/theforeman.org/pipelines/vars/katello/4.9.groovy @@ -1,5 +1,6 @@ def foreman_version = '3.7' def katello_version = '4.9' +def stage_source = 'koji' def foreman_el_releases = [ 'el8' ] diff --git a/theforeman.org/pipelines/vars/katello/nightly.groovy b/theforeman.org/pipelines/vars/katello/nightly.groovy index c3791972..c05f8861 100644 --- a/theforeman.org/pipelines/vars/katello/nightly.groovy +++ b/theforeman.org/pipelines/vars/katello/nightly.groovy @@ -1,5 +1,6 @@ def foreman_version = 'nightly' def katello_version = 'nightly' +def stage_source = 'koji' def foreman_el_releases = [ 'el8' ] diff --git a/theforeman.org/yaml/jobs/pipeline/foreman-release-pipelines.yaml b/theforeman.org/yaml/jobs/pipeline/foreman-release-pipelines.yaml index 2b196c05..c4b92164 100644 --- a/theforeman.org/yaml/jobs/pipeline/foreman-release-pipelines.yaml +++ b/theforeman.org/yaml/jobs/pipeline/foreman-release-pipelines.yaml @@ -24,6 +24,8 @@ sandbox: true triggers: - timed: 'H 21 * * *' + wrappers: + - ansicolor dsl: !include-raw: - 'pipelines/vars/foreman/{version}.groovy'