diff --git a/Vagrantfile b/Vagrantfile index 22f7f33cb..bc5bcfb5b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -134,4 +134,15 @@ Vagrant.configure("2") do |config| provider.vm.box_url = CENTOS_9_BOX_URL end end + + config.vm.define "repo-rpm" do |override| + override.vm.hostname = "repo-rpm" + override.vm.box = "centos/stream9" + + override.vm.provider "libvirt" do |libvirt, provider| + libvirt.memory = "2048" + libvirt.machine_virtual_size = 40 + provider.vm.box_url = CENTOS_9_BOX_URL + end + end end diff --git a/puppet/data/common.yaml b/puppet/data/common.yaml index 18e46c4e0..5aab609a7 100644 --- a/puppet/data/common.yaml +++ b/puppet/data/common.yaml @@ -2,6 +2,7 @@ stable_release: '3.11' profiles::web::stable: '%{alias("stable_release")}' profiles::repo::deb::stable: '%{alias("stable_release")}' +profiles::repo::rpm::stable_foreman: '%{alias("stable_release")}' backup_servicename: 'backups.theforeman.org' backup_username: 'backup-%{facts.networking.hostname}' @@ -241,3 +242,13 @@ sudo::wheel_config: password redmine::https: true apache::default_vhost: false + +rsync_usernames: + - 'ehelms' + - 'ekohl' + - 'evgeni' + - 'Odilhao' + - 'pcreech' + - 'zhunting' + +web::vhost::stagingrpm::usernames: '%{alias("rsync_usernames")}' diff --git a/puppet/data/vagrant.yaml b/puppet/data/vagrant.yaml index de705fbea..e556ddd3f 100644 --- a/puppet/data/vagrant.yaml +++ b/puppet/data/vagrant.yaml @@ -23,5 +23,6 @@ profiles::jenkins::node::swap_size_mb: 0 profiles::web::https: false profiles::repo::deb::https: false +profiles::repo::rpm::https: false redmine::https: false diff --git a/puppet/manifests/site.pp b/puppet/manifests/site.pp index 392eb3326..b43befa30 100644 --- a/puppet/manifests/site.pp +++ b/puppet/manifests/site.pp @@ -46,3 +46,8 @@ include profiles::base include profiles::repo::deb } + +node /^repo-rpm\d+\.[a-z]+\.theforeman\.org$/ { + include profiles::base + include profiles::repo::rpm +} diff --git a/puppet/modules/profiles/manifests/repo/rpm.pp b/puppet/modules/profiles/manifests/repo/rpm.pp new file mode 100644 index 000000000..0c1c45af8 --- /dev/null +++ b/puppet/modules/profiles/manifests/repo/rpm.pp @@ -0,0 +1,25 @@ +# @summary A profile for the rpm repo machines +# +# @param stable_foreman +# Latest Foreman release that users expect +# +# @param https +# Whether to enable HTTPS. This is typically wanted but can only be enabled +# in a 2 pass setup. First Apache needs to run for Letsencrypt to function. +# Then Letsencrypt can be enabled. Also useful to turn off in test setups. +class profiles::repo::rpm ( + String[1] $stable_foreman, + Boolean $https = true, +) { + class { 'web': + https => $https, + } + contain web + + class { 'web::vhost::rpm': + stable_foreman => $stable_foreman, + } + contain web::vhost::rpm + + contain web::vhost::stagingrpm +} diff --git a/puppet/modules/web/files/rpm/pulpcore-HEADER.html b/puppet/modules/web/files/rpm/pulpcore-HEADER.html new file mode 100644 index 000000000..701a77b8a --- /dev/null +++ b/puppet/modules/web/files/rpm/pulpcore-HEADER.html @@ -0,0 +1,3 @@ +

Pulpcore packages

+ +These are RPM builds for Pulp 3 and various plugins for use by Katello. They are only intended to be used by Katello. Only branches used by Katello are maintained. No explicit end of life announcements will be made. diff --git a/puppet/modules/web/files/rpm/robots.txt b/puppet/modules/web/files/rpm/robots.txt new file mode 100644 index 000000000..9fe8bff01 --- /dev/null +++ b/puppet/modules/web/files/rpm/robots.txt @@ -0,0 +1,3 @@ +User-agent: * +Disallow: /foreman/nightly/ +Disallow: /pulpcore/nightly/ diff --git a/puppet/modules/web/files/stagingrpm/robots.txt b/puppet/modules/web/files/stagingrpm/robots.txt new file mode 100644 index 000000000..1f53798bb --- /dev/null +++ b/puppet/modules/web/files/stagingrpm/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/puppet/modules/web/manifests/vhost/rpm.pp b/puppet/modules/web/manifests/vhost/rpm.pp new file mode 100644 index 000000000..6a14f6bcc --- /dev/null +++ b/puppet/modules/web/manifests/vhost/rpm.pp @@ -0,0 +1,106 @@ +# @summary Set up the rpm vhost +# @api private +class web::vhost::rpm ( + String[1] $stable_foreman, + Stdlib::Fqdn $servername = 'rpm.theforeman.org', + Stdlib::Absolutepath $rpm_directory = '/var/www/vhosts/rpm/htdocs', + Stdlib::Absolutepath $rpm_staging_directory = '/var/www/vhosts/stagingrpm/htdocs/', + String $user = 'rpmrepo', +) { + $rpm_directory_config = [ + { + path => $rpm_directory, + options => ['+Indexes', '+FollowSymLinks'], + expires_active => 'on', + expires_default => 'access plus 2 minutes', + }, + { + path => '.+\.(bz2|gz|rpm|xz)$', + provider => 'filesmatch', + expires_active => 'on', + expires_default => 'access plus 30 days', + }, + { + path => 'repomd.xml', + provider => 'files', + expires_active => 'on', + expires_default => 'access plus 2 minutes', + }, + ] + + $deploy_rpmrepo_context = { + 'servername' => $servername, + 'rpm_directory' => $rpm_directory, + 'rpm_staging_directory' => $rpm_staging_directory, + } + + secure_ssh::receiver_setup { $user: + user => $user, + foreman_search => 'host ~ node*.jenkins.osuosl.theforeman.org and (name = external_ip4 or name = external_ip6)', + script_content => epp('web/deploy-rpmrepo.sh.epp', $deploy_rpmrepo_context), + } + + include apache::mod::expires + include apache::mod::dir + include apache::mod::autoindex + include apache::mod::alias + include apache::mod::mime + + web::vhost { 'rpm': + servername => $servername, + docroot => $rpm_directory, + docroot_owner => $user, + docroot_group => $user, + docroot_mode => '0755', + directories => $rpm_directory_config, + } + + if $facts['os']['family'] == 'RedHat' { + package { 'createrepo_c': + ensure => present, + } + } + + file { "${rpm_directory}/robots.txt": + ensure => file, + owner => $user, + group => $user, + mode => '0644', + content => file('web/rpm/robots.txt'), + } + + file { "${rpm_directory}/HEADER.html": + ensure => file, + owner => $user, + group => $user, + mode => '0644', + content => epp("${module_name}/rpm/HEADER.html.epp", { + 'stable_foreman' => $stable_foreman, + 'servername' => $servername, + }), + } + + ['candlepin', 'foreman', 'pulpcore'].each |$directory| { + file { ["${rpm_directory}/${directory}"]: + ensure => directory, + owner => $user, + group => $user, + mode => '0755', + } + + exec { "fastly-purge-${directory}-latest": + command => "fastly-purge-find 'https://${servername}' ${rpm_directory} ${directory}/latest/", + path => '/bin:/usr/bin:/usr/local/bin', + require => File['/usr/local/bin/fastly-purge-find'], + refreshonly => true, + } + } + + file { "${rpm_directory}/pulpcore/HEADER.html": + ensure => file, + owner => $user, + group => $user, + mode => '0644', + content => file('web/rpm/pulpcore-HEADER.html'), + } +} diff --git a/puppet/modules/web/manifests/vhost/stagingrpm.pp b/puppet/modules/web/manifests/vhost/stagingrpm.pp new file mode 100644 index 000000000..0ffed108c --- /dev/null +++ b/puppet/modules/web/manifests/vhost/stagingrpm.pp @@ -0,0 +1,88 @@ +# @summary Set up the rpm staging vhost +# @api private +class web::vhost::stagingrpm ( + Array[String[1]] $usernames, + Stdlib::Fqdn $servername = 'stagingrpm.theforeman.org', + Stdlib::Absolutepath $rpm_staging_directory = '/var/www/vhosts/stagingrpm/htdocs', + String $user = 'rpmrepostage', + Stdlib::Absolutepath $home = "/home/${user}", +) { + $rpm_staging_directory_config = [ + { + path => $rpm_staging_directory, + options => ['Indexes', 'FollowSymLinks'], + expires_active => 'on', + expires_default => 'access plus 2 minutes', + }, + { + path => '.+\.(bz2|gz|rpm|xz)$', + provider => 'filesmatch', + expires_active => 'on', + expires_default => 'access plus 30 days', + }, + { + path => 'repomd.xml', + provider => 'files', + expires_active => 'on', + expires_default => 'access plus 2 minutes', + }, + ] + + include apache::mod::expires + include apache::mod::dir + include apache::mod::autoindex + include apache::mod::alias + include apache::mod::mime + + $authorized_keys = flatten($usernames.map |$name| { + split(file("users/${name}-authorized_keys"), "\n") + }) + + secure_ssh::rsync::receiver_setup { $user: + user => $user, + homedir => $home, + homedir_mode => '0750', + foreman_search => 'host ~ node*.jenkins.*.theforeman.org and (name = external_ip4 or name = external_ip6)', + authorized_keys => $authorized_keys, + script_content => epp("${module_name}/deploy-stagingrpm.sh.epp", { + 'home' => $home, + 'rpm_staging_directory' => $rpm_staging_directory, + }), + } + + web::vhost { 'stagingrpm': + servername => $servername, + docroot => $rpm_staging_directory, + docroot_owner => $user, + docroot_group => $user, + docroot_mode => '0755', + directories => $rpm_staging_directory_config, + } + + file { "${rpm_staging_directory}/robots.txt": + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + content => file('web/stagingrpm/robots.txt'), + } + + file { "${rpm_staging_directory}/HEADER.html": + ensure => file, + owner => 'root', + group => 'root', + mode => '0644', + content => epp("${module_name}/stagingrpm/HEADER.html.epp", { + 'servername' => $servername, + }), + } + + ['candlepin', 'foreman', 'pulpcore'].each |$directory| { + file { ["${rpm_staging_directory}/${directory}"]: + ensure => directory, + owner => $user, + group => $user, + mode => '0755', + } + } +} diff --git a/puppet/modules/web/manifests/vhost/stagingyum.pp b/puppet/modules/web/manifests/vhost/stagingyum.pp index c44b0e79f..fc0a345a8 100644 --- a/puppet/modules/web/manifests/vhost/stagingyum.pp +++ b/puppet/modules/web/manifests/vhost/stagingyum.pp @@ -1,11 +1,11 @@ # @summary Set up the yum vhost # @api private class web::vhost::stagingyum ( + Array[String[1]] $usernames, Stdlib::Fqdn $servername = 'stagingyum.theforeman.org', Stdlib::Absolutepath $yum_directory = '/var/www/vhosts/stagingyum/htdocs', String $user = 'yumrepostage', Stdlib::Absolutepath $home = "/home/${user}", - Array[String[1]] $usernames = ['ehelms', 'evgeni', 'ekohl', 'Odilhao', 'pcreech', 'zhunting'], ) { $yum_directory_config = [ { diff --git a/puppet/modules/web/templates/deploy-rpmrepo.sh.epp b/puppet/modules/web/templates/deploy-rpmrepo.sh.epp new file mode 100644 index 000000000..b8a3e6066 --- /dev/null +++ b/puppet/modules/web/templates/deploy-rpmrepo.sh.epp @@ -0,0 +1,123 @@ +<%- | + Stdlib::Fqdn $servername, + Stdlib::Absolutepath $rpm_directory, + Stdlib::Absolutepath $rpm_staging_directory, +| -%> +#!/bin/bash + +set -xe +# This is a forced SSH command - uncomment to test locally +set -f -- $SSH_ORIGINAL_COMMAND + +prepcache() { + if [[ -e $REPO_PATH ]]; then + if [[ $MERGE == false ]] && [[ $OVERWRITE == false ]] ; then + echo "Repo overwrite (${OVERWRITE}) and merge (${MERGE}) are disabled, but ${REPO_PATH} already exists" + exit 1 + fi + cp -al $REPO_PATH "$REPO_INSTANCE_PATH" + else + mkdir -p $REPO_INSTANCE_PATH + fi +} + +do_rsync() { + opts=(--archive --verbose --hard-links --log-file "$REPO_RSYNC_LOG") + if [[ $MERGE != true ]] ; then + opts+=('--delete') + fi + + for ARCH in x86_64 source; do + rsync "${opts[@]}" --log-file-format "CHANGED ${ARCH}/%n" "${REPO_SOURCE_RPM}/${ARCH}/" "${REPO_INSTANCE_PATH}/${ARCH}/" + done + + set +f + for d in "${REPO_INSTANCE_PATH}"/*; do + ( + cd "$d" + + latest=$(ls -t foreman-release-[0-9]*.rpm 2>/dev/null | head -n1) + if [[ -n "$latest" ]] ; then + ln -sf "$latest" foreman-release.rpm + fi + + latest=$(ls -t foreman-client-release-[0-9]*.rpm 2>/dev/null | head -n1) + if [[ -n "$latest" ]] ; then + ln -sf "$latest" foreman-client-release.rpm + fi + + latest=$(ls -t katello-repos-[0-9]*.rpm 2>/dev/null | head -n1) + if [[ -n "$latest" ]] ; then + ln -sf "$latest" katello-repos.rpm + fi + + if [[ $MERGE == true ]] ; then + HAS_MODULES_YAML=$(ls repodata/*-modules.yaml.gz >/dev/null 2>/dev/null && echo 'yes' || echo 'no') + + if [[ $HAS_MODULES_YAML == yes ]]; then + zcat repodata/*-modules.yaml.gz > modules.yaml + modifyrepo_c --remove modules repodata/ + rm -f repodata/*-modules.yaml.gz + fi + + createrepo_c --skip-symlinks --update . + + if [[ $HAS_MODULES_YAML == yes ]]; then + modifyrepo_c --mdtype=modules modules.yaml repodata/ + fi + fi + ) + done + set -f +} + +replace() { + if [[ -e $REPO_PATH ]]; then + mv "${REPO_PATH}" "${REPO_INSTANCE_PATH_PREV}" + fi + + mv "${REPO_INSTANCE_PATH}" "${REPO_PATH}" + + if [[ $MERGE == true ]] || [[ $OVERWRITE == true ]] ; then + if [[ -e "${REPO_INSTANCE_PATH_PREV}" ]]; then + rm -rf "${REPO_INSTANCE_PATH_PREV}" + fi + fi +} + +purgecdn() { + awk '/ CHANGED /{print $5}' "${REPO_RSYNC_LOG}" | xargs --no-run-if-empty fastly-purge "https://<%= $servername %>/${REPO_DEST}" + set +f + for d in "${REPO_PATH}"/*; do + purge_base="https://<%= $servername %>/${REPO_DEST}/$(basename $d)" + fastly-purge ${purge_base} foreman-release.rpm foreman-client-release.rpm katello-repos.rpm + done + set -f +} + +REPO_SOURCE=$1 +REPO_DEST=$2 +OVERWRITE=${3:-false} +MERGE=${4:-false} + +if [[ -z $REPO_SOURCE ]] || [[ -z $REPO_DEST ]] ; then + echo "Usage: $0 REPO_SOURCE REPO_DEST OVERWRITE MERGE" + exit 1 +fi + +REPO_SOURCE_BASE="<%= $rpm_staging_directory %>" +REPO_SOURCE_RPM="${REPO_SOURCE_BASE}/${REPO_SOURCE}" + +DEPLOY_TO="<%= $rpm_directory %>" +REPO_PATH="${DEPLOY_TO}/${REPO_DEST}" +REPO_INSTANCE_PATH="${DEPLOY_TO}/$(dirname $REPO_DEST)/.$(basename $REPO_DEST)-$(date "+%Y%m%d%H%M%S")" +REPO_INSTANCE_PATH_PREV="${REPO_INSTANCE_PATH}-previous" + +REPO_RSYNC_LOG=$(mktemp) + +trap "rm -rf $REPO_RSYNC_LOG $REPO_INSTANCE_PATH" EXIT + +prepcache +do_rsync +replace +purgecdn diff --git a/puppet/modules/web/templates/deploy-stagingrpm.sh.epp b/puppet/modules/web/templates/deploy-stagingrpm.sh.epp new file mode 100644 index 000000000..4e854f64c --- /dev/null +++ b/puppet/modules/web/templates/deploy-stagingrpm.sh.epp @@ -0,0 +1,15 @@ +<%- | + Stdlib::Absolutepath $home, + Stdlib::Absolutepath $rpm_staging_directory, +| -%> + # Make sure target dir can be created + RPM_PATH=`echo "${SSH_ORIGINAL_COMMAND}" | awk '{ print $NF }'` + PROJECT=`echo $RPM_PATH | /bin/cut -f2 -d/` + RELEASE=`echo $RPM_PATH | /bin/cut -f3 -d/` + mkdir -p <%= $home %>/rsync_cache/$PROJECT/$RELEASE + + # Permit transfer + $SSH_ORIGINAL_COMMAND + + # Publish the repo - stderr/out redirect is required to stop the noninteractive shell from hanging + rsync --recursive --times --verbose --one-file-system --delete-after <%= $home %>/rsync_cache/$PROJECT/$RELEASE <%= $rpm_staging_directory %>/$PROJECT/ 2>&1 >/dev/null ; diff --git a/puppet/modules/web/templates/rpm/HEADER.html.epp b/puppet/modules/web/templates/rpm/HEADER.html.epp new file mode 100644 index 000000000..cb0f55895 --- /dev/null +++ b/puppet/modules/web/templates/rpm/HEADER.html.epp @@ -0,0 +1,64 @@ +<%- | + String $stable_foreman, + Stdlib::Fqdn $servername, +| -%> +

<%= $servername %>

+ +

Foreman

+ +

Foreman is available under /releases/VERSION/DIST/ARCH, e.g.

+ + + +

foreman-release RPMs containing an appropriate .repo file are available with fixed URLs:

+ + + +

Release packages are signed with a new key for each major release. The public key is available in the RPM-GPG-KEY-foreman file within each version directory or the foreman-release RPMs.

+ +

Nightly builds of Foreman are available under /foreman/nightly/foreman/DIST/ARCH and are refreshed a few times a day, but are not GPG signed.

+ +

Plugins

+ +

A number of Foreman plugins are available in the plugin repos, see List of Plugins for more information.

+ +

Plugin repos are structured by the Foreman version that they're compatible with in the format /foreman/VERSION/plugins/DIST/ARCH, e.g.

+ + + +

Plugin repos are not GPG signed.

+ +

Katello

+ +

Katello is available under /foreman/VERSION/katello/DIST/ARCH with Candlepin under /candlepin/CANDLEPIN_VERSION/DIST/ARCH. + +

katello-repos RPMs containing an appropriate .repo file are available:

+ + + +

Accessing this repo

+ +This repository is available over HTTP and HTTPS: + + + +

Support

+ +

You can find the installation instructions here, but we strongly recommend using our Installer (which uses RPMs). + +

If you have any issues, you can find ways to reach us on our Support page.

diff --git a/puppet/modules/web/templates/stagingrpm/HEADER.html.epp b/puppet/modules/web/templates/stagingrpm/HEADER.html.epp new file mode 100644 index 000000000..661e76dd3 --- /dev/null +++ b/puppet/modules/web/templates/stagingrpm/HEADER.html.epp @@ -0,0 +1,19 @@ +<%- | + Stdlib::Fqdn $servername, +| -%> +

<%= $servername %>

+ +

RPM Staging Repositories for Foreman releases

+ +

Accessing this repo

+ +This repository is available over HTTP and HTTPS: + + + +

Support

+ +

These are staging repositories used for testing purposes only. They are NOT supported.

diff --git a/vagrant/manifests/default.pp b/vagrant/manifests/default.pp index 3e63ad1c5..1cf1dc2b3 100644 --- a/vagrant/manifests/default.pp +++ b/vagrant/manifests/default.pp @@ -29,3 +29,7 @@ node /^repo-deb.*/ { include profiles::repo::deb } + +node /^repo-rpm.*/ { + include profiles::repo::rpm +}