[VAULT-27917] fix(enos): handle SLES guestregister.service unreliabil…

…ity (#27380)

* [VAULT-27917] fix(enos): handle SLES guestregister.service unreliability

The SLES provided `guestregister.service` systemd unit is unreliable
enough that it will fail ~ 1/9 times when provisioning SLES instances.
When this happens the machine will never successfully exec SUSEConnect
to enroll and we'll get no access to the SLES repositories and
subsequently break our scenarios.

I resolved this by restructuring our `install_packages` module to to
separate repository synchronization, repository addition, and package
installation into different scripts and resources and by adding special
case handling for SLES and the `guestregister.service`.

I also make a distinction between `dnf` and `yum` because while they are
sort of the same thing on RHEL, it is not the case with Amazon2. I also
shimmed out the rest of the support for Apt in case we ever need to add
repos there.

* Revert "Temporarily remove SLES from samples (#27378)"

This reverts commit 490cdd9.

Signed-off-by: Ryan Cragun <[email protected]>

enos/modules/install_packages/main.tf

@@ -15,16 +15,15 @@ locals {
     "arm64" = "aarch64"
   }
   package_manager = {
-    # Note: though we generally use "amzn2" as our distro name for Amazon Linux 2, 
+    # Note: though we generally use "amzn2" as our distro name for Amazon Linux 2,
     # enos_host_info.hosts[each.key].distro returns "amzn", so that is what we reference here.
     "amzn"          = "yum"
     "opensuse-leap" = "zypper"
-    "rhel"          = "yum"
+    "rhel"          = "dnf"
     "sles"          = "zypper"
     "ubuntu"        = "apt"
   }
   distro_repos = {
-    # Currently sles is the only distro that requires setting up repos before installing packages
     "sles" = {
       "15.5" = "https://download.opensuse.org/repositories/network:utilities/SLE_15_SP5/network:utilities.repo"
     }
@@ -50,7 +49,7 @@ variable "hosts" {
 
 variable "timeout" {
   type        = number
-  description = "The max number of seconds to wait before timing out"
+  description = "The max number of seconds to wait before timing out. This is applied to each step so total timeout will be longer."
   default     = 120
 }
 
@@ -70,19 +69,41 @@ resource "enos_host_info" "hosts" {
   }
 }
 
-# Set up repos for each distro (in order to install some packages, some distros
-# require us to manually add the repo for that package first)
-resource "enos_remote_exec" "distro_repo_setup" {
+# Synchronize repositories on remote machines. This does not update packages but only ensures that
+# the remote hosts are configured with default upstream repositories that have been refreshed to
+# the latest metedata.
+resource "enos_remote_exec" "synchronize_repos" {
   for_each = var.hosts
 
   environment = {
     DISTRO          = enos_host_info.hosts[each.key].distro
+    PACKAGE_MANAGER = local.package_manager[enos_host_info.hosts[each.key].distro]
+    RETRY_INTERVAL  = var.retry_interval
+    TIMEOUT_SECONDS = var.timeout
+  }
+
+  scripts = [abspath("${path.module}/scripts/synchronize-repos.sh")]
+
+  transport = {
+    ssh = {
+      host = each.value.public_ip
+    }
+  }
+}
+
+# Add any additional repositories.
+resource "enos_remote_exec" "add_repos" {
+  for_each   = var.hosts
+  depends_on = [enos_remote_exec.synchronize_repos]
+
+  environment = {
     DISTRO_REPOS    = try(local.distro_repos[enos_host_info.hosts[each.key].distro][enos_host_info.hosts[each.key].distro_version], "__none")
+    PACKAGE_MANAGER = local.package_manager[enos_host_info.hosts[each.key].distro]
     RETRY_INTERVAL  = var.retry_interval
     TIMEOUT_SECONDS = var.timeout
   }
 
-  scripts = [abspath("${path.module}/scripts/distro-repo-setup.sh")]
+  scripts = [abspath("${path.module}/scripts/add-repos.sh")]
 
   transport = {
     ssh = {
@@ -91,9 +112,13 @@ resource "enos_remote_exec" "distro_repo_setup" {
   }
 }
 
+# Install any required packages.
 resource "enos_remote_exec" "install_packages" {
-  for_each   = var.hosts
-  depends_on = [enos_remote_exec.distro_repo_setup]
+  for_each = var.hosts
+  depends_on = [
+    enos_remote_exec.synchronize_repos,
+    enos_remote_exec.add_repos,
+  ]
 
   environment = {
     PACKAGE_MANAGER = local.package_manager[enos_host_info.hosts[each.key].distro]

enos/modules/install_packages/scripts/add-repos.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+set -e
+
+fail() {
+  echo "$1" 1>&2
+  exit 1
+}
+
+[[ -z "${PACKAGE_MANAGER}" ]] && fail "PACKAGE_MANAGER env variable has not been set"
+[[ -z "${RETRY_INTERVAL}" ]] && fail "RETRY_INTERVAL env variable has not been set"
+[[ -z "${TIMEOUT_SECONDS}" ]] && fail "TIMEOUT_SECONDS env variable has not been set"
+
+# Add any repositories that have have been passed in
+add_repos() {
+  # If we don't have any repos on the list for this distro, no action needed.
+  if [ ${#DISTRO_REPOS[@]} -lt 1 ]; then
+    echo "DISTRO_REPOS is empty; No repos required for the packages for this Linux distro."
+    return 0
+  fi
+
+  case $PACKAGE_MANAGER in
+    apt)
+      # NOTE: We do not currently add any apt repositories in our scenarios. I suspect if that time
+      # comes we'll need to add support for apt-key here.
+      for repo in ${DISTRO_REPOS}; do
+        if [ "$repo" == "__none" ]; then
+          continue
+        fi
+        sudo add-apt-repository "${repo}"
+      done
+    ;;
+    dnf)
+      for repo in ${DISTRO_REPOS}; do
+        if [ "$repo" == "__none" ]; then
+          continue
+        fi
+        sudo dnf install -y "${repo}"
+        sudo dnf makecache -y
+      done
+    ;;
+    yum)
+      for repo in ${DISTRO_REPOS}; do
+        if [ "$repo" == "__none" ]; then
+          continue
+        fi
+        sudo yum install -y "${repo}"
+        sudo yum makecache -y
+      done
+    ;;
+    zypper)
+      # Add each repo
+      for repo in ${DISTRO_REPOS}; do
+        if [ "$repo" == "__none" ]; then
+          continue
+        fi
+        if sudo zypper lr "${repo}"; then
+          echo "A repo named ${repo} already exists, skipping..."
+          continue
+        fi
+        sudo zypper --gpg-auto-import-keys --non-interactive addrepo "${repo}"
+      done
+      sudo zypper --gpg-auto-import-keys ref
+      sudo zypper --gpg-auto-import-keys refs
+    ;;
+    *)
+      fail "Unsupported package manager: ${PACKAGE_MANAGER}"
+  esac
+}
+
+begin_time=$(date +%s)
+end_time=$((begin_time + TIMEOUT_SECONDS))
+while [ "$(date +%s)" -lt "$end_time" ]; do
+  if add_repos; then
+    exit 0
+  fi
+
+  sleep "$RETRY_INTERVAL"
+done
+
+fail "Timed out waiting for distro repos to be set up"

enos/modules/install_packages/scripts/install-packages.sh

@@ -14,59 +14,81 @@ fail() {
 [[ -z "${PACKAGES}" ]] && fail "PACKAGES env variable has not been set"
 [[ -z "${PACKAGE_MANAGER}" ]] && fail "PACKAGE_MANAGER env variable has not been set"
 
+# Install packages based on the provided packages and package manager. We assume that the repositories
+# have already been synchronized by the repo setup that is a prerequisite for this script.
 install_packages() {
   if [[ "${PACKAGES}" = "__skip" ]]; then
     return 0
-  fi 
+  fi
 
   set -x
   echo "Installing Dependencies: ${PACKAGES}"
 
   # Use the default package manager of the current Linux distro to install packages
   case $PACKAGE_MANAGER in
-
-    "apt")
-      sudo apt update
+    apt)
       for package in ${PACKAGES}; do
         if dpkg -s "${package}"; then
+          echo "Skipping installation of ${package} because it is already installed"
           continue
         else
           echo "Installing ${package}"
-          sudo apt install -y "${package}"
+          local output
+          if ! output=$(sudo apt install -y "${package}" 2>&1); then
+            echo "Failed to install ${package}: ${output}" 1>&2
+            return 1
+          fi
         fi
       done
-      ;;
-
-    "yum")
+    ;;
+    dnf)
       for package in ${PACKAGES}; do
         if rpm -q "${package}"; then
+          echo "Skipping installation of ${package} because it is already installed"
           continue
         else
           echo "Installing ${package}"
-          sudo yum -y install "${package}"
+          local output
+          if ! output=$(sudo dnf -y install "${package}" 2>&1); then
+            echo "Failed to install ${package}: ${output}" 1>&2
+            return 1
+          fi
         fi
       done
-      ;;
-
-    "zypper")
-      cd /tmp
-      sudo zypper --gpg-auto-import-keys ref
+    ;;
+    yum)
       for package in ${PACKAGES}; do
         if rpm -q "${package}"; then
+          echo "Skipping installation of ${package} because it is already installed"
           continue
         else
           echo "Installing ${package}"
-          sudo zypper --non-interactive install "${package}"
-          date
+          local output
+          if ! output=$(sudo yum -y install "${package}" 2>&1); then
+            echo "Failed to install ${package}: ${output}" 1>&2
+            return 1
+          fi
         fi
-        sudo zypper search -i
       done
-      ;;
-
+    ;;
+    zypper)
+      for package in ${PACKAGES}; do
+        if rpm -q "${package}"; then
+          echo "Skipping installation of ${package} because it is already installed"
+          continue
+        else
+          echo "Installing ${package}"
+          local output
+          if ! output=$(sudo zypper --non-interactive install -y -l --force-resolution "${package}" 2>&1); then
+            echo "Failed to install ${package}: ${output}" 1>&2
+            return 1
+          fi
+        fi
+      done
+    ;;
     *)
       fail "No matching package manager provided."
-      ;;
-
+    ;;
   esac
 }