Pipeline 2024 05 08 (#2636)

* add ebsco reindexer topic

* new pipeline

pipeline/terraform/2024-05-08/main.tf

@@ -0,0 +1,30 @@
+module "pipeline" {
+  source = "../modules/stack"
+
+  reindexing_state = {
+    listen_to_reindexer      = false
+    scale_up_tasks           = false
+    scale_up_elastic_cluster = false
+    scale_up_id_minter_db    = false
+    scale_up_matcher_db      = false
+  }
+
+  index_config = {
+    works = {
+      identified = "works_identified.2023-05-26"
+      merged     = "works_merged.2023-05-26"
+      indexed    = "works_indexed.2024-01-09"
+    }
+    images = {
+      indexed        = "images_indexed.2024-01-09"
+      works_analysis = "works_indexed.2024-01-09"
+    }
+  }
+
+  pipeline_date = local.pipeline_date
+  release_label = local.pipeline_date
+
+  providers = {
+    aws.catalogue = aws.catalogue
+  }
+}

pipeline/terraform/2024-05-08/pipeline_config.tf

@@ -0,0 +1,18 @@
+# Note: this file is autogenerated by the run_terraform.sh script.
+#
+# Edits to this file may be reverted!
+
+locals {
+  pipeline_date = "2024-05-08"
+}
+
+terraform {
+  backend "s3" {
+    role_arn = "arn:aws:iam::760097843905:role/platform-developer"
+
+    bucket         = "wellcomecollection-platform-infra"
+    key            = "terraform/catalogue-pipeline/pipeline/2024-05-08.tfstate"
+    dynamodb_table = "terraform-locktable"
+    region         = "eu-west-1"
+  }
+}

pipeline/terraform/2024-05-08/run_terraform.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+
+# Get the path to the current directory, which we can use to find the
+# 'scripts' folder and the date of the current pipeline.
+#
+# https://stackoverflow.com/q/59895/1558022
+THIS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+SCRIPT_DIR="$(dirname "$THIS_DIR")/scripts"
+
+# Create the config file that tells Terraform which pipeline we're running
+# in and where to store the remote state.
+export PIPELINE_DATE="$(basename "$THIS_DIR")"
+. $SCRIPT_DIR/create_terraform_config_file.sh
+
+# Get the Elastic Cloud API key, which we need for the EC provider
+EC_API_KEY=$(. "$SCRIPT_DIR/get_elastic_cloud_api_key.sh")
+
+# Now run Terraform itself, passing any arguments directly to the underlying
+# Terraform binary.
+EC_API_KEY=$EC_API_KEY terraform "$@"

pipeline/terraform/2024-05-08/terraform.tf

@@ -0,0 +1,32 @@
+provider "aws" {
+  region = "eu-west-1"
+
+  assume_role {
+    role_arn = "arn:aws:iam::760097843905:role/platform-admin"
+  }
+}
+
+provider "aws" {
+  region = "eu-west-1"
+
+  alias = "catalogue"
+
+  assume_role {
+    role_arn = "arn:aws:iam::756629837203:role/catalogue-developer"
+  }
+}
+
+provider "ec" {}
+
+terraform {
+  required_version = ">= 0.13"
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+    }
+    ec = {
+      source  = "elastic/ec"
+      version = "0.2.1"
+    }
+  }
+}

pipeline/terraform/modules/stack/locals.tf

@@ -109,6 +109,7 @@ locals {
   calm_deletions_topic_arn = data.terraform_remote_state.calm_adapter.outputs.calm_deletions_topic_arn
 
   # Reindexer topics
+  ebsco_reindexer_topic_arn  = data.terraform_remote_state.reindexer.outputs.ebsco_reindexer_topic_arn
   miro_reindexer_topic_arn   = data.terraform_remote_state.shared_infra.outputs.catalogue_miro_reindex_topic_arn
   sierra_reindexer_topic_arn = data.terraform_remote_state.shared_infra.outputs.catalogue_sierra_reindex_topic_arn
   mets_reindexer_topic_arn   = data.terraform_remote_state.reindexer.outputs.mets_reindexer_topic_arn
@@ -168,7 +169,7 @@ locals {
       topics = [
         local.ebsco_adapter_topic_arn,
       ],
-      reindex_topic = null # TODO: Add reindexer topic (the adapter doesn't have one yet)
+      reindex_topic = local.ebsco_reindexer_topic_arn
       read_policy   = data.aws_iam_policy_document.read_ebsco_adapter_bucket.json
     }
   }

reindexer/terraform/locals.tf

@@ -11,14 +11,17 @@ locals {
   reporting_sierra_reindex_topic_arn = data.terraform_remote_state.shared_infra.outputs.reporting_sierra_reindex_topic_arn
   catalogue_miro_reindex_topic_arn   = data.terraform_remote_state.shared_infra.outputs.catalogue_miro_reindex_topic_arn
   catalogue_sierra_reindex_topic_arn = data.terraform_remote_state.shared_infra.outputs.catalogue_sierra_reindex_topic_arn
-  mets_reindexer_topic_name          = module.mets_reindexer_topic.name
-  mets_reindexer_topic_arn           = module.mets_reindexer_topic.arn
-  tei_reindexer_topic_arn            = module.tei_reindexer_topic.arn
-  calm_reindexer_topic_name          = module.calm_reindexer_topic.name
-  calm_reindexer_topic_arn           = module.calm_reindexer_topic.arn
-  calm_deletion_checker_topic_name   = module.calm_deletion_checker_topic.name
-  calm_deletion_checker_topic_arn    = module.calm_deletion_checker_topic.arn
-  miro_updates_topic_arn             = data.terraform_remote_state.shared_infra.outputs.miro_updates_topic_arn
+
+  ebsco_reindexer_topic_name       = module.ebsco_reindexer_topic.name
+  ebsco_reindexer_topic_arn        = module.ebsco_reindexer_topic.arn
+  mets_reindexer_topic_name        = module.mets_reindexer_topic.name
+  mets_reindexer_topic_arn         = module.mets_reindexer_topic.arn
+  tei_reindexer_topic_arn          = module.tei_reindexer_topic.arn
+  calm_reindexer_topic_name        = module.calm_reindexer_topic.name
+  calm_reindexer_topic_arn         = module.calm_reindexer_topic.arn
+  calm_deletion_checker_topic_name = module.calm_deletion_checker_topic.name
+  calm_deletion_checker_topic_arn  = module.calm_deletion_checker_topic.arn
+  miro_updates_topic_arn           = data.terraform_remote_state.shared_infra.outputs.miro_updates_topic_arn
 
   vpc_id          = local.catalogue_vpcs["catalogue_vpc_delta_id"]
   private_subnets = local.catalogue_vpcs["catalogue_vpc_delta_private_subnets"]

reindexer/terraform/outputs.tf

@@ -7,6 +7,9 @@ output "topic_arn" {
   value = module.reindex_worker.topic_arn
 }
 
+output "ebsco_reindexer_topic_arn" {
+  value = local.ebsco_reindexer_topic_arn
+}
 output "tei_reindexer_topic_arn" {
   value = local.tei_reindexer_topic_arn
 }

reindexer/terraform/topics.tf

@@ -1,3 +1,7 @@
+module "ebsco_reindexer_topic" {
+  source = "github.com/wellcomecollection/terraform-aws-sns-topic.git?ref=v1.0.0"
+  name   = "ebsco_reindexer_topic"
+}
 module "mets_reindexer_topic" {
   source = "github.com/wellcomecollection/terraform-aws-sns-topic.git?ref=v1.0.0"
   name   = "mets_reindexer_topic"