diff --git a/pipeline/terraform/2024-04-29/.terraform.lock.hcl b/pipeline/terraform/2024-04-29/.terraform.lock.hcl index 3909e1e83e..6737058d64 100644 --- a/pipeline/terraform/2024-04-29/.terraform.lock.hcl +++ b/pipeline/terraform/2024-04-29/.terraform.lock.hcl @@ -5,6 +5,7 @@ provider "registry.terraform.io/elastic/ec" { version = "0.2.1" constraints = "0.2.1" hashes = [ + "h1:Njh3l0Del+s0OKnOwuKvPZLcTPSdwx7fnLa8Ex/VJrg=", "h1:xkKP1EYtPN3kXPjuE9f4Ywe2AA/kM4X3ZPKen8xUMTs=", "zh:145414eaa5015fcad546eb17bd78a14720cadcfcca781a46c9a6493722eb3ba8", "zh:1cb7bacfe088c1145834e0a7d053f7aa7a8897a3554d040becad004774aa45f4", @@ -27,6 +28,7 @@ provider "registry.terraform.io/elastic/elasticstack" { constraints = "0.7.0" hashes = [ "h1:O0FhECkmPg0FUMmFR6vzpXz3Tb+ACs1pFCzkNT2yPF4=", + "h1:S8ZSgFIHsn++WvglAY93tpB7jmsBqXbX/xe0mRbW9mc=", "zh:036576eaa4ed8caf8da11b81cdac26940173cf8295bb69b5471278453a8db78b", "zh:20f3fb2e7f364d8750a56e56829fe9db46a014388fe266bf08b93ca7b2d72ba4", "zh:3250c8f46a3915db8234b98f453d1b712542767241fe9813bf092c2bcb948a05", @@ -47,6 +49,7 @@ provider "registry.terraform.io/elastic/elasticstack" { provider "registry.terraform.io/hashicorp/aws" { version = "5.47.0" hashes = [ + "h1:49aEnvHJ/M8BRGAXKzU6W3zSbf7HgIrjXkXjC5DGEWY=", "h1:T0tupfn2Ubj18Y7xmO0pFMvti1Qns2K6EGXenR6Hg30=", "zh:06037a14e47e8f82d0b3b326cd188566272b808b7970a9249a11db26d475b83d", "zh:116b7dd58ca964a1056249d2b6550f399b0a6bc9a7920b7ee134242114432c9f", @@ -70,6 +73,7 @@ provider "registry.terraform.io/hashicorp/null" { version = "3.2.2" hashes = [ "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", + "h1:vWAsYRd7MjYr3adj8BVKRohVfHpWQdvkIwUQ2Jf5FVM=", "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", @@ -88,6 +92,7 @@ provider "registry.terraform.io/hashicorp/null" { provider "registry.terraform.io/hashicorp/random" { version = "3.6.1" hashes = [ + "h1:8iqExjtAvirFTJkpm5YyYD+fC+DGV8NTJzKsE2c70VA=", "h1:a+Goawwh6Qtg4/bRWzfDtIdrEFfPlnVy0y4LdUQY3nI=", "zh:2a0ec154e39911f19c8214acd6241e469157489fc56b6c739f45fbed5896a176", "zh:57f4e553224a5e849c99131f5e5294be3a7adcabe2d867d8a4fef8d0976e0e52", diff --git a/pipeline/terraform/2024-05-08/.terraform.lock.hcl b/pipeline/terraform/2024-05-08/.terraform.lock.hcl new file mode 100644 index 0000000000..6737058d64 --- /dev/null +++ b/pipeline/terraform/2024-05-08/.terraform.lock.hcl @@ -0,0 +1,110 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/elastic/ec" { + version = "0.2.1" + constraints = "0.2.1" + hashes = [ + "h1:Njh3l0Del+s0OKnOwuKvPZLcTPSdwx7fnLa8Ex/VJrg=", + "h1:xkKP1EYtPN3kXPjuE9f4Ywe2AA/kM4X3ZPKen8xUMTs=", + "zh:145414eaa5015fcad546eb17bd78a14720cadcfcca781a46c9a6493722eb3ba8", + "zh:1cb7bacfe088c1145834e0a7d053f7aa7a8897a3554d040becad004774aa45f4", + "zh:20e245ae46316853ad18d2eff9107aec3e0ffcaa5407b2aa422669d80a872761", + "zh:4c9c52e4ee089d71a9f58f0f53173ded7a65012466dbb857b15e7697f25fdb40", + "zh:52877b6e934e932aa27d85868e055a907172df9b0ca6bcc4a2c820cc6890b152", + "zh:52cabbd5826ac10b5f4f9be25435f970ae0b9caed3cd3cdf42c71ee773c956ac", + "zh:999963c9f6ee6d61f3aea2c76d0e842d963662f71b04b9cf4e003d88bd849d0e", + "zh:9bd93ffef87de87b3f04bb8b572098ccb8d363a62a4c7fe43f74447ee4ee023f", + "zh:a39bdff8f72d1479a9fa6a2399dab687bf775bcb7a1c4db0e5ef87fbbf66bfa9", + "zh:aa3cd16eeae66ccaab372f3a6a819fe117516a4c96bc216663bc244983e11100", + "zh:c5d3cc19dcf190bea741e75c4f90e1cda4673bef7a6b86f2b7f2eb1fd926dce0", + "zh:fcdb180ef082876458ede8d1cdb2e703330120552c5d80268388d8437c9716fb", + "zh:fd595f502fcd0ca8ada078114fbf5341880ac8e53493105c42309560654f3729", + ] +} + +provider "registry.terraform.io/elastic/elasticstack" { + version = "0.7.0" + constraints = "0.7.0" + hashes = [ + "h1:O0FhECkmPg0FUMmFR6vzpXz3Tb+ACs1pFCzkNT2yPF4=", + "h1:S8ZSgFIHsn++WvglAY93tpB7jmsBqXbX/xe0mRbW9mc=", + "zh:036576eaa4ed8caf8da11b81cdac26940173cf8295bb69b5471278453a8db78b", + "zh:20f3fb2e7f364d8750a56e56829fe9db46a014388fe266bf08b93ca7b2d72ba4", + "zh:3250c8f46a3915db8234b98f453d1b712542767241fe9813bf092c2bcb948a05", + "zh:590126c06fd26e10fe0515f318e2f0aa6250bd0fa059f6ecd41e94ae4dd7e3bd", + "zh:59513920f1a07c4f3763940d151d16a765a5540e7db5f1e6334c3661e92d386b", + "zh:5b6e010ada8a84c722edbaacb179926b41808551737c60b5a0fa68c76679d2bf", + "zh:60db873c39a167a238ce67e1c76f075176ded92ba5fba102bd5611304ab14404", + "zh:7562e115f12d453507ba963a8277972deb506945fc99a1252ca2c4eb110b63f1", + "zh:828fe0a34a5ae29245752b012c7590b3f0fac3b23dd1cf20bfb6d803c922e770", + "zh:82c3a642fa0aab619b43ebe6c12846d3a38db96b13ee1842fbeb7f0a68315b40", + "zh:c0b671a672f2292c0e3f0206b5b3a9fc2fb865acc67e4c516038a00667c7b2dd", + "zh:c451cdbfeab83509a0cc3b09c5face658ece9d314072525f6e554d90dca65cd4", + "zh:c934918e470730e9d28ca6a14aeeb9d243f31b3b4a13ef2fd24090d9ff15bff7", + "zh:e846b3a4d8c6d0ed5b8652c14ff476844ffde9df4da4c088eb66e055276c7bbd", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.47.0" + hashes = [ + "h1:49aEnvHJ/M8BRGAXKzU6W3zSbf7HgIrjXkXjC5DGEWY=", + "h1:T0tupfn2Ubj18Y7xmO0pFMvti1Qns2K6EGXenR6Hg30=", + "zh:06037a14e47e8f82d0b3b326cd188566272b808b7970a9249a11db26d475b83d", + "zh:116b7dd58ca964a1056249d2b6550f399b0a6bc9a7920b7ee134242114432c9f", + "zh:1aa089c81459071c1d65ba7454f1122159e1fa1b5384e6e9ef85c8264f8a9ecb", + "zh:2c1471acba40c4944aa88dda761093c0c969db6408bdc1a4fb62417788cd6bb6", + "zh:3b950bea06ea4bf1ec359a97a4f1745b7efca7fc2da368843666020dd0ebc5d4", + "zh:7191c5c2fce834d584153dcd5269ed3042437f224d341ad85df06b2247bd09b2", + "zh:76d841b3f247f9bb3899dec3b4d871613a4ae8a83a581a827655d34b1bbee0ee", + "zh:7c656ce252fafc2c915dad43a0a7da17dba975207d75841a02f3f2b92d51ec25", + "zh:8ec97118cbdef64139c52b719e4e22443e67a1f37ea1597cd45b2e9b97332a35", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a369deca7938236a7da59f7ad1fe18137f736764c9015ed10e88edb6e8505980", + "zh:a743882fb099401eae0c86d9388a6faadbbc27b2ac9477aeef643e5de4eec3f9", + "zh:d5f960f58aff06fc58e244fea6e665800384cacb8cd64a556f8e145b98650372", + "zh:e31ffcfd560132ffbff2f574928ba392e663202a750750ed39a8950031b75623", + "zh:ebd9061b92a772144564f35a63d5a08cb45e14a9d39294fda185f2e0de9c8e28", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.2" + hashes = [ + "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", + "h1:vWAsYRd7MjYr3adj8BVKRohVfHpWQdvkIwUQ2Jf5FVM=", + "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", + "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", + "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", + "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", + "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", + "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", + "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", + "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", + "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", + "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.6.1" + hashes = [ + "h1:8iqExjtAvirFTJkpm5YyYD+fC+DGV8NTJzKsE2c70VA=", + "h1:a+Goawwh6Qtg4/bRWzfDtIdrEFfPlnVy0y4LdUQY3nI=", + "zh:2a0ec154e39911f19c8214acd6241e469157489fc56b6c739f45fbed5896a176", + "zh:57f4e553224a5e849c99131f5e5294be3a7adcabe2d867d8a4fef8d0976e0e52", + "zh:58f09948c608e601bd9d0a9e47dcb78e2b2c13b4bda4d8f097d09152ea9e91c5", + "zh:5c2a297146ed6fb3fe934c800e78380f700f49ff24dbb5fb5463134948e3a65f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7ce41e26f0603e31cdac849085fc99e5cd5b3b73414c6c6d955c0ceb249b593f", + "zh:8c9e8d30c4ef08ee8bcc4294dbf3c2115cd7d9049c6ba21422bd3471d92faf8a", + "zh:93e91be717a7ffbd6410120eb925ebb8658cc8f563de35a8b53804d33c51c8b0", + "zh:982542e921970d727ce10ed64795bf36c4dec77a5db0741d4665230d12250a0d", + "zh:b9d1873f14d6033e216510ef541c891f44d249464f13cc07d3f782d09c7d18de", + "zh:cfe27faa0bc9556391c8803ade135a5856c34a3fe85b9ae3bdd515013c0c87c1", + "zh:e4aabf3184bbb556b89e4b195eab1514c86a2914dd01c23ad9813ec17e863a8a", + ] +} diff --git a/pipeline/terraform/2024-05-08/main.tf b/pipeline/terraform/2024-05-08/main.tf new file mode 100644 index 0000000000..35cdbbc7e2 --- /dev/null +++ b/pipeline/terraform/2024-05-08/main.tf @@ -0,0 +1,30 @@ +module "pipeline" { + source = "../modules/stack" + + reindexing_state = { + listen_to_reindexer = false + scale_up_tasks = false + scale_up_elastic_cluster = false + scale_up_id_minter_db = false + scale_up_matcher_db = false + } + + index_config = { + works = { + identified = "works_identified.2023-05-26" + merged = "works_merged.2023-05-26" + indexed = "works_indexed.2024-01-09" + } + images = { + indexed = "images_indexed.2024-01-09" + works_analysis = "works_indexed.2024-01-09" + } + } + + pipeline_date = local.pipeline_date + release_label = local.pipeline_date + + providers = { + aws.catalogue = aws.catalogue + } +} diff --git a/pipeline/terraform/2024-05-08/pipeline_config.tf b/pipeline/terraform/2024-05-08/pipeline_config.tf new file mode 100644 index 0000000000..b625416ebd --- /dev/null +++ b/pipeline/terraform/2024-05-08/pipeline_config.tf @@ -0,0 +1,18 @@ +# Note: this file is autogenerated by the run_terraform.sh script. +# +# Edits to this file may be reverted! + +locals { + pipeline_date = "2024-05-08" +} + +terraform { + backend "s3" { + role_arn = "arn:aws:iam::760097843905:role/platform-developer" + + bucket = "wellcomecollection-platform-infra" + key = "terraform/catalogue-pipeline/pipeline/2024-05-08.tfstate" + dynamodb_table = "terraform-locktable" + region = "eu-west-1" + } +} diff --git a/pipeline/terraform/2024-05-08/run_terraform.sh b/pipeline/terraform/2024-05-08/run_terraform.sh new file mode 100755 index 0000000000..2154edc654 --- /dev/null +++ b/pipeline/terraform/2024-05-08/run_terraform.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset + +# Get the path to the current directory, which we can use to find the +# 'scripts' folder and the date of the current pipeline. +# +# https://stackoverflow.com/q/59895/1558022 +THIS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SCRIPT_DIR="$(dirname "$THIS_DIR")/scripts" + +# Create the config file that tells Terraform which pipeline we're running +# in and where to store the remote state. +export PIPELINE_DATE="$(basename "$THIS_DIR")" +. $SCRIPT_DIR/create_terraform_config_file.sh + +# Get the Elastic Cloud API key, which we need for the EC provider +EC_API_KEY=$(. "$SCRIPT_DIR/get_elastic_cloud_api_key.sh") + +# Now run Terraform itself, passing any arguments directly to the underlying +# Terraform binary. +EC_API_KEY=$EC_API_KEY terraform "$@" diff --git a/pipeline/terraform/2024-05-08/terraform.tf b/pipeline/terraform/2024-05-08/terraform.tf new file mode 100644 index 0000000000..8276e205db --- /dev/null +++ b/pipeline/terraform/2024-05-08/terraform.tf @@ -0,0 +1,32 @@ +provider "aws" { + region = "eu-west-1" + + assume_role { + role_arn = "arn:aws:iam::760097843905:role/platform-admin" + } +} + +provider "aws" { + region = "eu-west-1" + + alias = "catalogue" + + assume_role { + role_arn = "arn:aws:iam::756629837203:role/catalogue-developer" + } +} + +provider "ec" {} + +terraform { + required_version = ">= 0.13" + required_providers { + aws = { + source = "hashicorp/aws" + } + ec = { + source = "elastic/ec" + version = "0.2.1" + } + } +} diff --git a/pipeline/terraform/modules/stack/locals.tf b/pipeline/terraform/modules/stack/locals.tf index 8f1f06d952..db1d3c789d 100644 --- a/pipeline/terraform/modules/stack/locals.tf +++ b/pipeline/terraform/modules/stack/locals.tf @@ -109,6 +109,7 @@ locals { calm_deletions_topic_arn = data.terraform_remote_state.calm_adapter.outputs.calm_deletions_topic_arn # Reindexer topics + ebsco_reindexer_topic_arn = data.terraform_remote_state.reindexer.outputs.ebsco_reindexer_topic_arn miro_reindexer_topic_arn = data.terraform_remote_state.shared_infra.outputs.catalogue_miro_reindex_topic_arn sierra_reindexer_topic_arn = data.terraform_remote_state.shared_infra.outputs.catalogue_sierra_reindex_topic_arn mets_reindexer_topic_arn = data.terraform_remote_state.reindexer.outputs.mets_reindexer_topic_arn @@ -168,7 +169,7 @@ locals { topics = [ local.ebsco_adapter_topic_arn, ], - reindex_topic = null # TODO: Add reindexer topic (the adapter doesn't have one yet) + reindex_topic = local.ebsco_reindexer_topic_arn read_policy = data.aws_iam_policy_document.read_ebsco_adapter_bucket.json } } diff --git a/reindexer/terraform/locals.tf b/reindexer/terraform/locals.tf index f05ca60905..230b08c835 100644 --- a/reindexer/terraform/locals.tf +++ b/reindexer/terraform/locals.tf @@ -11,14 +11,17 @@ locals { reporting_sierra_reindex_topic_arn = data.terraform_remote_state.shared_infra.outputs.reporting_sierra_reindex_topic_arn catalogue_miro_reindex_topic_arn = data.terraform_remote_state.shared_infra.outputs.catalogue_miro_reindex_topic_arn catalogue_sierra_reindex_topic_arn = data.terraform_remote_state.shared_infra.outputs.catalogue_sierra_reindex_topic_arn - mets_reindexer_topic_name = module.mets_reindexer_topic.name - mets_reindexer_topic_arn = module.mets_reindexer_topic.arn - tei_reindexer_topic_arn = module.tei_reindexer_topic.arn - calm_reindexer_topic_name = module.calm_reindexer_topic.name - calm_reindexer_topic_arn = module.calm_reindexer_topic.arn - calm_deletion_checker_topic_name = module.calm_deletion_checker_topic.name - calm_deletion_checker_topic_arn = module.calm_deletion_checker_topic.arn - miro_updates_topic_arn = data.terraform_remote_state.shared_infra.outputs.miro_updates_topic_arn + + ebsco_reindexer_topic_name = module.ebsco_reindexer_topic.name + ebsco_reindexer_topic_arn = module.ebsco_reindexer_topic.arn + mets_reindexer_topic_name = module.mets_reindexer_topic.name + mets_reindexer_topic_arn = module.mets_reindexer_topic.arn + tei_reindexer_topic_arn = module.tei_reindexer_topic.arn + calm_reindexer_topic_name = module.calm_reindexer_topic.name + calm_reindexer_topic_arn = module.calm_reindexer_topic.arn + calm_deletion_checker_topic_name = module.calm_deletion_checker_topic.name + calm_deletion_checker_topic_arn = module.calm_deletion_checker_topic.arn + miro_updates_topic_arn = data.terraform_remote_state.shared_infra.outputs.miro_updates_topic_arn vpc_id = local.catalogue_vpcs["catalogue_vpc_delta_id"] private_subnets = local.catalogue_vpcs["catalogue_vpc_delta_private_subnets"] diff --git a/reindexer/terraform/outputs.tf b/reindexer/terraform/outputs.tf index 4445e86e4f..81a4620aa1 100644 --- a/reindexer/terraform/outputs.tf +++ b/reindexer/terraform/outputs.tf @@ -7,6 +7,9 @@ output "topic_arn" { value = module.reindex_worker.topic_arn } +output "ebsco_reindexer_topic_arn" { + value = local.ebsco_reindexer_topic_arn +} output "tei_reindexer_topic_arn" { value = local.tei_reindexer_topic_arn } diff --git a/reindexer/terraform/topics.tf b/reindexer/terraform/topics.tf index f61fdb7c03..68f43d2a98 100644 --- a/reindexer/terraform/topics.tf +++ b/reindexer/terraform/topics.tf @@ -1,3 +1,7 @@ +module "ebsco_reindexer_topic" { + source = "github.com/wellcomecollection/terraform-aws-sns-topic.git?ref=v1.0.0" + name = "ebsco_reindexer_topic" +} module "mets_reindexer_topic" { source = "github.com/wellcomecollection/terraform-aws-sns-topic.git?ref=v1.0.0" name = "mets_reindexer_topic"