From fb78a7740a0c6edadd12cb3375ba7da635b6847c Mon Sep 17 00:00:00 2001 From: Josh Garde Date: Thu, 13 Jun 2024 16:16:44 -0700 Subject: [PATCH] Update to utilize new commons (#21) * Update terraform to make PCM release tags optional * Update submit_to_sds to utilize new auto version selection * Fix tests --- podaac/swodlr_ingest_to_sds/submit_to_sds.py | 3 +-- terraform/bin/config.sh | 1 - terraform/environments/ops.env | 1 - terraform/environments/sit.env | 1 - terraform/environments/uat.env | 1 - terraform/lambdas.tf | 3 +++ terraform/variables.tf | 1 + tests/test_submit_to_sds.py | 5 ++++- 8 files changed, 9 insertions(+), 7 deletions(-) diff --git a/podaac/swodlr_ingest_to_sds/submit_to_sds.py b/podaac/swodlr_ingest_to_sds/submit_to_sds.py index f135900..79c0284 100644 --- a/podaac/swodlr_ingest_to_sds/submit_to_sds.py +++ b/podaac/swodlr_ingest_to_sds/submit_to_sds.py @@ -11,14 +11,13 @@ ACCEPTED_EXTS = ['nc'] INGEST_QUEUE_URL = utils.get_param('ingest_queue_url') INGEST_TABLE_NAME = utils.get_param('ingest_table_name') -PCM_RELEASE_TAG = utils.get_param('sds_pcm_release_tag') dynamodb = boto3.client('dynamodb') sqs = boto3.client('sqs') logger = utils.get_logger(__name__) ingest_job_type = utils.mozart_client.get_job_type( - f'job-INGEST_STAGED:{PCM_RELEASE_TAG}' + utils.get_latest_job_version('job-INGEST_STAGED') ) ingest_job_type.initialize() diff --git a/terraform/bin/config.sh b/terraform/bin/config.sh index f8165e1..95d3b29 100755 --- a/terraform/bin/config.sh +++ b/terraform/bin/config.sh @@ -15,6 +15,5 @@ export TF_INPUT=false # https://www.terraform.io/cli/config/environment-variabl export TF_VAR_region="$REGION" export TF_VAR_stage="$VENUE" -export TF_VAR_sds_pcm_release_tag="$SWODLR_sds_pcm_release_tag" terraform init -reconfigure -backend-config="bucket=$BUCKET" -backend-config="region=$REGION" \ No newline at end of file diff --git a/terraform/environments/ops.env b/terraform/environments/ops.env index 4515b04..72edfb5 100644 --- a/terraform/environments/ops.env +++ b/terraform/environments/ops.env @@ -1,3 +1,2 @@ export REGION=us-west-2 export BUCKET=podaac-services-ops-terraform -export SWODLR_sds_pcm_release_tag=pcm-v5.0.0-pge-v5.0.4 \ No newline at end of file diff --git a/terraform/environments/sit.env b/terraform/environments/sit.env index 6aea454..b74973e 100644 --- a/terraform/environments/sit.env +++ b/terraform/environments/sit.env @@ -1,3 +1,2 @@ export REGION=us-west-2 export BUCKET=podaac-services-sit-terraform -export SWODLR_sds_pcm_release_tag=pcm-v5.0.0-pge-v5.0.4 diff --git a/terraform/environments/uat.env b/terraform/environments/uat.env index daa52bf..22f59e9 100644 --- a/terraform/environments/uat.env +++ b/terraform/environments/uat.env @@ -1,3 +1,2 @@ export REGION=us-west-2 export BUCKET=podaac-services-uat-terraform -export SWODLR_sds_pcm_release_tag=pcm-v5.0.0-pge-v5.0.4 diff --git a/terraform/lambdas.tf b/terraform/lambdas.tf index 54e34f6..10a4523 100644 --- a/terraform/lambdas.tf +++ b/terraform/lambdas.tf @@ -13,6 +13,7 @@ resource "aws_lambda_function" "bootstrap" { resource "aws_lambda_function" "submit_to_sds" { function_name = "${local.service_prefix}-submit_to_sds" handler = "podaac.swodlr_ingest_to_sds.submit_to_sds.lambda_handler" + timeout = 300 role = aws_iam_role.lambda.arn runtime = "python3.9" @@ -29,6 +30,7 @@ resource "aws_lambda_function" "submit_to_sds" { resource "aws_lambda_function" "poll_status" { function_name = "${local.service_prefix}-poll_status" handler = "podaac.swodlr_ingest_to_sds.poll_status.lambda_handler" + timeout = 300 role = aws_iam_role.lambda.arn runtime = "python3.9" @@ -186,6 +188,7 @@ resource "aws_iam_role" "lambda" { # -- SSM Parameters -- resource "aws_ssm_parameter" "sds_pcm_release_tag" { + count = var.sds_pcm_release_tag == null ? 0 : 1 name = "${local.service_path}/sds_pcm_release_tag" type = "String" value = var.sds_pcm_release_tag diff --git a/terraform/variables.tf b/terraform/variables.tf index b553977..b98103a 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -23,6 +23,7 @@ variable "region" { variable "sds_pcm_release_tag" { type = string + default = null } variable "sds_host" { diff --git a/tests/test_submit_to_sds.py b/tests/test_submit_to_sds.py index a760772..d7177df 100644 --- a/tests/test_submit_to_sds.py +++ b/tests/test_submit_to_sds.py @@ -9,13 +9,16 @@ with ( patch('boto3.client'), patch('otello.mozart.Mozart.get_job_type'), + patch('podaac.swodlr_common.utilities.BaseUtilities.get_latest_job_version'), # pylint: disable-next=line-too-long # noqa: E501 + patch('podaac.swodlr_ingest_to_sds.utilities.utils.get_grq_es_client') as mock_es_client, # pylint: disable-next=line-too-long # noqa: E501 patch.dict(environ, { 'SWODLR_ENV': 'dev', 'SWODLR_sds_username': 'test_username', 'SWODLR_sds_password': 'test_password', 'SWODLR_ingest_table_name': 'test_ingest_table_name', 'SWODLR_ingest_queue_url': 'test_ingest_queue_url', - 'SWODLR_available_tiles_table': 'test_available_tiles_table' + 'SWODLR_available_tiles_table': 'test_available_tiles_table', + 'SWODLR_sds_host': 'http://sds-host.test/' }) ): from podaac.swodlr_ingest_to_sds import submit_to_sds