From d79ce59c6a9fc33fc08c69b0b41b9ce3241ae3e3 Mon Sep 17 00:00:00 2001 From: Josh Garde Date: Thu, 19 Oct 2023 10:00:14 -0700 Subject: [PATCH] Create table and integration into poll_status (#9) --- podaac/swodlr_ingest_to_sds/poll_status.py | 33 ++++++++++++++++++++-- podaac/swodlr_ingest_to_sds/utils.py | 15 ++++++++++ terraform/database_dynamodb.tf | 4 +++ terraform/lambdas.tf | 26 +++++++++++------ 4 files changed, 67 insertions(+), 11 deletions(-) diff --git a/podaac/swodlr_ingest_to_sds/poll_status.py b/podaac/swodlr_ingest_to_sds/poll_status.py index cca2521..0c4970a 100644 --- a/podaac/swodlr_ingest_to_sds/poll_status.py +++ b/podaac/swodlr_ingest_to_sds/poll_status.py @@ -2,10 +2,17 @@ from datetime import datetime from copy import deepcopy import logging -from podaac.swodlr_ingest_to_sds.utils import mozart_client, ingest_table +import re +from podaac.swodlr_ingest_to_sds.utils import ( + mozart_client, ingest_table, available_tiles_table +) + SUCCESS_STATUSES = {'job-completed'} FAIL_STATUSES = {'job-failed', 'job-offline', 'job-deduped'} +PRODUCT_REGEX = re.compile( + r'_(?PPIXC(Vec)?)_(?P\d{3})_(?P\d{3})_(?P\d{3}(R|L))_' # noqa: E501 +) def lambda_handler(event, _context): @@ -62,9 +69,31 @@ def lambda_handler(event, _context): new_event['jobs'].remove(item) elif status in SUCCESS_STATUSES: logging.info('Job id: %s; status: %s', job_id, status) - new_event['jobs'].remove(item) + + # Insert into available tiles table + cpt = _extract_cpt(granule_id) + if cpt is not None: + tile_id = f'{cpt["product"]},{cpt["cycle"]},{cpt["pass"]},{cpt["tile"]}' # pylint: disable=line-too-long # noqa: E501 + available_tiles_table.put_item( + Item={'tile_id': tile_id} + ) + + new_event['jobs'].remove(item) # Remove from queue # Otello raises very generic exceptions except Exception: # pylint: disable=broad-except logging.exception('Failed to get status: %s', job_id) return new_event + + +def _extract_cpt(granule_id): + parsed_id = PRODUCT_REGEX.match(granule_id) + if parsed_id is None: + return None + + return { + 'product': parsed_id.group('product'), + 'cycle': int(parsed_id.group('cycle')), + 'pass': int(parsed_id.group('pass')), + 'tile': int(parsed_id.group('tile')) + } diff --git a/podaac/swodlr_ingest_to_sds/utils.py b/podaac/swodlr_ingest_to_sds/utils.py index f94481a..9a894a8 100644 --- a/podaac/swodlr_ingest_to_sds/utils.py +++ b/podaac/swodlr_ingest_to_sds/utils.py @@ -93,9 +93,24 @@ def ingest_table(self): return self._ingest_table + @property + def available_tiles_table(self): + ''' + Lazily creates a DynamoDB table resource + ''' + if not hasattr(self, '_available_tiles_table'): + dynamodb = boto3.resource('dynamodb') + # pylint: disable=attribute-defined-outside-init + self._available_tiles_table = dynamodb.Table( + self.get_param('available_tiles_table_name') + ) + + return self._available_tiles_table + # Silence the linters ingest_table: Table +available_tiles_table: Table mozart_client: Mozart get_param: Callable[[str], str] diff --git a/terraform/database_dynamodb.tf b/terraform/database_dynamodb.tf index e8cdb5a..5189c72 100644 --- a/terraform/database_dynamodb.tf +++ b/terraform/database_dynamodb.tf @@ -5,3 +5,7 @@ data "aws_dynamodb_table" "ingest" { name = "${local.app_prefix}-ingest" } + +data "aws_dynamodb_table" "avalible_tiles" { + name = "${local.app_prefix}-avalible-tiles" +} diff --git a/terraform/lambdas.tf b/terraform/lambdas.tf index 5a52ab2..d418769 100644 --- a/terraform/lambdas.tf +++ b/terraform/lambdas.tf @@ -168,6 +168,16 @@ resource "aws_iam_role" "lambda" { ] Effect = "Allow" Resource = data.aws_dynamodb_table.ingest.arn + }, + { + Sid = "" + Action = [ + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem" + ] + Effect = "Allow" + Resource = data.aws_dynamodb_table.avalible_tiles.arn } ] }) @@ -178,55 +188,53 @@ resource "aws_iam_role" "lambda" { resource "aws_ssm_parameter" "sds_pcm_release_tag" { name = "${local.service_path}/sds_pcm_release_tag" type = "String" - overwrite = true value = var.sds_pcm_release_tag } resource "aws_ssm_parameter" "sds_host" { name = "${local.service_path}/sds_host" type = "String" - overwrite = true value = var.sds_host } resource "aws_ssm_parameter" "sds_username" { name = "${local.service_path}/sds_username" type = "String" - overwrite = true value = var.sds_username } resource "aws_ssm_parameter" "sds_password" { name = "${local.service_path}/sds_password" type = "SecureString" - overwrite = true value = var.sds_password } resource "aws_ssm_parameter" "sds_ca_cert" { name = "${local.service_path}/sds_ca_cert" type = "SecureString" - overwrite = true value = local.sds_ca_cert } resource "aws_ssm_parameter" "stepfunction_arn" { name = "${local.service_path}/stepfunction_arn" type = "String" - overwrite = true value = aws_sfn_state_machine.ingest_to_sds.arn } resource "aws_ssm_parameter" "ingest_queue_url" { name = "${local.service_path}/ingest_queue_url" type = "String" - overwrite = true value = data.aws_sqs_queue.ingest.id } resource "aws_ssm_parameter" "ingest_table_name" { name = "${local.service_path}/ingest_table_name" type = "String" - overwrite = true value = data.aws_dynamodb_table.ingest.name -} \ No newline at end of file +} + +resource "aws_ssm_parameter" "available_tiles_table_name" { + name = "${local.service_path}/available_tiles_table_name" + type = "String" + value = data.aws_dynamodb_table.avalible_tiles.name +}