From 451c886ab3d1dabb3c57852f43043c67b9f2de37 Mon Sep 17 00:00:00 2001 From: Kuntal Basu Date: Thu, 1 May 2025 18:12:19 -0400 Subject: [PATCH 1/2] added variable message retention sec --- autotagging.tf | 21 ++++++++++----------- glue_create.tf | 14 +++++++------- glue_sync.tf | 14 +++++++------- main.tf | 32 +++++++++++++++++++------------- variables.tf | 5 +++++ 5 files changed, 48 insertions(+), 38 deletions(-) diff --git a/autotagging.tf b/autotagging.tf index 2072c3e..12c33ad 100644 --- a/autotagging.tf +++ b/autotagging.tf @@ -1,17 +1,16 @@ # This is the optional Autotagging feature. resource "aws_lambda_function" "auto_tagging" { - count = var.enable_auto_tagging == true ? 1 : 0 - architectures = var.architectures - description = var.lambda_description - s3_key = var.auto_tagging_s3_key - s3_bucket = var.auto_tagging_s3_bucket - function_name = "${var.lambda_function_name}-auto_tagging" - role = aws_iam_role.auto_tagging_lambda[0].arn - handler = "provided" - runtime = "provided.al2023" - memory_size = var.lambda_memory_size - # lets set 2 minutes + count = var.enable_auto_tagging == true ? 1 : 0 + architectures = var.architectures + description = var.lambda_description + s3_key = var.auto_tagging_s3_key + s3_bucket = var.auto_tagging_s3_bucket + function_name = "${var.lambda_function_name}-auto_tagging" + role = aws_iam_role.auto_tagging_lambda[0].arn + handler = "provided" + runtime = "provided.al2023" + memory_size = var.lambda_memory_size timeout = var.lambda_timeout reserved_concurrent_executions = var.lambda_reserved_concurrent_executions diff --git a/glue_create.tf b/glue_create.tf index 7ccced0..dbdb812 100644 --- a/glue_create.tf +++ b/glue_create.tf @@ -73,8 +73,8 @@ data "aws_iam_policy_document" "glue_create_sqs_dl" { } resource "aws_sqs_queue" "glue_create" { - count = var.enable_glue_create ? 1 : 0 - + count = var.enable_glue_create ? 1 : 0 + message_retention_seconds = var.message_retention_seconds name = var.glue_create_config.sqs_queue_name policy = data.aws_iam_policy_document.glue_create_sqs[0].json visibility_timeout_seconds = var.sqs_visibility_timeout_seconds @@ -87,11 +87,11 @@ resource "aws_sqs_queue" "glue_create" { } resource "aws_sqs_queue" "glue_create_dl" { - count = var.enable_glue_create ? 1 : 0 - - name = var.glue_create_config.sqs_queue_name_dl - policy = data.aws_iam_policy_document.glue_create_sqs_dl[0].json - tags = var.tags + count = var.enable_glue_create ? 1 : 0 + message_retention_seconds = var.message_retention_seconds + name = var.glue_create_config.sqs_queue_name_dl + policy = data.aws_iam_policy_document.glue_create_sqs_dl[0].json + tags = var.tags } resource "aws_sqs_queue_redrive_allow_policy" "terraform_queue_redrive_allow_policy" { diff --git a/glue_sync.tf b/glue_sync.tf index 4f35499..67dfb46 100644 --- a/glue_sync.tf +++ b/glue_sync.tf @@ -37,8 +37,8 @@ data "aws_iam_policy_document" "glue_sync_sqs_dl" { } resource "aws_sqs_queue" "glue_sync" { - count = var.enable_glue_sync ? 1 : 0 - + count = var.enable_glue_sync ? 1 : 0 + message_retention_seconds = var.message_retention_seconds name = var.glue_sync_config.sqs_queue_name policy = data.aws_iam_policy_document.glue_sync_sqs[0].json visibility_timeout_seconds = var.sqs_visibility_timeout_seconds @@ -51,11 +51,11 @@ resource "aws_sqs_queue" "glue_sync" { } resource "aws_sqs_queue" "glue_sync_dl" { - count = var.enable_glue_sync ? 1 : 0 - - name = var.glue_sync_config.sqs_queue_name_dl - policy = data.aws_iam_policy_document.glue_sync_sqs_dl[0].json - tags = var.tags + count = var.enable_glue_sync ? 1 : 0 + message_retention_seconds = var.message_retention_seconds + name = var.glue_sync_config.sqs_queue_name_dl + policy = data.aws_iam_policy_document.glue_sync_sqs_dl[0].json + tags = var.tags } resource "aws_sqs_queue_redrive_allow_policy" "glue_syncredrive_allow_policy" { diff --git a/main.tf b/main.tf index cb4d456..5101ff3 100644 --- a/main.tf +++ b/main.tf @@ -141,6 +141,7 @@ resource "aws_sqs_queue" "oxbow_lambda_fifo_sqs" { policy = data.aws_iam_policy_document.oxbow_lambda_fifo_sqs[0].json visibility_timeout_seconds = var.sqs_visibility_timeout_seconds delay_seconds = var.sqs_delay_seconds + message_retention_seconds = var.message_retention_seconds content_based_deduplication = true fifo_queue = true tags = var.tags @@ -151,11 +152,12 @@ resource "aws_sqs_queue" "oxbow_lambda_fifo_sqs" { } resource "aws_sqs_queue" "oxbow_lambda_fifo_sqs_dlq" { - count = local.enable_group_events ? 1 : 0 - name = "${var.sqs_fifo_DL_queue_name}.fifo" - policy = data.aws_iam_policy_document.oxbow_lambda_fifo_sqs_dlq[0].json - fifo_queue = true - tags = var.tags + count = local.enable_group_events ? 1 : 0 + name = "${var.sqs_fifo_DL_queue_name}.fifo" + message_retention_seconds = var.message_retention_seconds + policy = data.aws_iam_policy_document.oxbow_lambda_fifo_sqs_dlq[0].json + fifo_queue = true + tags = var.tags } resource "aws_lambda_event_source_mapping" "group_events_lambda_sqs_trigger" { @@ -210,6 +212,7 @@ data "aws_iam_policy_document" "group_event_lambda_sqs_dlq" { resource "aws_sqs_queue" "group_events_lambda_sqs" { count = local.enable_group_events ? 1 : 0 name = var.sqs_group_queue_name + message_retention_seconds = var.message_retention_seconds policy = var.sns_topic_arn == "" ? data.aws_iam_policy_document.group_event_lambda_sqs[0].json : data.aws_iam_policy_document.this_sns_to_sqs[0].json visibility_timeout_seconds = var.sqs_visibility_timeout_seconds delay_seconds = var.sqs_delay_seconds @@ -221,10 +224,11 @@ resource "aws_sqs_queue" "group_events_lambda_sqs" { } resource "aws_sqs_queue" "group_events_lambda_sqs_dlq" { - count = local.enable_group_events ? 1 : 0 - policy = data.aws_iam_policy_document.group_event_lambda_sqs_dlq[0].json - name = var.sqs_group_DL_queue_name - tags = var.tags + count = local.enable_group_events ? 1 : 0 + message_retention_seconds = var.message_retention_seconds + policy = data.aws_iam_policy_document.group_event_lambda_sqs_dlq[0].json + name = var.sqs_group_DL_queue_name + tags = var.tags } @@ -239,6 +243,7 @@ resource "aws_lambda_event_source_mapping" "this_lambda_events" { resource "aws_sqs_queue" "this_sqs" { count = local.enable_group_events ? 0 : 1 name = var.sqs_queue_name + message_retention_seconds = var.message_retention_seconds policy = var.sns_topic_arn == "" ? data.aws_iam_policy_document.this_sqs_queue_policy_data.json : data.aws_iam_policy_document.this_sns_to_sqs[0].json visibility_timeout_seconds = var.sqs_visibility_timeout_seconds delay_seconds = var.sqs_delay_seconds @@ -250,10 +255,11 @@ resource "aws_sqs_queue" "this_sqs" { } resource "aws_sqs_queue" "this_DL" { - count = local.enable_group_events ? 0 : 1 - name = var.sqs_queue_name_dl - policy = data.aws_iam_policy_document.this_dead_letter_queue_policy.json - tags = var.tags + count = local.enable_group_events ? 0 : 1 + message_retention_seconds = var.message_retention_seconds + name = var.sqs_queue_name_dl + policy = data.aws_iam_policy_document.this_dead_letter_queue_policy.json + tags = var.tags } resource "aws_sns_topic_subscription" "this_sns_sub" { diff --git a/variables.tf b/variables.tf index fbb8b00..b3c0502 100644 --- a/variables.tf +++ b/variables.tf @@ -343,3 +343,8 @@ variable "architectures" { type = list(string) default = ["x86_64"] } + +variable "message_retention_seconds" { + type = number + default = 1209600 +} From e568946d20366d79d3a2b96f7e5a4985ab9941fe Mon Sep 17 00:00:00 2001 From: Kuntal Basu Date: Thu, 1 May 2025 18:12:45 -0400 Subject: [PATCH 2/2] fix: blank monitor creation in case gluesync/create is not enabled --- monitoring.tf | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/monitoring.tf b/monitoring.tf index 69f231c..89a1e62 100644 --- a/monitoring.tf +++ b/monitoring.tf @@ -1,8 +1,9 @@ locals { + base_dlq_name = local.enable_group_events ? lower("${var.sqs_fifo_DL_queue_name}.fifo") : lower(var.sqs_queue_name_dl) dlq_to_monitor = [ - local.enable_group_events ? lower("${var.sqs_fifo_DL_queue_name}.fifo") : lower(var.sqs_queue_name_dl), - var.enable_glue_create ? lower(var.glue_create_config.sqs_queue_name_dl) : "", - var.enable_glue_sync ? lower(var.glue_sync_config.sqs_queue_name_dl) : "", + local.base_dlq_name, + var.enable_glue_create ? lower(var.glue_create_config.sqs_queue_name_dl) : local.base_dlq_name, + var.enable_glue_sync ? lower(var.glue_sync_config.sqs_queue_name_dl) : local.base_dlq_name, ] }