From 8327b6100827f325b2f486130f75020380c4d954 Mon Sep 17 00:00:00 2001 From: Ajay Mudgal Date: Mon, 12 Jan 2026 12:49:13 +0000 Subject: [PATCH 1/2] Fixing race condition issue in cw queries --- .../blue-green-link/cloudwatch-queries.tf | 22 ------------------- .../stacks/blue-green-link/terraform.tf | 4 ---- .../stacks/common/common-provider.tf | 2 ++ 3 files changed, 2 insertions(+), 26 deletions(-) diff --git a/infrastructure/stacks/blue-green-link/cloudwatch-queries.tf b/infrastructure/stacks/blue-green-link/cloudwatch-queries.tf index 9e88fe56e..7e4283607 100644 --- a/infrastructure/stacks/blue-green-link/cloudwatch-queries.tf +++ b/infrastructure/stacks/blue-green-link/cloudwatch-queries.tf @@ -1,14 +1,4 @@ -// Introduce a short sleep BEFORE creating query definitions to allow -// recently-created log groups (from other modules in the same apply) -// to propagate. Note: This does NOT guarantee post-create consistency -// of the query definitions themselves, but can reduce immediate read -// failures when the provider plans many resources concurrently. -resource "time_sleep" "wait_for_propagation" { - create_duration = "45s" -} - resource "aws_cloudwatch_query_definition" "search_for_errors" { - depends_on = [time_sleep.wait_for_propagation] name = "${var.project_id}/${var.blue_green_environment}/search-for-errors" log_group_names = [ @@ -33,7 +23,6 @@ EOF } resource "aws_cloudwatch_query_definition" "search_by_correlation_id" { - depends_on = [aws_cloudwatch_query_definition.search_for_errors] name = "${var.project_id}/${var.blue_green_environment}/search-by-correlation-id" log_group_names = [ @@ -58,7 +47,6 @@ EOF } resource "aws_cloudwatch_query_definition" "search_by_correlation_id_expanded" { - depends_on = [aws_cloudwatch_query_definition.search_by_correlation_id] name = "${var.project_id}/${var.blue_green_environment}/search-by-correlation-id-expanded" log_group_names = [ @@ -83,7 +71,6 @@ EOF } resource "aws_cloudwatch_query_definition" "search_by_odscode" { - depends_on = [aws_cloudwatch_query_definition.search_by_correlation_id_expanded] name = "${var.project_id}/${var.blue_green_environment}/search-by-odscode" log_group_names = [ @@ -108,7 +95,6 @@ EOF } resource "aws_cloudwatch_query_definition" "search_by_odscode_expanded" { - depends_on = [aws_cloudwatch_query_definition.search_by_odscode] name = "${var.project_id}/${var.blue_green_environment}/search-by-odscode-expanded" log_group_names = [ @@ -133,7 +119,6 @@ EOF } resource "aws_cloudwatch_query_definition" "search_for_invalid_postcode" { - depends_on = [aws_cloudwatch_query_definition.search_by_odscode_expanded] name = "${var.project_id}/${var.blue_green_environment}/search-for-invalid-postcode" log_group_names = [ @@ -152,7 +137,6 @@ EOF } resource "aws_cloudwatch_query_definition" "search_for_invalid_opening_times" { - depends_on = [aws_cloudwatch_query_definition.search_for_invalid_postcode] name = "${var.project_id}/${var.blue_green_environment}/search-for-invalid-opening-times" log_group_names = [ @@ -171,7 +155,6 @@ EOF } resource "aws_cloudwatch_query_definition" "search_by_email_correlation_id" { - depends_on = [aws_cloudwatch_query_definition.search_for_invalid_opening_times] name = "${var.project_id}/${var.blue_green_environment}/search-by-email-correlation-id" log_group_names = [ @@ -191,7 +174,6 @@ EOF } resource "aws_cloudwatch_query_definition" "search_by_update_request_success" { - depends_on = [aws_cloudwatch_query_definition.search_by_email_correlation_id] name = "${var.project_id}/${var.blue_green_environment}/update-request-success" log_group_names = [ @@ -210,7 +192,6 @@ EOF } resource "aws_cloudwatch_query_definition" "search_by_update_request_failed" { - depends_on = [aws_cloudwatch_query_definition.search_by_update_request_success] name = "${var.project_id}/${var.blue_green_environment}/update-request-failed" log_group_names = [ @@ -229,7 +210,6 @@ EOF } resource "aws_cloudwatch_query_definition" "search_by_dos_data_item_updates" { - depends_on = [aws_cloudwatch_query_definition.search_by_update_request_failed] name = "${var.project_id}/${var.blue_green_environment}/dos-data-item-updates" log_group_names = [ @@ -250,7 +230,6 @@ EOF } resource "aws_cloudwatch_query_definition" "search_for_report_warnings" { - depends_on = [aws_cloudwatch_query_definition.search_by_dos_data_item_updates] name = "${var.project_id}/${var.blue_green_environment}/search-for-report-warnings" log_group_names = [ @@ -277,7 +256,6 @@ EOF resource "aws_cloudwatch_query_definition" "search_for_quality_checker_logs_with_odscode" { - depends_on = [aws_cloudwatch_query_definition.search_for_report_warnings] name = "${var.project_id}/${var.blue_green_environment}/search-for-quality-checker-logs-with-odscode" log_group_names = [ diff --git a/infrastructure/stacks/blue-green-link/terraform.tf b/infrastructure/stacks/blue-green-link/terraform.tf index 894f94b0e..5a3d2f3fc 100755 --- a/infrastructure/stacks/blue-green-link/terraform.tf +++ b/infrastructure/stacks/blue-green-link/terraform.tf @@ -7,9 +7,5 @@ terraform { source = "hashicorp/aws" version = "~> 6.14.1" } - time = { - source = "hashicorp/time" - version = "~> 0.13.1" - } } } diff --git a/infrastructure/stacks/common/common-provider.tf b/infrastructure/stacks/common/common-provider.tf index 520795211..68613811a 100644 --- a/infrastructure/stacks/common/common-provider.tf +++ b/infrastructure/stacks/common/common-provider.tf @@ -1,5 +1,7 @@ provider "aws" { region = var.aws_region + retry_mode = "adaptive" + max_retries = 25 default_tags { tags = { "DataClassification" = var.data_classification From 5a682610f2227cab65f27b8b1c499b16a31d310d Mon Sep 17 00:00:00 2001 From: Ajay Mudgal Date: Fri, 16 Jan 2026 11:37:37 +0000 Subject: [PATCH 2/2] Adding local provisioner to resolve consistency issue in queries --- .../blue-green-link/cloudwatch-queries.tf | 52 +++++++++---------- .../stacks/common/common-provider.tf | 2 - 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/infrastructure/stacks/blue-green-link/cloudwatch-queries.tf b/infrastructure/stacks/blue-green-link/cloudwatch-queries.tf index 7e4283607..968c08743 100644 --- a/infrastructure/stacks/blue-green-link/cloudwatch-queries.tf +++ b/infrastructure/stacks/blue-green-link/cloudwatch-queries.tf @@ -17,8 +17,8 @@ fields @timestamp, correlation_id, ods_code, function_name, message | sort @timestamp EOF - lifecycle { - create_before_destroy = false + provisioner "local-exec" { + command = "sleep 30" } } @@ -41,8 +41,8 @@ fields @timestamp, message | sort @timestamp EOF - lifecycle { - create_before_destroy = false + provisioner "local-exec" { + command = "sleep 30" } } @@ -65,8 +65,8 @@ fields @timestamp,correlation_id,ods_code,level,message_received,function_name, | sort @timestamp EOF - lifecycle { - create_before_destroy = false + provisioner "local-exec" { + command = "sleep 30" } } @@ -89,8 +89,8 @@ fields @timestamp, message | sort @timestamp EOF - lifecycle { - create_before_destroy = false + provisioner "local-exec" { + command = "sleep 30" } } @@ -113,8 +113,8 @@ fields @timestamp,correlation_id,ods_code,level,message_received,function_name, | sort @timestamp EOF - lifecycle { - create_before_destroy = false + provisioner "local-exec" { + command = "sleep 30" } } @@ -131,8 +131,8 @@ fields @timestamp,correlation_id,ods_code,level,message_received,function_name, | sort @timestamp EOF - lifecycle { - create_before_destroy = false + provisioner "local-exec" { + command = "sleep 30" } } @@ -149,8 +149,8 @@ fields @timestamp,correlation_id,ods_code,level,message_received,function_name, | sort @timestamp EOF - lifecycle { - create_before_destroy = false + provisioner "local-exec" { + command = "sleep 30" } } @@ -168,8 +168,8 @@ fields correlation_id | filter email_correlation_id == "ADD_EMAIL_CORRELATION_ID" EOF - lifecycle { - create_before_destroy = false + provisioner "local-exec" { + command = "sleep 30" } } @@ -186,8 +186,8 @@ fields @timestamp, correlation_id | sort @timestamp desc EOF - lifecycle { - create_before_destroy = false + provisioner "local-exec" { + command = "sleep 30" } } @@ -204,8 +204,8 @@ fields @timestamp, correlation_id, report_key | sort @timestamp desc EOF - lifecycle { - create_before_destroy = false + provisioner "local-exec" { + command = "sleep 30" } } @@ -224,8 +224,8 @@ fields @timestamp, correlation_id | sort @timestamp desc EOF - lifecycle { - create_before_destroy = false + provisioner "local-exec" { + command = "sleep 30" } } @@ -249,8 +249,8 @@ fields @timestamp, correlation_id, message | sort @timestamp desc EOF - lifecycle { - create_before_destroy = false + provisioner "local-exec" { + command = "sleep 30" } } @@ -268,7 +268,7 @@ fields @timestamp, level, message | sort @timestamp asc EOF - lifecycle { - create_before_destroy = false + provisioner "local-exec" { + command = "sleep 30" } } diff --git a/infrastructure/stacks/common/common-provider.tf b/infrastructure/stacks/common/common-provider.tf index 68613811a..520795211 100644 --- a/infrastructure/stacks/common/common-provider.tf +++ b/infrastructure/stacks/common/common-provider.tf @@ -1,7 +1,5 @@ provider "aws" { region = var.aws_region - retry_mode = "adaptive" - max_retries = 25 default_tags { tags = { "DataClassification" = var.data_classification