From c761890402563058d7d0c02541c676491f9477ac Mon Sep 17 00:00:00 2001 From: David Wass Date: Mon, 19 Jan 2026 11:30:20 +0000 Subject: [PATCH 1/5] workflow --- .../workflows/manual-proxy-environment-deploy.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/manual-proxy-environment-deploy.yaml b/.github/workflows/manual-proxy-environment-deploy.yaml index d5e502309..63f65ccf4 100644 --- a/.github/workflows/manual-proxy-environment-deploy.yaml +++ b/.github/workflows/manual-proxy-environment-deploy.yaml @@ -18,6 +18,10 @@ on: required: false default: false type: boolean + nodejs_version: + description: "Node.js version, set by the CI/CD pipeline workflow" + required: true + type: string permissions: contents: read @@ -36,11 +40,10 @@ jobs: node-version: 22 - name: Npm install - working-directory: . - env: - NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: npm ci - shell: bash + uses: ./.github/actions/node-install + with: + node-version: ${{ inputs.nodejs_version }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: "Check if pull request exists for this branch and set ENVIRONMENT/APIM_ENV" id: pr_exists From 779610e21e6759ce98f80042df130b68892f80bf Mon Sep 17 00:00:00 2001 From: David Wass Date: Fri, 16 Jan 2026 10:52:54 +0000 Subject: [PATCH 2/5] Enable firehose --- infrastructure/terraform/modules/eventsub/README.md | 6 +++--- infrastructure/terraform/modules/eventsub/variables.tf | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/infrastructure/terraform/modules/eventsub/README.md b/infrastructure/terraform/modules/eventsub/README.md index 859c5fd5b..b75c55450 100644 --- a/infrastructure/terraform/modules/eventsub/README.md +++ b/infrastructure/terraform/modules/eventsub/README.md @@ -14,9 +14,9 @@ | [aws\_account\_id](#input\_aws\_account\_id) | The AWS Account ID (numeric) | `string` | n/a | yes | | [component](#input\_component) | The name of the terraformscaffold component calling this module | `string` | n/a | yes | | [default\_tags](#input\_default\_tags) | Default tag map for application to all taggable resources in the module | `map(string)` | `{}` | no | -| [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `false` | no | -| [enable\_firehose\_raw\_message\_delivery](#input\_enable\_firehose\_raw\_message\_delivery) | Enables raw message delivery on firehose subscription | `bool` | `false` | no | -| [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `false` | no | +| [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `true` | no | +| [enable\_firehose\_raw\_message\_delivery](#input\_enable\_firehose\_raw\_message\_delivery) | Enables raw message delivery on firehose subscription | `bool` | `true` | no | +| [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `true` | no | | [environment](#input\_environment) | The name of the terraformscaffold environment the module is called for | `string` | n/a | yes | | [event\_cache\_buffer\_interval](#input\_event\_cache\_buffer\_interval) | The buffer interval for data firehose | `number` | `500` | no | | [event\_cache\_expiry\_days](#input\_event\_cache\_expiry\_days) | s3 archiving expiry in days | `number` | `30` | no | diff --git a/infrastructure/terraform/modules/eventsub/variables.tf b/infrastructure/terraform/modules/eventsub/variables.tf index 4b73d4523..79a1114c6 100644 --- a/infrastructure/terraform/modules/eventsub/variables.tf +++ b/infrastructure/terraform/modules/eventsub/variables.tf @@ -70,7 +70,7 @@ variable "event_cache_buffer_interval" { variable "enable_sns_delivery_logging" { type = bool description = "Enable SNS Delivery Failure Notifications" - default = false + default = true } variable "sns_success_logging_sample_percent" { @@ -94,13 +94,13 @@ variable "event_cache_expiry_days" { variable "enable_event_cache" { type = bool description = "Enable caching of events to an S3 bucket" - default = false + default = true } variable "enable_firehose_raw_message_delivery" { type = bool description = "Enables raw message delivery on firehose subscription" - default = false + default = true } variable "force_destroy" { From 365f6a910e4e9aee4bf2734e5cfa460aa6e6914b Mon Sep 17 00:00:00 2001 From: David Wass Date: Wed, 21 Jan 2026 11:10:34 +0000 Subject: [PATCH 3/5] Status updates for sub --- .../api/module_lambda_letter_status_update.tf | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/infrastructure/terraform/components/api/module_lambda_letter_status_update.tf b/infrastructure/terraform/components/api/module_lambda_letter_status_update.tf index 59393bd29..bd953b76f 100644 --- a/infrastructure/terraform/components/api/module_lambda_letter_status_update.tf +++ b/infrastructure/terraform/components/api/module_lambda_letter_status_update.tf @@ -82,4 +82,17 @@ data "aws_iam_policy_document" "letter_status_update" { module.letter_status_updates_queue.sqs_queue_arn ] } + + statement { + sid = "AllowSNSPublish" + effect = "Allow" + + actions = [ + "sns:Publish" + ] + + resources = [ + module.eventsub.sns_topic.arn + ] + } } From 99d6ab621b6940ae4df52fca7f69231ff08bb852 Mon Sep 17 00:00:00 2001 From: David Wass Date: Thu, 22 Jan 2026 14:53:23 +0000 Subject: [PATCH 4/5] glue table --- .../terraform/components/api/README.md | 1 + .../api/glue_catalog_database_supplier.tf | 4 + .../api/glue_catalog_table_events.tf | 86 +++++++++++++++++++ .../components/api/s3_event_reporting.tf | 19 ++++ .../terraform/components/api/variables.tf | 6 ++ 5 files changed, 116 insertions(+) create mode 100644 infrastructure/terraform/components/api/glue_catalog_database_supplier.tf create mode 100644 infrastructure/terraform/components/api/glue_catalog_table_events.tf create mode 100644 infrastructure/terraform/components/api/s3_event_reporting.tf diff --git a/infrastructure/terraform/components/api/README.md b/infrastructure/terraform/components/api/README.md index 01fe4c4be..03e0b0b28 100644 --- a/infrastructure/terraform/components/api/README.md +++ b/infrastructure/terraform/components/api/README.md @@ -17,6 +17,7 @@ No requirements. | [core\_environment](#input\_core\_environment) | Environment of Core | `string` | `"prod"` | no | | [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no | | [disable\_gateway\_execute\_endpoint](#input\_disable\_gateway\_execute\_endpoint) | Disable the execution endpoint for the API Gateway | `bool` | `true` | no | +| [enable\_backups](#input\_enable\_backups) | Enable backups | `bool` | `false` | no | | [enable\_api\_data\_trace](#input\_enable\_api\_data\_trace) | Enable API Gateway data trace logging | `bool` | `false` | no | | [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `false` | no | | [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `false` | no | diff --git a/infrastructure/terraform/components/api/glue_catalog_database_supplier.tf b/infrastructure/terraform/components/api/glue_catalog_database_supplier.tf new file mode 100644 index 000000000..ae64cab23 --- /dev/null +++ b/infrastructure/terraform/components/api/glue_catalog_database_supplier.tf @@ -0,0 +1,4 @@ +resource "aws_glue_catalog_database" "supplier" { + name = "${local.csi}-supplier" + description = "Glue catalog database for Suppliers API" +} diff --git a/infrastructure/terraform/components/api/glue_catalog_table_events.tf b/infrastructure/terraform/components/api/glue_catalog_table_events.tf new file mode 100644 index 000000000..f50b4a121 --- /dev/null +++ b/infrastructure/terraform/components/api/glue_catalog_table_events.tf @@ -0,0 +1,86 @@ +resource "aws_glue_catalog_table" "events" { + name = "events_history" + database_name = aws_glue_catalog_database.supplier.name + + table_type = "EXTERNAL_TABLE" + + parameters = { + classification = "json" + } + + storage_descriptor { + location = "s3://${aws_s3_bucket.event_reporting.bucket}/events/" + input_format = "org.apache.hadoop.mapred.TextInputFormat" + output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat" + + columns { + name = "type" + type = "string" + } + + columns { + name = "messageid" + type = "string" + } + + columns { + name = "topicarn" + type = "string" + } + + columns { + name = "message" + type = "string" + } + + columns { + name = "timestamp" + type = "string" + } + + columns { + name = "unsubscribeurl" + type = "string" + } + + columns { + name = "change" + type = "double" + } + + columns { + name = "price" + type = "double" + } + + columns { + name = "ticker_symbol" + type = "string" + } + + columns { + name = "sector" + type = "string" + } + + columns { + name = "partition_0" + type = "string" + } + + columns { + name = "partition_1" + type = "string" + } + + columns { + name = "partition_2" + type = "string" + } + + columns { + name = "partition_3" + type = "string" + } + } +} diff --git a/infrastructure/terraform/components/api/s3_event_reporting.tf b/infrastructure/terraform/components/api/s3_event_reporting.tf new file mode 100644 index 000000000..e61602d0c --- /dev/null +++ b/infrastructure/terraform/components/api/s3_event_reporting.tf @@ -0,0 +1,19 @@ +resource "aws_s3_bucket" "event_reporting" { + bucket = "${local.csi_global}-event-reporting" + + tags = merge(local.default_tags, { "Enable-Backup" = var.enable_backups }, { "Enable-S3-Continuous-Backup" = var.enable_backups }) +} +resource "aws_s3_bucket_ownership_controls" "event_reporting" { + bucket = aws_s3_bucket.event_reporting.id + + rule { + object_ownership = "BucketOwnerPreferred" + } +} +resource "aws_s3_bucket_versioning" "event_reporting" { + bucket = aws_s3_bucket.event_reporting.id + + versioning_configuration { + status = "Enabled" + } +} diff --git a/infrastructure/terraform/components/api/variables.tf b/infrastructure/terraform/components/api/variables.tf index 47928a960..410f59fb2 100644 --- a/infrastructure/terraform/components/api/variables.tf +++ b/infrastructure/terraform/components/api/variables.tf @@ -163,6 +163,12 @@ variable "core_environment" { } +variable "enable_backups" { + type = bool + description = "Enable backups" + default = false +} + # Event Pub/Sub cache settings variable "enable_event_cache" { type = bool From 0edb6f525d9cf95c299892d4083494447740fb0b Mon Sep 17 00:00:00 2001 From: David Wass Date: Mon, 26 Jan 2026 10:30:00 +0000 Subject: [PATCH 5/5] glue crawler --- .../terraform/components/api/README.md | 2 +- .../api/glue_catalog_table_events.tf | 2 +- .../api/glue_crawler_event_crawler.tf | 19 ++++ .../terraform/components/api/iam_role_glue.tf | 89 +++++++++++++++++++ .../api/s3_bucket_policy_eventcache.tf | 47 ++++++++++ 5 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 infrastructure/terraform/components/api/glue_crawler_event_crawler.tf create mode 100644 infrastructure/terraform/components/api/iam_role_glue.tf create mode 100644 infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf diff --git a/infrastructure/terraform/components/api/README.md b/infrastructure/terraform/components/api/README.md index 03e0b0b28..24dfd303d 100644 --- a/infrastructure/terraform/components/api/README.md +++ b/infrastructure/terraform/components/api/README.md @@ -17,8 +17,8 @@ No requirements. | [core\_environment](#input\_core\_environment) | Environment of Core | `string` | `"prod"` | no | | [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no | | [disable\_gateway\_execute\_endpoint](#input\_disable\_gateway\_execute\_endpoint) | Disable the execution endpoint for the API Gateway | `bool` | `true` | no | -| [enable\_backups](#input\_enable\_backups) | Enable backups | `bool` | `false` | no | | [enable\_api\_data\_trace](#input\_enable\_api\_data\_trace) | Enable API Gateway data trace logging | `bool` | `false` | no | +| [enable\_backups](#input\_enable\_backups) | Enable backups | `bool` | `false` | no | | [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `false` | no | | [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `false` | no | | [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes | diff --git a/infrastructure/terraform/components/api/glue_catalog_table_events.tf b/infrastructure/terraform/components/api/glue_catalog_table_events.tf index f50b4a121..a5fc161fc 100644 --- a/infrastructure/terraform/components/api/glue_catalog_table_events.tf +++ b/infrastructure/terraform/components/api/glue_catalog_table_events.tf @@ -1,5 +1,5 @@ resource "aws_glue_catalog_table" "events" { - name = "events_history" + name = "${local.csi}-events_history" database_name = aws_glue_catalog_database.supplier.name table_type = "EXTERNAL_TABLE" diff --git a/infrastructure/terraform/components/api/glue_crawler_event_crawler.tf b/infrastructure/terraform/components/api/glue_crawler_event_crawler.tf new file mode 100644 index 000000000..6d45a3ce1 --- /dev/null +++ b/infrastructure/terraform/components/api/glue_crawler_event_crawler.tf @@ -0,0 +1,19 @@ +resource "aws_glue_crawler" "event_crawler" { + name = "event-crawler-${aws_glue_catalog_table.events.name}" + database_name = aws_glue_catalog_database.supplier.name + role = aws_iam_role.glue_role.arn + + table_prefix = "" + s3_target { + path = "s3://${local.csi_global}-eventcache/" + } + recrawl_policy { + recrawl_behavior = "CRAWL_EVERYTHING" + } + + schema_change_policy { + delete_behavior = "LOG" + update_behavior = "UPDATE_IN_DATABASE" + } + +} diff --git a/infrastructure/terraform/components/api/iam_role_glue.tf b/infrastructure/terraform/components/api/iam_role_glue.tf new file mode 100644 index 000000000..f4910c3c9 --- /dev/null +++ b/infrastructure/terraform/components/api/iam_role_glue.tf @@ -0,0 +1,89 @@ +resource "aws_iam_role" "glue_role" { + name = "${local.csi}-glue-role" + assume_role_policy = data.aws_iam_policy_document.glue_assume_role.json +} + +data "aws_iam_policy_document" "glue_assume_role" { + statement { + sid = "AllowGlueServiceAssumeRole" + effect = "Allow" + + principals { + type = "Service" + identifiers = ["glue.amazonaws.com"] + } + + actions = [ + "sts:AssumeRole", + ] + } +} + +resource "aws_iam_policy" "glue_service_policy" { + name = "${local.csi}-glue-service-policy" + description = "Policy for ${local.csi} Glue Service Role" + policy = data.aws_iam_policy_document.glue_service_policy.json +} + +data "aws_iam_policy_document" "glue_service_policy" { + statement { + sid = "AllowGlueLogging" + effect = "Allow" + + actions = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + resources = ["arn:aws:logs:*:*:*"] + } + + statement { + sid = "AllowS3Access" + effect = "Allow" + + actions = [ + "s3:GetObject", + "s3:PutObject", + "s3:ListBucket", + "s3:GetBucketLocation", + "s3:DeleteObject" + ] + resources = ["arn:aws:s3:::${local.csi}-glue-bucket/*", + "arn:aws:s3:::${local.csi_global}-event-reporting/*"] + } + statement { + sid = "GlueCatalogAccess" + effect = "Allow" + actions = [ + "glue:GetDatabase", + "glue:GetDatabases", + "glue:GetTable", + "glue:GetTables", + "glue:CreateTable", + "glue:UpdateTable", + "glue:CreatePartition", + "glue:BatchCreatePartition", + "glue:GetPartition", + "glue:BatchGetPartition" + ] + resources = ["*"] + } + statement { + sid = "S3TempAndGlueETL" + effect = "Allow" + actions = [ + "s3:PutObject", + "s3:GetObject" + ] + resources = [ + "arn:aws:s3:::aws-glue-*", + "arn:aws:s3:::aws-glue-*/*" + ] + } +} + +resource "aws_iam_role_policy_attachment" "gllue_attach_policy" { + role = aws_iam_role.glue_role.name + policy_arn = aws_iam_policy.glue_service_policy.arn +} diff --git a/infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf b/infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf new file mode 100644 index 000000000..e7014be97 --- /dev/null +++ b/infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf @@ -0,0 +1,47 @@ +resource "aws_s3_bucket_policy" "eventcache" { + bucket = "${local.csi_global}-eventcache" + policy = data.aws_iam_policy_document.eventcache.json +} + +data "aws_iam_policy_document" "eventcache" { + statement { + sid = "AllowGlueListBucketAndGetLocation" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [aws_iam_role.glue_role.arn] + } + + actions = [ + "s3:ListBucket", + "s3:GetBucketLocation" + ] + + resources = [ + "arn:aws:s3:::${local.csi_global}-eventcache" + ] + } + + # Object-level permissions: Get/Put/Delete objects + statement { + sid = "AllowGlueObjectAccess" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [aws_iam_role.glue_role.arn] + } + + actions = [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:PutObject", + "s3:DeleteObject" + ] + + resources = [ + "arn:aws:s3:::${local.csi_global}-eventcache/*" + ] + } +}