diff --git a/.github/workflows/run-e2e-automation-tests.yml b/.github/workflows/run-e2e-automation-tests.yml index 54810aa86..d545fd344 100644 --- a/.github/workflows/run-e2e-automation-tests.yml +++ b/.github/workflows/run-e2e-automation-tests.yml @@ -68,6 +68,10 @@ on: - sandbox - proxy_smoke - Batch_File_Validation_Feature + mns_validation_required: + description: Set to true if you want the MNS validation to be performed as part of the tests. please keep in mind it will increase execution time. + default: "false" + type: boolean env: APIGEE_AUTH_ENV: ${{ inputs.apigee_environment == 'int' && inputs.apigee_environment || 'internal-dev' }} @@ -235,6 +239,7 @@ jobs: MEDICUS_client_Id: ${{ secrets.MEDICUS_client_Id }} MEDICUS_client_Secret: ${{ secrets.MEDICUS_client_Secret }} aws_account_id: ${{ vars.AWS_ACCOUNT_ID }} + mns_validation_required: ${{ inputs.mns_validation_required || startsWith(inputs.sub_environment, 'pr-') || inputs.apigee_environment == 'internal-dev' }} TEST_PATH: ${{ inputs.service_under_test == 'batch' && 'features/batchTests' || inputs.service_under_test == 'fhir_api' && 'features/APITests' || 'features' }} TEST_FILTER: ${{ inputs.suite_to_run == 'proxy_smoke' && 'Status_feature' || inputs.suite_to_run }} run: poetry run pytest "$TEST_PATH" -m "$TEST_FILTER" --junitxml=output/test-results.xml --alluredir=output/allure-results diff --git a/tests/e2e_automation/features/APITests/steps/common_steps.py b/tests/e2e_automation/features/APITests/steps/common_steps.py index 0533e461b..5e216711e 100644 --- a/tests/e2e_automation/features/APITests/steps/common_steps.py +++ b/tests/e2e_automation/features/APITests/steps/common_steps.py @@ -351,8 +351,10 @@ def send_update_for_immunization_event(context): def created_event_is_being_updated_twice(context): send_update_for_immunization_event(context) The_request_will_have_status_code(context, 200) + mns_event_will_be_triggered_with_correct_data(context=context, action="UPDATE") send_update_for_vaccination_detail(context) The_request_will_have_status_code(context, 200) + mns_event_will_be_triggered_with_correct_data(context=context, action="UPDATE") @given("created event is being deleted") @@ -379,9 +381,8 @@ def mns_event_will_not_be_triggered_for_the_event(context): message_body = read_message( context, queue_type="notification", - action="CREATE", wait_time_seconds=5, - max_empty_polls=1, + max_total_wait_seconds=20, ) print("No MNS create event is created") assert message_body is None, "Not expected a message but queue returned a message" @@ -392,9 +393,8 @@ def validate_mns_event_not_triggered_for_updated_event(context): message_body = read_message( context, queue_type="notification", - action="UPDATE", wait_time_seconds=5, - max_empty_polls=3, + max_total_wait_seconds=20, ) print("no MNS update event is created") assert message_body is None, "Not expected a message but queue returned a message" @@ -417,7 +417,7 @@ def normalize_param(value: str) -> str: def calculate_age(birth_date_str: str, occurrence_datetime_str: str) -> int: - birth = datetime.strptime(birth_date_str, "%Y-%m-%d").date() + birth = parse_birth_date(birth_date_str) occurrence = datetime.fromisoformat(occurrence_datetime_str).date() age = occurrence.year - birth.year if (occurrence.month, occurrence.day) < (birth.month, birth.day): @@ -425,6 +425,15 @@ def calculate_age(birth_date_str: str, occurrence_datetime_str: str) -> int: return age +def parse_birth_date(date_str: str) -> datetime.date: + for fmt in ("%Y-%m-%d", "%Y%m%d"): + try: + return datetime.strptime(date_str, fmt).date() + except ValueError: + pass + raise ValueError(f"Invalid birth date format: {date_str}") + + def is_valid_uuid(value: str) -> bool: try: uuid.UUID(value) @@ -462,36 +471,47 @@ def validate_sqs_message(context, message_body, action): f"msn event for {action} DataRef mismatch: expected {context.url}/{context.ImmsID}, got {message_body.dataref}", ) - check.is_true( - normalize(message_body.filtering.generalpractitioner) == normalize(context.gp_code), - f"msn event for {action} GP code mismatch: expected {context.gp_code}, got {message_body.filtering.generalpractitioner}", - ) + if context.S3_env not in ["int", "preprod"]: + check.is_true( + message_body.filtering is not None, + f"msn event for {action} Filtering is missing in the message body", + ) - expected_org = context.create_object.performer[1].actor.identifier.value - check.is_true( - normalize(message_body.filtering.sourceorganisation) == normalize(expected_org), - f"msn event for {action} Source org mismatch: expected {expected_org}, got {message_body.filtering.sourceorganisation}", - ) + check.is_true( + normalize(message_body.filtering.generalpractitioner) == normalize(context.gp_code), + f"msn event for {action} GP code mismatch: expected {context.gp_code}, got {message_body.filtering.generalpractitioner}", + ) - check.is_true( - message_body.filtering.sourceapplication.upper() == context.supplier_name.upper(), - f"msn event for {action} Source application mismatch: expected {context.supplier_name}, got {message_body.filtering.sourceapplication}", - ) + expected_org = context.immunization_object.performer[1].actor.identifier.value + check.is_true( + normalize(message_body.filtering.sourceorganisation) == normalize(expected_org), + f"msn event for {action} Source org mismatch: expected {expected_org}, got {message_body.filtering.sourceorganisation}", + ) - check.is_true( - message_body.filtering.subjectage == context.patient_age, - f"msn event for {action} Age mismatch: expected {context.patient_age}, got {message_body.filtering.subjectage}", - ) + check.is_true( + message_body.filtering.sourceapplication.upper() == context.supplier_name.upper(), + f"msn event for {action} Source application mismatch: expected {context.supplier_name}, got {message_body.filtering.sourceapplication}", + ) - check.is_true( - message_body.filtering.immunisationtype == context.vaccine_type.upper(), - f"msn event for {action} Immunisation type mismatch: expected {context.vaccine_type.upper()}, got {message_body.filtering.immunisationtype}", - ) + check.is_true( + message_body.filtering.subjectage == context.patient_age, + f"msn event for {action} Age mismatch: expected {context.patient_age}, got {message_body.filtering.subjectage}", + ) - check.is_true( - message_body.filtering.action == action.upper(), - f"msn event for {action} Action mismatch: expected {action.upper()}, got {message_body.filtering.action}", - ) + check.is_true( + message_body.filtering.immunisationtype == context.vaccine_type.upper(), + f"msn event for {action} Immunisation type mismatch: expected {context.vaccine_type.upper()}, got {message_body.filtering.immunisationtype}", + ) + + check.is_true( + message_body.filtering.action == action.upper(), + f"msn event for {action} Action mismatch: expected {action.upper()}, got {message_body.filtering.action}", + ) + else: + check.is_true( + message_body.filtering is None, + f"msn event for {action} Filtering is present in the message body when it shouldn't be for int environment", + ) def mns_event_will_be_triggered_with_correct_data_for_deleted_event(context): @@ -499,25 +519,29 @@ def mns_event_will_be_triggered_with_correct_data_for_deleted_event(context): message_body = read_message( context, queue_type="notification", - action="DELETE", wait_time_seconds=5, - max_empty_polls=3, + max_total_wait_seconds=20, ) print( "No MNS delete event is created as expected since NHS number is not present in the original immunization event" ) assert message_body is None, "Not expected a message but queue returned a message" else: - message_body = read_message(context, queue_type="notification", action="DELETE") + message_body = read_message(context, queue_type="notification") print(f"Read deleted message from SQS: {message_body}") assert message_body is not None, "Expected a delete message but queue returned empty" validate_sqs_message(context, message_body, "DELETE") def mns_event_will_be_triggered_with_correct_data(context, action): - message_body = read_message(context, queue_type="notification", action=action) - print(f"Read {action}d message from SQS: {message_body}") - assert message_body is not None, f"Expected a {action} message but queue returned empty" - context.gp_code = get_gp_code_by_nhs_number(context.patient.identifier[0].value) - context.patient_age = calculate_age(context.patient.birthDate, context.immunization_object.occurrenceDateTime) - validate_sqs_message(context, message_body, action) + if context.mns_validation_required.strip().lower() == "true": + message_body = read_message(context, queue_type="notification") + print(f"Read {action}d message from SQS: {message_body}") + assert message_body is not None, f"Expected a {action} message but queue returned empty" + context.gp_code = get_gp_code_by_nhs_number(context.patient.identifier[0].value) + context.patient_age = calculate_age(context.patient.birthDate, context.immunization_object.occurrenceDateTime) + validate_sqs_message(context, message_body, action) + else: + print( + f"MNS event validation is skipped since mns_validation_required is set to {context.mns_validation_required}" + ) diff --git a/tests/e2e_automation/features/batchTests/Steps/batch_common_steps.py b/tests/e2e_automation/features/batchTests/Steps/batch_common_steps.py index 38e48ffef..038c8edf9 100644 --- a/tests/e2e_automation/features/batchTests/Steps/batch_common_steps.py +++ b/tests/e2e_automation/features/batchTests/Steps/batch_common_steps.py @@ -2,6 +2,7 @@ import json import os import re +from collections import Counter from datetime import UTC, datetime import pandas as pd @@ -21,6 +22,7 @@ generate_file_name, save_record_to_batch_files_directory, ) +from src.objectModels.patient_loader import get_gp_code_by_nhs_number from utilities.batch_file_helper import ( read_and_validate_csv_bus_ack_file_content, validate_bus_ack_file_for_error, @@ -34,7 +36,15 @@ wait_and_read_ack_file, wait_for_file_to_move_archive, ) +from utilities.date_helper import iso_to_compact, normalize_utc_suffix from utilities.enums import ActionFlag, ActionMap, Operation +from utilities.sqs_message_halder import read_messages_for_batch + +from features.APITests.steps.common_steps import ( + calculate_age, + is_valid_uuid, + mns_event_will_be_triggered_with_correct_data, +) def ignore_if_local_run(func): @@ -299,6 +309,33 @@ def all_record_are_rejected_for_given_field_name(context): assert all_valid, "One or more records failed validation checks" +@then(parsers.parse("MNS event will be triggered with correct data for all '{event_type}' events where NHS is not null")) +def mns_event_will_be_triggered_with_correct_data_for_created_events_in_batch_file(context, event_type): + if context.mns_validation_required.strip().lower() != "true": + print( + f"MNS event validation is skipped since mns_validation_required is set to {context.mns_validation_required}" + ) + return + + action = event_type.upper() if event_type.upper() in ["CREATE", "UPDATE"] else "CREATE" + + df = context.vaccine_df.dropna(subset=["IMMS_ID"]).copy() + df["IMMS_ID_CLEAN"] = df["IMMS_ID"].astype(str).str.replace("Immunization#", "", regex=False) + + valid_rows = list(df.itertuples(index=False)) + + if not valid_rows: + print("No valid NHS rows found — skipping MNS validation.") + return + + mns_event_will_be_triggered_for_batch_record(context=context, action=action, valid_rows=valid_rows) + + +@then("Api updated event will trigger MNS event with correct data") +def mns_event_will_be_triggered_with_correct_data_for_api_updated_events(context): + mns_event_will_be_triggered_with_correct_data(context=context, action="UPDATE") + + def normalize(value): return "" if pd.isna(value) or value == "" else value @@ -443,3 +480,198 @@ def validate_imms_delta_table_for_deleted_records_in_batch_file(context): Operation.deleted.value, ActionFlag.deleted.value, ) + + +def _is_null_nhs_row(row) -> bool: + return str(row.UNIQUE_ID).startswith("NullNHS") or str(row.NHS_NUMBER).strip() in ("", "None", "nan") + + +def _assert_no_mns_events_for_null_nhs_rows(context, null_nhs_rows, wait_seconds=20): + if not null_nhs_rows: + print("No NullNHS rows — skipping negative MNS check.") + return + + # Use an unreachable expected_count so the poller runs for the full wait_seconds, + # then check that nothing arrived for these datarefs. + unexpected = read_messages_for_batch( + context, + queue_type="notification", + valid_rows=null_nhs_rows, + expected_count=len(null_nhs_rows) + 1, + max_total_wait_seconds=wait_seconds, + ) + + assert not unexpected, f"Unexpected MNS events received for NullNHS records: {[msg.dataref for msg in unexpected]}" + + +def mns_event_will_be_triggered_for_batch_record(context, action, valid_rows): + null_nhs_rows = [row for row in valid_rows if _is_null_nhs_row(row)] + positive_rows = [row for row in valid_rows if not _is_null_nhs_row(row)] + + row_lookup = {str(row.NHS_NUMBER): row for row in positive_rows} + + messages = read_messages_for_batch( + context, + queue_type="notification", + valid_rows=positive_rows, + expected_count=len(positive_rows), + ) + + print(f"Read {len(messages)} {action} message(s) from SQS") + + assert messages, f"Expected at least one {action} message but queue returned empty" + + for msg in messages: + nhs = msg.subject + + assert nhs in row_lookup, f"Received message for NHS {nhs} but it does not exist in valid_rows" + + row = row_lookup[nhs] + + context.nhs_number = row.NHS_NUMBER + context.gp_code = get_gp_code_by_nhs_number(row.NHS_NUMBER) + context.patient_age = calculate_age(row.PERSON_DOB, row.DATE_AND_TIME) + context.ImmsID = row.IMMS_ID_CLEAN + + print(f"Validating message for NHS {nhs}, IMMS ID {context.ImmsID}") + + validate_sqs_message_for_batch_record(context, msg, row) + + _assert_no_mns_events_for_null_nhs_rows(context, null_nhs_rows) + + +def validate_sqs_message_for_batch_record(context, message_body, row): + check.is_true(message_body.specversion == "1.0") + check.is_true(message_body.source == "uk.nhs.vaccinations-data-flow-management") + check.is_true(message_body.type == "imms-vaccination-record-change-1") + + check.is_true(is_valid_uuid(message_body.id), f"Invalid UUID: {message_body.id}") + + imms_date_time = normalize_utc_suffix(row.DATE_AND_TIME) + check.is_true( + message_body.time == f"{imms_date_time}Z", + f"msn event for {row.NHS_NUMBER} Time missing or mismatch: message_body.time = {message_body.time}, imms_date_time = {imms_date_time}", + ) + expected_nhs_number = row.NHS_NUMBER + if expected_nhs_number is None: + expected_nhs_number = "" + check.is_true( + message_body.subject == expected_nhs_number, + f"msn event for {row.NHS_NUMBER}Subject mismatch: expected {expected_nhs_number}, got {message_body.subject}", + ) + + check.is_true( + message_body.dataref == f"{context.url}/{row.IMMS_ID_CLEAN}", + f"msn event for {row.NHS_NUMBER} DataRef mismatch: expected {context.url}/{row.IMMS_ID_CLEAN}, got {message_body.dataref}", + ) + + if context.S3_env not in ["int", "preprod"]: + check.is_true( + message_body.filtering is not None, + f"msn event for {row.NHS_NUMBER} Filtering is missing in the message body", + ) + + check.is_true( + normalize(message_body.filtering.generalpractitioner) == normalize(context.gp_code), + f"msn event for {row.NHS_NUMBER} GP code mismatch: expected {context.gp_code}, got {message_body.filtering.generalpractitioner}", + ) + + expected_org = row.SITE_CODE + check.is_true( + normalize(message_body.filtering.sourceorganisation) == normalize(expected_org), + f"msn event for {row.NHS_NUMBER} Source org mismatch: expected {expected_org}, got {message_body.filtering.sourceorganisation}", + ) + + check.is_true( + message_body.filtering.sourceapplication.upper() == context.supplier_name.upper(), + f"msn event for {row.NHS_NUMBER} Source application mismatch: expected {context.supplier_name}, got {message_body.filtering.sourceapplication}", + ) + + check.is_true( + message_body.filtering.subjectage == context.patient_age, + f"msn event for {row.NHS_NUMBER} Age mismatch: expected {context.patient_age}, got {message_body.filtering.subjectage}", + ) + + check.is_true( + message_body.filtering.immunisationtype == context.vaccine_type.upper(), + f"msn event for {row.NHS_NUMBER} Immunisation type mismatch: expected {context.vaccine_type.upper()}, got {message_body.filtering.immunisationtype}", + ) + action = row.ACTION_FLAG.upper() if row.ACTION_FLAG.upper() in ["UPDATE", "DELETE"] else "CREATE" + check.is_true( + message_body.filtering.action == action.upper(), + f"msn event for {row.NHS_NUMBER} Action mismatch: expected {action.upper()}, got {message_body.filtering.action}", + ) + else: + check.is_true( + message_body.filtering is None, + f"msn event for {row.NHS_NUMBER} Filtering is present in the message body when it shouldn't be for int environment", + ) + + +@then("MNS event will be triggered with correct data for both events where NHS is not null") +def mns_event_will_be_triggered_with_correct_data_for_both_events_in_batch_file( + context, +): + if context.mns_validation_required.strip().lower() != "true": + print( + f"MNS event validation is skipped since mns_validation_required is set to {context.mns_validation_required}" + ) + return + + df = context.vaccine_df.dropna(subset=["IMMS_ID"]).copy() + df["IMMS_ID_CLEAN"] = df["IMMS_ID"].astype(str).str.replace("Immunization#", "", regex=False) + + all_rows = list(df.itertuples(index=False)) + + if not all_rows: + print("No rows found — skipping MNS validation.") + return + + null_nhs_rows = [row for row in all_rows if _is_null_nhs_row(row)] + expected_rows = [row for row in all_rows if not _is_null_nhs_row(row)] + + messages = read_messages_for_batch( + context, + queue_type="notification", + valid_rows=expected_rows, + expected_count=len(expected_rows), + ) + + print(f"Read {len(messages)} message(s) from SQS") + + assert len(messages) == len(expected_rows), f"Expected {len(expected_rows)} MNS events, but received {len(messages)}" + + nhs_counts = Counter(msg.subject for msg in messages) + expected_nhs_numbers = {row.NHS_NUMBER for row in expected_rows} + + assert len(nhs_counts) == len(expected_nhs_numbers), ( + f"Expected {len(expected_nhs_numbers)} NHS numbers, but got {len(nhs_counts)}: {list(nhs_counts.keys())}" + ) + + # Check each NHS number has exactly 2 events (one CREATE, one UPDATE) + for nhs, count in nhs_counts.items(): + assert count == 2, f"NHS {nhs} expected 2 events (CREATE + UPDATE) but received {count}" + + _assert_no_mns_events_for_null_nhs_rows(context, null_nhs_rows) + + +def build_batch_row_from_api_object(context, action): + patient = context.create_object.contained[1] + imms = context.create_object + performer_org = imms.performer[1].actor.identifier.value + + occurrenceDateTime = iso_to_compact(imms.occurrenceDateTime.replace("-", "").replace(":", "")) + + return { + "NHS_NUMBER": patient.identifier[0].value, + "PERSON_FORENAME": patient.name[0].given[0], + "PERSON_SURNAME": patient.name[0].family, + "PERSON_GENDER_CODE": patient.gender, + "PERSON_DOB": patient.birthDate.replace("-", ""), + "PERSON_POSTCODE": patient.address[0].postalCode, + "ACTION_FLAG": action.upper(), + "UNIQUE_ID": imms.identifier[0].value, + "UNIQUE_ID_URI": imms.identifier[0].system, + "SITE_CODE": performer_org, + "DATE_AND_TIME": occurrenceDateTime, + } diff --git a/tests/e2e_automation/features/batchTests/Steps/test_create_batch_steps.py b/tests/e2e_automation/features/batchTests/Steps/test_create_batch_steps.py index 57dca0955..6003379f2 100644 --- a/tests/e2e_automation/features/batchTests/Steps/test_create_batch_steps.py +++ b/tests/e2e_automation/features/batchTests/Steps/test_create_batch_steps.py @@ -2,7 +2,11 @@ from src.objectModels.batch.batch_file_builder import get_batch_date from utilities.text_helper import get_text -from .batch_common_steps import build_dataFrame_using_datatable, create_batch_file, ignore_if_local_run +from .batch_common_steps import ( + build_dataFrame_using_datatable, + create_batch_file, + ignore_if_local_run, +) scenarios("batchTests/create_batch.feature") @@ -12,7 +16,6 @@ def valid_batch_file_is_created_with_minimum_details(datatable, context): build_dataFrame_using_datatable(datatable, context) columns_to_clear = [ - "NHS_NUMBER", "VACCINATION_PROCEDURE_TERM", "VACCINE_PRODUCT_CODE", "VACCINE_PRODUCT_TERM", @@ -131,8 +134,14 @@ def valid_batch_file_is_created_with_missing_mandatory_fields(datatable, context context.vaccine_df.loc[1, "SITE_CODE_TYPE_URI"] = "" context.vaccine_df.loc[2, "LOCATION_CODE"] = "" context.vaccine_df.loc[3, "LOCATION_CODE_TYPE_URI"] = "" - context.vaccine_df.loc[4, ["UNIQUE_ID", "PERSON_SURNAME"]] = ["", "no_unique_identifiers"] - context.vaccine_df.loc[5, ["UNIQUE_ID_URI", "PERSON_SURNAME"]] = ["", "no_unique_identifiers"] + context.vaccine_df.loc[4, ["UNIQUE_ID", "PERSON_SURNAME"]] = [ + "", + "no_unique_identifiers", + ] + context.vaccine_df.loc[5, ["UNIQUE_ID_URI", "PERSON_SURNAME"]] = [ + "", + "no_unique_identifiers", + ] context.vaccine_df.loc[6, "PRIMARY_SOURCE"] = "" context.vaccine_df.loc[7, "VACCINATION_PROCEDURE_CODE"] = "" context.vaccine_df.loc[8, "SITE_CODE"] = " " @@ -140,7 +149,10 @@ def valid_batch_file_is_created_with_missing_mandatory_fields(datatable, context context.vaccine_df.loc[10, "LOCATION_CODE"] = " " context.vaccine_df.loc[11, "LOCATION_CODE_TYPE_URI"] = " " context.vaccine_df.loc[12, ["UNIQUE_ID", "PERSON_SURNAME"]] = [" ", "no_unique_id"] - context.vaccine_df.loc[13, ["UNIQUE_ID_URI", "PERSON_SURNAME"]] = [" ", "no_unique_id_uri"] + context.vaccine_df.loc[13, ["UNIQUE_ID_URI", "PERSON_SURNAME"]] = [ + " ", + "no_unique_id_uri", + ] context.vaccine_df.loc[14, "PRIMARY_SOURCE"] = " " context.vaccine_df.loc[15, "VACCINATION_PROCEDURE_CODE"] = " " context.vaccine_df.loc[16, "PRIMARY_SOURCE"] = "test" diff --git a/tests/e2e_automation/features/batchTests/Steps/test_delete_batch_steps.py b/tests/e2e_automation/features/batchTests/Steps/test_delete_batch_steps.py index 6d54c3d20..96006e751 100644 --- a/tests/e2e_automation/features/batchTests/Steps/test_delete_batch_steps.py +++ b/tests/e2e_automation/features/batchTests/Steps/test_delete_batch_steps.py @@ -2,11 +2,23 @@ from pytest_bdd import given, scenarios, then, when from src.objectModels.batch.batch_file_builder import build_batch_file -from features.APITests.steps.common_steps import validate_imms_event_table_by_operation, validVaccinationRecordIsCreated -from features.APITests.steps.test_create_steps import validate_imms_delta_table_by_ImmsID -from features.APITests.steps.test_delete_steps import validate_imms_delta_table_by_deleted_ImmsID - -from .batch_common_steps import build_dataFrame_using_datatable, create_batch_file, ignore_if_local_run +from features.APITests.steps.common_steps import ( + validate_imms_event_table_by_operation, + validVaccinationRecordIsCreated, +) +from features.APITests.steps.test_create_steps import ( + validate_imms_delta_table_by_ImmsID, +) +from features.APITests.steps.test_delete_steps import ( + validate_imms_delta_table_by_deleted_ImmsID, +) + +from .batch_common_steps import ( + build_batch_row_from_api_object, + build_dataFrame_using_datatable, + create_batch_file, + ignore_if_local_run, +) scenarios("batchTests/delete_batch.feature") @@ -31,31 +43,12 @@ def create_valid_vaccination_record_through_api(context): @when("An delete to above vaccination record is made through batch file upload") def upload_batch_file_to_s3_for_update(context): record = build_batch_file(context) - context.vaccine_df = pd.DataFrame([record.dict()]) - context.vaccine_df.loc[ - 0, - [ - "NHS_NUMBER", - "PERSON_FORENAME", - "PERSON_SURNAME", - "PERSON_GENDER_CODE", - "PERSON_DOB", - "PERSON_POSTCODE", - "ACTION_FLAG", - "UNIQUE_ID", - "UNIQUE_ID_URI", - ], - ] = [ - context.create_object.contained[1].identifier[0].value, - context.create_object.contained[1].name[0].given[0], - context.create_object.contained[1].name[0].family, - context.create_object.contained[1].gender, - context.create_object.contained[1].birthDate.replace("-", ""), - context.create_object.contained[1].address[0].postalCode, - "DELETE", - context.create_object.identifier[0].value, - context.create_object.identifier[0].system, - ] + df = pd.DataFrame([record.dict()]) + + batch_fields = build_batch_row_from_api_object(context, "DELETE") + df.loc[0, list(batch_fields.keys())] = list(batch_fields.values()) + + context.vaccine_df = df create_batch_file(context) context.vaccine_df.loc[0, "IMMS_ID"] = context.ImmsID @@ -69,21 +62,14 @@ def validate_imms_delta_table_for_api_created_event(context): @when("Delete above vaccination record is made through batch file upload with mandatory field missing") def upload_batch_file_to_s3_for_update_with_mandatory_field_missing(context): - # Build base record record = build_batch_file(context) - context.vaccine_df = pd.DataFrame([record.dict()]) - base_fields = { - "NHS_NUMBER": context.create_object.contained[1].identifier[0].value, - "PERSON_FORENAME": context.create_object.contained[1].name[0].given[0], - "PERSON_SURNAME": context.create_object.contained[1].name[0].family, - "PERSON_GENDER_CODE": context.create_object.contained[1].gender, - "PERSON_DOB": "", - "PERSON_POSTCODE": context.create_object.contained[1].address[0].postalCode, - "ACTION_FLAG": "DELETE", - "UNIQUE_ID": context.create_object.identifier[0].value, - "UNIQUE_ID_URI": context.create_object.identifier[0].system, - } - context.vaccine_df.loc[0, list(base_fields.keys())] = list(base_fields.values()) + df = pd.DataFrame([record.dict()]) + + batch_fields = build_batch_row_from_api_object(context, "DELETE") + batch_fields["PERSON_DOB"] = "" + df.loc[0, list(batch_fields.keys())] = list(batch_fields.values()) + + context.vaccine_df = df create_batch_file(context) context.vaccine_df.loc[0, "IMMS_ID"] = context.ImmsID diff --git a/tests/e2e_automation/features/batchTests/Steps/test_update_batch_steps.py b/tests/e2e_automation/features/batchTests/Steps/test_update_batch_steps.py index 43c149304..8ab8e4c42 100644 --- a/tests/e2e_automation/features/batchTests/Steps/test_update_batch_steps.py +++ b/tests/e2e_automation/features/batchTests/Steps/test_update_batch_steps.py @@ -1,4 +1,5 @@ import uuid +from datetime import datetime import pandas as pd from pytest_bdd import given, scenarios, then, when @@ -30,6 +31,7 @@ ) from .batch_common_steps import ( + build_batch_row_from_api_object, build_dataFrame_using_datatable, create_batch_file, ) @@ -69,33 +71,14 @@ def create_valid_vaccination_record_with_missing_mandatory_fields(context): @when("An update to above vaccination record is made through batch file upload") def upload_batch_file_to_s3_for_update(context): record = build_batch_file(context) - context.vaccine_df = pd.DataFrame([record.dict()]) - context.vaccine_df.loc[ - 0, - [ - "NHS_NUMBER", - "PERSON_FORENAME", - "PERSON_SURNAME", - "PERSON_GENDER_CODE", - "PERSON_DOB", - "PERSON_POSTCODE", - "ACTION_FLAG", - "UNIQUE_ID", - "UNIQUE_ID_URI", - ], - ] = [ - context.create_object.contained[1].identifier[0].value, - context.create_object.contained[1].name[0].given[0], - context.create_object.contained[1].name[0].family, - context.create_object.contained[1].gender, - context.create_object.contained[1].birthDate.replace("-", ""), - context.create_object.contained[1].address[0].postalCode, - "UPDATE", - context.create_object.identifier[0].value, - context.create_object.identifier[0].system, - ] - context.expected_version = 2 + df = pd.DataFrame([record.dict()]) + + batch_fields = build_batch_row_from_api_object(context, "UPDATE") + df.loc[0, list(batch_fields.keys())] = list(batch_fields.values()) + + context.vaccine_df = df create_batch_file(context) + context.expected_version = 2 @then("The delta and imms event table will be populated with the correct data for api created event") @@ -121,6 +104,14 @@ def send_update_for_immunization_event_with_vaccination_detail_updated(context): context.immunization_object.contained[1].address[0].postalCode = row["PERSON_POSTCODE"] context.immunization_object.identifier[0].value = row["UNIQUE_ID"] context.immunization_object.identifier[0].system = row["UNIQUE_ID_URI"] + context.immunization_object.performer[1].actor.identifier.value = row["SITE_CODE"] + base_date = row["DATE_AND_TIME"][:15] + frac = row["DATE_AND_TIME"][15:] + dt = datetime.strptime(base_date, "%Y%m%dT%H%M%S") + + formatted = dt.strftime(f"%Y-%m-%dT%H:%M:%S.{frac}+00:00") + + context.immunization_object.occurrenceDateTime = formatted send_update_for_immunization_event(context) diff --git a/tests/e2e_automation/features/batchTests/create_batch.feature b/tests/e2e_automation/features/batchTests/create_batch.feature index 206e87879..7e184d0e5 100644 --- a/tests/e2e_automation/features/batchTests/create_batch.feature +++ b/tests/e2e_automation/features/batchTests/create_batch.feature @@ -10,7 +10,9 @@ Feature: Create the immunization event for a patient through batch file | InvalidInPDS | InvalidInPDS_NhsNumber | | SFlag | SFlag_NhsNumber | | Mod11_NHS | Mod11_NhSNumber | - | OldNHSNo | OldNHSNo | + | NullNHS | NullNHS_NhsNumber | + | OldNHSNo | OldNHS_NhsNumber | + | NO_GP_NHS | NO_GP_NhsNumber | When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file @@ -20,6 +22,7 @@ Feature: Create the immunization event for a patient through batch file And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table will be populated with the correct data for 'created' event for records in batch file And The delta table will be populated with the correct data for all created records in batch file + And MNS event will be triggered with correct data for all 'CREATE' events where NHS is not null @smoke @delete_cleanup_batch @vaccine_type_MMR @supplier_name_TPP @@ -30,7 +33,9 @@ Feature: Create the immunization event for a patient through batch file | InvalidInPDS | InvalidInPDS_NhsNumber | | SFlag | SFlag_NhsNumber | | Mod11_NHS | Mod11_NhSNumber | - | OldNHSNo | OldNHSNo | + | NullNHS | NullNHS_NhsNumber | + | OldNHSNo | OldNHS_NhsNumber | + | NO_GP_NHS | NO_GP_NhsNumber | When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file @@ -40,6 +45,7 @@ Feature: Create the immunization event for a patient through batch file And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table will be populated with the correct data for 'created' event for records in batch file And The delta table will be populated with the correct data for all created records in batch file + And MNS event will be triggered with correct data for all 'CREATE' events where NHS is not null @vaccine_type_FLU @supplier_name_MAVIS Scenario: Verify that vaccination record will be get rejected if date_and_time is invalid in batch file @@ -142,6 +148,7 @@ Feature: Create the immunization event for a patient through batch file And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table will be populated with the correct data for 'created' event for records in batch file And The delta table will be populated with the correct data for all created records in batch file + And MNS event will be triggered with correct data for all 'CREATE' events where NHS is not null @vaccine_type_ROTAVIRUS @supplier_name_TPP Scenario: verify that vaccination record will be get successful with different valid value in gender field @@ -167,6 +174,7 @@ Feature: Create the immunization event for a patient through batch file And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table will be populated with the correct data for 'created' event for records in batch file And The delta table will be populated with the correct data for all created records in batch file + And MNS event will be triggered with correct data for all 'CREATE' events where NHS is not null @vaccine_type_FLU @supplier_name_MAVIS Scenario: verify that vaccination record will be get rejected if mandatory fields for site, location and unique identifiers are missing in batch file @@ -215,6 +223,7 @@ Feature: Create the immunization event for a patient through batch file And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table will be populated with the correct data for 'created' event for records in batch file And The delta table will be populated with the correct data for all created records in batch file + And MNS event will be triggered with correct data for all 'CREATE' events where NHS is not null @delete_cleanup_batch @vaccine_type_MENACWY @supplier_name_TPP @@ -234,6 +243,7 @@ Feature: Create the immunization event for a patient through batch file And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table will be populated with the correct data for 'created' event for records in batch file And The delta table will be populated with the correct data for all created records in batch file + And MNS event will be triggered with correct data for all 'CREATE' events where NHS is not null @vaccine_type_3IN1 @supplier_name_TPP @@ -289,3 +299,4 @@ Feature: Create the immunization event for a patient through batch file And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table will be populated with the correct data for 'created' event for records in batch file And The delta table will be populated with the correct data for all created records in batch file + And MNS event will be triggered with correct data for all 'CREATE' events where NHS is not null diff --git a/tests/e2e_automation/features/batchTests/delete_batch.feature b/tests/e2e_automation/features/batchTests/delete_batch.feature index ecc48ca20..36d150a55 100644 --- a/tests/e2e_automation/features/batchTests/delete_batch.feature +++ b/tests/e2e_automation/features/batchTests/delete_batch.feature @@ -1,51 +1,55 @@ @Delete_Batch_Feature @functional Feature: Create the immunization event for a patient through batch file and update the record from batch or Api calls -@smoke -@vaccine_type_BCG @supplier_name_TPP -Scenario: Delete immunization event for a patient through batch file - Given batch file is created for below data as full dataset and each record has a valid delete record in the same file - | patient_id | unique_id | - | Random | Valid_NhsNumber | - | InvalidInPDS | InvalidInPDS_NhsNumber| - | SFlag | SFlag_NhsNumber | - | Mod11_NHS | Mod11_NhSNumber | - | OldNHSNo | OldNHSNo | - When batch file is uploaded in s3 bucket - Then file will be moved to destination bucket and inf ack file will be created - And inf ack file has success status for processed batch file - And bus ack files will be created - And CSV bus ack will not have any entry of successfully processed records - And Json bus ack will only contain file metadata and no failure record entry - And Audit table will have correct status, queue name and record count for the processed batch file - And The imms event table will be populated with the correct data for 'deleted' event for records in batch file - And The delta table will be populated with the correct data for all created records in batch file - And The delta table will be populated with the correct data for all deleted records in batch file + @smoke + @vaccine_type_BCG @supplier_name_TPP + Scenario: Delete immunization event for a patient through batch file + Given batch file is created for below data as full dataset and each record has a valid delete record in the same file + | patient_id | unique_id | + | Random | Valid_NhsNumber | + | InvalidInPDS | InvalidInPDS_NhsNumber | + | SFlag | SFlag_NhsNumber | + | Mod11_NHS | Mod11_NhSNumber | + | NullNHS | NullNHS_NhsNumber | + | OldNHSNo | OldNHS_NhsNumber | + | NO_GP_NHS | NO_GP_NhsNumber | + When batch file is uploaded in s3 bucket + Then file will be moved to destination bucket and inf ack file will be created + And inf ack file has success status for processed batch file + And bus ack files will be created + And CSV bus ack will not have any entry of successfully processed records + And Json bus ack will only contain file metadata and no failure record entry + And Audit table will have correct status, queue name and record count for the processed batch file + And The imms event table will be populated with the correct data for 'deleted' event for records in batch file + And The delta table will be populated with the correct data for all created records in batch file + And The delta table will be populated with the correct data for all deleted records in batch file + And MNS event will be triggered with correct data for both events where NHS is not null -@vaccine_type_MENB @patient_id_Random @supplier_name_EMIS -Scenario: Verify that the API vaccination record will be successful deleted by batch file upload - Given I have created a valid vaccination record through API - And The delta and imms event table will be populated with the correct data for api created event - When An delete to above vaccination record is made through batch file upload - And batch file is uploaded in s3 bucket - Then file will be moved to destination bucket and inf ack file will be created - And inf ack file has success status for processed batch file - And bus ack files will be created - And CSV bus ack will not have any entry of successfully processed records - And Json bus ack will only contain file metadata and no failure record entry - And Audit table will have correct status, queue name and record count for the processed batch file - And The imms event table status will be updated to delete and no change to record detail - And The delta table will have delete entry with no change to record detail + @vaccine_type_MENB @patient_id_Random @supplier_name_EMIS + Scenario: Verify that the API vaccination record will be successful deleted by batch file upload + Given I have created a valid vaccination record through API + And The delta and imms event table will be populated with the correct data for api created event + When An delete to above vaccination record is made through batch file upload + And batch file is uploaded in s3 bucket + Then file will be moved to destination bucket and inf ack file will be created + And inf ack file has success status for processed batch file + And bus ack files will be created + And CSV bus ack will not have any entry of successfully processed records + And Json bus ack will only contain file metadata and no failure record entry + And Audit table will have correct status, queue name and record count for the processed batch file + And The imms event table status will be updated to delete and no change to record detail + And The delta table will have delete entry with no change to record detail + And MNS event will be triggered with correct data for all 'DELETE' events where NHS is not null -@vaccine_type_RSV @patient_id_Random @supplier_name_RAVS -Scenario: Verify that the API vaccination record will be successful deleted and batch file will successful with mandatory field missing - Given I have created a valid vaccination record through API - When Delete above vaccination record is made through batch file upload with mandatory field missing - And batch file is uploaded in s3 bucket - Then file will be moved to destination bucket and inf ack file will be created - And inf ack file has success status for processed batch file - And bus ack files will be created - And CSV bus ack will not have any entry of successfully processed records - And Json bus ack will only contain file metadata and no failure record entry - And The imms event table status will be updated to delete and no change to record detail - And The delta table will have delete entry with no change to record detail + @vaccine_type_RSV @patient_id_Random @supplier_name_RAVS + Scenario: Verify that the API vaccination record will be successful deleted and batch file will successful with mandatory field missing + Given I have created a valid vaccination record through API + When Delete above vaccination record is made through batch file upload with mandatory field missing + And batch file is uploaded in s3 bucket + Then file will be moved to destination bucket and inf ack file will be created + And inf ack file has success status for processed batch file + And bus ack files will be created + And CSV bus ack will not have any entry of successfully processed records + And Json bus ack will only contain file metadata and no failure record entry + And The imms event table status will be updated to delete and no change to record detail + And The delta table will have delete entry with no change to record detail diff --git a/tests/e2e_automation/features/batchTests/update_batch.feature b/tests/e2e_automation/features/batchTests/update_batch.feature index 9679736d6..aa2191ff3 100644 --- a/tests/e2e_automation/features/batchTests/update_batch.feature +++ b/tests/e2e_automation/features/batchTests/update_batch.feature @@ -10,7 +10,9 @@ Feature: Create the immunization event for a patient through batch file and upda | InvalidInPDS | InvalidInPDS_NhsNumber | | SFlag | SFlag_NhsNumber | | Mod11_NHS | Mod11_NhSNumber | - | OldNHSNo | OldNHSNo | + | NullNHS | NullNHS_NhsNumber | + | OldNHSNo | OldNHS_NhsNumber | + | NO_GP_NHS | NO_GP_NhsNumber | When batch file is uploaded in s3 bucket Then file will be moved to destination bucket and inf ack file will be created And inf ack file has success status for processed batch file @@ -21,6 +23,7 @@ Feature: Create the immunization event for a patient through batch file and upda And The imms event table will be populated with the correct data for 'updated' event for records in batch file And The delta table will be populated with the correct data for all created records in batch file And The delta table will be populated with the correct data for all updated records in batch file + And MNS event will be triggered with correct data for both events where NHS is not null @Delete_cleanUp @vaccine_type_ROTAVIRUS @patient_id_Random @supplier_name_EMIS Scenario: Verify that the API vaccination record will be successful updated by batch file upload @@ -36,6 +39,7 @@ Feature: Create the immunization event for a patient through batch file and upda And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table will be populated with the correct data for 'updated' event for records in batch file And The delta table will be populated with the correct data for all updated records in batch file + And MNS event will be triggered with correct data for all 'UPDATE' events where NHS is not null @Delete_cleanUp @vaccine_type_6IN1 @patient_id_Random @supplier_name_TPP Scenario: Verify that the batch vaccination record will be successful updated by API request @@ -51,8 +55,10 @@ Feature: Create the immunization event for a patient through batch file and upda And Audit table will have correct status, queue name and record count for the processed batch file And The imms event table will be populated with the correct data for 'created' event for records in batch file And The delta table will be populated with the correct data for all created records in batch file + And MNS event will be triggered with correct data for all 'CREATE' events where NHS is not null When Send a update for Immunization event created with vaccination detail being updated through API request Then Api request will be successful and tables will be updated correctly + And Api updated event will trigger MNS event with correct data @smoke @Delete_cleanUp @vaccine_type_RSV @patient_id_Random @supplier_name_RAVS diff --git a/tests/e2e_automation/features/conftest.py b/tests/e2e_automation/features/conftest.py index 98372333e..d20574c0e 100644 --- a/tests/e2e_automation/features/conftest.py +++ b/tests/e2e_automation/features/conftest.py @@ -17,7 +17,7 @@ from utilities.context import ScenarioContext from utilities.enums import SupplierNameWithODSCode from utilities.http_requests_session import http_requests_session -from utilities.sqs_message_halder import purge_all_queues +from utilities.sqs_message_halder import purge_all_queues # noqa: F403 from features.APITests.steps.common_steps import * # noqa: F403 @@ -75,7 +75,8 @@ def global_context(): s3_env = os.getenv("S3_env") aws_account_id = os.getenv("aws_account_id") - if s3_env and aws_account_id: + mns_validation_required = os.getenv("mns_validation_required", "false").strip().lower() == "true" + if s3_env and aws_account_id and mns_validation_required: purge_all_queues(s3_env, aws_account_id) @@ -115,6 +116,7 @@ def context(request, global_context, temp_apigee_apps: list[ApigeeApp] | None) - "sub_environment", "LOCAL_RUN_WITHOUT_S3_UPLOAD", "aws_account_id", + "mns_validation_required", ] for var in env_vars: setattr(ctx, var, os.getenv(var)) @@ -152,7 +154,10 @@ def pytest_bdd_after_scenario(request, feature, scenario): assert context.response.status_code == 204, ( f"Expected status code 204, but got {context.response.status_code}. Response: {get_response_body_for_display(context.response)}" ) - mns_event_will_be_triggered_with_correct_data_for_deleted_event(context) + if context.mns_validation_required.strip().lower() == "true": + mns_event_will_be_triggered_with_correct_data_for_deleted_event(context) + else: + print("MNS validation not required, skipping MNS event verification for deleted event.") else: print("Skipping delete: ImmsID is None") @@ -160,25 +165,25 @@ def pytest_bdd_after_scenario(request, feature, scenario): if "IMMS_ID" in context.vaccine_df.columns and context.vaccine_df["IMMS_ID"].notna().any(): get_tokens(context, context.supplier_name) - context.vaccine_df["IMMS_ID_CLEAN"] = ( - context.vaccine_df["IMMS_ID"].astype(str).str.replace("Immunization#", "", regex=False) - ) + df = context.vaccine_df.dropna(subset=["IMMS_ID"]).copy() + df["IMMS_ID_CLEAN"] = df["IMMS_ID"].astype(str).str.replace("Immunization#", "", regex=False) - for imms_id in context.vaccine_df["IMMS_ID_CLEAN"].dropna().unique(): + for row in df.itertuples(index=False): + imms_id = row.IMMS_ID_CLEAN delete_url = f"{context.url}/{imms_id}" - print(f"Sending DELETE request to: {delete_url}") + print(f"Sending DELETE request to: {delete_url}") response = http_requests_session.delete(delete_url, headers=context.headers) if response.status_code != 204: print( - f"Cleanup DELETE returned {response.status_code} for {imms_id} (teardown best-effort, not failing test). Response: {get_response_body_for_display(response)}" + f"Cleanup DELETE returned {response.status_code} for {imms_id} " + f"(teardown best-effort, not failing test). " + f"Response: {get_response_body_for_display(response)}" ) else: print(f"Deleted {imms_id} successfully.") - print("Batch cleanup finished.") + else: - print( - " No IMMS_ID column or no values present as test failed due to as exception — skipping delete cleanup." - ) + print("No IMMS_ID values available — skipping delete cleanup.") diff --git a/tests/e2e_automation/input/testData.csv b/tests/e2e_automation/input/testData.csv index 43b1b54ce..964a55032 100644 --- a/tests/e2e_automation/input/testData.csv +++ b/tests/e2e_automation/input/testData.csv @@ -6,7 +6,7 @@ SFlag,9449310475,test1,test2,unknown,1980-11-01,Validate Obf,"1, obf_2",obf_3,ob InvalidInPDS,7085531614,test1,test2,unknown,1980-11-01,Validate Obf,"1, obf_2",obf_3,obf_4,obf_5,LS01 1AB,obf_7,2000-01-01,2025-01-01,,, InvalidInPDS,1542293278,test1,test2,unknown,1980-11-01,Validate Obf,"1, obf_2",obf_3,obf_4,obf_5,LS01 1AB,obf_7,2000-01-01,2025-01-01,,, InvalidMOD11Check,1234567890,test1,test2,unknown,1980-11-01,Validate Obf,"1, obf_2",obf_3,obf_4,obf_5,LS01 1AB,obf_7,2000-01-01,2025-01-01,,, -OldNHSNo,9452372230,test1,test2,unknown,1980-11-01,Validate Obf,"1, obf_2",obf_3,obf_4,obf_5,LS01 1AB,obf_7,2000-01-01,2025-01-01,,, +OldNHSNo,9452372230,test1,test2,unknown,1980-11-01,Validate Obf,"1, obf_2",obf_3,obf_4,obf_5,LS01 1AB,obf_7,2000-01-01,2025-01-01,E82045,, SupersedeNhsNo,9467351307,test1,test2,unknown,1980-11-01,Validate Obf,"1, obf_2",obf_3,obf_4,obf_5,LS01 1AB,obf_7,2000-01-01,2025-01-01,,, Invalid_NHS,9461267665,STERLING,SOKHI,unknown,2007-11-25,217A,ROUNDHAY ROAD,City,district,State,LS8 4HS,UK,2013-12-22,2033-01-01,B83013,, Mod11_NHS,2788584652,STERLING,Sal,unknown,2007-11-25,217A,ROUNDHAY ROAD,City,district,State,LS8 4HS,UK,2013-12-22,2033-01-01,,, diff --git a/tests/e2e_automation/src/objectModels/batch/batch_file_builder.py b/tests/e2e_automation/src/objectModels/batch/batch_file_builder.py index 511a4168e..f6819bc21 100644 --- a/tests/e2e_automation/src/objectModels/batch/batch_file_builder.py +++ b/tests/e2e_automation/src/objectModels/batch/batch_file_builder.py @@ -8,7 +8,12 @@ from src.objectModels.patient_loader import load_patient_by_id from utilities.date_helper import generate_date from utilities.enums import GenderCode -from utilities.vaccination_constants import ROUTE_MAP, SITE_MAP, VACCINATION_PROCEDURE_MAP, VACCINE_CODE_MAP +from utilities.vaccination_constants import ( + ROUTE_MAP, + SITE_MAP, + VACCINATION_PROCEDURE_MAP, + VACCINE_CODE_MAP, +) def build_procedure_code(vaccine_type: str) -> dict[str, str]: @@ -45,21 +50,33 @@ def get_batch_date(date_str: str = "current_occurrence") -> str: def get_performing_professional(forename: str = "Automation", surname: str = "Tests") -> dict[str, str]: - return {"performing_professional_forename": forename, "performing_professional_surname": surname} + return { + "performing_professional_forename": forename, + "performing_professional_surname": surname, + } def build_site_of_vaccination() -> dict[str, str]: selected = random.choice(SITE_MAP) - return {"site_of_vaccination_code": selected["code"], "site_of_vaccination_term": selected["display"]} + return { + "site_of_vaccination_code": selected["code"], + "site_of_vaccination_term": selected["display"], + } def build_route_of_vaccination() -> dict[str, str]: selected = random.choice(ROUTE_MAP) - return {"route_of_vaccination_code": selected["code"], "route_of_vaccination_term": selected["display"]} + return { + "route_of_vaccination_code": selected["code"], + "route_of_vaccination_term": selected["display"], + } def build_dose_details( - dose_sequence: str = "1", dose_amount: str = "0.5", dose_unit_code: str = "ml", dose_unit_term: str = "millilitre" + dose_sequence: str = "1", + dose_amount: str = "0.5", + dose_unit_code: str = "ml", + dose_unit_term: str = "millilitre", ) -> dict[str, str]: return { "dose_sequence": dose_sequence, @@ -71,7 +88,10 @@ def build_dose_details( def build_unique_reference(unique_id: str | None = None) -> dict[str, str]: uid = unique_id or str(uuid.uuid4()) - return {"unique_id": uid, "unique_id_uri": "https://fhir.nhs.uk/Id/Automation-vaccine-administered-event-uk"} + return { + "unique_id": uid, + "unique_id_uri": "https://fhir.nhs.uk/Id/Automation-vaccine-administered-event-uk", + } def get_patient_details(context) -> dict[str, str]: @@ -79,7 +99,7 @@ def get_patient_details(context) -> dict[str, str]: return { "first_name": patient.name[0].given[0], "last_name": patient.name[0].family, - "nhs_number": patient.identifier[0].value, + "nhs_number": (patient.identifier[0].value if patient.identifier[0].value is not None else ""), "gender": GenderCode[patient.gender].value, "birth_date": patient.birthDate.replace("-", ""), "postal_code": patient.address[0].postalCode, diff --git a/tests/e2e_automation/src/objectModels/mns_event/msn_event.py b/tests/e2e_automation/src/objectModels/mns_event/msn_event.py index f9ab1450e..57f6a43b9 100644 --- a/tests/e2e_automation/src/objectModels/mns_event/msn_event.py +++ b/tests/e2e_automation/src/objectModels/mns_event/msn_event.py @@ -18,4 +18,4 @@ class MnsEvent(BaseModel): time: str subject: str dataref: str - filtering: Filtering + filtering: Filtering | None = None diff --git a/tests/e2e_automation/utilities/context.py b/tests/e2e_automation/utilities/context.py index 2ee7202e1..790135e64 100644 --- a/tests/e2e_automation/utilities/context.py +++ b/tests/e2e_automation/utilities/context.py @@ -49,3 +49,4 @@ def __init__(self): self.delta_cache = None self.aws_account_id = None self.gp_code = None + self.mns_validation_required = False diff --git a/tests/e2e_automation/utilities/sqs_message_halder.py b/tests/e2e_automation/utilities/sqs_message_halder.py index e82743d40..9d53616d9 100644 --- a/tests/e2e_automation/utilities/sqs_message_halder.py +++ b/tests/e2e_automation/utilities/sqs_message_halder.py @@ -1,4 +1,5 @@ import json +import time import boto3 from botocore.exceptions import ClientError @@ -8,10 +9,15 @@ "notification": "{env}-mns-test-notification-queue", "dead_letter": "{env}-mns-outbound-events-dead-letter-queue", "outbound": "{env}-mns-outbound-events-queue", + "int_notification": "imms-int-publisher-subscribe-test", + "int_dead_letter": "imms-int-publisher-subscribe-test-dlq", } def build_queue_url(env, aws_account_id, queue_type: str) -> str: + if env == "preprod" and queue_type in ["notification", "dead_letter"]: + queue_type = f"int_{queue_type}" + if queue_type not in QUEUE_TEMPLATES: raise ValueError(f"Invalid queue_type: {queue_type}") @@ -23,20 +29,25 @@ def build_queue_url(env, aws_account_id, queue_type: str) -> str: def read_message( context, queue_type="notification", - action="CREATE", - wait_for_message=True, - max_empty_polls=3, wait_time_seconds=20, + max_total_wait_seconds=120, ): sqs = boto3.client("sqs", region_name="eu-west-2") queue_url = build_queue_url(context.S3_env, context.aws_account_id, queue_type) expected_dataref = f"{context.url}/{context.ImmsID}" - empty_polls = 0 + start_time = time.time() + + print(f"Waiting for message with dataref: {expected_dataref}") while True: - print(f"Polling {queue_type} queue for {action} messages (wait {wait_time_seconds}s)...") + elapsed = time.time() - start_time + if elapsed > max_total_wait_seconds: + print("Stopping — reached max wait time.") + return None + + print(f"Polling {queue_type} queue (wait {wait_time_seconds}s)...") response = sqs.receive_message( QueueUrl=queue_url, @@ -48,33 +59,31 @@ def read_message( messages = response.get("Messages", []) if not messages: - empty_polls += 1 - print(f"No messages returned for {action} (empty poll {empty_polls}/{max_empty_polls})") - - if not wait_for_message or empty_polls >= max_empty_polls: - print("Stopping — queue quiet or wait disabled.") - return None - + print("No messages returned — continuing to poll...") continue - empty_polls = 0 - for msg in messages: body = MnsEvent(**json.loads(msg["Body"])) + dataref = body.dataref - if body.dataref == expected_dataref and body.filtering.action == action.upper(): + if dataref == expected_dataref: sqs.delete_message(QueueUrl=queue_url, ReceiptHandle=msg["ReceiptHandle"]) - print(f"Deleted {action.upper()} message from {queue_type} queue") + print(f"Matched and deleted message for {dataref}") return body sqs.delete_message(QueueUrl=queue_url, ReceiptHandle=msg["ReceiptHandle"]) - print(f"Deleted non-matching message from {queue_type} queue") + print(f"Deleted non-matching message: {dataref}") def purge_all_queues(env, aws_account_id): sqs = boto3.client("sqs", region_name="eu-west-2") - for queue_type in QUEUE_TEMPLATES.keys(): + if env == "preprod": + queue_types = ["notification", "dead_letter"] # will map to int_* + else: + queue_types = ["notification", "dead_letter", "outbound"] + + for queue_type in queue_types: queue_url = build_queue_url(env, aws_account_id, queue_type) print(f"Purging {queue_type} queue: {queue_url}") @@ -86,3 +95,57 @@ def purge_all_queues(env, aws_account_id): print(f"{queue_type.replace('_', ' ').title()} queue purge already in progress, skipping...\n") else: print(f"Error purging {queue_type} queue: {e}\n") + + +def read_messages_for_batch( + context, + queue_type="notification", + valid_rows=None, + wait_time_seconds=20, + max_total_wait_seconds=180, + expected_count=0, +): + sqs = boto3.client("sqs", region_name="eu-west-2") + queue_url = build_queue_url(context.S3_env, context.aws_account_id, queue_type) + + context.url = context.baseUrl + "/Immunization" + + expected_datarefs = {f"{context.url}/{str(row.IMMS_ID_CLEAN)}" for row in valid_rows} + + matched_messages = [] + start_time = time.time() + + print(f"Expecting {expected_count} messages for {len(valid_rows)} NHS numbers") + + while len(matched_messages) < expected_count: + elapsed = time.time() - start_time + if elapsed > max_total_wait_seconds: + print("Stopping — reached max wait time.") + break + + print(f"Polling SQS ({queue_type})...") + + response = sqs.receive_message( + QueueUrl=queue_url, + MaxNumberOfMessages=10, + WaitTimeSeconds=wait_time_seconds, + VisibilityTimeout=30, + ) + + messages = response.get("Messages", []) + + if not messages: + print("No messages returned — continuing to poll...") + continue + + for msg in messages: + body = MnsEvent(**json.loads(msg["Body"])) + dataref = body.dataref + + if dataref in expected_datarefs: + matched_messages.append(body) + print(f"Matched: {dataref}") + + sqs.delete_message(QueueUrl=queue_url, ReceiptHandle=msg["ReceiptHandle"]) + + return matched_messages