diff --git a/.github/workflows/shared-run-e2e-tests.yaml b/.github/workflows/shared-run-e2e-tests.yaml index 13d70757..459647dc 100644 --- a/.github/workflows/shared-run-e2e-tests.yaml +++ b/.github/workflows/shared-run-e2e-tests.yaml @@ -94,6 +94,9 @@ env: E2E_EUID_PROD_AWS_ARGS_JSON: ${{ secrets.E2E_EUID_PROD_AWS_ARGS_JSON }} E2E_EUID_PROD_AWS_OPERATOR_API_KEY: ${{ secrets.E2E_EUID_PROD_AWS_OPERATOR_API_KEY }} + E2E_UID2_INTEG_AKS_OPERATOR_KEY: ${{ secrets.E2E_UID2_INTEG_AKS_OPERATOR_KEY }} + E2E_UID2_PROD_AKS_OPERATOR_KEY: ${{ secrets.E2E_UID2_PROD_AKS_OPERATOR_KEY }} + jobs: e2e-test: name: E2E Test @@ -201,6 +204,37 @@ jobs: target_environment: ${{ inputs.target_environment }} aws_pcr0: ${{ inputs.aws_pcr0 }} + - name: Start AKS cluster + id: start_aks_cluster + if: ${{ inputs.operator_type == 'aks' }} + uses: IABTechLab/uid2-shared-actions/actions/start_aks_cluster@v3 + with: + azure_credentials: ${{ secrets.AZURE_CREDENTIALS }} + run_id: ${{ github.run_id }} + + - name: Get AKS operator key + id: get_aks_operator_key + if: ${{ inputs.operator_type == 'aks' }} + shell: bash + env: + IDENTITY_SCOPE: ${{ inputs.identity_scope }} + TARGET_ENVIRONMENT: ${{ inputs.target_environment }} + ENCLAVE_PROTOCOL: azure-cc + run: | + bash uid2-shared-actions/scripts/get_operator_key.sh + + - name: Prepare AKS metadata + id: prepare_aks_metadata + if: ${{ inputs.operator_type == 'aks' }} + uses: IABTechLab/uid2-shared-actions/actions/prepare_aks_metadata@v3 + with: + operator_image_version: ${{ inputs.operator_image_version }} + target_environment: ${{ inputs.target_environment }} + bore_url_core: ${{ steps.bore.outputs.bore_url_core }} + bore_url_optout: ${{ steps.bore.outputs.bore_url_optout }} + operator_key: ${{ steps.get_aks_operator_key.outputs.OPERATOR_KEY }} + run_id: ${{ github.run_id }} + - name: Bring up Docker Compose id: docker-compose if: ${{ inputs.target_environment == 'mock' }} @@ -255,6 +289,15 @@ jobs: target_environment: ${{ inputs.target_environment }} operator_key: ${{ steps.prepare_aws_metadata.outputs.operator_key }} + - name: Start AKS private operator + id: start_aks_private_operator + if: ${{ inputs.operator_type == 'aks' }} + uses: IABTechLab/uid2-shared-actions/actions/start_aks_private_operator@v3 + with: + template_file: ${{ steps.prepare_aks_metadata.outputs.template_file }} + azure_credentials: ${{ secrets.AZURE_CREDENTIALS }} + run_id: ${{ github.run_id }} + - name: Decide E2E test environment variables id: decide_env_var shell: bash @@ -315,6 +358,8 @@ jobs: - name: Stop AKS operator if: ${{ always() && !inputs.delay_operator_shutdown && inputs.operator_type == 'aks' }} + env: + RUN_ID: ${{ github.run_id }} run: | bash uid2-shared-actions/scripts/aks/stop_aks_enclave.sh @@ -358,3 +403,10 @@ jobs: with: aws_stack_name: ${{ needs.e2e-test.outputs.aws_stack_name }} aws_region: ${{ inputs.aws_region }} + + - name: Stop AKS private operator + if: ${{ inputs.operator_type == 'aks' }} + uses: IABTechLab/uid2-shared-actions/actions/stop_aks_private_operator@v3 + with: + azure_credentials: ${{ secrets.AZURE_CREDENTIALS }} + run_id: ${{ github.run_id }} diff --git a/actions/prepare_aks_metadata/action.yaml b/actions/prepare_aks_metadata/action.yaml index d169eefc..2be47577 100644 --- a/actions/prepare_aks_metadata/action.yaml +++ b/actions/prepare_aks_metadata/action.yaml @@ -14,6 +14,12 @@ inputs: bore_url_optout: description: The bore URL for optout service required: true + operator_key: + description: The operator key secret for the target environment + required: true + run_id: + description: Unique run identifier to avoid Azure resource conflicts + required: true outputs: template_file: description: The template file @@ -36,6 +42,8 @@ runs: BORE_URL_OPTOUT: ${{ inputs.bore_url_optout }} IMAGE_VERSION: ${{ inputs.operator_image_version }} TARGET_ENVIRONMENT: ${{ inputs.target_environment }} + OPERATOR_KEY: ${{ inputs.operator_key }} + RUN_ID: ${{ inputs.run_id }} run: | bash uid2-shared-actions/scripts/aks/prepare_aks_artifacts.sh diff --git a/actions/prepare_aws_metadata/action.yaml b/actions/prepare_aws_metadata/action.yaml index 3dd20c5c..ad86820c 100644 --- a/actions/prepare_aws_metadata/action.yaml +++ b/actions/prepare_aws_metadata/action.yaml @@ -39,4 +39,4 @@ runs: TARGET_ENVIRONMENT: ${{ inputs.target_environment }} ENCLAVE_PROTOCOL: aws-nitro run: | - bash ./uid2-shared-actions/scripts/get_operator_key.sh + bash uid2-shared-actions/scripts/get_operator_key.sh diff --git a/actions/prepare_gcp_metadata/action.yaml b/actions/prepare_gcp_metadata/action.yaml index fb950237..ee4d762b 100644 --- a/actions/prepare_gcp_metadata/action.yaml +++ b/actions/prepare_gcp_metadata/action.yaml @@ -89,4 +89,4 @@ runs: TARGET_ENVIRONMENT: ${{ inputs.target_environment }} ENCLAVE_PROTOCOL: gcp-oidc run: | - bash ./uid2-shared-actions/scripts/get_operator_key.sh + bash uid2-shared-actions/scripts/get_operator_key.sh diff --git a/actions/start_aks_cluster/action.yaml b/actions/start_aks_cluster/action.yaml new file mode 100644 index 00000000..2925b170 --- /dev/null +++ b/actions/start_aks_cluster/action.yaml @@ -0,0 +1,28 @@ +name: Start AKS Cluster +description: Creates the AKS cluster and infrastructure + +inputs: + azure_credentials: + description: The Azure credentials + required: true + run_id: + description: Unique run identifier for resource naming + required: true + +runs: + using: "composite" + + steps: + - name: Log in to Azure + uses: azure/login@v2 + with: + creds: ${{ inputs.azure_credentials }} + enable-AzPSSession: true + + - name: Start AKS cluster + id: start_aks + shell: bash + env: + RUN_ID: ${{ inputs.run_id }} + run: | + bash uid2-shared-actions/scripts/aks/start_aks_cluster.sh diff --git a/actions/start_aks_private_operator/action.yaml b/actions/start_aks_private_operator/action.yaml index 10b122ae..313a848b 100644 --- a/actions/start_aks_private_operator/action.yaml +++ b/actions/start_aks_private_operator/action.yaml @@ -8,6 +8,9 @@ inputs: azure_credentials: description: The Azure credentials required: true + run_id: + description: Unique run identifier for resource naming + required: true outputs: uid2_pipeline_e2e_operator_url: @@ -29,5 +32,6 @@ runs: shell: bash env: TEMPLATE_FILE: ${{ inputs.template_file }} + RUN_ID: ${{ inputs.run_id }} run: | bash uid2-shared-actions/scripts/aks/start_aks_enclave.sh diff --git a/actions/stop_aks_private_operator/action.yaml b/actions/stop_aks_private_operator/action.yaml index a41dcaae..dc4ee399 100644 --- a/actions/stop_aks_private_operator/action.yaml +++ b/actions/stop_aks_private_operator/action.yaml @@ -5,6 +5,9 @@ inputs: azure_credentials: description: The Azure credentials required: true + run_id: + description: Unique run identifier for resource naming + required: true runs: using: "composite" @@ -18,5 +21,7 @@ runs: - name: Stop AKS private operator shell: bash + env: + RUN_ID: ${{ inputs.run_id }} run: | bash uid2-shared-actions/scripts/aks/stop_aks_enclave.sh diff --git a/scripts/aks/aks_env.sh b/scripts/aks/aks_env.sh new file mode 100644 index 00000000..eb1f7a5f --- /dev/null +++ b/scripts/aks/aks_env.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Common AKS environment variables used by multiple scripts + +# Set the correct subscription for AKS E2E tests +az account set --subscription "63e97a70-d825-4b08-af6d-c0d8ad98bed3" + +# RUN_ID should be set by the caller (e.g., github.run_id) +# Use short suffix to stay within Azure naming limits (e.g., Key Vault max 24 chars) +if [ -z "${RUN_ID}" ]; then + echo "Warning: RUN_ID not set, using default names (may cause conflicts)" + RUN_SUFFIX="" +else + # Use last 8 digits of RUN_ID to keep names short + RUN_SUFFIX="-${RUN_ID: -8}" +fi + +export RESOURCE_GROUP="opr-e2e-aks${RUN_SUFFIX}" +export LOCATION="westus" +export VNET_NAME="opr-e2e-vnet${RUN_SUFFIX}" +export PUBLIC_IP_ADDRESS_NAME="opr-e2e-ip${RUN_SUFFIX}" +export NAT_GATEWAY_NAME="opr-e2e-nat${RUN_SUFFIX}" +export AKS_CLUSTER_NAME="opr-e2e-cluster${RUN_SUFFIX}" +export KEYVAULT_NAME="opre2evault${RUN_SUFFIX}" +export KEYVAULT_SECRET_NAME="opr-key${RUN_SUFFIX}" +export MANAGED_IDENTITY="opr-e2e-id${RUN_SUFFIX}" +export AKS_NODE_RESOURCE_GROUP="MC_${RESOURCE_GROUP}_${AKS_CLUSTER_NAME}_${LOCATION}" +export SUBSCRIPTION_ID="$(az account show --query id --output tsv)" +export DEPLOYMENT_ENV="integ" \ No newline at end of file diff --git a/scripts/aks/prepare_aks_artifacts.sh b/scripts/aks/prepare_aks_artifacts.sh index 592b16d0..8635c9b3 100644 --- a/scripts/aks/prepare_aks_artifacts.sh +++ b/scripts/aks/prepare_aks_artifacts.sh @@ -21,31 +21,58 @@ if [ -z "${TARGET_ENVIRONMENT}" ]; then exit 1 fi -# Below resources should be prepared ahead of running the E2E test. +if [ -z "${OPERATOR_KEY}" ]; then + echo "OPERATOR_KEY can not be empty" + exit 1 +fi + +if [ -z "${RUN_ID}" ]; then + echo "RUN_ID can not be empty" + exit 1 +fi + # See https://github.com/UnifiedID2/aks-demo/tree/master/vn-aks#setup-aks--node-pool -export RESOURCE_GROUP="pipeline-vn-aks" -export LOCATION="eastus" -export VNET_NAME="pipeline-vnet" -export PUBLIC_IP_ADDRESS_NAME="pipeline-public-ip" -export NAT_GATEWAY_NAME="pipeline-nat-gateway" -export AKS_CLUSTER_NAME="pipelinevncluster" -export KEYVAULT_NAME="pipeline-vn-aks-vault" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/aks_env.sh" + if [ ${TARGET_ENVIRONMENT} == "mock" ]; then - export KEYVAULT_SECRET_NAME="pipeline-vn-aks-opr-key-name" + export KEYVAULT_SECRET_NAME="opr-e2e-vn-aks-opr-key-name" elif [ ${TARGET_ENVIRONMENT} == "integ" ]; then - KEYVAULT_SECRET_NAME="pipeline-vn-aks-opr-key-name-integ" + export KEYVAULT_SECRET_NAME="opr-e2e-vn-aks-opr-key-name-integ" elif [ ${TARGET_ENVIRONMENT} == "prod" ]; then - KEYVAULT_SECRET_NAME="pipeline-vn-aks-opr-key-name-prod" + export KEYVAULT_SECRET_NAME="opr-e2e-vn-aks-opr-key-name-prod" else echo "Arguments not supported: TARGET_ENVIRONMENT=${TARGET_ENVIRONMENT}" exit 1 fi -export MANAGED_IDENTITY="pipeline-vn-aks-opr-id" -export AKS_NODE_RESOURCE_GROUP="MC_${RESOURCE_GROUP}_${AKS_CLUSTER_NAME}_${LOCATION}" -export SUBSCRIPTION_ID="$(az account show --query id --output tsv)" -export DEPLOYMENT_ENV="integ" -export MANAGED_IDENTITY_ID="/subscriptions/001a3882-eb1c-42ac-9edc-5e2872a07783/resourcegroups/pipeline-vn-aks/providers/Microsoft.ManagedIdentity/userAssignedIdentities/pipeline-vn-aks-opr-id" +# --- Create Key Vault & Managed Identity --- +# Login to AKS cluster +az aks get-credentials --name ${AKS_CLUSTER_NAME} --resource-group ${RESOURCE_GROUP} +# Create managed identity +az identity create --name "${MANAGED_IDENTITY}" --resource-group "${RESOURCE_GROUP}" --location "${LOCATION}" +# Create key vault with purge protection and RBAC authorization +# Check if vault exists in deleted state and recover it, otherwise create new +if az keyvault show-deleted --name "${KEYVAULT_NAME}" &>/dev/null; then + echo "Key vault '${KEYVAULT_NAME}' exists in deleted state, recovering..." + az keyvault recover --name "${KEYVAULT_NAME}" +elif az keyvault show --name "${KEYVAULT_NAME}" &>/dev/null; then + echo "Key vault '${KEYVAULT_NAME}' already exists." +else + echo "Creating key vault '${KEYVAULT_NAME}'..." + az keyvault create --name "${KEYVAULT_NAME}" --resource-group "${RESOURCE_GROUP}" --location "${LOCATION}" --enable-purge-protection --enable-rbac-authorization +fi +# Get keyvault resource ID +export KEYVAULT_RESOURCE_ID="$(az keyvault show --resource-group "${RESOURCE_GROUP}" --name "${KEYVAULT_NAME}" --query id --output tsv)" +# Set keyvault secret +az keyvault secret set --vault-name "${KEYVAULT_NAME}" --name "${KEYVAULT_SECRET_NAME}" --value "${OPERATOR_KEY}" +# Get identity principal ID +export IDENTITY_PRINCIPAL_ID="$(az identity show --name "${MANAGED_IDENTITY}" --resource-group "${RESOURCE_GROUP}" --query principalId --output tsv)" +# Create role assignment for Key Vault Secrets User +az role assignment create --assignee-object-id "${IDENTITY_PRINCIPAL_ID}" --role "Key Vault Secrets User" --scope "${KEYVAULT_RESOURCE_ID}" --assignee-principal-type ServicePrincipal + +# Get managed identity ID +export MANAGED_IDENTITY_ID="$(az identity show --name "${MANAGED_IDENTITY}" --resource-group "${RESOURCE_GROUP}" --query id --output tsv)" OPERATOR_ROOT="./uid2-operator" SHARED_ACTIONS_ROOT="./uid2-shared-actions" @@ -87,11 +114,12 @@ else sed -i "s#VAULT_NAME_PLACEHOLDER#${KEYVAULT_NAME}#g" "${OUTPUT_TEMPLATE_FILE}" sed -i "s#OPERATOR_KEY_SECRET_NAME_PLACEHOLDER#${KEYVAULT_SECRET_NAME}#g" "${OUTPUT_TEMPLATE_FILE}" sed -i "s#DEPLOYMENT_ENVIRONMENT_PLACEHOLDER#integ#g" "${OUTPUT_TEMPLATE_FILE}" + # Make deployment name unique per run to avoid Azure resource conflicts + sed -i "s#operator-deployment#operator-deployment-${RUN_ID}#g" "${OUTPUT_TEMPLATE_FILE}" cat ${OUTPUT_TEMPLATE_FILE} - if [ ${TARGET_ENVIRONMENT} == "mock" ]; then - python3 ${SHARED_ACTIONS_ROOT}/scripts/aks/add_env.py ${OUTPUT_TEMPLATE_FILE} uid2-operator CORE_BASE_URL ${BORE_URL_CORE} OPTOUT_BASE_URL ${BORE_URL_OPTOUT} SKIP_VALIDATIONS true - fi + # Add bore URLs for connecting to mock core/optout services (used in all E2E test environments) + python3 ${SHARED_ACTIONS_ROOT}/scripts/aks/add_env.py ${OUTPUT_TEMPLATE_FILE} uid2-operator CORE_BASE_URL ${BORE_URL_CORE} OPTOUT_BASE_URL ${BORE_URL_OPTOUT} SKIP_VALIDATIONS true cat ${OUTPUT_TEMPLATE_FILE} # --- Finished updating yaml file with resources --- diff --git a/scripts/aks/start_aks_cluster.sh b/scripts/aks/start_aks_cluster.sh new file mode 100644 index 00000000..d8c39a63 --- /dev/null +++ b/scripts/aks/start_aks_cluster.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +set -ex + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/aks_env.sh" + +# Setup AKS & Node Pool +az group create --name "${RESOURCE_GROUP}" --location "${LOCATION}" + +az network vnet create \ + --resource-group ${RESOURCE_GROUP} \ + --name ${VNET_NAME} \ + --location ${LOCATION} \ + --address-prefixes 10.0.0.0/8 + +# Default Subnet (10.0.0.0/24) +az network vnet subnet create \ + --resource-group ${RESOURCE_GROUP} \ + --vnet-name ${VNET_NAME} \ + --name default \ + --address-prefixes 10.0.0.0/24 + +# AKS Subnet (CIDR /16) +az network vnet subnet create \ + --resource-group ${RESOURCE_GROUP} \ + --vnet-name ${VNET_NAME} \ + --name aks \ + --address-prefixes 10.1.0.0/16 + +# Container Groups Subnet (CIDR /16) with Delegation +az network vnet subnet create \ + --resource-group ${RESOURCE_GROUP} \ + --vnet-name ${VNET_NAME} \ + --name cg \ + --address-prefixes 10.2.0.0/16 \ + --delegations Microsoft.ContainerInstance/containerGroups + +az network public-ip create --name ${PUBLIC_IP_ADDRESS_NAME} --resource-group ${RESOURCE_GROUP} --sku standard --allocation static + +az network nat gateway create \ + --resource-group ${RESOURCE_GROUP} \ + --name ${NAT_GATEWAY_NAME} \ + --public-ip-addresses ${PUBLIC_IP_ADDRESS_NAME} \ + --idle-timeout 4 + +az network vnet subnet update \ + --resource-group ${RESOURCE_GROUP} \ + --vnet-name ${VNET_NAME} \ + --name cg \ + --nat-gateway ${NAT_GATEWAY_NAME} + +export AKS_SUBNET_ID=$(az network vnet subnet show \ + --resource-group ${RESOURCE_GROUP} \ + --vnet-name ${VNET_NAME} \ + --name aks \ + --query id \ + --output tsv) + +# Create the AKS cluster if it doesn't exist +if az aks show --resource-group ${RESOURCE_GROUP} --name ${AKS_CLUSTER_NAME} &>/dev/null; then + echo "AKS cluster '${AKS_CLUSTER_NAME}' already exists, skipping creation." +else + echo "Creating AKS cluster '${AKS_CLUSTER_NAME}'..." + az aks create \ + --resource-group ${RESOURCE_GROUP} \ + --name ${AKS_CLUSTER_NAME} \ + --location ${LOCATION} \ + --kubernetes-version 1.33 \ + --network-plugin azure \ + --network-policy calico \ + --vnet-subnet-id ${AKS_SUBNET_ID} \ + --service-cidr 10.4.0.0/16 \ + --dns-service-ip 10.4.0.10 \ + --node-vm-size Standard_D4d_v5 \ + --node-count 2 \ + --enable-cluster-autoscaler \ + --min-count 2 \ + --max-count 5 \ + --auto-upgrade-channel patch \ + --enable-managed-identity \ + --nodepool-name oprnodepool \ + --os-sku Ubuntu +fi + +# Get the managed identity object ID for role assignments +export MANAGED_IDENTITY_OBJECT_ID="$(az aks show --resource-group ${RESOURCE_GROUP} --name ${AKS_CLUSTER_NAME} --query "identityProfile.kubeletidentity.objectId" --output tsv)" + +# Wait for managed identity to be available in AAD and create role assignments +echo "Waiting for managed identity to be available in AAD..." +until az role assignment create \ + --assignee-object-id ${MANAGED_IDENTITY_OBJECT_ID} \ + --assignee-principal-type ServicePrincipal \ + --scope /subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${AKS_NODE_RESOURCE_GROUP} \ + --role Contributor 2>/dev/null; do + echo "Managed identity not yet available, waiting 10 seconds..." + sleep 10 +done +echo "First role assignment created successfully." + +az role assignment create \ + --assignee-object-id ${MANAGED_IDENTITY_OBJECT_ID} \ + --assignee-principal-type ServicePrincipal \ + --scope /subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP} \ + --role Contributor + +# Setup AKS Cluster +az aks get-credentials --name ${AKS_CLUSTER_NAME} --resource-group ${RESOURCE_GROUP} +az provider register -n Microsoft.ContainerInstance +git clone https://github.com/microsoft/virtualnodesOnAzureContainerInstances.git +helm install virtualnode virtualnodesOnAzureContainerInstances/Helm/virtualnode +# Wait for virtualnode-0 to appear +echo "Waiting for virtualnode-0 to be ready..." +while ! kubectl get nodes | grep -q "virtualnode-0"; do + echo "virtualnode-0 not found yet, waiting 10 seconds..." + sleep 10 +done +echo "virtualnode-0 is ready!" +kubectl get nodes \ No newline at end of file diff --git a/scripts/aks/start_aks_enclave.sh b/scripts/aks/start_aks_enclave.sh index 2834891f..6d4673f6 100644 --- a/scripts/aks/start_aks_enclave.sh +++ b/scripts/aks/start_aks_enclave.sh @@ -6,23 +6,10 @@ if [[ ! -f ${TEMPLATE_FILE} ]]; then exit 1 fi -ROOT="./uid2-shared-actions/scripts" - -# below resources should be prepared ahead -export RESOURCE_GROUP="pipeline-vn-aks" -export LOCATION="eastus" -export VNET_NAME="pipeline-vnet" -export PUBLIC_IP_ADDRESS_NAME="pipeline-public-ip" -export NAT_GATEWAY_NAME="pipeline-nat-gateway" -export AKS_CLUSTER_NAME="pipelinevncluster" -export KEYVAULT_NAME="pipeline-vn-aks-vault" -export KEYVAULT_SECRET_NAME="pipeline-vn-aks-opr-key-name" -export MANAGED_IDENTITY="pipeline-vn-aks-opr-id" -export AKS_NODE_RESOURCE_GROUP="MC_${RESOURCE_GROUP}_${AKS_CLUSTER_NAME}_${LOCATION}" -export SUBSCRIPTION_ID="$(az account show --query id --output tsv)" -export DEPLOYMENT_ENV="integ" - -source "${ROOT}/healthcheck.sh" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${SCRIPT_DIR}/aks_env.sh" +source "${SCRIPT_DIR}/../healthcheck.sh" # --- Deploy operator service and make sure it starts --- az aks get-credentials --name ${AKS_CLUSTER_NAME} --resource-group ${RESOURCE_GROUP} @@ -33,8 +20,24 @@ if [ -z "${GITHUB_OUTPUT}" ]; then exit 1 fi -# Get public IP, need to trim quotes -IP=$(az network public-ip list --resource-group ${AKS_NODE_RESOURCE_GROUP} --query "[?starts_with(name, 'kubernetes')].ipAddress" --output tsv) +# Wait for public IP to be assigned (LoadBalancer provisioning can take time) +echo "Waiting for public IP to be assigned..." +for i in {1..30}; do + IP=$(az network public-ip list --resource-group ${AKS_NODE_RESOURCE_GROUP} --query "[?starts_with(name, 'kubernetes')].ipAddress" --output tsv) + if [ -n "${IP}" ]; then + echo "Public IP found: ${IP}" + break + fi + echo "Attempt ${i}/30: Public IP not yet available, waiting 10 seconds..." + sleep 10 +done + +if [ -z "${IP}" ]; then + echo "ERROR: Failed to get public IP after 5 minutes" + echo "Checking available public IPs in resource group:" + az network public-ip list --resource-group ${AKS_NODE_RESOURCE_GROUP} --output table + exit 1 +fi echo "Instance IP: ${IP}" echo "uid2_pipeline_e2e_operator_url=http://${IP}" >> ${GITHUB_OUTPUT} diff --git a/scripts/aks/stop_aks_enclave.sh b/scripts/aks/stop_aks_enclave.sh index 87d7c333..7a641036 100644 --- a/scripts/aks/stop_aks_enclave.sh +++ b/scripts/aks/stop_aks_enclave.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash set -ex -export RESOURCE_GROUP="pipeline-vn-aks" -export AKS_CLUSTER_NAME="pipelinevncluster" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/aks_env.sh" -az aks get-credentials --name ${AKS_CLUSTER_NAME} --resource-group ${RESOURCE_GROUP} -if kubectl get deployment operator-deployment -o name > /dev/null 2>&1; then - kubectl delete deployment operator-deployment - echo "Deployment 'operator-deployment' deleted." +if az group exists --name ${RESOURCE_GROUP} | grep -q true; then + echo "Deleting resource group '${RESOURCE_GROUP}'..." + az group delete --name ${RESOURCE_GROUP} --yes + echo "Resource group '${RESOURCE_GROUP}' successfully deleted." else - echo "Deployment 'operator-deployment' does not exist." + echo "Resource group '${RESOURCE_GROUP}' does not exist." fi \ No newline at end of file diff --git a/scripts/aws/start_aws_enclave.sh b/scripts/aws/start_aws_enclave.sh index 9458f071..0ad0fb33 100644 --- a/scripts/aws/start_aws_enclave.sh +++ b/scripts/aws/start_aws_enclave.sh @@ -41,9 +41,9 @@ if [ -z "${OPERATOR_KEY}" ]; then exit 1 fi -ROOT="./uid2-shared-actions/scripts" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${ROOT}/healthcheck.sh" +source "${SCRIPT_DIR}/../healthcheck.sh" DATE=$(date '+%Y%m%d%H%M%S') AWS_STACK_NAME="uid2-operator-e2e-${AWS_AMI}-${DATE}" @@ -66,8 +66,8 @@ case "${IDENTITY_SCOPE}" in exit 1 ;; esac -python ${ROOT}/aws/create_cloudformation_stack.py \ - --stack_fp "${ROOT}/aws/stacks" \ +python ${SCRIPT_DIR}/create_cloudformation_stack.py \ + --stack_fp "${SCRIPT_DIR}/stacks" \ --cftemplate_fp "../uid2-operator/scripts/aws" \ --core_url "${BORE_URL_CORE}" \ --optout_url "${BORE_URL_OPTOUT}" \ @@ -84,7 +84,7 @@ aws cloudformation describe-stacks \ --region "${AWS_REGION}" # Get public URL -AWS_INSTANCE_URL=$(python ${ROOT}/aws/get_instance_url.py \ +AWS_INSTANCE_URL=$(python ${SCRIPT_DIR}/get_instance_url.py \ --region "${AWS_REGION}" \ --stack "${AWS_STACK_NAME}") diff --git a/scripts/azure/prepare_azure_artifacts.sh b/scripts/azure/prepare_azure_artifacts.sh index 79e48945..7aa5801f 100644 --- a/scripts/azure/prepare_azure_artifacts.sh +++ b/scripts/azure/prepare_azure_artifacts.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash set -ex -ROOT="./uid2-shared-actions/scripts/azure" -INPUT_DIR="${ROOT}/artifacts_schema" -OUTPUT_DIR="${ROOT}/azure-artifacts" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INPUT_DIR="${SCRIPT_DIR}/artifacts_schema" +OUTPUT_DIR="${SCRIPT_DIR}/azure-artifacts" if [ -z "${IMAGE_VERSION}" ]; then echo "IMAGE_VERSION can not be empty" diff --git a/scripts/azure/start_azure_enclave.sh b/scripts/azure/start_azure_enclave.sh index c6064818..7cd65996 100644 --- a/scripts/azure/start_azure_enclave.sh +++ b/scripts/azure/start_azure_enclave.sh @@ -27,10 +27,10 @@ if [ -z "${TARGET_ENVIRONMENT}" ]; then fi # Below resources should be prepared ahead -ROOT="./uid2-shared-actions/scripts" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${ROOT}/jq_helper.sh" -source "${ROOT}/healthcheck.sh" +source "${SCRIPT_DIR}/../jq_helper.sh" +source "${SCRIPT_DIR}/../healthcheck.sh" RESOURCE_GROUP="uid-enclave-ci-cd" IDENTITY="uid-operator" diff --git a/scripts/gcp/start_gcp_enclave.sh b/scripts/gcp/start_gcp_enclave.sh index 7c56db8c..6174d505 100644 --- a/scripts/gcp/start_gcp_enclave.sh +++ b/scripts/gcp/start_gcp_enclave.sh @@ -31,11 +31,11 @@ if [ -z "${IMAGE_HASH}" ]; then exit 1 fi -ROOT="./uid2-shared-actions/scripts" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" GCP_INSTANCE_NAME="ci-test-${RANDOM}" OPERATOR_KEY_SECRET_NAME=${GCP_INSTANCE_NAME} -source "${ROOT}/healthcheck.sh" +source "${SCRIPT_DIR}/../healthcheck.sh" gcloud config set project ${GCP_PROJECT} diff --git a/scripts/get_operator_key.sh b/scripts/get_operator_key.sh index e27aa2e2..4fc652a3 100644 --- a/scripts/get_operator_key.sh +++ b/scripts/get_operator_key.sh @@ -34,6 +34,10 @@ elif [ "${IDENTITY_SCOPE}" == "EUID" ] && [ "${TARGET_ENVIRONMENT}" == "integ" ] OPERATOR_KEY=${E2E_EUID_INTEG_AWS_OPERATOR_API_KEY} elif [ "${IDENTITY_SCOPE}" == "EUID" ] && [ "${TARGET_ENVIRONMENT}" == "prod" ] && [ "${ENCLAVE_PROTOCOL}" == "aws-nitro" ]; then OPERATOR_KEY=${E2E_EUID_PROD_AWS_OPERATOR_API_KEY} +elif [ "${IDENTITY_SCOPE}" == "UID2" ] && [ "${TARGET_ENVIRONMENT}" == "integ" ] && [ "${ENCLAVE_PROTOCOL}" == "azure-cc" ]; then + OPERATOR_KEY=${E2E_UID2_INTEG_AKS_OPERATOR_KEY} +elif [ "${IDENTITY_SCOPE}" == "UID2" ] && [ "${TARGET_ENVIRONMENT}" == "prod" ] && [ "${ENCLAVE_PROTOCOL}" == "azure-cc" ]; then + OPERATOR_KEY=${E2E_UID2_PROD_AKS_OPERATOR_KEY} else echo "Arguments not supported: IDENTITY_SCOPE=${IDENTITY_SCOPE}, TARGET_ENVIRONMENT=${TARGET_ENVIRONMENT}, ENCLAVE_PROTOCOL=${ENCLAVE_PROTOCOL}" exit 1