feat: implement provisioning script hotfix system

.pipelines/scripts/verify_shell.sh

@@ -30,6 +30,7 @@ filesToCheck=$(find . -type f -name "*.sh" -not -path './pkg/agent/testdata/*' -
 # Known bash-only scripts that intentionally use bash specific syntax.
 BASH_ONLY_LIST=$(cat <<'EOF'
 ./vhdbuilder/packer/install-ig.sh
+./vhdbuilder/provisioning-manifest/build-hotfix-oci.sh
 EOF
 )
 

Makefile

@@ -263,4 +263,8 @@ validate-components:
 	@./hack/tools/bin/cue vet -c ./schemas/components.cue ./vhdbuilder/packer/windows/components-test.json
 	@./hack/tools/bin/cue vet -c ./schemas/windows_settings.cue ./vhdbuilder/packer/windows/windows_settings.json
 
+.PHONY: build-provisioning-scripts-hotfix
+build-provisioning-scripts-hotfix:
+	@bash vhdbuilder/provisioning-manifest/build-hotfix-oci.sh $(HOTFIX_ARGS)
+
 include versioning.mk

e2e/aks_model.go

@@ -260,14 +260,31 @@ func getFirewall(ctx context.Context, location, firewallSubnetID, publicIPID str
 		TargetFqdns: []*string{to.Ptr("download.microsoft.com")},
 	}
 
+	// Needed for hotfix e2e tests - allows VMs to pull test hotfix artifacts from the test ACR
+	// ACR redirects blob downloads to *.blob.core.windows.net, so that must also be allowed
+	hotfixACRFqdn := config.PrivateACRNameNotAnon(location) + ".azurecr.io"
+	hotfixACRDataFqdn := location + ".data.azurecr.io"
+	hotfixACRBlobFqdn := "*.blob.core.windows.net"
+	hotfixACRRule := armnetwork.AzureFirewallApplicationRule{
+		Name:            to.Ptr("hotfix-acr-fqdn"),
+		SourceAddresses: []*string{to.Ptr("*")},
+		Protocols: []*armnetwork.AzureFirewallApplicationRuleProtocol{
+			{
+				ProtocolType: to.Ptr(armnetwork.AzureFirewallApplicationRuleProtocolTypeHTTPS),
+				Port:         to.Ptr[int32](443),
+			},
+		},
+		TargetFqdns: []*string{to.Ptr(hotfixACRFqdn), to.Ptr(hotfixACRDataFqdn), to.Ptr(hotfixACRBlobFqdn)},
+	}
+
 	appRuleCollection := armnetwork.AzureFirewallApplicationRuleCollection{
 		Name: to.Ptr("aksfwar"),
 		Properties: &armnetwork.AzureFirewallApplicationRuleCollectionPropertiesFormat{
 			Priority: to.Ptr[int32](100),
 			Action: &armnetwork.AzureFirewallRCAction{
 				Type: to.Ptr(armnetwork.AzureFirewallRCActionTypeAllow),
 			},
-			Rules: []*armnetwork.AzureFirewallApplicationRule{&aksAppRule, &blobStorageAppRule, &mooncakeMARRule, &dmcRule},
+			Rules: []*armnetwork.AzureFirewallApplicationRule{&aksAppRule, &blobStorageAppRule, &mooncakeMARRule, &dmcRule, &hotfixACRRule},
 		},
 	}
 

e2e/scenario_test.go

@@ -3,6 +3,7 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -2505,3 +2506,118 @@ func Test_Ubuntu2204Gen2_ImagePullIdentityBinding_Disabled_Scriptless(t *testing
 		},
 	})
 }
+
+func Test_Ubuntu2204_NetworkIsolated_HotfixApplied(t *testing.T) {
+	RunScenario(t, &Scenario{
+		Description: "Tests that provisioning script hotfix detection on a network-isolated cluster correctly pulls and applies an available hotfix artifact from the registry",
+		Tags: Tags{
+			NetworkIsolated: true,
+			NonAnonymousACR: true,
+		},
+		Config: Config{
+			Cluster: ClusterAzureNetworkIsolated,
+			VHD:     config.VHDUbuntu2204Gen2Containerd,
+			BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
+				nbc.OutboundType = datamodel.OutboundTypeBlock
+				nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
+					PrivateEgress: &datamodel.PrivateEgress{
+						Enabled:                 true,
+						ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks-managed-repository", config.PrivateACRNameNotAnon(config.Config.DefaultLocation)),
+					},
+				}
+				nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity = true
+				nbc.AgentPoolProfile.KubernetesConfig.UseManagedIdentity = true
+				nbc.K8sComponents.LinuxCredentialProviderURL = fmt.Sprintf(
+					"https://packages.aks.azure.com/cloud-provider-azure/v%s/binaries/azure-acr-credential-provider-linux-amd64-v%s.tar.gz",
+					nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion,
+					nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion)
+				nbc.KubeletConfig["--image-credential-provider-config"] = "/var/lib/kubelet/credential-provider-config.yaml"
+				nbc.KubeletConfig["--image-credential-provider-bin-dir"] = "/var/lib/kubelet/credential-provider"
+			},
+			Validator: func(ctx context.Context, s *Scenario) {
+				hotfixRegistry := fmt.Sprintf("%s.azurecr.io", config.PrivateACRNameNotAnon(config.Config.DefaultLocation))
+
+				// Setup and run: extract function, write version stamp, clear state, run hotfix detection
+				execScriptOnVMForScenarioValidateExitCode(ctx, s,
+					strings.Join([]string{
+						"set -e",
+						`sudo sed -n '/^check_for_script_hotfix()/,/^}/p' /opt/azure/containers/provision_start.sh > /tmp/hotfix_func.sh`,
+						`sudo bash -c 'echo "999999.99.0" > /opt/azure/containers/.provisioning-scripts-version'`,
+						`sudo rm -f /opt/azure/containers/.hotfix-applied`,
+						`sudo bash -c '> /var/log/azure/hotfix-check.log'`,
+						fmt.Sprintf(`sudo bash -c 'export PATH=/opt/bin:$PATH && source /tmp/hotfix_func.sh && export HOTFIX_REGISTRY=%s && check_for_script_hotfix'`, hotfixRegistry),
+					}, "\n"),
+					0, "Setup and run hotfix detection")
+
+				// Verify: hotfix detected, applied, marker created, staging cleaned up
+				execScriptOnVMForScenarioValidateExitCode(ctx, s,
+					strings.Join([]string{
+						"set -e",
+						`sudo grep -q "found hotfix 999999.99.0-hotfix" /var/log/azure/hotfix-check.log`,
+						`sudo grep -q "applied 999999.99.0-hotfix successfully" /var/log/azure/hotfix-check.log`,
+						`sudo test -f /opt/azure/containers/.hotfix-applied`,
+						`sudo test ! -d /opt/azure/containers/.hotfix-staging`,
+					}, "\n"),
+					0, "Hotfix should be detected, applied, marker created, and staging cleaned up")
+
+				// Idempotency: running again should skip because marker exists
+				execScriptOnVMForScenarioValidateExitCode(ctx, s,
+					strings.Join([]string{
+						`sudo bash -c '> /var/log/azure/hotfix-check.log'`,
+						fmt.Sprintf(`sudo bash -c 'export PATH=/opt/bin:$PATH && source /tmp/hotfix_func.sh && export HOTFIX_REGISTRY=%s && check_for_script_hotfix'`, hotfixRegistry),
+						`sudo grep -q "hotfix already applied" /var/log/azure/hotfix-check.log`,
+					}, "\n"),
+					0, "Re-running should skip (idempotency)")
+			},
+		},
+	})
+}
+
+func Test_Ubuntu2204_HotfixApplied(t *testing.T) {
+	RunScenario(t, &Scenario{
+		Description: "Tests that provisioning script hotfix detection on a non-NI cluster correctly pulls and applies an available hotfix artifact from the registry",
+		Config: Config{
+			Cluster: ClusterAzureNetwork,
+			VHD:     config.VHDUbuntu2204Gen2Containerd,
+			BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
+				nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure)
+				nbc.AgentPoolProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure)
+			},
+			Validator: func(ctx context.Context, s *Scenario) {
+				hotfixRegistry := fmt.Sprintf("%s.azurecr.io", config.PrivateACRNameNotAnon(config.Config.DefaultLocation))
+
+				// Setup and run: extract function, write version stamp, clear state, run hotfix detection
+				execScriptOnVMForScenarioValidateExitCode(ctx, s,
+					strings.Join([]string{
+						"set -e",
+						`sudo sed -n '/^check_for_script_hotfix()/,/^}/p' /opt/azure/containers/provision_start.sh > /tmp/hotfix_func.sh`,
+						`sudo bash -c 'echo "999999.99.0" > /opt/azure/containers/.provisioning-scripts-version'`,
+						`sudo rm -f /opt/azure/containers/.hotfix-applied`,
+						`sudo bash -c '> /var/log/azure/hotfix-check.log'`,
+						fmt.Sprintf(`sudo bash -c 'export PATH=/opt/bin:$PATH && source /tmp/hotfix_func.sh && export HOTFIX_REGISTRY=%s && check_for_script_hotfix'`, hotfixRegistry),
+					}, "\n"),
+					0, "Setup and run hotfix detection")
+
+				// Verify: hotfix detected, applied, marker created, staging cleaned up
+				execScriptOnVMForScenarioValidateExitCode(ctx, s,
+					strings.Join([]string{
+						"set -e",
+						`sudo grep -q "found hotfix 999999.99.0-hotfix" /var/log/azure/hotfix-check.log`,
+						`sudo grep -q "applied 999999.99.0-hotfix successfully" /var/log/azure/hotfix-check.log`,
+						`sudo test -f /opt/azure/containers/.hotfix-applied`,
+						`sudo test ! -d /opt/azure/containers/.hotfix-staging`,
+					}, "\n"),
+					0, "Hotfix should be detected, applied, marker created, and staging cleaned up")
+
+				// Idempotency: running again should skip because marker exists
+				execScriptOnVMForScenarioValidateExitCode(ctx, s,
+					strings.Join([]string{
+						`sudo bash -c '> /var/log/azure/hotfix-check.log'`,
+						fmt.Sprintf(`sudo bash -c 'export PATH=/opt/bin:$PATH && source /tmp/hotfix_func.sh && export HOTFIX_REGISTRY=%s && check_for_script_hotfix'`, hotfixRegistry),
+						`sudo grep -q "hotfix already applied" /var/log/azure/hotfix-check.log`,
+					}, "\n"),
+					0, "Re-running should skip (idempotency)")
+			},
+		},
+	})
+}

e2e/vmss.go

@@ -570,6 +570,7 @@ func extractLogsFromVMLinux(ctx context.Context, s *Scenario, vm *ScenarioVM) er
 		"waagent.log":                      "sudo cat /var/log/waagent.log",
 		"aks-node-controller.log":          "sudo cat /var/log/azure/aks-node-controller.log",
 		"aks-node-controller-config.json":  "sudo cat /opt/azure/containers/aks-node-controller-config.json", // Only available in Scriptless.
+		"hotfix-check.log":                 "sudo cat /var/log/azure/hotfix-check.log",
 
 		// Only available in Scriptless. By default, e2e enables aks-node-controller-hack, so this is the actual config used. Only in e2e. Not used in production.
 		"aks-node-controller-config-hack.json": "sudo cat /opt/azure/containers/aks-node-controller-config-hack.json",

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -1120,7 +1120,7 @@ assert_refresh_token() {
     return 0
 }
 
-oras_login_with_kubelet_identity() {
+oras_login_with_managed_identity() {
     local acr_url=$1
     local client_id=$2
     local tenant_id=$3

parts/linux/cloud-init/artifacts/cse_main.sh

@@ -119,14 +119,19 @@ function basePrep {
 
     # oras login must be in front of configureKubeletAndKubectl and ensureKubelet
     if [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then
-        # Compute registry domain name for ORAS login
-        registry_domain_name="${MCR_REPOSITORY_BASE:-mcr.microsoft.com}"
-        registry_domain_name="${registry_domain_name%/}"
-        if [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then
-            registry_domain_name="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER%%/*}"
-        fi
+        # Skip login if already authenticated (e.g., by early login in cse_start.sh)
+        if [ -s "${ORAS_REGISTRY_CONFIG_FILE}" ]; then
+            echo "ORAS: reusing existing credentials from ${ORAS_REGISTRY_CONFIG_FILE}"
+        else
+            # Compute registry domain name for ORAS login
+            registry_domain_name="${MCR_REPOSITORY_BASE:-mcr.microsoft.com}"
+            registry_domain_name="${registry_domain_name%/}"
+            if [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then
+                registry_domain_name="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER%%/*}"
+            fi
 
-        logs_to_events "AKS.CSE.orasLogin.oras_login_with_kubelet_identity" oras_login_with_kubelet_identity "${registry_domain_name}" $USER_ASSIGNED_IDENTITY_ID $TENANT_ID || exit $?
+            logs_to_events "AKS.CSE.orasLogin.oras_login_with_managed_identity" oras_login_with_managed_identity "${registry_domain_name}" $USER_ASSIGNED_IDENTITY_ID $TENANT_ID || exit $?
+        fi
     fi
 
     logs_to_events "AKS.CSE.disableSystemdResolved" disableSystemdResolved