feat: windows network isolated cluster support anonymous-disabled bootstrap acr

e2e/scenario_win_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/Azure/agentbaker/e2e/components"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	"github.com/Masterminds/semver"
 	"github.com/stretchr/testify/require"
 
 	"github.com/Azure/agentbaker/e2e/config"
@@ -507,18 +508,35 @@ func Test_NetworkIsolatedCluster_Windows_WithEgress(t *testing.T) {
 		Description: "Tests that Windows nodes in network isolated clusters configure containerd to use the bootstrap profile container registry for MCR images",
 		Tags: Tags{
 			NetworkIsolated: true,
-			NonAnonymousACR: false,
+			NonAnonymousACR: true,
 		},
 		Config: Config{
 			Cluster: ClusterAzureBootstrapProfileCache,
 			VHD:     config.VHDWindows2025Gen2,
 			BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
+				Windows2025BootstrapConfigMutator(t, nbc)
 				nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
 					PrivateEgress: &datamodel.PrivateEgress{
 						Enabled:                 true,
-						ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks-managed-repository", config.PrivateACRName(config.Config.DefaultLocation)),
+						ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks-managed-repository", config.PrivateACRNameNotAnon(config.Config.DefaultLocation)),
 					},
 				}
+				nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity = true
+				nbc.AgentPoolProfile.KubernetesConfig.UseManagedIdentity = true
+				nbc.KubeletConfig["--image-credential-provider-config"] = "c:\\k\\credential-provider-config.yaml"
+				nbc.KubeletConfig["--image-credential-provider-bin-dir"] = "c:\\var\\lib\\kubelet\\credential-provider"
+				orchestratorVersion, _ := semver.NewVersion(nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion)
+				if orchestratorVersion.LessThan(semver.MustParse("1.32.0")) {
+					nbc.K8sComponents.WindowsCredentialProviderURL = fmt.Sprintf(
+						"https://packages.aks.azure.com/cloud-provider-azure/v%s/binaries/azure-acr-credential-provider-windows-amd64-v%s.tar.gz",
+						nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion,
+						nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion)
+				} else {
+					nbc.K8sComponents.WindowsCredentialProviderURL = fmt.Sprintf(
+						"https://packages.aks.azure.com/dalec-packages/azure-acr-credential-provider/%s/windows/amd64/azure-acr-credential-provider_%s-1_amd64.zip",
+						nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion,
+						nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion)
+				}
 			},
 			Validator: func(ctx context.Context, s *Scenario) {
 				// Verify mcr.microsoft.com host config exist

staging/cse/windows/networkisolatedclusterfunc.ps1

@@ -3,7 +3,7 @@
 # Initialize-Oras will install oras and login the registry if anonymous access is disabled. This is required for network isolated cluster to pull windowszip from private container registry.
 function Initialize-Oras {
     Install-Oras
-    # reserve for Invoke-OrasLogin to avoid frequent code changes in parts/windows/
+    Invoke-OrasLogin -Acr_Url $(Get-BootstrapRegistryDomainName) -ClientID $UserAssignedClientID -TenantID $global:TenantId
 }
 
 # unpackage and install oras from cache
@@ -143,3 +143,225 @@ function Set-PodInfraContainerImage {
     Remove-Item -Path $podInfraContainerImageDownloadDir -Recurse -Force -ErrorAction SilentlyContinue
     Remove-Item -Path $podInfraContainerImageTar -Force -ErrorAction SilentlyContinue
 }
+
+function Invoke-OrasLogin {
+    param(
+        [Parameter(Mandatory = $true)][string]
+        $Acr_Url,
+        [Parameter(Mandatory = $true)][string]
+        $ClientID,
+        [Parameter(Mandatory = $true)][string]
+        $TenantID
+    )
+
+    # Check for required variables
+    if ([string]::IsNullOrWhiteSpace($ClientID) -or [string]::IsNullOrWhiteSpace($TenantID)) {
+        Write-Host "ClientID or TenantID are not set. Oras login is not possible, proceeding with anonymous pull"
+        return $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED
+    }
+
+    # Attempt anonymous pull check (assumes helper function exists)
+    $retCode = Assert-AnonymousAcrAccess 10 5 $Acr_Url
+    if ($retCode -eq 0) {
+        Write-Host "anonymous pull is allowed for acr '$Acr_Url', proceeding with anonymous pull"
+        return
+    }
+    elseif ($retCode -ne 1) {
+        Write-Host "failed with an error other than unauthorized, exiting.."
+        Set-ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_NETWORK_TIMEOUT -ErrorMessage "failed with an error other than unauthorized, exiting"
+    }
+
+    # Get AAD Access Token using Managed Identity Metadata Service
+    $accessUrl = "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://management.azure.com/&client_id=$ClientID"
+    try {
+        $requestArgs = @{
+            Uri     = $accessUrl
+            Method  = "Get"
+            Headers = @{ Metadata = "true" }
+            TimeoutSec  = 10
+        }
+        $rawAccessTokenResponse = Retry-Command -Command "Invoke-RestMethod" -Args $requestArgs -Retries 10 -RetryDelaySeconds 5
+        $accessToken = $rawAccessTokenResponse.access_token
+    }
+    catch {
+        Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_IMDS_TIMEOUT -ErrorMessage "failed to retrieve AAD access token: $($_.Exception.Message)"
+    }
+
+    if ([string]::IsNullOrWhiteSpace($accessToken)) {
+        Set-ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED -ErrorMessage "failed to parse imds access token"
+    }
+
+    # Exchange AAD Access Token for ACR Refresh Token
+    try {
+        $exchangeUrl = "https://$Acr_Url/oauth2/exchange"
+        $body = @{
+            grant_type   = "access_token"
+            service      = $Acr_Url
+            tenant       = $TenantID
+            access_token = $accessToken
+        }
+        $requestArgs = @{
+            Uri         = $exchangeUrl
+            Method      = "Post"
+            ContentType = "application/x-www-form-urlencoded"
+            Body        = $body
+            TimeoutSec  = 10
+        }
+        $rawRefreshTokenResponse = Retry-Command -Command "Invoke-RestMethod" -Args $requestArgs -Retries 10 -RetryDelaySeconds 5
+        $refreshToken = $rawRefreshTokenResponse.refresh_token
+    }
+    catch {
+        Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED -ErrorMessage "failed to retrieve refresh token: $($_.Exception.Message)"
+    }
+
+    if ([string]::IsNullOrWhiteSpace($refreshToken)) {
+        Set-ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED -ErrorMessage "failed to parse refresh token"
+    }
+
+    # Pre-validate refresh token permissions
+    $retCode = Assert-RefreshToken -RefreshToken $refreshToken -RequiredActions @("read")
+    if ($retCode -ne 0) {
+        Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED -ErrorMessage "failed to validate refresh token permissions"
+    }
+
+    # Perform Oras Login (pipe refresh token to stdin for --identity-token-stdin)
+    $loginSuccess = $false
+    for ($i = 1; $i -le 3; $i++) {
+        try {
+            Write-Log "Retry $i : oras login $Acr_Url"
+            $loginOutput = $refreshToken | & $global:OrasPath login $Acr_Url --identity-token-stdin --registry-config $global:OrasRegistryConfigFile 2>&1
+            if ($LASTEXITCODE -eq 0) {
+                $loginSuccess = $true
+                break
+            }
+            Write-Log "oras login attempt $i failed (exit code $LASTEXITCODE): $loginOutput"
+        }
+        catch {
+            Write-Log "oras login attempt $i exception: $($_.Exception.Message)"
+        }
+        if ($i -lt 3) {
+            Start-Sleep -Seconds 5
+        }
+    }
+    if (-Not $loginSuccess) {
+        Set-ExitCode $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED -ErrorMessage "failed to login to acr '$Acr_Url' with identity token"
+    }
+
+    # Clean up sensitive data
+    Remove-Variable accessToken, refreshToken -ErrorAction SilentlyContinue
+
+    Write-Host "successfully logged in to acr '$Acr_Url' with identity token"
+}
+
+function Assert-AnonymousAcrAccess {
+    Param(
+        [Parameter(Mandatory = $true)][int]$Retries,
+        [Parameter(Mandatory = $true)][int]$WaitSleep,
+        [Parameter(Mandatory = $true)][string]$AcrUrl
+    )
+
+    for ($i = 1; $i -le $Retries; $i++) {
+        # Logout first to ensure insufficient ABAC token won't affect anonymous judging
+        try { & $global:OrasPath logout $AcrUrl  --registry-config $global:OrasRegistryConfigFile 2>$null } catch { }
+
+        $output = $null
+        try {
+            $output = & $global:OrasPath repo ls $AcrUrl  --registry-config $global:OrasRegistryConfigFile 2>&1
+        }
+        catch {
+            $output = $_.Exception.Message
+            # Ensure we do not rely on a stale success exit code when repo ls throws
+            $LASTEXITCODE = 1
+        }
+
+        if ($LASTEXITCODE -eq 0) {
+            Write-Host "acr is anonymously reachable"
+            return 0
+        }
+
+        if ($output -and ($output -like "*unauthorized: authentication required*")) {
+            Write-Host "ACR is not anonymously reachable: $output"
+            return 1
+        }
+
+        Start-Sleep -Seconds $WaitSleep
+    }
+
+    Write-Host "unexpected response from acr: $output"
+    return $global:WINDOWS_CSE_ERROR_ORAS_PULL_NETWORK_TIMEOUT
+}
+
+function Assert-RefreshToken {
+    Param(
+        [Parameter(Mandatory = $true)][string]$RefreshToken,
+        [Parameter(Mandatory = $true)][string[]]$RequiredActions
+    )
+
+    # Decode the refresh token (JWT format: header.payload.signature)
+    # Extract the payload (second part) and decode from base64
+    $tokenParts = $RefreshToken.Split('.')
+    if ($tokenParts.Length -lt 2) {
+        Write-Host "Invalid JWT token format"
+        return $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED
+    }
+
+    $tokenPayload = $tokenParts[1]
+    # Add padding if needed for base64url decoding
+    switch ($tokenPayload.Length % 4) {
+        2 { $tokenPayload += "==" }
+        3 { $tokenPayload += "=" }
+    }
+    # Replace base64url characters with standard base64
+    $tokenPayload = $tokenPayload -replace '-', '+' -replace '_', '/'
+
+    try {
+        $decodedBytes = [System.Convert]::FromBase64String($tokenPayload)
+        $decodedToken = [System.Text.Encoding]::UTF8.GetString($decodedBytes)
+    }
+    catch {
+        Write-Host "Failed to decode token payload: $($_.Exception.Message)"
+        return $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED
+    }
+
+    if (-Not [string]::IsNullOrWhiteSpace($decodedToken)) {
+        try {
+            $tokenObj = $decodedToken | ConvertFrom-Json
+        }
+        catch {
+            Write-Host "Failed to parse token JSON: $($_.Exception.Message)"
+            return $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED
+        }
+
+        # Check if permissions field exists (RBAC token vs ABAC token)
+        if ($null -ne $tokenObj.permissions) {
+            Write-Host "RBAC token detected, validating permissions"
+
+            $tokenActions = @()
+            if ($null -ne $tokenObj.permissions.actions) {
+                $tokenActions = @($tokenObj.permissions.actions)
+            }
+
+            foreach ($action in $RequiredActions) {
+                if ($tokenActions -notcontains $action) {
+                    Write-Host "Required action '$action' not found in token permissions"
+                    return $global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED
+                }
+            }
+            Write-Host "Token validation passed: all required actions present"
+        }
+        else {
+            Write-Host "No permissions field found in token. Assuming ABAC token, skipping permission validation"
+        }
+    }
+
+    return 0
+}
+
+function Get-BootstrapRegistryDomainName {
+    $registryDomainName = if ($global:MCRRepositoryBase) { $global:MCRRepositoryBase } else { "mcr.microsoft.com" } # default to mcr
+    $registryDomainName = $registryDomainName.TrimEnd("/")
+    if ($global:BootstrapProfileContainerRegistryServer) {
+        $registryDomainName = $global:BootstrapProfileContainerRegistryServer.Split("/")[0]
+    }
+    return $registryDomainName
+}