From 2613217c78db5c8351b9cbbb2ab2365347109a22 Mon Sep 17 00:00:00 2001
From: Hans Johnson <hans-johnson@uiowa.edu>
Date: Wed, 1 Apr 2026 20:31:25 -0500
Subject: [PATCH 1/3] ENH: Mirror dockcross images to GHCR for reliable Python
 wheel builds

Python wheel CI builds pull large dockcross/manylinux container images
(~1-2 GB) from Docker Hub and Quay.io on every run. These pulls
intermittently fail with "unexpected EOF" or Docker Hub rate limiting
(401/429), causing wheel builds to fail across all remote modules.

Add a scheduled workflow that mirrors the pinned dockcross images to
GHCR (ghcr.io/insightsoftwareconsortium/), which has much better
connectivity from GitHub Actions runners (same network).

Add pre-pull steps to the Linux x64 and ARM build jobs that:
1. Try pulling from the GHCR mirror first
2. Fall back to the original Docker Hub / Quay.io source
3. Retry up to 3 times with backoff
4. Tag the GHCR image with the original name so the downstream
   ITKPythonPackage build scripts find it already cached

This eliminates transient Docker image pull failures that have been
causing sporadic Python wheel build failures across ~55 remote modules.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .../workflows/build-test-package-python.yml   | 54 +++++++++++++++++++
 .github/workflows/mirror-dockcross-images.yml | 46 ++++++++++++++++
 2 files changed, 100 insertions(+)
 create mode 100644 .github/workflows/mirror-dockcross-images.yml

diff --git a/.github/workflows/build-test-package-python.yml b/.github/workflows/build-test-package-python.yml
index cac4545..3688a7c 100644
--- a/.github/workflows/build-test-package-python.yml
+++ b/.github/workflows/build-test-package-python.yml
@@ -84,6 +84,38 @@ jobs:
         large-packages: "false"  # Takes too long to remove apt-get packages
         mandb:          "true"  # Speeds up future apt-get installs (disables man page generation), this CI does not use apt-get
 
+    - name: 'Pre-pull dockcross image'
+      run: |
+        MANYLINUX_PLATFORM=${{ matrix.manylinux-platform }}
+        MANYLINUX_VERSION=$(echo ${MANYLINUX_PLATFORM} | cut -d '-' -f 1)
+        TARGET_ARCH=$(echo ${MANYLINUX_PLATFORM} | cut -d '-' -f 2)
+        if [[ ${MANYLINUX_VERSION} == _2_28 && ${TARGET_ARCH} == x64 ]]; then
+          IMAGE_TAG="20240304-9e57d2b"
+        elif [[ ${MANYLINUX_VERSION} == 2014 ]]; then
+          IMAGE_TAG="20240304-9e57d2b"
+        else
+          echo "Unknown manylinux platform ${MANYLINUX_PLATFORM}, skipping pre-pull"
+          exit 0
+        fi
+        IMAGE="dockcross/manylinux${MANYLINUX_VERSION}-${TARGET_ARCH}:${IMAGE_TAG}"
+        GHCR_IMAGE="ghcr.io/insightsoftwareconsortium/dockcross-manylinux${MANYLINUX_VERSION}-${TARGET_ARCH}:${IMAGE_TAG}"
+        echo "Pre-pulling ${GHCR_IMAGE} (mirror of docker.io/${IMAGE})"
+        for attempt in 1 2 3; do
+          if docker pull "${GHCR_IMAGE}" 2>/dev/null; then
+            docker tag "${GHCR_IMAGE}" "docker.io/${IMAGE}"
+            echo "Successfully pulled from GHCR mirror"
+            exit 0
+          fi
+          echo "GHCR pull attempt ${attempt} failed, trying Docker Hub..."
+          if docker pull "docker.io/${IMAGE}"; then
+            echo "Successfully pulled from Docker Hub"
+            exit 0
+          fi
+          echo "Docker Hub pull attempt ${attempt} failed, retrying in 15s..."
+          sleep 15
+        done
+        echo "WARNING: All pull attempts failed, build script will retry"
+
     - name: 'Fetch build script'
       run: |
         IPP_DOWNLOAD_GIT_TAG=${{ inputs.itk-python-package-tag }}
@@ -180,6 +212,28 @@ jobs:
         large-packages: "false"  # Takes too long to remove apt-get packages
         mandb:          "true"  # Speeds up future apt-get installs (disables man page generation), this CI does not use apt-get
 
+    - name: 'Pre-pull manylinux aarch64 image'
+      run: |
+        IMAGE_TAG="2024-03-25-9206bd9"
+        IMAGE="quay.io/pypa/manylinux_2_28_aarch64:${IMAGE_TAG}"
+        GHCR_IMAGE="ghcr.io/insightsoftwareconsortium/dockcross-manylinux_2_28-aarch64:${IMAGE_TAG}"
+        echo "Pre-pulling ${GHCR_IMAGE} (mirror of ${IMAGE})"
+        for attempt in 1 2 3; do
+          if docker pull "${GHCR_IMAGE}" 2>/dev/null; then
+            docker tag "${GHCR_IMAGE}" "${IMAGE}"
+            echo "Successfully pulled from GHCR mirror"
+            exit 0
+          fi
+          echo "GHCR pull attempt ${attempt} failed, trying upstream..."
+          if docker pull "${IMAGE}"; then
+            echo "Successfully pulled from upstream"
+            exit 0
+          fi
+          echo "Upstream pull attempt ${attempt} failed, retrying in 15s..."
+          sleep 15
+        done
+        echo "WARNING: All pull attempts failed, build script will retry"
+
     - name: 'Fetch build script'
       run: |
         IPP_DOWNLOAD_GIT_TAG=${{ inputs.itk-python-package-tag }}
diff --git a/.github/workflows/mirror-dockcross-images.yml b/.github/workflows/mirror-dockcross-images.yml
new file mode 100644
index 0000000..0aabfc7
--- /dev/null
+++ b/.github/workflows/mirror-dockcross-images.yml
@@ -0,0 +1,46 @@
+name: Mirror dockcross images to GHCR
+
+on:
+  schedule:
+    # Run weekly on Monday at 06:00 UTC
+    - cron: '0 6 * * 1'
+  workflow_dispatch:
+
+env:
+  GHCR_ORG: ghcr.io/insightsoftwareconsortium
+
+jobs:
+  mirror:
+    runs-on: ubuntu-22.04
+    permissions:
+      packages: write
+    strategy:
+      matrix:
+        include:
+          - source: docker.io/dockcross/manylinux_2_28-x64:20240304-9e57d2b
+            target: dockcross-manylinux_2_28-x64:20240304-9e57d2b
+          - source: docker.io/dockcross/manylinux2014-x64:20240304-9e57d2b
+            target: dockcross-manylinux2014-x64:20240304-9e57d2b
+          - source: quay.io/pypa/manylinux_2_28_aarch64:2024-03-25-9206bd9
+            target: dockcross-manylinux_2_28-aarch64:2024-03-25-9206bd9
+
+    steps:
+    - name: Log in to GHCR
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Pull source image
+      run: |
+        for attempt in 1 2 3; do
+          docker pull ${{ matrix.source }} && break
+          echo "Pull attempt $attempt failed, retrying in 15s..."
+          sleep 15
+        done
+
+    - name: Tag and push to GHCR
+      run: |
+        docker tag ${{ matrix.source }} ${{ env.GHCR_ORG }}/${{ matrix.target }}
+        docker push ${{ env.GHCR_ORG }}/${{ matrix.target }}

From 64376dc0822fa4fec796f17b3aad1f2ad57464c2 Mon Sep 17 00:00:00 2001
From: Hans Johnson <hans-johnson@uiowa.edu>
Date: Thu, 2 Apr 2026 07:42:31 -0500
Subject: [PATCH 2/3] ENH: Dynamically resolve dockcross image tags from
 ITKPythonPackage

The pre-pull step was hardcoding v5.4.x image tags, but modules using
v6.0b02 defaults pull different images (20260203-3dfb3ff for x64,
2025.08.12-1 for aarch64). The pre-pull cached the wrong image and
the actual build script still hit Docker Hub without caching.

Fetch dockcross-manylinux-set-vars.sh from the same ITKPythonPackage
tag the build will use, then source it to get the correct IMAGE_TAG
and CONTAINER_SOURCE. This ensures the pre-pull step always caches
exactly the image the build script will need.

Also add v6.0b02 image tags to the GHCR mirror workflow.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .../workflows/build-test-package-python.yml   | 67 +++++++++++++------
 .github/workflows/mirror-dockcross-images.yml |  6 ++
 2 files changed, 53 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/build-test-package-python.yml b/.github/workflows/build-test-package-python.yml
index 3688a7c..15c7704 100644
--- a/.github/workflows/build-test-package-python.yml
+++ b/.github/workflows/build-test-package-python.yml
@@ -86,32 +86,42 @@ jobs:
 
     - name: 'Pre-pull dockcross image'
       run: |
+        # Resolve the image tag dynamically from the ITKPythonPackage build scripts
+        # so it matches exactly what the build step will pull.
+        IPP_TAG=${{ inputs.itk-python-package-tag }}
+        IPP_TAG=${IPP_TAG:=main}
+        IPP_ORG=${{ inputs.itk-python-package-org }}
+        IPP_ORG=${IPP_ORG:=InsightSoftwareConsortium}
+
         MANYLINUX_PLATFORM=${{ matrix.manylinux-platform }}
-        MANYLINUX_VERSION=$(echo ${MANYLINUX_PLATFORM} | cut -d '-' -f 1)
-        TARGET_ARCH=$(echo ${MANYLINUX_PLATFORM} | cut -d '-' -f 2)
-        if [[ ${MANYLINUX_VERSION} == _2_28 && ${TARGET_ARCH} == x64 ]]; then
-          IMAGE_TAG="20240304-9e57d2b"
-        elif [[ ${MANYLINUX_VERSION} == 2014 ]]; then
-          IMAGE_TAG="20240304-9e57d2b"
-        else
-          echo "Unknown manylinux platform ${MANYLINUX_PLATFORM}, skipping pre-pull"
+        export MANYLINUX_VERSION=$(echo ${MANYLINUX_PLATFORM} | cut -d '-' -f 1)
+        export TARGET_ARCH=$(echo ${MANYLINUX_PLATFORM} | cut -d '-' -f 2)
+
+        VARS_URL="https://raw.githubusercontent.com/${IPP_ORG}/ITKPythonPackage/${IPP_TAG}/scripts/dockcross-manylinux-set-vars.sh"
+        echo "Fetching image tags from ${VARS_URL}"
+        curl -fsSL "${VARS_URL}" -o /tmp/set-vars.sh || { echo "Could not fetch set-vars.sh, skipping pre-pull"; exit 0; }
+        source /tmp/set-vars.sh
+
+        if [[ -z ${CONTAINER_SOURCE} ]]; then
+          echo "Could not determine container source, skipping pre-pull"
           exit 0
         fi
-        IMAGE="dockcross/manylinux${MANYLINUX_VERSION}-${TARGET_ARCH}:${IMAGE_TAG}"
+
+        # Build the GHCR mirror name
         GHCR_IMAGE="ghcr.io/insightsoftwareconsortium/dockcross-manylinux${MANYLINUX_VERSION}-${TARGET_ARCH}:${IMAGE_TAG}"
-        echo "Pre-pulling ${GHCR_IMAGE} (mirror of docker.io/${IMAGE})"
+        echo "Pre-pulling ${GHCR_IMAGE} (mirror of ${CONTAINER_SOURCE})"
         for attempt in 1 2 3; do
           if docker pull "${GHCR_IMAGE}" 2>/dev/null; then
-            docker tag "${GHCR_IMAGE}" "docker.io/${IMAGE}"
+            docker tag "${GHCR_IMAGE}" "${CONTAINER_SOURCE}"
             echo "Successfully pulled from GHCR mirror"
             exit 0
           fi
-          echo "GHCR pull attempt ${attempt} failed, trying Docker Hub..."
-          if docker pull "docker.io/${IMAGE}"; then
-            echo "Successfully pulled from Docker Hub"
+          echo "GHCR pull attempt ${attempt} failed, trying upstream ${CONTAINER_SOURCE}..."
+          if docker pull "${CONTAINER_SOURCE}"; then
+            echo "Successfully pulled from upstream"
             exit 0
           fi
-          echo "Docker Hub pull attempt ${attempt} failed, retrying in 15s..."
+          echo "Upstream pull attempt ${attempt} failed, retrying in 15s..."
           sleep 15
         done
         echo "WARNING: All pull attempts failed, build script will retry"
@@ -214,18 +224,35 @@ jobs:
 
     - name: 'Pre-pull manylinux aarch64 image'
       run: |
-        IMAGE_TAG="2024-03-25-9206bd9"
-        IMAGE="quay.io/pypa/manylinux_2_28_aarch64:${IMAGE_TAG}"
+        # Resolve the image tag dynamically from the ITKPythonPackage build scripts
+        IPP_TAG=${{ inputs.itk-python-package-tag }}
+        IPP_TAG=${IPP_TAG:=main}
+        IPP_ORG=${{ inputs.itk-python-package-org }}
+        IPP_ORG=${IPP_ORG:=InsightSoftwareConsortium}
+
+        export MANYLINUX_VERSION="_2_28"
+        export TARGET_ARCH="aarch64"
+
+        VARS_URL="https://raw.githubusercontent.com/${IPP_ORG}/ITKPythonPackage/${IPP_TAG}/scripts/dockcross-manylinux-set-vars.sh"
+        echo "Fetching image tags from ${VARS_URL}"
+        curl -fsSL "${VARS_URL}" -o /tmp/set-vars.sh || { echo "Could not fetch set-vars.sh, skipping pre-pull"; exit 0; }
+        source /tmp/set-vars.sh
+
+        if [[ -z ${CONTAINER_SOURCE} ]]; then
+          echo "Could not determine container source, skipping pre-pull"
+          exit 0
+        fi
+
         GHCR_IMAGE="ghcr.io/insightsoftwareconsortium/dockcross-manylinux_2_28-aarch64:${IMAGE_TAG}"
-        echo "Pre-pulling ${GHCR_IMAGE} (mirror of ${IMAGE})"
+        echo "Pre-pulling ${GHCR_IMAGE} (mirror of ${CONTAINER_SOURCE})"
         for attempt in 1 2 3; do
           if docker pull "${GHCR_IMAGE}" 2>/dev/null; then
-            docker tag "${GHCR_IMAGE}" "${IMAGE}"
+            docker tag "${GHCR_IMAGE}" "${CONTAINER_SOURCE}"
             echo "Successfully pulled from GHCR mirror"
             exit 0
           fi
           echo "GHCR pull attempt ${attempt} failed, trying upstream..."
-          if docker pull "${IMAGE}"; then
+          if docker pull "${CONTAINER_SOURCE}"; then
             echo "Successfully pulled from upstream"
             exit 0
           fi
diff --git a/.github/workflows/mirror-dockcross-images.yml b/.github/workflows/mirror-dockcross-images.yml
index 0aabfc7..df9a212 100644
--- a/.github/workflows/mirror-dockcross-images.yml
+++ b/.github/workflows/mirror-dockcross-images.yml
@@ -17,12 +17,18 @@ jobs:
     strategy:
       matrix:
         include:
+          # v5.4.x image tags
           - source: docker.io/dockcross/manylinux_2_28-x64:20240304-9e57d2b
             target: dockcross-manylinux_2_28-x64:20240304-9e57d2b
           - source: docker.io/dockcross/manylinux2014-x64:20240304-9e57d2b
             target: dockcross-manylinux2014-x64:20240304-9e57d2b
           - source: quay.io/pypa/manylinux_2_28_aarch64:2024-03-25-9206bd9
             target: dockcross-manylinux_2_28-aarch64:2024-03-25-9206bd9
+          # v6.0b02 image tags
+          - source: docker.io/dockcross/manylinux_2_28-x64:20260203-3dfb3ff
+            target: dockcross-manylinux_2_28-x64:20260203-3dfb3ff
+          - source: quay.io/pypa/manylinux_2_28_aarch64:2025.08.12-1
+            target: dockcross-manylinux_2_28-aarch64:2025.08.12-1
 
     steps:
     - name: Log in to GHCR

From d171504997770484733a9eba8aa56d4976a53da8 Mon Sep 17 00:00:00 2001
From: Hans Johnson <hans-johnson@uiowa.edu>
Date: Thu, 2 Apr 2026 07:47:32 -0500
Subject: [PATCH 3/3] ENH: Add v6.0b01 dockcross image to GHCR mirror

Add docker.io/dockcross/manylinux_2_28-x64:20250913-6ea98ba used by
ITKPythonPackage v6.0b01 (currently only ITKTotalVariation).

Complete image tag coverage across all ITKPythonPackage versions:
- v5.4.0-v5.4.5: 20240304-9e57d2b (x64), 2024-03-25-9206bd9 (aarch64)
- v6.0b01: 20250913-6ea98ba (x64), 2025.08.12-1 (aarch64)
- v6.0b02/main: 20260203-3dfb3ff (x64), 2025.08.12-1 (aarch64)
- manylinux2014-x64: 20240304-9e57d2b (all versions)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .github/workflows/mirror-dockcross-images.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/mirror-dockcross-images.yml b/.github/workflows/mirror-dockcross-images.yml
index df9a212..2e5d4df 100644
--- a/.github/workflows/mirror-dockcross-images.yml
+++ b/.github/workflows/mirror-dockcross-images.yml
@@ -24,7 +24,10 @@ jobs:
             target: dockcross-manylinux2014-x64:20240304-9e57d2b
           - source: quay.io/pypa/manylinux_2_28_aarch64:2024-03-25-9206bd9
             target: dockcross-manylinux_2_28-aarch64:2024-03-25-9206bd9
-          # v6.0b02 image tags
+          # v6.0b01 image tags
+          - source: docker.io/dockcross/manylinux_2_28-x64:20250913-6ea98ba
+            target: dockcross-manylinux_2_28-x64:20250913-6ea98ba
+          # v6.0b02 / main image tags
           - source: docker.io/dockcross/manylinux_2_28-x64:20260203-3dfb3ff
             target: dockcross-manylinux_2_28-x64:20260203-3dfb3ff
           - source: quay.io/pypa/manylinux_2_28_aarch64:2025.08.12-1