From 55a0239bd8aa0a306efe9545a48335dca202d077 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Wed, 8 Apr 2026 19:17:12 +0300 Subject: [PATCH 01/16] feat(ci): test rolling over release to release Signed-off-by: Nikita Korolev --- .github/workflows/e2e-test-releases.yml | 66 ++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml index d392304409..5a30c5480d 100644 --- a/.github/workflows/e2e-test-releases.yml +++ b/.github/workflows/e2e-test-releases.yml @@ -27,18 +27,72 @@ on: required: false # before merge to main, set to true type: string +concurrency: + group: "${{ github.workflow }}-${{ github.event.number || github.ref }}" + cancel-in-progress: true + defaults: run: shell: bash jobs: - release-test: - name: Release test + set-vars: + name: Set vars runs-on: ubuntu-latest + outputs: + date_start: ${{ steps.vars.outputs.date-start }} + randuuid4c: ${{ steps.vars.outputs.randuuid4c }} steps: - - name: Test + - name: Set vars + id: vars run: | - echo "🏷️ [INFO] Current release tag: ${{ github.event.inputs.current-release }}" - echo "🔜 [INFO] Next release tag: ${{ github.event.inputs.next-release }}" + echo "date-start=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_OUTPUT + echo "randuuid4c=$(openssl rand -hex 2)" >> $GITHUB_OUTPUT + + e2e-replicated: + name: E2E Release Pipeline (Replicated) + needs: + - set-vars + uses: ./.github/workflows/e2e-test-releases-reusable-pipeline.yml + with: + current_release: ${{ github.event.inputs.current-release }} + new_release: ${{ github.event.inputs.next-release }} + storage_type: replicated + nested_storageclass_name: nested-thin-r1 + branch: main + deckhouse_channel: alpha + default_user: cloud + go_version: "1.25.8" + date_start: ${{ needs.set-vars.outputs.date_start }} + randuuid4c: ${{ needs.set-vars.outputs.randuuid4c }} + cluster_config_workers_memory: "9Gi" + cluster_config_k8s_version: "1.34" + secrets: + DEV_REGISTRY_DOCKER_CFG: ${{ secrets.DEV_REGISTRY_DOCKER_CFG }} + VIRT_E2E_NIGHTLY_SA_TOKEN: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} + PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} + BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} - echo "🎉 [INFO] Test from ${{ github.event.inputs.current-release }} -> ${{ github.event.inputs.next-release }} SUCCESS" + e2e-nfs: + name: E2E Release Pipeline (NFS) + needs: + - set-vars + uses: ./.github/workflows/e2e-test-releases-reusable-pipeline.yml + with: + current_release: ${{ github.event.inputs.current-release }} + new_release: ${{ github.event.inputs.next-release }} + storage_type: nfs + nested_storageclass_name: nfs + branch: main + deckhouse_channel: alpha + default_user: cloud + go_version: "1.24.13" + date_start: ${{ needs.set-vars.outputs.date_start }} + randuuid4c: ${{ needs.set-vars.outputs.randuuid4c }} + cluster_config_workers_memory: "9Gi" + cluster_config_k8s_version: "Automatic" + secrets: + DEV_REGISTRY_DOCKER_CFG: ${{ secrets.DEV_REGISTRY_DOCKER_CFG }} + VIRT_E2E_NIGHTLY_SA_TOKEN: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} + PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} + BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} From 17e9178132da9773bb26ff1df574acac09b5de93 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Wed, 8 Apr 2026 19:27:47 +0300 Subject: [PATCH 02/16] add draft Signed-off-by: Nikita Korolev --- .../e2e-test-releases-reusable-pipeline.yml | 1213 ++++++++++++++++- 1 file changed, 1206 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 1ca30d5199..eba7e4b691 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -17,21 +17,1220 @@ name: E2E Release Test Reusable Pipeline on: workflow_call: inputs: - message: - description: "placeholder message" + current_release: + required: true + type: string + description: "Current virtualization release tag (e.g. v1.4.1)" + new_release: + required: true + type: string + description: "New virtualization release tag to upgrade to (e.g. v1.5.0)" + date_start: + required: true + type: string + description: "Date start" + randuuid4c: + required: true + type: string + description: "Random UUID first 4 chars" + cluster_config_k8s_version: + required: false + type: string + default: "Automatic" + description: "Set k8s version for cluster config, like 1.34, 1.36 (without patch version)" + cluster_config_workers_memory: + required: false + type: string + default: "8Gi" + description: "Set memory for workers node in cluster config" + storage_type: + required: true + type: string + description: "Storage type (replicated or nfs)" + nested_storageclass_name: + required: true + type: string + description: "Nested storage class name" + branch: + required: false + type: string + default: "main" + description: "Branch to use" + virtualization_image_url: + required: false + type: string + default: "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" + description: "Virtualization image url (default noble-server-cloudimg-amd64.img)" + deckhouse_channel: required: false - default: "placeholder message" type: string + default: "alpha" + description: "Deckhouse release channel" + pod_subnet_cidr: + required: false + type: string + default: "10.88.0.0/16" + description: "Pod subnet CIDR" + service_subnet_cidr: + required: false + type: string + default: "10.99.0.0/16" + description: "Service subnet CIDR" + default_user: + required: false + type: string + default: "ubuntu" + description: "Default user for vms" + go_version: + required: false + type: string + default: "1.24.6" + description: "Go version" + secrets: + DEV_REGISTRY_DOCKER_CFG: + required: true + VIRT_E2E_NIGHTLY_SA_TOKEN: + required: true + PROD_IO_REGISTRY_DOCKER_CFG: + required: true + BOOTSTRAP_DEV_PROXY: + required: true + +env: + BRANCH: ${{ inputs.branch }} + CURRENT_RELEASE: ${{ inputs.current_release }} + NEW_RELEASE: ${{ inputs.new_release }} + DECKHOUSE_CHANNEL: ${{ inputs.deckhouse_channel }} + DEFAULT_USER: ${{ inputs.default_user }} + GO_VERSION: ${{ inputs.go_version }} + SETUP_CLUSTER_TYPE_PATH: test/dvp-static-cluster + K8S_VERSION: ${{ inputs.cluster_config_k8s_version }} defaults: run: shell: bash jobs: - release-test: - name: Release test + bootstrap: + name: Bootstrap cluster + runs-on: ubuntu-latest + concurrency: + group: "${{ github.workflow }}-${{ github.event.number || github.ref }}-${{ inputs.storage_type }}" + cancel-in-progress: true + outputs: + kubeconfig: ${{ steps.generate-kubeconfig.outputs.kubeconfig }} + namespace: ${{ steps.vars.outputs.namespace }} + steps: + - uses: actions/checkout@v4 + + - name: Set outputs + env: + RANDUUID4C: ${{ inputs.randuuid4c }} + STORAGE_TYPE: ${{ inputs.storage_type }} + id: vars + run: | + GIT_SHORT_HASH=$(git rev-parse --short HEAD) + + namespace="nightly-e2e-$STORAGE_TYPE-$GIT_SHORT_HASH-$RANDUUID4C" + + echo "namespace=$namespace" >> $GITHUB_OUTPUT + echo "sha_short=$GIT_SHORT_HASH" >> $GITHUB_OUTPUT + + REGISTRY=$(base64 -d <<< ${{secrets.PROD_IO_REGISTRY_DOCKER_CFG}} | jq '.auths | to_entries | .[] | .key' -r) + USERNAME=$(base64 -d <<< ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} | jq '.auths | to_entries | .[] | .value.auth' -r | base64 -d | cut -d ':' -f1) + PASSWORD=$(base64 -d <<< ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} | jq '.auths | to_entries | .[] | .value.auth' -r | base64 -d | cut -d ':' -f2) + + echo "registry=$REGISTRY" >> $GITHUB_OUTPUT + echo "username=$USERNAME" >> $GITHUB_OUTPUT + echo "password=$PASSWORD" >> $GITHUB_OUTPUT + + - name: Install htpasswd utility + run: | + sudo apt-get update + sudo apt-get install -y apache2-utils + + - name: Install Task + uses: arduino/setup-task@v2 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup d8 + uses: ./.github/actions/install-d8 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Log in to private registry + uses: docker/login-action@v3 + with: + registry: ${{ steps.vars.outputs.registry }} + username: ${{ steps.vars.outputs.username }} + password: ${{ steps.vars.outputs.password }} + + - name: Configure kubectl via azure/k8s-set-context@v4 + uses: azure/k8s-set-context@v4 + with: + method: kubeconfig + context: e2e-cluster-nightly-e2e-virt-sa + kubeconfig: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} + + - name: Generate values.yaml + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} + run: | + defaultStorageClass=$(kubectl get storageclass -o json \ + | jq -r '.items[] | select(.metadata.annotations."storageclass.kubernetes.io/is-default-class" == "true") | .metadata.name') + + cat < values.yaml + namespace: ${{ steps.vars.outputs.namespace }} + storageType: ${{ inputs.storage_type }} + storageClass: ${defaultStorageClass} + sa: dkp-sa + deckhouse: + channel: ${{ env.DECKHOUSE_CHANNEL }} + podSubnetCIDR: ${{ inputs.pod_subnet_cidr }} + serviceSubnetCIDR: ${{ inputs.service_subnet_cidr }} + kubernetesVersion: ${{ env.K8S_VERSION }} + registryDockerCfg: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} + bundle: Default + proxyEnabled: false + image: + url: ${{ inputs.virtualization_image_url }} + defaultUser: ${{ env.DEFAULT_USER }} + bootloader: BIOS + ingressHosts: + - api + - grafana + - dex + - prometheus + - console + - virtualization + instances: + masterNodes: + count: 1 + cfg: + rootDiskSize: 60Gi + cpu: + cores: 4 + coreFraction: 50% + memory: + size: 12Gi + additionalNodes: + - name: worker + count: 3 + cfg: + cpu: + cores: 6 + coreFraction: 50% + memory: + size: ${{ inputs.cluster_config_workers_memory }} + additionalDisks: + - size: 50Gi + EOF + + mkdir -p tmp + touch tmp/discovered-values.yaml + + export REGISTRY=$(base64 -d <<< ${{secrets.DEV_REGISTRY_DOCKER_CFG}} | jq '.auths | to_entries | .[] | .key' -r) + export AUTH=$(base64 -d <<< ${{ secrets.DEV_REGISTRY_DOCKER_CFG }} | jq '.auths | to_entries | .[] | .value.auth' -r) + + yq eval --inplace '.discovered.registry_url = env(REGISTRY)' tmp/discovered-values.yaml + yq eval --inplace '.discovered.registry_auth = env(AUTH)' tmp/discovered-values.yaml + + - name: Bootstrap cluster [infra-deploy] + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} + run: | + task infra-deploy + - name: Bootstrap cluster [dhctl-bootstrap] + id: dhctl-bootstrap + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} + env: + HTTP_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} + HTTPS_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} + run: | + task dhctl-bootstrap + timeout-minutes: 30 + - name: Bootstrap cluster [show-connection-info] + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} + run: | + task show-connection-info + + - name: Save ssh to secrets in cluster + env: + NAMESPACE: ${{ steps.vars.outputs.namespace }} + if: always() && steps.dhctl-bootstrap.outcome == 'success' + run: | + kubectl -n $NAMESPACE create secret generic ssh-key --from-file=${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/ssh/cloud + + - name: Get info about nested cluster and master VM + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} + env: + NAMESPACE: ${{ steps.vars.outputs.namespace }} + PREFIX: ${{ inputs.storage_type }} + run: | + nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") + + d8vssh() { + local host=$1 + local cmd=$2 + d8 v ssh -i ./tmp/ssh/cloud \ + --local-ssh=true \ + --local-ssh-opts="-o StrictHostKeyChecking=no" \ + --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ + ${DEFAULT_USER}@${host}.${NAMESPACE} \ + -c "$cmd" + } + + echo "[INFO] Pods in namespace $NAMESPACE" + kubectl get pods -n "${NAMESPACE}" + echo "" + + echo "[INFO] VMs in namespace $NAMESPACE" + kubectl get vm -n "${NAMESPACE}" + echo "" + + echo "[INFO] VDs in namespace $NAMESPACE" + kubectl get vd -n "${NAMESPACE}" + echo "" + + echo "Check connection to master" + d8vssh "${nested_master}" 'echo master os-release: ; cat /etc/os-release; echo " "; echo master hostname: ; hostname' + echo "" + + - name: Generate nested kubeconfig + id: generate-kubeconfig + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} + env: + kubeConfigPath: tmp/kube.config + NAMESPACE: ${{ steps.vars.outputs.namespace }} + PREFIX: ${{ inputs.storage_type }} + run: | + nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") + + d8vscp() { + local source=$1 + local dest=$2 + d8 v scp -i ./tmp/ssh/cloud \ + --local-ssh=true \ + --local-ssh-opts="-o StrictHostKeyChecking=no" \ + --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ + "$source" "$dest" + echo "d8vscp: $source -> $dest - done" + } + + d8vssh() { + local cmd=$1 + d8 v ssh -i ./tmp/ssh/cloud \ + --local-ssh=true \ + --local-ssh-opts="-o StrictHostKeyChecking=no" \ + --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ + ${DEFAULT_USER}@${nested_master}.${NAMESPACE} \ + -c "$cmd" + } + + echo "[INFO] Copy script for generating kubeconfig in nested cluster" + echo "[INFO] Copy scripts/gen-kubeconfig.sh to master" + d8vscp "./scripts/gen-kubeconfig.sh" "${DEFAULT_USER}@${nested_master}.${NAMESPACE}:/tmp/gen-kubeconfig.sh" + echo "" + d8vscp "./scripts/deckhouse-queue.sh" "${DEFAULT_USER}@${nested_master}.${NAMESPACE}:/tmp/deckhouse-queue.sh" + echo "" + + echo "[INFO] Set file exec permissions" + d8vssh 'chmod +x /tmp/{gen-kubeconfig.sh,deckhouse-queue.sh}' + d8vssh 'ls -la /tmp/' + echo "[INFO] Check d8 queue in nested cluster" + d8vssh 'sudo /tmp/deckhouse-queue.sh' + + echo "[INFO] Generate kube conf in nested cluster" + echo "[INFO] Run gen-kubeconfig.sh in nested cluster" + d8vssh "sudo /tmp/gen-kubeconfig.sh nested-sa nested nested-e2e /${kubeConfigPath}" + echo "" + + echo "[INFO] Copy kubeconfig to runner" + echo "[INFO] ${DEFAULT_USER}@${nested_master}.$NAMESPACE:/${kubeConfigPath} ./${kubeConfigPath}" + d8vscp "${DEFAULT_USER}@${nested_master}.$NAMESPACE:/${kubeConfigPath}" "./${kubeConfigPath}" + + echo "[INFO] Set rights for kubeconfig" + echo "[INFO] sudo chown 1001:1001 ${kubeConfigPath}" + sudo chown 1001:1001 ${kubeConfigPath} + echo " " + + echo "[INFO] Kubeconf to github output" + CONFIG=$(cat ${kubeConfigPath} | base64 -w 0) + CONFIG=$(echo $CONFIG | base64 -w 0) + echo "kubeconfig=$CONFIG" >> $GITHUB_OUTPUT + + - name: cloud-init logs + if: steps.dhctl-bootstrap.outcome == 'failure' + env: + NAMESPACE: ${{ steps.vars.outputs.namespace }} + PREFIX: ${{ inputs.storage_type }} + run: | + nested_master=$(kubectl -n ${NAMESPACE} get vm -l group=${PREFIX}-master -o jsonpath="{.items[0].metadata.name}") + + d8vscp() { + local source=$1 + local dest=$2 + d8 v scp -i ./tmp/ssh/cloud \ + --local-ssh=true \ + --local-ssh-opts="-o StrictHostKeyChecking=no" \ + --local-ssh-opts="-o UserKnownHostsFile=/dev/null" \ + "$source" "$dest" + echo "d8vscp: $source -> $dest - done" + } + + d8vscp "${DEFAULT_USER}@${nested_master}.$NAMESPACE:/var/log/cloud-init*.log" "./${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/" + + - name: Prepare artifact + if: always() && steps.dhctl-bootstrap.outcome == 'success' + run: | + sudo chown -fR 1001:1001 ${{ env.SETUP_CLUSTER_TYPE_PATH }} + yq e '.deckhouse.registryDockerCfg = "None"' -i ./${{ env.SETUP_CLUSTER_TYPE_PATH }}/values.yaml + yq e 'select(.kind == "InitConfiguration").deckhouse.registryDockerCfg = "None"' -i ./${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/config.yaml || echo "The config.yaml file is not generated, skipping" + yq e '.discovered.registry_url = "None"' -i ./${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/discovered-values.yaml || echo "The discovered-values.yaml file is not generated, skipping editing registry_url" + yq e '.discovered.registry_auth = "None"' -i ./${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/discovered-values.yaml || echo "The discovered-values.yaml file is not generated, skipping editing registry_auth" + echo "${{ steps.generate-kubeconfig.outputs.kubeconfig }}" | base64 -d | base64 -d > ./${{ env.SETUP_CLUSTER_TYPE_PATH }}/kube-config + + - name: Upload generated files + uses: actions/upload-artifact@v4 + if: always() && steps.dhctl-bootstrap.outcome == 'success' + with: + name: ${{ inputs.storage_type }}-release-generated-files-${{ inputs.date_start }} + path: | + ${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp + ${{ env.SETUP_CLUSTER_TYPE_PATH }}/values.yaml + overwrite: true + include-hidden-files: true + retention-days: 3 + + - name: Upload ssh config + uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ inputs.storage_type }}-release-generated-files-ssh-${{ inputs.date_start }} + path: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/tmp/ssh + overwrite: true + include-hidden-files: true + retention-days: 3 + + - name: Upload kubeconfig + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.storage_type }}-release-generated-files-kubeconfig-${{ inputs.date_start }} + path: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/kube-config + overwrite: true + include-hidden-files: true + retention-days: 3 + + configure-storage: + name: Configure storage + runs-on: ubuntu-latest + needs: bootstrap + steps: + - uses: actions/checkout@v4 + + - name: Install Task + uses: arduino/setup-task@v2 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup d8 + uses: ./.github/actions/install-d8 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Install kubectl CLI + uses: azure/setup-kubectl@v4 + + - name: Check nested kube-api via generated kubeconfig + run: | + mkdir -p ~/.kube + echo "[INFO] Configure kubeconfig for nested cluster" + echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config + + echo "[INFO] Show paths and files content" + ls -la ~/.kube + echo "[INFO] Set permissions for kubeconfig" + chmod 600 ~/.kube/config + + echo "[INFO] Show current kubeconfig context" + kubectl config get-contexts + + echo "[INFO] Show nodes in cluster" + count=30 + success=false + for i in $(seq 1 $count); do + echo "[INFO] Attempt $i/$count..." + if kubectl get nodes; then + echo "[SUCCESS] Successfully retrieved nodes." + success=true + break + fi + + if [ $i -lt $count ]; then + echo "[INFO] Retrying in 10 seconds..." + sleep 10 + fi + done + + if [ "$success" = false ]; then + echo "[ERROR] Failed to retrieve nodes after $count attempts." + exit 1 + fi + + - name: Configure replicated storage + id: storage-replicated-setup + if: ${{ inputs.storage_type == 'replicated' }} + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/sds-replicated + run: | + d8_queue_list() { + d8 s queue list | grep -Po '([0-9]+)(?= active)' || echo "[WARNING] Failed to retrieve list queue" + } + + d8_queue() { + local count=90 + local queue_count + + for i in $(seq 1 $count) ; do + queue_count=$(d8_queue_list) + if [ -n "$queue_count" ] && [ "$queue_count" = "0" ]; then + echo "[SUCCESS] Queue is clear" + return 0 + fi + + echo "[INFO] Wait until queues are empty ${i}/${count}" + if (( i % 5 == 0 )); then + echo "[INFO] Show queue list" + d8 s queue list | head -n25 || echo "[WARNING] Failed to retrieve list queue" + echo " " + fi + + if (( i % 10 == 0 )); then + echo "[INFO] deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + echo " " + fi + sleep 10 + done + } + + sds_replicated_ready() { + local count=60 + for i in $(seq 1 $count); do + + sds_replicated_volume_status=$(kubectl get ns d8-sds-replicated-volume -o jsonpath='{.status.phase}' || echo "False") + + if [[ "${sds_replicated_volume_status}" = "Active" ]]; then + echo "[SUCCESS] Namespaces sds-replicated-volume are Active" + kubectl get ns d8-sds-replicated-volume + return 0 + fi + + echo "[INFO] Waiting 10s for sds-replicated-volume namespace to be ready (attempt ${i}/${count})" + if (( i % 5 == 0 )); then + echo "[INFO] Show namespaces sds-replicated-volume" + kubectl get ns | grep sds-replicated-volume || echo "Namespaces sds-replicated-volume are not ready" + echo "[DEBUG] Show queue (first 25 lines)" + d8 s queue list | head -n25 || echo "No queues" + fi + sleep 10 + done + + echo "[ERROR] Namespaces sds-replicated-volume are not ready after ${count} attempts" + echo "[DEBUG] Show namespaces sds" + kubectl get ns | grep sds || echo "Namespaces sds-replicated-volume are not ready" + echo "[DEBUG] Show queue" + echo "::group::Show queue" + d8 s queue list || echo "No queues" + echo "::endgroup::" + echo "[DEBUG] Show deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + exit 1 + } + + sds_pods_ready() { + local count=100 + local linstor_node + local csi_node + local webhooks + local workers=$(kubectl get nodes -o name | grep worker | wc -l || true) + workers=$((workers)) + + echo "[INFO] Wait while linstor-node csi-node webhooks pods are ready" + for i in $(seq 1 $count); do + linstor_node=$(kubectl -n d8-sds-replicated-volume get pods | grep "linstor-node.*Running" | wc -l || true) + csi_node=$(kubectl -n d8-sds-replicated-volume get pods | grep "csi-node.*Running" | wc -l || true) + + echo "[INFO] Check if sds-replicated pods are ready" + if [[ ${linstor_node} -ge ${workers} && ${csi_node} -ge ${workers} ]]; then + echo "[SUCCESS] sds-replicated-volume is ready" + return 0 + fi + + echo "[WARNING] Not all pods are ready, linstor_node=${linstor_node}, csi_node=${csi_node}" + echo "[INFO] Waiting 10s for pods to be ready (attempt ${i}/${count})" + if (( i % 5 == 0 )); then + echo "[DEBUG] Get pods" + kubectl -n d8-sds-replicated-volume get pods || true + echo "[DEBUG] Show queue (first 25 lines)" + d8 s queue list | head -n 25 || echo "Failed to retrieve list queue" + echo " " + fi + sleep 10 + done + + echo "[ERROR] sds-replicated-volume is not ready after ${count} attempts" + echo "[DEBUG] Get pods" + echo "::group::sds-replicated-volume pods" + kubectl -n d8-sds-replicated-volume get pods || true + echo "::endgroup::" + echo "[DEBUG] Show queue" + echo "::group::Show queue" + d8 s queue list || echo "Failed to retrieve list queue" + echo "::endgroup::" + echo "[DEBUG] Show deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + exit 1 + } + + blockdevices_ready() { + local count=60 + workers=$(kubectl get nodes -o name | grep worker | wc -l) + workers=$((workers)) + + if [[ $workers -eq 0 ]]; then + echo "[ERROR] No worker nodes found" + exit 1 + fi + + for i in $(seq 1 $count); do + blockdevices=$(kubectl get blockdevice -o name | wc -l || true) + if [ $blockdevices -ge $workers ]; then + echo "[SUCCESS] Blockdevices is greater or equal to $workers" + kubectl get blockdevice + return 0 + fi + + echo "[INFO] Wait 10 sec until blockdevices is greater or equal to $workers (attempt ${i}/${count})" + if (( i % 5 == 0 )); then + echo "[DEBUG] Show queue (first 25 lines)" + d8 s queue list | head -n25 || echo "No queues" + fi + + sleep 10 + done + + echo "[ERROR] Blockdevices is not 3" + echo "[DEBUG] Show cluster nodes" + kubectl get nodes + echo "[DEBUG] Show blockdevices" + kubectl get blockdevice + echo "[DEBUG] Show sds namespaces" + kubectl get ns | grep sds || echo "ns sds is not found" + echo "[DEBUG] Show pods in sds-replicated-volume" + echo "::group::pods in sds-replicated-volume" + kubectl -n d8-sds-replicated-volume get pods || true + echo "::endgroup::" + echo "[DEBUG] Show deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + exit 1 + } + + d8_queue + + kubectl apply -f mc.yaml + echo "[INFO] Wait for sds-node-configurator" + kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sds-node-configurator --timeout=300s + + echo "[INFO] Wait for sds-replicated-volume to be ready" + sds_replicated_ready + kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sds-replicated-volume --timeout=300s + + echo "[INFO] Wait BlockDevice are ready" + blockdevices_ready + + echo "[INFO] Wait pods and webhooks sds-replicated pods" + sds_pods_ready + + chmod +x lvg-gen.sh + ./lvg-gen.sh + + chmod +x rsc-gen.sh + ./rsc-gen.sh + + echo "[INFO] Show existing storageclasses" + if ! kubectl get storageclass | grep -q nested; then + echo "[WARNING] No nested storageclasses" + else + kubectl get storageclass | grep nested + echo "[SUCCESS] Done" + fi + + - name: Configure NFS storage + if: ${{ inputs.storage_type == 'nfs' }} + id: storage-nfs-setup + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/nfs + env: + NAMESPACE: ${{ needs.bootstrap.outputs.namespace }} + run: | + nfs_ready() { + local count=90 + local controller + local csi_controller + local csi_node_desired + local csi_node_ready + + for i in $(seq 1 $count); do + echo "[INFO] Check d8-csi-nfs pods (attempt ${i}/${count})" + controller=$(kubectl -n d8-csi-nfs get deploy controller -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0") + csi_controller=$(kubectl -n d8-csi-nfs get deploy csi-controller -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0") + csi_node_desired=$(kubectl -n d8-csi-nfs get ds csi-node -o jsonpath='{.status.desiredNumberScheduled}' 2>/dev/null || echo "0") + csi_node_ready=$(kubectl -n d8-csi-nfs get ds csi-node -o jsonpath='{.status.numberReady}' 2>/dev/null || echo "0") + + if [[ "$controller" -ge 1 && "$csi_controller" -ge 1 && "$csi_node_desired" -gt 0 && "$csi_node_ready" -eq "$csi_node_desired" ]]; then + echo "[SUCCESS] NFS CSI is ready (controller=${controller}, csi-controller=${csi_controller}, csi-node=${csi_node_ready}/${csi_node_desired})" + return 0 + fi + + echo "[WARNING] NFS CSI not ready: controller=${controller}, csi-controller=${csi_controller}, csi-node=${csi_node_ready}/${csi_node_desired}" + if (( i % 5 == 0 )); then + echo "[DEBUG] Pods in d8-csi-nfs:" + kubectl -n d8-csi-nfs get pods || echo "[WARNING] Failed to retrieve pods" + echo "[DEBUG] Deployments in d8-csi-nfs:" + kubectl -n d8-csi-nfs get deploy || echo "[WARNING] Failed to retrieve deployments" + echo "[DEBUG] DaemonSets in d8-csi-nfs:" + kubectl -n d8-csi-nfs get ds || echo "[WARNING] Failed to retrieve daemonsets" + echo "[DEBUG] csi-nfs module status:" + kubectl get modules csi-nfs -o wide || echo "[WARNING] Failed to retrieve module" + fi + sleep 10 + done + + echo "[ERROR] NFS CSI did not become ready in time" + kubectl -n d8-csi-nfs get pods || true + exit 1 + } + + echo "[INFO] Apply csi-nfs ModuleConfig, ModulePullOverride, snapshot-controller" + kubectl apply -f mc.yaml + + echo "[INFO] Wait for csi-nfs module to be ready" + kubectl wait --for=jsonpath='{.status.phase}'=Ready modules csi-nfs --timeout=300s + + echo "[INFO] Wait for csi-nfs pods to be ready" + nfs_ready + + echo "[INFO] Apply NFSStorageClass" + envsubst < storageclass.yaml | kubectl apply -f - + + echo "[INFO] Configure default storage class" + ./default-sc-configure.sh + + + echo "[INFO] Show existing storageclasses" + kubectl get storageclass + + configure-virtualization: + name: Configure Virtualization (current-release) + runs-on: ubuntu-latest + needs: + - bootstrap + - configure-storage + steps: + - uses: actions/checkout@v4 + - name: Install kubectl CLI + uses: azure/setup-kubectl@v4 + - name: Setup d8 + uses: ./.github/actions/install-d8 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Check kubeconfig + run: | + echo "[INFO] Configure kube config" + mkdir -p ~/.kube + echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config + chmod 600 ~/.kube/config + kubectl config use-context nested-e2e-nested-sa + + - name: Configure Virtualization + run: | + REGISTRY=$(base64 -d <<< "${{secrets.DEV_REGISTRY_DOCKER_CFG}}" | jq '.auths | to_entries | .[] | .key' -r) + + echo "[INFO] Apply ModuleSource prod config" + kubectl apply -f -< ~/.kube/config + chmod 600 ~/.kube/config + + - name: "Run E2E tests on current-release (STUB)" + run: | + echo "[INFO] Current release tag: ${{ env.CURRENT_RELEASE }}" + echo "[INFO] Storage type: ${{ inputs.storage_type }}" + echo "" + echo "[INFO] Verifying virtualization module is running" + kubectl get modules virtualization || true + kubectl get mpo virtualization || true + echo "" + echo "[STUB] E2E tests on current-release ${{ env.CURRENT_RELEASE }} -- PASSED" + echo "[INFO] Resources are intentionally left in the cluster for the upgrade test" + + patch-mpo: + name: "Patch MPO to new-release: ${{ inputs.new_release }}" + runs-on: ubuntu-latest + needs: + - bootstrap + - e2e-test-current + steps: + - uses: actions/checkout@v4 + + - name: Install kubectl CLI + uses: azure/setup-kubectl@v4 + + - name: Setup d8 + uses: ./.github/actions/install-d8 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup kubeconfig + run: | + mkdir -p ~/.kube + echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config + chmod 600 ~/.kube/config + kubectl config use-context nested-e2e-nested-sa + + - name: Show current MPO state + run: | + echo "[INFO] Current ModulePullOverride before patching:" + kubectl get mpo virtualization -o yaml + + - name: "Patch ModulePullOverride to new-release: ${{ env.NEW_RELEASE }}" + run: | + echo "[INFO] Patching MPO virtualization imageTag from ${{ env.CURRENT_RELEASE }} to ${{ env.NEW_RELEASE }}" + kubectl patch mpo virtualization --type merge -p '{"spec":{"imageTag":"${{ env.NEW_RELEASE }}"}}' + + echo "[INFO] Show patched ModulePullOverride:" + kubectl get mpo virtualization -o yaml + + - name: Wait for Virtualization to be ready after upgrade + run: | + d8_queue_list() { + d8 s queue list | grep -Po '([0-9]+)(?= active)' || echo "Failed to retrieve list queue" + } + + debug_output() { + local NODES + + echo "[ERROR] Virtualization module upgrade failed" + echo "[DEBUG] Show describe virtualization module" + echo "::group::describe virtualization module" + kubectl describe modules virtualization || true + echo "::endgroup::" + echo "[DEBUG] Show namespace d8-virtualization" + kubectl get ns d8-virtualization || true + echo "[DEBUG] Show pods in namespace d8-virtualization" + kubectl -n d8-virtualization get pods || true + echo "[DEBUG] Show pvc in namespace d8-virtualization" + kubectl get pvc -n d8-virtualization || true + echo "[DEBUG] Show cluster StorageClasses" + kubectl get storageclasses || true + echo "[DEBUG] Show cluster nodes" + kubectl get node + echo "[DEBUG] Show queue (first 25 lines)" + d8 s queue list | head -n 25 || echo "[WARNING] Failed to retrieve list queue" + echo "[DEBUG] Show deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + } + + d8_queue() { + local count=90 + local queue_count + + for i in $(seq 1 $count) ; do + queue_count=$(d8_queue_list) + if [ -n "$queue_count" ] && [ "$queue_count" = "0" ]; then + echo "[SUCCESS] Queue is clear" + return 0 + fi + + echo "[INFO] Wait until queues are empty ${i}/${count}" + if (( i % 5 == 0 )); then + echo "[INFO] Show queue list" + d8 s queue list | head -n25 || echo "[WARNING] Failed to retrieve list queue" + echo " " + fi + + if (( i % 10 == 0 )); then + echo "[INFO] deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + echo " " + fi + sleep 10 + done + } + + virtualization_ready() { + local count=90 + local virtualization_status + + for i in $(seq 1 $count) ; do + virtualization_status=$(kubectl get modules virtualization -o jsonpath='{.status.phase}') + if [ "$virtualization_status" == "Ready" ]; then + echo "[SUCCESS] Virtualization module is ready after upgrade to ${{ env.NEW_RELEASE }}" + kubectl get modules virtualization + kubectl -n d8-virtualization get pods + kubectl get vmclass || echo "[WARNING] no vmclasses found" + return 0 + fi + + echo "[INFO] Waiting 10s for Virtualization module to be ready (attempt $i/$count)" + + if (( i % 5 == 0 )); then + echo " " + echo "[DEBUG] Show additional info" + kubectl get ns d8-virtualization || echo "[WARNING] Namespace virtualization is not ready" + echo " " + kubectl -n d8-virtualization get pods || echo "[WARNING] Pods in namespace virtualization is not ready" + kubectl get pvc -n d8-virtualization || echo "[WARNING] PVC in namespace virtualization is not ready" + echo " " + fi + sleep 10 + done + + debug_output + exit 1 + } + + virt_handler_ready() { + local count=180 + local virt_handler_ready + local workers + local time_wait=10 + + workers=$(kubectl get nodes -o name | grep worker | wc -l || true) + workers=$((workers)) + + for i in $(seq 1 $count); do + virt_handler_ready=$(kubectl -n d8-virtualization get pods | grep "virt-handler.*Running" | wc -l || true) + + if [[ $virt_handler_ready -ge $workers ]]; then + echo "[SUCCESS] virt-handlers pods are ready" + return 0 + fi + + echo "[INFO] virt-handler pods $virt_handler_ready/$workers " + echo "[INFO] Wait ${time_wait}s virt-handler pods are ready (attempt $i/$count)" + if (( i % 5 == 0 )); then + echo "[DEBUG] Show pods in namespace d8-virtualization" + echo "::group::virtualization pods" + kubectl -n d8-virtualization get pods || echo "No pods in virtualization namespace found" + echo "::endgroup::" + echo "[DEBUG] Show cluster nodes" + echo "::group::cluster nodes" + kubectl get node + echo "::endgroup::" + fi + sleep ${time_wait} + done + + debug_output + exit 1 + } + + echo " " + echo "[INFO] Waiting for Virtualization module to be ready after upgrade to ${{ env.NEW_RELEASE }}" + d8_queue + + virtualization_ready + + echo "[INFO] Checking Virtualization module deployments" + kubectl -n d8-virtualization wait --for=condition=Available deploy --all --timeout 900s + echo "[INFO] Checking virt-handler pods" + virt_handler_ready + + - name: Show MPO state after upgrade + run: | + echo "[INFO] ModulePullOverride after upgrade:" + kubectl get mpo virtualization -o yaml + echo "[INFO] Virtualization module status:" + kubectl get modules virtualization + + e2e-test-new: + name: "E2E test (new-release: ${{ inputs.new_release }})" runs-on: ubuntu-latest + needs: + - bootstrap + - patch-mpo steps: - - name: Print message + - uses: actions/checkout@v4 + + - name: Install kubectl CLI + uses: azure/setup-kubectl@v4 + + - name: Setup kubeconfig + run: | + mkdir -p ~/.kube + echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config + chmod 600 ~/.kube/config + + - name: "Run E2E tests on new-release (STUB)" run: | - echo "${{ inputs.message }}" + echo "[INFO] New release tag: ${{ env.NEW_RELEASE }}" + echo "[INFO] Storage type: ${{ inputs.storage_type }}" + echo "" + echo "[INFO] Verifying virtualization module is running with new release" + kubectl get modules virtualization || true + kubectl get mpo virtualization || true + echo "" + echo "[STUB] E2E tests on new-release ${{ env.NEW_RELEASE }} -- PASSED" + echo "[INFO] Cluster is intentionally left running (no cleanup)" From 4c03f79f58e09d274b8ed93c97ec9009427ed6b0 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 9 Apr 2026 15:11:26 +0300 Subject: [PATCH 03/16] add sds local Signed-off-by: Nikita Korolev --- .../e2e-test-releases-reusable-pipeline.yml | 129 +++++++++++++++++- .github/workflows/e2e-test-releases.yml | 29 +--- .../charts/infra/templates/nfs/svc.yaml | 2 +- .../charts/infra/templates/nfs/vm.yaml | 2 +- .../storage/sds-local-volume/lsc-gen.sh | 109 +++++++++++++++ .../storage/sds-local-volume/mc.yaml | 8 ++ 6 files changed, 244 insertions(+), 35 deletions(-) create mode 100644 test/dvp-static-cluster/storage/sds-local-volume/lsc-gen.sh create mode 100644 test/dvp-static-cluster/storage/sds-local-volume/mc.yaml diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index eba7e4b691..1424388080 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -224,7 +224,7 @@ jobs: memory: size: ${{ inputs.cluster_config_workers_memory }} additionalDisks: - - size: 50Gi + - size: 100Gi EOF mkdir -p tmp @@ -480,7 +480,7 @@ jobs: - name: Configure replicated storage id: storage-replicated-setup - if: ${{ inputs.storage_type == 'replicated' }} + if: ${{ inputs.storage_type == 'replicated' || inputs.storage_type == 'mixed'}} working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/sds-replicated run: | d8_queue_list() { @@ -644,6 +644,18 @@ jobs: exit 1 } + lvg_ready() { + echo "[INFO] Wait for generated LVMVolumeGroup resources to be Ready" + if ! kubectl wait -f sds-lvg.yaml --for=jsonpath='{.status.phase}'=Ready --timeout=300s; then + echo "[ERROR] Generated LVMVolumeGroup resources did not become Ready" + kubectl get lvmvolumegroup -o wide || true + exit 1 + fi + + echo "[SUCCESS] Generated LVMVolumeGroup resources are Ready" + kubectl get lvmvolumegroup -o wide + } + d8_queue kubectl apply -f mc.yaml @@ -662,6 +674,7 @@ jobs: chmod +x lvg-gen.sh ./lvg-gen.sh + lvg_ready chmod +x rsc-gen.sh ./rsc-gen.sh @@ -674,12 +687,113 @@ jobs: echo "[SUCCESS] Done" fi + - name: Configure sds-local-volume + if: ${{ inputs.storage_type == 'local' || inputs.storage_type == 'mixed' }} + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/sds-local-volume + run: | + d8_queue_list() { + d8 s queue list | grep -Po '([0-9]+)(?= active)' || echo "[WARNING] Failed to retrieve list queue" + } + + d8_queue() { + local count=90 + local queue_count + + for i in $(seq 1 $count) ; do + queue_count=$(d8_queue_list) + if [ -n "$queue_count" ] && [ "$queue_count" = "0" ]; then + echo "[SUCCESS] Queue is clear" + return 0 + fi + + echo "[INFO] Wait until queues are empty ${i}/${count}" + if (( i % 5 == 0 )); then + echo "[INFO] Show queue list" + d8 s queue list | head -n25 || echo "[WARNING] Failed to retrieve list queue" + echo " " + fi + + if (( i % 10 == 0 )); then + echo "[INFO] deckhouse logs" + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + echo " " + fi + sleep 10 + done + } + + sds_local_volume_ready() { + local count=90 + local local_volume_status + local csi_node_desired + local csi_node_ready + local deploy_count + local controller_ready + + for i in $(seq 1 $count); do + local_volume_status=$(kubectl get modules sds-local-volume -o jsonpath='{.status.phase}' 2>/dev/null || echo "False") + csi_node_desired=$(kubectl -n d8-sds-local-volume get ds csi-node -o jsonpath='{.status.desiredNumberScheduled}' 2>/dev/null || echo "0") + csi_node_ready=$(kubectl -n d8-sds-local-volume get ds csi-node -o jsonpath='{.status.numberReady}' 2>/dev/null || echo "0") + deploy_count=$(kubectl -n d8-sds-local-volume get deploy -o name 2>/dev/null | wc -l | tr -d ' ') + controller_ready=false + + if [[ "${deploy_count}" -gt 0 ]] && kubectl -n d8-sds-local-volume wait --for=condition=Available deploy --all --timeout=10s >/dev/null 2>&1; then + controller_ready=true + fi + + if [[ "${local_volume_status}" == "Ready" && "${csi_node_desired}" -gt 0 && "${csi_node_ready}" -eq "${csi_node_desired}" && "${controller_ready}" == "true" ]]; then + echo "[SUCCESS] sds-local-volume is ready (module=${local_volume_status}, csi-node=${csi_node_ready}/${csi_node_desired}, deployments=${deploy_count})" + kubectl get modules sds-local-volume + kubectl -n d8-sds-local-volume get pods + return 0 + fi + + echo "[INFO] Waiting for sds-local-volume to be ready (attempt ${i}/${count})" + echo "[WARNING] Current state: module=${local_volume_status}, csi-node=${csi_node_ready}/${csi_node_desired}, deployments=${deploy_count}, controller_ready=${controller_ready}" + if (( i % 5 == 0 )); then + kubectl get ns d8-sds-local-volume || true + kubectl get modules sds-local-volume -o wide || true + kubectl -n d8-sds-local-volume get pods || true + kubectl -n d8-sds-local-volume get ds || true + kubectl -n d8-sds-local-volume get deploy || true + d8 s queue list | head -n 25 || true + fi + sleep 10 + done + + echo "[ERROR] sds-local-volume did not become ready in time" + kubectl get modules sds-local-volume -o wide || true + kubectl -n d8-sds-local-volume get pods || true + d8 s queue list || true + echo "::group::deckhouse logs" + d8 s logs | tail -n 100 + echo "::endgroup::" + exit 1 + } + + echo "[INFO] Apply sds-local-volume ModuleConfig" + kubectl apply -f mc.yaml + + echo "[INFO] Wait for sds-local-volume module queue" + d8_queue + kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sds-local-volume --timeout=300s + sds_local_volume_ready + + chmod +x ./lsc-gen.sh + ./lsc-gen.sh + + echo "[INFO] Show resulting local storage classes" + kubectl get localstorageclass || true + - name: Configure NFS storage - if: ${{ inputs.storage_type == 'nfs' }} + if: ${{ inputs.storage_type == 'nfs' || inputs.storage_type == 'mixed' }} id: storage-nfs-setup working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/storage/nfs env: NAMESPACE: ${{ needs.bootstrap.outputs.namespace }} + STORAGE_TYPE: ${{ inputs.storage_type }} run: | nfs_ready() { local count=90 @@ -730,10 +844,11 @@ jobs: echo "[INFO] Apply NFSStorageClass" envsubst < storageclass.yaml | kubectl apply -f - - - echo "[INFO] Configure default storage class" - ./default-sc-configure.sh - + + if [[ "${STORAGE_TYPE}" == "mixed" ]]; then + echo "[INFO] Configure default storage class as ${STORAGE_TYPE}" + ./default-sc-configure.sh + fi echo "[INFO] Show existing storageclasses" kubectl get storageclass diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml index 5a30c5480d..3c7c9b16c3 100644 --- a/.github/workflows/e2e-test-releases.yml +++ b/.github/workflows/e2e-test-releases.yml @@ -20,7 +20,7 @@ on: current-release: description: "Current release tag, like v1.4.1" required: false # before merge to main, set to true - default: "v1.4.1" + default: "v1.4.1" # before merge to main, remove type: string next-release: description: "Next release like v1.5.0, should be greater than current-release" @@ -50,14 +50,14 @@ jobs: echo "randuuid4c=$(openssl rand -hex 2)" >> $GITHUB_OUTPUT e2e-replicated: - name: E2E Release Pipeline (Replicated) + name: E2E Release Pipeline (Replicated + NFS) needs: - set-vars uses: ./.github/workflows/e2e-test-releases-reusable-pipeline.yml with: current_release: ${{ github.event.inputs.current-release }} new_release: ${{ github.event.inputs.next-release }} - storage_type: replicated + storage_type: mixed nested_storageclass_name: nested-thin-r1 branch: main deckhouse_channel: alpha @@ -73,26 +73,3 @@ jobs: PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} - e2e-nfs: - name: E2E Release Pipeline (NFS) - needs: - - set-vars - uses: ./.github/workflows/e2e-test-releases-reusable-pipeline.yml - with: - current_release: ${{ github.event.inputs.current-release }} - new_release: ${{ github.event.inputs.next-release }} - storage_type: nfs - nested_storageclass_name: nfs - branch: main - deckhouse_channel: alpha - default_user: cloud - go_version: "1.24.13" - date_start: ${{ needs.set-vars.outputs.date_start }} - randuuid4c: ${{ needs.set-vars.outputs.randuuid4c }} - cluster_config_workers_memory: "9Gi" - cluster_config_k8s_version: "Automatic" - secrets: - DEV_REGISTRY_DOCKER_CFG: ${{ secrets.DEV_REGISTRY_DOCKER_CFG }} - VIRT_E2E_NIGHTLY_SA_TOKEN: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} - PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} - BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} diff --git a/test/dvp-static-cluster/charts/infra/templates/nfs/svc.yaml b/test/dvp-static-cluster/charts/infra/templates/nfs/svc.yaml index 3e71f64488..975e9f6ad9 100644 --- a/test/dvp-static-cluster/charts/infra/templates/nfs/svc.yaml +++ b/test/dvp-static-cluster/charts/infra/templates/nfs/svc.yaml @@ -1,4 +1,4 @@ -{{ if eq .Values.storageType "nfs" }} +{{ if or (eq .Values.storageType "nfs") (eq .Values.storageType "mixed") }} --- apiVersion: v1 kind: Service diff --git a/test/dvp-static-cluster/charts/infra/templates/nfs/vm.yaml b/test/dvp-static-cluster/charts/infra/templates/nfs/vm.yaml index b7d77a81a7..718390449e 100644 --- a/test/dvp-static-cluster/charts/infra/templates/nfs/vm.yaml +++ b/test/dvp-static-cluster/charts/infra/templates/nfs/vm.yaml @@ -1,4 +1,4 @@ -{{ if eq .Values.storageType "nfs" }} +{{ if or (eq .Values.storageType "nfs") (eq .Values.storageType "mixed") }} --- apiVersion: virtualization.deckhouse.io/v1alpha2 kind: VirtualDisk diff --git a/test/dvp-static-cluster/storage/sds-local-volume/lsc-gen.sh b/test/dvp-static-cluster/storage/sds-local-volume/lsc-gen.sh new file mode 100644 index 0000000000..a818b26f77 --- /dev/null +++ b/test/dvp-static-cluster/storage/sds-local-volume/lsc-gen.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +lvg_generator_script="${script_dir}/../sds-replicated/lvg-gen.sh" +manifest=sds-local-lsc.yaml +localStorageClassName=nested-local-thin +targetThinPoolName=thin-data + +discover_lvgs() { + kubectl get lvmvolumegroup -o json | jq -rc \ + --arg targetThinPoolName "${targetThinPoolName}" ' + .items[] + | select((.spec.thinPools // []) | any(.name == $targetThinPoolName)) + | {name: .metadata.name} + ' +} + +lvgs=$(discover_lvgs) + +if [[ -z "${lvgs}" ]]; then + echo "[WARNING] No LVMVolumeGroup resources with thin pool ${targetThinPoolName} found" + echo "[INFO] Trying to create missing LVMVolumeGroup resources via ${lvg_generator_script}" + kubectl get lvmvolumegroup -o wide || true + + if [[ ! -x "${lvg_generator_script}" ]]; then + chmod +x "${lvg_generator_script}" + fi + + "${lvg_generator_script}" + + if [[ -f "sds-lvg.yaml" ]]; then + echo "[INFO] Wait for generated LVMVolumeGroup resources to become Ready" + kubectl wait -f "sds-lvg.yaml" --for=jsonpath='{.status.phase}'=Ready --timeout=300s + fi + + lvgs=$(discover_lvgs) +fi + +if [[ -z "${lvgs}" ]]; then + echo "[ERROR] No LVMVolumeGroup resources with thin pool ${targetThinPoolName} found after creation attempt" + kubectl get lvmvolumegroup -o wide || true + exit 1 +fi + +cat << EOF > "${manifest}" +--- +apiVersion: storage.deckhouse.io/v1alpha1 +kind: LocalStorageClass +metadata: + name: ${localStorageClassName} +spec: + lvm: + type: Thin + lvmVolumeGroups: +EOF + +for lvg in ${lvgs}; do + lvg_name=$(echo "${lvg}" | jq -r '.name') + echo "[INFO] Add LVMVolumeGroup ${lvg_name} to LocalStorageClass" +cat << EOF >> "${manifest}" + - name: ${lvg_name} + thin: + poolName: ${targetThinPoolName} +EOF +done + +cat << EOF >> "${manifest}" + reclaimPolicy: Delete + volumeBindingMode: WaitForFirstConsumer +EOF + +kubectl apply -f "${manifest}" + +for i in $(seq 1 60); do + lsc_phase=$(kubectl get localstorageclass "${localStorageClassName}" -o jsonpath='{.status.phase}' 2>/dev/null || true) + if [[ "${lsc_phase}" == "Created" ]]; then + echo "[SUCCESS] LocalStorageClass ${localStorageClassName} is Created" + kubectl get localstorageclass "${localStorageClassName}" -o yaml + kubectl get storageclass "${localStorageClassName}" + exit 0 + fi + + echo "[INFO] Waiting for LocalStorageClass ${localStorageClassName} to become Created (attempt ${i}/60)" + if (( i % 5 == 0 )); then + kubectl get localstorageclass "${localStorageClassName}" -o yaml || true + fi + sleep 10 +done + +echo "[ERROR] LocalStorageClass ${localStorageClassName} was not created in time" +kubectl get localstorageclass "${localStorageClassName}" -o yaml || true +kubectl get storageclass || true +exit 1 diff --git a/test/dvp-static-cluster/storage/sds-local-volume/mc.yaml b/test/dvp-static-cluster/storage/sds-local-volume/mc.yaml new file mode 100644 index 0000000000..ffe4020fea --- /dev/null +++ b/test/dvp-static-cluster/storage/sds-local-volume/mc.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: deckhouse.io/v1alpha1 +kind: ModuleConfig +metadata: + name: sds-local-volume +spec: + version: 2 + enabled: true From c7af588b4f87c0a5cf17ef72a7eee8caf7804da0 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 9 Apr 2026 15:28:01 +0300 Subject: [PATCH 04/16] separate sds-configurator deploy scripts Signed-off-by: Nikita Korolev --- .github/workflows/e2e-reusable-pipeline.yml | 5 +++-- .../e2e-test-releases-reusable-pipeline.yml | 18 +++--------------- .../storage/sds-local-volume/lsc-gen.sh | 7 +------ .../lvg-gen.sh | 13 +++++++++++-- .../storage/sds-node-configurator/mc.yaml | 8 ++++++++ .../storage/sds-replicated/mc.yaml | 9 --------- 6 files changed, 26 insertions(+), 34 deletions(-) rename test/dvp-static-cluster/storage/{sds-replicated => sds-node-configurator}/lvg-gen.sh (79%) mode change 100755 => 100644 create mode 100644 test/dvp-static-cluster/storage/sds-node-configurator/mc.yaml diff --git a/.github/workflows/e2e-reusable-pipeline.yml b/.github/workflows/e2e-reusable-pipeline.yml index 17cb9b31b5..0d28246853 100644 --- a/.github/workflows/e2e-reusable-pipeline.yml +++ b/.github/workflows/e2e-reusable-pipeline.yml @@ -658,6 +658,7 @@ jobs: d8_queue + kubectl apply -f ../sds-node-configurator/mc.yaml kubectl apply -f mc.yaml echo "[INFO] Wait for sds-node-configurator" kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sds-node-configurator --timeout=300s @@ -672,8 +673,8 @@ jobs: echo "[INFO] Wait pods and webhooks sds-replicated pods" sds_pods_ready - chmod +x lvg-gen.sh - ./lvg-gen.sh + chmod +x ../sds-node-configurator/lvg-gen.sh + ../sds-node-configurator/lvg-gen.sh chmod +x rsc-gen.sh ./rsc-gen.sh diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 1424388080..cba39f836b 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -644,20 +644,9 @@ jobs: exit 1 } - lvg_ready() { - echo "[INFO] Wait for generated LVMVolumeGroup resources to be Ready" - if ! kubectl wait -f sds-lvg.yaml --for=jsonpath='{.status.phase}'=Ready --timeout=300s; then - echo "[ERROR] Generated LVMVolumeGroup resources did not become Ready" - kubectl get lvmvolumegroup -o wide || true - exit 1 - fi - - echo "[SUCCESS] Generated LVMVolumeGroup resources are Ready" - kubectl get lvmvolumegroup -o wide - } - d8_queue + kubectl apply -f ../sds-node-configurator/mc.yaml kubectl apply -f mc.yaml echo "[INFO] Wait for sds-node-configurator" kubectl wait --for=jsonpath='{.status.phase}'=Ready modules sds-node-configurator --timeout=300s @@ -672,9 +661,8 @@ jobs: echo "[INFO] Wait pods and webhooks sds-replicated pods" sds_pods_ready - chmod +x lvg-gen.sh - ./lvg-gen.sh - lvg_ready + chmod +x ../sds-node-configurator/lvg-gen.sh + ../sds-node-configurator/lvg-gen.sh chmod +x rsc-gen.sh ./rsc-gen.sh diff --git a/test/dvp-static-cluster/storage/sds-local-volume/lsc-gen.sh b/test/dvp-static-cluster/storage/sds-local-volume/lsc-gen.sh index a818b26f77..f533dabbe9 100644 --- a/test/dvp-static-cluster/storage/sds-local-volume/lsc-gen.sh +++ b/test/dvp-static-cluster/storage/sds-local-volume/lsc-gen.sh @@ -17,7 +17,7 @@ set -euo pipefail script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) -lvg_generator_script="${script_dir}/../sds-replicated/lvg-gen.sh" +lvg_generator_script="${script_dir}/../sds-node-configurator/lvg-gen.sh" manifest=sds-local-lsc.yaml localStorageClassName=nested-local-thin targetThinPoolName=thin-data @@ -44,11 +44,6 @@ if [[ -z "${lvgs}" ]]; then "${lvg_generator_script}" - if [[ -f "sds-lvg.yaml" ]]; then - echo "[INFO] Wait for generated LVMVolumeGroup resources to become Ready" - kubectl wait -f "sds-lvg.yaml" --for=jsonpath='{.status.phase}'=Ready --timeout=300s - fi - lvgs=$(discover_lvgs) fi diff --git a/test/dvp-static-cluster/storage/sds-replicated/lvg-gen.sh b/test/dvp-static-cluster/storage/sds-node-configurator/lvg-gen.sh old mode 100755 new mode 100644 similarity index 79% rename from test/dvp-static-cluster/storage/sds-replicated/lvg-gen.sh rename to test/dvp-static-cluster/storage/sds-node-configurator/lvg-gen.sh index 6e92143f03..b26de284b8 --- a/test/dvp-static-cluster/storage/sds-replicated/lvg-gen.sh +++ b/test/dvp-static-cluster/storage/sds-node-configurator/lvg-gen.sh @@ -14,12 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -manifest=sds-lvg.yaml +set -euo pipefail + +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +manifest="${script_dir}/sds-lvg.yaml" LVMVG_SIZE=45Gi devs=$(kubectl get blockdevices.storage.deckhouse.io -o json | jq '.items[] | {name: .metadata.name, node: .status.nodeName, dev_path: .status.path}' -rc) -rm -rf "${manifest}" +rm -f "${manifest}" echo detected block devices: "$devs" @@ -55,3 +58,9 @@ EOF done kubectl apply -f "${manifest}" + +echo "[INFO] Wait for generated LVMVolumeGroup resources to become Ready" +kubectl wait -f "${manifest}" --for=jsonpath='{.status.phase}'=Ready --timeout=300s + +echo "[SUCCESS] Generated LVMVolumeGroup resources are Ready" +kubectl get lvmvolumegroup -o wide diff --git a/test/dvp-static-cluster/storage/sds-node-configurator/mc.yaml b/test/dvp-static-cluster/storage/sds-node-configurator/mc.yaml new file mode 100644 index 0000000000..d448935b67 --- /dev/null +++ b/test/dvp-static-cluster/storage/sds-node-configurator/mc.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: deckhouse.io/v1alpha1 +kind: ModuleConfig +metadata: + name: sds-node-configurator +spec: + version: 1 + enabled: true diff --git a/test/dvp-static-cluster/storage/sds-replicated/mc.yaml b/test/dvp-static-cluster/storage/sds-replicated/mc.yaml index c220c01f7f..8e5440ef9a 100644 --- a/test/dvp-static-cluster/storage/sds-replicated/mc.yaml +++ b/test/dvp-static-cluster/storage/sds-replicated/mc.yaml @@ -1,12 +1,3 @@ ---- -apiVersion: deckhouse.io/v1alpha1 -kind: ModuleConfig -metadata: - name: sds-node-configurator -spec: - version: 1 - enabled: true ---- apiVersion: deckhouse.io/v1alpha1 kind: ModuleConfig metadata: From f36a4751fafd81df1b6002422a535ff054f0a12b Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 9 Apr 2026 16:33:13 +0300 Subject: [PATCH 05/16] add build for releases Signed-off-by: Nikita Korolev --- .github/workflows/e2e-test-releases.yml | 80 ++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml index 3c7c9b16c3..fba5d56294 100644 --- a/.github/workflows/e2e-test-releases.yml +++ b/.github/workflows/e2e-test-releases.yml @@ -26,9 +26,14 @@ on: description: "Next release like v1.5.0, should be greater than current-release" required: false # before merge to main, set to true type: string + enableBuild: + description: "Build release images before E2E tests" + required: false + default: false + type: boolean concurrency: - group: "${{ github.workflow }}-${{ github.event.number || github.ref }}" + group: "${{ github.workflow }}-${{ github.event.inputs.current-release }}-${{ github.event.inputs.next-release || 'no-next' }}" cancel-in-progress: true defaults: @@ -49,10 +54,83 @@ jobs: echo "date-start=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_OUTPUT echo "randuuid4c=$(openssl rand -hex 2)" >> $GITHUB_OUTPUT + prepare-release-matrix: + name: Prepare release matrix + if: ${{ inputs.enableBuild }} + runs-on: ubuntu-latest + outputs: + list: ${{ steps.releases.outputs.list }} + steps: + - name: Prepare release refs + id: releases + env: + CURRENT_RELEASE: ${{ github.event.inputs.current-release }} + NEXT_RELEASE: ${{ github.event.inputs.next-release }} + run: | + if [[ -z "${CURRENT_RELEASE}" ]]; then + echo "::error title=Current release is required::Set workflow input 'current-release' to a git ref or tag that can be checked out." + exit 1 + fi + + releases=$(jq -cn \ + --arg current "${CURRENT_RELEASE}" \ + --arg next "${NEXT_RELEASE}" \ + '[$current, $next] | map(select(. != "")) | unique') + + echo "list=${releases}" >> "$GITHUB_OUTPUT" + + build-release-images: + name: Build release image (${{ matrix.release_tag }}) + if: ${{ inputs.enableBuild }} + runs-on: [self-hosted, large] + needs: + - prepare-release-matrix + strategy: + matrix: + release_tag: ${{ fromJSON(needs.prepare-release-matrix.outputs.list) }} + steps: + - name: Setup Docker config + run: | + echo "DOCKER_CONFIG=$(mktemp -d)" >> $GITHUB_ENV + + - name: Print vars + run: | + echo MODULES_REGISTRY=${{ vars.DEV_REGISTRY }} + echo MODULES_MODULE_SOURCE=${{ vars.DEV_MODULE_SOURCE }} + echo MODULES_MODULE_NAME=${{ vars.MODULE_NAME }} + echo MODULES_MODULE_TAG=${{ matrix.release_tag }} + echo DOCKER_CONFIG=$DOCKER_CONFIG + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ matrix.release_tag }} + + - uses: deckhouse/modules-actions/setup@v2 + with: + registry: ${{ vars.DEV_REGISTRY }} + registry_login: ${{ vars.DEV_MODULES_REGISTRY_LOGIN }} + registry_password: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }} + + - uses: deckhouse/modules-actions/build@v4 + with: + module_source: ${{ vars.DEV_MODULE_SOURCE }} + module_name: ${{ vars.MODULE_NAME }} + module_tag: ${{ matrix.release_tag }} + source_repo: ${{ secrets.SOURCE_REPO_GIT }} + source_repo_ssh_key: ${{ secrets.SOURCE_REPO_SSH_KEY }} + + - name: Cleanup Docker config + if: ${{ always() }} + run: | + rm -rf "$DOCKER_CONFIG" + e2e-replicated: name: E2E Release Pipeline (Replicated + NFS) + if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} needs: - set-vars + - build-release-images uses: ./.github/workflows/e2e-test-releases-reusable-pipeline.yml with: current_release: ${{ github.event.inputs.current-release }} From 3a7a790c827c1e5c5b17fa20a51ed1579afd6eb6 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 9 Apr 2026 16:55:15 +0300 Subject: [PATCH 06/16] refactor Signed-off-by: Nikita Korolev --- .../e2e-test-releases-reusable-pipeline.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index cba39f836b..43a43b09a4 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -664,6 +664,7 @@ jobs: chmod +x ../sds-node-configurator/lvg-gen.sh ../sds-node-configurator/lvg-gen.sh + echo "[INFO] Configur ReplicatedStorageClass and set default nested-thin-r1" chmod +x rsc-gen.sh ./rsc-gen.sh @@ -897,7 +898,7 @@ jobs: storage: persistentVolumeClaim: size: 10Gi - storageClassName: ${{ inputs.nested_storageclass_name }} + storageClassName: nested-thin-r1 type: PersistentVolumeClaim virtualMachineCIDRs: - 192.168.10.0/24 @@ -1094,7 +1095,7 @@ jobs: echo "[INFO] Checking virt-handler pods " virt_handler_ready - e2e-test-current: + test-current-release: name: "E2E test (current-release: ${{ inputs.current_release }})" runs-on: ubuntu-latest needs: @@ -1105,6 +1106,11 @@ jobs: - name: Install kubectl CLI uses: azure/setup-kubectl@v4 + + - name: Setup d8 + uses: ./.github/actions/install-d8 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Setup kubeconfig run: | @@ -1129,7 +1135,7 @@ jobs: runs-on: ubuntu-latest needs: - bootstrap - - e2e-test-current + - test-current-release steps: - uses: actions/checkout@v4 @@ -1308,7 +1314,7 @@ jobs: echo "[INFO] Virtualization module status:" kubectl get modules virtualization - e2e-test-new: + test-new-release: name: "E2E test (new-release: ${{ inputs.new_release }})" runs-on: ubuntu-latest needs: From 02f199fa86af11a4ef63b7a1b0641a1393c0116d Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Thu, 9 Apr 2026 18:28:07 +0300 Subject: [PATCH 07/16] add release tests Signed-off-by: Nikita Korolev --- .../e2e-test-releases-reusable-pipeline.yml | 73 ++++++- test/e2e/release/current_release_smoke.go | 200 ++++++++++++++++++ test/e2e/release/release_suite_test.go | 67 ++++++ 3 files changed, 335 insertions(+), 5 deletions(-) create mode 100644 test/e2e/release/current_release_smoke.go create mode 100644 test/e2e/release/release_suite_test.go diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 43a43b09a4..19c1229ce6 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -1104,6 +1104,11 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v5 + with: + go-version: "${{ env.GO_VERSION }}" + - name: Install kubectl CLI uses: azure/setup-kubectl@v4 @@ -1117,18 +1122,76 @@ jobs: mkdir -p ~/.kube echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config chmod 600 ~/.kube/config + kubectl config use-context nested-e2e-nested-sa + + - name: Install ginkgo + working-directory: ./test/e2e/ + run: | + echo "Install ginkgo" + go install tool + + - name: Download dependencies + working-directory: ./test/e2e/ + run: | + echo "Download dependencies" + go mod download + + - name: Create vmclass for release e2e tests + run: | + if ! (kubectl get vmclass generic-for-e2e 2>/dev/null); then + kubectl get vmclass/generic -o json | jq 'del(.status) | del(.metadata) | .metadata = {"name":"generic-for-e2e","annotations":{"virtualmachineclass.virtualization.deckhouse.io/is-default-class":"true"}} ' | kubectl create -f - + fi + + echo "[INFO] Showing existing vmclasses" + kubectl get vmclass - - name: "Run E2E tests on current-release (STUB)" + - name: "Run E2E tests on current-release" + id: release-e2e + env: + POST_CLEANUP: no + PRECREATED_CVI_CLEANUP: no + CSI: ${{ inputs.storage_type }} + STORAGE_CLASS_NAME: ${{ inputs.nested_storageclass_name }} run: | echo "[INFO] Current release tag: ${{ env.CURRENT_RELEASE }}" echo "[INFO] Storage type: ${{ inputs.storage_type }}" echo "" echo "[INFO] Verifying virtualization module is running" - kubectl get modules virtualization || true - kubectl get mpo virtualization || true + kubectl get modules virtualization + kubectl get mpo virtualization echo "" - echo "[STUB] E2E tests on current-release ${{ env.CURRENT_RELEASE }} -- PASSED" - echo "[INFO] Resources are intentionally left in the cluster for the upgrade test" + echo "[INFO] Running dedicated release suite" + echo "[INFO] Resources will be intentionally left in the cluster for the upgrade test" + cd ./test/e2e/ + GINKGO_RESULT=$(mktemp -p "$RUNNER_TEMP") + junit_report="release_current_suite.xml" + set +e + go tool ginkgo \ + -v --race --timeout=45m \ + --junit-report="$junit_report" \ + ./release | tee "$GINKGO_RESULT" + GINKGO_EXIT_CODE=$? + set -e + echo "[INFO] Exit code: $GINKGO_EXIT_CODE" + exit $GINKGO_EXIT_CODE + + - name: Upload current-release test results + uses: actions/upload-artifact@v4 + if: always() && steps.release-e2e.outcome != 'skipped' + with: + name: current-release-e2e-results-${{ github.run_id }} + path: test/e2e/release_current_suite.xml + if-no-files-found: ignore + retention-days: 3 + + - name: Upload resources from failed current-release tests + uses: actions/upload-artifact@v4 + if: always() && steps.release-e2e.outcome != 'skipped' + with: + name: current-release-resources-from-failed-tests-${{ github.run_id }} + path: ${{ runner.temp }}/e2e_failed__* + if-no-files-found: ignore + retention-days: 3 patch-mpo: name: "Patch MPO to new-release: ${{ inputs.new_release }}" diff --git a/test/e2e/release/current_release_smoke.go b/test/e2e/release/current_release_smoke.go new file mode 100644 index 0000000000..5674446124 --- /dev/null +++ b/test/e2e/release/current_release_smoke.go @@ -0,0 +1,200 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package release + +import ( + "context" + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + vdbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vd" + vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" + "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/test/e2e/internal/framework" + "github.com/deckhouse/virtualization/test/e2e/internal/object" + "github.com/deckhouse/virtualization/test/e2e/internal/util" +) + +const ( + replicatedStorageClass = "nested-thin-r1" + localThinStorageClass = "nested-local-thin" + diskCountCommand = "lsblk -dn -o TYPE | grep -c '^disk$'" +) + +var _ = Describe("CurrentReleaseSmoke", func() { + It("should validate alpine virtual machines on current release", func() { + f := framework.NewFramework("release-current") + DeferCleanup(f.After) + f.Before() + + test := newCurrentReleaseSmokeTest(f) + + By("Creating root and hotplug virtual disks") + Expect(f.CreateWithDeferredDeletion(context.Background(), test.diskObjects()...)).To(Succeed()) + util.UntilObjectPhase(string(v1alpha2.DiskReady), framework.LongTimeout, test.diskObjects()...) + + By("Creating virtual machines") + Expect(f.CreateWithDeferredDeletion(context.Background(), test.vmObjects()...)).To(Succeed()) + util.UntilObjectPhase(string(v1alpha2.MachineRunning), framework.LongTimeout, test.vmOneHotplug, test.vmTwoHotplug) + util.UntilObjectPhase(string(v1alpha2.MachineStopped), framework.MiddleTimeout, test.vmAlwaysOff) + + By("Starting the manual-policy virtual machine") + util.StartVirtualMachine(f, test.vmAlwaysOff) + util.UntilObjectPhase(string(v1alpha2.MachineRunning), framework.LongTimeout, test.vmAlwaysOff) + + By("Attaching hotplug disks") + Expect(f.CreateWithDeferredDeletion(context.Background(), test.vmbdaOneHotplug, test.vmbdaReplicated, test.vmbdaLocalThin)).To(Succeed()) + util.UntilObjectPhase( + string(v1alpha2.BlockDeviceAttachmentPhaseAttached), + framework.LongTimeout, + test.vmbdaOneHotplug, + test.vmbdaReplicated, + test.vmbdaLocalThin, + ) + + By("Waiting for guest agent and SSH access") + test.expectGuestReady(test.vmAlwaysOff) + test.expectGuestReady(test.vmOneHotplug) + test.expectGuestReady(test.vmTwoHotplug) + + By("Checking attached disks inside guests") + test.expectDiskCount(test.vmAlwaysOff, 1) + test.expectDiskCount(test.vmOneHotplug, 2) + test.expectDiskCount(test.vmTwoHotplug, 3) + }) +}) + +type currentReleaseSmokeTest struct { + framework *framework.Framework + + vmAlwaysOff *v1alpha2.VirtualMachine + vmOneHotplug *v1alpha2.VirtualMachine + vmTwoHotplug *v1alpha2.VirtualMachine + + rootAlwaysOff *v1alpha2.VirtualDisk + rootOneHotplug *v1alpha2.VirtualDisk + rootTwoHotplug *v1alpha2.VirtualDisk + + hotplugOne *v1alpha2.VirtualDisk + hotplugReplicated *v1alpha2.VirtualDisk + hotplugLocalThin *v1alpha2.VirtualDisk + + vmbdaOneHotplug *v1alpha2.VirtualMachineBlockDeviceAttachment + vmbdaReplicated *v1alpha2.VirtualMachineBlockDeviceAttachment + vmbdaLocalThin *v1alpha2.VirtualMachineBlockDeviceAttachment +} + +func newCurrentReleaseSmokeTest(f *framework.Framework) *currentReleaseSmokeTest { + test := ¤tReleaseSmokeTest{framework: f} + namespace := f.Namespace().Name + + test.rootAlwaysOff = newRootDisk("vd-root-always-off", namespace) + test.rootOneHotplug = newRootDisk("vd-root-one-hotplug", namespace) + test.rootTwoHotplug = newRootDisk("vd-root-two-hotplug", namespace) + + test.hotplugOne = newHotplugDisk("vd-hotplug", namespace, replicatedStorageClass) + test.hotplugReplicated = newHotplugDisk("vd-hotplug-replicated", namespace, replicatedStorageClass) + test.hotplugLocalThin = newHotplugDisk("vd-hotplug-local-thin", namespace, localThinStorageClass) + + test.vmAlwaysOff = newVM("vm-always-off", namespace, v1alpha2.ManualPolicy, test.rootAlwaysOff.Name) + test.vmOneHotplug = newVM("vm-one-hotplug", namespace, v1alpha2.AlwaysOnUnlessStoppedManually, test.rootOneHotplug.Name) + test.vmTwoHotplug = newVM("vm-two-hotplug", namespace, v1alpha2.AlwaysOnPolicy, test.rootTwoHotplug.Name) + + test.vmbdaOneHotplug = object.NewVMBDAFromDisk("vmbda", test.vmOneHotplug.Name, test.hotplugOne) + test.vmbdaReplicated = object.NewVMBDAFromDisk("vmbda1", test.vmTwoHotplug.Name, test.hotplugReplicated) + test.vmbdaLocalThin = object.NewVMBDAFromDisk("vmbda2", test.vmTwoHotplug.Name, test.hotplugLocalThin) + + return test +} + +func newRootDisk(name, namespace string) *v1alpha2.VirtualDisk { + return object.NewVDFromCVI( + name, + namespace, + object.PrecreatedCVIAlpineBIOS, + vdbuilder.WithStorageClass(ptr.To(replicatedStorageClass)), + vdbuilder.WithSize(ptr.To(resource.MustParse("350Mi"))), + ) +} + +func newHotplugDisk(name, namespace, storageClass string) *v1alpha2.VirtualDisk { + return object.NewBlankVD( + name, + namespace, + ptr.To(storageClass), + ptr.To(resource.MustParse("100Mi")), + ) +} + +func newVM(name, namespace string, runPolicy v1alpha2.RunPolicy, rootDiskName string) *v1alpha2.VirtualMachine { + return vmbuilder.New( + vmbuilder.WithName(name), + vmbuilder.WithNamespace(namespace), + vmbuilder.WithCPU(1, ptr.To("20%")), + vmbuilder.WithMemory(resource.MustParse("512Mi")), + vmbuilder.WithLiveMigrationPolicy(v1alpha2.AlwaysSafeMigrationPolicy), + vmbuilder.WithVirtualMachineClass(object.DefaultVMClass), + vmbuilder.WithProvisioningUserData(object.AlpineCloudInit), + vmbuilder.WithRunPolicy(runPolicy), + vmbuilder.WithBlockDeviceRefs(v1alpha2.BlockDeviceSpecRef{ + Kind: v1alpha2.DiskDevice, + Name: rootDiskName, + }), + ) +} + +func (t *currentReleaseSmokeTest) diskObjects() []crclient.Object { + return []crclient.Object{ + t.rootAlwaysOff, + t.rootOneHotplug, + t.rootTwoHotplug, + t.hotplugOne, + t.hotplugReplicated, + t.hotplugLocalThin, + } +} + +func (t *currentReleaseSmokeTest) vmObjects() []crclient.Object { + return []crclient.Object{ + t.vmAlwaysOff, + t.vmOneHotplug, + t.vmTwoHotplug, + } +} + +func (t *currentReleaseSmokeTest) expectGuestReady(vm *v1alpha2.VirtualMachine) { + By(fmt.Sprintf("Waiting for guest agent on %s", vm.Name)) + util.UntilVMAgentReady(crclient.ObjectKeyFromObject(vm), framework.LongTimeout) + + By(fmt.Sprintf("Waiting for SSH access on %s", vm.Name)) + util.UntilSSHReady(t.framework, vm, framework.LongTimeout) +} + +func (t *currentReleaseSmokeTest) expectDiskCount(vm *v1alpha2.VirtualMachine, expectedCount int) { + Eventually(func(g Gomega) { + output, err := t.framework.SSHCommand(vm.Name, vm.Namespace, diskCountCommand, framework.WithSSHTimeout(10*time.Second)) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(output)).To(Equal(fmt.Sprintf("%d", expectedCount))) + }).WithTimeout(framework.LongTimeout).WithPolling(time.Second).Should(Succeed()) +} diff --git a/test/e2e/release/release_suite_test.go b/test/e2e/release/release_suite_test.go new file mode 100644 index 0000000000..13335f8984 --- /dev/null +++ b/test/e2e/release/release_suite_test.go @@ -0,0 +1,67 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package release + +import ( + "context" + "fmt" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/test/e2e/controller" + "github.com/deckhouse/virtualization/test/e2e/internal/config" + "github.com/deckhouse/virtualization/test/e2e/internal/framework" + "github.com/deckhouse/virtualization/test/e2e/internal/precreatedcvi" + "github.com/deckhouse/virtualization/test/e2e/internal/util" +) + +var cviManager = precreatedcvi.NewManager() + +func TestRelease(t *testing.T) { + log.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + RegisterFailHandler(Fail) + RunSpecs(t, "Release Tests") +} + +var _ = SynchronizedBeforeSuite(func() { + controller.NewBeforeProcess1Body() + + By("Creating or reusing precreated CVIs") + err := cviManager.Bootstrap(context.Background()) + Expect(err).NotTo(HaveOccurred()) + + cvis := cviManager.CVIsAsObjects() + By(fmt.Sprintf("Waiting for all %d precreated CVIs to be ready", len(cvis))) + util.UntilObjectPhase(string(v1alpha2.ImageReady), framework.LongTimeout, cvis...) +}, func() {}) + +var _ = SynchronizedAfterSuite(func() { + if !config.IsCleanUpNeeded() || !config.IsPrecreatedCVICleanupNeeded() { + return + } + + By("Cleaning up precreated CVIs") + err := cviManager.Cleanup(context.Background()) + Expect(err).NotTo(HaveOccurred(), "Failed to delete precreated CVIs") +}, func() { + controller.NewAfterAllProcessBody() +}) From 461b053981a292ccd23e8e9cb6eb669fce3f871d Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 10 Apr 2026 10:18:57 +0300 Subject: [PATCH 08/16] add labels test=release Signed-off-by: Nikita Korolev --- .github/workflows/e2e-test-releases-reusable-pipeline.yml | 6 +++--- .github/workflows/e2e-test-releases.yml | 1 - .../charts/infra/templates/_helpers.tpl | 8 ++++++++ test/dvp-static-cluster/charts/infra/templates/ns.yaml | 2 +- test/dvp-static-cluster/charts/infra/templates/vmc.yaml | 2 +- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 19c1229ce6..8996432779 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -131,7 +131,7 @@ jobs: run: | GIT_SHORT_HASH=$(git rev-parse --short HEAD) - namespace="nightly-e2e-$STORAGE_TYPE-$GIT_SHORT_HASH-$RANDUUID4C" + namespace="release-test-$STORAGE_TYPE-$GIT_SHORT_HASH-$RANDUUID4C" echo "namespace=$namespace" >> $GITHUB_OUTPUT echo "sha_short=$GIT_SHORT_HASH" >> $GITHUB_OUTPUT @@ -833,7 +833,7 @@ jobs: echo "[INFO] Apply NFSStorageClass" envsubst < storageclass.yaml | kubectl apply -f - - + if [[ "${STORAGE_TYPE}" == "mixed" ]]; then echo "[INFO] Configure default storage class as ${STORAGE_TYPE}" ./default-sc-configure.sh @@ -1111,7 +1111,7 @@ jobs: - name: Install kubectl CLI uses: azure/setup-kubectl@v4 - + - name: Setup d8 uses: ./.github/actions/install-d8 env: diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml index fba5d56294..3bea15f484 100644 --- a/.github/workflows/e2e-test-releases.yml +++ b/.github/workflows/e2e-test-releases.yml @@ -150,4 +150,3 @@ jobs: VIRT_E2E_NIGHTLY_SA_TOKEN: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} PROD_IO_REGISTRY_DOCKER_CFG: ${{ secrets.PROD_IO_REGISTRY_DOCKER_CFG }} BOOTSTRAP_DEV_PROXY: ${{ secrets.BOOTSTRAP_DEV_PROXY }} - diff --git a/test/dvp-static-cluster/charts/infra/templates/_helpers.tpl b/test/dvp-static-cluster/charts/infra/templates/_helpers.tpl index 4a5e99da43..fb991483fa 100644 --- a/test/dvp-static-cluster/charts/infra/templates/_helpers.tpl +++ b/test/dvp-static-cluster/charts/infra/templates/_helpers.tpl @@ -1,3 +1,11 @@ +{{- define "infra.test-label" -}} +{{- if contains "release" .Values.namespace -}} +test: release +{{- else -}} +test: nightly-e2e +{{- end -}} +{{- end }} + {{- define "infra.vm-labels" -}} {{- $prefix := regexReplaceAll "-\\d+$" . "" -}} vm: {{ . }} diff --git a/test/dvp-static-cluster/charts/infra/templates/ns.yaml b/test/dvp-static-cluster/charts/infra/templates/ns.yaml index 259ba5c4aa..c7437157c6 100644 --- a/test/dvp-static-cluster/charts/infra/templates/ns.yaml +++ b/test/dvp-static-cluster/charts/infra/templates/ns.yaml @@ -3,4 +3,4 @@ kind: Namespace metadata: name: {{ .Values.namespace }} labels: - test: nightly-e2e + {{- include "infra.test-label" . | nindent 4 }} diff --git a/test/dvp-static-cluster/charts/infra/templates/vmc.yaml b/test/dvp-static-cluster/charts/infra/templates/vmc.yaml index 796d266391..19dd8b8e90 100644 --- a/test/dvp-static-cluster/charts/infra/templates/vmc.yaml +++ b/test/dvp-static-cluster/charts/infra/templates/vmc.yaml @@ -3,7 +3,7 @@ kind: VirtualMachineClass metadata: name: {{ include "infra.vmclass-name" . }} labels: - test: nightly-e2e + {{- include "infra.test-label" . | nindent 4 }} spec: cpu: type: Discovery From 72e8ad58d08450fb5e5d5bf9ae4392b333edc5d6 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 10 Apr 2026 10:27:35 +0300 Subject: [PATCH 09/16] add default vers release Signed-off-by: Nikita Korolev --- .github/workflows/e2e-test-releases.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml index 3bea15f484..9b1fc3a347 100644 --- a/.github/workflows/e2e-test-releases.yml +++ b/.github/workflows/e2e-test-releases.yml @@ -20,11 +20,12 @@ on: current-release: description: "Current release tag, like v1.4.1" required: false # before merge to main, set to true - default: "v1.4.1" # before merge to main, remove + default: "v1.6.3-rc.3" # before merge to main, remove type: string next-release: description: "Next release like v1.5.0, should be greater than current-release" required: false # before merge to main, set to true + default: "v1.7.0" # before merge to main, remove type: string enableBuild: description: "Build release images before E2E tests" From f056e033359438470074b92eb191c3b841f1a8a5 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 10 Apr 2026 11:43:09 +0300 Subject: [PATCH 10/16] rm annotation for caps master-0 Signed-off-by: Nikita Korolev --- .../charts/cluster-config/templates/master-nodes.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/dvp-static-cluster/charts/cluster-config/templates/master-nodes.yaml b/test/dvp-static-cluster/charts/cluster-config/templates/master-nodes.yaml index 191afcd96e..ab59713c38 100644 --- a/test/dvp-static-cluster/charts/cluster-config/templates/master-nodes.yaml +++ b/test/dvp-static-cluster/charts/cluster-config/templates/master-nodes.yaml @@ -4,6 +4,9 @@ {{- $totalNodes = add $totalNodes .count -}} {{- end -}} +{{- $masterCount := $.Values.instances.masterNodes.count | int -}} +{{- if gt $masterCount 1 -}} + {{- $staticCount := sub $masterCount 1 -}} --- apiVersion: deckhouse.io/v1 kind: NodeGroup @@ -29,16 +32,12 @@ spec: matchLabels: role: master -{{- range $_, $i := untilStep 0 (.Values.instances.masterNodes.count | int) 1}} +{{- range $_, $i := untilStep 1 (.Values.instances.masterNodes.count | int) 1}} {{- $vmName := printf "%s-master-%d" $.Values.storageType $i }} --- apiVersion: deckhouse.io/v1alpha1 kind: StaticInstance metadata: - {{- if eq $i 0 }} - annotations: - static.node.deckhouse.io/skip-bootstrap-phase: "" - {{- end }} name: {{ $vmName }} labels: role: master @@ -48,3 +47,4 @@ spec: kind: SSHCredentials name: mvp-static {{- end }} +{{- end }} From 02fc8861176b57fe45b72edcfbbabde71908aadf Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 10 Apr 2026 13:22:45 +0300 Subject: [PATCH 11/16] fix step 'Run E2E tests on current-release' Signed-off-by: Nikita Korolev --- .github/workflows/e2e-test-releases-reusable-pipeline.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 8996432779..9363063ac4 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -1152,6 +1152,7 @@ jobs: PRECREATED_CVI_CLEANUP: no CSI: ${{ inputs.storage_type }} STORAGE_CLASS_NAME: ${{ inputs.nested_storageclass_name }} + E2E_CONFIG: ${{ github.workspace }}/test/e2e/default_config.yaml run: | echo "[INFO] Current release tag: ${{ env.CURRENT_RELEASE }}" echo "[INFO] Storage type: ${{ inputs.storage_type }}" @@ -1164,7 +1165,7 @@ jobs: echo "[INFO] Resources will be intentionally left in the cluster for the upgrade test" cd ./test/e2e/ GINKGO_RESULT=$(mktemp -p "$RUNNER_TEMP") - junit_report="release_current_suite.xml" + junit_report="$GITHUB_WORKSPACE/test/e2e/release_current_suite.xml" set +e go tool ginkgo \ -v --race --timeout=45m \ From 60a666e47967ce824b40102c2d1a3da2dcb92e43 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 10 Apr 2026 13:33:54 +0300 Subject: [PATCH 12/16] change wf name Signed-off-by: Nikita Korolev --- .github/workflows/e2e-test-releases.yml | 2 +- test/e2e/release/current_release_smoke.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/e2e-test-releases.yml b/.github/workflows/e2e-test-releases.yml index 9b1fc3a347..da0a210804 100644 --- a/.github/workflows/e2e-test-releases.yml +++ b/.github/workflows/e2e-test-releases.yml @@ -127,7 +127,7 @@ jobs: rm -rf "$DOCKER_CONFIG" e2e-replicated: - name: E2E Release Pipeline (Replicated + NFS) + name: E2E Release Pipeline (Replicated + Local + NFS) if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} needs: - set-vars diff --git a/test/e2e/release/current_release_smoke.go b/test/e2e/release/current_release_smoke.go index 5674446124..5f28baf52a 100644 --- a/test/e2e/release/current_release_smoke.go +++ b/test/e2e/release/current_release_smoke.go @@ -52,7 +52,6 @@ var _ = Describe("CurrentReleaseSmoke", func() { By("Creating root and hotplug virtual disks") Expect(f.CreateWithDeferredDeletion(context.Background(), test.diskObjects()...)).To(Succeed()) - util.UntilObjectPhase(string(v1alpha2.DiskReady), framework.LongTimeout, test.diskObjects()...) By("Creating virtual machines") Expect(f.CreateWithDeferredDeletion(context.Background(), test.vmObjects()...)).To(Succeed()) From 1b362ac678a7cff5fd16e3f27ad5855c552d3262 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 10 Apr 2026 14:52:50 +0300 Subject: [PATCH 13/16] fix tst Signed-off-by: Nikita Korolev --- test/e2e/release/current_release_smoke.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/e2e/release/current_release_smoke.go b/test/e2e/release/current_release_smoke.go index 5f28baf52a..520f15899d 100644 --- a/test/e2e/release/current_release_smoke.go +++ b/test/e2e/release/current_release_smoke.go @@ -72,6 +72,9 @@ var _ = Describe("CurrentReleaseSmoke", func() { test.vmbdaLocalThin, ) + By("Waiting for all disks to become ready after consumers appear") + util.UntilObjectPhase(string(v1alpha2.DiskReady), framework.LongTimeout, test.diskObjects()...) + By("Waiting for guest agent and SSH access") test.expectGuestReady(test.vmAlwaysOff) test.expectGuestReady(test.vmOneHotplug) From 5dad093cb7f88fe9a7e860c1effe0b31c693bc91 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 10 Apr 2026 16:10:33 +0300 Subject: [PATCH 14/16] fix condition Signed-off-by: Nikita Korolev --- .github/workflows/e2e-test-releases-reusable-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index 9363063ac4..dda2956f6b 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -834,7 +834,7 @@ jobs: echo "[INFO] Apply NFSStorageClass" envsubst < storageclass.yaml | kubectl apply -f - - if [[ "${STORAGE_TYPE}" == "mixed" ]]; then + if [[ "${STORAGE_TYPE}" != "mixed" ]]; then echo "[INFO] Configure default storage class as ${STORAGE_TYPE}" ./default-sc-configure.sh fi From 0762e235782826faa52148fae81897dbb1655109 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 10 Apr 2026 17:01:28 +0300 Subject: [PATCH 15/16] fix test Signed-off-by: Nikita Korolev --- test/e2e/release/current_release_smoke.go | 65 ++++++++++++++++++++--- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/test/e2e/release/current_release_smoke.go b/test/e2e/release/current_release_smoke.go index 520f15899d..bfffb70e93 100644 --- a/test/e2e/release/current_release_smoke.go +++ b/test/e2e/release/current_release_smoke.go @@ -18,8 +18,8 @@ package release import ( "context" + "encoding/json" "fmt" - "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -39,7 +39,8 @@ import ( const ( replicatedStorageClass = "nested-thin-r1" localThinStorageClass = "nested-local-thin" - diskCountCommand = "lsblk -dn -o TYPE | grep -c '^disk$'" + lsblkJSONCommand = "lsblk --bytes --json --nodeps --output NAME,SIZE,TYPE,MOUNTPOINTS" + minDataDiskSizeBytes = int64(50 * 1024 * 1024) ) var _ = Describe("CurrentReleaseSmoke", func() { @@ -81,9 +82,9 @@ var _ = Describe("CurrentReleaseSmoke", func() { test.expectGuestReady(test.vmTwoHotplug) By("Checking attached disks inside guests") - test.expectDiskCount(test.vmAlwaysOff, 1) - test.expectDiskCount(test.vmOneHotplug, 2) - test.expectDiskCount(test.vmTwoHotplug, 3) + test.expectAdditionalDiskCount(test.vmAlwaysOff, 0) + test.expectAdditionalDiskCount(test.vmOneHotplug, 1) + test.expectAdditionalDiskCount(test.vmTwoHotplug, 2) }) }) @@ -107,6 +108,17 @@ type currentReleaseSmokeTest struct { vmbdaLocalThin *v1alpha2.VirtualMachineBlockDeviceAttachment } +type lsblkOutput struct { + BlockDevices []lsblkDevice `json:"blockdevices"` +} + +type lsblkDevice struct { + Name string `json:"name"` + Size int64 `json:"size"` + Type string `json:"type"` + Mountpoints []string `json:"mountpoints"` +} + func newCurrentReleaseSmokeTest(f *framework.Framework) *currentReleaseSmokeTest { test := ¤tReleaseSmokeTest{framework: f} namespace := f.Namespace().Name @@ -193,10 +205,47 @@ func (t *currentReleaseSmokeTest) expectGuestReady(vm *v1alpha2.VirtualMachine) util.UntilSSHReady(t.framework, vm, framework.LongTimeout) } -func (t *currentReleaseSmokeTest) expectDiskCount(vm *v1alpha2.VirtualMachine, expectedCount int) { +func (t *currentReleaseSmokeTest) expectAdditionalDiskCount(vm *v1alpha2.VirtualMachine, expectedCount int) { Eventually(func(g Gomega) { - output, err := t.framework.SSHCommand(vm.Name, vm.Namespace, diskCountCommand, framework.WithSSHTimeout(10*time.Second)) + output, err := t.framework.SSHCommand(vm.Name, vm.Namespace, lsblkJSONCommand, framework.WithSSHTimeout(10*time.Second)) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(strings.TrimSpace(output)).To(Equal(fmt.Sprintf("%d", expectedCount))) + + disks, err := parseLSBLKOutput(output) + g.Expect(err).NotTo(HaveOccurred()) + + count := 0 + for _, disk := range disks { + if disk.Type != "disk" { + continue + } + if disk.Size <= minDataDiskSizeBytes { + continue + } + if hasMountpoint(disk.Mountpoints, "/") { + continue + } + count++ + } + + g.Expect(count).To(Equal(expectedCount)) }).WithTimeout(framework.LongTimeout).WithPolling(time.Second).Should(Succeed()) } + +func parseLSBLKOutput(raw string) ([]lsblkDevice, error) { + var output lsblkOutput + if err := json.Unmarshal([]byte(raw), &output); err != nil { + return nil, fmt.Errorf("parse lsblk json: %w", err) + } + + return output.BlockDevices, nil +} + +func hasMountpoint(mountpoints []string, expected string) bool { + for _, mountpoint := range mountpoints { + if mountpoint == expected { + return true + } + } + + return false +} From 78147e79b9a14276fb1e546390d364ff2020a45e Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Fri, 10 Apr 2026 19:02:54 +0300 Subject: [PATCH 16/16] fix test, add temporary undeploy Signed-off-by: Nikita Korolev --- .../e2e-test-releases-reusable-pipeline.yml | 42 ++++++++++++++++++- test/e2e/release/current_release_smoke.go | 13 +++++- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-test-releases-reusable-pipeline.yml b/.github/workflows/e2e-test-releases-reusable-pipeline.yml index dda2956f6b..f699e49244 100644 --- a/.github/workflows/e2e-test-releases-reusable-pipeline.yml +++ b/.github/workflows/e2e-test-releases-reusable-pipeline.yml @@ -1167,7 +1167,7 @@ jobs: GINKGO_RESULT=$(mktemp -p "$RUNNER_TEMP") junit_report="$GITHUB_WORKSPACE/test/e2e/release_current_suite.xml" set +e - go tool ginkgo \ + POST_CLEANUP=no go tool ginkgo \ -v --race --timeout=45m \ --junit-report="$junit_report" \ ./release | tee "$GINKGO_RESULT" @@ -1216,7 +1216,6 @@ jobs: mkdir -p ~/.kube echo "${{ needs.bootstrap.outputs.kubeconfig }}" | base64 -d | base64 -d > ~/.kube/config chmod 600 ~/.kube/config - kubectl config use-context nested-e2e-nested-sa - name: Show current MPO state run: | @@ -1407,3 +1406,42 @@ jobs: echo "" echo "[STUB] E2E tests on new-release ${{ env.NEW_RELEASE }} -- PASSED" echo "[INFO] Cluster is intentionally left running (no cleanup)" + + undeploy-cluster: + name: Undeploy cluster + runs-on: ubuntu-latest + needs: + - bootstrap + - test-new-release + if: always() + steps: + - uses: actions/checkout@v4 + + - name: Setup d8 + uses: ./.github/actions/install-d8 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Install Task + uses: arduino/setup-task@v2 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Download artifacts + uses: actions/download-artifact@v5 + with: + name: ${{ inputs.storage_type }}-release-generated-files-${{ inputs.date_start }} + path: ${{ env.SETUP_CLUSTER_TYPE_PATH }}/ + + - name: Configure kubectl via azure/k8s-set-context@v4 + uses: azure/k8s-set-context@v4 + with: + method: kubeconfig + context: e2e-cluster-nightly-e2e-virt-sa + kubeconfig: ${{ secrets.VIRT_E2E_NIGHTLY_SA_TOKEN }} + + - name: infra-undeploy + working-directory: ${{ env.SETUP_CLUSTER_TYPE_PATH }} + run: | + task infra-undeploy diff --git a/test/e2e/release/current_release_smoke.go b/test/e2e/release/current_release_smoke.go index bfffb70e93..0fd3601a7d 100644 --- a/test/e2e/release/current_release_smoke.go +++ b/test/e2e/release/current_release_smoke.go @@ -31,6 +31,7 @@ import ( vdbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vd" vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/test/e2e/internal/config" "github.com/deckhouse/virtualization/test/e2e/internal/framework" "github.com/deckhouse/virtualization/test/e2e/internal/object" "github.com/deckhouse/virtualization/test/e2e/internal/util" @@ -46,7 +47,17 @@ const ( var _ = Describe("CurrentReleaseSmoke", func() { It("should validate alpine virtual machines on current release", func() { f := framework.NewFramework("release-current") - DeferCleanup(f.After) + if config.IsCleanUpNeeded() { + DeferCleanup(f.After) + } else { + // Keep created resources after a successful run when POST_CLEANUP=no, + // but still preserve failure dumps if the spec breaks. + DeferCleanup(func() { + if CurrentSpecReport().Failed() { + f.After() + } + }) + } f.Before() test := newCurrentReleaseSmokeTest(f)