From 7088f9cd31b34688dbd9397262099e73cf435f74 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 10 Dec 2025 15:04:31 +0900 Subject: [PATCH 01/39] chore: improve wrapper scripts for flexibility Signed-off-by: Chris Butler --- rhdp/rhdp-cluster-define.py | 34 ++++++++++++++++++--- rhdp/wrapper.sh | 61 ++++++++++++++++++++++++++++++++----- 2 files changed, 84 insertions(+), 11 deletions(-) diff --git a/rhdp/rhdp-cluster-define.py b/rhdp/rhdp-cluster-define.py index 522c0bb3..b65cb4b3 100644 --- a/rhdp/rhdp-cluster-define.py +++ b/rhdp/rhdp-cluster-define.py @@ -13,8 +13,22 @@ from typing_extensions import Annotated -def get_default_cluster_configs() -> List[Dict]: - """Get default cluster configurations""" +def get_default_cluster_configs(prefix: str = "") -> List[Dict]: + """Get default cluster configurations + + Args: + prefix: Optional prefix to add to cluster name and directory + """ + if prefix: + return [ + { + "name": f"coco-{prefix}", + "directory": f"openshift-install-{prefix}", + "cluster_network_cidr": "10.128.0.0/14", + "machine_network_cidr": "10.0.0.0/16", + "service_network_cidr": "172.30.0.0/16", + } + ] return [ { "name": "coco", @@ -135,6 +149,9 @@ def run( multicluster: Annotated[ bool, typer.Option("--multicluster", help="Deploy hub and spoke clusters") ] = False, + prefix: Annotated[ + str, typer.Option("--prefix", help="Prefix for cluster name and directory") + ] = "", ): """ Region flag requires an azure region key which can be (authoritatively) @@ -142,16 +159,25 @@ def run( Use --multicluster flag to deploy both hub (coco-hub) and spoke (coco-spoke) clusters. + + Use --prefix to add a prefix to cluster name and install directory, enabling + multiple cluster deployments (e.g., --prefix cluster1 creates coco-cluster1 + in openshift-install-cluster1). """ validate_dir() # Choose cluster configurations based on multicluster flag if multicluster: + if prefix: + rprint("WARNING: --prefix is ignored when using --multicluster") cluster_configs = get_multicluster_configs() rprint("Setting up multicluster deployment (hub and spoke)") else: - cluster_configs = get_default_cluster_configs() - rprint("Setting up single cluster deployment") + cluster_configs = get_default_cluster_configs(prefix) + if prefix: + rprint(f"Setting up single cluster deployment with prefix: {prefix}") + else: + rprint("Setting up single cluster deployment") cleanup(pathlib.Path.cwd(), cluster_configs) setup_install( diff --git a/rhdp/wrapper.sh b/rhdp/wrapper.sh index 5fbf6994..5bc1f992 100755 --- a/rhdp/wrapper.sh +++ b/rhdp/wrapper.sh @@ -14,13 +14,56 @@ get_python_cmd() { fi } -if [ "$#" -ne 1 ]; then - echo "Error: Exactly one argument is required." - echo "Usage: $0 {azure-region-code}" +# Parse arguments +AZUREREGION="" +PREFIX="" + +while [[ $# -gt 0 ]]; do + case $1 in + --prefix) + PREFIX="$2" + shift 2 + ;; + --prefix=*) + PREFIX="${1#*=}" + shift + ;; + -*) + echo "Error: Unknown option $1" + echo "Usage: $0 [--prefix ] {azure-region-code}" + echo "Example: $0 eastasia" + echo "Example: $0 --prefix cluster1 eastasia" + exit 1 + ;; + *) + if [ -z "$AZUREREGION" ]; then + AZUREREGION="$1" + else + echo "Error: Too many positional arguments." + echo "Usage: $0 [--prefix ] {azure-region-code}" + exit 1 + fi + shift + ;; + esac +done + +if [ -z "$AZUREREGION" ]; then + echo "Error: Azure region is required." + echo "Usage: $0 [--prefix ] {azure-region-code}" echo "Example: $0 eastasia" + echo "Example: $0 --prefix cluster1 eastasia" exit 1 fi -AZUREREGION=$1 + +# Set install directory based on prefix +if [ -n "$PREFIX" ]; then + INSTALL_DIR="openshift-install-${PREFIX}" + echo "Using prefix: $PREFIX" + echo "Install directory: $INSTALL_DIR" +else + INSTALL_DIR="openshift-install" +fi echo "---------------------" echo "Validating configuration" @@ -113,7 +156,11 @@ echo "---------------------" echo "defining cluster" echo "---------------------" PYTHON_CMD=$(get_python_cmd) -$PYTHON_CMD rhdp/rhdp-cluster-define.py ${AZUREREGION} +if [ -n "$PREFIX" ]; then + $PYTHON_CMD rhdp/rhdp-cluster-define.py --prefix "${PREFIX}" ${AZUREREGION} +else + $PYTHON_CMD rhdp/rhdp-cluster-define.py ${AZUREREGION} +fi echo "---------------------" echo "cluster defined" echo "---------------------" @@ -121,7 +168,7 @@ sleep 10 echo "---------------------" echo "openshift-install" echo "---------------------" -openshift-install create cluster --dir=./openshift-install +openshift-install create cluster --dir=./${INSTALL_DIR} echo "openshift-install done" echo "---------------------" echo "setting up secrets" @@ -133,7 +180,7 @@ sleep 60 echo "---------------------" echo "pattern install" echo "---------------------" -export KUBECONFIG="$(pwd)/openshift-install/auth/kubeconfig" +export KUBECONFIG="$(pwd)/${INSTALL_DIR}/auth/kubeconfig" ./pattern.sh make install From 4b6407225c2ff7a85b28a7cf066433c286c5d8c8 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 10 Dec 2025 15:13:24 +0900 Subject: [PATCH 02/39] feat: trustee GA Signed-off-by: Chris Butler --- values-simple.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/values-simple.yaml b/values-simple.yaml index a6405a2d..9e14023f 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -26,14 +26,14 @@ clusterGroup: source: redhat-operators channel: stable installPlanApproval: Manual - csv: sandboxed-containers-operator.v1.10.1 + csv: sandboxed-containers-operator.v1.11.0 trustee: name: trustee-operator namespace: trustee-operator-system source: redhat-operators channel: stable installPlanApproval: Manual - csv: trustee-operator.v0.4.1 + csv: trustee-operator.v1.0.0 cert-manager: name: openshift-cert-manager-operator namespace: cert-manager-operator From 9f0074bb7d510d3662d6acb08e329205316a5e79 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 17 Dec 2025 11:23:27 +0900 Subject: [PATCH 03/39] fix: test uplifting posture Signed-off-by: Chris Butler --- ansible/init-data-gzipper.yaml | 19 ++++++++++++++++++- ansible/initdata-default.toml.tpl | 23 ++++++++++++++++++++--- values-secret.yaml.template | 30 +++++++++++++++++++++--------- values-simple.yaml | 26 +++++++++++++++++++++----- 4 files changed, 80 insertions(+), 18 deletions(-) diff --git a/ansible/init-data-gzipper.yaml b/ansible/init-data-gzipper.yaml index 20459b84..c6e04f05 100644 --- a/ansible/init-data-gzipper.yaml +++ b/ansible/init-data-gzipper.yaml @@ -1,4 +1,4 @@ -- name: Gzip initdata +- name: Gzip initdata and register init data become: false connection: local hosts: localhost @@ -55,6 +55,22 @@ path: "{{ gz_path }}" register: gz_slurped + # This block runs a shell script that calculates a hash value (PCR8_HASH) derived from the contents of 'initdata.toml'. + # The script performs the following steps: + # 1. hash=$(sha256sum initdata.toml | cut -d' ' -f1): Computes the sha256 hash of 'initdata.toml' and assigns it to $hash. + # 2. initial_pcr=000000000000000000000000000000000000000000000000000000000000000: Initializes a string of zeros as the initial PCR value. + # 3. PCR8_HASH=$(echo -n "$initial_pcr$hash" | xxd -r -p | sha256sum | cut -d' ' -f1): Concatenates initial_pcr and $hash, converts from hex to binary, computes its sha256 hash, and stores the result as PCR8_HASH. + # 4. echo $PCR8_HASH: Outputs the PCR hash value. + # The important part: The 'register: pcr8_hash' registers the **stdout of the command**, which is the value output by 'echo $PCR8_HASH', as 'pcr8_hash.stdout' in Ansible. + # It does NOT register an environment variable, but rather the value actually printed by 'echo'. + - name: Register init data pcr into a var + ansible.builtin.shell: | + hash=$(sha256sum initdata.toml | cut -d' ' -f1) + initial_pcr=000000000000000000000000000000000000000000000000000000000000000 + PCR8_HASH=$(echo -n "$initial_pcr$hash" | xxd -r -p | sha256sum | cut -d' ' -f1) && echo $PCR8_HASH + register: pcr8_hash + + - name: Create/update ConfigMap with gzipped+base64 content kubernetes.core.k8s: kubeconfig: "{{ kubeconfig | default(omit) }}" @@ -67,3 +83,4 @@ namespace: "imperative" data: INITDATA: "{{ gz_slurped.content }}" + PCR8_HASH: "{{ pcr8_hash.stdout }}" diff --git a/ansible/initdata-default.toml.tpl b/ansible/initdata-default.toml.tpl index 9cadbc1c..b1cc5a0b 100644 --- a/ansible/initdata-default.toml.tpl +++ b/ansible/initdata-default.toml.tpl @@ -26,6 +26,10 @@ kbs_cert = """ """ ''' +[image] +image_security_policy_uri = 'kbs:///default/security-policy/osc +''' + "policy.rego" = ''' package agent_policy @@ -36,7 +40,6 @@ default CopyFileRequest := true default CreateContainerRequest := true default CreateSandboxRequest := true default DestroySandboxRequest := true -default ExecProcessRequest := false default GetMetricsRequest := true default GetOOMEventRequest := true default GuestDetailsRequest := true @@ -52,7 +55,6 @@ default RemoveStaleVirtiofsShareMountsRequest := true default ReseedRandomDevRequest := true default ResumeContainerRequest := true default SetGuestDateTimeRequest := true -default SetPolicyRequest := true default SignalProcessRequest := true default StartContainerRequest := true default StartTracingRequest := true @@ -64,5 +66,20 @@ default UpdateEphemeralMountsRequest := true default UpdateInterfaceRequest := true default UpdateRoutesRequest := true default WaitProcessRequest := true -default WriteStreamRequest := true +default ExecProcessRequest := false +default SetPolicyRequest := false +default WriteStreamRequest := false + +ExecProcessRequest if { + input_command = concat(" ", input.process.Args) + some allowed_command in policy_data.allowed_commands + input_command == allowed_command +} + +policy_data := { + "allowed_commands": [ + "curl http://127.0.0.1:8006/cdh/resource/default/attestation-status/status", + "curl http://127.0.0.1:8006/cdh/resource/default/attestation-status/random" + ] +} ''' \ No newline at end of file diff --git a/values-secret.yaml.template b/values-secret.yaml.template index fe410d42..8d614d8b 100644 --- a/values-secret.yaml.template +++ b/values-secret.yaml.template @@ -6,21 +6,32 @@ version: "2.0" # automatically generated inside the vault this should not really matter) secrets: - - name: 'sshKey' - vaultPrefixes: - - global - fields: - - name: id_rsa.pub - path: ~/.coco-pattern/id_rsa.pub - - name: id_rsa - path: ~/.coco-pattern/id_rsa + - name: 'securityPolicyConfig' vaultPrefixes: - hub fields: - name: osc - path: ~/.coco-pattern/security-policy-config.json + value: | + { + "default": [ + { + "type": "insecureAcceptAnything" + }], + "transports": {} + } + + - name: attestationStatus + vaultPrefixes: + - hub + fields: + - name: status + value: 'attested' + - name: random + value: '' + onMissingValue: generate + vaultPolicy: validatedPatternDefaultPolicy - name: kbsPublicKey vaultPrefixes: @@ -61,3 +72,4 @@ secrets: value: '' onMissingValue: generate vaultPolicy: validatedPatternDefaultPolicy + diff --git a/values-simple.yaml b/values-simple.yaml index 9e14023f..99f90c3c 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -77,23 +77,34 @@ clusterGroup: name: trustee namespace: trustee-operator-system #upstream config project: trustee - chart: trustee - chartVersion: 0.1.* + repoURL: https://github.com/butler54/trustee-chart.git + targetRevision: merge-certs + # chart: trustee + # chartVersion: 0.1.* # Use the override file to specify the list of secrets accessible to trustee from the ESO backend (today by default, Vault). extraValueFiles: - '$patternref/overrides/values-trustee.yaml' + # sandbox: + # name: sandbox + # namespace: openshift-sandboxed-containers-operator #upstream config + # project: sandbox + # chart: sandboxed-containers + # chartVersion: 0.0.* sandbox: name: sandbox namespace: openshift-sandboxed-containers-operator #upstream config project: sandbox - chart: sandboxed-containers - chartVersion: 0.0.* + repoURL: https://github.com/butler54/sandboxed-containers-chart.git + targetRevision: remove-ssh + sandbox-policies: name: sandbox-policies namespace: openshift-sandboxed-containers-operator #upstream config chart: sandboxed-policies chartVersion: 0.0.* - +# path: applications/pipeline +# repoURL: https://github.com/you/applications.git +# targetRevision: stable # Letsencrypt is not required anymore for trustee. # It's only here if you need it for your needs. letsencrypt: @@ -117,6 +128,11 @@ clusterGroup: project: workloads path: charts/coco-supported/kbs-access + tmp: + name: tmp + namespace: default + project: hub + path: charts/tmp imperative: # NOTE: We *must* use lists and not hashes. As hashes lose ordering once parsed by helm From 04c9e3c628501b26f6bcfbea20345a38d14cdbed Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 17 Dec 2025 11:52:23 +0900 Subject: [PATCH 04/39] fix: add url path for a repo Signed-off-by: Chris Butler --- values-simple.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/values-simple.yaml b/values-simple.yaml index 99f90c3c..65a45dac 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -79,6 +79,7 @@ clusterGroup: project: trustee repoURL: https://github.com/butler54/trustee-chart.git targetRevision: merge-certs + path: / # chart: trustee # chartVersion: 0.1.* # Use the override file to specify the list of secrets accessible to trustee from the ESO backend (today by default, Vault). @@ -96,6 +97,7 @@ clusterGroup: project: sandbox repoURL: https://github.com/butler54/sandboxed-containers-chart.git targetRevision: remove-ssh + path: / sandbox-policies: name: sandbox-policies From af4185179d4e6d056f93a9902955928eb23eb8ac Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 17 Dec 2025 11:54:53 +0900 Subject: [PATCH 05/39] fix: remove dead chart Signed-off-by: Chris Butler --- values-simple.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/values-simple.yaml b/values-simple.yaml index 65a45dac..8bbd7832 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -130,12 +130,6 @@ clusterGroup: project: workloads path: charts/coco-supported/kbs-access - tmp: - name: tmp - namespace: default - project: hub - path: charts/tmp - imperative: # NOTE: We *must* use lists and not hashes. As hashes lose ordering once parsed by helm # The default schedule is every 10 minutes: imperative.schedule From 71dcba59f67b5770d22f1092a44181f21bb002fe Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 17 Dec 2025 11:56:58 +0900 Subject: [PATCH 06/39] fix: relative urls Signed-off-by: Chris Butler --- values-simple.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/values-simple.yaml b/values-simple.yaml index 8bbd7832..595c848f 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -79,7 +79,7 @@ clusterGroup: project: trustee repoURL: https://github.com/butler54/trustee-chart.git targetRevision: merge-certs - path: / + path: ./ # chart: trustee # chartVersion: 0.1.* # Use the override file to specify the list of secrets accessible to trustee from the ESO backend (today by default, Vault). @@ -97,7 +97,7 @@ clusterGroup: project: sandbox repoURL: https://github.com/butler54/sandboxed-containers-chart.git targetRevision: remove-ssh - path: / + path: ./ sandbox-policies: name: sandbox-policies From a3a23c80544abd5989426b824bdc793a6b148fde Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 17 Dec 2025 14:41:28 +0900 Subject: [PATCH 07/39] fix: update with security policies Signed-off-by: Chris Butler --- ansible/init-data-gzipper.yaml | 1 + ansible/initdata-default.toml.tpl | 2 +- values-global.yaml | 2 ++ values-secret.yaml.template | 50 ++++++++++++++++++++++++++----- values-simple.yaml | 5 ++-- 5 files changed, 50 insertions(+), 10 deletions(-) diff --git a/ansible/init-data-gzipper.yaml b/ansible/init-data-gzipper.yaml index c6e04f05..1d22dfea 100644 --- a/ansible/init-data-gzipper.yaml +++ b/ansible/init-data-gzipper.yaml @@ -7,6 +7,7 @@ kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}" cluster_platform: "{{ global.clusterPlatform | default('none') | lower }}" hub_domain: "{{ global.hubClusterDomain | default('none') | lower}}" + security_policy_flavour: "{{ global.coco.securityPolicyFlavour | default('insecure') }}" template_src: "initdata-default.toml.tpl" tasks: - name: Create temporary working directory diff --git a/ansible/initdata-default.toml.tpl b/ansible/initdata-default.toml.tpl index b1cc5a0b..9d9442dd 100644 --- a/ansible/initdata-default.toml.tpl +++ b/ansible/initdata-default.toml.tpl @@ -27,7 +27,7 @@ kbs_cert = """ ''' [image] -image_security_policy_uri = 'kbs:///default/security-policy/osc +image_security_policy_uri = 'kbs:///default/security-policy/{{ security_policy_flavour }} ''' "policy.rego" = ''' diff --git a/values-global.yaml b/values-global.yaml index e91c7f0c..bead30eb 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -11,6 +11,7 @@ global: # This defines whether or not to use upstream resources for CoCo. # Defines whether or not the hub cluster can be used for confidential containers coco: + securityPolicyFlavour: "insecure" # insecure, signed or reject is expected. azure: defaultVMFlavour: "Standard_DC2as_v5" VMFlavours: "Standard_DC2as_v5,Standard_DC4as_v5,Standard_DC8as_v5,Standard_DC16as_v5" @@ -24,6 +25,7 @@ main: clusterGroupChartVersion: 0.9.* # Common secret store configuration used across multiple charts +# Warning do not rely on this. it does not consistently apply. secretStore: name: vault-backend kind: ClusterSecretStore diff --git a/values-secret.yaml.template b/values-secret.yaml.template index 8d614d8b..288b78f5 100644 --- a/values-secret.yaml.template +++ b/values-secret.yaml.template @@ -8,19 +8,55 @@ version: "2.0" secrets: - - name: 'securityPolicyConfig' + - name: securityPolicyConfig vaultPrefixes: - hub fields: - - name: osc + # Accept all images without verification (INSECURE - dev/testing only) + - name: insecure value: | { - "default": [ - { - "type": "insecureAcceptAnything" - }], - "transports": {} + "default": [{"type": "insecureAcceptAnything"}], + "transports": {} } + # Reject all images (useful for testing policy enforcement) + - name: reject + value: | + { + "default": [{"type": "reject"}], + "transports": {} + } + # Only accept signed images (production) + # Edit the transports section to add your signed images. + # Each image needs a corresponding cosign public key in cosign-keys secret. + # The keys much line up with the keys below + - name: signed + value: | + { + "default": [{"type": "reject"}], + "transports": { + "docker": { + "registry.example.com/my-image": [ + { + "type": "sigstoreSigned", + "keyPath": "kbs:///default/cosign-keys/key-0" + } + ] + } + } + } + + # Cosign public keys for image signature verification + # Required when using the "signed" policy above. + # Add your cosign public key files here. + # Generate a cosign key pair: cosign generate-key-pair + #- name: cosign-keys + # vaultPrefixes: + # - hub + # fields: + # - name: key-0 + # path: ~/.coco-pattern/trustee/cosign-key-0.pub + - name: attestationStatus vaultPrefixes: diff --git a/values-simple.yaml b/values-simple.yaml index 595c848f..09dda1e3 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -80,11 +80,12 @@ clusterGroup: repoURL: https://github.com/butler54/trustee-chart.git targetRevision: merge-certs path: ./ + extraValueFiles: + - '$patternref/overrides/values-trustee.yaml' # chart: trustee # chartVersion: 0.1.* # Use the override file to specify the list of secrets accessible to trustee from the ESO backend (today by default, Vault). - extraValueFiles: - - '$patternref/overrides/values-trustee.yaml' + # sandbox: # name: sandbox # namespace: openshift-sandboxed-containers-operator #upstream config From 7051aa942ed230b1ede6e2216981f7c447a8371d Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 17 Dec 2025 14:51:54 +0900 Subject: [PATCH 08/39] fix: clean up secrets and gen secrets script Signed-off-by: Chris Butler --- scripts/gen-secrets.sh | 22 ---------------------- values-secret.yaml.template | 8 +------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/scripts/gen-secrets.sh b/scripts/gen-secrets.sh index 25c4713a..1196b51d 100755 --- a/scripts/gen-secrets.sh +++ b/scripts/gen-secrets.sh @@ -4,8 +4,6 @@ echo "Creating secrets as required" echo COCO_SECRETS_DIR="${HOME}/.coco-pattern" -SECURITY_POLICY_FILE="${COCO_SECRETS_DIR}/security-policy-config.json" -SSH_KEY_FILE="${COCO_SECRETS_DIR}/id_rsa" KBS_PRIVATE_KEY="${COCO_SECRETS_DIR}/kbsPrivateKey" KBS_PUBLIC_KEY="${COCO_SECRETS_DIR}/kbsPublicKey" SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" @@ -13,19 +11,6 @@ VALUES_FILE="${HOME}/values-secret-coco-pattern.yaml" mkdir -p ${COCO_SECRETS_DIR} -if [ ! -f "${SECURITY_POLICY_FILE}" ]; then -echo "Creating security policy" -cat > ${SECURITY_POLICY_FILE} < Date: Mon, 22 Dec 2025 16:59:18 +0900 Subject: [PATCH 09/39] fix: correct charts Signed-off-by: Chris Butler --- ansible/initdata-default.toml.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/initdata-default.toml.tpl b/ansible/initdata-default.toml.tpl index 9d9442dd..47246798 100644 --- a/ansible/initdata-default.toml.tpl +++ b/ansible/initdata-default.toml.tpl @@ -24,10 +24,10 @@ url = "https://kbs.{{ hub_domain }}" kbs_cert = """ {{ trustee_cert }} """ -''' + [image] -image_security_policy_uri = 'kbs:///default/security-policy/{{ security_policy_flavour }} +image_security_policy_uri = 'kbs:///default/security-policy/{{ security_policy_flavour }}' ''' "policy.rego" = ''' From 7c90be737be15945d17aa265bbe135ac66d5ae75 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 5 Jan 2026 14:33:44 +0900 Subject: [PATCH 10/39] chore: add flag Signed-off-by: Chris Butler --- values-global.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/values-global.yaml b/values-global.yaml index bead30eb..ceeb1241 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -12,6 +12,7 @@ global: # Defines whether or not the hub cluster can be used for confidential containers coco: securityPolicyFlavour: "insecure" # insecure, signed or reject is expected. + secured: true # true or false. If true, the cluster will be secured. If false, the cluster will be insecure. azure: defaultVMFlavour: "Standard_DC2as_v5" VMFlavours: "Standard_DC2as_v5,Standard_DC4as_v5,Standard_DC8as_v5,Standard_DC16as_v5" From 73d8cac1e6c77f5faadbb08c22c3c351b1d659b6 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 5 Jan 2026 16:32:54 +0900 Subject: [PATCH 11/39] feat: add components required for pcr securing Signed-off-by: Chris Butler --- scripts/get-pcr.sh | 103 ++++++++++++++++++++++++++++++++++++ values-secret.yaml.template | 12 +++++ 2 files changed, 115 insertions(+) create mode 100755 scripts/get-pcr.sh diff --git a/scripts/get-pcr.sh b/scripts/get-pcr.sh new file mode 100755 index 00000000..acb34986 --- /dev/null +++ b/scripts/get-pcr.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +set -e + +# Script to retrieve the sandboxed container operator CSV for the current clusterGroup +# using the pull secret for authentication if needed. + +# 1. Locate pull secret +PULL_SECRET_PATH="${HOME}/pull-secret.json" +if [ ! -f "$PULL_SECRET_PATH" ]; then + if [ -n "${PULL_SECRET}" ]; then + PULL_SECRET_PATH="${PULL_SECRET}" + if [ ! -f "$PULL_SECRET_PATH" ]; then + echo "ERROR: Pull secret file not found at path specified in PULL_SECRET: $PULL_SECRET_PATH" + exit 1 + fi + else + echo "ERROR: Pull secret not found at ~/pull-secret.json" + echo "Please either place your pull secret at ~/pull-secret.json or set the PULL_SECRET environment variable" + exit 1 + fi +fi + +echo "Using pull secret: $PULL_SECRET_PATH" + +# 2. Check for required tools +if ! command -v yq &> /dev/null; then + echo "ERROR: yq is required but not installed" + echo "Please install yq: https://github.com/mikefarah/yq#install" + exit 1 +fi + +# 3. Check values-global.yaml exists +if [ ! -f "values-global.yaml" ]; then + echo "ERROR: values-global.yaml not found in current directory" + echo "Please run this script from the root directory of the project" + exit 1 +fi + +# 4. Get the active clusterGroupName from values-global.yaml +CLUSTER_GROUP_NAME=$(yq eval '.main.clusterGroupName' values-global.yaml) + +if [ -z "$CLUSTER_GROUP_NAME" ] || [ "$CLUSTER_GROUP_NAME" == "null" ]; then + echo "ERROR: Could not determine clusterGroupName from values-global.yaml" + echo "Expected: main.clusterGroupName to be set" + exit 1 +fi + +echo "Active clusterGroup: $CLUSTER_GROUP_NAME" + +# 5. Locate the values file for the active clusterGroup +VALUES_FILE="values-${CLUSTER_GROUP_NAME}.yaml" + +if [ ! -f "$VALUES_FILE" ]; then + echo "ERROR: Values file for clusterGroup not found: $VALUES_FILE" + exit 1 +fi + +# 6. Get the sandboxed container operator CSV from the clusterGroup values +SANDBOX_CSV=$(yq eval '.clusterGroup.subscriptions.sandbox.csv' "$VALUES_FILE") + +if [ -z "$SANDBOX_CSV" ] || [ "$SANDBOX_CSV" == "null" ]; then + echo "WARNING: No sandboxed container operator CSV found in $VALUES_FILE" + echo "The subscription clusterGroup.subscriptions.sandbox.csv is not defined" + exit 0 +fi + +# Extract version from CSV (e.g., "sandboxed-containers-operator.v1.11.0" -> "1.11.0") +# Remove everything up to and including ".v" +SANDBOX_VERSION="${SANDBOX_CSV##*.v}" + +echo "Sandboxed container operator CSV: $SANDBOX_CSV" +echo "Version: $SANDBOX_VERSION" +# alternatively, use the operator-version tag. +# OSC_VERSION=1.11.1 +VERITY_IMAGE=registry.redhat.io/openshift-sandboxed-containers/osc-dm-verity-image + +TAG=$(skopeo inspect --authfile $PULL_SECRET_PATH docker://${VERITY_IMAGE}:${SANDBOX_VERSION} | jq -r .Digest) + +IMAGE=${VERITY_IMAGE}@${TAG} + +echo "IMAGE: $IMAGE" + +curl -L https://tuf-default.apps.rosa.rekor-prod.2jng.p3.openshiftapps.com/targets/rekor.pub -o rekor.pub +curl -L https://security.access.redhat.com/data/63405576.txt -o cosign-pub-key.pem +# export REGISTRY_AUTH_FILE=${PULL_SECRET_PATH} +# echo "REGISTRY_AUTH_FILE: $REGISTRY_AUTH_FILE" +# export SIGSTORE_REKOR_PUBLIC_KEY=${PWD}/rekor.pub +# echo "SIGSTORE_REKOR_PUBLIC_KEY: $SIGSTORE_REKOR_PUBLIC_KEY" +# cosign verify --key cosign-pub-key.pem --output json --rekor-url=https://rekor-server-default.apps.rosa.rekor-prod.2jng.p3.openshiftapps.com $IMAGE > cosign_verify.log + + +# Ensure output directory exists +mkdir -p ~/.coco-pattern + +# Download the measurements using podman cp (works on macOS with remote podman) +podman pull --authfile $PULL_SECRET_PATH $IMAGE + +cid=$(podman create --entrypoint /bin/true $IMAGE) +echo "CID: ${cid}" +podman cp $cid:/image/measurements.json ~/.coco-pattern/measurements.json +podman rm $cid + +echo "Measurements saved to ~/.coco-pattern/measurements.json" \ No newline at end of file diff --git a/values-secret.yaml.template b/values-secret.yaml.template index 1a017a10..b3df87dd 100644 --- a/values-secret.yaml.template +++ b/values-secret.yaml.template @@ -58,6 +58,18 @@ secrets: # path: ~/.coco-pattern/cosign-key-0.pub + # Cosign public keys for image signature verification + # Required when using the "signed" policy above. + # Add your cosign public key files here. + # Generate a cosign key pair: cosign generate-key-pair + #- name: pcrStash + # vaultPrefixes: + # - hub + # fields: + # - name: json + # path: ~/.coco-pattern/measurements.json + + - name: attestationStatus vaultPrefixes: - hub From 9d09e23c1074c806463111240176bc7c13a8bee8 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 6 Jan 2026 11:57:45 +0900 Subject: [PATCH 12/39] fix: add corrected health check Signed-off-by: Chris Butler --- values-simple.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/values-simple.yaml b/values-simple.yaml index 09dda1e3..c6fd4ce5 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -3,6 +3,36 @@ clusterGroup: name: simple isHubCluster: true + # Override health check for Subscriptions to treat UpgradePending as healthy + # Only applies to pinned CSV subscriptions (sandbox and trustee) + argoCD: + resourceHealthChecks: + - group: operators.coreos.com + kind: Subscription + check: | + local hs = {} + -- Only apply custom logic to pinned subscriptions + local isPinned = (obj.metadata.name == "sandboxed-containers-operator" or + obj.metadata.name == "trustee-operator") + if obj.status ~= nil and obj.status.state ~= nil then + local state = obj.status.state + if state == "AtLatestKnown" then + hs.status = "Healthy" + hs.message = state + return hs + elseif state == "UpgradePending" and isPinned then + hs.status = "Healthy" + hs.message = "Pinned subscription at desired version" + return hs + elseif state == "UpgradePending" then + hs.status = "Progressing" + hs.message = "Upgrade pending approval" + return hs + end + end + hs.status = "Progressing" + hs.message = "Waiting for Subscription to be ready" + return hs namespaces: - open-cluster-management - vault From e3c0559bc3563e74cb14ab527400782ba5c9e86d Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 6 Jan 2026 16:10:21 +0900 Subject: [PATCH 13/39] fix: correctly override values Signed-off-by: Chris Butler --- overrides/values-trustee.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/overrides/values-trustee.yaml b/overrides/values-trustee.yaml index ee42e416..03dd120a 100644 --- a/overrides/values-trustee.yaml +++ b/overrides/values-trustee.yaml @@ -6,4 +6,9 @@ kbs: - name: "kbsres1" # name is the name of the k8s secret that will be presented to trustee and accessible via the CDH key: "secret/data/hub/kbsres1" # this is the path to the secret in vault. - name: "passphrase" - key: "secret/data/hub/passphrase" \ No newline at end of file + key: "secret/data/hub/passphrase" +# Override the default values for the coco pattern this is because when testing against a branch strange stuff happens +# FIXME: Don't commit this to main +global: + coco: + secured: true # true or false. If true, the cluster will be secured. If false, the cluster will be insecure. \ No newline at end of file From ab3a9267b0ddbafbc8bebca3be605abad8b1c6f6 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 6 Jan 2026 16:24:34 +0900 Subject: [PATCH 14/39] fix: temporarily change for single source charts Signed-off-by: Chris Butler --- values-simple.yaml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/values-simple.yaml b/values-simple.yaml index c6fd4ce5..4689805d 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -110,11 +110,19 @@ clusterGroup: repoURL: https://github.com/butler54/trustee-chart.git targetRevision: merge-certs path: ./ - extraValueFiles: - - '$patternref/overrides/values-trustee.yaml' - # chart: trustee - # chartVersion: 0.1.* - # Use the override file to specify the list of secrets accessible to trustee from the ESO backend (today by default, Vault). + # Note: extraValueFiles with $patternref don't work for external repoURL (single-source app) + # Using overrides instead to pass values directly + overrides: + - name: global.coco.secured + value: "true" + - name: kbs.secretResources[0].name + value: kbsres1 + - name: kbs.secretResources[0].key + value: secret/data/hub/kbsres1 + - name: kbs.secretResources[1].name + value: passphrase + - name: kbs.secretResources[1].key + value: secret/data/hub/passphrase # sandbox: # name: sandbox From 5da188f3d5704f7acf9c404e3ad8e7a5272f7954 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 12 Jan 2026 10:34:00 +0900 Subject: [PATCH 15/39] fix: clean up measurements Signed-off-by: Chris Butler --- scripts/get-pcr.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/get-pcr.sh b/scripts/get-pcr.sh index acb34986..f55bd5f5 100755 --- a/scripts/get-pcr.sh +++ b/scripts/get-pcr.sh @@ -97,7 +97,11 @@ podman pull --authfile $PULL_SECRET_PATH $IMAGE cid=$(podman create --entrypoint /bin/true $IMAGE) echo "CID: ${cid}" -podman cp $cid:/image/measurements.json ~/.coco-pattern/measurements.json +podman cp $cid:/image/measurements.json ~/.coco-pattern/measurements-raw.json podman rm $cid -echo "Measurements saved to ~/.coco-pattern/measurements.json" \ No newline at end of file +# Trim leading "0x" from all measurement values +jq 'walk(if type == "string" and startswith("0x") then .[2:] else . end)' \ + ~/.coco-pattern/measurements-raw.json > ~/.coco-pattern/measurements.json + +echo "Measurements saved to ~/.coco-pattern/measurements.json (0x prefixes removed)" \ No newline at end of file From 5b48dba24e861a55d31c2a84890fe1ac1acc889a Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 12 Jan 2026 10:57:04 +0900 Subject: [PATCH 16/39] fix: deal with existing file Signed-off-by: Chris Butler --- scripts/get-pcr.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/get-pcr.sh b/scripts/get-pcr.sh index f55bd5f5..c55baa7d 100755 --- a/scripts/get-pcr.sh +++ b/scripts/get-pcr.sh @@ -92,6 +92,9 @@ curl -L https://security.access.redhat.com/data/63405576.txt -o cosign-pub-key.p # Ensure output directory exists mkdir -p ~/.coco-pattern +# Clean up any existing measurement files +rm -f ~/.coco-pattern/measurements-raw.json ~/.coco-pattern/measurements.json + # Download the measurements using podman cp (works on macOS with remote podman) podman pull --authfile $PULL_SECRET_PATH $IMAGE From 4c9ea24430bf97f98f4c542bbdc2349ced4e5170 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 13 Jan 2026 12:31:37 +0900 Subject: [PATCH 17/39] fix: remove uneeded resource constrains Signed-off-by: Chris Butler --- values-simple.yaml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/values-simple.yaml b/values-simple.yaml index 4689805d..718d7672 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -5,34 +5,6 @@ clusterGroup: isHubCluster: true # Override health check for Subscriptions to treat UpgradePending as healthy # Only applies to pinned CSV subscriptions (sandbox and trustee) - argoCD: - resourceHealthChecks: - - group: operators.coreos.com - kind: Subscription - check: | - local hs = {} - -- Only apply custom logic to pinned subscriptions - local isPinned = (obj.metadata.name == "sandboxed-containers-operator" or - obj.metadata.name == "trustee-operator") - if obj.status ~= nil and obj.status.state ~= nil then - local state = obj.status.state - if state == "AtLatestKnown" then - hs.status = "Healthy" - hs.message = state - return hs - elseif state == "UpgradePending" and isPinned then - hs.status = "Healthy" - hs.message = "Pinned subscription at desired version" - return hs - elseif state == "UpgradePending" then - hs.status = "Progressing" - hs.message = "Upgrade pending approval" - return hs - end - end - hs.status = "Progressing" - hs.message = "Waiting for Subscription to be ready" - return hs namespaces: - open-cluster-management - vault From 7ca63ce5b3db7831815d51dcb38cf43b90b4b4d5 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Thu, 5 Feb 2026 21:29:43 +0900 Subject: [PATCH 18/39] fix: add policy keywords Signed-off-by: Chris Butler --- ansible/initdata-default.toml.tpl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ansible/initdata-default.toml.tpl b/ansible/initdata-default.toml.tpl index 47246798..46417dd4 100644 --- a/ansible/initdata-default.toml.tpl +++ b/ansible/initdata-default.toml.tpl @@ -9,9 +9,7 @@ url = "https://kbs.{{ hub_domain }}" [token_configs.kbs] url = "https://kbs.{{ hub_domain }}" -cert = """ -{{ trustee_cert }} -""" +cert = """{{ trustee_cert }}""" ''' "cdh.toml" = ''' @@ -21,9 +19,7 @@ credentials = [] [kbc] name = "cc_kbc" url = "https://kbs.{{ hub_domain }}" -kbs_cert = """ -{{ trustee_cert }} -""" +kbs_cert = """{{ trustee_cert }}""" [image] @@ -33,6 +29,10 @@ image_security_policy_uri = 'kbs:///default/security-policy/{{ security_policy_f "policy.rego" = ''' package agent_policy +import future.keywords.in +import future.keywords.if +import future.keywords.every + default AddARPNeighborsRequest := true default AddSwapRequest := true default CloseStdinRequest := true @@ -82,4 +82,4 @@ policy_data := { "curl http://127.0.0.1:8006/cdh/resource/default/attestation-status/random" ] } -''' \ No newline at end of file +''' From 82fbc25fdfa15aad2050d9e0d2f5b941af1e6aab Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Thu, 5 Feb 2026 23:38:28 +0900 Subject: [PATCH 19/39] fix: allow policy changes Signed-off-by: Chris Butler --- ansible/initdata-default.toml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/initdata-default.toml.tpl b/ansible/initdata-default.toml.tpl index 46417dd4..f162f44d 100644 --- a/ansible/initdata-default.toml.tpl +++ b/ansible/initdata-default.toml.tpl @@ -67,7 +67,7 @@ default UpdateInterfaceRequest := true default UpdateRoutesRequest := true default WaitProcessRequest := true default ExecProcessRequest := false -default SetPolicyRequest := false +default SetPolicyRequest := true default WriteStreamRequest := false ExecProcessRequest if { From 1ea2b53705052185a1f0468f36aad9fa6ea2109b Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Thu, 5 Feb 2026 23:52:43 +0900 Subject: [PATCH 20/39] fix: correct pcr8 rendering Signed-off-by: Chris Butler --- ansible/init-data-gzipper.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/init-data-gzipper.yaml b/ansible/init-data-gzipper.yaml index 1d22dfea..495e2b21 100644 --- a/ansible/init-data-gzipper.yaml +++ b/ansible/init-data-gzipper.yaml @@ -66,7 +66,7 @@ # It does NOT register an environment variable, but rather the value actually printed by 'echo'. - name: Register init data pcr into a var ansible.builtin.shell: | - hash=$(sha256sum initdata.toml | cut -d' ' -f1) + hash=$(sha256sum {{ rendered_path }} | cut -d' ' -f1) initial_pcr=000000000000000000000000000000000000000000000000000000000000000 PCR8_HASH=$(echo -n "$initial_pcr$hash" | xxd -r -p | sha256sum | cut -d' ' -f1) && echo $PCR8_HASH register: pcr8_hash From d8e70deabe69261868327167827dd4f7249e8774 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Fri, 6 Feb 2026 00:00:14 +0900 Subject: [PATCH 21/39] fix: stuff Signed-off-by: Chris Butler --- ansible/init-data-gzipper.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/init-data-gzipper.yaml b/ansible/init-data-gzipper.yaml index 495e2b21..18b2c245 100644 --- a/ansible/init-data-gzipper.yaml +++ b/ansible/init-data-gzipper.yaml @@ -66,7 +66,7 @@ # It does NOT register an environment variable, but rather the value actually printed by 'echo'. - name: Register init data pcr into a var ansible.builtin.shell: | - hash=$(sha256sum {{ rendered_path }} | cut -d' ' -f1) + hash=$(sha256sum "{{ rendered_path }}" | cut -d' ' -f1) initial_pcr=000000000000000000000000000000000000000000000000000000000000000 PCR8_HASH=$(echo -n "$initial_pcr$hash" | xxd -r -p | sha256sum | cut -d' ' -f1) && echo $PCR8_HASH register: pcr8_hash From b70dcd2a9613a8b62cc8f835d7da031956d480f8 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Fri, 6 Feb 2026 10:36:08 +0900 Subject: [PATCH 22/39] fix: make sure xxd is in the imperative container Signed-off-by: Chris Butler --- ansible/initdata-default.toml.tpl | 2 +- values-simple.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/initdata-default.toml.tpl b/ansible/initdata-default.toml.tpl index f162f44d..fc64b3c9 100644 --- a/ansible/initdata-default.toml.tpl +++ b/ansible/initdata-default.toml.tpl @@ -82,4 +82,4 @@ policy_data := { "curl http://127.0.0.1:8006/cdh/resource/default/attestation-status/random" ] } -''' +''' \ No newline at end of file diff --git a/values-simple.yaml b/values-simple.yaml index 718d7672..bea146e0 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -148,6 +148,7 @@ clusterGroup: # imagePullPolicy is set to always: imperative.imagePullPolicy # For additional overrides that apply to the jobs, please refer to # https://validatedpatterns.io/imperative-actions/#additional-job-customizations + image: ghcr.io/butler54/imperative-container:latest jobs: - name: install-deps playbook: ansible/install-deps.yaml From 35762ba19de7b3f7fa19fae9068ece0129ae61de Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Fri, 6 Feb 2026 10:51:56 +0900 Subject: [PATCH 23/39] fix: decrease encoding variations Signed-off-by: Chris Butler --- ansible/init-data-gzipper.yaml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/ansible/init-data-gzipper.yaml b/ansible/init-data-gzipper.yaml index 18b2c245..79099158 100644 --- a/ansible/init-data-gzipper.yaml +++ b/ansible/init-data-gzipper.yaml @@ -46,15 +46,11 @@ mode: "0600" - - name: Gzip the rendered content + - name: Gzip and base64 encode the rendered content ansible.builtin.shell: | - gzip -c "{{ rendered_path }}" > "{{ gz_path }}" - changed_when: true - - - name: Read gzip as base64 - ansible.builtin.slurp: - path: "{{ gz_path }}" - register: gz_slurped + cat "{{ rendered_path }}" | gzip | base64 -w0 + register: initdata_encoded + changed_when: false # This block runs a shell script that calculates a hash value (PCR8_HASH) derived from the contents of 'initdata.toml'. # The script performs the following steps: From 80c2df45f22cb8e85db5a4114219a6ab604b17ea Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Fri, 6 Feb 2026 11:00:28 +0900 Subject: [PATCH 24/39] fix: use consistent gzip | base64 approach for initdata encoding Replace separate gzip-to-file and slurp approach with direct pipe: cat file | gzip | base64 -w0 This matches the approach used elsewhere in the codebase and fixes deployment failures caused by inconsistent encoding. Co-Authored-By: Claude Opus 4.5 --- ansible/init-data-gzipper.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ansible/init-data-gzipper.yaml b/ansible/init-data-gzipper.yaml index 79099158..e2fcce12 100644 --- a/ansible/init-data-gzipper.yaml +++ b/ansible/init-data-gzipper.yaml @@ -37,7 +37,6 @@ - name: Define temp file paths ansible.builtin.set_fact: rendered_path: "{{ tmpdir.path }}/rendered.toml" - gz_path: "{{ tmpdir.path }}/rendered.toml.gz" - name: Render template to temp file ansible.builtin.template: @@ -79,5 +78,5 @@ name: "initdata" namespace: "imperative" data: - INITDATA: "{{ gz_slurped.content }}" + INITDATA: "{{ initdata_encoded.stdout }}" PCR8_HASH: "{{ pcr8_hash.stdout }}" From c4c57d2f8ed6897227fac3b9cacffa26317073a6 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Fri, 6 Feb 2026 11:06:46 +0900 Subject: [PATCH 25/39] fix: correct initial_pcr to have 64 zeros (32 bytes) SHA-256 produces 256 bits = 32 bytes = 64 hex characters. The initial PCR value was missing one zero. Co-Authored-By: Claude Opus 4.5 --- ansible/init-data-gzipper.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/init-data-gzipper.yaml b/ansible/init-data-gzipper.yaml index e2fcce12..3ffeddbf 100644 --- a/ansible/init-data-gzipper.yaml +++ b/ansible/init-data-gzipper.yaml @@ -54,7 +54,7 @@ # This block runs a shell script that calculates a hash value (PCR8_HASH) derived from the contents of 'initdata.toml'. # The script performs the following steps: # 1. hash=$(sha256sum initdata.toml | cut -d' ' -f1): Computes the sha256 hash of 'initdata.toml' and assigns it to $hash. - # 2. initial_pcr=000000000000000000000000000000000000000000000000000000000000000: Initializes a string of zeros as the initial PCR value. + # 2. initial_pcr=0000000000000000000000000000000000000000000000000000000000000000: Initializes a string of zeros as the initial PCR value. # 3. PCR8_HASH=$(echo -n "$initial_pcr$hash" | xxd -r -p | sha256sum | cut -d' ' -f1): Concatenates initial_pcr and $hash, converts from hex to binary, computes its sha256 hash, and stores the result as PCR8_HASH. # 4. echo $PCR8_HASH: Outputs the PCR hash value. # The important part: The 'register: pcr8_hash' registers the **stdout of the command**, which is the value output by 'echo $PCR8_HASH', as 'pcr8_hash.stdout' in Ansible. @@ -62,7 +62,7 @@ - name: Register init data pcr into a var ansible.builtin.shell: | hash=$(sha256sum "{{ rendered_path }}" | cut -d' ' -f1) - initial_pcr=000000000000000000000000000000000000000000000000000000000000000 + initial_pcr=0000000000000000000000000000000000000000000000000000000000000000 PCR8_HASH=$(echo -n "$initial_pcr$hash" | xxd -r -p | sha256sum | cut -d' ' -f1) && echo $PCR8_HASH register: pcr8_hash From 36014e92d8bf05dbca0e4363a836e1f1975a4e7f Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Fri, 6 Feb 2026 11:21:21 +0900 Subject: [PATCH 26/39] fix: hard return at EoL Signed-off-by: Chris Butler --- ansible/initdata-default.toml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/initdata-default.toml.tpl b/ansible/initdata-default.toml.tpl index fc64b3c9..f162f44d 100644 --- a/ansible/initdata-default.toml.tpl +++ b/ansible/initdata-default.toml.tpl @@ -82,4 +82,4 @@ policy_data := { "curl http://127.0.0.1:8006/cdh/resource/default/attestation-status/random" ] } -''' \ No newline at end of file +''' From aa924a4d969cede6faca780404799e6a556faf3e Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Sun, 8 Feb 2026 16:28:59 +0900 Subject: [PATCH 27/39] fix: stuff Signed-off-by: Chris Butler --- ansible/initdata-default.toml.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/initdata-default.toml.tpl b/ansible/initdata-default.toml.tpl index f162f44d..3fd1ecc3 100644 --- a/ansible/initdata-default.toml.tpl +++ b/ansible/initdata-default.toml.tpl @@ -1,4 +1,4 @@ -algorithm = "sha384" +algorithm = "sha256" version = "0.1.0" [data] From 8668989ac69161c6b1cac523ba2c871d4521fde2 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 9 Feb 2026 11:25:37 +0800 Subject: [PATCH 28/39] chore: pin for testing stability Signed-off-by: Chris Butler --- values-simple.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values-simple.yaml b/values-simple.yaml index bea146e0..7a549201 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -114,7 +114,7 @@ clusterGroup: name: sandbox-policies namespace: openshift-sandboxed-containers-operator #upstream config chart: sandboxed-policies - chartVersion: 0.0.* + chartVersion: 0.0.4 # pin to ensure latest for now # path: applications/pipeline # repoURL: https://github.com/you/applications.git # targetRevision: stable From f9769dd9925cc081c6c7c268b41e9c8bcd0487bb Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Mon, 9 Feb 2026 10:19:10 -0500 Subject: [PATCH 29/39] feat: optional SSH debug key injection for podvm Signed-off-by: Beraldo Leal --- scripts/gen-secrets.sh | 27 ++++++++++++++++++--------- values-global.yaml | 4 ++++ values-secret.yaml.template | 13 +++++++++++++ values-simple.yaml | 4 ++-- 4 files changed, 37 insertions(+), 11 deletions(-) diff --git a/scripts/gen-secrets.sh b/scripts/gen-secrets.sh index 1196b51d..c487bcac 100755 --- a/scripts/gen-secrets.sh +++ b/scripts/gen-secrets.sh @@ -1,27 +1,36 @@ #!/usr/bin/env bash echo "Creating secrets as required" -echo +echo COCO_SECRETS_DIR="${HOME}/.coco-pattern" KBS_PRIVATE_KEY="${COCO_SECRETS_DIR}/kbsPrivateKey" KBS_PUBLIC_KEY="${COCO_SECRETS_DIR}/kbsPublicKey" -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" VALUES_FILE="${HOME}/values-secret-coco-pattern.yaml" mkdir -p ${COCO_SECRETS_DIR} +SSH_KEY_FILE="${COCO_SECRETS_DIR}/id_rsa" + +if [ "${COCO_ENABLE_SSH_DEBUG:-false}" = "true" ]; then + if [ ! -f "${SSH_KEY_FILE}" ]; then + echo "Creating ssh keys for podvm debug access" + rm -f "${SSH_KEY_FILE}.pub" + ssh-keygen -f "${SSH_KEY_FILE}" -N "" + fi +fi if [ ! -f "${KBS_PRIVATE_KEY}" ]; then - echo "Creating kbs keys" - rm -f "${KBS_PUBLIC_KEY}" - openssl genpkey -algorithm ed25519 > ${KBS_PRIVATE_KEY} - openssl pkey -in "${KBS_PRIVATE_KEY}" -pubout -out "${KBS_PUBLIC_KEY}" + echo "Creating kbs keys" + rm -f "${KBS_PUBLIC_KEY}" + openssl genpkey -algorithm ed25519 >${KBS_PRIVATE_KEY} + openssl pkey -in "${KBS_PRIVATE_KEY}" -pubout -out "${KBS_PUBLIC_KEY}" fi ## Copy a sample values file if this stuff doesn't exist if [ ! -f "${VALUES_FILE}" ]; then - echo "No values file was found copying template.. please review before deploying" - cp "${SCRIPT_DIR}/../values-secret.yaml.template" "${VALUES_FILE}" -fi \ No newline at end of file + echo "No values file was found copying template.. please review before deploying" + cp "${SCRIPT_DIR}/../values-secret.yaml.template" "${VALUES_FILE}" +fi diff --git a/values-global.yaml b/values-global.yaml index ceeb1241..7a1a8071 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -13,6 +13,10 @@ global: coco: securityPolicyFlavour: "insecure" # insecure, signed or reject is expected. secured: true # true or false. If true, the cluster will be secured. If false, the cluster will be insecure. + # Enable SSH key injection into podvm for debugging. Do not enable in production. + # Also requires: COCO_ENABLE_SSH_DEBUG=true ./scripts/gen-secrets.sh + # and uncommenting the sshKey block in values-secret.yaml.template. + enableSSHDebug: true azure: defaultVMFlavour: "Standard_DC2as_v5" VMFlavours: "Standard_DC2as_v5,Standard_DC4as_v5,Standard_DC8as_v5,Standard_DC16as_v5" diff --git a/values-secret.yaml.template b/values-secret.yaml.template index b3df87dd..6eee0149 100644 --- a/values-secret.yaml.template +++ b/values-secret.yaml.template @@ -8,6 +8,19 @@ version: "2.0" secrets: + # SSH keys for podvm debug access (optional). + # To enable: set global.coco.enableSSHDebug=true in values-global.yaml, + # run COCO_ENABLE_SSH_DEBUG=true ./scripts/gen-secrets.sh, + # then uncomment the block below. + #- name: sshKey + # vaultPrefixes: + # - global + # fields: + # - name: id_rsa.pub + # path: ~/.coco-pattern/id_rsa.pub + # - name: id_rsa + # path: ~/.coco-pattern/id_rsa + - name: securityPolicyConfig vaultPrefixes: - hub diff --git a/values-simple.yaml b/values-simple.yaml index 7a549201..c5c4e8ad 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -106,8 +106,8 @@ clusterGroup: name: sandbox namespace: openshift-sandboxed-containers-operator #upstream config project: sandbox - repoURL: https://github.com/butler54/sandboxed-containers-chart.git - targetRevision: remove-ssh + repoURL: https://github.com/beraldoleal/sandboxed-containers-chart.git + targetRevision: coco-chart-issue path: ./ sandbox-policies: From a9713e0824b73d88c531b9c4bc847c949f4dafac Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Mon, 9 Feb 2026 13:15:46 -0500 Subject: [PATCH 30/39] fix: pass required values to sandbox chart overrides --- values-simple.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/values-simple.yaml b/values-simple.yaml index c5c4e8ad..7d4a720c 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -109,6 +109,15 @@ clusterGroup: repoURL: https://github.com/beraldoleal/sandboxed-containers-chart.git targetRevision: coco-chart-issue path: ./ + overrides: + - name: global.coco.enableSSHDebug + value: "true" + - name: global.secretStore.backend + value: vault + - name: secretStore.name + value: vault-backend + - name: secretStore.kind + value: ClusterSecretStore sandbox-policies: name: sandbox-policies From 564e2e464161e4a8c662ddf688f04cf4901e5200 Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Mon, 9 Feb 2026 13:41:06 -0500 Subject: [PATCH 31/39] fix: point trustee chart to beraldoleal fork --- values-simple.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/values-simple.yaml b/values-simple.yaml index 7d4a720c..d4369ea6 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -79,8 +79,8 @@ clusterGroup: name: trustee namespace: trustee-operator-system #upstream config project: trustee - repoURL: https://github.com/butler54/trustee-chart.git - targetRevision: merge-certs + repoURL: https://github.com/beraldoleal/trustee-chart.git + targetRevision: coco-chart-issue path: ./ # Note: extraValueFiles with $patternref don't work for external repoURL (single-source app) # Using overrides instead to pass values directly From 7a41c258a623d5332453e1b355366a34c27c2e78 Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Mon, 9 Feb 2026 14:23:36 -0500 Subject: [PATCH 32/39] fix: point sandbox-policies to beraldoleal fork --- values-simple.yaml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/values-simple.yaml b/values-simple.yaml index d4369ea6..35e5d3e1 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -122,11 +122,20 @@ clusterGroup: sandbox-policies: name: sandbox-policies namespace: openshift-sandboxed-containers-operator #upstream config - chart: sandboxed-policies - chartVersion: 0.0.4 # pin to ensure latest for now -# path: applications/pipeline -# repoURL: https://github.com/you/applications.git -# targetRevision: stable + repoURL: https://github.com/beraldoleal/sandboxed-policies-chart.git + targetRevision: coco-chart-issue + path: ./ + overrides: + - name: global.coco.enableSSHDebug + value: "true" + - name: global.coco.azure.defaultVMFlavour + value: Standard_DC2as_v5 + - name: global.coco.azure.VMFlavours + value: "Standard_DC2as_v5,Standard_DC4as_v5,Standard_DC8as_v5,Standard_DC16as_v5" + - name: global.coco.azure.tags + value: "key1=value1,key2=value2" + - name: global.coco.azure.rootVolumeSize + value: "20" # Letsencrypt is not required anymore for trustee. # It's only here if you need it for your needs. letsencrypt: From d24ad297c4db4ef496329010e84f0a527960cf42 Mon Sep 17 00:00:00 2001 From: Beraldo Leal Date: Mon, 9 Feb 2026 14:31:33 -0500 Subject: [PATCH 33/39] fix: uncomment pcrStash in template, required for attestation --- values-secret.yaml.template | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/values-secret.yaml.template b/values-secret.yaml.template index 6eee0149..4ed9d158 100644 --- a/values-secret.yaml.template +++ b/values-secret.yaml.template @@ -71,16 +71,14 @@ secrets: # path: ~/.coco-pattern/cosign-key-0.pub - # Cosign public keys for image signature verification - # Required when using the "signed" policy above. - # Add your cosign public key files here. - # Generate a cosign key pair: cosign generate-key-pair - #- name: pcrStash - # vaultPrefixes: - # - hub - # fields: - # - name: json - # path: ~/.coco-pattern/measurements.json + # PCR measurements for attestation. + # Required: run ./scripts/get-pcr.sh before deploying. + - name: pcrStash + vaultPrefixes: + - hub + fields: + - name: json + path: ~/.coco-pattern/measurements.json - name: attestationStatus From a549be783c4a66e2c912079724ea5814db1ffca3 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 16 Feb 2026 10:38:56 +0900 Subject: [PATCH 34/39] fix: update chart references to butler54 repos Co-Authored-By: Claude Opus 4.6 (1M context) --- values-simple.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/values-simple.yaml b/values-simple.yaml index 35e5d3e1..00d649e9 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -79,8 +79,8 @@ clusterGroup: name: trustee namespace: trustee-operator-system #upstream config project: trustee - repoURL: https://github.com/beraldoleal/trustee-chart.git - targetRevision: coco-chart-issue + repoURL: https://github.com/butler54/trustee-chart.git + targetRevision: merge-certs path: ./ # Note: extraValueFiles with $patternref don't work for external repoURL (single-source app) # Using overrides instead to pass values directly @@ -106,8 +106,8 @@ clusterGroup: name: sandbox namespace: openshift-sandboxed-containers-operator #upstream config project: sandbox - repoURL: https://github.com/beraldoleal/sandboxed-containers-chart.git - targetRevision: coco-chart-issue + repoURL: https://github.com/butler54/sandboxed-containers-chart.git + targetRevision: remove-ssh path: ./ overrides: - name: global.coco.enableSSHDebug @@ -122,8 +122,8 @@ clusterGroup: sandbox-policies: name: sandbox-policies namespace: openshift-sandboxed-containers-operator #upstream config - repoURL: https://github.com/beraldoleal/sandboxed-policies-chart.git - targetRevision: coco-chart-issue + repoURL: https://github.com/butler54/sandboxed-policies-chart.git + targetRevision: rootvolume path: ./ overrides: - name: global.coco.enableSSHDebug From 2b8b29cf5652f842cd592e0ee803e88ed3d5ea25 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 16 Feb 2026 10:51:52 +0900 Subject: [PATCH 35/39] fix: add agents.md Signed-off-by: Chris Butler --- AGENTS.md | 150 +++++++++++++++++++++++++++++++++ docs/dell-tdx-configuration.md | 149 ++++++++++++++++++++++++++++++++ values-simple.yaml | 8 +- 3 files changed, 303 insertions(+), 4 deletions(-) create mode 100644 AGENTS.md create mode 100644 docs/dell-tdx-configuration.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..ef9bfa42 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,150 @@ +# CoCo Pattern — AI Coding Assistant Guidance + +This is a [Validated Pattern](https://validatedpatterns.io/) for deploying confidential containers (CoCo) on OpenShift. +This file provides rules and context for any AI coding assistant working in this repository. + +## Critical Rules + +- **DO NOT** edit anything under `/common/`. It is a read-only git subtree from the upstream validated patterns framework. +- **DO NOT** commit secrets, credentials, or private keys. `values-secret.yaml.template` is a template only. +- **DO NOT** use Kustomize. This project uses Helm exclusively. +- **DO NOT** create charts with `apiVersion: v1`. Use `apiVersion: v2` (Helm 3+). +- **DO NOT** place cloud-provider-specific logic in chart templates. Use `/overrides/` via `sharedValueFiles` instead. +- **DO NOT** hardcode secrets in templates. Use External Secrets Operator with vault paths (see `charts/hub/trustee/templates/dynamic-eso.yaml` for reference). + +## Feature Development Precedence Order + +Use the **first** approach that fits your requirement: + +1. **Helm charts** — Declarative Kubernetes resources in `/charts/`, deployed by ArgoCD. Preferred for installing operators, configuring CRDs, and creating Kubernetes resources. +2. **ACM policies** — Red Hat Advanced Cluster Management policies for propagating configuration from hub to spoke clusters and enforcing multi-cluster governance. Reference: `charts/hub/sandbox-policies/templates/`. +3. **Imperative framework (Ansible)** — Playbooks in `/ansible/`, executed as Kubernetes Jobs on a 10-minute schedule. **Must be idempotent.** Use for API calls, runtime data lookups, and multi-step orchestration that cannot be expressed declaratively. Register playbooks in `clusterGroup.imperative.jobs` as an ordered list. +4. **Out-of-band scripts** — `/scripts/` or `/rhdp/`. Last resort for one-time setup or local development tooling. These are not managed by GitOps. + +## Project Structure + +``` +├── ansible/ # Ansible playbooks (imperative jobs) +├── charts/ +│ ├── all/ +│ │ └── letsencrypt/ # Shared across cluster groups +│ ├── coco-supported/ +│ │ ├── baremetal/ # Bare-metal TDX configuration +│ │ ├── hello-openshift/ # Sample workloads +│ │ ├── kbs-access/ # KBS access verification workload +│ │ └── sandbox/ # Sandboxed containers runtime +│ └── hub/ +│ ├── lvm-storage/ # LVM storage for bare-metal +│ ├── sandbox-policies/ # ACM policies (hub → spoke) +│ └── trustee/ # Trustee / KBS +├── common/ # READ-ONLY — upstream framework subtree +├── overrides/ # Cloud-provider value overrides +│ ├── values-AWS.yaml +│ ├── values-Azure.yaml +│ └── values-IBMCloud.yaml +├── rhdp/ # Red Hat Demo Platform tooling +├── scripts/ # Utility scripts +├── values-global.yaml # Global configuration +├── values-simple.yaml # Cluster group: simple +├── values-baremetal.yaml # Cluster group: baremetal +├── values-trusted-hub.yaml # Cluster group: trusted-hub +├── values-untrusted-spoke.yaml # Cluster group: untrusted-spoke +└── values-secret.yaml.template # Secrets template (never commit filled-in copy) +``` + +## Companion Chart Repositories + +Several charts in this repo have companion repositories for independent versioning and reuse. Develop and test in this repo first (charts deploy via `path:`), then sync changes to the companion repo. + +| Local Path | Companion Repo | Purpose | +|---|---|---| +| `charts/hub/trustee/` | `trustee-chart` | Trustee / KBS on hub | +| `charts/hub/sandbox-policies/` | `sandboxed-policies-chart` | ACM policies hub → spoke | +| `charts/coco-supported/sandbox/` | `sandboxed-containers-chart` | Sandboxed runtime on spoke | + +Large features may require coordinated changes across multiple companion repos. References are org-agnostic — contributors should fork all relevant repos as needed. + +## Cluster Groups + +Set via `main.clusterGroupName` in `values-global.yaml`. + +| Cluster Group | Values File | Role | Description | +|---|---|---|---| +| `simple` | `values-simple.yaml` | Hub (single cluster) | All components on one cluster | +| `baremetal` | `values-baremetal.yaml` | Hub (single cluster) | TDX + LVM storage on bare metal | +| `trusted-hub` | `values-trusted-hub.yaml` | Multi-cluster hub | Trustee + ACM policies | +| `untrusted-spoke` | `values-untrusted-spoke.yaml` | Multi-cluster spoke | Sandbox runtime + workloads | + +## Values File Hierarchy + +Merge order (last wins): + +1. Chart defaults (`charts///values.yaml`) +2. `values-global.yaml` +3. `values-.yaml` +4. `/overrides/values-{{ clusterPlatform }}.yaml` (via `sharedValueFiles`) +5. `values-secret.yaml` (runtime only, never committed) + +Key conventions: + +- Global settings go under the `global:` key in `values-global.yaml`. +- Subscriptions go under `clusterGroup.subscriptions:` in the cluster group values file. +- Applications go under `clusterGroup.applications:` in the cluster group values file. +- Local charts use `path:` (e.g., `path: charts/hub/trustee`). Shared framework charts use `chart:` + `chartVersion:`. +- Imperative jobs go under `clusterGroup.imperative.jobs:` as an **ordered list** (not a hash — hashes lose ordering in Helm). + +## Helm Chart Conventions + +- Use `apiVersion: v2`. Place charts in `charts///`. +- Use ArgoCD sync-wave annotations to control deployment ordering. +- Use `ExternalSecret` resources to pull secrets from vault. Reference: `charts/hub/trustee/templates/dynamic-eso.yaml`. +- Use `.Values.global.clusterPlatform` for platform-conditional logic only when overrides files are insufficient. +- Reference patterns: + - ESO integration: `charts/hub/trustee/templates/dynamic-eso.yaml` + - Template helpers: `charts/coco-supported/hello-openshift/templates/_helpers.tpl` + +## Ansible Playbook Conventions + +- Place playbooks in `/ansible/`. They **must be idempotent**. +- Use `connection: local`, `hosts: localhost`, `become: false`. +- Use `kubernetes.core.k8s` and `kubernetes.core.k8s_info` modules for cluster interaction. +- Register playbooks in the cluster group values file under `clusterGroup.imperative.jobs` with `name`, `playbook`, `verbosity`, and `timeout` fields. + +## Git Workflow + +- **Fork-first**: ArgoCD reconciles against your fork. Clone and push to your own fork. +- **Conventional commits**: Enforced by commitlint (`@commitlint/config-conventional`). +- **Branch-based deployment**: The branch of your local checkout determines the ArgoCD deployment target. +- **Changes require commit + push** to take effect — ArgoCD watches the remote. + +## Commands Reference + +All commands run via `./pattern.sh make `: + +| Command | Purpose | +|---|---| +| `install` | Install the pattern and load secrets | +| `show` | Render the starting template without installing | +| `preview-all` | Preview all applications across cluster groups | +| `validate-schema` | Validate values files against JSON schema | +| `validate-cluster` | Validate cluster prerequisites | +| `super-linter` | Run super-linter locally | +| `load-secrets` | Load secrets into the configured backend | +| `uninstall` | Uninstall the pattern | + +See the README for secrets backend configuration, RHDP environment variables, and additional maintenance commands. + +## Validation and CI + +CI runs the following checks on pull requests: + +- **JSON schema validation** — values files validated against `common/clustergroup` schema +- **Super Linter** — multi-language linting +- **Conventional PR title lint** — PR titles must follow conventional commit format + +Run locally before pushing: + +```bash +./pattern.sh make preview-all +./pattern.sh make validate-schema +``` diff --git a/docs/dell-tdx-configuration.md b/docs/dell-tdx-configuration.md new file mode 100644 index 00000000..f3b13a01 --- /dev/null +++ b/docs/dell-tdx-configuration.md @@ -0,0 +1,149 @@ +# Enable Intel TDX on Dell PowerEdge via iDRAC + +This guide provides step-by-step instructions for enabling Intel Trust Domain Extensions (TDX) on Dell PowerEdge servers using the iDRAC console. + +## Prerequisites + +- Dell 16th Generation PowerEdge server: + - PowerEdge R660, R660xs + - PowerEdge R760, R760xs, R760xd2, R760XA + - PowerEdge R860, R960 + - PowerEdge XE8640, XE9640, XE9680 + - PowerEdge C6620, MX760c + - PowerEdge XR5610, XR7620, XR8610t, XR8620t + - PowerEdge T360, T560 +- 5th Gen Intel Xeon Scalable processor with TDX support +- **8 or 16 DIMMs per socket** (required memory configuration) +- Latest BIOS firmware installed + +## Step-by-Step Instructions (Order Matters) + +> **IMPORTANT:** Settings must be configured in this exact order. Some options (like "Multiple Keys") will be greyed out until prerequisite settings are applied. You may need to **save and reboot between steps** for dependent options to become available. + +### 1. Access BIOS Setup via iDRAC + +1. Log into the iDRAC web console +2. Navigate to **Configuration → BIOS Settings** +3. Alternatively, launch **Virtual Console** and press **F2** during POST to enter System Setup + +### 2. Configure Memory Settings (FIRST) + +Navigate to: **System BIOS → Memory Settings** + +| Setting | Value | +| ---------------------- | ----------- | +| **Node Interleaving** | Disabled | + +**Save and reboot** before proceeding. + +### 3. Configure Processor Prerequisites (SECOND) + +Navigate to: **System BIOS → Processor Settings** + +| Setting | Value | +| -------------------------------- | -------- | +| **Logical Processor (x2APIC)** | Enabled | +| **CPU Physical Address Limit** | Disabled | + +**Save and reboot** before proceeding. + +### 4. Enable Memory Encryption - Multiple Keys (THIRD) + +Navigate to: **System BIOS → System Security** + +| Setting | Value | +| --------------------- | -------------- | +| **Memory Encryption** | Multiple Keys | + +> If "Multiple Keys" is still greyed out, verify steps 2 and 3 were applied and the system was rebooted. + +**Save and reboot** before proceeding. + +### 5. Configure TDX Settings (FOURTH) + +Navigate to: **System BIOS → System Security** (or **Processor Settings** depending on BIOS version) + +| Setting | Value | +| ------------------------------------------------- | ------- | +| **Global Memory Integrity** | Disabled | +| **Intel TDX (Trust Domain Extension)** | Enabled | +| **TME-MT/TDX Key Split** | 1 | +| **TDX Secure Arbitration Mode Loader (SEAM)** | Enabled | + +### 6. Configure SGX Settings (FIFTH) + +Navigate to: **System BIOS → Processor Settings → Software Guard Extensions (SGX)** + +| Setting | Value | +| -------------------- | ------------------------- | +| **Intel SGX** | Enabled | +| **SGX Factory Reset** | Off | +| **SGX PRMRR Size** | As needed (e.g., 64GB) | + +### 7. Final Save and Reboot + +1. Press **Escape** to exit menus +2. Select **Save Changes and Exit** +3. System will reboot with TDX enabled + +## Configuration Summary (Order of Operations) + +``` +1. Disable Node Interleaving → Save & Reboot +2. Enable x2APIC Mode → Save & Reboot +3. Disable CPU Physical Address Limit → Save & Reboot +4. Set Memory Encryption = Multiple Keys → Save & Reboot +5. Disable Global Memory Integrity +6. Enable Intel TDX +7. Set TME-MT/TDX Key Split = 1 +8. Enable SEAM Loader +9. Enable Intel SGX → Final Save & Reboot +``` + +## Verification + +After the OS boots, verify TDX is enabled: + +```bash +# Check kernel messages for TDX +dmesg | grep -i tdx +# Should show: "virt/tdx: BIOS enabled: private KeyID range: [X, Y)" + +# Check for TDX module +ls /sys/firmware/tdx_seam/ +``` + +## Troubleshooting + +### "Multiple Keys" Option is Greyed Out + +This is typically caused by: + +1. **Node Interleaving is Enabled** - Must be disabled first +2. **x2APIC Mode is Disabled** - Must be enabled first +3. **CPU Physical Address Limit is Enabled** - Must be disabled first +4. **System not rebooted** - Some changes require reboot before dependent options appear +5. **Insufficient DIMMs** - Requires 8 or 16 DIMMs per socket + +### Settings Not Available + +If TDX-related settings are not visible: + +1. Ensure BIOS firmware is updated to the latest version +2. Verify your processor supports TDX (5th Gen Xeon Scalable required) +3. Contact Dell support for BIOS with TDX support + +### TDX Not Detected by OS + +If the OS doesn't detect TDX after configuration: + +1. Verify all settings are correctly applied in the order specified +2. Ensure the OS/kernel supports TDX (Linux 6.2+ recommended) +3. Check that Memory Encryption is set to "Multiple Keys" (not "Single Key") + +## References + +- [Dell: Enable Intel TDX on Dell 16G Intel Servers](https://www.dell.com/support/kbdoc/en-us/000226452/enableinteltdxondell16g) +- [Intel TDX Enabling Guide - Hardware Setup](https://cc-enabling.trustedservices.intel.com/intel-tdx-enabling-guide/04/hardware_setup/) +- [Dell Info Hub: Enable Intel TDX in BIOS](https://infohub.delltechnologies.com/en-us/l/securing-ai-workloads-on-dell-poweredge-with-intel-xeon-processors-using-intel-trust-domain-extensions/appendix-b-enable-intel-r-tdx-in-bios/) +- [Linux Kernel TDX Documentation](https://docs.kernel.org/arch/x86/tdx.html) diff --git a/values-simple.yaml b/values-simple.yaml index 00d649e9..74dba729 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -110,8 +110,8 @@ clusterGroup: targetRevision: remove-ssh path: ./ overrides: - - name: global.coco.enableSSHDebug - value: "true" + # - name: global.coco.enableSSHDebug + # value: "true" - name: global.secretStore.backend value: vault - name: secretStore.name @@ -126,8 +126,8 @@ clusterGroup: targetRevision: rootvolume path: ./ overrides: - - name: global.coco.enableSSHDebug - value: "true" + # - name: global.coco.enableSSHDebug + # value: "true" - name: global.coco.azure.defaultVMFlavour value: Standard_DC2as_v5 - name: global.coco.azure.VMFlavours From 6eccff50a6b861081d441baf6bcd159d8d1f14a1 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 17 Feb 2026 15:50:54 +0900 Subject: [PATCH 36/39] feat: add support for multiple clusters Signed-off-by: Chris Butler --- rhdp/requirements.txt | 3 ++- values-spoke.yaml | 23 ++++++++++++-------- values-trusted-hub.yaml | 48 +++++++++++++++++++++++++++++------------ 3 files changed, 50 insertions(+), 24 deletions(-) diff --git a/rhdp/requirements.txt b/rhdp/requirements.txt index e02a4479..3a98310e 100644 --- a/rhdp/requirements.txt +++ b/rhdp/requirements.txt @@ -1,3 +1,4 @@ typer rich -Jinja2 \ No newline at end of file +Jinja2 +typing_extensions \ No newline at end of file diff --git a/values-spoke.yaml b/values-spoke.yaml index 270e4cfd..47c084e8 100644 --- a/values-spoke.yaml +++ b/values-spoke.yaml @@ -17,7 +17,7 @@ clusterGroup: source: redhat-operators channel: stable installPlanApproval: Manual - csv: sandboxed-containers-operator.v1.10.1 + csv: sandboxed-containers-operator.v1.11.0 cert-manager: name: openshift-cert-manager-operator namespace: cert-manager-operator @@ -48,8 +48,18 @@ clusterGroup: name: sandbox namespace: openshift-sandboxed-containers-operator #upstream config project: sandbox - chart: sandboxed-containers - chartVersion: 0.0.* + repoURL: https://github.com/butler54/sandboxed-containers-chart.git + targetRevision: remove-ssh + path: ./ + overrides: + # - name: global.coco.enableSSHDebug + # value: "true" + - name: global.secretStore.backend + value: vault + - name: secretStore.name + value: vault-backend + - name: secretStore.kind + value: ClusterSecretStore hello-openshift: name: hello-openshift @@ -64,12 +74,7 @@ clusterGroup: path: charts/coco-supported/kbs-access imperative: - # NOTE: We *must* use lists and not hashes. As hashes lose ordering once parsed by helm - # The default schedule is every 10 minutes: imperative.schedule - # Total timeout of all jobs is 1h: imperative.activeDeadlineSeconds - # imagePullPolicy is set to always: imperative.imagePullPolicy - # For additional overrides that apply to the jobs, please refer to - # https://validatedpatterns.io/imperative-actions/#additional-job-customizations + image: ghcr.io/butler54/imperative-container:latest jobs: - name: install-deps playbook: ansible/install-deps.yaml diff --git a/values-trusted-hub.yaml b/values-trusted-hub.yaml index be5a2da6..be2f452d 100644 --- a/values-trusted-hub.yaml +++ b/values-trusted-hub.yaml @@ -22,7 +22,7 @@ clusterGroup: source: redhat-operators channel: stable installPlanApproval: Manual - csv: trustee-operator.v0.4.1 + csv: trustee-operator.v1.0.0 cert-manager: name: openshift-cert-manager-operator namespace: cert-manager-operator @@ -66,31 +66,51 @@ clusterGroup: name: trustee namespace: trustee-operator-system #upstream config project: trustee - chart: trustee - chartVersion: 0.1.* - # Use the override file to specify the list of secrets accessible to trustee from the ESO backend (today by default, Vault). - extraValueFiles: - - '$patternref/overrides/values-trustee.yaml' + repoURL: https://github.com/butler54/trustee-chart.git + targetRevision: merge-certs + path: ./ + overrides: + - name: global.coco.secured + value: "true" + - name: kbs.secretResources[0].name + value: kbsres1 + - name: kbs.secretResources[0].key + value: secret/data/hub/kbsres1 + - name: kbs.secretResources[1].name + value: passphrase + - name: kbs.secretResources[1].key + value: secret/data/hub/passphrase sandbox-policies: name: sandbox-policies namespace: openshift-sandboxed-containers-operator #upstream config - chart: sandboxed-policies - chartVersion: 0.0.* + repoURL: https://github.com/butler54/sandboxed-policies-chart.git + targetRevision: rootvolume + path: ./ + overrides: + # - name: global.coco.enableSSHDebug + # value: "true" + - name: global.coco.azure.defaultVMFlavour + value: Standard_DC2as_v5 + - name: global.coco.azure.VMFlavours + value: "Standard_DC2as_v5,Standard_DC4as_v5,Standard_DC8as_v5,Standard_DC16as_v5" + - name: global.coco.azure.tags + value: "key1=value1,key2=value2" + - name: global.coco.azure.rootVolumeSize + value: "20" imperative: - # NOTE: We *must* use lists and not hashes. As hashes lose ordering once parsed by helm - # The default schedule is every 10 minutes: imperative.schedule - # Total timeout of all jobs is 1h: imperative.activeDeadlineSeconds - # imagePullPolicy is set to always: imperative.imagePullPolicy - # For additional overrides that apply to the jobs, please refer to - # https://validatedpatterns.io/imperative-actions/#additional-job-customizations + image: ghcr.io/butler54/imperative-container:latest jobs: - name: install-deps playbook: ansible/install-deps.yaml verbosity: -vvv timeout: 3600 + - name: configure-azure-dns + playbook: ansible/configure-issuer.yaml + verbosity: -vvv + timeout: 3600 - name: init-data-gzipper playbook: ansible/init-data-gzipper.yaml verbosity: -vvv From 209b4e4dd16b29128f35aa8d6985a661212b5fd6 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 17 Feb 2026 15:51:42 +0900 Subject: [PATCH 37/39] chore: prep for multi-cluster deployment Signed-off-by: Chris Butler --- values-global.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values-global.yaml b/values-global.yaml index 7a1a8071..48d12c4e 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -24,7 +24,7 @@ main: # WARNING # This default configuration uses a single cluster on azure. # It fundamentally violates the separation of duties. - clusterGroupName: simple + clusterGroupName: trusted-hub multiSourceConfig: enabled: true clusterGroupChartVersion: 0.9.* From 3b4014de177b9d080c3159849c0b1ff174df9698 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 18 Feb 2026 14:31:00 +0900 Subject: [PATCH 38/39] feat: documentatin update Signed-off-by: Chris Butler --- README.md | 233 +++++++++++------------------- charts/all/letsencrypt/Chart.yaml | 6 +- rhdp/wrapper.sh | 15 ++ values-simple.yaml | 12 -- 4 files changed, 104 insertions(+), 162 deletions(-) diff --git a/README.md b/README.md index 8afd14a7..496716dc 100644 --- a/README.md +++ b/README.md @@ -1,199 +1,134 @@ # coco-pattern -This is a validated pattern for deploying confidential containers on OpenShift. +Validated pattern for deploying confidential containers on OpenShift using the [Validated Patterns](https://validatedpatterns.io/) framework. -There are two topologies for deploying this pattern: +Confidential containers use hardware-backed Trusted Execution Environments (TEEs) to isolate workloads from cluster and hypervisor administrators. This pattern deploys and configures the Red Hat CoCo stack — including the sandboxed containers operator, Trustee (Key Broker Service), and peer-pod infrastructure — on Azure. -1. *Default* using a single cluster. This breaks the RACI expected in a remote attestation architecture, however, makes it easier to test. This uses the `simple` `clusterGroup`. -2. A more secure operating model that has two clusters: - - One in a "trusted" zone where the remote attestation, KMS and Key Broker infrastructure are deployed. This is also the Advanced Cluster Manager Hub cluster. It uses the `trusted-hub` `clusterGroup`. - - A second where a subset of workloads are deployed in confidential containers. It uses the `spoke` `clusterGroup` +## Topologies -The current version of this application the confidential containers assumes deployment to Azure. +The pattern provides two deployment topologies: -On the cluster where confidential workloads are deployed two sample applications are deployed: +1. **Single cluster** (`simple` clusterGroup) — deploys all components (Trustee, Vault, ACM, sandboxed containers, workloads) in one cluster. This breaks the RACI separation expected in a remote attestation architecture but simplifies testing and demonstrations. -1. Sample hello world applications to allow users to experiment with the policies for CoCo and the KBS (trustee). -2. A sample application `kbs-access` which presents secrets obtained from trustee to a web service. This is designed to allow users to test locked down environments. +2. **Multi-cluster** (`trusted-hub` + `spoke` clusterGroups) — separates the trusted zone from the untrusted workload zone: + - **Hub** (`trusted-hub`): Runs Trustee (KBS + attestation service), HashiCorp Vault, ACM, and cert-manager. This cluster is the trust anchor. + - **Spoke** (`spoke`): Runs the sandboxed containers operator and confidential workloads. The spoke is imported into ACM and managed from the hub. -Future work includes: +The topology is controlled by the `main.clusterGroupName` field in `values-global.yaml`. -1. ~~Supporting a multiple cluster deployment~~ Done -2. Supporting multiple infrastructure providers - Work in Progress. -3. Supporting air-gapped deployments - Work in Progress. -4. Supporting a more sophisticated workload such as confidential AI inference with protected GPUs. +Currently supports Azure via peer-pods. Peer-pods provision confidential VMs (`Standard_DCas_v5` family) directly on the Azure hypervisor rather than nesting VMs inside worker nodes. -## Current constraints and assumptions +## Current version (4.*) -- Only currently is known to work with `azure` as the provider of confidential vms via peer-pods. -- Below version 3.1, if not using ARO you must either provide your own CA signed certs, or use let's encrypt. -- Must be on 4.16.14 or later. +Breaking change from v3. This is the first version using GA (Generally Available) releases of the CoCo stack: -## Major versions +- **OpenShift Sandboxed Containers 1.11+** (requires OCP 4.17+) +- **Red Hat Build of Trustee 1.0** (first GA release; all prior versions were Technology Preview) +- External chart repositories for [Trustee](https://github.com/butler54/trustee-chart), [sandboxed-containers](https://github.com/butler54/sandboxed-containers-chart), and [sandboxed-policies](https://github.com/butler54/sandboxed-policies-chart) +- Self-signed certificates via cert-manager (Let's Encrypt no longer required) +- Multi-cluster support via ACM -### `3.*` +### Previous versions -Version `3.*` of the pattern is currently constrained to support the general availability releases of coco. +All previous versions used pre-GA (Technology Preview) releases of Trustee: -- (OpenShift Sandboxed Containers Operator) `1.10.*` and above -- Trustee `0.4.*` +| Version | Trustee | OSC | Min OCP | +|---------|---------|-----|---------| +| **3.*** | 0.4.* (Tech Preview) | 1.10.* | 4.16+ | +| **2.*** | 0.3.* (Tech Preview) | 1.9.* | 4.16+ | +| **1.0.0** | 0.2.0 (Tech Preview) | 1.8.1 | 4.16+ | -This limits support to OpenShift 4.16 and higher. +## Setup -The pattern has been tested on Azure for two installation methods: +### Prerequisites -1. Installing onto an ARO cluster -2. Self managed OpenShift install using the `openshift-install` CLI. +- OpenShift 4.17+ cluster on Azure (self-managed via `openshift-install` or ARO) +- Azure `Standard_DCas_v5` VM quota in your target region (these are confidential computing VMs and are not available in all regions). See the note below for more details. +- Azure DNS hosting the cluster's DNS zone +- Tools on your workstation: `podman`, `yq`, `jq`, `skopeo` +- OpenShift pull secret saved at `~/pull-secret.json` (download from [console.redhat.com](https://console.redhat.com/openshift/downloads)) +- Fork the repo — ArgoCD reconciles cluster state against your fork, so changes must be pushed to your remote -#### Known limitations -[Additional configuration](https://issues.redhat.com/browse/KATA-4107) is required to pull secrets from authenticated registries. +### Secrets and PCR setup -### `2.*` +These scripts generate the cryptographic material and attestation measurements needed by Trustee and the peer-pod VMs. Run them once before your first deployment. -Version `2.*` of the pattern is currently constrained to support: +1. `bash scripts/gen-secrets.sh` — generates KBS key pairs, attestation policy seeds, and copies `values-secret.yaml.template` to `~/values-secret-coco-pattern.yaml` +2. `bash scripts/get-pcr.sh` — retrieves PCR measurements from the peer-pod VM image and stores them at `~/.coco-pattern/measurements.json` (requires `podman`, `skopeo`, and `~/pull-secret.json`) +3. Review and customise `~/values-secret-coco-pattern.yaml` — this file is loaded into Vault and provides secrets to the pattern -- (OpenShift Sandboxed Containers Operator) `1.9.*` -- Trustee `0.3.*` +> **Note:** `gen-secrets.sh` will not overwrite existing secrets. Delete `~/.coco-pattern/` if you need to regenerate. -This limits support to OpenShift 4.16 and higher. +### Single cluster deployment -The pattern has been tested on Azure for two installation methods: +1. Set `main.clusterGroupName: simple` in `values-global.yaml` +2. Ensure your Azure configuration is populated in `values-global.yaml` (see `global.azure.*` fields) +3. `./pattern.sh make install` +4. Wait for the cluster to reboot all nodes (the sandboxed containers operator triggers a MachineConfig update). Monitor progress in the ArgoCD UI. -1. Installing onto an ARO cluster -2. Self managed OpenShift install using the `openshift-install` CLI. +### Multi-cluster deployment -### `1.0.0` +1. Set `main.clusterGroupName: trusted-hub` in `values-global.yaml` +2. Deploy the hub cluster: `./pattern.sh make install` +3. Wait for ACM (`MultiClusterHub`) to reach `Running` state on the hub +4. Provision a second OpenShift 4.17+ cluster on Azure for the spoke +5. Import the spoke into ACM with label `clusterGroup=spoke` + (see [importing a cluster](https://validatedpatterns.io/learn/importing-a-cluster/)) +6. ACM will automatically deploy the `spoke` clusterGroup applications (sandboxed containers, workloads) to the imported cluster -1.0.0 supports OpenShift Sandboxed containers version `1.8.1` along with Trustee version `0.2.0`. +## Sample applications -The pattern has been tested on Azure for one installation method: +Two sample applications are deployed on the cluster running confidential workloads (the single cluster in `simple` mode, or the spoke in multi-cluster mode): -1. Self managed OpenShift install using the `openshift-install` CLI -2. Installing on top of an existing Azure Red Hat OpenShift (ARO) cluster +- **hello-openshift**: Three pods demonstrating CoCo security boundaries: + - `standard` — a regular Kubernetes pod (no confidential computing) + - `secure` — a confidential container with a strict policy; `oc exec` is denied even for `kubeadmin` + - `insecure-policy` — a confidential container with a relaxed policy allowing `oc exec` (useful for testing the Confidential Data Hub) -## Changing deployment topoloiges + Each confidential pod runs on its own `Standard_DC2as_v5` Azure VM (visible in the Azure portal). Pods use `runtimeClassName: kata-remote`. -**Today the demo has two deployment topologies** -The most important change is what `clusterGroup` is deployed to your main or 'hub' cluster. +- **kbs-access**: A web service that retrieves and presents secrets obtained from the Trustee Key Broker Service (KBS) via the Confidential Data Hub (CDH). Useful for verifying end-to-end attestation and secret delivery in locked-down environments. -You can change between behaviour by configuring [`global.main.clusterGroupName`](https://validatedpatterns.io/learn/values-files/) key in the `values-global.yaml` file. +## Confidential computing virtual machine availability on Microsoft Azure -- `values-simple.yaml`: or the `simple` cluster group is the default for the pattern. It deploys everything in one cluster. --`values-trusted-hub`: or the `trusted-hub` cluster group can be configured as the main cluster group. A second cluster should be deployed with the `spoke` cluster group. Follow [instructions here](https://validatedpatterns.io/learn/importing-a-cluster/) to add the second cluster. +Confidential computing VM availability on Azure varies by region. Not all regions offer the required VM families, and available sizes differ between regions. Before deploying, verify the following: -## Setup instructions +1. **Check regional availability.** Confirm that your target Azure region supports confidential computing VMs. Microsoft's [products available by region](https://azure.microsoft.com/en-us/explore/global-infrastructure/products-by-region/) page lists which services and VM families are offered in each region. -### Default single cluster setup with `values-simple.yaml` +2. **Check your subscription quota.** Even in supported regions, your subscription may have zero default quota for confidential VM sizes. Go to **Azure Portal > Subscriptions > Usage + quotas** and filter for the DCas/DCads/ECas/ECads families. Request a quota increase if needed. -The instructions here presume you have a cluster. See further down for provisioning instructions for a cluster. +3. **Select a VM size.** The pattern defaults to `Standard_DC2as_v5` but supports a configurable list of sizes. The following VM families are relevant for confidential containers on Azure: -#### Fork and Clone the GitHub repository +| VM Family | CPU | Architecture | Notes | +|-----------|-----|--------------|-------| +| `Standard_DC2as_v5` | AMD SEV-SNP | AMD EPYC (Genoa) | Default for this pattern. Smallest CoCo-capable size. | +| `Standard_DC4as_v5` | AMD SEV-SNP | AMD EPYC (Genoa) | More vCPUs/memory for larger workloads. | +| `Standard_DC2ads_v5` | AMD SEV-SNP | AMD EPYC (Genoa) | Same as DC2as_v5 with a local temp disk. | +| `Standard_DC2es_v5` | Intel TDX | Intel Xeon (Sapphire Rapids) | Intel-based confidential VMs. Regional availability is more limited than AMD. | -1. Following [standard validated patterns workflow](https://validatedpatterns.io/learn/workflow/) fork the repository and clone to your development environment which has `podman` and `git` -2. If using a particular version (e.g. `1.0.0`) checkout the correct tag. +The available sizes can be configured via the `global.coco.azure.VMFlavours` field in `values-global.yaml` and the sandbox-policies chart overrides. The default VM flavour is set in `global.coco.azure.defaultVMFlavour`. -> [!TIP] -> Forking is essential as the validated pattern uses ArgoCD to reconcile it's state against your remote (forked) repository. +### RHDP deployment (Red Hat Demo Platform) -#### Configuring required secrets / parameters +For Red Hat associates and partners, the pattern includes wrapper scripts that automate cluster provisioning and deployment using RHDP Azure Open Environments. -The secrets here secure Trustee and the peer-pod vms. Mostly they are for demonstration purposes. -This only has to be done once. - -1. Run `sh scripts/gen-secrets.sh` - -> [!NOTE] -> Once generated this script will not override secrets. Be careful when doing multiple tests. - -#### Configuring let's encrypt (deprecated) - -> [!IMPORTANT] -> Ensure you have password login available to the cluster. Let's encrypt will replace the API certificate in addition to the certificates to user with routes. - -Trustee (guest agents) requires that Trustee uses a Mozilla trusted CA issued certificate, or a specific certificate which is known in advance. Today the pattern uses specific self signed certs. Let's encrypt was an option for getting a trusted certificate onto OpenShift's routes, and therefore Trustee. Ths functionality will be removed at a later date. - -If you need a Let's Encrypt certificate to be issued the `letsencrypt` application configuration needs to be changed as below. - -```yaml - --- - # Default configuration, safe for ARO - letsencrypt: - name: letsencrypt - namespace: letsencrypt - project: hub - path: charts/all/letsencrypt - # Default to 'safe' for ARO - overrides: - - name: letsencrypt.enabled - value: false - --- - # Explicitly correct configuration for enabling let's encrypt - letsencrypt: - name: letsencrypt - namespace: letsencrypt - project: hub - path: charts/all/letsencrypt - overrides: - - name: letsencrypt.enabled - value: true -``` - -> [!WARNING] -> Configuration changes are only effective once committed and pushed to your remote repository. - -#### Installing onto a cluster - -Once you configuration is pushed (if required) `./pattern.sh make install` to provision a cluster. - -> [!TIP] -> The branch and default origin you have checked-out in your local repository is used to determine what ArgoCD and the patterns operator should reconcile against. Typical choices are to use the main for your fork. - -## Cluster setup (if not already setup) - -### Single cluster install on an OCP cluster on azure using Red Hat Demo Platform - -Red Hat a demo platform. This allows easy access for Red Hat associates and partners to ephemeral cloud resources. The pattern is known to work with this setup. - -1. Get the [openshift installer](https://console.redhat.com/openshift/downloads) - 1. **NOTE: openshift installer must be updated regularly if you want to automatically provision the latest versions of OCP** -2. Get access to an [Azure Subscription Based Blank Open Environment](https://catalog.demo.redhat.com/catalog?category=Open_Environments&search=azure&item=babylon-catalog-prod%2Fazure-gpte.open-environment-azure-subscription.prod). -3. Import the required azure environmental variables (see code block below) -4. Ensure certificates are configured (via let's encrypt or do so manually) -5. Run the wrapper install script - 1. `bash ./rhdp/wrapper.sh azure-region-code` - 2. Where azure region code is `eastasia`, `useast2` etc. -6. You *should* be done - 1. You *may* need to recreate the hello world peer-pods depending on timeouts. +Required environment variables (provided by your RHDP environment): ```shell - export GUID= - export CLIENT_ID= - export PASSWORD= - export TENANT= - export SUBSCRIPTION= - export RESOURCEGROUP= +export GUID= +export CLIENT_ID= +export PASSWORD= +export TENANT= +export SUBSCRIPTION= +export RESOURCEGROUP= ``` -### Single cluster install on plain old azure *not* using Red Hat Demo Platform +Deployment commands: -> [!TIP] -> Don't use the default node sizes.. increase the node sizes such as below +- Single cluster: `bash rhdp/wrapper.sh ` (e.g. `bash rhdp/wrapper.sh eastasia`) +- Multi-cluster: `bash rhdp/wrapper-multicluster.sh ` -1. Login to console.redhat.com -2. Get the openshift installer -3. Login to azure locally. -4. `openshift-install create install-config` - 1. Select azure - 2. For Red Hatter's and partners using RHDP make sure you select the same region for your account that you selected in RHDP -5. Change worker machine type e.g. change `type: Standard_D4s_v5` to `type: Standard_D8s_v5` or similar based on your needs. -6. `mkdir ./ocp-install && mv openshift-install.yaml ./ocp-install` -7. `openshift-install create cluster --dir=./ocp-install` -8. Once installed: - 1. Login to `oc` - 2. Configure Let's Encrypt (if required) - 3. `./pattern.sh make install` +The wrapper scripts handle cluster provisioning via `openshift-install`, secret generation, PCR retrieval, and pattern installation. -### Multi cluster setup -TBD diff --git a/charts/all/letsencrypt/Chart.yaml b/charts/all/letsencrypt/Chart.yaml index c4d83704..e3203347 100644 --- a/charts/all/letsencrypt/Chart.yaml +++ b/charts/all/letsencrypt/Chart.yaml @@ -1,6 +1,10 @@ apiVersion: v2 name: letsencrypt -description: A Helm chart to add letsencrypt support to Validated Patterns. +description: >- + DEPRECATED: This chart is unsupported and will be removed in a future release. + Trustee 1.0 uses cert-manager for certificate management, making Let's Encrypt + integration unnecessary. A Helm chart to add letsencrypt support to Validated Patterns. +deprecated: true type: application diff --git a/rhdp/wrapper.sh b/rhdp/wrapper.sh index 5bc1f992..18763936 100755 --- a/rhdp/wrapper.sh +++ b/rhdp/wrapper.sh @@ -83,6 +83,17 @@ if ! command -v yq &> /dev/null; then exit 1 fi +# Check if podman is available and running +if ! command -v podman &> /dev/null; then + echo "ERROR: podman is required but not installed" + exit 1 +fi + +if ! podman info &> /dev/null; then + echo "ERROR: podman is installed but not responding" + exit 1 +fi + # Extract clusterGroupName from values-global.yaml using yq CLUSTER_GROUP_NAME=$(yq eval '.main.clusterGroupName' values-global.yaml) @@ -175,6 +186,10 @@ echo "setting up secrets" bash ./scripts/gen-secrets.sh +echo "---------------------" +echo "retrieving PCR measurements" +echo "---------------------" +bash ./scripts/get-pcr.sh sleep 60 echo "---------------------" diff --git a/values-simple.yaml b/values-simple.yaml index 74dba729..c292bcb5 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -14,7 +14,6 @@ clusterGroup: - hello-openshift - cert-manager-operator - cert-manager - - letsencrypt - kbs-access - encrypted-storage subscriptions: @@ -136,17 +135,6 @@ clusterGroup: value: "key1=value1,key2=value2" - name: global.coco.azure.rootVolumeSize value: "20" - # Letsencrypt is not required anymore for trustee. - # It's only here if you need it for your needs. - letsencrypt: - name: letsencrypt - namespace: letsencrypt - project: hub - path: charts/all/letsencrypt - # Default to 'safe' for ARO - overrides: - - name: letsencrypt.enabled - value: false hello-openshift: name: hello-openshift namespace: hello-openshift From 3bba8d35039b6907745cc42bb6ee2e116f4744e6 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Thu, 19 Feb 2026 12:01:52 +0900 Subject: [PATCH 39/39] fix: resolve CI lint failures for PR #69 Address ansible-lint (risky-shell-pipe, no-changed-when, line-length), super-linter markdown and natural language errors, and JSON schema validation by resetting clusterGroupName to simple. Co-Authored-By: Claude Opus 4.6 (1M context) --- AGENTS.md | 10 +++++----- README.md | 5 +---- ansible/init-data-gzipper.yaml | 10 ++++++++-- docs/dell-tdx-configuration.md | 2 +- values-global.yaml | 2 +- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ef9bfa42..d88316da 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,7 +5,7 @@ This file provides rules and context for any AI coding assistant working in this ## Critical Rules -- **DO NOT** edit anything under `/common/`. It is a read-only git subtree from the upstream validated patterns framework. +- **DO NOT** edit anything under `/common/`. It is a read-only Git subtree from the upstream validated patterns framework. - **DO NOT** commit secrets, credentials, or private keys. `values-secret.yaml.template` is a template only. - **DO NOT** use Kustomize. This project uses Helm exclusively. - **DO NOT** create charts with `apiVersion: v1`. Use `apiVersion: v2` (Helm 3+). @@ -23,7 +23,7 @@ Use the **first** approach that fits your requirement: ## Project Structure -``` +```text ├── ansible/ # Ansible playbooks (imperative jobs) ├── charts/ │ ├── all/ @@ -54,9 +54,9 @@ Use the **first** approach that fits your requirement: ## Companion Chart Repositories -Several charts in this repo have companion repositories for independent versioning and reuse. Develop and test in this repo first (charts deploy via `path:`), then sync changes to the companion repo. +Several charts in this repository have companion repositories for independent versioning and reuse. Develop and test in this repository first (charts deploy via `path:`), then sync changes to the companion repository. -| Local Path | Companion Repo | Purpose | +| Local Path | Companion Repository | Purpose | |---|---|---| | `charts/hub/trustee/` | `trustee-chart` | Trustee / KBS on hub | | `charts/hub/sandbox-policies/` | `sandboxed-policies-chart` | ACM policies hub → spoke | @@ -132,7 +132,7 @@ All commands run via `./pattern.sh make `: | `load-secrets` | Load secrets into the configured backend | | `uninstall` | Uninstall the pattern | -See the README for secrets backend configuration, RHDP environment variables, and additional maintenance commands. +See the readme for secrets backend configuration, RHDP environment variables, and additional maintenance commands. ## Validation and CI diff --git a/README.md b/README.md index 496716dc..9c7b61bd 100644 --- a/README.md +++ b/README.md @@ -47,8 +47,7 @@ All previous versions used pre-GA (Technology Preview) releases of Trustee: - Azure DNS hosting the cluster's DNS zone - Tools on your workstation: `podman`, `yq`, `jq`, `skopeo` - OpenShift pull secret saved at `~/pull-secret.json` (download from [console.redhat.com](https://console.redhat.com/openshift/downloads)) -- Fork the repo — ArgoCD reconciles cluster state against your fork, so changes must be pushed to your remote - +- Fork the repository — ArgoCD reconciles cluster state against your fork, so changes must be pushed to your remote ### Secrets and PCR setup @@ -130,5 +129,3 @@ Deployment commands: - Multi-cluster: `bash rhdp/wrapper-multicluster.sh ` The wrapper scripts handle cluster provisioning via `openshift-install`, secret generation, PCR retrieval, and pattern installation. - - diff --git a/ansible/init-data-gzipper.yaml b/ansible/init-data-gzipper.yaml index 3ffeddbf..c5a161ca 100644 --- a/ansible/init-data-gzipper.yaml +++ b/ansible/init-data-gzipper.yaml @@ -47,6 +47,7 @@ - name: Gzip and base64 encode the rendered content ansible.builtin.shell: | + set -o pipefail cat "{{ rendered_path }}" | gzip | base64 -w0 register: initdata_encoded changed_when: false @@ -55,16 +56,21 @@ # The script performs the following steps: # 1. hash=$(sha256sum initdata.toml | cut -d' ' -f1): Computes the sha256 hash of 'initdata.toml' and assigns it to $hash. # 2. initial_pcr=0000000000000000000000000000000000000000000000000000000000000000: Initializes a string of zeros as the initial PCR value. - # 3. PCR8_HASH=$(echo -n "$initial_pcr$hash" | xxd -r -p | sha256sum | cut -d' ' -f1): Concatenates initial_pcr and $hash, converts from hex to binary, computes its sha256 hash, and stores the result as PCR8_HASH. + # 3. PCR8_HASH=$(echo -n "$initial_pcr$hash" | xxd -r -p | sha256sum | cut -d' ' -f1): + # Concatenates initial_pcr and $hash, converts from hex to binary, + # computes its sha256 hash, and stores the result as PCR8_HASH. # 4. echo $PCR8_HASH: Outputs the PCR hash value. - # The important part: The 'register: pcr8_hash' registers the **stdout of the command**, which is the value output by 'echo $PCR8_HASH', as 'pcr8_hash.stdout' in Ansible. + # The important part: The 'register: pcr8_hash' registers the **stdout of the command**, + # which is the value output by 'echo $PCR8_HASH', as 'pcr8_hash.stdout' in Ansible. # It does NOT register an environment variable, but rather the value actually printed by 'echo'. - name: Register init data pcr into a var ansible.builtin.shell: | + set -o pipefail hash=$(sha256sum "{{ rendered_path }}" | cut -d' ' -f1) initial_pcr=0000000000000000000000000000000000000000000000000000000000000000 PCR8_HASH=$(echo -n "$initial_pcr$hash" | xxd -r -p | sha256sum | cut -d' ' -f1) && echo $PCR8_HASH register: pcr8_hash + changed_when: false - name: Create/update ConfigMap with gzipped+base64 content diff --git a/docs/dell-tdx-configuration.md b/docs/dell-tdx-configuration.md index f3b13a01..ecf078fd 100644 --- a/docs/dell-tdx-configuration.md +++ b/docs/dell-tdx-configuration.md @@ -88,7 +88,7 @@ Navigate to: **System BIOS → Processor Settings → Software Guard Extensions ## Configuration Summary (Order of Operations) -``` +```text 1. Disable Node Interleaving → Save & Reboot 2. Enable x2APIC Mode → Save & Reboot 3. Disable CPU Physical Address Limit → Save & Reboot diff --git a/values-global.yaml b/values-global.yaml index 48d12c4e..7a1a8071 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -24,7 +24,7 @@ main: # WARNING # This default configuration uses a single cluster on azure. # It fundamentally violates the separation of duties. - clusterGroupName: trusted-hub + clusterGroupName: simple multiSourceConfig: enabled: true clusterGroupChartVersion: 0.9.*