CP-37371: bundle Prometheus binary in CloudZero Agent image

.tools/go.mod

@@ -1,6 +1,6 @@
 module github.com/cloudzero/cloudzero-agent/.tools
 
-go 1.25.5
+go 1.25.6
 
 require (
 	github.com/homeport/dyff v1.10.3

app/functions/helmless/default-values.yaml

@@ -553,10 +553,27 @@ components:
   # prometheus contains details about where to find the Prometheus image.
   # Prometheus is critical to the functionality of this chart, and is used to
   # scrape metrics.
+  #
+  # By default, this uses the CloudZero Agent image which bundles the Prometheus
+  # binary in a minimal scratch-based image.
+  # To use the official Prometheus image instead:
+  #
+  #   image:
+  #     repository: quay.io/prometheus/prometheus
+  #     tag: vX.Y.Z
+  #   command: []
+  #
   prometheus:
+    # Container image configuration for Prometheus.
     image:
-      repository: quay.io/prometheus/prometheus
-      tag:  # This will fall back on .Chart.AppVersion if not set.
+      repository:
+      tag:
+    # Command to run in the container.
+    #
+    # - null (default): Uses /app/prometheus (for CloudZero Agent image)
+    # - []: Uses the image's default entrypoint (for official Prometheus image)
+    # - ["/custom/path"]: Uses the specified command
+    command:
 
   # prometheusReloader contains details about where to find the Prometheus
   # reloader image.

docker/Dockerfile

@@ -1,14 +1,18 @@
 ARG DEPLOY_IMAGE=scratch
 
+# Versions for bundled third-party binaries
+ARG PROMETHEUS_VERSION=v3.9.1
+
 # Multi-stage Docker build with platform-specific cache optimization:
 # 1. base-tools: Install system packages and tools (cached per platform)
 # 2. dependencies: Download Go modules (cached per platform)
 # 3. builder: Build Go binaries (cached per platform)
 # 4. certs: Extract certificates from distroless image
-# 5. final: Minimal runtime image with compiled binaries
+# 5. prometheus-source: Extract Prometheus binary from official image
+# 6. final: Minimal runtime image with compiled binaries
 
 # Stage 1: Base tools installation
-FROM --platform=$BUILDPLATFORM golang:1.25.5-alpine AS base-tools
+FROM --platform=$BUILDPLATFORM golang:1.25.6-alpine AS base-tools
 ARG TARGETPLATFORM
 ARG TARGETOS TARGETARCH
 
@@ -69,7 +73,12 @@ RUN --mount=type=cache,target=/go/pkg/mod,id=gomod-$TARGETPLATFORM \
 # Stage 4: Access current certs
 FROM gcr.io/distroless/static-debian12:debug@sha256:20d9c135406d8029d30d59eaaa9d62d2edcd4ec5915dbcda324243c40460e8df AS certs
 
-# Stage 5: Final runtime image
+# Stage 5: Extract Prometheus binary from official image
+# This allows us to bundle Prometheus in our hardened scratch image
+ARG PROMETHEUS_VERSION
+FROM quay.io/prometheus/prometheus:${PROMETHEUS_VERSION} AS prometheus-source
+
+# Stage 6: Final runtime image
 # Note: For debugging, you can temporarily change the image used for building by
 # passing in something like this to 'docker build':
 #
@@ -102,7 +111,7 @@ LABEL io.artifacthub.package.license="Apache-2.0"
 VOLUME [ "/app/config" ]
 ENV PATH=/app:$PATH
 
-# Copy the Go binary from the builder stage
+# Copy the Go binaries from the builder stage
 COPY --from=builder /go/bin/cloudzero-agent-inspector /app/cloudzero-agent-inspector
 COPY --from=builder /go/bin/cloudzero-agent-validator /app/cloudzero-agent-validator
 COPY --from=builder /go/bin/cloudzero-collector /app/cloudzero-collector
@@ -113,6 +122,9 @@ COPY --from=builder /go/bin/cloudzero-helmless /app/cloudzero-helmless
 COPY --from=builder /go/bin/cloudzero-scout /app/cloudzero-scout
 COPY --from=builder /go/bin/cloudzero-certifik8s /app/cloudzero-certifik8s
 
+# Copy executables from other images
+COPY --from=prometheus-source /bin/prometheus /app/prometheus
+
 # Allow the default ENTRYPOINT from busybox to be the default,
 # however run the app as the default command
 CMD ["/app/cloudzero-agent-validator", "-h"]

go.mod

@@ -1,6 +1,6 @@
 module github.com/cloudzero/cloudzero-agent
 
-go 1.25.5
+go 1.25.6
 
 require (
 	github.com/google/go-cmp v0.7.0

helm/templates/_helpers.tpl

@@ -838,6 +838,26 @@ configuration will not overwrite values from the first configuration.
 {{- end -}}
 {{- end -}}
 
+{{/*
+Generate container command with special handling:
+- null/not set: Uses provided default command array
+- empty array []: No command output (uses image's default entrypoint)
+- non-empty array: Uses the specified command
+
+Usage: {{ include "cloudzero-agent.generateContainerCommand" (dict "command" .Values.components.prometheus.command "default" (list "/app/prometheus")) | nindent 10 }}
+*/}}
+{{- define "cloudzero-agent.generateContainerCommand" -}}
+{{- $isEmptyArray := and (kindIs "slice" .command) (empty .command) -}}
+{{- if not $isEmptyArray -}}
+command:
+  {{- if kindIs "invalid" .command }}
+  {{- toYaml .default | nindent 2 }}
+  {{- else }}
+  {{- toYaml .command | nindent 2 }}
+  {{- end }}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Generate image configuration with defaults.
 */}}
@@ -1410,8 +1430,8 @@ Returns: string (either "--agent", "--enable-feature=agent", or empty string)
 {{- define "cloudzero-agent.prometheusAgentFlag" -}}
   {{- $mode := include "cloudzero-agent.Values.components.agent.mode" . -}}
   {{- if or (eq $mode "agent") (eq $mode "federated") -}}
-    {{- /* Use same fallback chain as image generation: server.image.tag -> components.prometheus.image.tag -> Chart.AppVersion */ -}}
-    {{- $tag := .Values.server.image.tag | default .Values.components.prometheus.image.tag | default .Chart.AppVersion -}}
+    {{- /* Use same fallback chain as image generation: server.image.tag -> components.prometheus.image.tag -> components.agent.image.tag -> Chart.AppVersion */ -}}
+    {{- $tag := .Values.server.image.tag | default .Values.components.prometheus.image.tag | default .Values.components.agent.image.tag | default .Chart.AppVersion -}}
     {{- if hasPrefix "v2." $tag -}}
       --enable-feature=agent
     {{- else -}}

helm/templates/agent-daemonset.yaml

@@ -127,8 +127,8 @@ spec:
               readOnly: true
         {{- end }}
         - name: {{ template "cloudzero-agent.name" . }}-server
-          {{/* This is a little special because we want to fall back on the .Chart.AppVersion */}}
-          {{- include "cloudzero-agent.generateImage" (dict "defaults" .Values.defaults.image "image" .Values.components.prometheus.image "compat" (dict "repository" .Values.server.image.repository "tag" (.Values.server.image.tag | default .Values.components.prometheus.image.tag | default .Chart.AppVersion) "digest" .Values.server.image.digest "pullPolicy" .Values.server.image.pullPolicy)) | nindent 10 }}
+          {{/* Falls back to components.agent.image (CloudZero Agent) when prometheus.image is not set */}}
+          {{- include "cloudzero-agent.generateImage" (dict "defaults" .Values.components.agent.image "image" .Values.components.prometheus.image "compat" (dict "repository" .Values.server.image.repository "tag" (.Values.server.image.tag | default .Values.components.prometheus.image.tag) "digest" .Values.server.image.digest "pullPolicy" .Values.server.image.pullPolicy)) | nindent 10 }}
           env:
             - name: NODE_NAME
               valueFrom:
@@ -137,6 +137,7 @@ spec:
             {{- if .Values.server.env }}
             {{- toYaml .Values.server.env | indent 12}}
             {{- end }}
+          {{- include "cloudzero-agent.generateContainerCommand" (dict "command" .Values.components.prometheus.command "default" (list "/app/prometheus")) | nindent 10 }}
           args:
             - --config.file=/etc/config/prometheus.yml
             - --web.enable-lifecycle

helm/templates/agent-deploy.yaml

@@ -153,8 +153,18 @@ spec:
         {{- if ne (include "cloudzero-agent.Values.components.agent.mode" .) "clustered" }}
         # Prometheus server container
         - name: {{ template "cloudzero-agent.name" . }}-server
-          {{/* This is a little special because we want to fall back on the .Chart.AppVersion */}}
-          {{- include "cloudzero-agent.generateImage" (dict "defaults" .Values.defaults.image "image" .Values.components.prometheus.image "compat" (dict "repository" .Values.server.image.repository "tag" (.Values.server.image.tag | default .Values.components.prometheus.image.tag | default .Chart.AppVersion) "digest" .Values.server.image.digest "pullPolicy" .Values.server.image.pullPolicy)) | nindent 10 }}
+          {{/* Falls back to components.agent.image (CloudZero Agent) when prometheus.image is not set */}}
+          {{- include "cloudzero-agent.generateImage" (
+                dict
+                  "defaults" .Values.components.agent.image
+                  "image" .Values.components.prometheus.image
+                  "compat" (dict
+                    "repository" .Values.server.image.repository
+                    "tag" (
+                      .Values.server.image.tag
+                      | default .Values.components.prometheus.image.tag)
+                    "digest" .Values.server.image.digest
+                    "pullPolicy" .Values.server.image.pullPolicy)) | nindent 10 }}
           {{- if .Values.server.env }}
           env:
 {{ toYaml .Values.server.env | indent 12}}
@@ -177,6 +187,7 @@ spec:
                   - pre-stop
                   - -f
                   - /checks/app/config/validator.yml
+          {{- include "cloudzero-agent.generateContainerCommand" (dict "command" .Values.components.prometheus.command "default" (list "/app/prometheus")) | nindent 10 }}
           args:
             {{ toYaml .Values.server.args | nindent 12}}
             {{- $agentFlag := include "cloudzero-agent.prometheusAgentFlag" . }}

helm/tests/README.md

@@ -14,14 +14,32 @@ The helm unittest plugin allows us to test the actual rendered output of Helm te
 
 ### Prerequisites
 
-The helm unittest plugin must be installed:
+Install the helm unittest plugin via Make:
 
 ```bash
-helm plugin install https://github.com/helm-unittest/helm-unittest
+make install-tools-helm-unittest
 ```
 
-### Running the Tests
+### Running All Tests
 
 ```bash
-helm unittest .
+make helm-test-unittest
 ```
+
+### Running a Single Test File
+
+```bash
+make helm/tests/<test-file>.yaml-unittest
+```
+
+For example:
+
+```bash
+make helm/tests/prometheus_command_test.yaml-unittest
+```
+
+**Note:** Always use the Makefile targets rather than running `helm unittest` directly. The Makefile ensures:
+
+- Correct plugin version is installed
+- Required base values are applied (`helm/tests/values.yaml` provides `existingSecretName` to satisfy schema validation)
+- Consistent test environment

helm/tests/alloy_deployment_test.yaml

@@ -10,9 +10,11 @@ tests:
           path: spec.template.spec.containers[1].name
           value: cloudzero-agent-server
         template: templates/agent-deploy.yaml
+      # The default Prometheus container uses the CloudZero agent image which
+      # bundles Prometheus in a hardened scratch image.
       - matchRegex:
           path: spec.template.spec.containers[1].image
-          pattern: prometheus
+          pattern: cloudzero-agent
         template: templates/agent-deploy.yaml
       - isNotEmpty:
           path: data["prometheus.yml"]

helm/tests/alloy_image_configuration_test.yaml

@@ -33,12 +33,9 @@ tests:
         repository: custom.registry.io/custom/alloy
         tag: v2.0.0
     asserts:
-      - matchRegex:
-          path: spec.template.spec.containers[1].image
-          pattern: prometheus
       - notMatchRegex:
           path: spec.template.spec.containers[1].image
-          pattern: alloy
+          pattern: custom.registry.io/custom/alloy
 
   # Test image pull policy from components.agent.clusteredNode.image
   - it: should respect pullPolicy from components.agent.clusteredNode.image

helm/values.schema.json

@@ -6045,6 +6045,16 @@
         "prometheus": {
           "additionalProperties": false,
           "properties": {
+            "command": {
+              "oneOf": [
+                {
+                  "$ref": "#/$defs/io.k8s.api.core.v1.Container/properties/command"
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            },
             "image": {
               "$ref": "#/$defs/com.cloudzero.agent.image"
             }

helm/values.schema.yaml

@@ -952,6 +952,15 @@ properties:
             description: |
               Container image configuration for Prometheus.
             $ref: "#/$defs/com.cloudzero.agent.image"
+          command:
+            description: |
+              Command to run in the container.
+              - null (default): Uses /app/prometheus (for CloudZero Agent image)
+              - []: Uses the image's default entrypoint (for official Prometheus image)
+              - ["/custom/path"]: Uses the specified command
+            oneOf:
+              - $ref: "#/$defs/io.k8s.api.core.v1.Container/properties/command"
+              - type: "null"
 
       prometheusReloader:
         description: |

helm/values.yaml

@@ -553,10 +553,27 @@ components:
   # prometheus contains details about where to find the Prometheus image.
   # Prometheus is critical to the functionality of this chart, and is used to
   # scrape metrics.
+  #
+  # By default, this uses the CloudZero Agent image which bundles the Prometheus
+  # binary in a minimal scratch-based image.
+  # To use the official Prometheus image instead:
+  #
+  #   image:
+  #     repository: quay.io/prometheus/prometheus
+  #     tag: vX.Y.Z
+  #   command: []
+  #
   prometheus:
+    # Container image configuration for Prometheus.
     image:
-      repository: quay.io/prometheus/prometheus
-      tag:  # This will fall back on .Chart.AppVersion if not set.
+      repository:
+      tag:
+    # Command to run in the container.
+    #
+    # - null (default): Uses /app/prometheus (for CloudZero Agent image)
+    # - []: Uses the image's default entrypoint (for official Prometheus image)
+    # - ["/custom/path"]: Uses the specified command
+    command:
 
   # prometheusReloader contains details about where to find the Prometheus
   # reloader image.

scripts/ci-checks.sh

@@ -24,7 +24,7 @@ function check_go_version() {
             echo "${DOCKERFILE} does not have the desired Go version (${DESIRED_GO_VERSION})" >&2
             FAILED=true
         }
-    done < <(find . -type f -iname 'Dockerfile*' -not -path '*/node_modules/*' -print0)
+    done < <(find . -type f -iname 'Dockerfile*' -not -path '*/node_modules/*' -not -path './.tools/*' -print0)
 
     # go.mod
     for GO_MOD in \

tests/docker/Dockerfile.collector

@@ -1,4 +1,4 @@
-FROM golang:1.25.5 AS builder
+FROM golang:1.25.6 AS builder
 
 WORKDIR /app
 

tests/docker/Dockerfile.controller

@@ -1,4 +1,4 @@
-FROM golang:1.25.5 AS builder
+FROM golang:1.25.6 AS builder
 
 WORKDIR /app
 

tests/docker/Dockerfile.remotewrite

@@ -1,4 +1,4 @@
-FROM golang:1.25.5 AS builder
+FROM golang:1.25.6 AS builder
 
 WORKDIR /app
 

tests/docker/Dockerfile.shipper

@@ -1,4 +1,4 @@
-FROM golang:1.25.5 AS builder
+FROM golang:1.25.6 AS builder
 
 # Copy source code
 COPY go.mod go.sum ./

tests/go.mod

@@ -1,6 +1,6 @@
 module github.com/cloudzero/cloudzero-agent/tests
 
-go 1.25.5
+go 1.25.6
 
 require (
 	github.com/andybalholm/brotli v1.2.0

tests/helm/schema/prometheus.command.empty.pass.yaml

@@ -0,0 +1,4 @@
+# Test empty prometheus.command (uses image default entrypoint)
+components:
+  prometheus:
+    command: []

tests/helm/schema/prometheus.command.invalid-items.fail.yaml

@@ -0,0 +1,6 @@
+# Test invalid prometheus.command items (numbers instead of strings)
+components:
+  prometheus:
+    command:
+      - 123
+      - 456

tests/helm/schema/prometheus.command.invalid-type.fail.yaml

@@ -0,0 +1,4 @@
+# Test invalid prometheus.command type (string instead of array)
+components:
+  prometheus:
+    command: "/app/prometheus"

tests/helm/schema/prometheus.command.multiple.pass.yaml

@@ -0,0 +1,7 @@
+# Test valid prometheus.command with multiple elements
+components:
+  prometheus:
+    command:
+      - /app/prometheus
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus

tests/helm/schema/prometheus.command.null.pass.yaml

@@ -0,0 +1,4 @@
+# Test null prometheus.command (uses default /app/prometheus)
+components:
+  prometheus:
+    command: null

tests/helm/schema/prometheus.command.valid.pass.yaml

@@ -0,0 +1,5 @@
+# Test valid prometheus.command configuration with single element
+components:
+  prometheus:
+    command:
+      - /app/prometheus