diff --git a/.github/workflows/mutants.yml b/.github/workflows/mutants.yml new file mode 100644 index 0000000..0a5da4c --- /dev/null +++ b/.github/workflows/mutants.yml @@ -0,0 +1,139 @@ +name: Mutation Testing + +on: + pull_request: + paths: + - ".cargo/*.toml" + - ".github/workflows/mutants.yml" + - "Cargo.*" + - "src/**" + - "tests/**" + +env: + CARGO_TERM_COLOR: always + +permissions: + contents: read + pull-requests: write + +jobs: + mutants: + name: Mutation Testing + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache dependencies + uses: Swatinem/rust-cache@v2 + + - name: Install cargo-mutants + uses: taiki-e/install-action@v2 + with: + tool: cargo-mutants + + - name: Relative diff + run: | + git branch -av + git diff origin/${{ github.base_ref }}.. | tee git.diff + + - name: Run cargo-mutants + run: cargo mutants --no-shuffle --all-features --in-diff git.diff --in-place + continue-on-error: true + + - name: Check mutants report + id: report + run: | + if [ -f mutants.out/outcomes.json ]; then + echo "exists=true" >> "$GITHUB_OUTPUT" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + fi + + - name: Comment on PR + if: steps.report.outputs.exists == 'true' && (github.event.pull_request.head.repo.full_name == github.repository) + uses: actions/github-script@v8 + with: + script: | + const fs = require('fs'); + const report = JSON.parse(fs.readFileSync('mutants.out/outcomes.json', 'utf8')); + + const counts = { + caught: Number(report.caught ?? 0), + missed: Number(report.missed ?? 0), + timeout: Number(report.timeout ?? 0), + unviable: Number(report.unviable ?? 0), + }; + + const missed = (report.outcomes ?? []) + .filter(o => o.summary === 'MissedMutant' && o.scenario?.Mutant) + .map(o => { + const mutant = o.scenario.Mutant; + const functionName = mutant.function?.function_name ?? mutant.name; + const replacement = mutant.replacement ?? 'unknown replacement'; + return `${functionName}: ${replacement}`; + }); + + const totalMutants = Number(report.total_mutants ?? 0); + const testable = counts.caught + counts.missed; + const score = testable > 0 ? (counts.caught / testable * 100).toFixed(1) : 'N/A'; + + let body = '## 🧬 Mutation Testing Results\n\n'; + body += 'Scope: PR diff (`--in-diff`)\n\n'; + body += '| Metric | Count |\n|--------|-------|\n'; + body += `| Caught | ${counts.caught} |\n`; + body += `| Missed | ${counts.missed} |\n`; + body += `| Timeout | ${counts.timeout} |\n`; + body += `| Unviable | ${counts.unviable} |\n`; + body += `| **Mutation Score** | **${score}%** |\n\n`; + + if (totalMutants === 0) { + body += 'No mutants were generated for the PR diff. This is expected for test-only or non-Rust changes.\n'; + } else if (missed.length > 0) { + body += '
Missed mutants (test gaps)\n\n```\n'; + body += missed.join('\n'); + body += '\n```\n\n
\n'; + } else { + body += 'All mutants in the PR diff were caught by tests.\n'; + } + + const marker = '## 🧬 Mutation Testing Results'; + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + const existing = comments.find(c => c.body?.includes(marker)); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + + - name: No report notice + if: steps.report.outputs.exists == 'false' + run: echo "cargo-mutants did not produce mutants.out/outcomes.json; skipping PR comment and artifact upload." + + - name: Upload mutants report + if: steps.report.outputs.exists == 'true' + uses: actions/upload-artifact@v7 + with: + name: mutants-report + path: mutants.out/ + retention-days: 14 diff --git a/.gitignore b/.gitignore index 6520452..ceeae4f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ # Generated by Cargo target/ +# Mutation testing output +mutants.out/ + .vscode/ diff --git a/src/ast.rs b/src/ast.rs index 19029a0..ccf5fa5 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -501,6 +501,30 @@ mod helper_function_tests { assert!(builtin_fn != func); // cross-variant catch-all arm + // PartialEq: Functions with different params are not equal + let func_different_params = + eval(&parse_scheme("(lambda (y) (+ y 1))").unwrap(), &mut env).unwrap(); + assert!(func != func_different_params); + + // PartialEq: Functions with same params/body but different captured environments are not equal + let mut env3 = create_global_env(); + eval(&parse_scheme("(define z 99)").unwrap(), &mut env3).unwrap(); + let func_different_env = + eval(&parse_scheme("(lambda (x) (+ x 1))").unwrap(), &mut env3).unwrap(); + assert!(func != func_different_env); + + // PartialEq: PrecompiledOps with same op but different args are not equal + let precompiled_different_args = parse_scheme("(+ 3 4)").unwrap(); + assert!(precompiled != precompiled_different_args); + + // PartialEq: Unspecified never equals non-Unspecified values + assert!(Value::Unspecified != val(0)); + assert!(Value::Unspecified != val(false)); + assert!(Value::Unspecified != val("")); + assert!(Value::Unspecified != nil()); + assert!(val(0) != Value::Unspecified); + assert!(val(false) != Value::Unspecified); + // Error Display: (error, expected_substring) let error_cases: Vec<(crate::Error, &str)> = vec![ ( diff --git a/src/builtinops.rs b/src/builtinops.rs index a237a55..8c6f3a2 100644 --- a/src/builtinops.rs +++ b/src/builtinops.rs @@ -674,6 +674,10 @@ mod tests { && (add_ref.op_kind != if_ref.op_kind) && (*add_ref == add_ref.clone()) ); + + // BuiltinOp::eq: different builtins are not equal + let sub_ref = find_scheme_op("-").unwrap(); + assert!(add_ref != sub_ref); } /// Macro to create test cases, invoking builtins via the registry. diff --git a/src/evaluator.rs b/src/evaluator.rs index 374d05d..81182ff 100644 --- a/src/evaluator.rs +++ b/src/evaluator.rs @@ -1089,11 +1089,18 @@ mod tests { ("(+ 1 (car \"not-a-list\"))", Error), ("(if (not 42) 1 2)", Error), // Test lambda parameter errors - ("(lambda (x x) x)", Error), // Duplicate params - ("(lambda \"not-a-list\" 42)", Error), // Invalid params + ("(lambda (x x) x)", Error), // Duplicate params + ( + "(lambda \"not-a-list\" 42)", + SpecificError("must be a list"), + ), // Invalid params + ("(lambda 42 x)", SpecificError("must be a list")), // Non-list params // Test define errors - ("(define 123 42)", Error), // Invalid var name - ("(define \"not-symbol\" 42)", Error), // Invalid var name + ("(define 123 42)", SpecificError("define requires a symbol")), + ( + "(define \"not-symbol\" 42)", + SpecificError("define requires a symbol"), + ), // === ERROR CASES === // Unbound variables ( diff --git a/src/jsonlogic.rs b/src/jsonlogic.rs index 52031bf..4710a7c 100644 --- a/src/jsonlogic.rs +++ b/src/jsonlogic.rs @@ -512,6 +512,8 @@ mod tests { r#"{"123invalid": [1, 2]}"#, SpecificError("Invalid operator name"), ), + // Invalid var names should be rejected + (r#"{"var": "123"}"#, SpecificError("valid symbol")), // Design validation tests - operations intentionally rejected/different ( r#"{"unknown_not": [true]}"#, @@ -803,10 +805,38 @@ mod tests { "{label} should fail conversion" ); } + } - // Non-symbol list converts to JSON array - let list_val = Value::List(vec![Value::Number(1), Value::Number(2)]); - assert_eq!(ast_to_jsonlogic(&list_val).unwrap(), "[1,2]"); + #[test] + fn test_ast_to_jsonlogic_roundtrip() { + // AST values that should convert to specific JSON representations + let roundtrip_cases: Vec<(Value, &str)> = vec![ + // Non-symbol list converts to JSON array + ( + Value::List(vec![Value::Number(1), Value::Number(2)]), + "[1,2]", + ), + // Empty list converts to empty JSON array + (Value::List(vec![]), "[]"), + // PrecompiledOp "list" roundtrips to JSON array + (parse_scheme("(list 1 2 3)").unwrap(), "[1,2,3]"), + // Unprecompiled Value::List with Symbol("list") also roundtrips to JSON array + ( + Value::List(vec![ + Value::Symbol("list".into()), + Value::Number(4), + Value::Number(5), + ]), + "[4,5]", + ), + ]; + for (value, expected_json) in &roundtrip_cases { + assert_eq!( + ast_to_jsonlogic(value).unwrap(), + *expected_json, + "Roundtrip failed for {value:?}" + ); + } } /// Helper function to test AST equivalence and roundtrip (shared by Identical and IdenticalWithEvalError) diff --git a/src/scheme.rs b/src/scheme.rs index fede57b..754c103 100644 --- a/src/scheme.rs +++ b/src/scheme.rs @@ -695,6 +695,8 @@ mod tests { ("(not #f)", precompiled_op("not", vec![val(false)])), // Test nested arity errors are also caught ("(list (not) 42)", SpecificError("ArityError")), + // Arity error nested inside unprecompiled list (unknown-fn isn't a builtin, so outer remains Value::List) + ("(unknown-fn (if #t 1))", SpecificError("ArityError")), // ===== PARSE-TIME SYNTAX ERRORS ===== // Unclosed parentheses - should contain parse error information ("(+ 1 (- 2", SpecificError("ParseError")), @@ -737,6 +739,12 @@ mod tests { ")".repeat(MAX_PARSE_DEPTH) ); let deep_quotes_at_limit = format!("{}a", "'".repeat(MAX_PARSE_DEPTH)); + // Longhand (quote ...) nesting exercises the depth tracking in parse_list's quote path + let deep_longhand_quote_at_limit = format!( + "{}a{}", + "(quote ".repeat(MAX_PARSE_DEPTH), + ")".repeat(MAX_PARSE_DEPTH) + ); let depth_test_cases = vec![ // At/over limit should fail at parse time with specific error @@ -748,6 +756,11 @@ mod tests { deep_quotes_at_limit.as_str(), SpecificError("Invalid syntax"), ), + // Longhand (quote ...) nesting should also be caught + ( + deep_longhand_quote_at_limit.as_str(), + SpecificError("Invalid syntax"), + ), ]; run_parse_tests(depth_test_cases);