From 78c69f650bac8c93e6aeebb55fbbe8dd3317c852 Mon Sep 17 00:00:00 2001 From: Val Redchenko Date: Wed, 8 Apr 2026 16:04:34 +0100 Subject: [PATCH] chore: remove detect-secrets, trufflehog, and pip-audit Consolidate secret scanning to gitleaks, vulnerability scanning to osv-scanner. --- .github/workflows/leaked-secrets-scan.yml | 96 -------- .pre-commit-config.yaml | 12 - .secrets.baseline | 214 ------------------ claude-code/ARCHITECTURE.md | 2 +- claude-code/shared/skills/devops/SKILL.md | 2 +- .../smartem-decisions/agents/devops.md | 2 +- ...0005-detect-secrets-for-secret-scanning.md | 4 +- lefthook.yml | 2 - scripts/pip-audit-scan-repos.sh | 152 ------------- 9 files changed, 6 insertions(+), 480 deletions(-) delete mode 100644 .secrets.baseline delete mode 100755 scripts/pip-audit-scan-repos.sh diff --git a/.github/workflows/leaked-secrets-scan.yml b/.github/workflows/leaked-secrets-scan.yml index 68f4ac2..493da8b 100644 --- a/.github/workflows/leaked-secrets-scan.yml +++ b/.github/workflows/leaked-secrets-scan.yml @@ -10,77 +10,6 @@ on: branches: [main] jobs: - detect-secrets: - runs-on: ubuntu-latest - continue-on-error: true - name: detect-secrets - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - name: Install detect-secrets - run: pipx install detect-secrets - - name: Verify baseline exists - run: | - if [ ! -f .secrets.baseline ]; then - echo "::error::.secrets.baseline not found!" - exit 1 - fi - echo "Found .secrets.baseline" - - name: Scan for secrets - run: | - echo "Scanning for secrets..." - detect-secrets scan \ - --baseline .secrets.baseline \ - --exclude-files '.*\.lock$' \ - --force-use-all-plugins - - name: Audit baseline for unaudited secrets - run: | - echo "Auditing secrets baseline..." - if grep -q '"is_secret": null' .secrets.baseline; then - echo "::error::Found unaudited secrets in baseline! Run: detect-secrets audit .secrets.baseline" - detect-secrets audit .secrets.baseline --report - exit 1 - fi - echo "All secrets in baseline have been audited" - detect-secrets audit .secrets.baseline --report - - name: Check for new secrets in PR - if: github.event_name == 'pull_request' - run: | - echo "Checking for new secrets in PR..." - mkdir -p /tmp/pr-scan - git diff origin/main...HEAD --name-only | while read -r file; do - if [ -f "$file" ]; then - mkdir -p "/tmp/pr-scan/$(dirname "$file")" 2>/dev/null || true - cp "$file" "/tmp/pr-scan/$file" 2>/dev/null || true - fi - done - if [ "$(ls -A /tmp/pr-scan 2>/dev/null)" ]; then - echo "Scanning changed files..." - detect-secrets scan \ - --baseline .secrets.baseline \ - --exclude-files '.*\.lock$' \ - --force-use-all-plugins \ - /tmp/pr-scan || echo "No new secrets found" - else - echo "No files to scan" - fi - - name: Full repository scan (scheduled) - if: github.event_name == 'schedule' - run: | - echo "Performing full repository scan..." - detect-secrets scan \ - --exclude-files '.*\.lock$' \ - --force-use-all-plugins - - name: Upload baseline on failure - uses: actions/upload-artifact@v7 - if: failure() - with: - name: detect-secrets-report - path: .secrets.baseline - retention-days: 30 - gitleaks-cli: name: gitleaks (CLI) runs-on: ubuntu-latest @@ -105,28 +34,3 @@ jobs: name: gitleaks-report path: gitleaks-report.sarif retention-days: 30 - - trufflehog: - name: trufflehog - runs-on: ubuntu-latest - continue-on-error: true - env: - TRUFFLEHOG_VERSION: '3.93.8' - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - name: Install trufflehog - run: | - curl -sSfL "https://github.com/trufflesecurity/trufflehog/releases/download/v${TRUFFLEHOG_VERSION}/trufflehog_${TRUFFLEHOG_VERSION}_linux_amd64.tar.gz" \ - | tar -xz -C /usr/local/bin trufflehog - trufflehog --version - - name: Run trufflehog - run: trufflehog git file://. --only-verified --fail --json 2>&1 | tee trufflehog-report.json - - name: Upload report - uses: actions/upload-artifact@v7 - if: always() - with: - name: trufflehog-report - path: trufflehog-report.json - retention-days: 30 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 01ecd3c..644d8d7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,20 +8,8 @@ repos: - id: check-merge-conflict - id: end-of-file-fixer - - repo: https://github.com/Yelp/detect-secrets - rev: v1.5.0 - hooks: - - id: detect-secrets - args: ['--baseline', '.secrets.baseline', '--exclude-files', '.*\.lock$'] - - repo: https://github.com/gitleaks/gitleaks rev: v8.30.0 hooks: - id: gitleaks stages: [pre-push] - - - repo: https://github.com/trufflesecurity/trufflehog - rev: v3.93.8 - hooks: - - id: trufflehog - stages: [pre-push] diff --git a/.secrets.baseline b/.secrets.baseline deleted file mode 100644 index 28a89c4..0000000 --- a/.secrets.baseline +++ /dev/null @@ -1,214 +0,0 @@ -{ - "version": "1.5.0", - "plugins_used": [ - { - "name": "ArtifactoryDetector" - }, - { - "name": "AWSKeyDetector" - }, - { - "name": "AzureStorageKeyDetector" - }, - { - "name": "Base64HighEntropyString", - "limit": 4.5 - }, - { - "name": "BasicAuthDetector" - }, - { - "name": "CloudantDetector" - }, - { - "name": "DiscordBotTokenDetector" - }, - { - "name": "GitHubTokenDetector" - }, - { - "name": "GitLabTokenDetector" - }, - { - "name": "HexHighEntropyString", - "limit": 3.0 - }, - { - "name": "IbmCloudIamDetector" - }, - { - "name": "IbmCosHmacDetector" - }, - { - "name": "IPPublicDetector" - }, - { - "name": "JwtTokenDetector" - }, - { - "name": "KeywordDetector", - "keyword_exclude": "" - }, - { - "name": "MailchimpDetector" - }, - { - "name": "NpmDetector" - }, - { - "name": "OpenAIDetector" - }, - { - "name": "PrivateKeyDetector" - }, - { - "name": "PypiTokenDetector" - }, - { - "name": "SendGridDetector" - }, - { - "name": "SlackDetector" - }, - { - "name": "SoftlayerDetector" - }, - { - "name": "SquareOAuthDetector" - }, - { - "name": "StripeDetector" - }, - { - "name": "TelegramBotTokenDetector" - }, - { - "name": "TwilioKeyDetector" - } - ], - "filters_used": [ - { - "path": "detect_secrets.filters.allowlist.is_line_allowlisted" - }, - { - "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", - "min_level": 2 - }, - { - "path": "detect_secrets.filters.heuristic.is_indirect_reference" - }, - { - "path": "detect_secrets.filters.heuristic.is_likely_id_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_lock_file" - }, - { - "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_potential_uuid" - }, - { - "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" - }, - { - "path": "detect_secrets.filters.heuristic.is_sequential_string" - }, - { - "path": "detect_secrets.filters.heuristic.is_swagger_file" - }, - { - "path": "detect_secrets.filters.heuristic.is_templated_secret" - }, - { - "path": "detect_secrets.filters.regex.should_exclude_file", - "pattern": [ - ".*\\.lock$" - ] - } - ], - "results": { - "claude-code/smartem-frontend/skills/playwright/API_REFERENCE.md": [ - { - "type": "Secret Keyword", - "filename": "claude-code/smartem-frontend/skills/playwright/API_REFERENCE.md", - "hashed_secret": "f0578f1e7174b1a41c4ea8c6e17f7a8a3b88c92a", - "is_verified": false, - "line_number": 499 - }, - { - "type": "Secret Keyword", - "filename": "claude-code/smartem-frontend/skills/playwright/API_REFERENCE.md", - "hashed_secret": "8be52126a6fde450a7162a3651d589bb51e9579d", - "is_verified": false, - "line_number": 500 - } - ], - "docs/decision-records/smartem-workspace-developer-guide.md": [ - { - "type": "Secret Keyword", - "filename": "docs/decision-records/smartem-workspace-developer-guide.md", - "hashed_secret": "8bed7940e0c3ea61eca107fdb36282d4ceef45d4", - "is_verified": false, - "line_number": 1135 - }, - { - "type": "Secret Keyword", - "filename": "docs/decision-records/smartem-workspace-developer-guide.md", - "hashed_secret": "46192f4a6b473b1d14528bad8da6300e84cb4d0e", - "is_verified": false, - "line_number": 1140 - } - ], - "docs/development/e2e-simulation.md": [ - { - "type": "Basic Auth Credentials", - "filename": "docs/development/e2e-simulation.md", - "hashed_secret": "35675e68f4b5af7b995d9205ad0fc43842f16450", - "is_verified": false, - "line_number": 410 - } - ], - "docs/operations/kubernetes-secrets.md": [ - { - "type": "Secret Keyword", - "filename": "docs/operations/kubernetes-secrets.md", - "hashed_secret": "142879ed2e01ae92cf503554bfd2241c3e024a0f", - "is_verified": false, - "line_number": 115 - }, - { - "type": "Secret Keyword", - "filename": "docs/operations/kubernetes-secrets.md", - "hashed_secret": "3ba67f062408ecadaabf84c340240f69c1a0a318", - "is_verified": false, - "line_number": 117 - }, - { - "type": "Secret Keyword", - "filename": "docs/operations/kubernetes-secrets.md", - "hashed_secret": "3a12b92524e2bb602238d9f3e28f885d097668bb", - "is_verified": false, - "line_number": 318 - }, - { - "type": "Secret Keyword", - "filename": "docs/operations/kubernetes-secrets.md", - "hashed_secret": "76a3330d1ebe1e20c9824c59cafac491bcfa43c3", - "is_verified": false, - "line_number": 334 - } - ], - "k8s/secret.example.yaml": [ - { - "type": "Secret Keyword", - "filename": "k8s/secret.example.yaml", - "hashed_secret": "da19880411eca9df7c34f4f4690fb783e9c38d38", - "is_verified": false, - "line_number": 9 - } - ] - }, - "generated_at": "2026-01-26T18:21:21Z" -} diff --git a/claude-code/ARCHITECTURE.md b/claude-code/ARCHITECTURE.md index 447f096..d7e5d6f 100644 --- a/claude-code/ARCHITECTURE.md +++ b/claude-code/ARCHITECTURE.md @@ -182,7 +182,7 @@ export ARIA_GQL_LOCAL=http://localhost:9002/graphql ### smartem-decisions CI Features - Schema drift checking (Alembic vs SQLModel) -- Security scanning (detect-secrets) +- Security scanning (gitleaks) - Versioned docs to GitHub Pages - Windows .exe builds (PyInstaller) for agent and epuplayer diff --git a/claude-code/shared/skills/devops/SKILL.md b/claude-code/shared/skills/devops/SKILL.md index 854df67..fbf7842 100644 --- a/claude-code/shared/skills/devops/SKILL.md +++ b/claude-code/shared/skills/devops/SKILL.md @@ -174,7 +174,7 @@ kubectl get secret db-credentials -n smartem-decisions -o jsonpath='{.data.POSTG | ci.yml | Push/PR to main | Tests, lint, type check | | _container.yml | Tag push (v*) | Build and push Docker image | | _docs.yml | Tag push | Build and publish docs | -| security-scan.yml | Schedule/manual | detect-secrets scanning | +| security-scan.yml | Schedule/manual | gitleaks scanning | | build_win_smartem_agent.yml | Push to main | Windows .exe build | ### Debugging CI Failures diff --git a/claude-code/smartem-decisions/agents/devops.md b/claude-code/smartem-decisions/agents/devops.md index 3d5c63f..8c78d75 100644 --- a/claude-code/smartem-decisions/agents/devops.md +++ b/claude-code/smartem-decisions/agents/devops.md @@ -95,7 +95,7 @@ Tools and technologies you work with: - Docker/Podman for containerisation and image management - kubectl, kustomize, Helm for Kubernetes deployment and configuration management - Development scripts and local cluster management (k3d, kind, development tools) -- Security scanning tools (detect-secrets, container vulnerability scanners) +- Security scanning tools (gitleaks, container vulnerability scanners) - Monitoring stacks (Prometheus, Grafana, logging aggregation systems) - Infrastructure automation and GitOps tools for configuration management diff --git a/docs/decision-records/decisions/0005-detect-secrets-for-secret-scanning.md b/docs/decision-records/decisions/0005-detect-secrets-for-secret-scanning.md index d6cbd61..9c5438f 100644 --- a/docs/decision-records/decisions/0005-detect-secrets-for-secret-scanning.md +++ b/docs/decision-records/decisions/0005-detect-secrets-for-secret-scanning.md @@ -4,7 +4,9 @@ Date: 21/08/2025 ## Status -Accepted +Superseded by gitleaks consolidation + +**Note:** detect-secrets was removed in favour of consolidating on gitleaks as the single secret scanning tool across the project. This simplifies the security tool stack and aligns with the organisational preference for gitleaks standardisation. ## Context diff --git a/lefthook.yml b/lefthook.yml index 420112b..d40e2a7 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -29,5 +29,3 @@ pre-push: run: npm run format:check gitleaks: run: gitleaks protect --staged --redact - trufflehog: - run: trufflehog git file://. --only-verified diff --git a/scripts/pip-audit-scan-repos.sh b/scripts/pip-audit-scan-repos.sh deleted file mode 100755 index 0bea9b9..0000000 --- a/scripts/pip-audit-scan-repos.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env bash -# -# Local pip-audit vulnerability scanner for ERIC workspace repos (Python only) -# Scans specified repositories and outputs results to stdout or a file -# Uses uvx to run pip-audit without permanent installation -# -# Usage: -# ./pip-audit-scan-repos.sh # Scan default repos, output to stdout -# ./pip-audit-scan-repos.sh -o results.md # Output as markdown to file -# ./pip-audit-scan-repos.sh repo1 repo2 # Scan specific repo paths -# -# Requirements: -# - uvx (from uv package manager) - -set -euo pipefail - -# Default repos relative to ERIC workspace root -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ERIC_ROOT="${SCRIPT_DIR}/../../../.." - -# Discover all repos in ERIC workspace (repos/*/* directories with .git) -discover_repos() { - local repos_dir="${ERIC_ROOT}/repos" - if [[ -d "$repos_dir" ]]; then - find "$repos_dir" -mindepth 2 -maxdepth 2 -type d -exec test -d '{}/.git' \; -print | sort - fi -} - -OUTPUT_FILE="" -REPOS=() - -# Parse arguments -while [[ $# -gt 0 ]]; do - case $1 in - -o|--output) - OUTPUT_FILE="$2" - shift 2 - ;; - -h|--help) - echo "Usage: $0 [-o output.md] [repo_path ...]" - echo "" - echo "Options:" - echo " -o, --output FILE Write results to FILE (markdown format)" - echo " -h, --help Show this help" - echo "" - echo "If no repos specified, scans all ERIC workspace Python repos." - echo "Note: Only scans Python repos with requirements.txt files." - exit 0 - ;; - *) - REPOS+=("$1") - shift - ;; - esac -done - -# Use discovered repos if none specified -if [[ ${#REPOS[@]} -eq 0 ]]; then - while IFS= read -r repo; do - REPOS+=("$repo") - done < <(discover_repos) -fi - -scan_repo() { - local repo_path="$1" - local repo_name - repo_name=$(basename "$repo_path") - - if [[ ! -d "$repo_path" ]]; then - echo "SKIP: $repo_name (directory not found)" - return - fi - - if [[ ! -f "$repo_path/requirements.txt" ]]; then - echo "SKIP: $repo_name (no requirements.txt)" - return - fi - - echo "Scanning: $repo_name" - uvx pip-audit -r "$repo_path/requirements.txt" 2>&1 || true -} - -scan_repo_markdown() { - local repo_path="$1" - local repo_name - repo_name=$(basename "$repo_path") - - echo "### $repo_name" - echo "" - - if [[ ! -d "$repo_path" ]]; then - echo "**Status:** Skipped (directory not found)" - echo "" - return - fi - - if [[ ! -f "$repo_path/requirements.txt" ]]; then - echo "**Status:** Skipped (no requirements.txt - not a Python project or uses different dep format)" - echo "" - return - fi - - local output - output=$(uvx pip-audit -r "$repo_path/requirements.txt" 2>&1) || true - - if echo "$output" | grep -q "No known vulnerabilities found"; then - echo "**Status:** Clean - no vulnerabilities found" - elif echo "$output" | grep -q "found [0-9]* known vulnerabilit"; then - local vuln_count - vuln_count=$(echo "$output" | grep -oP "found \K[0-9]+" | head -1) - echo "**Status:** $vuln_count vulnerabilities found" - echo "" - echo '```' - echo "$output" - echo '```' - else - echo "**Status:** Scan completed" - echo "" - echo '```' - echo "$output" - echo '```' - fi - echo "" -} - -# Run scans -if [[ -n "$OUTPUT_FILE" ]]; then - { - echo "# pip-audit Scanner Results" - echo "" - echo "**Scan date:** $(date -Iseconds)" - echo "" - echo "**Scanner:** pip-audit (via uvx)" - echo "" - echo "**Note:** pip-audit only scans Python dependencies (requirements.txt). PHP repos are skipped." - echo "" - echo "---" - echo "" - - for repo in "${REPOS[@]}"; do - scan_repo_markdown "$repo" - done - } > "$OUTPUT_FILE" - - echo "Results written to: $OUTPUT_FILE" -else - for repo in "${REPOS[@]}"; do - echo "========================================" - scan_repo "$repo" - echo "" - done -fi