evals.yml

name: Evals

on:
  push:
    branches: ['**']

jobs:
  run-evals:
    runs-on: ubuntu-latest
    timeout-minutes: 360
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Check commit message
        id: check_commit
        env:
          COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
        run: |
          shopt -s nocasematch
          if [[ "$COMMIT_MESSAGE" == *"[buffbench]"* ]]; then
            echo "should_run_evals=true" >> $GITHUB_OUTPUT
            echo "Commit message contains [buffbench] - will run evals"
          else
            echo "should_run_evals=false" >> $GITHUB_OUTPUT
            echo "Commit message does not contain [buffbench] - skipping evals"
          fi

      - name: Set up Bun
        if: ${{ steps.check_commit.outputs.should_run_evals == 'true' }}
        uses: oven-sh/setup-bun@v2
        with:
          bun-version: '1.3.5'

      - name: Cache dependencies
        if: ${{ steps.check_commit.outputs.should_run_evals == 'true' }}
        uses: actions/cache@v4
        with:
          path: |
            node_modules
            */node_modules
            packages/*/node_modules
          key: ${{ runner.os }}-deps-${{ hashFiles('**/bun.lock*') }}
          restore-keys: |
            ${{ runner.os }}-deps-

      - name: Install dependencies
        if: ${{ steps.check_commit.outputs.should_run_evals == 'true' }}
        run: bun install --frozen-lockfile

      - name: Set environment variables
        if: ${{ steps.check_commit.outputs.should_run_evals == 'true' }}
        env:
          SECRETS_CONTEXT: ${{ toJSON(secrets) }}
        run: |
          VAR_NAMES=$(bun scripts/generate-ci-env.ts)
          echo "$SECRETS_CONTEXT" | jq -r --argjson vars "$VAR_NAMES" '
            to_entries | .[] | select(.key as $k | $vars | index($k)) | .key + "=" + .value
          ' >> $GITHUB_ENV
          echo "CODEBUFF_GITHUB_ACTIONS=true" >> $GITHUB_ENV
          echo "NEXT_PUBLIC_CB_ENVIRONMENT=test" >> $GITHUB_ENV
          echo "NEXT_PUBLIC_INFISICAL_UP=true" >> $GITHUB_ENV
          echo "CODEBUFF_GITHUB_TOKEN=${{ secrets.CODEBUFF_GITHUB_TOKEN }}" >> $GITHUB_ENV

      - name: Run evals
        if: ${{ steps.check_commit.outputs.should_run_evals == 'true' }}
        run: cd evals && bun run-eval-set --concurrency 10 --email --title "Git Eval (${{ github.sha }} ${{ github.event.head_commit.message }}})"

      - name: Workflow completed
        run: echo "Evals workflow completed successfully"