nightly-evals.yml

name: Nightly Evals

on:
  schedule:
    # Run every night at 1:00 AM PT (9:00 AM UTC)
    - cron: '0 9 * * *'
  workflow_dispatch: # Allow manual triggering

jobs:
  run-nightly-evals:
    runs-on: ubuntu-latest
    timeout-minutes: 360 # 6 hours is the max for any hosted github action
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Bun
        uses: oven-sh/setup-bun@v2
        with:
          bun-version: '1.3.5'

      - name: Cache dependencies
        uses: actions/cache@v4
        with:
          path: |
            node_modules
            */node_modules
            packages/*/node_modules
          key: ${{ runner.os }}-deps-${{ hashFiles('**/bun.lock*') }}
          restore-keys: |
            ${{ runner.os }}-deps-

      - name: Install dependencies
        run: bun install --frozen-lockfile

      - name: Set environment variables
        env:
          SECRETS_CONTEXT: ${{ toJSON(secrets) }}
        run: |
          VAR_NAMES=$(bun scripts/generate-ci-env.ts)
          echo "$SECRETS_CONTEXT" | jq -r --argjson vars "$VAR_NAMES" '
            to_entries | .[] | select(.key as $k | $vars | index($k)) | .key + "=" + .value
          ' >> $GITHUB_ENV
          echo "CODEBUFF_GITHUB_ACTIONS=true" >> $GITHUB_ENV
          echo "NEXT_PUBLIC_CB_ENVIRONMENT=test" >> $GITHUB_ENV
          echo "NEXT_PUBLIC_INFISICAL_UP=true" >> $GITHUB_ENV
          echo "CODEBUFF_GITHUB_TOKEN=${{ secrets.CODEBUFF_GITHUB_TOKEN }}" >> $GITHUB_ENV

      - name: Run nightly evals
        run: cd evals && bun run-buffbench-nightly

      - name: Workflow completed
        run: echo "Nightly evals workflow completed successfully"