# Copy to .github/workflows/judge-gate.yml in your repo and adjust the paths.
# Before the first run: `judgekeeper freeze evals/anchors.jsonl`, judge and validate once,
# then `judgekeeper baseline set reports/report.json` and commit .judgekeeper/baseline.json.
name: judge gate

on:
  # Weekly: the provider can change what a model id serves without any change in your repo.
  # A scheduled run catches judge drift (a new snapshot, a silent update) between PRs.
  schedule:
    - cron: "17 6 * * 1"  # Mondays 06:17 UTC
  # Pull requests that change the judge itself: its prompt, the anchor set it is measured
  # against, the committed baseline or the gate thresholds. Other PRs cannot move the judge,
  # so they do not pay for judge calls.
  pull_request:
    paths:
      - "prompts/judge.md"
      - "evals/anchors.jsonl"
      - "evals/anchors.manifest.json"
      - ".judgekeeper/baseline.json"
      - "judgekeeper.toml"

permissions:
  contents: read

jobs:
  gate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: judgekeeper/judgekeeper@main  # pin to a release tag or commit sha
        env:
          # Keys come from repository secrets, never from action inputs.
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        with:
          anchors: evals/anchors.jsonl
          runner: anthropic
          model: claude-haiku-4-5-20251001
          prompt: prompts/judge.md
          runs: 3
          flaky-as: pass  # FLAKY is reported in the summary but does not fail the build

  # Weekly only: the gate job above has just re-judged the frozen anchor set. Compare those
  # verdicts item by item with the baseline's to tell judge drift from a change in your system,
  # even when the declared fingerprint is identical (a silent provider-side update).
  # The baseline must carry per-item verdicts (`items` in report.json, schema_version 2): if it
  # predates them, re-run `validate` on the baseline runs and `baseline set` again.
  attribute:
    needs: gate
    if: always() && github.event_name == 'schedule'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      # pin @main to a release tag or commit sha, as for the action above
      - run: pip install "judgekeeper @ git+https://github.com/judgekeeper/judgekeeper@main"
      - uses: actions/download-artifact@v4
        with:
          name: judgekeeper-gate  # the gate action's artifact-name
          path: judgekeeper-out
      - name: Attribute
        run: |
          set +e
          judgekeeper attribute judgekeeper-out/report.json --baseline .judgekeeper/baseline.json
          code=$?
          cat judgekeeper-out/attribution.md >> "$GITHUB_STEP_SUMMARY"
          exit $code  # 0 STABLE, 6 JUDGE_DRIFT, 7 SYSTEM_CHANGE, 2 usage, 3 anchors changed