在 GitHub Actions 和 CI 中运行 NiceEval

NICEEVAL_LANG=en npx niceeval exp ci \
  --output ci \
  --strict \
  --json .niceeval/ci-summary.json \
  --junit .niceeval/junit.xml

name: evals
on:
  pull_request:
  workflow_dispatch:

jobs:
  evals:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 22
          cache: npm
      - run: npm ci
      - run: >-
          npx niceeval exp ci --output ci --strict
          --json .niceeval/ci-summary.json
          --junit .niceeval/junit.xml
        env:
          NICEEVAL_LANG: en
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

niceeval: start total=24 configs=3 concurrency=10 reused=18
niceeval: errored locator=@12h8m4k1 eval=fixtures/button experiment=ci/codex phase=sandbox.create reason="E2B sandbox allocation failed after 5 attempts"
niceeval: result=failed passed=23 failed=0 errored=1 reused=18 duration=128s
niceeval: json=.niceeval/ci-summary.json
niceeval: junit=.niceeval/junit.xml

npx niceeval list

npx niceeval exp ci --output ci --max-concurrency 2