为你的 Agent 项目设置评估

pnpm add -D niceeval
pnpm exec niceeval init

// agents/my-agent.ts
import { completeEvidenceCoverage, defineAgent } from "niceeval/adapter";

export default defineAgent({
  name: "my-agent",
  evidenceCoverage: completeEvidenceCoverage,
  async send(input, ctx) {
    // 示例地址：换成你自己 agent 的真实端点（HTTP、CLI、SDK 都行，只要 send 里能拿到回复文本）
    const r = await fetch("http://localhost:3000/chat", {
      method: "POST",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({ message: input.text, model: ctx.model }), // ← experiment.model 经 ctx.model 到这里
      signal: ctx.signal,
    });
    const body = await r.json();
    return {
      status: r.ok ? "completed" : "failed",
      events: [{ type: "message", role: "assistant", text: body.reply }],
    };
  },
});

// experiments/my-agent.ts
import { defineExperiment } from "niceeval";
import myAgent from "../agents/my-agent.ts";

export default defineExperiment({
  description: "my-agent 基线",
  model: "gpt-4o",
  agent: myAgent,
  attempts: 1,
});

// evals/refund-policy.eval.ts
import { defineEval } from "niceeval";
import { pattern } from "niceeval/expect";

export default defineEval({
  description: "退款政策问答",
  async test(t) {
    await t.send("你们的退款政策是什么?");
    t.succeeded();
    t.check(t.reply, pattern(/30\s*天|一个月/));  // 措辞会变，用等价正则而不是单一精确短语
  },
});

pnpm exec niceeval exp my-agent   # 跑起来
pnpm exec niceeval show --experiment my-agent  # 在终端读全部仍有效的结果
pnpm exec niceeval view --experiment my-agent  # 在网页查看同一份 Sample

name: evals
on: [pull_request]
jobs:
  evals:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
      - run: npm ci
      - run: npm exec niceeval exp my-agent