为你的 Agent 项目设置评估

pnpm add -D niceeval
pnpm exec niceeval init

// agents/my-agent.ts
import { defineAgent } from "niceeval/adapter";

export default defineAgent({
  name: "my-agent",
  async send(input, ctx) {
    // 示例地址：换成你自己 agent 的真实端点（HTTP、CLI、SDK 都行，只要 send 里能拿到回复文本）
    const r = await fetch("http://localhost:3000/chat", {
      method: "POST",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({ message: input.text, model: ctx.model }), // ← experiment.model 经 ctx.model 到这里
      signal: ctx.signal,
    });
    const body = await r.json();
    return {
      status: r.ok ? "completed" : "failed",
      events: [{ type: "message", role: "assistant", text: body.reply }],
    };
  },
});

// experiments/my-agent.ts
import { defineExperiment } from "niceeval";
import myAgent from "../agents/my-agent.ts";

export default defineExperiment({
  description: "my-agent 基线",
  model: "gpt-4o",
  agent: myAgent,
  runs: 1,
});

// evals/refund-policy.eval.ts
import { defineEval } from "niceeval";
import { includes } from "niceeval/expect";

export default defineEval({
  description: "退款政策问答",
  async test(t) {
    await t.send("你们的退款政策是什么?");
    t.succeeded();
    t.check(t.reply, includes("30 天"));
  },
});

pnpm exec niceeval exp my-agent   # 跑起来
pnpm exec niceeval show         # 在终端读结果；失败项会给出继续下钻的命令
pnpm exec niceeval view         # 需要交互浏览时打开本地查看器

name: evals
on: [pull_request]
jobs:
  evals:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
      - run: npm ci
      - run: npm exec niceeval exp my-agent