快速开始：跑通第一条评估用例

pnpm add -D niceeval
pnpm exec niceeval init

// agents/my-agent.ts
import { completeEvidenceCoverage, defineAgent } from "niceeval/adapter";

export default defineAgent({
  name: "my-agent",
  evidenceCoverage: completeEvidenceCoverage,
  async send(input, ctx) {
    // 换成你自己应用的地址
    const r = await fetch("http://localhost:3000/chat", {
      method: "POST",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({ message: input.text, model: ctx.model }), // ctx.model 来自 Experiment
      signal: ctx.signal,
    });
    const body = await r.json();
    return {
      status: r.ok ? "completed" : "failed",
      events: [{ type: "message", role: "assistant", text: body.reply }],
    };
  },
});

// experiments/my-agent.ts
import { defineExperiment } from "niceeval";
import myAgent from "../agents/my-agent.ts";

export default defineExperiment({
  description: "my-agent 基线",
  agent: myAgent,
  model: "gpt-4o",
  attempts: 1,
});

// evals/refund-policy.eval.ts
import { defineEval } from "niceeval";
import { pattern } from "niceeval/expect";

export default defineEval({
  description: "能正确回答退款政策",
  async test(t) {
    await t.send("你们的退款政策是什么？");
    t.succeeded();
    t.check(t.reply, pattern(/30\s*天|一个月/));
  },
});

pnpm exec niceeval exp my-agent

╭─ PASSED ─────────────────────────────────────────────────────── 12s ─╮
│ 1 passed · 0 failed · 0 errored  (0 reused)                          │
╰──────────────────────────────────────────────────────────────────────╯

╭─ RESULTS ───────────────────────────────────── 1 run configuration ─╮
│ my-agent                                                             │
│   refund-policy  1/1 passed                                          │
╰──────────────────────────────────────────────────────────────────────╯

╭─ NEXT ───────────────────────────────────────────────────────────────╮
│ my-agent                                                             │
│   show: niceeval show --run 8f3d6f62-1d34-4cf3-99c7-84ba3c483706     │
│   view: niceeval view --run 8f3d6f62-1d34-4cf3-99c7-84ba3c483706     │
╰──────────────────────────────────────────────────────────────────────╯