编写评估用例：单轮、多轮和测试集

// evals/billing/refund.eval.ts → ID 是 billing/refund
import { defineEval } from "niceeval";

export default defineEval({
  description: "能正确回答退款政策",
  tags: ["billing"],
  async test(t) {
    // 发消息，再检查结果
  },
});

import { defineEval } from "niceeval";
import { includes, jsonMatch, toolMatch } from "niceeval/expect";

export default defineEval({
  description: "问天气时调用工具",
  async test(t) {
    await t.send("布鲁克林今天天气怎么样？");

    t.succeeded();
    t.calledTool(toolMatch("get_weather", { input: jsonMatch({ city: "Brooklyn" }) }).exactly(1));
    t.check(t.reply, includes("晴"));
  },
});

t.calledTool("get_weather").label("调用了天气工具");

export default defineEval({
  description: "先起草邮件，确认后再发送",
  async test(t) {
    const draft = await t.send("帮我起草一封跟进邮件。");
    draft.succeeded();
    t.check(draft.message, includes("此致"));

    await t.send("可以，发出去吧。");
    t.calledTool("send_email");
  },
});

import { defineEval, defineJudge } from "niceeval";

const grounded = defineJudge({
  name: "grounded-answer",
  rubric: "根据 question 评价 answer 是否基于事实作答，而不是编造或含糊其辞。",
});

export default defineEval({
  async test(t) {
    const turn = await t.send("北京今天天气怎么样？");
    t.judge({ question: turn.input, answer: turn.message }, grounded).gate(0.7).label("回答有依据");
  },
});

// evals/sql.eval.ts
import { defineEval } from "niceeval";
import { equals } from "niceeval/expect";

const rows = [
  { task: "统计订单数", prompt: "一共有多少订单？", expected: "42" },
  { task: "最贵商品", prompt: "最贵的商品是什么？", expected: "MacBook Pro" },
];

export default rows.map((row) =>
  defineEval({
    description: row.task,
    async test(t) {
      await t.send(row.prompt);
      t.check(t.reply, equals(row.expected));
    },
  }),
);

export default defineEval({
  description: "创建 Button 组件",
  async test(t) {
    await t.sandbox.uploadDirectory("../workspaces/ts-starter");
    const turn = await t.send("创建 src/components/Button.tsx，接受 label 和 onClick 两个 props。");
    turn.succeeded();
    t.sandbox.fileChanged("src/components/Button.tsx");
  },
});

export default defineEval({
  async test(t) {
    t.progress({ message: "上传项目", current: 1, total: 2 });
    await t.sandbox.uploadDirectory("../fixtures/project");

    const preflight = await inspectFixture();
    if (preflight.usedFallback) {
      t.diagnostic({
        code: "fixture-check-degraded",
        level: "warning",
        message: "Fixture 预检使用了备用检查器",
        data: { checker: preflight.checker },
      });
    }

    await t.send("完成任务");
  },
});

// evals/pr-review/close-outdated.eval.ts
import { defineEval } from "niceeval";
import type { Sandbox } from "niceeval/sandbox";

// 同一条用例的多个 Attempt 会并发执行、共享这个模块，所以按 sandbox 存取，不用普通模块变量
const fixtures = new WeakMap<Sandbox, { repoUrl: string; destroy(): Promise<void> }>();

export default defineEval({
  async setup(sandbox, ctx) {
    ctx.progress({ message: "创建临时仓库" });
    const fixture = await createFixtureRepo("pr-review/close-outdated");
    fixtures.set(sandbox, fixture);
    await sandbox.runCommand("git", ["clone", fixture.repoUrl, "workspace"]);
  },
  async teardown(sandbox) {
    await fixtures.get(sandbox)?.destroy(); // setup 中途失败时可能没建成
  },
  async test(t) { /* 让 Agent 清理过期 PR，再断言 */ },
});