编写评估用例: 单轮、多轮和数据集模式

import { defineEval } from "niceeval";

export default defineEval({
  description?: string;
  tags?: string[];
  judge?: JudgeConfig;
  reporters?: Reporter[];
  timeoutMs?: number;
  environment?: string;
  metadata?: Record<string, unknown>;
  async setup(sandbox, ctx) { /* task fixture + progress/diagnostic */ },
  async test(t) { /* interactions + assertions */ },
});

// evals/coding/fix-button.eval.ts → id: coding/fix-button
export default defineEval({
  description: "修复 Button 组件",
  tags: ["coding", "frontend"],
  environment: "node-22",
  async test(t) { /* coding task */ },
});

// evals/research/gpu-literature.eval.ts → id: research/gpu-literature
export default defineEval({
  description: "在 GPU 环境检索论文",
  tags: ["research"],
  environment: "gpu",
  async test(t) { /* research task */ },
});

import { defineEval } from "niceeval";
import { includes } from "niceeval/expect";

export default defineEval({
  description: "Brooklyn weather query",
  async test(t) {
    await t.send("What's the weather like in Brooklyn today?");
    t.succeeded();
    t.calledTool("get_weather", { input: { city: "Brooklyn" }, count: 1 });
    t.check(t.reply, includes("sunny"));
  },
});

export default defineEval({
  description: "Draft an email, then send it on confirmation",
  async test(t) {
    const draft = await t.send("Draft a follow-up email.");
    draft.succeeded();
    t.check(draft.message, includes("Best"));

    await t.send("Looks good, send it.");
    t.calledTool("send_email");
  },
});

export default rows.map((row) =>
  defineEval({
    description: row.task,
    async test(t) {
      await t.send(row.prompt);
      t.check(t.reply, equals(row.expected));
    },
  }),
);

import { defineEval } from "niceeval";

export default defineEval({
  description: "Create a Button component",
  async test(t) {
    await t.sandbox.uploadDirectory("../workspaces/ts-starter");
    await t.send("Create src/components/Button.tsx with label and onClick props.").then((turn) => turn.succeeded());
    t.sandbox.fileChanged("src/components/Button.tsx");
  },
});

export default defineEval({
  async setup(sandbox, ctx) {
    ctx.progress({ message: "安装 fixture 依赖" });
    await sandbox.runCommand("npm", ["install"]);
  },

  async test(t) {
    t.progress({ message: "上传隐藏测试", current: 1, total: 2 });
    await t.sandbox.uploadDirectory("../fixtures/project");

    const preflight = await inspectFixture();
    if (preflight.usedFallback) {
      t.diagnostic({
        code: "fixture-check-degraded",
        level: "warning",
        message: "Fixture 预检使用了备用检查器",
        data: { checker: preflight.checker },
      });
    }

    await t.send("完成任务");
  },
});

// evals/pr-review/close-stale.eval.ts
import { defineEval } from "niceeval";
import type { Sandbox } from "niceeval/sandbox";

// 并发 Attempt 共享本模块：句柄按 sandbox 键控，不用普通模块变量
const fixtures = new WeakMap<Sandbox, { repoUrl: string; destroy(): Promise<void> }>();

export default defineEval({
  async setup(sandbox, ctx) {
    ctx.progress({ message: "seeding fixture repo" });
    const fixture = await createFixtureRepo("pr-review/close-stale"); // 沙箱外的临时资源
    fixtures.set(sandbox, fixture);
    await sandbox.runCommand("git", ["clone", fixture.repoUrl, "workspace"]);
  },
  async teardown(sandbox) {
    await fixtures.get(sandbox)?.destroy(); // setup 抛错也会进来：没建成就跳过
  },
  async test(t) { /* 驱动 agent 清理 stale PR，断言 */ },
});