NiceEval 中的评估用例：生命周期、判定与文件

import { defineEval } from "niceeval";
import { includes, jsonMatch, toolMatch } from "niceeval/expect";

export default defineEval({
  description: "Brooklyn weather query",
  async test(t) {
    await t.send("What's the weather like in Brooklyn today?");
    t.succeeded();
    t.calledTool(
      toolMatch("get_weather", { input: jsonMatch({ city: "Brooklyn" }) }).exactly(1),
    );
    t.check(t.reply, includes("sunny"));
  },
});

npx niceeval exp local weather
npx niceeval exp local weather/brooklyn

evals/
└─ billing/
   └─ refund.eval.ts  # id: billing/refund

export default rows.map((row) =>
  defineEval({
    description: row.task,
    async test(t) {
      await t.send(row.prompt);
      t.check(t.reply, equals(row.expected));
    },
  }),
);