通过制与计分制评估

import { defineEval } from "niceeval";
import { commandSucceeded, includes } from "niceeval/expect";

export default defineEval({
  async test(t) {
    const turn = await t.send("修复测试并说明原因。");

    t.check(turn.message, includes("原因")).label("说明失败原因");

    const tests = await t.sandbox.runCommand("pnpm", ["test"]);
    await t.check(tests, commandSucceeded()).label("测试通过").orStop();
  },
});

t.check(turn.message, similarity(expected))
  .atLeast(0.8)
  .label("回答接近期望内容");

import { defineScoreEval } from "niceeval";
import { commandSucceeded, includes, similarity } from "niceeval/expect";

export default defineScoreEval({
  description: "安装并启动 DB-GPT",
  async test(t) {
    const turn = await t.send("把 DB-GPT 装起来并通过健康检查。");

    t.sandbox.fileChanged("db-gpt/.env").score(1).label("配置运行环境");

    const health = await t.sandbox.runCommand("curl", ["-s", "localhost:5670/health"]);
    t.check(health, commandSucceeded()).score(1).label("健康检查可达");
    t.check(health.stdout, includes("ok")).score(1).label("健康检查内容正确");

    t.check(turn.message, similarity("说明安装步骤和验证结果")).score(2).label("说明质量");

    t.score(1).label("代码精简");
  },
});

export default defineScoreEval({
  judge: true,
  async test(t) {
    const turn = await t.send("说明这项变更的动机和风险。");
    turn.judge.autoevals.closedQA("说明是否具体、准确？")
      .score(20)
      .atLeast(0.7)
      .label("说明质量");
  },
});

await t.check(
  await t.sandbox.runCommand("pnpm", ["test"]),
  commandSucceeded(),
).label("项目测试通过");