NiceEval 里的 Judge：评开放式输出

import { defineJudge } from "niceeval";

const explainsRisk = defineJudge({
  name: "explains-risk",
  rubric: "根据 change 评价 explanation 是否说明兼容性、回滚方式和数据风险。",
  anchors: [
    { measurement: 0, description: "没有说明具体风险或回滚方式" },
    { measurement: 0.5, description: "说明部分风险，但遗漏一项关键内容" },
    { measurement: 1, description: "完整说明兼容性、回滚方式和数据风险" },
  ],
});

export default defineEval({
  judge: "judge-model",
  async test(t) {
    const change = "修改持久化字段并提供回滚方案";
    const turn = await t.send(`解释这项变更：${change}`);

    t.judge({ change, explanation: turn.message }, explainsRisk)
      .gate(0.8)
      .label("风险说明质量");
  },
});

import { eventMatch } from "niceeval/expect";
const answer = await t.send("列出风险与回滚步骤。");
answer.closeQA("这轮是否说明风险和可执行的回滚步骤？")
  .gate(0.8).label("回滚说明");

// 特殊检查可以显式选择全部助手发言。
answer.closeQA(
  eventMatch("message", { role: "assistant" }),
  "全部发言是否清楚说明风险？",
).gate(0.8);

import { defineConfig } from "niceeval";
import { OpenAIProvider } from "niceeval/judge";

export default defineConfig({
  judgeRuntime: OpenAIProvider({
    model: "judge-model",
    baseUrl: "https://gateway.example.com/v1",
    apiKeyEnv: "JUDGE_GATEWAY_KEY",
    timeoutMs: 180_000,
    maxOutputTokens: 512,
  }),
});

export default defineEval({
  judge: "another-model",
  async test(t) {
    const answer = await t.send("法国首都是哪里？");
    t.factuality({ input: "法国首都是哪里？", output: answer.message, expected: "巴黎" }).gate(1);
  },
});