defineEval：声明、配置并运行 NiceEval 评估用例

import { defineEval } from "niceeval";

export default defineEval({
  description: "Brooklyn weather query",
  async test(t) {
    const turn = await t.send("What's the weather like in Brooklyn today?");
    turn.succeeded().label("Turn 完成");
  },
});

description?: string;

tags?: string[];

sandbox?: SandboxLayer;

plugins?: readonly PluginInstance<"eval">[];

judge?: JudgeDeclaration;

reporters?: Reporter[];

timeoutMs?: number;

metadata?: globalThis.Record<string, JsonValue>;

diff?: { include?: string[]; ignore?: string[] };

test(t: TestContext): EvalTestReturn;

readonly evaluationKind: Kind;

send(input: string | { readonly text: string; readonly files?: readonly InputFile[] }): Promise<TurnHandle<Kind>>;

sendFile(path: string, text?: string): Promise<TurnHandle<Kind>>;

requireInputRequest(filter?: InputRequestFilter): InputRequest;

respond(...responses: readonly (string | InputAnswer)[]): Promise<TurnHandle<Kind>>;

respondAll(optionId: string): Promise<TurnHandle<Kind>>;

readonly reply: string;

readonly sessionId: string | undefined;

readonly events: readonly StreamEvent[];

newSession(): SessionHandle<Kind>;

readonly signal: AbortSignal;

readonly model?: string;

readonly reasoningEffort?: string;

readonly flags: Readonly<globalThis.Record<string, JsonValue>>;

progress(update: import("../types.ts").ProgressUpdate): void;

diagnostic(input: import("../types.ts").DiagnosticInput): void;

log(message: string): void;

skip(reason: string): never;

group<Value>(
  title: string,
  body: () => Value | PromiseLike<Value>,
): Promise<Awaited<Value>>;

check: AssertionsRuntime<Kind>["t"]["check"];

readonly sandbox: Sandbox<Kind>;

readonly o11y: import("../o11y/types.ts").O11ySummary;

readonly usage: Usage;

succeeded(): BooleanAssertionHandle<Kind, void>;

usedNoTools(): BooleanAssertionHandle<Kind, void>;

maxToolCalls(max: number): BooleanAssertionHandle<Kind, void>;

noFailedActions(): BooleanAssertionHandle<Kind, void>;

event(match: EventMatch, options?: EventOptions): BooleanAssertionHandle<Kind, void>;

notEvent(match: EventMatch): BooleanAssertionHandle<Kind, void>;

maxTokens(max: number): BooleanAssertionHandle<Kind, void>;

maxCost(usd: number): BooleanAssertionHandle<Kind, void>;

readonly judge: RootJudge<Kind>;

const turn = await t.send("总结需求。");
turn.judge.autoevals.factuality(expected).atLeast(0.8).label("事实一致");
turn.judge.autoevals.closedQA(question).atLeast(0.7).label("回答质量");
turn.judge.autoevals.summarizes(sourceText).score(20).label("摘要质量");

readonly events: readonly StreamEvent[];

readonly toolCalls: readonly import("../o11y/types.ts").ToolCall[];

readonly status: "completed" | "failed" | "waiting";

readonly message: string;

readonly data?: JsonValue;

readonly usage?: Usage;

succeeded(): BooleanAssertionHandle<Kind, void>;

toolOrder(matches: readonly [ToolMatch, ToolMatch, ...ToolMatch[]]): BooleanAssertionHandle<Kind, void>;

usedNoTools(): BooleanAssertionHandle<Kind, void>;

maxToolCalls(max: number): BooleanAssertionHandle<Kind, void>;

noFailedActions(): BooleanAssertionHandle<Kind, void>;

event(match: EventMatch, options?: EventOptions): BooleanAssertionHandle<Kind, void>;

notEvent(match: EventMatch): BooleanAssertionHandle<Kind, void>;

eventOrder(matches: readonly [EventMatch, EventMatch, ...EventMatch[]]): BooleanAssertionHandle<Kind, void>;

maxTokens(max: number): BooleanAssertionHandle<Kind, void>;

maxCost(usd: number): BooleanAssertionHandle<Kind, void>;

readonly judge: TurnJudge<Kind>;

export default rows.map((row) =>
  defineEval({
    description: row.task,
    async test(t) {
      await t.send(row.prompt);
    },
  }),
);