AI-Native Eval for Agents.

1import { defineEval } from "niceeval";
2import { pattern } from "niceeval/expect";
3
4export default defineEval({
5  judge: true,
6  description: "Evaluate an agent's multimodal ability across a multi-turn conversation",
7
8  async test(t) {
    const first = await t.sendFile("evals/sample.png", "What is in this image?");
    await first.succeeded().orStop();
    first.usedNoTools();
    const second = await t.send("What color is the background?");
    t.check(second.message, pattern(/blue|white|square/i));
    const third = await t.send("What color is the shape in the middle?");
15
16    await t.group("follow-ups stay grounded in the image context", () => {
      t.check(third.message, pattern(/white/i));
18    });
19
20    t.judge.autoevals
21      .closedQA("Does the assistant keep grounding every answer in the turn-one image, across all three turns, instead of making things up?", {
22        input: "A user asks about a blue image with a white square, then asks for its background and shape colors.",
23        output: [first.message, second.message, third.message].join("\n"),
24      })
      .gate(0.7);1/0.7
26  },
27});