Evals built for agents.

1import { defineEval, defineJudge } from "niceeval";
2import { pattern } from "niceeval/expect";
3
4const imageContext = defineJudge({ name: "image-context", rubric: "Does the last reply stay grounded in the earlier image?" });
5
6export default defineEval({
7  description: "Evaluate an agent's multimodal ability across a multi-turn conversation",
8
9  async test(t) {
    const first = await t.sendFile("evals/sample.png", "What is in this image?");
    await first.succeeded().orStop();
    first.usedNoTools();
    const second = await t.send("What color is the background?");
    t.check(second.message, pattern(/blue|white|square/i));
    const third = await t.send("What color is the shape in the middle?");
16
17    await t.group("follow-ups stay grounded in the image context", () => {
      t.check(third.message, pattern(/white/i));
19    });
20
    t.judge({ earlierImageFacts: ["blue background", "white square"], question: third.input, answer: third.message }, imageContext).gate(0.7);≥ 0.7
22  },
23});