Set up evals for your agent project

pnpm add -D niceeval
pnpm exec niceeval init

// agents/my-agent.ts
import { completeEvidenceCoverage, defineAgent } from "niceeval/adapter";

export default defineAgent({
  name: "my-agent",
  evidenceCoverage: completeEvidenceCoverage,
  async send(input, ctx) {
    // Example endpoint: replace with your agent's real endpoint (HTTP, CLI, or SDK all work, as long as send() can get the reply text)
    const r = await fetch("http://localhost:3000/chat", {
      method: "POST",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({ message: input.text, model: ctx.model }), // ← experiment.model flows through ctx.model here
      signal: ctx.signal,
    });
    const body = await r.json();
    return {
      status: r.ok ? "completed" : "failed",
      events: [{ type: "message", role: "assistant", text: body.reply }],
    };
  },
});

// experiments/my-agent.ts
import { defineExperiment } from "niceeval";
import myAgent from "../agents/my-agent.ts";

export default defineExperiment({
  description: "my-agent baseline",
  model: "gpt-4o",
  agent: myAgent,
  attempts: 1,
});

// evals/refund-policy.eval.ts
import { defineEval } from "niceeval";
import { includes } from "niceeval/expect";

export default defineEval({
  description: "Refund policy question",
  async test(t) {
    const turn = await t.send("What is your refund policy?");
    await turn.succeeded().stopOnFailure();
    t.check(t.reply, includes("30 days"));
  },
});

pnpm exec niceeval exp my-agent   # run it
pnpm exec niceeval show --experiment my-agent  # read all still-valid results in the terminal
pnpm exec niceeval view --experiment my-agent  # browse the same Sample locally

name: evals
on: [pull_request]
jobs:
  evals:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
      - run: npm ci
      - run: npm exec niceeval exp my-agent