Set Up Evals for Your Agent Project

pnpm add -D niceeval
pnpm exec niceeval init

// agents/my-agent.ts
import { completeEvidenceCoverage, defineAgent } from "niceeval/adapter";

export default defineAgent({
  name: "my-agent",
  evidenceCoverage: completeEvidenceCoverage,
  async send(input, ctx) {
    // Example address: replace it with your Agent's real endpoint. HTTP, a CLI, or an SDK all work if send() can get reply text.
    const r = await fetch("http://localhost:3000/chat", {
      method: "POST",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({ message: input.text, model: ctx.model }), // ← experiment.model arrives here through ctx.model
      signal: ctx.signal,
    });
    const body = await r.json();
    return {
      status: r.ok ? "completed" : "failed",
      events: [{ type: "message", role: "assistant", text: body.reply }],
    };
  },
});

// experiments/my-agent.ts
import { defineExperiment } from "niceeval";
import myAgent from "../agents/my-agent.ts";

export default defineExperiment({
  description: "my-agent baseline",
  model: "gpt-4o",
  agent: myAgent,
  attempts: 1,
});

// evals/refund-policy.eval.ts
import { defineEval } from "niceeval";
import { pattern } from "niceeval/expect";

export default defineEval({
  description: "Refund-policy question",
  async test(t) {
    await t.send("What is your refund policy?");
    t.succeeded();
    t.check(t.reply, pattern(/30\s*days|one month/i));  // Wording varies, so use an equivalent regex rather than one exact phrase
  },
});

pnpm exec niceeval exp my-agent   # Run it
pnpm exec niceeval query discover              # Discover available read operations
pnpm exec niceeval view                        # Review the current published results in the browser

name: evals
on: [pull_request]
jobs:
  evals:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
      - run: npm ci
      - run: npm exec niceeval exp my-agent