NiceEval: an eval tool for agents

// evals/eval-tool-call.eval.ts
import { defineEval } from "niceeval";

export default defineEval({
  description: "Verify the agent calls the weather tool correctly for a live weather question and answers from its result",

  async test(t) {
    const turn = await t.send("What's the weather in Beijing today?");
    t.succeeded();

    await t.group("calls get_weather with the right city", () => {
      t.calledTool("get_weather", { input: { city: "Beijing" } });
      t.messageIncludes(/°C|sunny|cloudy|rain/);
    });

    const second = await t.send("What about Shanghai tomorrow?");
    second.messageIncludes("Shanghai");

    t.judge.autoevals
      .closedQA("Does the assistant answer from the tool's weather data instead of making up a temperature?")
      .atLeast(0.7);
  },
});

pnpm exec niceeval exp local eval-tool-call  # run only eval-tool-call under the local experiment
pnpm exec niceeval show                      # read current-project results in the terminal
pnpm exec niceeval view                      # browse the same Sample in the web viewer