NiceEval: an eval tool for Agents

// evals/eval-tool-call.eval.ts
import { defineEval, defineJudge } from "niceeval";
import { includes } from "niceeval/expect";

const groundedWeatherAnswer = defineJudge({
  name: "grounded-weather-answer",
  rubric: "Assess whether the answer gives specific weather information based on the question instead of refusing or answering vaguely.",
});

export default defineEval({
  judge: groundedWeatherAnswer,
  description: "Verify that the agent calls a tool for a live weather question and answers from the result",

  async test(t) {
    const turn = await t.send("What's the weather like in Beijing today?");
    turn.succeeded().label("Turn completed");

    await t.group("Calls get_weather and answers about the weather", () => {
      t.calledTool("get_weather").label("Calls the weather tool");
      t.check(turn.message, includes("weather")).label("Describes the weather");
    });

    const second = await t.send("What's the weather like in Shanghai tomorrow?");
    t.check(second.message, includes("Shanghai")).label("Second answer mentions Shanghai");

    t.judge({ question: turn.input, answer: turn.message }, groundedWeatherAnswer).gate(0.7)
      .label("Answer is grounded");
  },
});

pnpm exec niceeval exp local eval-tool-call  # Run only eval-tool-call with the local Experiment
pnpm exec niceeval query discover            # Discover operations for AI, scripts, and CI
pnpm exec niceeval view                      # Review results in a browser