examples/zh/ai-sdk/
  ai-sdk-agent/            # 被测 web agent(POST /api/turn,内含 3 个工具 + 图片理解)
  agents/web-agent.ts      # niceeval adapter:工厂 webAgent({ baseUrl })
  evals/                   # 会话型 eval
    weather-tool.eval.ts        # 问天气 → 调 get_weather
    image-understanding.eval.ts # 图片理解
  experiments/
    compare-models/        # 实验组:每文件钉一个 model
      gpt-4o-mini.ts
      gpt-4o.ts
  niceeval.config.ts       # 注册 adapter、judge、并发

// agents/web-agent.ts
import { defineAgent } from "niceeval/adapter";
import type { Agent } from "niceeval/adapter";
import type { StreamEvent } from "niceeval";
import type { AgentEvent, AgentResponse } from "../ai-sdk-agent/src/protocol.ts";

export function webAgent(opts: { baseUrl: string }): Agent {
  const baseUrl = opts.baseUrl.replace(/\/$/, "");
  return defineAgent({
    name: "web-agent",
    capabilities: { conversation: true, toolObservability: true, tracing: true },
    async send(input, ctx) {
      const response = await fetch(`${baseUrl}/api/turn`, {
        method: "POST",
        headers: { "content-type": "application/json" },
        body: JSON.stringify({
          sessionId: ctx.session.id,
          message: input.text,
          model: ctx.model,
          otelEndpoint: ctx.telemetry?.endpoint, // 双可观测:让 app 把本轮 span 也发回 niceeval
        }),
        signal: ctx.signal,
      });
      // 同一 workspace 的共享契约,直接按 AgentResponse 读,不必当 unknown 再校验一遍。
      const body = (await response.json()) as AgentResponse;
      ctx.session.id = body.sessionId;
      return {
        events: body.events.map(toStreamEvent),
        data: body.data,
        status: "completed" as const,
      };
    },
  });
}

function toStreamEvent(event: AgentEvent): StreamEvent {
  if (event.type === "action.called") return { ...event, tool: "unknown" };
  return event;
}

// evals/weather-tool.eval.ts
import { defineEval } from "niceeval";

export default defineEval({
  description: "AI 助手:问天气调用 get_weather",
  async test(t) {
    const turn = await t.send("北京今天天气怎么样？");
    turn.expectOk();
    await t.group("调用 get_weather 且城市正确", () => {
      t.calledTool("get_weather", { input: { city: "北京" } });
      t.messageIncludes(/°C|气温|天气/);
    });
    t.judge.autoevals.closedQA("助手是否基于工具返回的天气数据作答，而不是凭空编造？").atLeast(0.7);
  },
});

// evals/image-understanding.eval.ts
import { SAMPLE_IMAGE_URL } from "../ai-sdk-agent/src/assistant.ts";

const turn = await t.send(`这张图片里是什么？${SAMPLE_IMAGE_URL}`);
t.messageIncludes(/猫|cat/i); // 示例图是一只猫

// experiments/compare-models/gpt-4o.ts
import { defineExperiment } from "niceeval";
import { webAgent } from "../../agents/web-agent.ts";

export default defineExperiment({
  description: "AI 助手:gpt-4o",
  agent: webAgent({ baseUrl: "http://127.0.0.1:5188" }),
  model: "gpt-4o", // 单个字符串;另一个模型复制一份文件改这一行
  runs: 3,
});

cd examples/zh/ai-sdk && pnpm install && pnpm dev

cd examples/zh/ai-sdk
pnpm exec niceeval exp compare-models   # 跑模型对比实验组
pnpm exec niceeval exp compare-models weather-tool   # 在实验组里只跑某个 eval