Write Send

// agents/chat-app.ts
import { completeEvidenceCoverage, defineAgent } from "niceeval/adapter";

const BASE_URL = "http://localhost:8080";   // To switch addresses by Experiment, make this a factory argument; see the connection guide.

export default defineAgent({
  name: "chat-app",
  evidenceCoverage: completeEvidenceCoverage,
  async send(input, ctx) {
    ctx.progress({ message: "Waiting for the Chat API" });
    const res = await fetch(`${BASE_URL}/v1/chat/completions`, {  // ← The interface the frontend already uses
      method: "POST",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({
        messages: [{ role: "user", content: input.text }],
        model: ctx.model,                                         // ← experiment.model arrives here through ctx.model. Delete this line if the application interface cannot select a model.
      }),
      signal: ctx.signal,                                         // ← Runner timeout and cancellation
    }).then((r) => r.json());

    return {
      status: "completed",
      events: [{ type: "message", role: "assistant", text: res.choices[0].message.content }],
    };
  },
});

if (!res.requestId) {
  ctx.diagnostic({
    code: "missing-request-id",
    level: "warning",
    message: "The response has no request id and cannot be correlated with server logs",
  });
}

// agents/chat-app.ts
import { completeEvidenceCoverage, createSessionSlot, defineAgent } from "niceeval/adapter";

const BASE_URL = "http://localhost:8080";

interface Msg { role: "user" | "assistant"; content: string | null }
const historySlot = createSessionSlot<Msg[]>("chat-app/history");

export default defineAgent({
  name: "chat-app",
  evidenceCoverage: completeEvidenceCoverage,
  async send(input, ctx) {
    const history = ctx.session.get(historySlot) ?? []; // ← This slot has not been written on a new session line.
    const messages = [...history, { role: "user" as const, content: input.text }];

    const res = await fetch(`${BASE_URL}/v1/chat/completions`, {
      // … method, headers, and signal are the same as step one; omitted
      body: JSON.stringify({ messages, model: ctx.model }),        // ← The change: send complete history. Forward model as in step one.
    }).then((r) => r.json());

    const reply = res.choices[0].message;
    ctx.session.set(historySlot, [...messages, reply]); // ← Write it back. The next Turn on this line gets it automatically.

    return {
      status: "completed",
      events: [{ type: "message", role: "assistant", text: reply.content }],
    };
  },
});

// Inside send:
//   Send the prior Turn's id in the request body … previous_response_id: ctx.session.id   ← It is naturally undefined on a new session line.
//   Capture a returned id … ctx.session.capture(res.id)                                   ← The next Turn continues from it.

    // … transport and session are the same as step two; omitted
    return {
      status: "completed",
      events: [{ type: "message", role: "assistant", text: reply.content }],
      usage: {                                        // ← The change: report the usage returned by the interface truthfully.
        inputTokens: res.usage.prompt_tokens,
        outputTokens: res.usage.completion_tokens,
      },
    };

import type { CommandProjection } from "niceeval/adapter";

type StreamEvent =
  | { type: "message"; role: "assistant" | "user"; text: string }
  | { type: "operation.started"; operationId: string; operation:
      | { kind: "tool"; name: string; input: JsonValue;
          tool?: ToolName; command?: CommandProjection }
      | { kind: "subagent"; name: string; remoteUrl?: string } }
  | { type: "operation.finished"; operationId: string; kind: "tool";
      output?: JsonValue; status: "completed" | "failed" | "rejected" }
  | { type: "operation.finished"; operationId: string; kind: "subagent";
      output?: JsonValue; status: "completed" | "failed" }
  | { type: "skill.loaded"; skill: string; operationId?: string }
  | { type: "input.requested"; request: InputRequest }
  | { type: "thinking"; text: string }
  | { type: "compaction"; reason?: string }
  | { type: "error"; message: string };

    // … transport and session are the same as step two; usage is the same as step three; omitted
    const reply = res.choices[0].message;

    const events: StreamEvent[] = [];
    for (const call of reply.tool_calls ?? []) {
      events.push({
        type: "operation.started",
        operationId: call.id,
        operation: {
          kind: "tool",
          name: call.function.name,
          input: JSON.parse(call.function.arguments),
        },
      });
    }
    if (reply.content) events.push({ type: "message", role: "assistant", text: reply.content });

    return { events, status: "completed", usage };

    // … transport and session are the same as step two; omitted
    return turnFromChatCompletion(res);            // ← The mapping above plus step three's usage, in the official version.

// agents/my-app.ts
import { completeEvidenceCoverage, createSessionSlot, defineAgent, sseJsonFrames, createPiAgentEventStream, driveFrameStream } from "niceeval/adapter";
import type { AgentContext, PiAgentStream, SseFrameCursor } from "niceeval/adapter";
import type { Turn, TurnInput } from "niceeval";

const BASE_URL = "http://localhost:5299";               // The application's own port; evals do not manage its process.

interface Pending { cursor: SseFrameCursor<Frame>; stream: PiAgentStream; callId: string }
const pendingSlot = createSessionSlot<Pending>("my-app/pending");

function readStream(cursor: SseFrameCursor<Frame>, ctx: AgentContext, stream: PiAgentStream): Promise<Turn> {
  return driveFrameStream(cursor, stream, ctx, (frame) => {
    if (frame.type === "session") {
      ctx.session.capture(frame.sessionId);             // ← Step two: write the returned id back so the next Turn continues.
      return;
    }
    if (frame.type === "approval_request") {            // ← This step: the application pauses for approval, so retain the scene.
      ctx.session.set(pendingSlot, { cursor, stream, callId: frame.toolCallId });
      return { pause: { id: frame.toolCallId, action: frame.toolName,
                        options: [{ id: "approve" }, { id: "deny" }] } };
      // On pause, driveFrameStream adds input.requested, sets status to "waiting", and stops reading the stream.
    }
    if (frame.type === "server_error") return { fail: frame.message };
  });
}

export default defineAgent({
  name: "my-app",
  evidenceCoverage: completeEvidenceCoverage,
  async send(input: TurnInput, ctx: AgentContext): Promise<Turn> {
    const held = ctx.session.take(pendingSlot);
    if (held) {                                         // ← t.respond("approve"): an ordinary send; deliver the decision first.
      const approved = input.responses?.[0]?.optionId === "approve";  // Match by requestId when several requests are pending.
      if (!approved) held.stream.markRejected(held.callId);
      await fetch(`${BASE_URL}/api/chat/approve`, { method: "POST", signal: ctx.signal,
        body: JSON.stringify({ toolUseId: held.callId, approved }) });
      return readStream(held.cursor, ctx, held.stream); // Then keep reading the same stream; do not send a new request.
    }

    const res = await fetch(`${BASE_URL}/api/chat`, {   // ← Transport: the only part you truly hand-write.
      method: "POST",
      body: JSON.stringify({
        message: input.text,                            // ← Step one: send the message.
        model: ctx.model,                               // ← Step one: forward experiment.model.
        sessionId: ctx.session.id,                      // ← Step two: naturally absent on the first send and present afterward.
      }),
      signal: ctx.signal,
    });
    return readStream(sseJsonFrames<Frame>(res.body!), ctx, createPiAgentEventStream());  // ← Step four: official converter with no mapping.
  },
});

// niceeval.config.ts
import { defineConfig } from "niceeval";

export default defineConfig({
  telemetry: { port: 4318 },   // The receiver always listens on http://localhost:4318/v1/traces.
});

OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318/v1/traces node server.js

    const res = await fetch(`${BASE_URL}/v1/chat/completions`, {
      // … the rest is the same as step four; omitted
      headers: { "content-type": "application/json", ...ctx.telemetry?.headers },  // ← The only added line.
    }).then((r) => r.json());

      // … the rest of send is the same as before; omitted
      body: JSON.stringify({
        messages,
        model: ctx.model,        // ← experiment.model has been forwarded since step one; it appears here with flags.
        flags: ctx.flags,      // ← experiment.flags passes through unchanged. It is {} when not configured.
      }),

// experiments/concise.ts
import { defineExperiment } from "niceeval";
import chatApp from "../agents/chat-app.ts";

export default defineExperiment({
  agent: chatApp,
  model: "gpt-5.4",                      // ← Reaches send through ctx.model.
  flags: { promptVariant: "concise" },  // ← Reaches send through ctx.flags every Turn.
});