Sandbox Agent：评估 Claude Code、Codex 和 bub

// experiments/local.ts
import { defineExperiment } from "niceeval";
import { claudeCodeAgent } from "niceeval/adapter";
import { dockerSandbox } from "niceeval/sandbox";

export default defineExperiment({
  agent: claudeCodeAgent(),
  model: "claude-sonnet-4-6",
  sandbox: dockerSandbox({ source: { type: "image", image: "node:24-slim" } }),
});

export ANTHROPIC_API_KEY=sk-ant-...
npx niceeval exp local fixtures/button

npx niceeval exp local fixtures/button --attempts 10

import { codexAgent } from "niceeval/adapter";
import { e2bSandbox } from "niceeval/sandbox";
import { NICEEVAL_CODEX_E2B_TEMPLATE } from "niceeval/sandbox/e2b-template";

export default defineExperiment({
  agent: codexAgent(),
  model: "gpt-5.4",
  sandbox: e2bSandbox({ template: NICEEVAL_CODEX_E2B_TEMPLATE }),
});

import { defineExperiment } from "niceeval";
import { claudeCodeAgent } from "niceeval/adapter";
import { dockerSandbox } from "niceeval/sandbox";

export default defineExperiment({
  agent: claudeCodeAgent({
    apiKey: process.env.ANTHROPIC_API_KEY,
    baseUrl: process.env.ANTHROPIC_BASE_URL,
    maxTurns: 8,
    mcpServers: [
      {
        name: "browser",
        command: "npx",
        args: ["-y", "@anthropic/mcp-browser"],
      },
    ],
    skills: [{ kind: "repo", source: "Effect-TS/skills", ref: "8f3c1a2" }],
  }),
  model: "claude-sonnet-4-6",
  sandbox: dockerSandbox({ source: { type: "image", image: "node:24-slim" } }),
});

创建 Sandbox
  → 运行 Sandbox 的 setup 生命周期
  → 运行评估的 setup 生命周期，准备 Fixture
  → 检查 Agent CLI 版本，需要时安装并复检
  → 运行 Adapter 的 setup，写入鉴权和运行配置
  → 运行 test(t)，写入起始文件
  → Adapter 调用 Agent，记录期间的文件变化
  → 运行验证命令，完成评分和判定
  → 运行 Adapter 和 Sandbox 的 teardown 生命周期
  → 停止 Sandbox

export default defineEval({
  diff: {
    ignore: ["fixtures/**"],               // agent 改这里不算它的改动
    include: ["node_modules/some-pkg/**"],  // 被 patch 的依赖要算
  },
  async test(t) { /* ... */ },
});

import {
  completeEvidenceCoverage,
  createNpmCliInstaller,
  defineSandboxAgent,
  resolveAgentBin,
} from "niceeval/adapter";

const { ensure, installer } = createNpmCliInstaller({
  identity: { agent: "my-agent", version: "1.0.0", revision: "1" },
  packageName: "my-agent",
  bin: "my-agent",
});

export default defineSandboxAgent({
  name: "my-agent",
  evidenceCoverage: completeEvidenceCoverage,
  ensure,
  installers: [installer],
  async send(input, ctx) {
    ctx.progress({ message: "运行 my-agent CLI" });
    const bin = await resolveAgentBin(ctx.sandbox, "my-agent");
    await ctx.sandbox.runCommand(bin, ["run", input.text, "--json-out", "agent-events.json"]);
    const transcript = await ctx.sandbox.readText("agent-events.json");
    if (transcript.trim() === "") {
      ctx.diagnostic({
        code: "empty-transcript",
        level: "warning",
        message: "my-agent 没有写出 transcript，工具断言可能缺少证据",
      });
    }
    return {
      status: "completed",
      events: parseTranscript(transcript),
    };
  },
});

✗ @5TB8167MXJ30SYZCNAVRHPQ4D2 fixtures/button [local] errored · agent setup
    agent-install-failed: npm install my-agent exited with code 1
    Inspect: niceeval show --run <runId> --page attempt-5TB8167MXJ30SYZCNAVRHPQ4D2

import { defineExperiment } from "niceeval";
import { dockerSandbox } from "niceeval/sandbox";
import myAgent from "./agents/my-agent";

export default defineExperiment({
  agent: myAgent,
  model: "claude-sonnet-4-6",
  sandbox: dockerSandbox({ source: { type: "image", image: "node:24-slim" } }),
  attempts: 3,
});