Sandbox Agents: Evaluate Claude Code, Codex, and bub

// experiments/local.ts
import { defineExperiment } from "niceeval";
import { claudeCodeAgent } from "niceeval/adapter";
import { dockerSandbox } from "niceeval/sandbox";

export default defineExperiment({
  agent: claudeCodeAgent(),
  model: "claude-sonnet-4-6",
  sandbox: dockerSandbox({ source: { type: "image", image: "node:24-slim" } }),
});

export ANTHROPIC_API_KEY=sk-ant-...
npx niceeval exp local fixtures/button

npx niceeval exp local fixtures/button --attempts 10

import { codexAgent } from "niceeval/adapter";
import { e2bSandbox } from "niceeval/sandbox";
import { NICEEVAL_CODEX_E2B_TEMPLATE } from "niceeval/sandbox/e2b-template";

export default defineExperiment({
  agent: codexAgent(),
  model: "gpt-5.4",
  sandbox: e2bSandbox({ template: NICEEVAL_CODEX_E2B_TEMPLATE }),
});

import { defineExperiment } from "niceeval";
import { claudeCodeAgent } from "niceeval/adapter";
import { dockerSandbox } from "niceeval/sandbox";

export default defineExperiment({
  agent: claudeCodeAgent({
    apiKey: process.env.ANTHROPIC_API_KEY,
    baseUrl: process.env.ANTHROPIC_BASE_URL,
    maxTurns: 8,
    mcpServers: [
      {
        name: "browser",
        command: "npx",
        args: ["-y", "@anthropic/mcp-browser"],
      },
    ],
    skills: [{ kind: "repo", source: "Effect-TS/skills", ref: "8f3c1a2" }],
  }),
  model: "claude-sonnet-4-6",
  sandbox: dockerSandbox({ source: { type: "image", image: "node:24-slim" } }),
});

Create a Sandbox
  → Run the Sandbox setup lifecycle
  → Run the Eval setup lifecycle and prepare Fixtures
  → Check the Agent CLI version; install it and check again when needed
  → Run Adapter setup and write authentication and run configuration
  → Run test(t) and write starting files
  → The Adapter calls the Agent and records its file changes
  → Run verification commands, score the result, and form a Verdict
  → Run Adapter and Sandbox teardown lifecycles
  → Stop the Sandbox

export default defineEval({
  diff: {
    ignore: ["fixtures/**"],                 // A change here is not Agent work
    include: ["node_modules/some-pkg/**"],  // Include a patched dependency
  },
  async test(t) { /* ... */ },
});

import {
  completeEvidenceCoverage,
  createNpmCliInstaller,
  defineSandboxAgent,
  resolveAgentBin,
} from "niceeval/adapter";

const { ensure, installer } = createNpmCliInstaller({
  identity: { agent: "my-agent", version: "1.0.0", revision: "1" },
  packageName: "my-agent",
  bin: "my-agent",
});

export default defineSandboxAgent({
  name: "my-agent",
  evidenceCoverage: completeEvidenceCoverage,
  ensure,
  installers: [installer],
  async send(input, ctx) {
    ctx.progress({ message: "Running my-agent CLI" });
    const bin = await resolveAgentBin(ctx.sandbox, "my-agent");
    await ctx.sandbox.runCommand(bin, ["run", input.text, "--json-out", "agent-events.json"]);
    const transcript = await ctx.sandbox.readText("agent-events.json");
    if (transcript.trim() === "") {
      ctx.diagnostic({
        code: "empty-transcript",
        level: "warning",
        message: "my-agent did not write a transcript; tool assertions might lack evidence",
      });
    }
    return {
      status: "completed",
      events: parseTranscript(transcript),
    };
  },
});

✗ @5TB8167MXJ30SYZCNAVRHPQ4D2 fixtures/button [local] errored · agent setup
    agent-install-failed: npm install my-agent exited with code 1
    Inspect: niceeval show --run <runId> --page attempt-5TB8167MXJ30SYZCNAVRHPQ4D2

import { defineExperiment } from "niceeval";
import { dockerSandbox } from "niceeval/sandbox";
import myAgent from "./agents/my-agent";

export default defineExperiment({
  agent: myAgent,
  model: "claude-sonnet-4-6",
  sandbox: dockerSandbox({ source: { type: "image", image: "node:24-slim" } }),
  attempts: 3,
});