Install a CLI for a Custom Sandbox Agent

// agents/my-agent.ts
import {
  createNpmCliInstaller,
  defineSandboxAgent,
  agentBin,
  responsesEvidenceCoverage,
  turnFromResponses,
  type ResponseLike,
} from "niceeval/adapter";

const { ensure, installer } = createNpmCliInstaller({
  identity: {
    agent: "my-agent",
    version: "1.0.0",
    revision: "1",
  },
  packageName: "my-agent",
  bin: "my-agent",
  progress: {
    checking: "Checking my-agent CLI 1.0.0",
    installing: "Installing my-agent CLI 1.0.0",
    ready: "my-agent CLI 1.0.0 is ready",
  },
});

export default defineSandboxAgent({
  name: "my-agent",
  evidenceCoverage: responsesEvidenceCoverage,
  ensure,
  installers: [installer],

  async setup(sandbox) {
    await sandbox.writeText(".my-agent/config.json", JSON.stringify({ telemetry: true }));
  },

  async send(input, ctx) {
    const result = await ctx.sandbox.runCommand("sh", [
      "-lc",
      `exec ${agentBin("my-agent")} run --json "$1"`,
      "my-agent",
      input.text,
    ]);
    if (result.exitCode !== 0) {
      throw new Error(`my-agent exited with code ${result.exitCode}: ${result.stderr}`);
    }
    return turnFromResponses(JSON.parse(result.stdout) as ResponseLike);
  },
});

// experiments/my-agent.ts
import { defineExperiment } from "niceeval";
import { dockerSandbox } from "niceeval/sandbox";
import myAgent from "../agents/my-agent.ts";

export default defineExperiment({
  agent: myAgent,
  model: "acme-model-v2",
  sandbox: dockerSandbox({ source: { type: "image", image: "node:22-slim" } }),
});

npx niceeval exp my-agent fixtures/button

pnpm exec niceeval show @<attempt-locator>