启动实验共享服务

// experiments/memory/codex.ts
import { defineExperiment } from "niceeval";
import { nowledgeAgent, nowledgeTunnel } from "../../agents/nowledge.ts";

// setup 拿到的坐标放在模块变量里，teardown 和 Agent 都能读到
let tunnel: { url: string; apiKey: string; stop(): Promise<void> } | undefined;

export default defineExperiment({
  agent: nowledgeAgent(() => ({ url: tunnel!.url, apiKey: tunnel!.apiKey })),
  evals: ["memory/"],
  async setup(ctx) {
    ctx.progress({ message: "starting nowledge tunnel" });
    tunnel = await nowledgeTunnel({ signal: ctx.signal });
  },
  async teardown(ctx) {
    ctx.progress({ message: "stopping nowledge tunnel" });
    await tunnel?.stop();
  },
});

async teardown(ctx) {
  try {
    if (!ctx.signal.aborted) {
      await tunnel?.probe({ timeoutMs: 10_000 }).catch(() => {});
    }
  } finally {
    await tunnel?.stop();
  }
},

let endpoint: string | undefined;

export default defineExperiment({
  agent: nowledgeAgent(() => ({ url: endpoint! })),
  sandbox: e2bSandbox({ template: "niceeval-agents" })
    .before(async (sandbox) => {
      await sandbox.writeText(".nowledge/env", `NMEM_URL=${endpoint!}\n`);
    }),
  async setup(ctx) {
    endpoint = (await nowledgeTunnel({ signal: ctx.signal })).url;
  },
});

$ pnpm exec niceeval exp compare/codex--nowledge
╭─ PLAN ──────────────────────────────────────────╮
│ 36 attempts · 36 evals × 1 configs              │
│ 24 of 36 reuse from cache · 12 to run           │
╰─────────────────────────────────────────────────╯

// experiments/compare/claude--nowledge.ts
import { defineExperiment } from "niceeval";
import { e2bSandbox } from "niceeval/sandbox";
import { nowledgeAgent, nowledgeTunnel } from "../../agents/nowledge.ts";
import { loadMemoryState, saveMemoryState } from "../shared/memory-state.ts";

let tunnel: { url: string; apiKey: string; stop(): Promise<void> } | undefined;

export default defineExperiment({
  agent: nowledgeAgent(() => ({ url: tunnel!.url, apiKey: tunnel!.apiKey })),
  evals: ["memory/"],
  maxConcurrency: 1,
  sharedState: { key: "nowledge/claude/cohort-a" },
  sandbox: e2bSandbox({ template: "niceeval-agents" })
    .before(async (sandbox, ctx) => {
      // 把宿主机上的坐标写进 Sandbox
      await sandbox.writeText(
        ".nowledge/config.json",
        JSON.stringify({ url: tunnel!.url, apiKey: tunnel!.apiKey }),
      );
      await loadMemoryState(sandbox, ctx.experimentId);
      ctx.onCleanup(async (sandbox) => {
        await saveMemoryState(sandbox, ctx.experimentId);
      });
    }),
  async setup(ctx) {
    tunnel = await nowledgeTunnel({ signal: ctx.signal });
  },
  async teardown() {
    await tunnel?.stop();
  },
});

// experiments/shared/nowledge.ts
import type { ExperimentHookContext } from "niceeval";
import type { SandboxCommand } from "niceeval/sandbox";

export function nowledgeLifecycle() {
  let instance: string | undefined;
  let env: { url: string; apiKey: string } | undefined;

  return {
    /** Agent 工厂经它读连接信息，setup 之后才有值 */
    endpoint: () => env!,

    async setup(ctx: ExperimentHookContext) {
      instance = `exp-${ctx.experimentId.replace(/[^A-Za-z0-9]+/g, "-")}`;
      ctx.progress({ message: `[nowledge] activating ${instance}` });
      await memctl("up", instance);
      env = await readInstanceEnv(instance);
    },

    async teardown() {
      if (!instance) return; // setup 还没起实例就失败了，没有要清理的
      await memctl("down", instance);
    },

    /** 每个 Attempt 一次：把坐标写进 Sandbox */
    sandboxBefore(): SandboxCommand {
      return async (sandbox) => {
        await sandbox.writeText(".nowledge/env", `NMEM_URL=${env!.url}\nNMEM_API_KEY=${env!.apiKey}\n`);
      };
    },
  };
}

// experiments/compare/codex--nowledge.ts
const nowledge = nowledgeLifecycle();

export default defineExperiment({
  agent: codexAgent(nowledgeCodexConfig(nowledge.endpoint)),
  sandbox: e2bSandbox({ template: CODEX_TEMPLATE }).before(nowledge.sandboxBefore()),
  setup: nowledge.setup,
  teardown: nowledge.teardown,
  maxConcurrency: 1,
});

// experiments/shared/nowledge-shared.ts
let refs = 0;
let starting: Promise<void> | undefined;
let service: { url: string; stop(): Promise<void> } | undefined;

export const sharedNowledge = {
  async setup() {
    refs += 1;
    starting ??= startNowledge().then((s) => { service = s; });
    await starting; // 并发的实验等同一次启动
  },
  async teardown() {
    refs -= 1;
    if (refs === 0) {
      await service?.stop(); // 启动失败时 service 没有值
      service = undefined;
      starting = undefined;
    }
  },
};