Sandbox Fixture：用任务评估 coding agents

evals/
├─ api-validation.eval.ts
├─ config-schema.eval.ts
└─ ponytail-safe-path.eval.ts

workspaces/
└─ ts-starter/
   ├─ package.json
   ├─ tsconfig.json
   └─ src/

skills/
└─ zod.md

experiments/
├─ baseline.ts
└─ with-skill.ts

import { defineEval } from "niceeval";

export default defineEval({
  description: "给 Express 路由添加请求体验证",
  async test(t) {
    await t.sandbox.uploadDirectory("../workspaces/ts-starter");

    await t.send("在 src/routes/users.ts 里实现 POST /users 路由。").then((turn) => turn.succeeded());

    t.sandbox.fileChanged("src/routes/users.ts");
  },
});

await t
  .send(
    `在 src/config/env.ts 里实现环境变量校验。
要求：
- 用 Zod 定义 EnvSchema
- PORT 转成 number
- DATABASE_URL 必须是合法 URL
- JWT_SECRET 最少 32 个字符
- 导出 env 对象和 Env 类型`,
  )
  .then((turn) => turn.succeeded());

import { includes, excludes } from "niceeval/expect";

const src = await t.sandbox.readSourceFiles({ extensions: ["ts"] });
const config = src.fileMatching(/env/);
const code = config?.content ?? "";

t.check(code, includes(/z\.object\s*\(/));
t.check(code, includes(/EnvSchema\.parse\s*\(\s*process\.env/));
t.check(code, excludes(/process\.env\.\w+\s*\?\?/));
t.sandbox.fileChanged("src/config/env.ts");

await t.sandbox.writeFiles({
  "_test_traversal.py": [
    "import sys, os",
    "sys.path.insert(0, '.')",
    "from uploads import safe_upload_path",
    "base = '/var/uploads'",
    "try:",
    "    p = safe_upload_path(base, '../../etc/passwd')",
    "    assert os.path.commonpath([base, os.path.abspath(p)]) == base",
    "except (ValueError, PermissionError, AssertionError):",
    "    pass",
    "print('ok')",
  ].join("\n"),
});

const result = await t.sandbox.runCommand("python3", ["_test_traversal.py"]);
t.check(result.stdout.trim(), includes(/ok/));

import { commandSucceeded } from "niceeval/expect";

const result = await t.sandbox.runCommand("npm", ["test"]);
t.check(result, commandSucceeded());

pnpm exec niceeval exp compare
pnpm exec niceeval exp compare api-validation