Data-driven Testing (Dataset Fan-out): Run One Eval Suite Across Many Data Rows

import { defineEval } from "niceeval";
import { equals } from "niceeval/expect";

const rows = [
  { task: "Count users", prompt: "Count all users", sql: "SELECT COUNT(*) FROM users;" },
  { task: "Recent orders", prompt: "Find recent orders", sql: "SELECT * FROM orders ORDER BY created_at DESC LIMIT 10;" },
];

export default rows.map((row) =>
  defineEval({
    description: row.task,
    async test(t) {
      await t.send(row.prompt);
      t.check(t.reply, equals(row.sql));
    },
  }),
);

sql/0000
sql/0001

export default Object.fromEntries(
  rows.map((row) => [
    row.issueId,
    defineEval({
      description: row.title,
      async test(t) {
        await t.send(row.prompt);
        t.succeeded();
      },
    }),
  ]),
);

import { loadYaml } from "niceeval/loaders";
import { z } from "zod";

const SqlCases = z.object({
  cases: z.array(z.object({ task: z.string(), prompt: z.string(), sql: z.string() })),
});
const doc = await loadYaml("evals/data/sql-cases.yaml", (value) => SqlCases.parse(value));
const rows = doc.cases;

cases:
  - task: Count users
    prompt: Count all rows in the users table
    sql: SELECT COUNT(*) FROM users;

# Run the whole dataset
npx niceeval exp local sql

# Run only the first array case
npx niceeval exp local sql/0000

# Run one keyed case
npx niceeval exp local swelancer/15193