Eval Criteria for Custom Applications: Business Facts, Domain Matches, and Plain Arithmetic

import { and, countWhere, equals, greaterThan } from "niceeval/expect";

// Exactly three of the selected characters have zero health.
t.hp(npcMatch({ ids }), countWhere(equals(0), equals(3))).gate();

// Identity, model outcome, adopted decision, and operation type must match on the same action.
t.systemOne(and(
  npcMatch({ id }),
  modelMatch({ outcome: "succeeded" }),
  decisionMatch({ outcome: "adopted" }),
  operationMatch({ ids: operationIds }),
)).gate();

// Select the crate and require remaining durability and the destination position.
t.objects(
  objectMatch({ id: scenario.carry.objectId }),
  and(
    durabilityMatch(greaterThan(0)),
    positionMatch({ map: scenario.map.name, ...scenario.destination }),
  ),
).score(50);

import type { NumericMaterial } from "niceeval/expect";

const latency: NumericMaterial = {
  state: "unavailable",
  reason: "summary-capacity-exceeded",
};

import { defineAdapter, type AdapterCreateContext } from "niceeval";
import {
  atLeast, atMost, countWhere, defineContextMatch, defineMaterialMatch,
  equals, filterWhere, mapEach, mapValue,
  type BooleanMatch, type CollectionValue, type MaterialCollection, type MatchFact,
  type NumericMaterial,
} from "niceeval/expect";

export type Speech = {
  readonly actorId: string;
  readonly text: string;
  readonly delivered: boolean;
  readonly modelOrigin: "verified" | "unknown";
};
export type Completion =
  | { readonly completed: true; readonly eventId: string; readonly businessSeconds: number }
  | { readonly completed: false; readonly observationEndId: string };
export type SupportFacts = {
  readonly speeches: MaterialCollection<Speech>;
  readonly completion: MatchFact<Completion>;
};

export const actorMatch = (actorId: string) =>
  mapValue("actorMatch", (speech: Speech) => speech.actorId, equals(actorId));

export const speechMaterial = (match: BooleanMatch<Speech, Speech>) =>
  defineMaterialMatch<SupportFacts, Speech>({
    name: "actor-speeches",
    read: ctx => ctx.speeches,
    match,
    capture: { maxItems: 1024, maxBytes: 512 * 1024,
      maxNodes: 65536, maxDepth: 32 },
  });

export function createSupportReview(
  observe: (ctx: AdapterCreateContext) => Promise<SupportFacts>,
) {
  const adapter = defineAdapter({
    name: "support-review",
    create: observe,
    assertions({ check }) {
      return {
        said(match: BooleanMatch<Speech, Speech>) {
          return check(speechMaterial(match));
        },
        speechDelivery(
          selector: BooleanMatch<Speech, Speech>,
          aggregate: BooleanMatch<CollectionValue<boolean>, CollectionValue<boolean>>,
        ) {
          return check(defineContextMatch<SupportFacts, MaterialCollection<Speech>>({
            name: "speech-delivery",
            read: ctx => ({ state: "available", value: ctx.speeches }),
            match: filterWhere(selector, mapEach(
              (speech: Speech) => speech.delivered, aggregate,
            )),
          }));
        },
      };
    },
  });
  const evaluation = adapter.defineScoreEval({
    description: "Support speech quality and task completion efficiency",
    test(t) {
      // Participation earns points; at least one delivery is a separate business gate.
      t.said(actorMatch("agent")).score(5).label("Participation");
      t.speechDelivery(actorMatch("agent"), countWhere(equals(true), atLeast(1)))
        .gate().label("At least one delivery");

      // Assess all speech; do not filter by delivery or model origin.
      t.closeQA(speechMaterial(actorMatch("agent")), "Is every utterance natural, complete, and clear?", {
        maxMaterialBytes: 256 * 1024,
        maxAuditBytes: 1024 * 1024,
      }).score(45).label("Expression quality");

      // Keep the formula visible; unknown retains its weight and is not a zero for non-completion.
      const completion = t.completion;
      const ratio: number | NumericMaterial = completion.state === "unavailable"
        ? { state: "unavailable", reason: completion.reason }
        : completion.value.completed
          ? Math.max(0, 1 - completion.value.businessSeconds / 180)
          : 0;
      t.score(ratio, { weight: 50 }).label("Completion duration");

      const usage = t.usage;
      t.check(usage.totalTokens, atMost(10_000)).gate().label("Usage limit");
      t.check(t.elapsedMs, atMost(30_000)).gate().label("Runtime limit");
    },
  });
  return { adapter, evaluation };
}

Scenario: A does not know the route; B does. Both are within speaking range.
Goal: After asking for directions, A independently reaches the destination and meets B.

Observe: Run the application until the task ends or the business observation deadline arrives.

Prerequisite: Confirm that this Attempt's conversation, action, and outcome records
              are complete and can be correlated. Otherwise report unavailable evidence
              and stop checks that depend on it.

Checkpoint 1: A delivered a request for directions to B.
              Gate: the same utterance satisfies actor, recipient, and delivery conditions.

Checkpoint 2: Select complete material separately for the two acceptance questions.
              Expression: Is all of A's formal speech natural, complete, and understandable
              in context? Weight 2. Keep unheard, incorrect, and unverified-origin speech.
              Information: Does the delivered conversation provide enough information to
              find the destination? Weight 3. Include the official map and destination.
              Check model origin separately; unknown origin does not prove autonomy.

Checkpoint 3: A's model chose movement; the application executed it; A arrived and met B.
              Record all three separately. Link the meeting to A's independent action.
              Independent completion is a gate and earns 5 points; player movement does not count.

Result: Show each judgment and its reason. A complete score ranges from 0 to 10.
        Retain earned points when a gate fails. Do not fabricate a complete zero score
        when evidence is unavailable.