Case input  →  support agent  →  reply text
                                    ↓
                         Jev: { faithful: 0.91, helpful: 2 }
                                    ↓
                         pass if faithful >= 0.7

npm install @agentium/eval @typesafe-ai/sdk
export TYPESAFE_API_KEY="..."

import { Agent, jev, noul, openai, score } from "@agentium/core";
import { EvalSuite, custom, ConsoleReporter } from "@agentium/eval";

const agent = new Agent({
  name: "support",
  model: openai("gpt-4o-mini"),
  instructions: "Help the customer. Stay on the ticket. Do not invent facts.",
});

const judge = new Agent({
  name: "jev-judge",
  model: jev("jev-latest"),
});

const suite = new EvalSuite({
  name: "support quality",
  agent,
  cases: [
    { name: "double-charge", input: "I was charged twice for last month." },
    { name: "outage", input: "The API returns 500 on /v1/shipments." },
  ],
  scorers: [
    custom("faithful", async (input, output) => {
      const result = await judge.run(
        JSON.stringify({ ticket: input, reply: output.text }),
        {
          questions: {
            faithful: noul(
              "Does the reply stay true to the ticket — no fake systems, no weather, no denying what the customer said?",
            ),
            helpful: score("How useful is this reply?", [
              "unhelpful",
              "partial",
              "useful",
              "excellent",
            ]),
          },
        },
      );
      const a = JSON.parse(result.text);
      const s = a.faithful.noul as number;
      return {
        score: s,
        pass: s >= 0.7,
        reason: `faithful=${s} helpful=${a.helpful.score}`,
      };
    }),
  ],
  threshold: 0.7,
});

const result = await suite.run([new ConsoleReporter()]);
console.log(`${result.passed}/${result.total} passed`);

// Broken — Jev cannot write this JSON
llmJudge({ model: jev("jev-latest"), criteria: ["relevance"] });

// Broken — same chat-shaped prompt
new AgentJudgeEval({ judge: jev("jev-latest"), criteria: ["…"] });