Conversational Testing

import { Agent, openai } from "@agentium/core";
import { ConversationSuite, ConversationRunner } from "@agentium/eval";

const agent = new Agent({
  name: "support-agent",
  model: openai("gpt-4o"),
  instructions: "You are a customer support agent.",
});

const suite = new ConversationSuite(
  {
    name: "Support Scenarios",
    scenarios: [
      {
        name: "Password Reset",
        persona: {
          name: "Frustrated User",
          description: "Non-technical user who is frustrated",
          goal: "Successfully reset their password",
          maxTurns: 10,
        },
        initialMessage: "I can't log in! I forgot my password.",
        successCriteria: "User successfully resets their password",
        expectedTrajectory: {
          requiredTools: ["send_reset_email"],
          forbiddenTools: ["delete_account"],
        },
      },
    ],
    concurrency: 3,
  },
  openai("gpt-4o-mini"), // Model for synthetic user
);

const results = await suite.run(agent);
console.log(`Passed: ${results.passed}/${results.total}`);
console.log(`Average turns: ${results.averageTurns}`);

import { SyntheticUser } from "@agentium/eval";

const user = new SyntheticUser(
  {
    name: "Impatient Executive",
    description: "C-level executive with no time for details",
    goal: "Get a summary of Q4 revenue",
    maxTurns: 5,
  },
  openai("gpt-4o-mini"),
);

const scenario = {
  name: "Order Lookup",
  expectedTrajectory: {
    requiredTools: ["search_orders", "get_order_details"],
    orderedTools: ["search_orders", "get_order_details"],
    forbiddenTools: ["cancel_order", "refund_order"],
    maxToolCalls: 5,
  },
};

const runner = new ConversationRunner(openai("gpt-4o-mini"));
const result = await runner.runComparison(agentA, agentB, scenario);
// result.winner: "A" | "B" | "tie"
// result.resultA: full conversation results
// result.resultB: full conversation results

interface ConversationSuiteResult {
  name: string;
  results: ConversationEvalResult[];
  passed: number;
  failed: number;
  total: number;
  averageTurns: number;
  averageScore: number;
  durationMs: number;
}