Build a quality gate with traces

import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { pathToFileURL } from "node:url";
import { Agent, openai, type ModelProvider } from "@agentium/core";
import { custom, EvalSuite } from "@agentium/eval";
import { instrument, JsonFileExporter } from "@agentium/observability";
import { z } from "zod";

const classification = z.object({ queue: z.enum(["billing", "shipping"]) });

// A deterministic provider makes the evaluation pipeline runnable offline.
function createFixture(broken: boolean): ModelProvider {
  return {
    providerId: "fixture", modelId: "ticket-classifier",
    async generate(messages) {
      const input = String(messages.filter((message) => message.role === "user").at(-1)?.content ?? "");
      const queue = !broken && input.includes("delivery") ? "shipping" : "billing";
      return {
        message: { role: "assistant", content: JSON.stringify({ queue }) },
        finishReason: "stop", raw: null,
        usage: { promptTokens: 10, completionTokens: 5, totalTokens: 15 },
      };
    },
    async *stream() { throw new Error("This recipe uses generate(), not stream()"); },
  };
}

export async function runQualityGate(model: ModelProvider, outputRoot = "artifacts") {
  await mkdir(outputRoot, { recursive: true });
  const directory = await mkdtemp(join(outputRoot, "quality-"));
  const agent = new Agent({
    name: "ticket-classifier", model, register: false,
    instructions: "Classify invoices as billing and delivery questions as shipping.",
    structuredOutput: classification,
  });
  const telemetry = instrument(agent, {
    exporters: [new JsonFileExporter({ path: join(directory, "traces.jsonl") })],
  });
  try {
    const result = await new EvalSuite({
      name: "ticket-routing", agent, concurrency: 1, timeoutMs: 10_000,
      cases: [
        { name: "invoice", input: "Explain my invoice", expected: "billing" },
        { name: "delivery", input: "Where is my delivery?", expected: "shipping" },
      ],
      scorers: [custom("correct-queue", (_input, output, expected) => {
        const parsed = classification.safeParse(output.structured);
        const pass = parsed.success && parsed.data.queue === expected;
        return { score: pass ? 1 : 0, pass, reason: pass ? "Correct queue" : `Expected ${expected}` };
      })],
    }).run();
    const summary = {
      passed: result.passed, failed: result.failed, total: result.total,
      cases: result.results.map((item) => ({ name: item.caseName, pass: item.pass, scores: item.scores, error: item.error })),
      metrics: telemetry.metrics?.getMetrics(),
    };
    // Await the write so report failures cannot silently produce a passing gate.
    await writeFile(join(directory, "report.json"), JSON.stringify(summary, null, 2));
    console.log(`passed: ${result.passed}/${result.total}; failed: ${result.failed}`);
    console.log("artifacts:", directory);
    return { failed: result.failed, directory };
  } finally {
    try { await agent.close(); } finally { await telemetry.shutdown(); }
  }
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  const live = process.argv.includes("--live");
  if (live && !process.env.OPENAI_API_KEY) throw new Error("Set OPENAI_API_KEY before using --live");
  const model = live
    ? openai(process.env.OPENAI_MODEL ?? "gpt-6-luna")
    : createFixture(process.argv.includes("--broken"));
  const result = await runQualityGate(model);
  if (result.failed > 0) process.exitCode = 1;
}