npm install @agentium/core @google/genai

import { VisionAgent, geminiVisionLive } from "@agentium/core";
import { readFileSync } from "fs";

const agent = new VisionAgent({
  name: "vision-assistant",
  provider: geminiVisionLive("gemini-3.1-flash-live-preview"),
  instructions: "You can see and hear. Describe what you see when asked.",
  voice: "Aoede",
  fps: 1,
  thinkingLevel: "minimal",
});

const session = await agent.connect();

// Send an image frame (JPEG from camera, screenshot, etc.)
const frame = readFileSync("./photo.jpg");
session.sendImage(frame, "image/jpeg");

// Send audio (PCM 16kHz)
session.sendAudio(micBuffer);

// Send text
session.sendText("What do you see?");

// Listen for responses
session.on("transcript", ({ text, role }) => {
  console.log(`[${role}] ${text}`);
});

session.on("audio", ({ data }) => {
  // Play audio response (PCM 24kHz)
});

session.on("interrupted", () => {
  // User started speaking — stop playback
});

interface VisionProvider {
  readonly providerId: string;
  readonly modelId: string;
  connect(config: VisionSessionConfig): Promise<VisionConnection>;
}

interface VisionConnection {
  sendAudio(data: Buffer): void;
  sendImage(data: Buffer, mimeType?: string): void;
  sendText(text: string): void;
  sendToolResult(callId: string, result: string): void;
  interrupt(): void;
  close(): Promise<void>;
  on(event, handler): void;
  off(event, handler): void;
}

import { geminiVisionLive } from "@agentium/core";

const provider = geminiVisionLive("gemini-3.1-flash-live-preview");

const agent = new VisionAgent({
  name: "assistant",
  provider: geminiVisionLive(),
  instructions: "You are a helpful vision assistant.",
  voice: "Aoede",              // voice selection (30 voices available)
  language: "hi-IN",           // optional — force a BCP-47 language code
  fps: 1,                      // frame rate hint
  thinkingLevel: "minimal",    // lowest latency
  temperature: 0.7,
  tools: [myTool],
  memory: { /* UnifiedMemoryConfig */ },
});

const agent = new VisionAgent({
  name: "assistant",
  provider: geminiVisionLive(),
  voice: "Puck",  // Upbeat personality
});

socket.on("vision.start", async ({ voice }) => {
  const agent = new VisionAgent({
    name: "assistant",
    provider: geminiVisionLive(),
    instructions: "You are a helpful assistant.",
    voice: voice || undefined,  // user's choice or provider default
  });
  const session = await agent.connect();
});

const agent = new VisionAgent({
  name: "assistant",
  provider: geminiVisionLive(),
  instructions: [
    "You are a helpful multilingual vision assistant.",
    "LANGUAGE RULE: Always detect the language the user is speaking and respond in that same language.",
    "If the user switches languages mid-conversation, switch with them immediately.",
  ].join(" "),
});

const agent = new VisionAgent({
  name: "assistant",
  provider: geminiVisionLive(),
  language: "hi-IN",  // always respond in Hindi
});

const session = await agent.connect();

session.on("interrupted", () => {
  console.log("User interrupted — response stopped");
  // Forward to client to stop audio playback
});

let activeSources = [];

socket.on("vision.audio", (data) => {
  const source = playbackCtx.createBufferSource();
  // ... decode and play audio ...
  activeSources.push(source);
  source.onended = () => {
    activeSources = activeSources.filter(s => s !== source);
  };
});

socket.on("vision.interrupted", () => {
  activeSources.forEach(s => { try { s.stop(); } catch(e) {} });
  activeSources = [];
  nextPlayTime = 0;
});

import { readFileSync } from "fs";

const frame = readFileSync("./screenshot.jpg");
session.sendImage(frame, "image/jpeg");

const stream = await navigator.mediaDevices.getDisplayMedia({ video: true });
const video = document.createElement("video");
video.srcObject = stream;
await video.play();

const canvas = document.createElement("canvas");
const nw = video.videoWidth;
const nh = video.videoHeight;
const scale = Math.min(1, 1280 / nw);
canvas.width = Math.round(nw * scale);
canvas.height = Math.round(nh * scale);
const ctx = canvas.getContext("2d");

setInterval(() => {
  ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
  canvas.toBlob((blob) => {
    blob.arrayBuffer().then((buf) => {
      const base64 = btoa(String.fromCharCode(...new Uint8Array(buf)));
      socket.emit("vision.image", {
        data: base64,
        mimeType: "image/jpeg",
        source: "screen",
      });
    });
  }, "image/jpeg", 0.7);
}, 2000);

const stream = await navigator.mediaDevices.getUserMedia({
  video: { width: 640, height: 480, facingMode: "user" },
  audio: false,
});
const video = document.createElement("video");
video.srcObject = stream;
await video.play();

const canvas = document.createElement("canvas");
canvas.width = video.videoWidth || 640;
canvas.height = video.videoHeight || 480;
const ctx = canvas.getContext("2d");

setInterval(() => {
  ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
  canvas.toBlob((blob) => {
    blob.arrayBuffer().then((buf) => {
      const base64 = btoa(String.fromCharCode(...new Uint8Array(buf)));
      socket.emit("vision.image", {
        data: base64,
        mimeType: "image/jpeg",
        source: "camera",
      });
    });
  }, "image/jpeg", 0.7);
}, 2000);

import { createVisionGateway } from "@agentium/transport";

createVisionGateway({
  agents: { assistant: visionAgent },
  io: socketIoServer,
});

import { defineTool } from "@agentium/core";
import { z } from "zod";

const weatherTool = defineTool({
  name: "getWeather",
  description: "Get weather for a city",
  parameters: z.object({ city: z.string() }),
  execute: async ({ city }) => `Sunny, 22C in ${city}`,
});

const agent = new VisionAgent({
  name: "assistant",
  provider: geminiVisionLive(),
  tools: [weatherTool],
});

const agent = new VisionAgent({
  name: "assistant",
  provider: geminiVisionLive(),
  memory: {
    storage: new SqliteStorage("vision-memory.db"),
    summary: { enabled: true },
    userProfile: { enabled: true },
  },
});

GOOGLE_API_KEY=your-key npx tsx examples/voice/screen-assistant.ts

import { VisionAgent, geminiVisionLive } from "@agentium/core";

const INSTRUCTIONS = [
  "You are a helpful multilingual vision assistant.",
  "You can see the user's screen and/or camera feed, and hear them speak.",
  "LANGUAGE RULE: Always detect the language the user is speaking and respond in that same language.",
  "If the user switches languages mid-conversation, switch with them immediately.",
  "Keep responses conversational and concise. Speak naturally in the user's language.",
].join(" ");

// Create a per-session agent with the user's chosen voice
const agent = new VisionAgent({
  name: "VisionAssistant",
  provider: geminiVisionLive("gemini-3.1-flash-live-preview"),
  instructions: INSTRUCTIONS,
  voice: "Puck",     // or let the user pick
  fps: 1,
  thinkingLevel: "low",
});

const session = await agent.connect();

// Forward screen/camera frames
session.sendImage(jpegBuffer, "image/jpeg");

// Forward microphone audio
session.sendAudio(pcm16kBuffer);

// Handle responses
session.on("audio", ({ data }) => { /* play PCM 24kHz */ });
session.on("transcript", ({ text, role }) => { /* show text */ });
session.on("interrupted", () => { /* stop audio playback */ });