Pattern 1: Memory Middleware (Real-Time)

import { Sonzai } from "@sonzai-labs/agents";

const sonzai = new Sonzai({ apiKey: process.env.SONZAI_API_KEY! });

async function runConversation(agentId: string, userId: string) {
const sessionId = `session-${Date.now()}`;
const history: { role: string; content: string }[] = [];

// Open a Session handle. agentId/userId/sessionId and provider/model
// defaults live on the handle so you don't repeat them on every call.
const session = await sonzai.agents.sessions.start(agentId, {
  userId,
  sessionId,
  toolDefinitions: yourTools,                   // optional — register session-scoped tool schemas
  provider: "gemini",                           // optional — default for .turn()
  model: "gemini-3.1-flash-lite-preview",       // optional — default for .turn()
});

async function turn(userMessage: string): Promise<string> {
  // Fresh enriched context for this specific message
  const ctx = await session.context({ query: userMessage });

  // Your LLM — swap in any provider you like
  let reply = await yourLLM.chat({
    system: buildSystemPrompt(ctx),
    messages: [...history, { role: "user", content: userMessage }],
    tools: yourTools,
  });

  // Tool-calling loop is entirely yours — Sonzai is OUT of the loop here.
  const toolMessages: any[] = [];
  while (reply.tool_calls?.length) {
    for (const call of reply.tool_calls) {
      const result = await runYourTool(call);
      toolMessages.push(
        { role: "assistant", tool_calls: [call] },
        { role: "tool", tool_call_id: call.id, content: result },
      );
    }
    reply = await yourLLM.chat({
      system: buildSystemPrompt(ctx),
      messages: [...history, { role: "user", content: userMessage }, ...toolMessages],
      tools: yourTools,
    });
  }

  sendToUser(reply.content); // send first; don't block on Sonzai

  // Submit just the new turn. Sync mood ~300ms, deferred extraction
  // (facts, personality, habits) runs asynchronously 5–15s later.
  // Pass the FULL exchange — including tool calls and tool results —
  // so Sonzai can extract facts from tool outputs too.
  const { mood, extraction_id } = await session.turn({
    messages: [
      { role: "user", content: userMessage },
      ...toolMessages,                          // assistant tool_calls + tool results
      { role: "assistant", content: reply.content },
    ],
  });

  history.push({ role: "user", content: userMessage });
  history.push({ role: "assistant", content: reply.content });

  return reply.content;
}

return { turn, end: () => session.end() };
}

// The /context response is a flat object — there is no nested
// `profile` / `behavioral` / `memory` envelope.
function buildSystemPrompt(ctx: any): string {
const facts = (ctx.loaded_facts ?? []).map((f: any) => `- ${f.atomic_text}`).join("\n");
const goals = (ctx.active_goals ?? []).map((g: any) => g.description).join(", ");
return `${ctx.personality_prompt ?? "You are a helpful AI companion."}
Personality (Big5): ${JSON.stringify(ctx.big5 ?? {})}
Current mood: ${JSON.stringify(ctx.current_mood ?? {})}
Active goals: ${goals || "none"}
Relevant memories:
${facts || "none yet"}`;
}

while (conversationActive) {
const userMsg = await getUserInput();

// 1. PULL FRESH CONTEXT — happens every turn, before the LLM call.
//    ctx is a flat object — no `profile` / `behavioral` / `memory` envelope.
//    Fields you'll usually read:
//      ctx.personality_prompt          — agent identity / instructions
//      ctx.bio, ctx.speech_patterns    — agent identity bits
//      ctx.big5                        — Big5 trait object
//      ctx.current_mood                — fresh inline (~300ms after .turn())
//      ctx.habits, ctx.active_goals    — behavioral state
//      ctx.loaded_facts                — recalled facts (5-15s lag from extraction)
//      ctx.proactive_memories          — pending proactive signals
//      ctx.knowledge.results           — KB hits (only nested key)
//      ctx.recent_turns                — buffered messages from this session
const ctx = await session.context({ query: userMsg });

// 2. Build system prompt from the context layers
const systemPrompt = renderPromptFromContext(ctx);

// 3. Run YOUR LLM — Sonzai is OUT of the loop here
const reply = await yourLLM.chat({
  system: systemPrompt,
  messages: [...history, { role: "user", content: userMsg }],
});

// 4. Submit the just-completed turn — sync mood + async deferred extraction
await session.turn({
  messages: [
    { role: "user", content: userMsg },
    { role: "assistant", content: reply.content },
  ],
});
}

function renderPromptFromContext(ctx: any): string {
const parts: string[] = [];
if (ctx.personality_prompt) parts.push(ctx.personality_prompt);
if (ctx.big5) parts.push(`Personality (Big5): ${JSON.stringify(ctx.big5)}`);
if (ctx.speech_patterns?.length) parts.push(`Speech patterns: ${ctx.speech_patterns.join(", ")}`);
if (ctx.current_mood) parts.push(`Current mood: ${JSON.stringify(ctx.current_mood)}`);
const facts = (ctx.loaded_facts ?? []).slice(0, 5).map((f: any) => `- ${f.atomic_text ?? ""}`).join("\n");
if (facts) parts.push(`Relevant memories:\n${facts}`);
const kb = (ctx.knowledge?.results ?? []).slice(0, 3).map((r: any) => `- ${r.label}: ${(r.content ?? "").slice(0, 120)}`).join("\n");
if (kb) parts.push(`Knowledge base:\n${kb}`);
return parts.join("\n\n");
}

const ctx = await session.context({ query: userMessage });

// Sonzai is the source of truth — no local history list needed.
const history = (ctx.recent_turns ?? []).map((t) => ({
role: t.role,
content: t.content,
}));

const reply = await yourLLM.chat({
system:   buildSystemPrompt(ctx),
messages: [...history, { role: "user", content: userMessage }],
});

await session.turn({
messages: [
  { role: "user", content: "Where did my last order ship from?" },
  {
    role: "assistant",
    tool_calls: [{ id: "call_1", type: "function", function: { name: "order-lookup", arguments: "{}" } }],
  },
  {
    role: "tool",
    tool_call_id: "call_1",
    content: '{"order_id":"42","origin":"Tokyo","carrier":"DHL"}',
  },
  { role: "assistant", content: "Your last order shipped from Tokyo via DHL." },
],
});

const { extraction_id } = await session.turn({ messages });

// Optional — only poll if you need to wait for facts/personality before doing something
let status = await session.status(extraction_id);
while (status.state !== "done" && status.state !== "failed") {
await new Promise((r) => setTimeout(r, 1000));
status = await session.status(extraction_id);
}

// TypeScript — agents.memory.search is available directly
import { Sonzai } from "@sonzai-labs/agents";
import { tool } from "ai";
import { z } from "zod";

const sonzai = new Sonzai({ apiKey: process.env.SONZAI_API_KEY! });

const kbSearch = tool({
description: "Search the agent's knowledge base.",
parameters: z.object({ query: z.string() }),
execute: async ({ query }) => {
  const res = await sonzai.agents.knowledgeSearch("agent-id", { query, limit: 5 });
  return res.results.map((r) => `- ${r.label}: ${r.content}`).join("\n") || "No matching knowledge.";
},
});

const memorySearch = tool({
description: "Search the user's long-term memory.",
parameters: z.object({ query: z.string() }),
execute: async ({ query }) => {
  const res = await sonzai.agents.memory.search("agent-id", {
    query,
    user_id: "user-123",
    limit: 5,
  });
  return res.results.map((r) => `- ${r.text}`).join("\n") || "No matching memories.";
},
});

import OpenAI from "openai";
import { Sonzai } from "@sonzai-labs/agents";

const gemini = new OpenAI({
baseURL: "https://generativelanguage.googleapis.com/v1beta/openai/",
apiKey: process.env.GEMINI_API_KEY!,
});
const sonzai = new Sonzai({ apiKey: process.env.SONZAI_API_KEY! });

const SYSTEM_PROMPT_IMAGE_AWARE = `You are a friendly companion. When the user shares an image, respond warmly
to what's emotionally important to THEM.

After your reply, ALWAYS include a single line:
[MEMORY: <detailed factual description of the image — setting, objects,
people, mood, time of day, what the user appears to be doing>]

The user does NOT see the [MEMORY: ...] line.`;

async function processImageTurn(session: any, userMsg: string, imageUrl: string): Promise<string> {
const result = await gemini.chat.completions.create({
  model: "gemini-3.1-flash-lite-preview",
  messages: [
    { role: "system", content: SYSTEM_PROMPT_IMAGE_AWARE },
    {
      role: "user",
      content: [
        { type: "text", text: userMsg },
        { type: "image_url", image_url: { url: imageUrl } },
      ],
    },
  ],
});
const raw = result.choices[0].message.content ?? "";

// Split the dual output
const m = raw.match(/\[MEMORY:\s*([\s\S]+?)\]/);
const memoryNote = m ? m[1].trim() : "";
const reply = raw.replace(/\[MEMORY:[\s\S]+?\]/, "").trim();

sendToUser(reply);

await session.turn({
  messages: [
    { role: "user", content: `${userMsg}\n\n[Image attached: ${memoryNote}, URL: ${imageUrl}]` },
    { role: "assistant", content: reply },
  ],
});
return reply;
}

import { Sonzai } from "@sonzai-labs/agents";

const sonzai = new Sonzai({ apiKey: process.env.SONZAI_API_KEY! });

async function processVoiceTurn(
session: any, // Session handle from sonzai.agents.sessions.start
audioBuffer: Buffer
): Promise<Buffer> {
// Your STT
const transcript = await yourSTT.transcribe(audioBuffer);

// Inject memory into a concise voice-friendly system prompt
const ctx = await session.context({ query: transcript });

const systemPrompt = `${ctx.personality_prompt ?? "You are a voice companion."} Keep replies under 2 sentences for voice.
Mood: ${JSON.stringify(ctx.current_mood)}.
Key memory: ${ctx.loaded_facts?.[0]?.atomic_text ?? "none"}.`;

const reply = await yourLLM.chat({ system: systemPrompt, message: transcript });

// Submit the turn while TTS synthesizes (run in parallel)
const [audioResponse] = await Promise.all([
  yourTTS.synthesize(reply),
  session.turn({
    messages: [
      { role: "user", content: transcript },
      { role: "assistant", content: reply },
    ],
  }),
]);

return audioResponse;
}

import { ChatOpenAI } from "@langchain/openai";
import { SystemMessage, HumanMessage, AIMessage } from "@langchain/core/messages";
import { Sonzai } from "@sonzai-labs/agents";

const sonzai = new Sonzai({ apiKey: process.env.SONZAI_API_KEY! });
const llm = new ChatOpenAI({ model: "gpt-4o", tools: yourToolSchemas });

async function agentTurn(
session: any,
userInput: string,
messageHistory: (HumanMessage | AIMessage)[]
): Promise<string> {
const ctx = await session.context({ query: userInput });

const messages = [
  new SystemMessage(buildSystemPrompt(ctx)),
  ...messageHistory,
  new HumanMessage(userInput),
];

// Run the agent's full tool-calling loop on your side, then surface
// every intermediate message (assistant tool_calls + tool results)
// to Sonzai so it can extract from them.
const { reply, intermediate } = await runLangchainAgent(llm, messages);

await session.turn({
  messages: [
    { role: "user", content: userInput },
    ...intermediate,
    { role: "assistant", content: reply },
  ],
});

return reply;
}

import Anthropic from "@anthropic-ai/sdk";
import { GoogleGenAI } from "@google/genai";
import { Sonzai } from "@sonzai-labs/agents";

const sonzai = new Sonzai({ apiKey: process.env.SONZAI_API_KEY! });
const claude = new Anthropic();
const gemini = new GoogleGenAI({ apiKey: process.env.GEMINI_API_KEY! });

type TaskType = "creative" | "analytical" | "casual";

function classifyTask(message: string): TaskType {
if (/write|story|poem|imagine/i.test(message)) return "creative";
if (/analyze|compare|explain|why/i.test(message)) return "analytical";
return "casual";
}

async function routedTurn(session: any, userMessage: string): Promise<string> {
const ctx = await session.context({ query: userMessage });
const systemPrompt = buildSystemPrompt(ctx);
const task = classifyTask(userMessage);

let reply: string;

if (task === "creative") {
  const response = await claude.messages.create({
    model: "claude-sonnet-4-6",
    max_tokens: 1024,
    system: systemPrompt,
    messages: [{ role: "user", content: userMessage }],
  });
  reply = response.content[0].type === "text" ? response.content[0].text : "";
} else {
  const response = await gemini.models.generateContent({
    model: "gemini-2.5-flash",
    contents: [{ role: "user", parts: [{ text: systemPrompt + "\n\n" + userMessage }] }],
  });
  reply = response.text ?? "";
}

// Same .turn() call regardless of which chat model answered.
await session.turn({
  messages: [
    { role: "user", content: userMessage },
    { role: "assistant", content: reply },
  ],
});

return reply;
}

async function privacyTurn(session: any, userMessage: string): Promise<string> {
const ctx = await session.context({ query: userMessage });

// Scrub PII from facts before they reach your LLM
const sanitizedFacts = (ctx.loaded_facts ?? []).map((f: any) => ({
  ...f,
  atomic_text: redactPII(f.atomic_text), // your PII redaction logic
}));

const sanitizedCtx = { ...ctx, loaded_facts: sanitizedFacts };
const systemPrompt = buildSystemPrompt(sanitizedCtx);

const reply = await yourLLM.chat({ system: systemPrompt, message: userMessage });

// Send unredacted transcript to Sonzai for extraction
// (Sonzai stores structured facts, not raw text)
await session.turn({
  messages: [
    { role: "user", content: userMessage },
    { role: "assistant", content: reply },
  ],
});

return reply;
}