Merge pull request 'Route chat through local LiteLLM gateway, drop !claude/!ai prefixes' (#8) from feat/litellm-router into main

Reviewed-on: #8

.env.example

@@ -25,10 +25,20 @@ CLAUDE_CODE_OAUTH_TOKEN=
 MATRIX_HOMESERVER_URL=https://matrix.apps.williamturner.eu
 MATRIX_BOT_TOKEN=
 MATRIX_CONTROL_ROOM_ID=
+# The bot's own Matrix ID (@username:server), e.g. @claude-bot:matrix.apps.williamturner.eu
+# — set explicitly rather than fetched via the API (that call 404s against Continuwuity).
+# Required: without it the bot can't tell its own messages apart from real ones and would
+# reply to itself in a loop, so it refuses to start.
+MATRIX_BOT_USER_ID=
+# Comma-separated "owner/repo" list the chat router is allowed to open code-change PRs
+# against. A plain chat message mentioning a repo NOT in this list is treated as chat,
+# never as a code task — the router only matches confidently against known repos.
+KNOWN_REPOS=william/gitops-automation
 
-# --- openrouter (the "!ai" chat command — other models, not the coding agent) ---
+# --- litellm (local LLM gateway — see litellm-config.yaml) ---
 OPENROUTER_API_KEY=
-OPENROUTER_DEFAULT_MODEL=openai/gpt-4o-mini
+# Any random string; also used as litellm's general_settings.master_key.
+LITELLM_MASTER_KEY=
 
 # --- portainer (GitOps redeploy) ---
 PORTAINER_STACK_WEBHOOK_URL=

agent/src/litellm.js

@@ -0,0 +1,37 @@
+const LITELLM_BASE_URL = process.env.LITELLM_BASE_URL || "http://litellm:4000";
+const LITELLM_MASTER_KEY = process.env.LITELLM_MASTER_KEY;
+
+// OpenAI-compatible chat completion, for OpenRouter-backed models routed through the
+// local LiteLLM gateway (e.g. "auto" — OpenRouter's own prompt-aware auto-router).
+export async function chatCompletion(model, prompt) {
+  if (!LITELLM_MASTER_KEY) {
+    throw new Error("LITELLM_MASTER_KEY is not set");
+  }
+
+  const res = await fetch(`${LITELLM_BASE_URL}/v1/chat/completions`, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${LITELLM_MASTER_KEY}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      model,
+      messages: [{ role: "user", content: prompt }],
+      // Some models default max_tokens to their full context window (e.g. 65536), which
+      // can exceed available credit balance before a single token is generated. This is a
+      // quick chat reply, not a long-form task — cap it.
+      max_tokens: 1024,
+    }),
+  });
+
+  if (!res.ok) {
+    throw new Error(`LiteLLM request failed: ${res.status} ${await res.text()}`);
+  }
+
+  const data = await res.json();
+  const content = data.choices?.[0]?.message?.content;
+  if (!content) {
+    throw new Error(`LiteLLM returned no content: ${JSON.stringify(data)}`);
+  }
+  return content;
+}

agent/src/matrixBot.js

@@ -1,37 +1,23 @@
 import { MatrixClient, SimpleFsStorageProvider } from "matrix-bot-sdk";
 import { runChatTask } from "./runner.js";
-import { askOpenRouter, DEFAULT_MODEL } from "./openrouter.js";
+import { routeMessage, chatReply } from "./router.js";
 
 const HOMESERVER_URL = process.env.MATRIX_HOMESERVER_URL;
 const ACCESS_TOKEN = process.env.MATRIX_BOT_TOKEN;
 const CONTROL_ROOM_ID = process.env.MATRIX_CONTROL_ROOM_ID;
 const GITEA_URL = process.env.GITEA_URL;
+// Set explicitly rather than fetched via client.getUserId() — that call hits /whoami,
+// which (like /joined_rooms before it) 404s against Continuwuity for reasons unrelated
+// to the endpoint itself. This is also the only reliable way to filter the bot's own
+// messages now that there's no command prefix to naturally exclude them by.
+const BOT_USER_ID = process.env.MATRIX_BOT_USER_ID;
+const KNOWN_REPOS = (process.env.KNOWN_REPOS || "")
+  .split(",")
+  .map((r) => r.trim())
+  .filter(Boolean);
 
 const MAX_REPLY_LENGTH = 4000;
 
-// "!claude owner/repo do the thing" — everything after the repo slug is the instruction.
-function parseClaudeCommand(text) {
-  const match = text.match(/^!claude\s+([^\s/]+\/[^\s/]+)\s+(.+)$/s);
-  if (!match) return null;
-  const [, repoFullName, instruction] = match;
-  return { repoFullName, instruction: instruction.trim() };
-}
-
-// "!ai <prompt>" uses the default model. "!ai provider/model <prompt>" (first token
-// contains a "/") picks a specific OpenRouter model, e.g. "!ai google/gemini-2.0-flash-001
-// summarize this repo's README".
-function parseAiCommand(text) {
-  const match = text.match(/^!ai\s+(.+)$/s);
-  if (!match) return null;
-  const rest = match[1].trim();
-  const firstSpace = rest.search(/\s/);
-  const firstToken = firstSpace === -1 ? rest : rest.slice(0, firstSpace);
-  if (firstToken.includes("/") && firstSpace !== -1) {
-    return { model: firstToken, prompt: rest.slice(firstSpace + 1).trim() };
-  }
-  return { model: DEFAULT_MODEL, prompt: rest };
-}
-
 function truncate(text) {
   if (text.length <= MAX_REPLY_LENGTH) return text;
   return `${text.slice(0, MAX_REPLY_LENGTH)}\n\n[truncated]`;
@@ -42,15 +28,14 @@ export async function startMatrixBot() {
     console.warn("Matrix env vars not set — skipping bot startup");
     return;
   }
+  if (!BOT_USER_ID) {
+    console.warn("MATRIX_BOT_USER_ID not set — bot could reply to its own messages, skipping startup");
+    return;
+  }
 
   const storage = new SimpleFsStorageProvider("/workspace/matrix-bot-storage.json");
   const client = new MatrixClient(HOMESERVER_URL, ACCESS_TOKEN, storage);
 
-  // Not using AutojoinRoomsMixin: it calls /joined_rooms at startup to build its initial
-  // state, which — like the /whoami call removed earlier — 404s against Continuwuity for
-  // reasons unrelated to the endpoint itself (curling it directly works fine). This
-  // simpler handler does the one thing we actually need — auto-join on invite — without
-  // that startup scan.
   client.on("room.invite", async (roomId) => {
     try {
       await client.joinRoom(roomId);
@@ -60,38 +45,27 @@ export async function startMatrixBot() {
   });
 
   client.on("room.message", async (roomId, event) => {
-    // Invite-only control room enforces who can reach the bot at all; this just scopes
-    // command handling to that one room. No need to fetch/compare the bot's own user ID
-    // to filter out its own messages — its replies never match either command pattern
-    // below, so they're ignored the same as any other non-command message.
     if (roomId !== CONTROL_ROOM_ID) return;
+    if (event.sender === BOT_USER_ID) return;
     const body = event.content?.body;
     if (!body) return;
 
-    const aiCmd = parseAiCommand(body);
-    if (aiCmd) {
-      try {
-        const reply = await askOpenRouter(aiCmd.model, aiCmd.prompt);
-        await client.sendText(roomId, `[${aiCmd.model}] ${truncate(reply)}`);
-      } catch (err) {
-        console.error("openrouter query failed", err);
-        await client.sendText(roomId, `Failed: ${err.message}`);
-      }
-      return;
-    }
-
-    const cmd = parseClaudeCommand(body);
-    if (!cmd) return;
-
-    const [owner, repo] = cmd.repoFullName.split("/");
-    await client.sendText(roomId, `Working on it: ${cmd.repoFullName} — ${cmd.instruction}`);
-
     try {
-      const cloneUrl = `${GITEA_URL}/${owner}/${repo}.git`;
+      const decision = await routeMessage(body, KNOWN_REPOS);
-      const pr = await runChatTask({ owner, repo, cloneUrl, instruction: cmd.instruction });
+
-      await client.sendText(roomId, `Opened PR: ${pr.html_url}`);
+      if (decision.type === "code_task") {
+        const [owner, repo] = decision.repo.split("/");
+        await client.sendText(roomId, `Working on it: ${decision.repo} — ${decision.instruction}`);
+        const cloneUrl = `${GITEA_URL}/${owner}/${repo}.git`;
+        const pr = await runChatTask({ owner, repo, cloneUrl, instruction: decision.instruction });
+        await client.sendText(roomId, `Opened PR: ${pr.html_url}`);
+        return;
+      }
+
+      const reply = await chatReply(body);
+      await client.sendText(roomId, truncate(reply));
     } catch (err) {
-      console.error("chat task failed", err);
+      console.error("message handling failed", err);
       await client.sendText(roomId, `Failed: ${err.message}`);
     }
   });

agent/src/openrouter.js

@@ -1,37 +0,0 @@
-const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY;
-const DEFAULT_MODEL = process.env.OPENROUTER_DEFAULT_MODEL || "openai/gpt-4o-mini";
-
-export async function askOpenRouter(model, prompt) {
-  if (!OPENROUTER_API_KEY) {
-    throw new Error("OPENROUTER_API_KEY is not set");
-  }
-
-  const res = await fetch("https://openrouter.ai/api/v1/chat/completions", {
-    method: "POST",
-    headers: {
-      Authorization: `Bearer ${OPENROUTER_API_KEY}`,
-      "Content-Type": "application/json",
-    },
-    body: JSON.stringify({
-      model: model || DEFAULT_MODEL,
-      messages: [{ role: "user", content: prompt }],
-      // Some models default max_tokens to their full context window (e.g. 65536),
-      // which can exceed available credit balance before a single token is generated.
-      // This is a quick chat reply, not a long-form task — cap it.
-      max_tokens: 1024,
-    }),
-  });
-
-  if (!res.ok) {
-    throw new Error(`OpenRouter request failed: ${res.status} ${await res.text()}`);
-  }
-
-  const data = await res.json();
-  const content = data.choices?.[0]?.message?.content;
-  if (!content) {
-    throw new Error(`OpenRouter returned no content: ${JSON.stringify(data)}`);
-  }
-  return content;
-}
-
-export { DEFAULT_MODEL };

agent/src/router.js

@@ -0,0 +1,45 @@
+import { chatCompletion } from "./litellm.js";
+
+const ROUTER_MODEL = "router-classifier";
+const CHAT_MODEL = "auto";
+
+function systemPrompt(knownRepos) {
+  return [
+    "You are a routing classifier for a chat bot. Given a user message, decide whether it is:",
+    '- "chat": a question, discussion, or anything that just needs a text reply.',
+    '- "code_task": a request to change a specific code repository (add/edit/fix something)',
+    "  where the repository is clearly one of the known repositories below.",
+    "",
+    `Known repositories: ${knownRepos.join(", ") || "(none configured)"}`,
+    "",
+    "Reply with ONLY a JSON object, nothing else:",
+    '{"type":"chat"}',
+    'or',
+    '{"type":"code_task","repo":"owner/repo","instruction":"clear imperative instruction"}',
+    "",
+    "If it sounds like a code change but you can't confidently match it to one of the known",
+    'repositories, reply {"type":"chat"} instead of guessing.',
+  ].join("\n");
+}
+
+function parseDecision(raw) {
+  try {
+    const cleaned = raw.trim().replace(/^```(?:json)?\n?/, "").replace(/```$/, "");
+    const parsed = JSON.parse(cleaned);
+    if (parsed.type === "code_task" && parsed.repo && parsed.instruction) {
+      return parsed;
+    }
+  } catch {
+    // fall through to chat — an unparseable classification is not a reason to edit a repo
+  }
+  return { type: "chat" };
+}
+
+export async function routeMessage(text, knownRepos) {
+  const raw = await chatCompletion(ROUTER_MODEL, `${systemPrompt(knownRepos)}\n\nMessage: ${text}`);
+  return parseDecision(raw);
+}
+
+export async function chatReply(text) {
+  return chatCompletion(CHAT_MODEL, text);
+}

docker-compose.yml

@@ -36,8 +36,35 @@ services:
       - "traefik.http.routers.matrix.tls.certresolver=letsencrypt"
       - "traefik.http.services.matrix.loadbalancer.server.port=8008"
 
+  litellm:
+    # Pinned deliberately, not :latest or :main-latest — litellm==1.82.7/1.82.8 on PyPI
+    # were compromised with credential-stealing malware in March 2026 (fixed within the
+    # hour, but a floating tag could still land on a bad release in the future). v1.98.0
+    # verified clean as of this writing.
+    image: ghcr.io/berriai/litellm:v1.98.0
+    container_name: litellm
+    restart: unless-stopped
+    environment:
+      OPENROUTER_API_KEY: ${OPENROUTER_API_KEY}
+      LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY}
+    volumes:
+      # Absolute host path, NOT a repo-relative one — Portainer's git-stack deploy clones
+      # into its own directory (/data/compose/N/) whose checkout doesn't reliably persist
+      # for the container's runtime (see the act_runner config comment below for the same
+      # failure mode). An absolute path on the actual host filesystem always resolves the
+      # same way regardless of which tool ran `docker compose up`. Keep this local clone
+      # (/home/william/gitops-automation) pulled to latest when the config changes.
+      - /home/william/gitops-automation/litellm-config.yaml:/app/config.yaml:ro
+    command: ["--config", "/app/config.yaml", "--port", "4000"]
+    networks:
+      - web
+    # Internal only — no Traefik labels. No reason to expose an LLM gateway holding a
+    # master key and OAuth-forwarding config to the public internet.
+
   claude-agent:
     image: ${GITEA_REGISTRY_IMAGE}
+    depends_on:
+      - litellm
     container_name: claude-agent
     restart: unless-stopped
     # Explicit vars, not env_file: .env — Portainer's git-based stack deploy clones the
@@ -53,8 +80,13 @@ services:
       MATRIX_HOMESERVER_URL: ${MATRIX_HOMESERVER_URL}
       MATRIX_BOT_TOKEN: ${MATRIX_BOT_TOKEN}
       MATRIX_CONTROL_ROOM_ID: ${MATRIX_CONTROL_ROOM_ID}
-      OPENROUTER_API_KEY: ${OPENROUTER_API_KEY}
+      MATRIX_BOT_USER_ID: ${MATRIX_BOT_USER_ID}
-      OPENROUTER_DEFAULT_MODEL: ${OPENROUTER_DEFAULT_MODEL:-openai/gpt-4o-mini}
+      KNOWN_REPOS: ${KNOWN_REPOS}
+      # All model calls now go through the local litellm service, not OpenRouter directly —
+      # one gateway for OpenRouter's models (incl. its auto-router) and, for the
+      # claude-subscription route, Anthropic itself via the forwarded OAuth token above.
+      LITELLM_BASE_URL: http://litellm:4000
+      LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY}
     volumes:
       - agent_workspace:/workspace
     networks:

litellm-config.yaml

@@ -0,0 +1,23 @@
+model_list:
+  # General chat — OpenRouter's own auto-router picks the best underlying model per prompt.
+  - model_name: auto
+    litellm_params:
+      model: openrouter/openrouter/auto
+      api_key: os.environ/OPENROUTER_API_KEY
+
+  # Cheap/fast model used by the agent's own chat-vs-code-task classifier, not by users directly.
+  - model_name: router-classifier
+    litellm_params:
+      model: openrouter/openai/gpt-4o-mini
+      api_key: os.environ/OPENROUTER_API_KEY
+
+  # NOT included: a "claude-subscription" route forwarding the Claude Pro/Max OAuth token
+  # (from `claude setup-token`) through to Anthropic's raw API. Tested and confirmed
+  # non-functional — Anthropic returns a generic rate_limit_error for ANY direct API call
+  # using this token type outside the real Claude Code CLI client (reproduced with plain
+  # curl straight to api.anthropic.com, bypassing LiteLLM entirely, same result). The
+  # subscription token only works through the actual Claude Code CLI, which is what
+  # claude-agent already uses directly for code tasks — it was never routed through here.
+
+general_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY