Route all chat through a local LiteLLM gateway, drop command prefixes
- New litellm service (pinned v1.98.0 — litellm 1.82.7/1.82.8 on PyPI were compromised with credential-stealing malware in March 2026; internal-only, no Traefik route, no reason to expose an LLM gateway with a master key publicly). - Replaces !claude/!ai command prefixes with automatic routing: every plain message in the control room goes through a classifier (router.js) that decides chat vs code_task. Chat replies use OpenRouter's own auto-router (openrouter/auto) via LiteLLM; code_task requests go through the existing runChatTask() flow (Claude Code CLI, unchanged, still using the subscription token directly). - Investigated routing Claude itself through LiteLLM via OAuth token forwarding (general_settings.forward_client_headers_to_llm_api) so the Pro/Max subscription could be one of the auto-routable options. Confirmed non-functional: Anthropic returns a generic rate_limit_error for any direct API call using this token type outside the real Claude Code CLI, reproduced with plain curl straight to api.anthropic.com. Not included. - MATRIX_BOT_USER_ID now required and set explicitly (self-message filtering can no longer rely on a command-prefix mismatch once there isn't one).
This commit is contained in:
@@ -0,0 +1,37 @@
|
||||
const LITELLM_BASE_URL = process.env.LITELLM_BASE_URL || "http://litellm:4000";
|
||||
const LITELLM_MASTER_KEY = process.env.LITELLM_MASTER_KEY;
|
||||
|
||||
// OpenAI-compatible chat completion, for OpenRouter-backed models routed through the
|
||||
// local LiteLLM gateway (e.g. "auto" — OpenRouter's own prompt-aware auto-router).
|
||||
export async function chatCompletion(model, prompt) {
|
||||
if (!LITELLM_MASTER_KEY) {
|
||||
throw new Error("LITELLM_MASTER_KEY is not set");
|
||||
}
|
||||
|
||||
const res = await fetch(`${LITELLM_BASE_URL}/v1/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${LITELLM_MASTER_KEY}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
messages: [{ role: "user", content: prompt }],
|
||||
// Some models default max_tokens to their full context window (e.g. 65536), which
|
||||
// can exceed available credit balance before a single token is generated. This is a
|
||||
// quick chat reply, not a long-form task — cap it.
|
||||
max_tokens: 1024,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`LiteLLM request failed: ${res.status} ${await res.text()}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
const content = data.choices?.[0]?.message?.content;
|
||||
if (!content) {
|
||||
throw new Error(`LiteLLM returned no content: ${JSON.stringify(data)}`);
|
||||
}
|
||||
return content;
|
||||
}
|
||||
+29
-55
@@ -1,37 +1,23 @@
|
||||
import { MatrixClient, SimpleFsStorageProvider } from "matrix-bot-sdk";
|
||||
import { runChatTask } from "./runner.js";
|
||||
import { askOpenRouter, DEFAULT_MODEL } from "./openrouter.js";
|
||||
import { routeMessage, chatReply } from "./router.js";
|
||||
|
||||
const HOMESERVER_URL = process.env.MATRIX_HOMESERVER_URL;
|
||||
const ACCESS_TOKEN = process.env.MATRIX_BOT_TOKEN;
|
||||
const CONTROL_ROOM_ID = process.env.MATRIX_CONTROL_ROOM_ID;
|
||||
const GITEA_URL = process.env.GITEA_URL;
|
||||
// Set explicitly rather than fetched via client.getUserId() — that call hits /whoami,
|
||||
// which (like /joined_rooms before it) 404s against Continuwuity for reasons unrelated
|
||||
// to the endpoint itself. This is also the only reliable way to filter the bot's own
|
||||
// messages now that there's no command prefix to naturally exclude them by.
|
||||
const BOT_USER_ID = process.env.MATRIX_BOT_USER_ID;
|
||||
const KNOWN_REPOS = (process.env.KNOWN_REPOS || "")
|
||||
.split(",")
|
||||
.map((r) => r.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
const MAX_REPLY_LENGTH = 4000;
|
||||
|
||||
// "!claude owner/repo do the thing" — everything after the repo slug is the instruction.
|
||||
function parseClaudeCommand(text) {
|
||||
const match = text.match(/^!claude\s+([^\s/]+\/[^\s/]+)\s+(.+)$/s);
|
||||
if (!match) return null;
|
||||
const [, repoFullName, instruction] = match;
|
||||
return { repoFullName, instruction: instruction.trim() };
|
||||
}
|
||||
|
||||
// "!ai <prompt>" uses the default model. "!ai provider/model <prompt>" (first token
|
||||
// contains a "/") picks a specific OpenRouter model, e.g. "!ai google/gemini-2.0-flash-001
|
||||
// summarize this repo's README".
|
||||
function parseAiCommand(text) {
|
||||
const match = text.match(/^!ai\s+(.+)$/s);
|
||||
if (!match) return null;
|
||||
const rest = match[1].trim();
|
||||
const firstSpace = rest.search(/\s/);
|
||||
const firstToken = firstSpace === -1 ? rest : rest.slice(0, firstSpace);
|
||||
if (firstToken.includes("/") && firstSpace !== -1) {
|
||||
return { model: firstToken, prompt: rest.slice(firstSpace + 1).trim() };
|
||||
}
|
||||
return { model: DEFAULT_MODEL, prompt: rest };
|
||||
}
|
||||
|
||||
function truncate(text) {
|
||||
if (text.length <= MAX_REPLY_LENGTH) return text;
|
||||
return `${text.slice(0, MAX_REPLY_LENGTH)}\n\n[truncated]`;
|
||||
@@ -42,15 +28,14 @@ export async function startMatrixBot() {
|
||||
console.warn("Matrix env vars not set — skipping bot startup");
|
||||
return;
|
||||
}
|
||||
if (!BOT_USER_ID) {
|
||||
console.warn("MATRIX_BOT_USER_ID not set — bot could reply to its own messages, skipping startup");
|
||||
return;
|
||||
}
|
||||
|
||||
const storage = new SimpleFsStorageProvider("/workspace/matrix-bot-storage.json");
|
||||
const client = new MatrixClient(HOMESERVER_URL, ACCESS_TOKEN, storage);
|
||||
|
||||
// Not using AutojoinRoomsMixin: it calls /joined_rooms at startup to build its initial
|
||||
// state, which — like the /whoami call removed earlier — 404s against Continuwuity for
|
||||
// reasons unrelated to the endpoint itself (curling it directly works fine). This
|
||||
// simpler handler does the one thing we actually need — auto-join on invite — without
|
||||
// that startup scan.
|
||||
client.on("room.invite", async (roomId) => {
|
||||
try {
|
||||
await client.joinRoom(roomId);
|
||||
@@ -60,38 +45,27 @@ export async function startMatrixBot() {
|
||||
});
|
||||
|
||||
client.on("room.message", async (roomId, event) => {
|
||||
// Invite-only control room enforces who can reach the bot at all; this just scopes
|
||||
// command handling to that one room. No need to fetch/compare the bot's own user ID
|
||||
// to filter out its own messages — its replies never match either command pattern
|
||||
// below, so they're ignored the same as any other non-command message.
|
||||
if (roomId !== CONTROL_ROOM_ID) return;
|
||||
if (event.sender === BOT_USER_ID) return;
|
||||
const body = event.content?.body;
|
||||
if (!body) return;
|
||||
|
||||
const aiCmd = parseAiCommand(body);
|
||||
if (aiCmd) {
|
||||
try {
|
||||
const reply = await askOpenRouter(aiCmd.model, aiCmd.prompt);
|
||||
await client.sendText(roomId, `[${aiCmd.model}] ${truncate(reply)}`);
|
||||
} catch (err) {
|
||||
console.error("openrouter query failed", err);
|
||||
await client.sendText(roomId, `Failed: ${err.message}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const cmd = parseClaudeCommand(body);
|
||||
if (!cmd) return;
|
||||
|
||||
const [owner, repo] = cmd.repoFullName.split("/");
|
||||
await client.sendText(roomId, `Working on it: ${cmd.repoFullName} — ${cmd.instruction}`);
|
||||
|
||||
try {
|
||||
const cloneUrl = `${GITEA_URL}/${owner}/${repo}.git`;
|
||||
const pr = await runChatTask({ owner, repo, cloneUrl, instruction: cmd.instruction });
|
||||
await client.sendText(roomId, `Opened PR: ${pr.html_url}`);
|
||||
const decision = await routeMessage(body, KNOWN_REPOS);
|
||||
|
||||
if (decision.type === "code_task") {
|
||||
const [owner, repo] = decision.repo.split("/");
|
||||
await client.sendText(roomId, `Working on it: ${decision.repo} — ${decision.instruction}`);
|
||||
const cloneUrl = `${GITEA_URL}/${owner}/${repo}.git`;
|
||||
const pr = await runChatTask({ owner, repo, cloneUrl, instruction: decision.instruction });
|
||||
await client.sendText(roomId, `Opened PR: ${pr.html_url}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const reply = await chatReply(body);
|
||||
await client.sendText(roomId, truncate(reply));
|
||||
} catch (err) {
|
||||
console.error("chat task failed", err);
|
||||
console.error("message handling failed", err);
|
||||
await client.sendText(roomId, `Failed: ${err.message}`);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY;
|
||||
const DEFAULT_MODEL = process.env.OPENROUTER_DEFAULT_MODEL || "openai/gpt-4o-mini";
|
||||
|
||||
export async function askOpenRouter(model, prompt) {
|
||||
if (!OPENROUTER_API_KEY) {
|
||||
throw new Error("OPENROUTER_API_KEY is not set");
|
||||
}
|
||||
|
||||
const res = await fetch("https://openrouter.ai/api/v1/chat/completions", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${OPENROUTER_API_KEY}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: model || DEFAULT_MODEL,
|
||||
messages: [{ role: "user", content: prompt }],
|
||||
// Some models default max_tokens to their full context window (e.g. 65536),
|
||||
// which can exceed available credit balance before a single token is generated.
|
||||
// This is a quick chat reply, not a long-form task — cap it.
|
||||
max_tokens: 1024,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`OpenRouter request failed: ${res.status} ${await res.text()}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
const content = data.choices?.[0]?.message?.content;
|
||||
if (!content) {
|
||||
throw new Error(`OpenRouter returned no content: ${JSON.stringify(data)}`);
|
||||
}
|
||||
return content;
|
||||
}
|
||||
|
||||
export { DEFAULT_MODEL };
|
||||
@@ -0,0 +1,45 @@
|
||||
import { chatCompletion } from "./litellm.js";
|
||||
|
||||
const ROUTER_MODEL = "router-classifier";
|
||||
const CHAT_MODEL = "auto";
|
||||
|
||||
function systemPrompt(knownRepos) {
|
||||
return [
|
||||
"You are a routing classifier for a chat bot. Given a user message, decide whether it is:",
|
||||
'- "chat": a question, discussion, or anything that just needs a text reply.',
|
||||
'- "code_task": a request to change a specific code repository (add/edit/fix something)',
|
||||
" where the repository is clearly one of the known repositories below.",
|
||||
"",
|
||||
`Known repositories: ${knownRepos.join(", ") || "(none configured)"}`,
|
||||
"",
|
||||
"Reply with ONLY a JSON object, nothing else:",
|
||||
'{"type":"chat"}',
|
||||
'or',
|
||||
'{"type":"code_task","repo":"owner/repo","instruction":"clear imperative instruction"}',
|
||||
"",
|
||||
"If it sounds like a code change but you can't confidently match it to one of the known",
|
||||
'repositories, reply {"type":"chat"} instead of guessing.',
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function parseDecision(raw) {
|
||||
try {
|
||||
const cleaned = raw.trim().replace(/^```(?:json)?\n?/, "").replace(/```$/, "");
|
||||
const parsed = JSON.parse(cleaned);
|
||||
if (parsed.type === "code_task" && parsed.repo && parsed.instruction) {
|
||||
return parsed;
|
||||
}
|
||||
} catch {
|
||||
// fall through to chat — an unparseable classification is not a reason to edit a repo
|
||||
}
|
||||
return { type: "chat" };
|
||||
}
|
||||
|
||||
export async function routeMessage(text, knownRepos) {
|
||||
const raw = await chatCompletion(ROUTER_MODEL, `${systemPrompt(knownRepos)}\n\nMessage: ${text}`);
|
||||
return parseDecision(raw);
|
||||
}
|
||||
|
||||
export async function chatReply(text) {
|
||||
return chatCompletion(CHAT_MODEL, text);
|
||||
}
|
||||
Reference in New Issue
Block a user