MrSphay
f29339cf12
Some checks failed
Build and publish Docker image / docker (push) Failing after 2m33s
69 lines
1.7 KiB
TypeScript
69 lines
1.7 KiB
TypeScript
import { env } from "@/lib/env";
|
|
|
|
type ChatMessage = {
|
|
role: "system" | "user" | "assistant";
|
|
content: string;
|
|
};
|
|
|
|
type ChatCompletion = {
|
|
choices?: Array<{
|
|
message?: {
|
|
content?: string;
|
|
};
|
|
}>;
|
|
usage?: {
|
|
total_tokens?: number;
|
|
};
|
|
};
|
|
|
|
export type LlmResult = {
|
|
content: string;
|
|
model: string;
|
|
tokenCount?: number;
|
|
raw?: unknown;
|
|
};
|
|
|
|
export async function runLlmChat(messages: ChatMessage[], model = env.DEFAULT_LLM_MODEL): Promise<LlmResult> {
|
|
if (!env.LITELLM_BASE_URL || !env.LITELLM_API_KEY) {
|
|
return {
|
|
content: localFallback(messages),
|
|
model: `${model}:fallback`,
|
|
};
|
|
}
|
|
|
|
const response = await fetch(`${env.LITELLM_BASE_URL.replace(/\/$/, "")}/chat/completions`, {
|
|
method: "POST",
|
|
headers: {
|
|
"content-type": "application/json",
|
|
authorization: `Bearer ${env.LITELLM_API_KEY}`,
|
|
},
|
|
body: JSON.stringify({
|
|
model,
|
|
messages,
|
|
temperature: 0.2,
|
|
}),
|
|
});
|
|
|
|
if (!response.ok) {
|
|
const errorBody = await response.text();
|
|
throw new Error(`LiteLLM request failed: ${response.status} ${errorBody}`);
|
|
}
|
|
|
|
const data = (await response.json()) as ChatCompletion;
|
|
return {
|
|
content: data.choices?.[0]?.message?.content?.trim() ?? "",
|
|
model,
|
|
tokenCount: data.usage?.total_tokens,
|
|
raw: data,
|
|
};
|
|
}
|
|
|
|
function localFallback(messages: ChatMessage[]) {
|
|
const latestUserMessage = [...messages].reverse().find((message) => message.role === "user")?.content ?? "";
|
|
return [
|
|
"Lokaler Fallback: Es ist kein LiteLLM-Endpunkt konfiguriert.",
|
|
"Die Anfrage wurde strukturiert angenommen und muss nach Konfiguration eines Modells erneut ausgeführt werden.",
|
|
`Auszug der Anfrage: ${latestUserMessage.slice(0, 420)}`,
|
|
].join("\n\n");
|
|
}
|