Initial human archive app

src/lib/moderation.ts

@@ -0,0 +1,111 @@
+import { runLlmChat } from "@/lib/litellm";
+import type { ModerationResult, SourceInput } from "@/lib/types";
+
+export const MODERATION_PROMPT_VERSION = "v1-source-required";
+
+type ModerationInput = {
+  title: string;
+  content: unknown;
+  sources: SourceInput[];
+};
+
+export async function moderateContribution(input: ModerationInput): Promise<ModerationResult> {
+  const sourceRisks = validateSources(input.sources);
+  if (sourceRisks.length > 0) {
+    return {
+      outcome: "REJECT",
+      riskCategories: sourceRisks,
+      confidence: 0.98,
+      reasoning: "Der Beitrag verletzt die Quellenpflicht oder enthält nicht verwertbare Quellen.",
+      model: "rules:v1",
+      promptVersion: MODERATION_PROMPT_VERSION,
+    };
+  }
+
+  const llmResult = await runLlmChat([
+    {
+      role: "system",
+      content:
+        "Du moderierst Enzyklopaedie-Beitraege. Antworte als JSON mit outcome APPROVE, REJECT oder ESCALATE, riskCategories, confidence und reasoning.",
+    },
+    {
+      role: "user",
+      content: JSON.stringify({
+        title: input.title,
+        content: input.content,
+        sourceCount: input.sources.length,
+        sources: input.sources.map((source) => ({
+          title: source.title,
+          url: source.url,
+          license: source.license,
+          trustLevel: source.trustLevel,
+        })),
+      }),
+    },
+  ]);
+
+  const parsed = parseModerationJson(llmResult.content);
+  return {
+    outcome: parsed.outcome,
+    riskCategories: parsed.riskCategories,
+    confidence: parsed.confidence,
+    reasoning: parsed.reasoning,
+    model: llmResult.model,
+    promptVersion: MODERATION_PROMPT_VERSION,
+    rawResponse: llmResult.raw ?? llmResult.content,
+  };
+}
+
+export function validateSources(sources: SourceInput[]) {
+  const risks: string[] = [];
+
+  if (sources.length === 0) {
+    risks.push("missing_sources");
+  }
+
+  for (const source of sources) {
+    if (!source.title.trim()) {
+      risks.push("source_missing_title");
+    }
+    if (!isHttpUrl(source.url)) {
+      risks.push("source_invalid_url");
+    }
+    if (!source.license?.trim()) {
+      risks.push("source_missing_license");
+    }
+  }
+
+  return Array.from(new Set(risks));
+}
+
+function parseModerationJson(content: string): Pick<ModerationResult, "outcome" | "riskCategories" | "confidence" | "reasoning"> {
+  try {
+    const jsonStart = content.indexOf("{");
+    const jsonEnd = content.lastIndexOf("}");
+    const parsed = JSON.parse(content.slice(jsonStart, jsonEnd + 1)) as Partial<ModerationResult>;
+    const outcome = parsed.outcome === "REJECT" || parsed.outcome === "ESCALATE" ? parsed.outcome : "APPROVE";
+
+    return {
+      outcome,
+      riskCategories: Array.isArray(parsed.riskCategories) ? parsed.riskCategories.map(String) : [],
+      confidence: typeof parsed.confidence === "number" ? Math.min(Math.max(parsed.confidence, 0), 1) : 0.72,
+      reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : "KI-Moderation abgeschlossen.",
+    };
+  } catch {
+    return {
+      outcome: "ESCALATE",
+      riskCategories: ["moderation_parse_error"],
+      confidence: 0.4,
+      reasoning: "Die KI-Moderation konnte nicht eindeutig geparst werden und wird zur Admin-Pruefung eskaliert.",
+    };
+  }
+}
+
+function isHttpUrl(value: string) {
+  try {
+    const url = new URL(value);
+    return url.protocol === "http:" || url.protocol === "https:";
+  } catch {
+    return false;
+  }
+}