human-archive/src/lib/moderation.ts

import { runLlmChat } from "@/lib/litellm";
import type { ModerationResult, SourceInput } from "@/lib/types";

export const MODERATION_PROMPT_VERSION = "v1-source-required";

type ModerationInput = {
  title: string;
  content: unknown;
  sources: SourceInput[];
};

export async function moderateContribution(input: ModerationInput): Promise<ModerationResult> {
  const sourceRisks = validateSources(input.sources);
  if (sourceRisks.length > 0) {
    return {
      outcome: "REJECT",
      riskCategories: sourceRisks,
      confidence: 0.98,
      reasoning: "Der Beitrag verletzt die Quellenpflicht oder enthält nicht verwertbare Quellen.",
      model: "rules:v1",
      promptVersion: MODERATION_PROMPT_VERSION,
    };
  }

  const llmResult = await runLlmChat([
    {
      role: "system",
      content:
        "Du moderierst Enzyklopaedie-Beitraege. Antworte als JSON mit outcome APPROVE, REJECT oder ESCALATE, riskCategories, confidence und reasoning.",
    },
    {
      role: "user",
      content: JSON.stringify({
        title: input.title,
        content: input.content,
        sourceCount: input.sources.length,
        sources: input.sources.map((source) => ({
          title: source.title,
          url: source.url,
          license: source.license,
          trustLevel: source.trustLevel,
        })),
      }),
    },
  ]);

  const parsed = parseModerationJson(llmResult.content);
  return {
    outcome: parsed.outcome,
    riskCategories: parsed.riskCategories,
    confidence: parsed.confidence,
    reasoning: parsed.reasoning,
    model: llmResult.model,
    promptVersion: MODERATION_PROMPT_VERSION,
    rawResponse: llmResult.raw ?? llmResult.content,
  };
}

export function validateSources(sources: SourceInput[]) {
  const risks: string[] = [];

  if (sources.length === 0) {
    risks.push("missing_sources");
  }

  for (const source of sources) {
    if (!source.title.trim()) {
      risks.push("source_missing_title");
    }
    if (!isHttpUrl(source.url)) {
      risks.push("source_invalid_url");
    }
    if (!source.license?.trim()) {
      risks.push("source_missing_license");
    }
  }

  return Array.from(new Set(risks));
}

function parseModerationJson(content: string): Pick<ModerationResult, "outcome" | "riskCategories" | "confidence" | "reasoning"> {
  try {
    const jsonStart = content.indexOf("{");
    const jsonEnd = content.lastIndexOf("}");
    const parsed = JSON.parse(content.slice(jsonStart, jsonEnd + 1)) as Partial<ModerationResult>;
    const outcome = parsed.outcome === "REJECT" || parsed.outcome === "ESCALATE" ? parsed.outcome : "APPROVE";

    return {
      outcome,
      riskCategories: Array.isArray(parsed.riskCategories) ? parsed.riskCategories.map(String) : [],
      confidence: typeof parsed.confidence === "number" ? Math.min(Math.max(parsed.confidence, 0), 1) : 0.72,
      reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : "KI-Moderation abgeschlossen.",
    };
  } catch {
    return {
      outcome: "ESCALATE",
      riskCategories: ["moderation_parse_error"],
      confidence: 0.4,
      reasoning: "Die KI-Moderation konnte nicht eindeutig geparst werden und wird zur Admin-Pruefung eskaliert.",
    };
  }
}

function isHttpUrl(value: string) {
  try {
    const url = new URL(value);
    return url.protocol === "http:" || url.protocol === "https:";
  } catch {
    return false;
  }
}