fix: exclude slash-command/setup messages from LLM context (#2634) (#2640)

Slash-command replies and the echoed /setup command are persisted to session
history so they render in the transcript, but they are UI chatter the user
never meant as conversation. They were sent to the model on the next turn,
which then commented on '/setup ...' and exposed transient values (e.g. the
Copilot device user_code) to the LLM.

- get_context_messages() (the LLM-API view) now skips messages tagged
  metadata.source == 'slash'. Display/history-load paths use raw history and
  are unaffected.
- slashCommands.js tags the echoed user command with source:'slash' too (the
  assistant replies already carried it); the user line was the one untagged
  path that still reached context.

Fixes #2634.
This commit is contained in:
Kenny Van de Maele
2026-06-04 21:42:23 +02:00
committed by GitHub
parent baf9179d94
commit 67782e684e
3 changed files with 61 additions and 3 deletions

View File

@@ -0,0 +1,44 @@
"""Regression: slash-command / setup messages must not reach LLM context.
Slash replies (and the echoed `/setup ...` command) are persisted to history so
they render in the transcript, tagged ``metadata.source == "slash"``. They are
UI chatter the user never meant as conversation, so ``get_context_messages``
(the LLM-API view) must exclude them while the raw history keeps them for
display. See issue #2634.
"""
from core.models import Session, ChatMessage
def _session_with_slash():
s = Session(id="s1", name="t", endpoint_url="http://x/v1", model="m")
s.add_message(ChatMessage("user", "hi, give me a recipe"))
s.add_message(ChatMessage("user", "/setup copilot", metadata={"source": "slash"}))
s.add_message(ChatMessage("assistant", "Starting GitHub Copilot sign-in...", metadata={"source": "slash"}))
s.add_message(ChatMessage("assistant", "Here is a recipe", metadata={"model": "m"}))
return s
def test_context_excludes_slash_messages():
ctx = _session_with_slash().get_context_messages()
contents = [m["content"] for m in ctx]
assert "hi, give me a recipe" in contents
assert "Here is a recipe" in contents
# Slash command + its status reply are filtered out of LLM context.
assert "/setup copilot" not in contents
assert all("sign-in" not in c for c in contents)
assert len(ctx) == 2
def test_history_still_keeps_slash_messages_for_display():
s = _session_with_slash()
# Raw history (what the UI renders) is untouched.
assert len(s.history) == 4
assert any(m.content == "/setup copilot" for m in s.history)
def test_no_metadata_messages_are_kept():
s = Session(id="s2", name="t", endpoint_url="http://x/v1", model="m")
s.add_message(ChatMessage("user", "plain"))
s.add_message(ChatMessage("assistant", "reply"))
assert [m["content"] for m in s.get_context_messages()] == ["plain", "reply"]