From 67782e684e374e4ea2b2aaa6fbe74fd535c1d2ea Mon Sep 17 00:00:00 2001
From: Kenny Van de Maele <kenny@kvandemaele.be>
Date: Thu, 4 Jun 2026 21:42:23 +0200
Subject: [PATCH] fix: exclude slash-command/setup messages from LLM context
 (#2634) (#2640)

Slash-command replies and the echoed /setup command are persisted to session
history so they render in the transcript, but they are UI chatter the user
never meant as conversation. They were sent to the model on the next turn,
which then commented on '/setup ...' and exposed transient values (e.g. the
Copilot device user_code) to the LLM.

- get_context_messages() (the LLM-API view) now skips messages tagged
  metadata.source == 'slash'. Display/history-load paths use raw history and
  are unaffected.
- slashCommands.js tags the echoed user command with source:'slash' too (the
  assistant replies already carried it); the user line was the one untagged
  path that still reached context.

Fixes #2634.
---
 core/models.py                               | 16 ++++++-
 static/js/slashCommands.js                   |  4 +-
 tests/test_session_context_excludes_slash.py | 44 ++++++++++++++++++++
 3 files changed, 61 insertions(+), 3 deletions(-)
 create mode 100644 tests/test_session_context_excludes_slash.py

diff --git a/core/models.py b/core/models.py
index 6914b20..1adae65 100644
--- a/core/models.py
+++ b/core/models.py
@@ -76,8 +76,20 @@ class Session:
             _session_manager._persist_message(self.id, message)
 
     def get_context_messages(self) -> List[Dict[str, Any]]:
-        """Get messages in format for LLM API."""
-        return [msg.to_dict() for msg in self.history]
+        """Get messages in format for LLM API.
+
+        Slash-command / setup replies are persisted to history so they render
+        in the transcript, but they are UI chatter (e.g. ``/setup ...`` and its
+        status lines) the user never meant as conversation. They carry
+        ``metadata.source == "slash"``; exclude them here so they never reach
+        the model. Display/history-load paths use the raw ``history`` and are
+        unaffected.
+        """
+        return [
+            msg.to_dict()
+            for msg in self.history
+            if (msg.metadata or {}).get("source") != "slash"
+        ]
 
     def get(self, key: str, default=None):
         """Dict-like access for compatibility."""
diff --git a/static/js/slashCommands.js b/static/js/slashCommands.js
index 97b3fb3..19bcd7a 100644
--- a/static/js/slashCommands.js
+++ b/static/js/slashCommands.js
@@ -5880,7 +5880,9 @@ async function handleSlashCommand(input) {
   let args = parts.slice(1);
   const ctx = _makeCtx();
   let _userShown = false;
-  function _showUser() { if (!_userShown) { _userShown = true; _addMessage('user', input); _persistMsg('user', input); } }
+  // Tag the echoed command with source:'slash' so it renders in the transcript
+  // but is excluded from LLM context (get_context_messages), like the replies.
+  function _showUser() { if (!_userShown) { _userShown = true; _addMessage('user', input); _persistMsg('user', input, { source: 'slash' }); } }
 
   try {
     // --- Check for --help / -h on any command ---
diff --git a/tests/test_session_context_excludes_slash.py b/tests/test_session_context_excludes_slash.py
new file mode 100644
index 0000000..e9ff152
--- /dev/null
+++ b/tests/test_session_context_excludes_slash.py
@@ -0,0 +1,44 @@
+"""Regression: slash-command / setup messages must not reach LLM context.
+
+Slash replies (and the echoed `/setup ...` command) are persisted to history so
+they render in the transcript, tagged ``metadata.source == "slash"``. They are
+UI chatter the user never meant as conversation, so ``get_context_messages``
+(the LLM-API view) must exclude them while the raw history keeps them for
+display. See issue #2634.
+"""
+
+from core.models import Session, ChatMessage
+
+
+def _session_with_slash():
+    s = Session(id="s1", name="t", endpoint_url="http://x/v1", model="m")
+    s.add_message(ChatMessage("user", "hi, give me a recipe"))
+    s.add_message(ChatMessage("user", "/setup copilot", metadata={"source": "slash"}))
+    s.add_message(ChatMessage("assistant", "Starting GitHub Copilot sign-in...", metadata={"source": "slash"}))
+    s.add_message(ChatMessage("assistant", "Here is a recipe", metadata={"model": "m"}))
+    return s
+
+
+def test_context_excludes_slash_messages():
+    ctx = _session_with_slash().get_context_messages()
+    contents = [m["content"] for m in ctx]
+    assert "hi, give me a recipe" in contents
+    assert "Here is a recipe" in contents
+    # Slash command + its status reply are filtered out of LLM context.
+    assert "/setup copilot" not in contents
+    assert all("sign-in" not in c for c in contents)
+    assert len(ctx) == 2
+
+
+def test_history_still_keeps_slash_messages_for_display():
+    s = _session_with_slash()
+    # Raw history (what the UI renders) is untouched.
+    assert len(s.history) == 4
+    assert any(m.content == "/setup copilot" for m in s.history)
+
+
+def test_no_metadata_messages_are_kept():
+    s = Session(id="s2", name="t", endpoint_url="http://x/v1", model="m")
+    s.add_message(ChatMessage("user", "plain"))
+    s.add_message(ChatMessage("assistant", "reply"))
+    assert [m["content"] for m in s.get_context_messages()] == ["plain", "reply"]