fix: exclude slash-command/setup messages from LLM context (#2634) (#2640)

Slash-command replies and the echoed /setup command are persisted to session
history so they render in the transcript, but they are UI chatter the user
never meant as conversation. They were sent to the model on the next turn,
which then commented on '/setup ...' and exposed transient values (e.g. the
Copilot device user_code) to the LLM.

- get_context_messages() (the LLM-API view) now skips messages tagged
  metadata.source == 'slash'. Display/history-load paths use raw history and
  are unaffected.
- slashCommands.js tags the echoed user command with source:'slash' too (the
  assistant replies already carried it); the user line was the one untagged
  path that still reached context.

Fixes #2634.

core/models.py

@@ -76,8 +76,20 @@ class Session:
             _session_manager._persist_message(self.id, message)
 
     def get_context_messages(self) -> List[Dict[str, Any]]:
-        """Get messages in format for LLM API."""
+        """Get messages in format for LLM API.
-        return [msg.to_dict() for msg in self.history]
+
+        Slash-command / setup replies are persisted to history so they render
+        in the transcript, but they are UI chatter (e.g. ``/setup ...`` and its
+        status lines) the user never meant as conversation. They carry
+        ``metadata.source == "slash"``; exclude them here so they never reach
+        the model. Display/history-load paths use the raw ``history`` and are
+        unaffected.
+        """
+        return [
+            msg.to_dict()
+            for msg in self.history
+            if (msg.metadata or {}).get("source") != "slash"
+        ]
 
     def get(self, key: str, default=None):
         """Dict-like access for compatibility."""

static/js/slashCommands.js

@@ -5880,7 +5880,9 @@ async function handleSlashCommand(input) {
   let args = parts.slice(1);
   const ctx = _makeCtx();
   let _userShown = false;
-  function _showUser() { if (!_userShown) { _userShown = true; _addMessage('user', input); _persistMsg('user', input); } }
+  // Tag the echoed command with source:'slash' so it renders in the transcript
+  // but is excluded from LLM context (get_context_messages), like the replies.
+  function _showUser() { if (!_userShown) { _userShown = true; _addMessage('user', input); _persistMsg('user', input, { source: 'slash' }); } }
 
   try {
     // --- Check for --help / -h on any command ---

tests/test_session_context_excludes_slash.py

@@ -0,0 +1,44 @@
+"""Regression: slash-command / setup messages must not reach LLM context.
+
+Slash replies (and the echoed `/setup ...` command) are persisted to history so
+they render in the transcript, tagged ``metadata.source == "slash"``. They are
+UI chatter the user never meant as conversation, so ``get_context_messages``
+(the LLM-API view) must exclude them while the raw history keeps them for
+display. See issue #2634.
+"""
+
+from core.models import Session, ChatMessage
+
+
+def _session_with_slash():
+    s = Session(id="s1", name="t", endpoint_url="http://x/v1", model="m")
+    s.add_message(ChatMessage("user", "hi, give me a recipe"))
+    s.add_message(ChatMessage("user", "/setup copilot", metadata={"source": "slash"}))
+    s.add_message(ChatMessage("assistant", "Starting GitHub Copilot sign-in...", metadata={"source": "slash"}))
+    s.add_message(ChatMessage("assistant", "Here is a recipe", metadata={"model": "m"}))
+    return s
+
+
+def test_context_excludes_slash_messages():
+    ctx = _session_with_slash().get_context_messages()
+    contents = [m["content"] for m in ctx]
+    assert "hi, give me a recipe" in contents
+    assert "Here is a recipe" in contents
+    # Slash command + its status reply are filtered out of LLM context.
+    assert "/setup copilot" not in contents
+    assert all("sign-in" not in c for c in contents)
+    assert len(ctx) == 2
+
+
+def test_history_still_keeps_slash_messages_for_display():
+    s = _session_with_slash()
+    # Raw history (what the UI renders) is untouched.
+    assert len(s.history) == 4
+    assert any(m.content == "/setup copilot" for m in s.history)
+
+
+def test_no_metadata_messages_are_kept():
+    s = Session(id="s2", name="t", endpoint_url="http://x/v1", model="m")
+    s.add_message(ChatMessage("user", "plain"))
+    s.add_message(ChatMessage("assistant", "reply"))
+    assert [m["content"] for m in s.get_context_messages()] == ["plain", "reply"]