Merge pull request 'docs: record tagged tool parser fix' (#78) from codex/handoff-chatml-tool-parser into codex/production-intelligence-terminal

docs/agent-handoff.md

@@ -19,6 +19,11 @@ Last updated: 2026-07-05
 
 ## Latest Completed Work
 
+- Issue #76 / PR #77 fixed a second local-model protocol leak: ChatML-style output such as `<|tool_call>call:get_evidence{query:"...",limit:5}<tool_call|>` was previously treated as user-facing text. Merge commit: `7f140021f037cbc351c9a3b39a5eb3610bfc4939`.
+- The controlled agent now parses bounded tagged calls into the normal allowlisted tool path without `eval`, recognizes tagged JSON envelopes, and treats malformed protocol-like output as a repairable protocol error. Tool tags are never accepted as final Telegram text.
+- The exact observed Russia-query format and a malformed-tag variant are regression fixtures. PR runs 935-936 and production runs 937-939 passed.
+- The registry image was redeployed to Dockge. Live health reported `running/healthy`, Telegram agent enabled with 14 tools, dynamic DAVE presence enabled, encrypted profile preserved, and no sweep error.
+
 - Canonical repository: `https://git.wilkensxl.de/Code-Inc/intelligence-terminal`
 - LiteLLM implementation merge: `5c4bf80eb0c19bd59080f5432a2a344798d7a3ce`
 - Merged PR: `#48 feat: add LiteLLM provider and publish Code-Inc image`

lib/agent/terminal-agent.mjs

@@ -91,6 +91,10 @@ export class TerminalAgent {
       const decision = parseDecision(response?.text);
       if (!decision) {
         const answer = String(response?.text || '').trim() || 'The agent returned no usable response.';
+        if (looksLikeProtocolPayload(answer)) {
+          working += '\n\nPROTOCOL ERROR: The previous tool syntax was malformed. Return the documented JSON tool_call or final object only.';
+          continue;
+        }
         this.lastTraceByChat.set(key, trace);
         return { answer, trace };
       }
@@ -272,6 +276,8 @@ Output exactly one JSON object without markdown:
 
 function parseDecision(text) {
   let value = String(text || '').trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '');
+  const tagged = parseTaggedToolCall(value);
+  if (tagged) return tagged;
   const match = value.match(/\{[\s\S]*\}/);
   if (match) value = match[0];
   try {
@@ -282,9 +288,53 @@ function parseDecision(text) {
   }
 }
 
+function parseTaggedToolCall(text) {
+  const callMatch = String(text || '').match(/<\|?tool_call\|?>\s*call:([a-z0-9_]+)\s*(\{[\s\S]*?\})\s*<(?:\/tool_call|tool_call\|)>/i);
+  if (callMatch) {
+    const argumentsValue = parseTaggedArguments(callMatch[2]);
+    if (argumentsValue) {
+      return {
+        type: 'tool_call',
+        tool: callMatch[1],
+        arguments: argumentsValue,
+        rationale: 'Model requested an allowlisted tool through tagged protocol.',
+      };
+    }
+  }
+
+  const jsonMatch = String(text || '').match(/<\|?tool_call\|?>\s*(\{[\s\S]*\})\s*<(?:\/tool_call|tool_call\|)>/i);
+  if (!jsonMatch) return null;
+  try {
+    const value = JSON.parse(jsonMatch[1]);
+    const tool = value.name || value.tool;
+    const args = value.arguments || value.parameters || {};
+    if (!/^[a-z0-9_]+$/i.test(String(tool || '')) || !args || Array.isArray(args) || typeof args !== 'object') return null;
+    return { type: 'tool_call', tool: String(tool), arguments: args, rationale: 'Model requested an allowlisted tool through tagged protocol.' };
+  } catch {
+    return null;
+  }
+}
+
+function parseTaggedArguments(value) {
+  const input = String(value || '').trim().slice(0, 4000);
+  try {
+    const parsed = JSON.parse(input);
+    return parsed && !Array.isArray(parsed) && typeof parsed === 'object' ? parsed : null;
+  } catch {
+    try {
+      const normalized = input.replace(/([{,]\s*)([a-z_][a-z0-9_]*)\s*:/gi, '$1"$2":');
+      const parsed = JSON.parse(normalized);
+      return parsed && !Array.isArray(parsed) && typeof parsed === 'object' ? parsed : null;
+    } catch {
+      return null;
+    }
+  }
+}
+
 function looksLikeProtocolPayload(text) {
   return /"?type"?\s*:\s*"?(?:tool_call|final)\b/i.test(text)
-    || /"?tool"?\s*:\s*"?[a-z0-9_]+/i.test(text);
+    || /"?tool"?\s*:\s*"?[a-z0-9_]+/i.test(text)
+    || /<\|?tool_call\|?>|<tool_call\|>|call:[a-z0-9_]+\s*\{/i.test(text);
 }
 
 function finalizationFailureMessage(input) {

test/terminal-agent.test.mjs

@@ -150,3 +150,43 @@ test('scheduled presence cannot create pending mutating actions', async () => {
   assert.equal(result.trace[0].status, 'rejected');
   assert.equal(executions, 0);
 });
+
+test('ChatML-style tagged tool calls are executed without leaking protocol text', async () => {
+  const calls = [];
+  let responseIndex = 0;
+  const responses = [
+    '<|tool_call>call:get_evidence{query:"Russia planned attack military escalation",limit:5}<tool_call|>',
+    JSON.stringify({ type: 'final', answer: 'The retrieved evidence does not confirm a specific planned attack.', confidence: 'medium', evidence: ['evt-russia'], notify: false, priority: 'routine' }),
+  ];
+  const agent = new TerminalAgent({
+    provider: {
+      isConfigured: true,
+      async complete() { return { text: responses[responseIndex++] }; },
+    },
+    registry: new TerminalToolRegistry([{
+      name: 'get_evidence',
+      handler: async args => { calls.push(args); return [{ id: 'evt-russia' }]; },
+    }]),
+  });
+
+  const result = await agent.run('Wie sieht es mit dem angeblich geplanten Angriff von Russland aus?');
+  assert.deepEqual(calls, [{ query: 'Russia planned attack military escalation', limit: 5 }]);
+  assert.equal(result.trace[0].tool, 'get_evidence');
+  assert.doesNotMatch(result.answer, /tool_call|call:get_evidence/i);
+});
+
+test('malformed tagged protocol is repaired instead of returned to the user', async () => {
+  let responseIndex = 0;
+  const responses = [
+    '<|tool_call>call:get_evidence{broken arguments}<tool_call|>',
+    JSON.stringify({ type: 'final', answer: 'I could not run the malformed tool request.', confidence: 'low', evidence: [], notify: false, priority: 'routine' }),
+  ];
+  const agent = new TerminalAgent({
+    provider: { isConfigured: true, async complete() { return { text: responses[responseIndex++] }; } },
+    registry: new TerminalToolRegistry([]),
+  });
+
+  const result = await agent.run('Check the claim');
+  assert.equal(result.answer, 'I could not run the malformed tool request.');
+  assert.doesNotMatch(result.answer, /tool_call|call:get_evidence/i);
+});