Merge pull request 'docs: record tagged tool parser fix' (#78) from codex/handoff-chatml-tool-parser into codex/production-intelligence-terminal

docs/agent-handoff.md

@@ -2,21 +2,28 @@
 
 Last updated: 2026-07-05
 
-## Active WIP: Dynamic DAVE Presence
+## Dynamic DAVE Presence
 
-- Gitea issue #70 tracks dynamic autonomous Telegram presence for DAVE.
-- Local branch: `codex/issue-70-dave-presence`.
-- Local implementation commit: `08b6016 feat: add dynamic autonomous DAVE presence`.
+- Gitea issue #70 was closed by merged PR #71. Production merge commit: `ca68541adf9931e311f1a79a09cb87b3cae85928`.
+- Implementation commit: `08b6016 feat: add dynamic autonomous DAVE presence`; handoff commit: `f2d8e89`.
 - The implementation deliberately uses no fixed daily schedule. It evaluates at randomized intervals, is pulled forward by material/critical sweep deltas, delays itself after operator chat activity, and lengthens quiet periods.
 - Hard controls: profile quiet hours, daily cap, minimum message gap, bounded evaluation interval, persisted state in `runs/dave-presence-state.json`, and agent-core rejection of mutating tools outside operator-initiated chat.
 - Presence can produce a grounded warning, update, all-clear, practical suggestion, or one useful question; it stays silent when another message would add noise. It never claims consciousness or continuous observation.
-- New env keys start with `DAVE_PRESENCE_`; the feature defaults off and must be enabled explicitly in the Dockge `.env`.
+- New env keys start with `DAVE_PRESENCE_`; the repository default remains opt-in.
 - Local lightweight verification passed: Node syntax checks, `package.json` parse, Compose config, and `git diff --check`. Heavy tests/build were not run locally per kit policy.
-- Push was blocked by the Codex usage-limit approval service until 2026-07-06 02:28 local time. The branch and commit currently exist only in this workspace.
-- Required continuation: push the branch, create a PR closing #70, poll Gitea runners, fix failures if any, merge, poll production publish workflows, add `DAVE_PRESENCE_ENABLED=true` to the Dockge environment, redeploy `latest`, verify `/api/health.davePresence`, then update this handoff with final commits/run IDs/live state.
+- PR runs 915-916 and production runs 917-919 passed, including unit tests, Compose validation, Docker build/publish, release dry-run, and template compliance.
+- Dockge was updated from the Gitea Registry and explicitly configured with `DAVE_PRESENCE_ENABLED=true`, max 4 messages/day, 75-minute minimum gap, randomized 45-180 minute evaluation bounds, 60-minute post-interaction idle delay, and a 5-minute timer resolution.
+- Live `/api/health` reported `davePresence.enabled=true`, `running=true`, `dynamic=true`, `sentToday=0`, a variable `nextEvaluationAt`, the existing encrypted profile preserved, and no sweep error. The container reported `running/healthy`.
+- Live verification then exposed issue #73: an invalid optional profile timezone caused `lastReason=invalid_timezone`. PR #74 fixed runtime resolution to fall back to `DAVE_PRESENCE_TIMEZONE` without logging or modifying encrypted profile data. Merge commit: `f0a8162202bfe0a0a8ed2a00dc4f0f7092677eea`.
+- PR runs 925-926 and production runs 927-929 passed. After redeployment and the startup timer, live health reported `lastReason=not_due` instead of `invalid_timezone`, with dynamic presence running, the profile preserved, the container healthy, and no sweep error.
 
 ## Latest Completed Work
 
+- Issue #76 / PR #77 fixed a second local-model protocol leak: ChatML-style output such as `<|tool_call>call:get_evidence{query:"...",limit:5}<tool_call|>` was previously treated as user-facing text. Merge commit: `7f140021f037cbc351c9a3b39a5eb3610bfc4939`.
+- The controlled agent now parses bounded tagged calls into the normal allowlisted tool path without `eval`, recognizes tagged JSON envelopes, and treats malformed protocol-like output as a repairable protocol error. Tool tags are never accepted as final Telegram text.
+- The exact observed Russia-query format and a malformed-tag variant are regression fixtures. PR runs 935-936 and production runs 937-939 passed.
+- The registry image was redeployed to Dockge. Live health reported `running/healthy`, Telegram agent enabled with 14 tools, dynamic DAVE presence enabled, encrypted profile preserved, and no sweep error.
+
 - Canonical repository: `https://git.wilkensxl.de/Code-Inc/intelligence-terminal`
 - LiteLLM implementation merge: `5c4bf80eb0c19bd59080f5432a2a344798d7a3ce`
 - Merged PR: `#48 feat: add LiteLLM provider and publish Code-Inc image`

lib/agent/dave-presence.mjs

@@ -78,8 +78,12 @@ export class DavePresence {
     const profile = this.profileStore?.getAgentProfile();
     if (!profile) return this._result(false, 'profile_missing');
 
-    const timezone = profile.timezone || this.fallbackTimezone;
-    const clock = localClock(now, timezone);
+    let timezone = profile.timezone || this.fallbackTimezone;
+    let clock = localClock(now, timezone);
+    if (!clock && timezone !== this.fallbackTimezone) {
+      timezone = this.fallbackTimezone;
+      clock = localClock(now, timezone);
+    }
     if (!clock) return this._result(false, 'invalid_timezone');
     this._rollDay(clock.day);
     if (this.state.sentCount >= this.maxPerDay) return this._result(false, 'daily_limit');

lib/agent/terminal-agent.mjs

@@ -91,6 +91,10 @@ export class TerminalAgent {
       const decision = parseDecision(response?.text);
       if (!decision) {
         const answer = String(response?.text || '').trim() || 'The agent returned no usable response.';
+        if (looksLikeProtocolPayload(answer)) {
+          working += '\n\nPROTOCOL ERROR: The previous tool syntax was malformed. Return the documented JSON tool_call or final object only.';
+          continue;
+        }
         this.lastTraceByChat.set(key, trace);
         return { answer, trace };
       }
@@ -272,6 +276,8 @@ Output exactly one JSON object without markdown:
 
 function parseDecision(text) {
   let value = String(text || '').trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '');
+  const tagged = parseTaggedToolCall(value);
+  if (tagged) return tagged;
   const match = value.match(/\{[\s\S]*\}/);
   if (match) value = match[0];
   try {
@@ -282,9 +288,53 @@ function parseDecision(text) {
   }
 }
 
+function parseTaggedToolCall(text) {
+  const callMatch = String(text || '').match(/<\|?tool_call\|?>\s*call:([a-z0-9_]+)\s*(\{[\s\S]*?\})\s*<(?:\/tool_call|tool_call\|)>/i);
+  if (callMatch) {
+    const argumentsValue = parseTaggedArguments(callMatch[2]);
+    if (argumentsValue) {
+      return {
+        type: 'tool_call',
+        tool: callMatch[1],
+        arguments: argumentsValue,
+        rationale: 'Model requested an allowlisted tool through tagged protocol.',
+      };
+    }
+  }
+
+  const jsonMatch = String(text || '').match(/<\|?tool_call\|?>\s*(\{[\s\S]*\})\s*<(?:\/tool_call|tool_call\|)>/i);
+  if (!jsonMatch) return null;
+  try {
+    const value = JSON.parse(jsonMatch[1]);
+    const tool = value.name || value.tool;
+    const args = value.arguments || value.parameters || {};
+    if (!/^[a-z0-9_]+$/i.test(String(tool || '')) || !args || Array.isArray(args) || typeof args !== 'object') return null;
+    return { type: 'tool_call', tool: String(tool), arguments: args, rationale: 'Model requested an allowlisted tool through tagged protocol.' };
+  } catch {
+    return null;
+  }
+}
+
+function parseTaggedArguments(value) {
+  const input = String(value || '').trim().slice(0, 4000);
+  try {
+    const parsed = JSON.parse(input);
+    return parsed && !Array.isArray(parsed) && typeof parsed === 'object' ? parsed : null;
+  } catch {
+    try {
+      const normalized = input.replace(/([{,]\s*)([a-z_][a-z0-9_]*)\s*:/gi, '$1"$2":');
+      const parsed = JSON.parse(normalized);
+      return parsed && !Array.isArray(parsed) && typeof parsed === 'object' ? parsed : null;
+    } catch {
+      return null;
+    }
+  }
+}
+
 function looksLikeProtocolPayload(text) {
   return /"?type"?\s*:\s*"?(?:tool_call|final)\b/i.test(text)
-    || /"?tool"?\s*:\s*"?[a-z0-9_]+/i.test(text);
+    || /"?tool"?\s*:\s*"?[a-z0-9_]+/i.test(text)
+    || /<\|?tool_call\|?>|<tool_call\|>|call:[a-z0-9_]+\s*\{/i.test(text);
 }
 
 function finalizationFailureMessage(input) {

test/dave-presence.test.mjs

@@ -89,3 +89,11 @@ test('local clock uses the profile timezone', () => {
   assert.deepEqual(localClock(new Date('2026-07-06T12:30:00Z'), 'Europe/Berlin'), { day: '2026-07-06', time: '14:30' });
   assert.equal(localClock(new Date(), 'Invalid/Timezone'), null);
 });
+
+test('invalid profile timezone falls back to configured operational timezone', async () => {
+  const { presence, messages, calls } = setup({ profile: { timezone: 'not-a-timezone' } });
+  const outcome = await presence.tick(new Date('2026-07-06T12:00:00Z'));
+  assert.equal(outcome.reason, 'sent');
+  assert.equal(messages.length, 1);
+  assert.match(calls[0].prompt, /timezone: UTC/);
+});

test/terminal-agent.test.mjs

@@ -150,3 +150,43 @@ test('scheduled presence cannot create pending mutating actions', async () => {
   assert.equal(result.trace[0].status, 'rejected');
   assert.equal(executions, 0);
 });
+
+test('ChatML-style tagged tool calls are executed without leaking protocol text', async () => {
+  const calls = [];
+  let responseIndex = 0;
+  const responses = [
+    '<|tool_call>call:get_evidence{query:"Russia planned attack military escalation",limit:5}<tool_call|>',
+    JSON.stringify({ type: 'final', answer: 'The retrieved evidence does not confirm a specific planned attack.', confidence: 'medium', evidence: ['evt-russia'], notify: false, priority: 'routine' }),
+  ];
+  const agent = new TerminalAgent({
+    provider: {
+      isConfigured: true,
+      async complete() { return { text: responses[responseIndex++] }; },
+    },
+    registry: new TerminalToolRegistry([{
+      name: 'get_evidence',
+      handler: async args => { calls.push(args); return [{ id: 'evt-russia' }]; },
+    }]),
+  });
+
+  const result = await agent.run('Wie sieht es mit dem angeblich geplanten Angriff von Russland aus?');
+  assert.deepEqual(calls, [{ query: 'Russia planned attack military escalation', limit: 5 }]);
+  assert.equal(result.trace[0].tool, 'get_evidence');
+  assert.doesNotMatch(result.answer, /tool_call|call:get_evidence/i);
+});
+
+test('malformed tagged protocol is repaired instead of returned to the user', async () => {
+  let responseIndex = 0;
+  const responses = [
+    '<|tool_call>call:get_evidence{broken arguments}<tool_call|>',
+    JSON.stringify({ type: 'final', answer: 'I could not run the malformed tool request.', confidence: 'low', evidence: [], notify: false, priority: 'routine' }),
+  ];
+  const agent = new TerminalAgent({
+    provider: { isConfigured: true, async complete() { return { text: responses[responseIndex++] }; } },
+    registry: new TerminalToolRegistry([]),
+  });
+
+  const result = await agent.run('Check the claim');
+  assert.equal(result.answer, 'I could not run the malformed tool request.');
+  assert.doesNotMatch(result.answer, /tool_call|call:get_evidence/i);
+});