mo commited on
Commit
4fc4b38
·
1 Parent(s): 0b356a1

feat(agent): self-healing loop — auto-debug, auto-reasoning, iteration

Browse files

Fail is never allowed:

- think() retries transient failures (network/429/5xx) with backoff
- malformed or empty brain replies trigger corrective nudges (max 2)
- tool errors inject a diagnostic hint + auto-capture a debug snapshot
(sysinfo, uname, node version) into the activity feed
- step exhaustion forces one plain-text conclusion instead of giving up
- task results are reported with retries — the conversation is never
left dangling
- daemon survives uncaught exceptions / unhandled rejections (log only)

Live-verified: broken command (exit 127) → auto-debug → brain pivoted to
a working tool → real answer.

apps/desktop/bin/mona-agent.js CHANGED
@@ -311,6 +311,14 @@ async function gui() {
311
 
312
  // ── start (headless) ──────────────────────────────────────────────
313
  async function start() {
 
 
 
 
 
 
 
 
314
  const creds = requireCreds();
315
 
316
  console.log(`\n ${BOLD}${CYAN}mona-agent${RESET} ${DIM}v${DEFAULTS.version}${RESET}`);
 
311
 
312
  // ── start (headless) ──────────────────────────────────────────────
313
  async function start() {
314
+ // Fail is never allowed: the daemon logs and survives unexpected errors.
315
+ process.on('uncaughtException', (err) => {
316
+ process.stderr.write(` ${YELLOW}⚠ uncaught: ${err?.message || err}${RESET}\n`);
317
+ });
318
+ process.on('unhandledRejection', (err) => {
319
+ process.stderr.write(` ${YELLOW}⚠ unhandled rejection: ${String(err?.message || err)}${RESET}\n`);
320
+ });
321
+
322
  const creds = requireCreds();
323
 
324
  console.log(`\n ${BOLD}${CYAN}mona-agent${RESET} ${DIM}v${DEFAULTS.version}${RESET}`);
apps/desktop/src/agent.js CHANGED
@@ -12,9 +12,14 @@ import { CLOUD } from './config.js';
12
  import { log } from './log.js';
13
 
14
  const MAX_ITERS = 8; // max tool steps per task
 
 
15
  const TASK_POLL_MS = 2000; // sngine platform: poll the cloud task queue
16
  const TOOL_OUT_MAX = 4000; // chars of tool output fed back to the brain
17
 
 
 
 
18
  function truncate(s, n) {
19
  s = String(s ?? '');
20
  return s.length > n ? s.slice(0, n) + '…(truncated)' : s;
@@ -166,6 +171,61 @@ export class AgentDaemon extends EventEmitter {
166
  }
167
 
168
  // ── Task execution (cloud reasoning + local tools) ──────────────
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
169
  async #runTask(task, runId, cloudTask = null) {
170
  if (!task) {
171
  this.#control.result(runId, { error: 'No task provided' });
@@ -195,21 +255,30 @@ export class AgentDaemon extends EventEmitter {
195
  { role: 'user', content: task },
196
  ];
197
 
 
198
  for (let i = 0; i < MAX_ITERS; i++) {
199
- const answer = await think({
200
- apiKey: this.#creds.apiKey,
201
- messages,
202
- tools: tools.list(),
203
- onChunk: (delta) => {
204
- this.#control.token(delta, runId);
205
- this.emit('task:token', delta, runId);
206
- },
207
- onUsage: (usage) => this.emit('task:usage', usage),
208
- });
209
-
210
  const toolCall = parseToolCall(answer);
211
 
212
- if (!toolCall) { final = answer; break; }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
213
 
214
  // Execute the tool locally
215
  steps.push({ type: 'tool.call', tool: toolCall.tool, args: toolCall.args });
@@ -233,10 +302,24 @@ export class AgentDaemon extends EventEmitter {
233
  await postActivity(this.#creds.apiKey, 'tool.result', { tool: toolCall.tool, output: truncate(out, 200) }, runId, this.#creds.agentId).catch(() => {});
234
  log.info(`Tool result (${toolCall.tool}): ${truncate(out, 120)}`);
235
 
236
- messages.push({ role: 'user', content: `TOOL RESULT (${toolCall.tool}):\n${out}` });
 
 
 
 
 
 
237
  }
238
 
239
- if (!final) final = 'I ran out of steps — the last tool results are above.';
 
 
 
 
 
 
 
 
240
  }
241
  } catch (err) {
242
  this.#stats.errors++;
@@ -244,8 +327,9 @@ export class AgentDaemon extends EventEmitter {
244
  this.#control.step('task.error', { error: err.message });
245
  this.emit('task:error', err);
246
  log.error(`Think failed: ${err.message}`);
247
- const msg = `Error: ${err.message}`;
248
- if (cloudTask) await taskResult(this.#creds.apiKey, cloudTask.id, { result: msg, steps }).catch(() => {});
 
249
  return;
250
  }
251
 
@@ -254,7 +338,7 @@ export class AgentDaemon extends EventEmitter {
254
 
255
  this.#currentTask = null;
256
  if (cloudTask) {
257
- await taskResult(this.#creds.apiKey, cloudTask.id, { result: final, steps }).catch(() => {});
258
  }
259
  this.#control.result(runId, { text: final });
260
  this.#control.step('task.done', { runId, tokens: final.length, chars: final.length });
 
12
  import { log } from './log.js';
13
 
14
  const MAX_ITERS = 8; // max tool steps per task
15
+ const MAX_RETRIES = 3; // transient failures (network, 429, 5xx)
16
+ const MAX_CORRECTIONS = 2; // malformed/empty brain replies per task
17
  const TASK_POLL_MS = 2000; // sngine platform: poll the cloud task queue
18
  const TOOL_OUT_MAX = 4000; // chars of tool output fed back to the brain
19
 
20
+ const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
21
+ const RETRIABLE = /429|5\d\d|fetch failed|network|ECONN|ETIMEDOUT|socket|timeout/i;
22
+
23
  function truncate(s, n) {
24
  s = String(s ?? '');
25
  return s.length > n ? s.slice(0, n) + '…(truncated)' : s;
 
171
  }
172
 
173
  // ── Task execution (cloud reasoning + local tools) ──────────────
174
+ /** think() with auto-retry for transient failures — fail is never allowed. */
175
+ async #thinkWithRetry(messages, runId) {
176
+ let lastErr = null;
177
+ for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
178
+ try {
179
+ return await think({
180
+ apiKey: this.#creds.apiKey,
181
+ messages,
182
+ tools: tools.list(),
183
+ onChunk: (delta) => {
184
+ this.#control.token(delta, runId);
185
+ this.emit('task:token', delta, runId);
186
+ },
187
+ onUsage: (usage) => this.emit('task:usage', usage),
188
+ });
189
+ } catch (err) {
190
+ lastErr = err;
191
+ const msg = String(err?.message || err);
192
+ const retriable = RETRIABLE.test(msg);
193
+ log.warn(`Think attempt ${attempt}/${MAX_RETRIES} failed: ${msg}`);
194
+ await postActivity(this.#creds.apiKey, 'auto.retry', { attempt, error: msg.slice(0, 200) }, runId, this.#creds.agentId).catch(() => {});
195
+ if (!retriable || attempt === MAX_RETRIES) throw err;
196
+ await sleep(500 * attempt + Math.random() * 400);
197
+ }
198
+ }
199
+ throw lastErr ?? new Error('think failed');
200
+ }
201
+
202
+ /** Report the result with retries so the cloud conversation is never left dangling. */
203
+ async #reportResult(cloudTask, result, steps) {
204
+ for (let a = 1; a <= 3; a++) {
205
+ try {
206
+ await taskResult(this.#creds.apiKey, cloudTask.id, { result, steps });
207
+ return;
208
+ } catch (err) {
209
+ log.warn(`Task result POST attempt ${a} failed: ${err.message}`);
210
+ await sleep(800 * a);
211
+ }
212
+ }
213
+ log.error('Could not report task result to cloud');
214
+ }
215
+
216
+ /** Auto-debug: capture a system snapshot when things go wrong. */
217
+ async #debugSnapshot(runId, why) {
218
+ try {
219
+ const info = await tools.run('sysinfo', {});
220
+ const uname = await tools.run('shell', { cmd: 'uname -a && which node && node -v' });
221
+ await postActivity(this.#creds.apiKey, 'auto.debug', {
222
+ why,
223
+ sysinfo: truncate(JSON.stringify(info), 300),
224
+ uname: truncate(JSON.stringify(uname), 300),
225
+ }, runId, this.#creds.agentId).catch(() => {});
226
+ } catch { /* never break the task for debugging */ }
227
+ }
228
+
229
  async #runTask(task, runId, cloudTask = null) {
230
  if (!task) {
231
  this.#control.result(runId, { error: 'No task provided' });
 
255
  { role: 'user', content: task },
256
  ];
257
 
258
+ let corrections = 0;
259
  for (let i = 0; i < MAX_ITERS; i++) {
260
+ const answer = await this.#thinkWithRetry(messages, runId);
 
 
 
 
 
 
 
 
 
 
261
  const toolCall = parseToolCall(answer);
262
 
263
+ if (!toolCall) {
264
+ const hasText = answer && answer.trim();
265
+ const looksLikeAttempt = hasText && /"tool"\s*:/.test(answer);
266
+ if (hasText && !looksLikeAttempt) { final = answer; break; }
267
+ // empty or malformed → corrective nudge (auto-reasoning)
268
+ if (corrections >= MAX_CORRECTIONS) {
269
+ final = hasText ? answer : 'The brain produced no usable reply. Check the activity feed for the trace.';
270
+ break;
271
+ }
272
+ corrections++;
273
+ const hint = looksLikeAttempt
274
+ ? 'Your last message was not valid JSON. Reply with ONLY one JSON object: {"tool":"<tool name>","args":{...}} — or plain text if you are done.'
275
+ : 'Your reply was empty. Either answer in plain text or emit ONE JSON tool call.';
276
+ messages.push({ role: 'assistant', content: answer });
277
+ messages.push({ role: 'user', content: hint });
278
+ await postActivity(this.#creds.apiKey, 'auto.correct', { reason: looksLikeAttempt ? 'malformed' : 'empty', attempt: corrections }, runId, this.#creds.agentId).catch(() => {});
279
+ continue;
280
+ }
281
+ corrections = 0;
282
 
283
  // Execute the tool locally
284
  steps.push({ type: 'tool.call', tool: toolCall.tool, args: toolCall.args });
 
302
  await postActivity(this.#creds.apiKey, 'tool.result', { tool: toolCall.tool, output: truncate(out, 200) }, runId, this.#creds.agentId).catch(() => {});
303
  log.info(`Tool result (${toolCall.tool}): ${truncate(out, 120)}`);
304
 
305
+ // Auto-debug: on tool error, give the brain a diagnostic nudge
306
+ const failed = result && (result.error || result.exitCode);
307
+ const debugHint = failed
308
+ ? `\n\nThe tool returned an error. Diagnose it, then either fix the call or use a different tool/approach. Do not give up — verify your fix by running it again.`
309
+ : '';
310
+ messages.push({ role: 'user', content: `TOOL RESULT (${toolCall.tool}):\n${out}${debugHint}` });
311
+ if (failed) await this.#debugSnapshot(runId, `tool ${toolCall.tool} failed`);
312
  }
313
 
314
+ // Never give up: one forced plain-text conclusion when steps run out
315
+ if (!final) {
316
+ messages.push({ role: 'user', content: 'You must now answer in plain text. Summarize what you did, what the last tool result showed, and the current state. Do not call tools.' });
317
+ try {
318
+ final = await this.#thinkWithRetry(messages, runId);
319
+ } catch {
320
+ final = 'The agent hit its step limit. See the activity feed for the full execution trace.';
321
+ }
322
+ }
323
  }
324
  } catch (err) {
325
  this.#stats.errors++;
 
327
  this.#control.step('task.error', { error: err.message });
328
  this.emit('task:error', err);
329
  log.error(`Think failed: ${err.message}`);
330
+ await this.#debugSnapshot(runId, 'task error: ' + err.message);
331
+ const msg = `I hit an error and could not complete the task: ${err.message}. A debug snapshot was captured — retry the request or check the activity feed.`;
332
+ if (cloudTask) await this.#reportResult(cloudTask, msg, steps);
333
  return;
334
  }
335
 
 
338
 
339
  this.#currentTask = null;
340
  if (cloudTask) {
341
+ await this.#reportResult(cloudTask, final, steps);
342
  }
343
  this.#control.result(runId, { text: final });
344
  this.#control.step('task.done', { runId, tokens: final.length, chars: final.length });