""" NeuraPrompt — Reply Engine (v1.0) ================================= Solves the "reply cut off mid-sentence" class of bugs. Why replies were truncated -------------------------- The Groq organisation is capped at a small number of OUTPUT tokens per minute (OTPM) for the primary model. One long answer can consume the whole minute's budget, so the next request either: * is rejected with 429, or * gets a tiny remaining budget and stops mid-sentence with finish_reason == "length" (this is the "**Physics" fragment users saw). The old code never inspected finish_reason, so a cut-off answer was shown as if it were complete. What this module guarantees --------------------------- 1. A per-request output cap so one answer cannot drain the whole minute. 2. On 429: fail over to a DIFFERENT model (separate quota) with backoff, instead of retrying the same exhausted model. 3. On finish_reason == "length": the partial text is fed back and the model is asked to continue where it stopped; pieces are joined. Repeats up to MAX_CONTINUATIONS times. 4. Returns metadata (model actually used, finish reason, whether the answer still looks incomplete) so callers can be honest with the user. The module is dependency-free: it accepts a tool executor callable instead of importing the app, so it cannot create an import cycle with main.py. """ from __future__ import annotations import asyncio import json import logging import os from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple import httpx log = logging.getLogger("reply_engine") GROQ_URL = "https://api.groq.com/openai/v1/chat/completions" # Output cap per single request. Deliberately modest: the OTPM budget is shared # across ALL users of the organisation, so one huge answer starves everyone. MAX_REPLY_TOKENS = int(os.getenv("MAX_REPLY_TOKENS", "1200")) # A different model on a separate quota, used when the primary hits 429. FAILOVER_MODEL = os.getenv("GROQ_FAILOVER_MODEL", "openai/gpt-oss-120b") # How many times we may ask the model to continue a truncated answer. MAX_CONTINUATIONS = int(os.getenv("MAX_CONTINUATIONS", "2")) MAX_HARD_TOKENS = int(os.getenv("MAX_HARD_TOKENS", "4000")) # Attempts per round (includes the failover attempt). MAX_ATTEMPTS_PER_ROUND = 3 CONTINUE_PROMPT = ( "Your previous answer was cut off because of a length limit. Continue " "exactly where you stopped. Do not repeat anything you already wrote, do " "not restart the answer, and do not add a new introduction." ) def _headers() -> Dict[str, str]: return { "Authorization": f"Bearer {os.getenv('GROQ_API_KEY', '')}", "Content-Type": "application/json", } def looks_incomplete(text: str) -> bool: """Cheap heuristic: does this text look like it was chopped? Catches the visible symptoms users report: an unclosed bold marker (the classic '**Physics'), an unterminated code fence, a dangling sentence, or an answer that is absurdly short. """ t = (text or "").strip() if not t: return True if len(t) < 12: return True if t.count("**") % 2: # unbalanced bold return True if t.count("```") % 2: # unterminated code fence return True if t.count("`") % 2: # unterminated inline code return True if t.endswith((",", "and", "or", "the", "of", "to", "—", "-", ":")): return True return False async def _call_once( messages: List[Dict[str, Any]], model: str, temperature: float, max_tokens: int, tools: Optional[List[Dict[str, Any]]] = None, ) -> Tuple[Optional[Dict[str, Any]], str, int, int]: """One Groq call. Returns (message_dict|None, finish_reason, status, retry_after).""" payload: Dict[str, Any] = { "model": model, "messages": messages, "temperature": temperature, "max_tokens": max_tokens, } if tools: payload["tools"] = tools payload["tool_choice"] = "auto" try: async with httpx.AsyncClient(timeout=90.0) as client: r = await client.post(GROQ_URL, headers=_headers(), json=payload) except Exception as e: log.error(f"[ReplyEngine] request failed: {e}") return None, "request_error", 0, 0 if r.status_code == 429: try: retry_after = int(r.headers.get("retry-after", "0") or 0) except (TypeError, ValueError): retry_after = 0 return None, "rate_limit", 429, retry_after if r.status_code >= 400: log.error(f"[ReplyEngine] HTTP {r.status_code}: {r.text[:200]}") return None, f"http_{r.status_code}", r.status_code, 0 try: data = r.json() choice = data["choices"][0] return choice.get("message", {}), choice.get("finish_reason", "") or "", 200, 0 except Exception as e: log.error(f"[ReplyEngine] bad response: {e}") return None, "bad_response", 200, 0 async def generate_complete_reply( messages: List[Dict[str, Any]], model_cfg: Dict[str, Any], *, temperature: Optional[float] = None, max_tokens: Optional[int] = None, tools: Optional[List[Dict[str, Any]]] = None, tool_executor: Optional[Callable[..., Awaitable[Any]]] = None, user_id: str = "", ) -> Tuple[Optional[str], Dict[str, Any]]: """Generate a reply that is never silently cut off. Returns (text or None, meta) where meta carries the model actually used, the last finish_reason, and whether the answer still looks incomplete. """ primary = model_cfg.get("groq_model") or FAILOVER_MODEL # Failover must be a DIFFERENT model, because the per-minute budget is # per model. If the caller is already on the failover model, use the other. secondary = FAILOVER_MODEL if primary != FAILOVER_MODEL else "qwen/qwen3.8-27b" temp = model_cfg.get("temperature", 0.7) if temperature is None else temperature # Per-model budget: reply_cap guards the per-minute output quota, fast_cap # keeps short questions snappy, an explicit max_tokens from the caller wins. if max_tokens: budget = int(max_tokens) else: budget = int(model_cfg.get("reply_cap") or MAX_REPLY_TOKENS) budget = max(256, min(budget, MAX_HARD_TOKENS)) base_messages = [dict(m) for m in messages] convo = [dict(m) for m in base_messages] parts: List[str] = [] used_models: List[str] = [] last_finish = "" failover_used = False for round_no in range(MAX_CONTINUATIONS + 1): got: Optional[Tuple[Dict[str, Any], str]] = None model_try = secondary if failover_used else primary backoff = 3 for attempt in range(MAX_ATTEMPTS_PER_ROUND): msg, finish, status, retry_after = await _call_once( convo, model_try, temp, budget, tools=tools ) if status == 200 and msg is not None: got = (msg, finish) break if status == 429: if model_try != secondary: # Primary quota exhausted: hop to the failover model, which # has its own per-minute budget. log.warning( "[ReplyEngine] 429 on %s — failing over to %s", model_try, secondary ) model_try = secondary failover_used = True continue wait = min(retry_after or backoff, 8) log.warning("[ReplyEngine] failover also 429 — waiting %ss", wait) await asyncio.sleep(wait) backoff = min(backoff * 2, 10) continue # Non-retryable error break if got is None: break msg, last_finish = got used_models.append(model_try) content = (msg.get("content") or "").strip() # Tool round: run the tools, then let the loop continue for the answer. if tools and msg.get("tool_calls") and tool_executor is not None: convo.append( { "role": "assistant", "content": msg.get("content"), "tool_calls": msg["tool_calls"], } ) for tc in msg["tool_calls"]: name = "" args: Dict[str, Any] = {} try: name = tc["function"]["name"] args = json.loads(tc["function"].get("arguments") or "{}") except Exception: args = {} try: output = await tool_executor(name, user_id, **args) except Exception as e: log.error(f"[ReplyEngine] tool {name} failed: {e}") output = {"error": "Tool execution failed."} convo.append( { "role": "tool", "tool_call_id": tc.get("id", ""), "content": json.dumps(output, ensure_ascii=False, default=str)[:4000], } ) continue if content: parts.append(content) if last_finish != "length": break # Hit the output ceiling. Ask it to keep going from where it stopped. convo = [dict(m) for m in base_messages] convo.append({"role": "assistant", "content": "\n".join(parts)}) convo.append({"role": "user", "content": CONTINUE_PROMPT}) log.info("[ReplyEngine] finish_reason=length — continuing (round %s)", round_no + 1) full = "\n".join(p for p in parts if p).strip() meta = { "model": used_models[-1] if used_models else primary, "finish": last_finish, "segments": len(parts), "failover": failover_used, "incomplete": (last_finish == "length") or looks_incomplete(full), } return (full or None), meta