self-trained2 / reply_engine.py
DeepImagix's picture
Neurones Pro 1.5: upgrade neurones_pro.py in place (id neurones-pro-1.5, default everywhere; neurones-pro-1.0 remapped). Fixes the old prompt that hard-capped answers at 100-200 words and banned tables. New: citation-backed automatic web search for time-sensitive questions, two-speed output budget (fast_cap 900 / reply_cap 2400), rewritten system prompt (complete answers, markdown when useful, honest uncertainty, ask-once-when-ambiguous, brief refusals).
68b2187 verified
Raw History Blame Contribute Delete
10 kB
"""
NeuraPrompt β€” Reply Engine (v1.0)
=================================
Solves the "reply cut off mid-sentence" class of bugs.
Why replies were truncated
--------------------------
The Groq organisation is capped at a small number of OUTPUT tokens per minute
(OTPM) for the primary model. One long answer can consume the whole minute's
budget, so the next request either:
* is rejected with 429, or
* gets a tiny remaining budget and stops mid-sentence with
finish_reason == "length" (this is the "**Physics" fragment users saw).
The old code never inspected finish_reason, so a cut-off answer was shown as
if it were complete.
What this module guarantees
---------------------------
1. A per-request output cap so one answer cannot drain the whole minute.
2. On 429: fail over to a DIFFERENT model (separate quota) with backoff,
instead of retrying the same exhausted model.
3. On finish_reason == "length": the partial text is fed back and the model is
asked to continue where it stopped; pieces are joined. Repeats up to
MAX_CONTINUATIONS times.
4. Returns metadata (model actually used, finish reason, whether the answer
still looks incomplete) so callers can be honest with the user.
The module is dependency-free: it accepts a tool executor callable instead of
importing the app, so it cannot create an import cycle with main.py.
"""
from __future__ import annotations
import asyncio
import json
import logging
import os
from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple
import httpx
log = logging.getLogger("reply_engine")
GROQ_URL = "https://api.groq.com/openai/v1/chat/completions"
# Output cap per single request. Deliberately modest: the OTPM budget is shared
# across ALL users of the organisation, so one huge answer starves everyone.
MAX_REPLY_TOKENS = int(os.getenv("MAX_REPLY_TOKENS", "1200"))
# A different model on a separate quota, used when the primary hits 429.
FAILOVER_MODEL = os.getenv("GROQ_FAILOVER_MODEL", "openai/gpt-oss-120b")
# How many times we may ask the model to continue a truncated answer.
MAX_CONTINUATIONS = int(os.getenv("MAX_CONTINUATIONS", "2"))
MAX_HARD_TOKENS = int(os.getenv("MAX_HARD_TOKENS", "4000"))
# Attempts per round (includes the failover attempt).
MAX_ATTEMPTS_PER_ROUND = 3
CONTINUE_PROMPT = (
"Your previous answer was cut off because of a length limit. Continue "
"exactly where you stopped. Do not repeat anything you already wrote, do "
"not restart the answer, and do not add a new introduction."
)
def _headers() -> Dict[str, str]:
return {
"Authorization": f"Bearer {os.getenv('GROQ_API_KEY', '')}",
"Content-Type": "application/json",
}
def looks_incomplete(text: str) -> bool:
"""Cheap heuristic: does this text look like it was chopped?
Catches the visible symptoms users report: an unclosed bold marker
(the classic '**Physics'), an unterminated code fence, a dangling
sentence, or an answer that is absurdly short.
"""
t = (text or "").strip()
if not t:
return True
if len(t) < 12:
return True
if t.count("**") % 2: # unbalanced bold
return True
if t.count("```") % 2: # unterminated code fence
return True
if t.count("`") % 2: # unterminated inline code
return True
if t.endswith((",", "and", "or", "the", "of", "to", "β€”", "-", ":")):
return True
return False
async def _call_once(
messages: List[Dict[str, Any]],
model: str,
temperature: float,
max_tokens: int,
tools: Optional[List[Dict[str, Any]]] = None,
) -> Tuple[Optional[Dict[str, Any]], str, int, int]:
"""One Groq call. Returns (message_dict|None, finish_reason, status, retry_after)."""
payload: Dict[str, Any] = {
"model": model,
"messages": messages,
"temperature": temperature,
"max_tokens": max_tokens,
}
if tools:
payload["tools"] = tools
payload["tool_choice"] = "auto"
try:
async with httpx.AsyncClient(timeout=90.0) as client:
r = await client.post(GROQ_URL, headers=_headers(), json=payload)
except Exception as e:
log.error(f"[ReplyEngine] request failed: {e}")
return None, "request_error", 0, 0
if r.status_code == 429:
try:
retry_after = int(r.headers.get("retry-after", "0") or 0)
except (TypeError, ValueError):
retry_after = 0
return None, "rate_limit", 429, retry_after
if r.status_code >= 400:
log.error(f"[ReplyEngine] HTTP {r.status_code}: {r.text[:200]}")
return None, f"http_{r.status_code}", r.status_code, 0
try:
data = r.json()
choice = data["choices"][0]
return choice.get("message", {}), choice.get("finish_reason", "") or "", 200, 0
except Exception as e:
log.error(f"[ReplyEngine] bad response: {e}")
return None, "bad_response", 200, 0
async def generate_complete_reply(
messages: List[Dict[str, Any]],
model_cfg: Dict[str, Any],
*,
temperature: Optional[float] = None,
max_tokens: Optional[int] = None,
tools: Optional[List[Dict[str, Any]]] = None,
tool_executor: Optional[Callable[..., Awaitable[Any]]] = None,
user_id: str = "",
) -> Tuple[Optional[str], Dict[str, Any]]:
"""Generate a reply that is never silently cut off.
Returns (text or None, meta) where meta carries the model actually used,
the last finish_reason, and whether the answer still looks incomplete.
"""
primary = model_cfg.get("groq_model") or FAILOVER_MODEL
# Failover must be a DIFFERENT model, because the per-minute budget is
# per model. If the caller is already on the failover model, use the other.
secondary = FAILOVER_MODEL if primary != FAILOVER_MODEL else "qwen/qwen3.8-27b"
temp = model_cfg.get("temperature", 0.7) if temperature is None else temperature
# Per-model budget: reply_cap guards the per-minute output quota, fast_cap
# keeps short questions snappy, an explicit max_tokens from the caller wins.
if max_tokens:
budget = int(max_tokens)
else:
budget = int(model_cfg.get("reply_cap") or MAX_REPLY_TOKENS)
budget = max(256, min(budget, MAX_HARD_TOKENS))
base_messages = [dict(m) for m in messages]
convo = [dict(m) for m in base_messages]
parts: List[str] = []
used_models: List[str] = []
last_finish = ""
failover_used = False
for round_no in range(MAX_CONTINUATIONS + 1):
got: Optional[Tuple[Dict[str, Any], str]] = None
model_try = secondary if failover_used else primary
backoff = 3
for attempt in range(MAX_ATTEMPTS_PER_ROUND):
msg, finish, status, retry_after = await _call_once(
convo, model_try, temp, budget, tools=tools
)
if status == 200 and msg is not None:
got = (msg, finish)
break
if status == 429:
if model_try != secondary:
# Primary quota exhausted: hop to the failover model, which
# has its own per-minute budget.
log.warning(
"[ReplyEngine] 429 on %s β€” failing over to %s", model_try, secondary
)
model_try = secondary
failover_used = True
continue
wait = min(retry_after or backoff, 8)
log.warning("[ReplyEngine] failover also 429 β€” waiting %ss", wait)
await asyncio.sleep(wait)
backoff = min(backoff * 2, 10)
continue
# Non-retryable error
break
if got is None:
break
msg, last_finish = got
used_models.append(model_try)
content = (msg.get("content") or "").strip()
# Tool round: run the tools, then let the loop continue for the answer.
if tools and msg.get("tool_calls") and tool_executor is not None:
convo.append(
{
"role": "assistant",
"content": msg.get("content"),
"tool_calls": msg["tool_calls"],
}
)
for tc in msg["tool_calls"]:
name = ""
args: Dict[str, Any] = {}
try:
name = tc["function"]["name"]
args = json.loads(tc["function"].get("arguments") or "{}")
except Exception:
args = {}
try:
output = await tool_executor(name, user_id, **args)
except Exception as e:
log.error(f"[ReplyEngine] tool {name} failed: {e}")
output = {"error": "Tool execution failed."}
convo.append(
{
"role": "tool",
"tool_call_id": tc.get("id", ""),
"content": json.dumps(output, ensure_ascii=False, default=str)[:4000],
}
)
continue
if content:
parts.append(content)
if last_finish != "length":
break
# Hit the output ceiling. Ask it to keep going from where it stopped.
convo = [dict(m) for m in base_messages]
convo.append({"role": "assistant", "content": "\n".join(parts)})
convo.append({"role": "user", "content": CONTINUE_PROMPT})
log.info("[ReplyEngine] finish_reason=length β€” continuing (round %s)", round_no + 1)
full = "\n".join(p for p in parts if p).strip()
meta = {
"model": used_models[-1] if used_models else primary,
"finish": last_finish,
"segments": len(parts),
"failover": failover_used,
"incomplete": (last_finish == "length") or looks_incomplete(full),
}
return (full or None), meta