Spaces:
Running
Running
Neurones Pro 1.5: upgrade neurones_pro.py in place (id neurones-pro-1.5, default everywhere; neurones-pro-1.0 remapped). Fixes the old prompt that hard-capped answers at 100-200 words and banned tables. New: citation-backed automatic web search for time-sensitive questions, two-speed output budget (fast_cap 900 / reply_cap 2400), rewritten system prompt (complete answers, markdown when useful, honest uncertainty, ask-once-when-ambiguous, brief refusals).
68b2187 verified Download reply_engine.py from DeepImagix/self-trained2: direct link, hf CLI and curl.
- Browser
- Download file 10 kB
-
https://huggingface.co/spaces/DeepImagix/self-trained2/resolve/main/reply_engine.py
- Command line
-
hf download hf://spaces/DeepImagix/self-trained2/reply_engine.py
-
curl -L -o reply_engine.py https://huggingface.co/spaces/DeepImagix/self-trained2/resolve/main/reply_engine.py
10 kB
| """ | |
| NeuraPrompt β Reply Engine (v1.0) | |
| ================================= | |
| Solves the "reply cut off mid-sentence" class of bugs. | |
| Why replies were truncated | |
| -------------------------- | |
| The Groq organisation is capped at a small number of OUTPUT tokens per minute | |
| (OTPM) for the primary model. One long answer can consume the whole minute's | |
| budget, so the next request either: | |
| * is rejected with 429, or | |
| * gets a tiny remaining budget and stops mid-sentence with | |
| finish_reason == "length" (this is the "**Physics" fragment users saw). | |
| The old code never inspected finish_reason, so a cut-off answer was shown as | |
| if it were complete. | |
| What this module guarantees | |
| --------------------------- | |
| 1. A per-request output cap so one answer cannot drain the whole minute. | |
| 2. On 429: fail over to a DIFFERENT model (separate quota) with backoff, | |
| instead of retrying the same exhausted model. | |
| 3. On finish_reason == "length": the partial text is fed back and the model is | |
| asked to continue where it stopped; pieces are joined. Repeats up to | |
| MAX_CONTINUATIONS times. | |
| 4. Returns metadata (model actually used, finish reason, whether the answer | |
| still looks incomplete) so callers can be honest with the user. | |
| The module is dependency-free: it accepts a tool executor callable instead of | |
| importing the app, so it cannot create an import cycle with main.py. | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| import json | |
| import logging | |
| import os | |
| from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple | |
| import httpx | |
| log = logging.getLogger("reply_engine") | |
| GROQ_URL = "https://api.groq.com/openai/v1/chat/completions" | |
| # Output cap per single request. Deliberately modest: the OTPM budget is shared | |
| # across ALL users of the organisation, so one huge answer starves everyone. | |
| MAX_REPLY_TOKENS = int(os.getenv("MAX_REPLY_TOKENS", "1200")) | |
| # A different model on a separate quota, used when the primary hits 429. | |
| FAILOVER_MODEL = os.getenv("GROQ_FAILOVER_MODEL", "openai/gpt-oss-120b") | |
| # How many times we may ask the model to continue a truncated answer. | |
| MAX_CONTINUATIONS = int(os.getenv("MAX_CONTINUATIONS", "2")) | |
| MAX_HARD_TOKENS = int(os.getenv("MAX_HARD_TOKENS", "4000")) | |
| # Attempts per round (includes the failover attempt). | |
| MAX_ATTEMPTS_PER_ROUND = 3 | |
| CONTINUE_PROMPT = ( | |
| "Your previous answer was cut off because of a length limit. Continue " | |
| "exactly where you stopped. Do not repeat anything you already wrote, do " | |
| "not restart the answer, and do not add a new introduction." | |
| ) | |
| def _headers() -> Dict[str, str]: | |
| return { | |
| "Authorization": f"Bearer {os.getenv('GROQ_API_KEY', '')}", | |
| "Content-Type": "application/json", | |
| } | |
| def looks_incomplete(text: str) -> bool: | |
| """Cheap heuristic: does this text look like it was chopped? | |
| Catches the visible symptoms users report: an unclosed bold marker | |
| (the classic '**Physics'), an unterminated code fence, a dangling | |
| sentence, or an answer that is absurdly short. | |
| """ | |
| t = (text or "").strip() | |
| if not t: | |
| return True | |
| if len(t) < 12: | |
| return True | |
| if t.count("**") % 2: # unbalanced bold | |
| return True | |
| if t.count("```") % 2: # unterminated code fence | |
| return True | |
| if t.count("`") % 2: # unterminated inline code | |
| return True | |
| if t.endswith((",", "and", "or", "the", "of", "to", "β", "-", ":")): | |
| return True | |
| return False | |
| async def _call_once( | |
| messages: List[Dict[str, Any]], | |
| model: str, | |
| temperature: float, | |
| max_tokens: int, | |
| tools: Optional[List[Dict[str, Any]]] = None, | |
| ) -> Tuple[Optional[Dict[str, Any]], str, int, int]: | |
| """One Groq call. Returns (message_dict|None, finish_reason, status, retry_after).""" | |
| payload: Dict[str, Any] = { | |
| "model": model, | |
| "messages": messages, | |
| "temperature": temperature, | |
| "max_tokens": max_tokens, | |
| } | |
| if tools: | |
| payload["tools"] = tools | |
| payload["tool_choice"] = "auto" | |
| try: | |
| async with httpx.AsyncClient(timeout=90.0) as client: | |
| r = await client.post(GROQ_URL, headers=_headers(), json=payload) | |
| except Exception as e: | |
| log.error(f"[ReplyEngine] request failed: {e}") | |
| return None, "request_error", 0, 0 | |
| if r.status_code == 429: | |
| try: | |
| retry_after = int(r.headers.get("retry-after", "0") or 0) | |
| except (TypeError, ValueError): | |
| retry_after = 0 | |
| return None, "rate_limit", 429, retry_after | |
| if r.status_code >= 400: | |
| log.error(f"[ReplyEngine] HTTP {r.status_code}: {r.text[:200]}") | |
| return None, f"http_{r.status_code}", r.status_code, 0 | |
| try: | |
| data = r.json() | |
| choice = data["choices"][0] | |
| return choice.get("message", {}), choice.get("finish_reason", "") or "", 200, 0 | |
| except Exception as e: | |
| log.error(f"[ReplyEngine] bad response: {e}") | |
| return None, "bad_response", 200, 0 | |
| async def generate_complete_reply( | |
| messages: List[Dict[str, Any]], | |
| model_cfg: Dict[str, Any], | |
| *, | |
| temperature: Optional[float] = None, | |
| max_tokens: Optional[int] = None, | |
| tools: Optional[List[Dict[str, Any]]] = None, | |
| tool_executor: Optional[Callable[..., Awaitable[Any]]] = None, | |
| user_id: str = "", | |
| ) -> Tuple[Optional[str], Dict[str, Any]]: | |
| """Generate a reply that is never silently cut off. | |
| Returns (text or None, meta) where meta carries the model actually used, | |
| the last finish_reason, and whether the answer still looks incomplete. | |
| """ | |
| primary = model_cfg.get("groq_model") or FAILOVER_MODEL | |
| # Failover must be a DIFFERENT model, because the per-minute budget is | |
| # per model. If the caller is already on the failover model, use the other. | |
| secondary = FAILOVER_MODEL if primary != FAILOVER_MODEL else "qwen/qwen3.8-27b" | |
| temp = model_cfg.get("temperature", 0.7) if temperature is None else temperature | |
| # Per-model budget: reply_cap guards the per-minute output quota, fast_cap | |
| # keeps short questions snappy, an explicit max_tokens from the caller wins. | |
| if max_tokens: | |
| budget = int(max_tokens) | |
| else: | |
| budget = int(model_cfg.get("reply_cap") or MAX_REPLY_TOKENS) | |
| budget = max(256, min(budget, MAX_HARD_TOKENS)) | |
| base_messages = [dict(m) for m in messages] | |
| convo = [dict(m) for m in base_messages] | |
| parts: List[str] = [] | |
| used_models: List[str] = [] | |
| last_finish = "" | |
| failover_used = False | |
| for round_no in range(MAX_CONTINUATIONS + 1): | |
| got: Optional[Tuple[Dict[str, Any], str]] = None | |
| model_try = secondary if failover_used else primary | |
| backoff = 3 | |
| for attempt in range(MAX_ATTEMPTS_PER_ROUND): | |
| msg, finish, status, retry_after = await _call_once( | |
| convo, model_try, temp, budget, tools=tools | |
| ) | |
| if status == 200 and msg is not None: | |
| got = (msg, finish) | |
| break | |
| if status == 429: | |
| if model_try != secondary: | |
| # Primary quota exhausted: hop to the failover model, which | |
| # has its own per-minute budget. | |
| log.warning( | |
| "[ReplyEngine] 429 on %s β failing over to %s", model_try, secondary | |
| ) | |
| model_try = secondary | |
| failover_used = True | |
| continue | |
| wait = min(retry_after or backoff, 8) | |
| log.warning("[ReplyEngine] failover also 429 β waiting %ss", wait) | |
| await asyncio.sleep(wait) | |
| backoff = min(backoff * 2, 10) | |
| continue | |
| # Non-retryable error | |
| break | |
| if got is None: | |
| break | |
| msg, last_finish = got | |
| used_models.append(model_try) | |
| content = (msg.get("content") or "").strip() | |
| # Tool round: run the tools, then let the loop continue for the answer. | |
| if tools and msg.get("tool_calls") and tool_executor is not None: | |
| convo.append( | |
| { | |
| "role": "assistant", | |
| "content": msg.get("content"), | |
| "tool_calls": msg["tool_calls"], | |
| } | |
| ) | |
| for tc in msg["tool_calls"]: | |
| name = "" | |
| args: Dict[str, Any] = {} | |
| try: | |
| name = tc["function"]["name"] | |
| args = json.loads(tc["function"].get("arguments") or "{}") | |
| except Exception: | |
| args = {} | |
| try: | |
| output = await tool_executor(name, user_id, **args) | |
| except Exception as e: | |
| log.error(f"[ReplyEngine] tool {name} failed: {e}") | |
| output = {"error": "Tool execution failed."} | |
| convo.append( | |
| { | |
| "role": "tool", | |
| "tool_call_id": tc.get("id", ""), | |
| "content": json.dumps(output, ensure_ascii=False, default=str)[:4000], | |
| } | |
| ) | |
| continue | |
| if content: | |
| parts.append(content) | |
| if last_finish != "length": | |
| break | |
| # Hit the output ceiling. Ask it to keep going from where it stopped. | |
| convo = [dict(m) for m in base_messages] | |
| convo.append({"role": "assistant", "content": "\n".join(parts)}) | |
| convo.append({"role": "user", "content": CONTINUE_PROMPT}) | |
| log.info("[ReplyEngine] finish_reason=length β continuing (round %s)", round_no + 1) | |
| full = "\n".join(p for p in parts if p).strip() | |
| meta = { | |
| "model": used_models[-1] if used_models else primary, | |
| "finish": last_finish, | |
| "segments": len(parts), | |
| "failover": failover_used, | |
| "incomplete": (last_finish == "length") or looks_incomplete(full), | |
| } | |
| return (full or None), meta | |