File size: 10,033 Bytes
4b8904a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
68b2187
4b8904a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
68b2187
 
 
 
 
 
 
4b8904a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
"""
NeuraPrompt β€” Reply Engine (v1.0)
=================================
Solves the "reply cut off mid-sentence" class of bugs.

Why replies were truncated
--------------------------
The Groq organisation is capped at a small number of OUTPUT tokens per minute
(OTPM) for the primary model. One long answer can consume the whole minute's
budget, so the next request either:
  * is rejected with 429, or
  * gets a tiny remaining budget and stops mid-sentence with
    finish_reason == "length"  (this is the "**Physics" fragment users saw).

The old code never inspected finish_reason, so a cut-off answer was shown as
if it were complete.

What this module guarantees
---------------------------
1. A per-request output cap so one answer cannot drain the whole minute.
2. On 429: fail over to a DIFFERENT model (separate quota) with backoff,
   instead of retrying the same exhausted model.
3. On finish_reason == "length": the partial text is fed back and the model is
   asked to continue where it stopped; pieces are joined. Repeats up to
   MAX_CONTINUATIONS times.
4. Returns metadata (model actually used, finish reason, whether the answer
   still looks incomplete) so callers can be honest with the user.

The module is dependency-free: it accepts a tool executor callable instead of
importing the app, so it cannot create an import cycle with main.py.
"""

from __future__ import annotations

import asyncio
import json
import logging
import os
from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple

import httpx

log = logging.getLogger("reply_engine")

GROQ_URL = "https://api.groq.com/openai/v1/chat/completions"

# Output cap per single request. Deliberately modest: the OTPM budget is shared
# across ALL users of the organisation, so one huge answer starves everyone.
MAX_REPLY_TOKENS = int(os.getenv("MAX_REPLY_TOKENS", "1200"))

# A different model on a separate quota, used when the primary hits 429.
FAILOVER_MODEL = os.getenv("GROQ_FAILOVER_MODEL", "openai/gpt-oss-120b")

# How many times we may ask the model to continue a truncated answer.
MAX_CONTINUATIONS = int(os.getenv("MAX_CONTINUATIONS", "2"))
MAX_HARD_TOKENS = int(os.getenv("MAX_HARD_TOKENS", "4000"))

# Attempts per round (includes the failover attempt).
MAX_ATTEMPTS_PER_ROUND = 3

CONTINUE_PROMPT = (
    "Your previous answer was cut off because of a length limit. Continue "
    "exactly where you stopped. Do not repeat anything you already wrote, do "
    "not restart the answer, and do not add a new introduction."
)


def _headers() -> Dict[str, str]:
    return {
        "Authorization": f"Bearer {os.getenv('GROQ_API_KEY', '')}",
        "Content-Type": "application/json",
    }


def looks_incomplete(text: str) -> bool:
    """Cheap heuristic: does this text look like it was chopped?

    Catches the visible symptoms users report: an unclosed bold marker
    (the classic '**Physics'), an unterminated code fence, a dangling
    sentence, or an answer that is absurdly short.
    """
    t = (text or "").strip()
    if not t:
        return True
    if len(t) < 12:
        return True
    if t.count("**") % 2:                 # unbalanced bold
        return True
    if t.count("```") % 2:                # unterminated code fence
        return True
    if t.count("`") % 2:                  # unterminated inline code
        return True
    if t.endswith((",", "and", "or", "the", "of", "to", "β€”", "-", ":")):
        return True
    return False


async def _call_once(
    messages: List[Dict[str, Any]],
    model: str,
    temperature: float,
    max_tokens: int,
    tools: Optional[List[Dict[str, Any]]] = None,
) -> Tuple[Optional[Dict[str, Any]], str, int, int]:
    """One Groq call. Returns (message_dict|None, finish_reason, status, retry_after)."""
    payload: Dict[str, Any] = {
        "model": model,
        "messages": messages,
        "temperature": temperature,
        "max_tokens": max_tokens,
    }
    if tools:
        payload["tools"] = tools
        payload["tool_choice"] = "auto"

    try:
        async with httpx.AsyncClient(timeout=90.0) as client:
            r = await client.post(GROQ_URL, headers=_headers(), json=payload)
    except Exception as e:
        log.error(f"[ReplyEngine] request failed: {e}")
        return None, "request_error", 0, 0

    if r.status_code == 429:
        try:
            retry_after = int(r.headers.get("retry-after", "0") or 0)
        except (TypeError, ValueError):
            retry_after = 0
        return None, "rate_limit", 429, retry_after

    if r.status_code >= 400:
        log.error(f"[ReplyEngine] HTTP {r.status_code}: {r.text[:200]}")
        return None, f"http_{r.status_code}", r.status_code, 0

    try:
        data = r.json()
        choice = data["choices"][0]
        return choice.get("message", {}), choice.get("finish_reason", "") or "", 200, 0
    except Exception as e:
        log.error(f"[ReplyEngine] bad response: {e}")
        return None, "bad_response", 200, 0


async def generate_complete_reply(
    messages: List[Dict[str, Any]],
    model_cfg: Dict[str, Any],
    *,
    temperature: Optional[float] = None,
    max_tokens: Optional[int] = None,
    tools: Optional[List[Dict[str, Any]]] = None,
    tool_executor: Optional[Callable[..., Awaitable[Any]]] = None,
    user_id: str = "",
) -> Tuple[Optional[str], Dict[str, Any]]:
    """Generate a reply that is never silently cut off.

    Returns (text or None, meta) where meta carries the model actually used,
    the last finish_reason, and whether the answer still looks incomplete.
    """
    primary = model_cfg.get("groq_model") or FAILOVER_MODEL
    # Failover must be a DIFFERENT model, because the per-minute budget is
    # per model. If the caller is already on the failover model, use the other.
    secondary = FAILOVER_MODEL if primary != FAILOVER_MODEL else "qwen/qwen3.8-27b"
    temp = model_cfg.get("temperature", 0.7) if temperature is None else temperature
    # Per-model budget: reply_cap guards the per-minute output quota, fast_cap
    # keeps short questions snappy, an explicit max_tokens from the caller wins.
    if max_tokens:
        budget = int(max_tokens)
    else:
        budget = int(model_cfg.get("reply_cap") or MAX_REPLY_TOKENS)
    budget = max(256, min(budget, MAX_HARD_TOKENS))

    base_messages = [dict(m) for m in messages]
    convo = [dict(m) for m in base_messages]
    parts: List[str] = []
    used_models: List[str] = []
    last_finish = ""
    failover_used = False

    for round_no in range(MAX_CONTINUATIONS + 1):
        got: Optional[Tuple[Dict[str, Any], str]] = None
        model_try = secondary if failover_used else primary
        backoff = 3

        for attempt in range(MAX_ATTEMPTS_PER_ROUND):
            msg, finish, status, retry_after = await _call_once(
                convo, model_try, temp, budget, tools=tools
            )

            if status == 200 and msg is not None:
                got = (msg, finish)
                break

            if status == 429:
                if model_try != secondary:
                    # Primary quota exhausted: hop to the failover model, which
                    # has its own per-minute budget.
                    log.warning(
                        "[ReplyEngine] 429 on %s β€” failing over to %s", model_try, secondary
                    )
                    model_try = secondary
                    failover_used = True
                    continue
                wait = min(retry_after or backoff, 8)
                log.warning("[ReplyEngine] failover also 429 β€” waiting %ss", wait)
                await asyncio.sleep(wait)
                backoff = min(backoff * 2, 10)
                continue

            # Non-retryable error
            break

        if got is None:
            break

        msg, last_finish = got
        used_models.append(model_try)
        content = (msg.get("content") or "").strip()

        # Tool round: run the tools, then let the loop continue for the answer.
        if tools and msg.get("tool_calls") and tool_executor is not None:
            convo.append(
                {
                    "role": "assistant",
                    "content": msg.get("content"),
                    "tool_calls": msg["tool_calls"],
                }
            )
            for tc in msg["tool_calls"]:
                name = ""
                args: Dict[str, Any] = {}
                try:
                    name = tc["function"]["name"]
                    args = json.loads(tc["function"].get("arguments") or "{}")
                except Exception:
                    args = {}
                try:
                    output = await tool_executor(name, user_id, **args)
                except Exception as e:
                    log.error(f"[ReplyEngine] tool {name} failed: {e}")
                    output = {"error": "Tool execution failed."}
                convo.append(
                    {
                        "role": "tool",
                        "tool_call_id": tc.get("id", ""),
                        "content": json.dumps(output, ensure_ascii=False, default=str)[:4000],
                    }
                )
            continue

        if content:
            parts.append(content)

        if last_finish != "length":
            break

        # Hit the output ceiling. Ask it to keep going from where it stopped.
        convo = [dict(m) for m in base_messages]
        convo.append({"role": "assistant", "content": "\n".join(parts)})
        convo.append({"role": "user", "content": CONTINUE_PROMPT})
        log.info("[ReplyEngine] finish_reason=length β€” continuing (round %s)", round_no + 1)

    full = "\n".join(p for p in parts if p).strip()
    meta = {
        "model": used_models[-1] if used_models else primary,
        "finish": last_finish,
        "segments": len(parts),
        "failover": failover_used,
        "incomplete": (last_finish == "length") or looks_incomplete(full),
    }
    return (full or None), meta