File size: 20,151 Bytes
e29c7ce
 
 
 
 
 
 
d1d1729
e29c7ce
 
addec5c
e29c7ce
 
 
ab0a14d
caa1f90
34bd4b4
7e8acb6
e29c7ce
6afc342
31404a1
c57e87b
e29c7ce
5cd1a47
 
bdc8662
e29c7ce
ab0a14d
e29c7ce
5cd1a47
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
773e767
17ba573
 
 
 
 
 
 
 
e29c7ce
bdc8662
 
 
addec5c
 
bdc8662
 
 
 
 
 
 
addec5c
34bd4b4
8880040
c0d4963
ddc024a
 
ed3adb7
ddc024a
7e8acb6
e29c7ce
 
 
 
d1d1729
 
e29c7ce
addec5c
e29c7ce
ab0a14d
 
 
31c3c78
 
 
 
 
 
 
e29c7ce
31c3c78
 
 
 
 
 
 
 
 
 
 
e29c7ce
773e767
 
7e8acb6
 
 
 
832430f
ab0a14d
773e767
ab0a14d
 
832430f
ab0a14d
 
 
 
 
832430f
ab0a14d
 
 
addec5c
 
 
 
 
 
 
6afc342
addec5c
 
e29c7ce
bdc8662
31c3c78
 
17ba573
 
 
7e8acb6
31c3c78
 
 
 
 
 
 
bdc8662
7e8acb6
8880040
bdc8662
832430f
17ba573
34bd4b4
 
 
832430f
bdc8662
7e8acb6
 
bdc8662
 
8880040
bdc8662
 
7e8acb6
8880040
5cd1a47
 
8880040
5cd1a47
 
 
 
 
 
ddc024a
5cd1a47
ddc024a
ed3adb7
eaaf029
ddc024a
e29c7ce
 
 
 
 
 
bdc8662
 
8880040
e29c7ce
c57e87b
6afc342
 
 
addec5c
 
5cd1a47
7e8acb6
5cd1a47
 
 
7e8acb6
 
 
 
5cd1a47
 
 
 
 
 
 
 
 
 
 
 
 
7e8acb6
5cd1a47
 
 
 
 
 
 
832430f
 
 
 
e29c7ce
ab0a14d
7e8acb6
 
773e767
e29c7ce
7e8acb6
 
e29c7ce
ab0a14d
bdc8662
 
 
17ba573
 
bdc8662
 
 
 
 
 
ddc024a
bdc8662
 
7e8acb6
 
bdc8662
34bd4b4
 
bdc8662
 
17ba573
 
bdc8662
 
 
 
 
 
ddc024a
bdc8662
 
7e8acb6
 
bdc8662
34bd4b4
 
bdc8662
ab0a14d
 
7e8acb6
 
 
 
34bd4b4
 
 
 
 
 
 
 
 
 
 
ab0a14d
e29c7ce
7e8acb6
 
 
832430f
34bd4b4
 
 
 
addec5c
34bd4b4
addec5c
e29c7ce
bdc8662
 
832430f
7e8acb6
17ba573
7e8acb6
 
34bd4b4
 
 
bdc8662
 
 
8880040
caa1f90
 
 
 
 
8880040
 
 
 
7e8acb6
 
addec5c
 
c0d4963
addec5c
 
 
 
 
 
 
 
c57e87b
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
"""
Chat engine β€” core conversation loop.

Connects prompt_builder + memory + parser into a single streaming generator
that app.py (or any UI layer) can consume with `yield from`.

Separation of concerns:
    - app.py      : standalone browser UI and HTTP server entry point
    - chat.py     : LLM call, streaming, post-stream memory save
    - prompt_builder.py : system prompt assembly
    - memory.py   : session-scoped snapshots and periodic memory extraction
    - parser.py   : raw response β†’ typed segments β†’ markdown
"""

import logging
import re
import time
from uuid import uuid4
from huggingface_hub import InferenceClient
from engine.memory import sync_history, append_exchange, bounded_recent_context
from engine.memory_jobs import schedule_memory_update
from engine.relationship import relationship_to_prompt
from engine.parser import parse_response, segments_to_markdown
from engine.prompt_builder import (build_messages, load_character, build_story_prompt,
                                   select_active_story_characters, build_compact_story_character_prompt)
from engine.translator import translate_from_english, translate_to_english

logger = logging.getLogger(__name__)

# Shared conservative context envelope for selectable providers with different limits.
TOTAL_CONTEXT_TOKEN_BUDGET = 12_000
CONTEXT_SAFETY_TOKENS = 600
CHARS_PER_ESTIMATED_TOKEN = 3


def _fit_messages_to_budget(messages: list[dict[str, str]], max_tokens: int) -> tuple[list[dict[str, str]], int, int]:
    """Keep output and provider safety headroom inside a shared estimated context budget."""
    prompt_budget = max(1_000, TOTAL_CONTEXT_TOKEN_BUDGET - max(0, max_tokens) - CONTEXT_SAFETY_TOKENS)
    char_budget = prompt_budget * CHARS_PER_ESTIMATED_TOKEN
    result = [dict(message) for message in messages]
    # Current input (last message) and system rules are protected; discard the oldest history first.
    while sum(len(item.get("content", "")) for item in result) > char_budget and len(result) > 2:
        result.pop(1)
    excess = sum(len(item.get("content", "")) for item in result) - char_budget
    if excess > 0 and len(result) > 1:
        system = result[0].get("content", "")
        # Compress optional system context in priority order, retaining a marker and some content.
        priorities = (
            ("LONG-TERM MEMORY", 500), ("CONVERSATION SUMMARY", 700),
            ("PLAYER'S PREFERRED STORY DIRECTION", 300),
            ("USER'S OPTIONAL STORY PREMISE", 300),
            ("PLAYER CHARACTER DETAILS", 300), ("THE USER IS PLAYING", 300),
            ("STORY GROUP:", 6_000),
        )
        sections = system.split("\n\n")
        for marker, minimum in priorities:
            index = next((i for i, section in enumerate(sections) if marker in section), None)
            if index is None or len(sections[index]) <= minimum:
                continue
            reduction = min(excess, len(sections[index]) - minimum)
            keep = max(minimum, len(sections[index]) - reduction)
            sections[index] = sections[index][:keep].rstrip() + "\n[Context shortened to fit the shared token budget.]"
            excess -= reduction
            if excess <= 0:
                break
        result[0]["content"] = "\n\n".join(sections)
    estimated = (sum(len(item.get("content", "")) for item in result) + CHARS_PER_ESTIMATED_TOKEN - 1) // CHARS_PER_ESTIMATED_TOKEN
    return result, prompt_budget, estimated


def _chat_completion(client, messages, **kwargs):
    """Adapt chat completion calls for HF InferenceClient or OpenAI HF Router."""
    if isinstance(kwargs.get("model"), str) and kwargs["model"].endswith(":featherless-ai"):
        model = kwargs.pop("model")
        return client.chat.completions.create(model=model, messages=messages, **kwargs)
    return client.chat_completion(messages, **kwargs)


def generate_response(
    message:            str,
    character_id:       str,
    client:             InferenceClient,
    session_id:         str,
    history:            list[dict],
    persona:            dict | None = None,
    model:              str   = "Qwen/Qwen3-235B-A22B-Instruct-2507",
    max_tokens:         int   = 512,
    temperature:        float = 0.7,
    top_p:              float = 0.95,
    language:           str   = "English",
    translation_model:  str | None = None,
    use_translator:     bool  = False,
    deadline:           float | None = None,
    story:              dict | None = None,
    hidden_opening:     bool = False,
    story_length:       str = "balanced",
    story_creativity:   str = "balanced",
    point_of_view:      str = "third_person_limited",
    frequency_penalty:  float = 0.3,
    trace_id:           str | None = None,
):
    """
    Core streaming generator.

    Yields the accumulated raw response after each token arrives so the web
    layer can display it. After the stream ends, yields one final
    formatted version (actions italic, thoughts em-dashed) and saves
    the exchange to session-scoped memory.

    All errors are caught and surfaced as in-chat messages so the UI
    never silently breaks.

    Translation modes (when language β‰  English):
        use_translator=True  β€” dedicated translation_model translates user input
                               to English and response back to user's language.
                               Main model always operates in English.
        use_translator=False β€” no separate model. Main model receives the language
                               instruction directly and responds in user's language.

    Args:
        message:          Current user input.
        character_id:     Which character to respond as.
        client:           Initialised HuggingFace InferenceClient.
        persona:          Optional user persona dict.
        model:            HuggingFace model string.
        max_tokens:       Maximum tokens to generate.
        temperature:      Sampling temperature.
        top_p:            Nucleus sampling probability.
        language:         UI language selected by the user.
        translation_model: Model used when use_translator=True.
        use_translator:   True = separate translator, False = main model handles language.
    """
    
    
    trace_id = trace_id or uuid4().hex[:10]
    request_started = time.monotonic()
    logger.info("[Chat][%s] start character=%s model=%s language=%s story=%s hidden_opening=%s",
                trace_id, character_id, model, language, bool(story), hidden_opening)

    # ── Load character ─────────────────────────────────────────────────────────
    
    try:
        character = load_character(character_id)
        logger.debug(f"[Chat] Character loaded: '{character.get('name', character_id)}'")
    except FileNotFoundError:
        logger.error(f"[Chat] Character not found: {character_id}")
        yield f"*(Character '{character_id}' could not be loaded.)*"
        return
    except Exception as e:
        logger.error(f"[Chat] Unexpected error loading character: {e}", exc_info=True)
        yield "*(Something went wrong loading the character. Check the logs.)*"
        return

    # Synchronize the visible transcript before assembling model context.
    try:
        conversation = sync_history(session_id, character_id, history)
    except Exception:
        logger.exception("Could not synchronize conversation memory.")
        yield "*(Conversation history could not be loaded. Please try again.)*"
        return
    recent_messages, context_omitted = bounded_recent_context(conversation)
    summary = conversation['summary']
    long_term_memory = conversation['long_term_memory']

    # ── Translation mode? ──────────────────────────────────────────────────────
    is_non_english = language.strip().lower() != "english"
    use_translation = use_translator and is_non_english and bool(translation_model)
    translation_client = client
    if use_translation and isinstance(model, str) and model.endswith(":featherless-ai"):
        from huggingface_hub import InferenceClient
        translation_client = InferenceClient(token=getattr(client, "api_key", None), timeout=240)

    if is_non_english and not use_translator:
        logger.info(f"[Chat] Translation: OFF β€” main model will respond in '{language}' directly")
    elif use_translation:
        logger.info(f"[Chat] Translation: ON β†’ model='{translation_model}'")
    else:
        logger.info("[Chat] Translation: not needed (language is English)")

    translation_started = time.monotonic()
    if use_translation and message.strip():
        # Translate user message β†’ English before sending to main model
        logger.debug(f"[Chat] Translating user message: {message[:80]!r}")
        en_message = translate_to_english(message, language, translation_client, translation_model)
        if deadline is not None and time.monotonic() >= deadline:
            yield "[[CHAT_ERROR:timeout]]"
            return
        logger.debug(f"[Chat] User message (EN): {en_message[:80]!r}")
        prompt_language = "English"   # main model always responds in English
        logger.info("[Chat][%s] input_translation_seconds=%.2f input_chars=%d output_chars=%d",
                    trace_id, time.monotonic() - translation_started, len(message), len(en_message))
    else:
        en_message      = message
        prompt_language = "English" if use_translation else language

    # ── Build prompt ───────────────────────────────────────────────────────────
    prompt_started = time.monotonic()
    story_prompt = None
    active_story_group = None
    full_cast_count = 0
    if story:
        full_cast_count = len(story.get("group", {}).get("characters", []))
        active_story_group = select_active_story_characters(
            story["group"], story["role"], recent_messages, en_message,
            summary=summary, long_term_memory=long_term_memory,
            max_major=15, max_supporting=5,
        )
        story_prompt = build_story_prompt(
            active_story_group, story["role"], story.get("topic", ""),
            response_length=story_length, creativity=story_creativity,
            point_of_view=point_of_view,
            themes=story.get("themes"), next_plot=story.get("next_plot", ""),
        )
    messages = build_messages(
        character=character,
        persona=persona,
        summary=summary,
        long_term_memory=long_term_memory,
        recent_messages=recent_messages,
        current_message=en_message,
        language=prompt_language,
        story_prompt=story_prompt,
    )
    messages[0]['content'] += '\n\n' + relationship_to_prompt(conversation['relationship'])
    if context_omitted:
        messages[0]['content'] += ('\n\nSome unsummarized history is temporarily omitted to keep context bounded. '
                                   'Do not invent details from missing turns.')
    if use_translation:
        messages[0]['content'] += "\n\nReply in English, even if earlier messages use another language."
    messages, prompt_budget_tokens, estimated_prompt_tokens = _fit_messages_to_budget(messages, max_tokens)
    prompt_chars = sum(len(item.get("content", "")) for item in messages)
    active_cast = active_story_group.get("characters", []) if active_story_group else []
    cast_catalog_chars = sum(len(build_compact_story_character_prompt(load_character(member["id"])))
                             for member in active_cast)
    user_context_chars = sum(len(str(value or "")) for value in (persona or {}).values())
    story_context_chars = sum(len(str(story.get(key, "") or "")) for key in ("topic", "next_plot")) if story else 0
    if story:
        story_context_chars += sum(len(str(value or "")) for value in story.get("themes", {}).values())
    summary_chars = min(len(summary or ""), 2200)
    memory_chars = 0
    remaining_memory_chars = 2600
    for item in sorted(long_term_memory, key=lambda mem: (0 if mem.get("origin") == "manual" else 1,
                                                          {"high": 0, "medium": 1, "low": 2}.get(mem.get("importance", "medium"), 1))):
        if remaining_memory_chars <= 32:
            break
        memory_chars += min(len(str(item.get("content", ""))), 360, remaining_memory_chars - 32) + 8
        remaining_memory_chars -= min(len(str(item.get("content", ""))), 360, remaining_memory_chars - 32) + 8
    has_current_user = bool(messages and messages[-1].get("role") == "user")
    history_messages = messages[1:-1] if has_current_user else messages[1:]
    history_chars = sum(len(item.get("content", "")) for item in history_messages)
    current_message_chars = len(messages[-1].get("content", "")) if has_current_user else 0
    logger.info(
        "[Chat][%s] prompt_ready seconds=%.2f total_prompt_chars=%d estimated_prompt_tokens=%d prompt_budget_tokens=%d output_tokens_reserved=%d safety_tokens=%d estimated_total_tokens=%d system_chars=%d history_messages=%d history_chars=%d current_message_chars=%d story_prompt_chars=%d active_cast_chars=%d active_cast_count=%d full_cast_count=%d summary_chars=%d long_term_memory_chars=%d persona_chars=%d story_context_chars=%d omitted_history=%s",
        trace_id, time.monotonic() - prompt_started, prompt_chars, estimated_prompt_tokens,
        prompt_budget_tokens, max_tokens, CONTEXT_SAFETY_TOKENS,
        estimated_prompt_tokens + max_tokens + CONTEXT_SAFETY_TOKENS,
        len(messages[0].get("content", "")), len(history_messages), history_chars, current_message_chars,
        len(story_prompt) if story_prompt else 0, cast_catalog_chars, len(active_cast), full_cast_count,
        summary_chars, memory_chars, user_context_chars, story_context_chars, context_omitted)
    logger.debug(
        f"[Chat] Prompt built: {len(messages[0]['content'])} chars system | "
        f"{len(messages)} total messages"
    )

    # ── LLM call ──────────────────────────────────────────────────────────────
    logger.info("[Chat][%s] llm_request model=%s max_tokens=%d temperature=%.2f top_p=%.2f frequency_penalty=%.2f translation=%s",
                trace_id, model, max_tokens, temperature, top_p, frequency_penalty, use_translation)
    
    raw_response = ""
    llm_started = time.monotonic()
    first_token_seconds = None

    try:
        if use_translation:
            # Collect silently β€” yield placeholder while model generates
            yield "*(…)*"
            for chunk in _chat_completion(
                client,
                messages,
                model=model,
                max_tokens=max_tokens,
                stream=True,
                temperature=temperature,
                top_p=top_p,
                frequency_penalty=frequency_penalty,
            ):
                if chunk.choices and chunk.choices[0].delta.content:
                    if first_token_seconds is None:
                        first_token_seconds = time.monotonic() - llm_started
                    raw_response += chunk.choices[0].delta.content
                if deadline is not None and time.monotonic() >= deadline:
                    raise TimeoutError("The model response exceeded its deadline.")
        else:
            # Stream tokens in real time
            for chunk in _chat_completion(
                client,
                messages,
                model=model,
                max_tokens=max_tokens,
                stream=True,
                temperature=temperature,
                top_p=top_p,
                frequency_penalty=frequency_penalty,
            ):
                if chunk.choices and chunk.choices[0].delta.content:
                    if first_token_seconds is None:
                        first_token_seconds = time.monotonic() - llm_started
                    raw_response += chunk.choices[0].delta.content
                if deadline is not None and time.monotonic() >= deadline:
                    raise TimeoutError("The model response exceeded its deadline.")
                yield raw_response

    except Exception as e:
        logger.error("[Chat][%s] llm_failed model=%s elapsed_seconds=%.2f first_token_seconds=%s error=%s",
                     trace_id, model, time.monotonic() - llm_started,
                     f"{first_token_seconds:.2f}" if first_token_seconds is not None else "none", e,
                     exc_info=True)
        status = getattr(e, "status_code", getattr(getattr(e, "response", None), "status_code", None))
        detail = str(e).casefold()
        if deadline is not None and time.monotonic() >= deadline:
            code = "timeout"
        elif status in (429, 503) or any(word in detail for word in ("rate limit", "too busy", "overloaded", "capacity")):
            code = "busy"
        elif status in (408, 504) or isinstance(e, (TimeoutError,)) or any(word in detail for word in ("timeout", "timed out", "deadline")):
            code = "timeout"
        else:
            code = "no_response"
        yield f"[[CHAT_ERROR:{code}]]"
        return

    logger.info("[Chat][%s] llm_stream_complete elapsed_seconds=%.2f first_token_seconds=%s response_chars=%d",
                trace_id, time.monotonic() - llm_started,
                f"{first_token_seconds:.2f}" if first_token_seconds is not None else "none", len(raw_response))

    if deadline is not None and time.monotonic() >= deadline:
        yield "[[CHAT_ERROR:timeout]]"
        return

    if not raw_response.strip():
        yield "[[CHAT_ERROR:empty]]"
        return

    # ── Translate response β†’ user's language, format, yield ───────────────────
    if use_translation:
        logger.debug(f"[Chat] Translating response β†’ {language} ({len(raw_response)} chars)")
        output_translation_started = time.monotonic()
        final_text = translate_from_english(raw_response, language, translation_client, translation_model)
        logger.info("[Chat][%s] output_translation_seconds=%.2f input_chars=%d output_chars=%d",
                    trace_id, time.monotonic() - output_translation_started, len(raw_response), len(final_text))
        if deadline is not None and time.monotonic() >= deadline:
            yield "[[CHAT_ERROR:timeout]]"
            return
    else:
        final_text = raw_response

    if story:
        # Old and instruction-following models may still include these wrappers.
        # Keep physical action as prose, but never expose private-thought blocks.
        formatted = re.sub(r"<thought\b[^>]*>.*?</thought\s*>", "", final_text,
                           flags=re.IGNORECASE | re.DOTALL)
        formatted = re.sub(r"</?action\b[^>]*>", "", formatted, flags=re.IGNORECASE).strip()
    else:
        segments  = parse_response(final_text)
        logger.debug(f"[Chat] Parser: {len(segments)} segments β†’ {[s['type'] for s in segments]}")
        formatted = segments_to_markdown(segments)
    logger.info("[Chat][%s] response_ready chars=%d total_engine_seconds=%.2f",
                trace_id, len(formatted), time.monotonic() - request_started)
    # Store exactly what the user sees, including translated text.
    try:
        saved = append_exchange(session_id, character_id, conversation, "" if hidden_opening else message, formatted)
    except Exception:
        logger.exception("Could not save conversation memory.")
        yield formatted + "\n\n*(This response could not be saved. Please try again.)*"
        return
    if not saved:
        yield "*(Conversation changed while generating. Please try again.)*"
        return
    yield formatted
    schedule_memory_update(session_id, character_id, saved, client, model, persona=persona)