Spaces:
Running
Running
File size: 20,151 Bytes
e29c7ce d1d1729 e29c7ce addec5c e29c7ce ab0a14d caa1f90 34bd4b4 7e8acb6 e29c7ce 6afc342 31404a1 c57e87b e29c7ce 5cd1a47 bdc8662 e29c7ce ab0a14d e29c7ce 5cd1a47 773e767 17ba573 e29c7ce bdc8662 addec5c bdc8662 addec5c 34bd4b4 8880040 c0d4963 ddc024a ed3adb7 ddc024a 7e8acb6 e29c7ce d1d1729 e29c7ce addec5c e29c7ce ab0a14d 31c3c78 e29c7ce 31c3c78 e29c7ce 773e767 7e8acb6 832430f ab0a14d 773e767 ab0a14d 832430f ab0a14d 832430f ab0a14d addec5c 6afc342 addec5c e29c7ce bdc8662 31c3c78 17ba573 7e8acb6 31c3c78 bdc8662 7e8acb6 8880040 bdc8662 832430f 17ba573 34bd4b4 832430f bdc8662 7e8acb6 bdc8662 8880040 bdc8662 7e8acb6 8880040 5cd1a47 8880040 5cd1a47 ddc024a 5cd1a47 ddc024a ed3adb7 eaaf029 ddc024a e29c7ce bdc8662 8880040 e29c7ce c57e87b 6afc342 addec5c 5cd1a47 7e8acb6 5cd1a47 7e8acb6 5cd1a47 7e8acb6 5cd1a47 832430f e29c7ce ab0a14d 7e8acb6 773e767 e29c7ce 7e8acb6 e29c7ce ab0a14d bdc8662 17ba573 bdc8662 ddc024a bdc8662 7e8acb6 bdc8662 34bd4b4 bdc8662 17ba573 bdc8662 ddc024a bdc8662 7e8acb6 bdc8662 34bd4b4 bdc8662 ab0a14d 7e8acb6 34bd4b4 ab0a14d e29c7ce 7e8acb6 832430f 34bd4b4 addec5c 34bd4b4 addec5c e29c7ce bdc8662 832430f 7e8acb6 17ba573 7e8acb6 34bd4b4 bdc8662 8880040 caa1f90 8880040 7e8acb6 addec5c c0d4963 addec5c c57e87b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 | """
Chat engine β core conversation loop.
Connects prompt_builder + memory + parser into a single streaming generator
that app.py (or any UI layer) can consume with `yield from`.
Separation of concerns:
- app.py : standalone browser UI and HTTP server entry point
- chat.py : LLM call, streaming, post-stream memory save
- prompt_builder.py : system prompt assembly
- memory.py : session-scoped snapshots and periodic memory extraction
- parser.py : raw response β typed segments β markdown
"""
import logging
import re
import time
from uuid import uuid4
from huggingface_hub import InferenceClient
from engine.memory import sync_history, append_exchange, bounded_recent_context
from engine.memory_jobs import schedule_memory_update
from engine.relationship import relationship_to_prompt
from engine.parser import parse_response, segments_to_markdown
from engine.prompt_builder import (build_messages, load_character, build_story_prompt,
select_active_story_characters, build_compact_story_character_prompt)
from engine.translator import translate_from_english, translate_to_english
logger = logging.getLogger(__name__)
# Shared conservative context envelope for selectable providers with different limits.
TOTAL_CONTEXT_TOKEN_BUDGET = 12_000
CONTEXT_SAFETY_TOKENS = 600
CHARS_PER_ESTIMATED_TOKEN = 3
def _fit_messages_to_budget(messages: list[dict[str, str]], max_tokens: int) -> tuple[list[dict[str, str]], int, int]:
"""Keep output and provider safety headroom inside a shared estimated context budget."""
prompt_budget = max(1_000, TOTAL_CONTEXT_TOKEN_BUDGET - max(0, max_tokens) - CONTEXT_SAFETY_TOKENS)
char_budget = prompt_budget * CHARS_PER_ESTIMATED_TOKEN
result = [dict(message) for message in messages]
# Current input (last message) and system rules are protected; discard the oldest history first.
while sum(len(item.get("content", "")) for item in result) > char_budget and len(result) > 2:
result.pop(1)
excess = sum(len(item.get("content", "")) for item in result) - char_budget
if excess > 0 and len(result) > 1:
system = result[0].get("content", "")
# Compress optional system context in priority order, retaining a marker and some content.
priorities = (
("LONG-TERM MEMORY", 500), ("CONVERSATION SUMMARY", 700),
("PLAYER'S PREFERRED STORY DIRECTION", 300),
("USER'S OPTIONAL STORY PREMISE", 300),
("PLAYER CHARACTER DETAILS", 300), ("THE USER IS PLAYING", 300),
("STORY GROUP:", 6_000),
)
sections = system.split("\n\n")
for marker, minimum in priorities:
index = next((i for i, section in enumerate(sections) if marker in section), None)
if index is None or len(sections[index]) <= minimum:
continue
reduction = min(excess, len(sections[index]) - minimum)
keep = max(minimum, len(sections[index]) - reduction)
sections[index] = sections[index][:keep].rstrip() + "\n[Context shortened to fit the shared token budget.]"
excess -= reduction
if excess <= 0:
break
result[0]["content"] = "\n\n".join(sections)
estimated = (sum(len(item.get("content", "")) for item in result) + CHARS_PER_ESTIMATED_TOKEN - 1) // CHARS_PER_ESTIMATED_TOKEN
return result, prompt_budget, estimated
def _chat_completion(client, messages, **kwargs):
"""Adapt chat completion calls for HF InferenceClient or OpenAI HF Router."""
if isinstance(kwargs.get("model"), str) and kwargs["model"].endswith(":featherless-ai"):
model = kwargs.pop("model")
return client.chat.completions.create(model=model, messages=messages, **kwargs)
return client.chat_completion(messages, **kwargs)
def generate_response(
message: str,
character_id: str,
client: InferenceClient,
session_id: str,
history: list[dict],
persona: dict | None = None,
model: str = "Qwen/Qwen3-235B-A22B-Instruct-2507",
max_tokens: int = 512,
temperature: float = 0.7,
top_p: float = 0.95,
language: str = "English",
translation_model: str | None = None,
use_translator: bool = False,
deadline: float | None = None,
story: dict | None = None,
hidden_opening: bool = False,
story_length: str = "balanced",
story_creativity: str = "balanced",
point_of_view: str = "third_person_limited",
frequency_penalty: float = 0.3,
trace_id: str | None = None,
):
"""
Core streaming generator.
Yields the accumulated raw response after each token arrives so the web
layer can display it. After the stream ends, yields one final
formatted version (actions italic, thoughts em-dashed) and saves
the exchange to session-scoped memory.
All errors are caught and surfaced as in-chat messages so the UI
never silently breaks.
Translation modes (when language β English):
use_translator=True β dedicated translation_model translates user input
to English and response back to user's language.
Main model always operates in English.
use_translator=False β no separate model. Main model receives the language
instruction directly and responds in user's language.
Args:
message: Current user input.
character_id: Which character to respond as.
client: Initialised HuggingFace InferenceClient.
persona: Optional user persona dict.
model: HuggingFace model string.
max_tokens: Maximum tokens to generate.
temperature: Sampling temperature.
top_p: Nucleus sampling probability.
language: UI language selected by the user.
translation_model: Model used when use_translator=True.
use_translator: True = separate translator, False = main model handles language.
"""
trace_id = trace_id or uuid4().hex[:10]
request_started = time.monotonic()
logger.info("[Chat][%s] start character=%s model=%s language=%s story=%s hidden_opening=%s",
trace_id, character_id, model, language, bool(story), hidden_opening)
# ββ Load character βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
try:
character = load_character(character_id)
logger.debug(f"[Chat] Character loaded: '{character.get('name', character_id)}'")
except FileNotFoundError:
logger.error(f"[Chat] Character not found: {character_id}")
yield f"*(Character '{character_id}' could not be loaded.)*"
return
except Exception as e:
logger.error(f"[Chat] Unexpected error loading character: {e}", exc_info=True)
yield "*(Something went wrong loading the character. Check the logs.)*"
return
# Synchronize the visible transcript before assembling model context.
try:
conversation = sync_history(session_id, character_id, history)
except Exception:
logger.exception("Could not synchronize conversation memory.")
yield "*(Conversation history could not be loaded. Please try again.)*"
return
recent_messages, context_omitted = bounded_recent_context(conversation)
summary = conversation['summary']
long_term_memory = conversation['long_term_memory']
# ββ Translation mode? ββββββββββββββββββββββββββββββββββββββββββββββββββββββ
is_non_english = language.strip().lower() != "english"
use_translation = use_translator and is_non_english and bool(translation_model)
translation_client = client
if use_translation and isinstance(model, str) and model.endswith(":featherless-ai"):
from huggingface_hub import InferenceClient
translation_client = InferenceClient(token=getattr(client, "api_key", None), timeout=240)
if is_non_english and not use_translator:
logger.info(f"[Chat] Translation: OFF β main model will respond in '{language}' directly")
elif use_translation:
logger.info(f"[Chat] Translation: ON β model='{translation_model}'")
else:
logger.info("[Chat] Translation: not needed (language is English)")
translation_started = time.monotonic()
if use_translation and message.strip():
# Translate user message β English before sending to main model
logger.debug(f"[Chat] Translating user message: {message[:80]!r}")
en_message = translate_to_english(message, language, translation_client, translation_model)
if deadline is not None and time.monotonic() >= deadline:
yield "[[CHAT_ERROR:timeout]]"
return
logger.debug(f"[Chat] User message (EN): {en_message[:80]!r}")
prompt_language = "English" # main model always responds in English
logger.info("[Chat][%s] input_translation_seconds=%.2f input_chars=%d output_chars=%d",
trace_id, time.monotonic() - translation_started, len(message), len(en_message))
else:
en_message = message
prompt_language = "English" if use_translation else language
# ββ Build prompt βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
prompt_started = time.monotonic()
story_prompt = None
active_story_group = None
full_cast_count = 0
if story:
full_cast_count = len(story.get("group", {}).get("characters", []))
active_story_group = select_active_story_characters(
story["group"], story["role"], recent_messages, en_message,
summary=summary, long_term_memory=long_term_memory,
max_major=15, max_supporting=5,
)
story_prompt = build_story_prompt(
active_story_group, story["role"], story.get("topic", ""),
response_length=story_length, creativity=story_creativity,
point_of_view=point_of_view,
themes=story.get("themes"), next_plot=story.get("next_plot", ""),
)
messages = build_messages(
character=character,
persona=persona,
summary=summary,
long_term_memory=long_term_memory,
recent_messages=recent_messages,
current_message=en_message,
language=prompt_language,
story_prompt=story_prompt,
)
messages[0]['content'] += '\n\n' + relationship_to_prompt(conversation['relationship'])
if context_omitted:
messages[0]['content'] += ('\n\nSome unsummarized history is temporarily omitted to keep context bounded. '
'Do not invent details from missing turns.')
if use_translation:
messages[0]['content'] += "\n\nReply in English, even if earlier messages use another language."
messages, prompt_budget_tokens, estimated_prompt_tokens = _fit_messages_to_budget(messages, max_tokens)
prompt_chars = sum(len(item.get("content", "")) for item in messages)
active_cast = active_story_group.get("characters", []) if active_story_group else []
cast_catalog_chars = sum(len(build_compact_story_character_prompt(load_character(member["id"])))
for member in active_cast)
user_context_chars = sum(len(str(value or "")) for value in (persona or {}).values())
story_context_chars = sum(len(str(story.get(key, "") or "")) for key in ("topic", "next_plot")) if story else 0
if story:
story_context_chars += sum(len(str(value or "")) for value in story.get("themes", {}).values())
summary_chars = min(len(summary or ""), 2200)
memory_chars = 0
remaining_memory_chars = 2600
for item in sorted(long_term_memory, key=lambda mem: (0 if mem.get("origin") == "manual" else 1,
{"high": 0, "medium": 1, "low": 2}.get(mem.get("importance", "medium"), 1))):
if remaining_memory_chars <= 32:
break
memory_chars += min(len(str(item.get("content", ""))), 360, remaining_memory_chars - 32) + 8
remaining_memory_chars -= min(len(str(item.get("content", ""))), 360, remaining_memory_chars - 32) + 8
has_current_user = bool(messages and messages[-1].get("role") == "user")
history_messages = messages[1:-1] if has_current_user else messages[1:]
history_chars = sum(len(item.get("content", "")) for item in history_messages)
current_message_chars = len(messages[-1].get("content", "")) if has_current_user else 0
logger.info(
"[Chat][%s] prompt_ready seconds=%.2f total_prompt_chars=%d estimated_prompt_tokens=%d prompt_budget_tokens=%d output_tokens_reserved=%d safety_tokens=%d estimated_total_tokens=%d system_chars=%d history_messages=%d history_chars=%d current_message_chars=%d story_prompt_chars=%d active_cast_chars=%d active_cast_count=%d full_cast_count=%d summary_chars=%d long_term_memory_chars=%d persona_chars=%d story_context_chars=%d omitted_history=%s",
trace_id, time.monotonic() - prompt_started, prompt_chars, estimated_prompt_tokens,
prompt_budget_tokens, max_tokens, CONTEXT_SAFETY_TOKENS,
estimated_prompt_tokens + max_tokens + CONTEXT_SAFETY_TOKENS,
len(messages[0].get("content", "")), len(history_messages), history_chars, current_message_chars,
len(story_prompt) if story_prompt else 0, cast_catalog_chars, len(active_cast), full_cast_count,
summary_chars, memory_chars, user_context_chars, story_context_chars, context_omitted)
logger.debug(
f"[Chat] Prompt built: {len(messages[0]['content'])} chars system | "
f"{len(messages)} total messages"
)
# ββ LLM call ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
logger.info("[Chat][%s] llm_request model=%s max_tokens=%d temperature=%.2f top_p=%.2f frequency_penalty=%.2f translation=%s",
trace_id, model, max_tokens, temperature, top_p, frequency_penalty, use_translation)
raw_response = ""
llm_started = time.monotonic()
first_token_seconds = None
try:
if use_translation:
# Collect silently β yield placeholder while model generates
yield "*(β¦)*"
for chunk in _chat_completion(
client,
messages,
model=model,
max_tokens=max_tokens,
stream=True,
temperature=temperature,
top_p=top_p,
frequency_penalty=frequency_penalty,
):
if chunk.choices and chunk.choices[0].delta.content:
if first_token_seconds is None:
first_token_seconds = time.monotonic() - llm_started
raw_response += chunk.choices[0].delta.content
if deadline is not None and time.monotonic() >= deadline:
raise TimeoutError("The model response exceeded its deadline.")
else:
# Stream tokens in real time
for chunk in _chat_completion(
client,
messages,
model=model,
max_tokens=max_tokens,
stream=True,
temperature=temperature,
top_p=top_p,
frequency_penalty=frequency_penalty,
):
if chunk.choices and chunk.choices[0].delta.content:
if first_token_seconds is None:
first_token_seconds = time.monotonic() - llm_started
raw_response += chunk.choices[0].delta.content
if deadline is not None and time.monotonic() >= deadline:
raise TimeoutError("The model response exceeded its deadline.")
yield raw_response
except Exception as e:
logger.error("[Chat][%s] llm_failed model=%s elapsed_seconds=%.2f first_token_seconds=%s error=%s",
trace_id, model, time.monotonic() - llm_started,
f"{first_token_seconds:.2f}" if first_token_seconds is not None else "none", e,
exc_info=True)
status = getattr(e, "status_code", getattr(getattr(e, "response", None), "status_code", None))
detail = str(e).casefold()
if deadline is not None and time.monotonic() >= deadline:
code = "timeout"
elif status in (429, 503) or any(word in detail for word in ("rate limit", "too busy", "overloaded", "capacity")):
code = "busy"
elif status in (408, 504) or isinstance(e, (TimeoutError,)) or any(word in detail for word in ("timeout", "timed out", "deadline")):
code = "timeout"
else:
code = "no_response"
yield f"[[CHAT_ERROR:{code}]]"
return
logger.info("[Chat][%s] llm_stream_complete elapsed_seconds=%.2f first_token_seconds=%s response_chars=%d",
trace_id, time.monotonic() - llm_started,
f"{first_token_seconds:.2f}" if first_token_seconds is not None else "none", len(raw_response))
if deadline is not None and time.monotonic() >= deadline:
yield "[[CHAT_ERROR:timeout]]"
return
if not raw_response.strip():
yield "[[CHAT_ERROR:empty]]"
return
# ββ Translate response β user's language, format, yield βββββββββββββββββββ
if use_translation:
logger.debug(f"[Chat] Translating response β {language} ({len(raw_response)} chars)")
output_translation_started = time.monotonic()
final_text = translate_from_english(raw_response, language, translation_client, translation_model)
logger.info("[Chat][%s] output_translation_seconds=%.2f input_chars=%d output_chars=%d",
trace_id, time.monotonic() - output_translation_started, len(raw_response), len(final_text))
if deadline is not None and time.monotonic() >= deadline:
yield "[[CHAT_ERROR:timeout]]"
return
else:
final_text = raw_response
if story:
# Old and instruction-following models may still include these wrappers.
# Keep physical action as prose, but never expose private-thought blocks.
formatted = re.sub(r"<thought\b[^>]*>.*?</thought\s*>", "", final_text,
flags=re.IGNORECASE | re.DOTALL)
formatted = re.sub(r"</?action\b[^>]*>", "", formatted, flags=re.IGNORECASE).strip()
else:
segments = parse_response(final_text)
logger.debug(f"[Chat] Parser: {len(segments)} segments β {[s['type'] for s in segments]}")
formatted = segments_to_markdown(segments)
logger.info("[Chat][%s] response_ready chars=%d total_engine_seconds=%.2f",
trace_id, len(formatted), time.monotonic() - request_started)
# Store exactly what the user sees, including translated text.
try:
saved = append_exchange(session_id, character_id, conversation, "" if hidden_opening else message, formatted)
except Exception:
logger.exception("Could not save conversation memory.")
yield formatted + "\n\n*(This response could not be saved. Please try again.)*"
return
if not saved:
yield "*(Conversation changed while generating. Please try again.)*"
return
yield formatted
schedule_memory_update(session_id, character_id, saved, client, model, persona=persona)
|