StoryChatBot / engine /chat.py
cereeenn120's picture
Add point of view for narrative
ed3adb7
Raw History Blame Contribute Delete
20.2 kB
"""
Chat engine β€” core conversation loop.
Connects prompt_builder + memory + parser into a single streaming generator
that app.py (or any UI layer) can consume with `yield from`.
Separation of concerns:
- app.py : standalone browser UI and HTTP server entry point
- chat.py : LLM call, streaming, post-stream memory save
- prompt_builder.py : system prompt assembly
- memory.py : session-scoped snapshots and periodic memory extraction
- parser.py : raw response β†’ typed segments β†’ markdown
"""
import logging
import re
import time
from uuid import uuid4
from huggingface_hub import InferenceClient
from engine.memory import sync_history, append_exchange, bounded_recent_context
from engine.memory_jobs import schedule_memory_update
from engine.relationship import relationship_to_prompt
from engine.parser import parse_response, segments_to_markdown
from engine.prompt_builder import (build_messages, load_character, build_story_prompt,
select_active_story_characters, build_compact_story_character_prompt)
from engine.translator import translate_from_english, translate_to_english
logger = logging.getLogger(__name__)
# Shared conservative context envelope for selectable providers with different limits.
TOTAL_CONTEXT_TOKEN_BUDGET = 12_000
CONTEXT_SAFETY_TOKENS = 600
CHARS_PER_ESTIMATED_TOKEN = 3
def _fit_messages_to_budget(messages: list[dict[str, str]], max_tokens: int) -> tuple[list[dict[str, str]], int, int]:
"""Keep output and provider safety headroom inside a shared estimated context budget."""
prompt_budget = max(1_000, TOTAL_CONTEXT_TOKEN_BUDGET - max(0, max_tokens) - CONTEXT_SAFETY_TOKENS)
char_budget = prompt_budget * CHARS_PER_ESTIMATED_TOKEN
result = [dict(message) for message in messages]
# Current input (last message) and system rules are protected; discard the oldest history first.
while sum(len(item.get("content", "")) for item in result) > char_budget and len(result) > 2:
result.pop(1)
excess = sum(len(item.get("content", "")) for item in result) - char_budget
if excess > 0 and len(result) > 1:
system = result[0].get("content", "")
# Compress optional system context in priority order, retaining a marker and some content.
priorities = (
("LONG-TERM MEMORY", 500), ("CONVERSATION SUMMARY", 700),
("PLAYER'S PREFERRED STORY DIRECTION", 300),
("USER'S OPTIONAL STORY PREMISE", 300),
("PLAYER CHARACTER DETAILS", 300), ("THE USER IS PLAYING", 300),
("STORY GROUP:", 6_000),
)
sections = system.split("\n\n")
for marker, minimum in priorities:
index = next((i for i, section in enumerate(sections) if marker in section), None)
if index is None or len(sections[index]) <= minimum:
continue
reduction = min(excess, len(sections[index]) - minimum)
keep = max(minimum, len(sections[index]) - reduction)
sections[index] = sections[index][:keep].rstrip() + "\n[Context shortened to fit the shared token budget.]"
excess -= reduction
if excess <= 0:
break
result[0]["content"] = "\n\n".join(sections)
estimated = (sum(len(item.get("content", "")) for item in result) + CHARS_PER_ESTIMATED_TOKEN - 1) // CHARS_PER_ESTIMATED_TOKEN
return result, prompt_budget, estimated
def _chat_completion(client, messages, **kwargs):
"""Adapt chat completion calls for HF InferenceClient or OpenAI HF Router."""
if isinstance(kwargs.get("model"), str) and kwargs["model"].endswith(":featherless-ai"):
model = kwargs.pop("model")
return client.chat.completions.create(model=model, messages=messages, **kwargs)
return client.chat_completion(messages, **kwargs)
def generate_response(
message: str,
character_id: str,
client: InferenceClient,
session_id: str,
history: list[dict],
persona: dict | None = None,
model: str = "Qwen/Qwen3-235B-A22B-Instruct-2507",
max_tokens: int = 512,
temperature: float = 0.7,
top_p: float = 0.95,
language: str = "English",
translation_model: str | None = None,
use_translator: bool = False,
deadline: float | None = None,
story: dict | None = None,
hidden_opening: bool = False,
story_length: str = "balanced",
story_creativity: str = "balanced",
point_of_view: str = "third_person_limited",
frequency_penalty: float = 0.3,
trace_id: str | None = None,
):
"""
Core streaming generator.
Yields the accumulated raw response after each token arrives so the web
layer can display it. After the stream ends, yields one final
formatted version (actions italic, thoughts em-dashed) and saves
the exchange to session-scoped memory.
All errors are caught and surfaced as in-chat messages so the UI
never silently breaks.
Translation modes (when language β‰  English):
use_translator=True β€” dedicated translation_model translates user input
to English and response back to user's language.
Main model always operates in English.
use_translator=False β€” no separate model. Main model receives the language
instruction directly and responds in user's language.
Args:
message: Current user input.
character_id: Which character to respond as.
client: Initialised HuggingFace InferenceClient.
persona: Optional user persona dict.
model: HuggingFace model string.
max_tokens: Maximum tokens to generate.
temperature: Sampling temperature.
top_p: Nucleus sampling probability.
language: UI language selected by the user.
translation_model: Model used when use_translator=True.
use_translator: True = separate translator, False = main model handles language.
"""
trace_id = trace_id or uuid4().hex[:10]
request_started = time.monotonic()
logger.info("[Chat][%s] start character=%s model=%s language=%s story=%s hidden_opening=%s",
trace_id, character_id, model, language, bool(story), hidden_opening)
# ── Load character ─────────────────────────────────────────────────────────
try:
character = load_character(character_id)
logger.debug(f"[Chat] Character loaded: '{character.get('name', character_id)}'")
except FileNotFoundError:
logger.error(f"[Chat] Character not found: {character_id}")
yield f"*(Character '{character_id}' could not be loaded.)*"
return
except Exception as e:
logger.error(f"[Chat] Unexpected error loading character: {e}", exc_info=True)
yield "*(Something went wrong loading the character. Check the logs.)*"
return
# Synchronize the visible transcript before assembling model context.
try:
conversation = sync_history(session_id, character_id, history)
except Exception:
logger.exception("Could not synchronize conversation memory.")
yield "*(Conversation history could not be loaded. Please try again.)*"
return
recent_messages, context_omitted = bounded_recent_context(conversation)
summary = conversation['summary']
long_term_memory = conversation['long_term_memory']
# ── Translation mode? ──────────────────────────────────────────────────────
is_non_english = language.strip().lower() != "english"
use_translation = use_translator and is_non_english and bool(translation_model)
translation_client = client
if use_translation and isinstance(model, str) and model.endswith(":featherless-ai"):
from huggingface_hub import InferenceClient
translation_client = InferenceClient(token=getattr(client, "api_key", None), timeout=240)
if is_non_english and not use_translator:
logger.info(f"[Chat] Translation: OFF β€” main model will respond in '{language}' directly")
elif use_translation:
logger.info(f"[Chat] Translation: ON β†’ model='{translation_model}'")
else:
logger.info("[Chat] Translation: not needed (language is English)")
translation_started = time.monotonic()
if use_translation and message.strip():
# Translate user message β†’ English before sending to main model
logger.debug(f"[Chat] Translating user message: {message[:80]!r}")
en_message = translate_to_english(message, language, translation_client, translation_model)
if deadline is not None and time.monotonic() >= deadline:
yield "[[CHAT_ERROR:timeout]]"
return
logger.debug(f"[Chat] User message (EN): {en_message[:80]!r}")
prompt_language = "English" # main model always responds in English
logger.info("[Chat][%s] input_translation_seconds=%.2f input_chars=%d output_chars=%d",
trace_id, time.monotonic() - translation_started, len(message), len(en_message))
else:
en_message = message
prompt_language = "English" if use_translation else language
# ── Build prompt ───────────────────────────────────────────────────────────
prompt_started = time.monotonic()
story_prompt = None
active_story_group = None
full_cast_count = 0
if story:
full_cast_count = len(story.get("group", {}).get("characters", []))
active_story_group = select_active_story_characters(
story["group"], story["role"], recent_messages, en_message,
summary=summary, long_term_memory=long_term_memory,
max_major=15, max_supporting=5,
)
story_prompt = build_story_prompt(
active_story_group, story["role"], story.get("topic", ""),
response_length=story_length, creativity=story_creativity,
point_of_view=point_of_view,
themes=story.get("themes"), next_plot=story.get("next_plot", ""),
)
messages = build_messages(
character=character,
persona=persona,
summary=summary,
long_term_memory=long_term_memory,
recent_messages=recent_messages,
current_message=en_message,
language=prompt_language,
story_prompt=story_prompt,
)
messages[0]['content'] += '\n\n' + relationship_to_prompt(conversation['relationship'])
if context_omitted:
messages[0]['content'] += ('\n\nSome unsummarized history is temporarily omitted to keep context bounded. '
'Do not invent details from missing turns.')
if use_translation:
messages[0]['content'] += "\n\nReply in English, even if earlier messages use another language."
messages, prompt_budget_tokens, estimated_prompt_tokens = _fit_messages_to_budget(messages, max_tokens)
prompt_chars = sum(len(item.get("content", "")) for item in messages)
active_cast = active_story_group.get("characters", []) if active_story_group else []
cast_catalog_chars = sum(len(build_compact_story_character_prompt(load_character(member["id"])))
for member in active_cast)
user_context_chars = sum(len(str(value or "")) for value in (persona or {}).values())
story_context_chars = sum(len(str(story.get(key, "") or "")) for key in ("topic", "next_plot")) if story else 0
if story:
story_context_chars += sum(len(str(value or "")) for value in story.get("themes", {}).values())
summary_chars = min(len(summary or ""), 2200)
memory_chars = 0
remaining_memory_chars = 2600
for item in sorted(long_term_memory, key=lambda mem: (0 if mem.get("origin") == "manual" else 1,
{"high": 0, "medium": 1, "low": 2}.get(mem.get("importance", "medium"), 1))):
if remaining_memory_chars <= 32:
break
memory_chars += min(len(str(item.get("content", ""))), 360, remaining_memory_chars - 32) + 8
remaining_memory_chars -= min(len(str(item.get("content", ""))), 360, remaining_memory_chars - 32) + 8
has_current_user = bool(messages and messages[-1].get("role") == "user")
history_messages = messages[1:-1] if has_current_user else messages[1:]
history_chars = sum(len(item.get("content", "")) for item in history_messages)
current_message_chars = len(messages[-1].get("content", "")) if has_current_user else 0
logger.info(
"[Chat][%s] prompt_ready seconds=%.2f total_prompt_chars=%d estimated_prompt_tokens=%d prompt_budget_tokens=%d output_tokens_reserved=%d safety_tokens=%d estimated_total_tokens=%d system_chars=%d history_messages=%d history_chars=%d current_message_chars=%d story_prompt_chars=%d active_cast_chars=%d active_cast_count=%d full_cast_count=%d summary_chars=%d long_term_memory_chars=%d persona_chars=%d story_context_chars=%d omitted_history=%s",
trace_id, time.monotonic() - prompt_started, prompt_chars, estimated_prompt_tokens,
prompt_budget_tokens, max_tokens, CONTEXT_SAFETY_TOKENS,
estimated_prompt_tokens + max_tokens + CONTEXT_SAFETY_TOKENS,
len(messages[0].get("content", "")), len(history_messages), history_chars, current_message_chars,
len(story_prompt) if story_prompt else 0, cast_catalog_chars, len(active_cast), full_cast_count,
summary_chars, memory_chars, user_context_chars, story_context_chars, context_omitted)
logger.debug(
f"[Chat] Prompt built: {len(messages[0]['content'])} chars system | "
f"{len(messages)} total messages"
)
# ── LLM call ──────────────────────────────────────────────────────────────
logger.info("[Chat][%s] llm_request model=%s max_tokens=%d temperature=%.2f top_p=%.2f frequency_penalty=%.2f translation=%s",
trace_id, model, max_tokens, temperature, top_p, frequency_penalty, use_translation)
raw_response = ""
llm_started = time.monotonic()
first_token_seconds = None
try:
if use_translation:
# Collect silently β€” yield placeholder while model generates
yield "*(…)*"
for chunk in _chat_completion(
client,
messages,
model=model,
max_tokens=max_tokens,
stream=True,
temperature=temperature,
top_p=top_p,
frequency_penalty=frequency_penalty,
):
if chunk.choices and chunk.choices[0].delta.content:
if first_token_seconds is None:
first_token_seconds = time.monotonic() - llm_started
raw_response += chunk.choices[0].delta.content
if deadline is not None and time.monotonic() >= deadline:
raise TimeoutError("The model response exceeded its deadline.")
else:
# Stream tokens in real time
for chunk in _chat_completion(
client,
messages,
model=model,
max_tokens=max_tokens,
stream=True,
temperature=temperature,
top_p=top_p,
frequency_penalty=frequency_penalty,
):
if chunk.choices and chunk.choices[0].delta.content:
if first_token_seconds is None:
first_token_seconds = time.monotonic() - llm_started
raw_response += chunk.choices[0].delta.content
if deadline is not None and time.monotonic() >= deadline:
raise TimeoutError("The model response exceeded its deadline.")
yield raw_response
except Exception as e:
logger.error("[Chat][%s] llm_failed model=%s elapsed_seconds=%.2f first_token_seconds=%s error=%s",
trace_id, model, time.monotonic() - llm_started,
f"{first_token_seconds:.2f}" if first_token_seconds is not None else "none", e,
exc_info=True)
status = getattr(e, "status_code", getattr(getattr(e, "response", None), "status_code", None))
detail = str(e).casefold()
if deadline is not None and time.monotonic() >= deadline:
code = "timeout"
elif status in (429, 503) or any(word in detail for word in ("rate limit", "too busy", "overloaded", "capacity")):
code = "busy"
elif status in (408, 504) or isinstance(e, (TimeoutError,)) or any(word in detail for word in ("timeout", "timed out", "deadline")):
code = "timeout"
else:
code = "no_response"
yield f"[[CHAT_ERROR:{code}]]"
return
logger.info("[Chat][%s] llm_stream_complete elapsed_seconds=%.2f first_token_seconds=%s response_chars=%d",
trace_id, time.monotonic() - llm_started,
f"{first_token_seconds:.2f}" if first_token_seconds is not None else "none", len(raw_response))
if deadline is not None and time.monotonic() >= deadline:
yield "[[CHAT_ERROR:timeout]]"
return
if not raw_response.strip():
yield "[[CHAT_ERROR:empty]]"
return
# ── Translate response β†’ user's language, format, yield ───────────────────
if use_translation:
logger.debug(f"[Chat] Translating response β†’ {language} ({len(raw_response)} chars)")
output_translation_started = time.monotonic()
final_text = translate_from_english(raw_response, language, translation_client, translation_model)
logger.info("[Chat][%s] output_translation_seconds=%.2f input_chars=%d output_chars=%d",
trace_id, time.monotonic() - output_translation_started, len(raw_response), len(final_text))
if deadline is not None and time.monotonic() >= deadline:
yield "[[CHAT_ERROR:timeout]]"
return
else:
final_text = raw_response
if story:
# Old and instruction-following models may still include these wrappers.
# Keep physical action as prose, but never expose private-thought blocks.
formatted = re.sub(r"<thought\b[^>]*>.*?</thought\s*>", "", final_text,
flags=re.IGNORECASE | re.DOTALL)
formatted = re.sub(r"</?action\b[^>]*>", "", formatted, flags=re.IGNORECASE).strip()
else:
segments = parse_response(final_text)
logger.debug(f"[Chat] Parser: {len(segments)} segments β†’ {[s['type'] for s in segments]}")
formatted = segments_to_markdown(segments)
logger.info("[Chat][%s] response_ready chars=%d total_engine_seconds=%.2f",
trace_id, len(formatted), time.monotonic() - request_started)
# Store exactly what the user sees, including translated text.
try:
saved = append_exchange(session_id, character_id, conversation, "" if hidden_opening else message, formatted)
except Exception:
logger.exception("Could not save conversation memory.")
yield formatted + "\n\n*(This response could not be saved. Please try again.)*"
return
if not saved:
yield "*(Conversation changed while generating. Please try again.)*"
return
yield formatted
schedule_memory_update(session_id, character_id, saved, client, model, persona=persona)