Spaces:
Running
Running
Download engine/chat.py from cereeenn120/StoryChatBot: direct link, hf CLI and curl.
- Browser
- Download file 20.2 kB
-
https://huggingface.co/spaces/cereeenn120/StoryChatBot/resolve/main/engine/chat.py
- Command line
-
hf download hf://spaces/cereeenn120/StoryChatBot/engine/chat.py
-
curl -L -o chat.py https://huggingface.co/spaces/cereeenn120/StoryChatBot/resolve/main/engine/chat.py
20.2 kB
| """ | |
| Chat engine β core conversation loop. | |
| Connects prompt_builder + memory + parser into a single streaming generator | |
| that app.py (or any UI layer) can consume with `yield from`. | |
| Separation of concerns: | |
| - app.py : standalone browser UI and HTTP server entry point | |
| - chat.py : LLM call, streaming, post-stream memory save | |
| - prompt_builder.py : system prompt assembly | |
| - memory.py : session-scoped snapshots and periodic memory extraction | |
| - parser.py : raw response β typed segments β markdown | |
| """ | |
| import logging | |
| import re | |
| import time | |
| from uuid import uuid4 | |
| from huggingface_hub import InferenceClient | |
| from engine.memory import sync_history, append_exchange, bounded_recent_context | |
| from engine.memory_jobs import schedule_memory_update | |
| from engine.relationship import relationship_to_prompt | |
| from engine.parser import parse_response, segments_to_markdown | |
| from engine.prompt_builder import (build_messages, load_character, build_story_prompt, | |
| select_active_story_characters, build_compact_story_character_prompt) | |
| from engine.translator import translate_from_english, translate_to_english | |
| logger = logging.getLogger(__name__) | |
| # Shared conservative context envelope for selectable providers with different limits. | |
| TOTAL_CONTEXT_TOKEN_BUDGET = 12_000 | |
| CONTEXT_SAFETY_TOKENS = 600 | |
| CHARS_PER_ESTIMATED_TOKEN = 3 | |
| def _fit_messages_to_budget(messages: list[dict[str, str]], max_tokens: int) -> tuple[list[dict[str, str]], int, int]: | |
| """Keep output and provider safety headroom inside a shared estimated context budget.""" | |
| prompt_budget = max(1_000, TOTAL_CONTEXT_TOKEN_BUDGET - max(0, max_tokens) - CONTEXT_SAFETY_TOKENS) | |
| char_budget = prompt_budget * CHARS_PER_ESTIMATED_TOKEN | |
| result = [dict(message) for message in messages] | |
| # Current input (last message) and system rules are protected; discard the oldest history first. | |
| while sum(len(item.get("content", "")) for item in result) > char_budget and len(result) > 2: | |
| result.pop(1) | |
| excess = sum(len(item.get("content", "")) for item in result) - char_budget | |
| if excess > 0 and len(result) > 1: | |
| system = result[0].get("content", "") | |
| # Compress optional system context in priority order, retaining a marker and some content. | |
| priorities = ( | |
| ("LONG-TERM MEMORY", 500), ("CONVERSATION SUMMARY", 700), | |
| ("PLAYER'S PREFERRED STORY DIRECTION", 300), | |
| ("USER'S OPTIONAL STORY PREMISE", 300), | |
| ("PLAYER CHARACTER DETAILS", 300), ("THE USER IS PLAYING", 300), | |
| ("STORY GROUP:", 6_000), | |
| ) | |
| sections = system.split("\n\n") | |
| for marker, minimum in priorities: | |
| index = next((i for i, section in enumerate(sections) if marker in section), None) | |
| if index is None or len(sections[index]) <= minimum: | |
| continue | |
| reduction = min(excess, len(sections[index]) - minimum) | |
| keep = max(minimum, len(sections[index]) - reduction) | |
| sections[index] = sections[index][:keep].rstrip() + "\n[Context shortened to fit the shared token budget.]" | |
| excess -= reduction | |
| if excess <= 0: | |
| break | |
| result[0]["content"] = "\n\n".join(sections) | |
| estimated = (sum(len(item.get("content", "")) for item in result) + CHARS_PER_ESTIMATED_TOKEN - 1) // CHARS_PER_ESTIMATED_TOKEN | |
| return result, prompt_budget, estimated | |
| def _chat_completion(client, messages, **kwargs): | |
| """Adapt chat completion calls for HF InferenceClient or OpenAI HF Router.""" | |
| if isinstance(kwargs.get("model"), str) and kwargs["model"].endswith(":featherless-ai"): | |
| model = kwargs.pop("model") | |
| return client.chat.completions.create(model=model, messages=messages, **kwargs) | |
| return client.chat_completion(messages, **kwargs) | |
| def generate_response( | |
| message: str, | |
| character_id: str, | |
| client: InferenceClient, | |
| session_id: str, | |
| history: list[dict], | |
| persona: dict | None = None, | |
| model: str = "Qwen/Qwen3-235B-A22B-Instruct-2507", | |
| max_tokens: int = 512, | |
| temperature: float = 0.7, | |
| top_p: float = 0.95, | |
| language: str = "English", | |
| translation_model: str | None = None, | |
| use_translator: bool = False, | |
| deadline: float | None = None, | |
| story: dict | None = None, | |
| hidden_opening: bool = False, | |
| story_length: str = "balanced", | |
| story_creativity: str = "balanced", | |
| point_of_view: str = "third_person_limited", | |
| frequency_penalty: float = 0.3, | |
| trace_id: str | None = None, | |
| ): | |
| """ | |
| Core streaming generator. | |
| Yields the accumulated raw response after each token arrives so the web | |
| layer can display it. After the stream ends, yields one final | |
| formatted version (actions italic, thoughts em-dashed) and saves | |
| the exchange to session-scoped memory. | |
| All errors are caught and surfaced as in-chat messages so the UI | |
| never silently breaks. | |
| Translation modes (when language β English): | |
| use_translator=True β dedicated translation_model translates user input | |
| to English and response back to user's language. | |
| Main model always operates in English. | |
| use_translator=False β no separate model. Main model receives the language | |
| instruction directly and responds in user's language. | |
| Args: | |
| message: Current user input. | |
| character_id: Which character to respond as. | |
| client: Initialised HuggingFace InferenceClient. | |
| persona: Optional user persona dict. | |
| model: HuggingFace model string. | |
| max_tokens: Maximum tokens to generate. | |
| temperature: Sampling temperature. | |
| top_p: Nucleus sampling probability. | |
| language: UI language selected by the user. | |
| translation_model: Model used when use_translator=True. | |
| use_translator: True = separate translator, False = main model handles language. | |
| """ | |
| trace_id = trace_id or uuid4().hex[:10] | |
| request_started = time.monotonic() | |
| logger.info("[Chat][%s] start character=%s model=%s language=%s story=%s hidden_opening=%s", | |
| trace_id, character_id, model, language, bool(story), hidden_opening) | |
| # ββ Load character βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| try: | |
| character = load_character(character_id) | |
| logger.debug(f"[Chat] Character loaded: '{character.get('name', character_id)}'") | |
| except FileNotFoundError: | |
| logger.error(f"[Chat] Character not found: {character_id}") | |
| yield f"*(Character '{character_id}' could not be loaded.)*" | |
| return | |
| except Exception as e: | |
| logger.error(f"[Chat] Unexpected error loading character: {e}", exc_info=True) | |
| yield "*(Something went wrong loading the character. Check the logs.)*" | |
| return | |
| # Synchronize the visible transcript before assembling model context. | |
| try: | |
| conversation = sync_history(session_id, character_id, history) | |
| except Exception: | |
| logger.exception("Could not synchronize conversation memory.") | |
| yield "*(Conversation history could not be loaded. Please try again.)*" | |
| return | |
| recent_messages, context_omitted = bounded_recent_context(conversation) | |
| summary = conversation['summary'] | |
| long_term_memory = conversation['long_term_memory'] | |
| # ββ Translation mode? ββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| is_non_english = language.strip().lower() != "english" | |
| use_translation = use_translator and is_non_english and bool(translation_model) | |
| translation_client = client | |
| if use_translation and isinstance(model, str) and model.endswith(":featherless-ai"): | |
| from huggingface_hub import InferenceClient | |
| translation_client = InferenceClient(token=getattr(client, "api_key", None), timeout=240) | |
| if is_non_english and not use_translator: | |
| logger.info(f"[Chat] Translation: OFF β main model will respond in '{language}' directly") | |
| elif use_translation: | |
| logger.info(f"[Chat] Translation: ON β model='{translation_model}'") | |
| else: | |
| logger.info("[Chat] Translation: not needed (language is English)") | |
| translation_started = time.monotonic() | |
| if use_translation and message.strip(): | |
| # Translate user message β English before sending to main model | |
| logger.debug(f"[Chat] Translating user message: {message[:80]!r}") | |
| en_message = translate_to_english(message, language, translation_client, translation_model) | |
| if deadline is not None and time.monotonic() >= deadline: | |
| yield "[[CHAT_ERROR:timeout]]" | |
| return | |
| logger.debug(f"[Chat] User message (EN): {en_message[:80]!r}") | |
| prompt_language = "English" # main model always responds in English | |
| logger.info("[Chat][%s] input_translation_seconds=%.2f input_chars=%d output_chars=%d", | |
| trace_id, time.monotonic() - translation_started, len(message), len(en_message)) | |
| else: | |
| en_message = message | |
| prompt_language = "English" if use_translation else language | |
| # ββ Build prompt βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| prompt_started = time.monotonic() | |
| story_prompt = None | |
| active_story_group = None | |
| full_cast_count = 0 | |
| if story: | |
| full_cast_count = len(story.get("group", {}).get("characters", [])) | |
| active_story_group = select_active_story_characters( | |
| story["group"], story["role"], recent_messages, en_message, | |
| summary=summary, long_term_memory=long_term_memory, | |
| max_major=15, max_supporting=5, | |
| ) | |
| story_prompt = build_story_prompt( | |
| active_story_group, story["role"], story.get("topic", ""), | |
| response_length=story_length, creativity=story_creativity, | |
| point_of_view=point_of_view, | |
| themes=story.get("themes"), next_plot=story.get("next_plot", ""), | |
| ) | |
| messages = build_messages( | |
| character=character, | |
| persona=persona, | |
| summary=summary, | |
| long_term_memory=long_term_memory, | |
| recent_messages=recent_messages, | |
| current_message=en_message, | |
| language=prompt_language, | |
| story_prompt=story_prompt, | |
| ) | |
| messages[0]['content'] += '\n\n' + relationship_to_prompt(conversation['relationship']) | |
| if context_omitted: | |
| messages[0]['content'] += ('\n\nSome unsummarized history is temporarily omitted to keep context bounded. ' | |
| 'Do not invent details from missing turns.') | |
| if use_translation: | |
| messages[0]['content'] += "\n\nReply in English, even if earlier messages use another language." | |
| messages, prompt_budget_tokens, estimated_prompt_tokens = _fit_messages_to_budget(messages, max_tokens) | |
| prompt_chars = sum(len(item.get("content", "")) for item in messages) | |
| active_cast = active_story_group.get("characters", []) if active_story_group else [] | |
| cast_catalog_chars = sum(len(build_compact_story_character_prompt(load_character(member["id"]))) | |
| for member in active_cast) | |
| user_context_chars = sum(len(str(value or "")) for value in (persona or {}).values()) | |
| story_context_chars = sum(len(str(story.get(key, "") or "")) for key in ("topic", "next_plot")) if story else 0 | |
| if story: | |
| story_context_chars += sum(len(str(value or "")) for value in story.get("themes", {}).values()) | |
| summary_chars = min(len(summary or ""), 2200) | |
| memory_chars = 0 | |
| remaining_memory_chars = 2600 | |
| for item in sorted(long_term_memory, key=lambda mem: (0 if mem.get("origin") == "manual" else 1, | |
| {"high": 0, "medium": 1, "low": 2}.get(mem.get("importance", "medium"), 1))): | |
| if remaining_memory_chars <= 32: | |
| break | |
| memory_chars += min(len(str(item.get("content", ""))), 360, remaining_memory_chars - 32) + 8 | |
| remaining_memory_chars -= min(len(str(item.get("content", ""))), 360, remaining_memory_chars - 32) + 8 | |
| has_current_user = bool(messages and messages[-1].get("role") == "user") | |
| history_messages = messages[1:-1] if has_current_user else messages[1:] | |
| history_chars = sum(len(item.get("content", "")) for item in history_messages) | |
| current_message_chars = len(messages[-1].get("content", "")) if has_current_user else 0 | |
| logger.info( | |
| "[Chat][%s] prompt_ready seconds=%.2f total_prompt_chars=%d estimated_prompt_tokens=%d prompt_budget_tokens=%d output_tokens_reserved=%d safety_tokens=%d estimated_total_tokens=%d system_chars=%d history_messages=%d history_chars=%d current_message_chars=%d story_prompt_chars=%d active_cast_chars=%d active_cast_count=%d full_cast_count=%d summary_chars=%d long_term_memory_chars=%d persona_chars=%d story_context_chars=%d omitted_history=%s", | |
| trace_id, time.monotonic() - prompt_started, prompt_chars, estimated_prompt_tokens, | |
| prompt_budget_tokens, max_tokens, CONTEXT_SAFETY_TOKENS, | |
| estimated_prompt_tokens + max_tokens + CONTEXT_SAFETY_TOKENS, | |
| len(messages[0].get("content", "")), len(history_messages), history_chars, current_message_chars, | |
| len(story_prompt) if story_prompt else 0, cast_catalog_chars, len(active_cast), full_cast_count, | |
| summary_chars, memory_chars, user_context_chars, story_context_chars, context_omitted) | |
| logger.debug( | |
| f"[Chat] Prompt built: {len(messages[0]['content'])} chars system | " | |
| f"{len(messages)} total messages" | |
| ) | |
| # ββ LLM call ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| logger.info("[Chat][%s] llm_request model=%s max_tokens=%d temperature=%.2f top_p=%.2f frequency_penalty=%.2f translation=%s", | |
| trace_id, model, max_tokens, temperature, top_p, frequency_penalty, use_translation) | |
| raw_response = "" | |
| llm_started = time.monotonic() | |
| first_token_seconds = None | |
| try: | |
| if use_translation: | |
| # Collect silently β yield placeholder while model generates | |
| yield "*(β¦)*" | |
| for chunk in _chat_completion( | |
| client, | |
| messages, | |
| model=model, | |
| max_tokens=max_tokens, | |
| stream=True, | |
| temperature=temperature, | |
| top_p=top_p, | |
| frequency_penalty=frequency_penalty, | |
| ): | |
| if chunk.choices and chunk.choices[0].delta.content: | |
| if first_token_seconds is None: | |
| first_token_seconds = time.monotonic() - llm_started | |
| raw_response += chunk.choices[0].delta.content | |
| if deadline is not None and time.monotonic() >= deadline: | |
| raise TimeoutError("The model response exceeded its deadline.") | |
| else: | |
| # Stream tokens in real time | |
| for chunk in _chat_completion( | |
| client, | |
| messages, | |
| model=model, | |
| max_tokens=max_tokens, | |
| stream=True, | |
| temperature=temperature, | |
| top_p=top_p, | |
| frequency_penalty=frequency_penalty, | |
| ): | |
| if chunk.choices and chunk.choices[0].delta.content: | |
| if first_token_seconds is None: | |
| first_token_seconds = time.monotonic() - llm_started | |
| raw_response += chunk.choices[0].delta.content | |
| if deadline is not None and time.monotonic() >= deadline: | |
| raise TimeoutError("The model response exceeded its deadline.") | |
| yield raw_response | |
| except Exception as e: | |
| logger.error("[Chat][%s] llm_failed model=%s elapsed_seconds=%.2f first_token_seconds=%s error=%s", | |
| trace_id, model, time.monotonic() - llm_started, | |
| f"{first_token_seconds:.2f}" if first_token_seconds is not None else "none", e, | |
| exc_info=True) | |
| status = getattr(e, "status_code", getattr(getattr(e, "response", None), "status_code", None)) | |
| detail = str(e).casefold() | |
| if deadline is not None and time.monotonic() >= deadline: | |
| code = "timeout" | |
| elif status in (429, 503) or any(word in detail for word in ("rate limit", "too busy", "overloaded", "capacity")): | |
| code = "busy" | |
| elif status in (408, 504) or isinstance(e, (TimeoutError,)) or any(word in detail for word in ("timeout", "timed out", "deadline")): | |
| code = "timeout" | |
| else: | |
| code = "no_response" | |
| yield f"[[CHAT_ERROR:{code}]]" | |
| return | |
| logger.info("[Chat][%s] llm_stream_complete elapsed_seconds=%.2f first_token_seconds=%s response_chars=%d", | |
| trace_id, time.monotonic() - llm_started, | |
| f"{first_token_seconds:.2f}" if first_token_seconds is not None else "none", len(raw_response)) | |
| if deadline is not None and time.monotonic() >= deadline: | |
| yield "[[CHAT_ERROR:timeout]]" | |
| return | |
| if not raw_response.strip(): | |
| yield "[[CHAT_ERROR:empty]]" | |
| return | |
| # ββ Translate response β user's language, format, yield βββββββββββββββββββ | |
| if use_translation: | |
| logger.debug(f"[Chat] Translating response β {language} ({len(raw_response)} chars)") | |
| output_translation_started = time.monotonic() | |
| final_text = translate_from_english(raw_response, language, translation_client, translation_model) | |
| logger.info("[Chat][%s] output_translation_seconds=%.2f input_chars=%d output_chars=%d", | |
| trace_id, time.monotonic() - output_translation_started, len(raw_response), len(final_text)) | |
| if deadline is not None and time.monotonic() >= deadline: | |
| yield "[[CHAT_ERROR:timeout]]" | |
| return | |
| else: | |
| final_text = raw_response | |
| if story: | |
| # Old and instruction-following models may still include these wrappers. | |
| # Keep physical action as prose, but never expose private-thought blocks. | |
| formatted = re.sub(r"<thought\b[^>]*>.*?</thought\s*>", "", final_text, | |
| flags=re.IGNORECASE | re.DOTALL) | |
| formatted = re.sub(r"</?action\b[^>]*>", "", formatted, flags=re.IGNORECASE).strip() | |
| else: | |
| segments = parse_response(final_text) | |
| logger.debug(f"[Chat] Parser: {len(segments)} segments β {[s['type'] for s in segments]}") | |
| formatted = segments_to_markdown(segments) | |
| logger.info("[Chat][%s] response_ready chars=%d total_engine_seconds=%.2f", | |
| trace_id, len(formatted), time.monotonic() - request_started) | |
| # Store exactly what the user sees, including translated text. | |
| try: | |
| saved = append_exchange(session_id, character_id, conversation, "" if hidden_opening else message, formatted) | |
| except Exception: | |
| logger.exception("Could not save conversation memory.") | |
| yield formatted + "\n\n*(This response could not be saved. Please try again.)*" | |
| return | |
| if not saved: | |
| yield "*(Conversation changed while generating. Please try again.)*" | |
| return | |
| yield formatted | |
| schedule_memory_update(session_id, character_id, saved, client, model, persona=persona) | |