Buckets:

download
raw
331 kB
from __future__ import annotations import weakref import sys import asyncio import contextvars import html import csv import inspect import json import logging import logging.handlers import math import os import re import time import uuid import threading import warnings from concurrent.futures import Future as ConcurrentFuture, ThreadPoolExecutor from dataclasses import dataclass from datetime import datetime from functools import partial, wraps from hashlib import md5 from pathlib import Path from typing import ( Any, Awaitable, Protocol, Callable, TYPE_CHECKING, List, NamedTuple, Optional, Iterable, Sequence, Collection, ) import numpy as np from dotenv import load_dotenv import json_repair from lightrag.exceptions import ( ChunkBlockMatchError, CommitBookkeepingError, EmptyTruncatedResponseError, ) from lightrag.constants import ( DEFAULT_LOG_MAX_BYTES, DEFAULT_LOG_BACKUP_COUNT, DEFAULT_LOG_FILENAME, GRAPH_FIELD_SEP, DEFAULT_MAX_TOTAL_TOKENS, DEFAULT_PROCESSING_PRIORITY, DEFAULT_SOURCE_IDS_LIMIT_METHOD, VALID_SOURCE_IDS_LIMIT_METHODS, SOURCE_IDS_LIMIT_METHOD_FIFO, PARSED_DIR_NAME, DEFAULT_GLOBAL_SLOT_POLL_MIN, DEFAULT_GLOBAL_SLOT_POLL_DEFERRED_MAX, DEFAULT_GLOBAL_SLOT_DRAIN_LIMIT, DEFAULT_ZOMBIE_COMPACT_THRESHOLD, DEFAULT_COMPACT_BATCH_LIMIT, DEFAULT_QUEUE_STATS_MIN_PUBLISH_INTERVAL, ) # Precompile regex pattern for JSON sanitization (module-level, compiled once) _SURROGATE_PATTERN = re.compile(r"[\uD800-\uDFFF\uFFFE\uFFFF]") _CONTROL_CHAR_PATTERN_ALL = re.compile(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]") class SafeStreamHandler(logging.StreamHandler): """StreamHandler that gracefully handles closed streams during shutdown. This handler prevents "ValueError: I/O operation on closed file" errors that can occur when pytest or other test frameworks close stdout/stderr before Python's logging cleanup runs. """ def flush(self): """Flush the stream, ignoring errors if the stream is closed.""" try: super().flush() except (ValueError, OSError): # Stream is closed or otherwise unavailable, silently ignore pass def close(self): """Close the handler, ignoring errors if the stream is already closed.""" try: super().close() except (ValueError, OSError): # Stream is closed or otherwise unavailable, silently ignore pass # Initialize logger with basic configuration logger = logging.getLogger("lightrag") logger.propagate = False # prevent log message send to root logger logger.setLevel(logging.INFO) # Add console handler if no handlers exist if not logger.handlers: console_handler = SafeStreamHandler() console_handler.setLevel(logging.INFO) formatter = logging.Formatter("%(levelname)s: %(message)s") console_handler.setFormatter(formatter) logger.addHandler(console_handler) # Set httpx logging level to WARNING logging.getLogger("httpx").setLevel(logging.WARNING) def _patch_ascii_colors_console_handler() -> None: """Prevent ascii_colors from printing flush errors during interpreter exit.""" try: from ascii_colors import ConsoleHandler except ImportError: return if getattr(ConsoleHandler, "_lightrag_patched", False): return original_handle_error = ConsoleHandler.handle_error def _safe_handle_error(self, message: str) -> None: # type: ignore[override] exc_type, _, _ = sys.exc_info() if exc_type in (ValueError, OSError) and "close" in message.lower(): return original_handle_error(self, message) ConsoleHandler.handle_error = _safe_handle_error # type: ignore[assignment] ConsoleHandler._lightrag_patched = True # type: ignore[attr-defined] _patch_ascii_colors_console_handler() # Global import for pypinyin with startup-time logging try: import pypinyin _PYPINYIN_AVAILABLE = True # logger.info("pypinyin loaded successfully for Chinese pinyin sorting") except ImportError: pypinyin = None _PYPINYIN_AVAILABLE = False logger.warning( "pypinyin is not installed. Chinese pinyin sorting will use simple string sorting." ) async def safe_vdb_operation_with_exception( operation: Callable, operation_name: str, entity_name: str = "", max_retries: int = 3, retry_delay: float = 0.2, logger_func: Optional[Callable] = None, timeout_seconds: float | None = None, log_start: bool = False, success_log_threshold_seconds: float = 10.0, ) -> None: """ Safely execute vector database operations with retry mechanism and exception handling. This function ensures that VDB operations are executed with proper error handling and retry logic. If all retries fail, it raises an exception to maintain data consistency. Args: operation: The async operation to execute operation_name: Operation name for logging purposes entity_name: Entity name for logging purposes max_retries: Maximum number of retry attempts retry_delay: Delay between retries in seconds logger_func: Logger function to use for error messages timeout_seconds: Optional timeout for a single operation attempt log_start: Whether to emit start/success logs for each attempt success_log_threshold_seconds: Log successful attempts when duration exceeds this threshold Raises: Exception: When operation fails after all retry attempts """ log_func = logger_func or logger.warning for attempt in range(max_retries): start_ts = time.perf_counter() attempt_label = f"{attempt + 1}/{max_retries}" try: if log_start: logger.info( "VDB %s start for %s (attempt %s, timeout=%s)", operation_name, entity_name or "<unknown>", attempt_label, f"{timeout_seconds:.1f}s" if timeout_seconds is not None else "none", ) if timeout_seconds is not None and timeout_seconds > 0: await asyncio.wait_for(operation(), timeout=timeout_seconds) else: await operation() elapsed = time.perf_counter() - start_ts if log_start or elapsed >= success_log_threshold_seconds: logger.info( "VDB %s success for %s in %.2fs (attempt %s)", operation_name, entity_name or "<unknown>", elapsed, attempt_label, ) return # Success, return immediately except asyncio.TimeoutError as e: elapsed = time.perf_counter() - start_ts timeout_msg = ( f"VDB {operation_name} timeout for {entity_name or '<unknown>'} " f"after {elapsed:.2f}s (attempt {attempt_label}, timeout={timeout_seconds}s)" ) if attempt >= max_retries - 1: log_func(timeout_msg) raise TimeoutError(timeout_msg) from e log_func(f"{timeout_msg}, retrying...") if retry_delay > 0: await asyncio.sleep(retry_delay) except Exception as e: elapsed = time.perf_counter() - start_ts if attempt >= max_retries - 1: error_msg = ( f"VDB {operation_name} failed for {entity_name or '<unknown>'} " f"after {max_retries} attempts in {elapsed:.2f}s: {e}" ) log_func(error_msg) raise Exception(error_msg) from e else: log_func( f"VDB {operation_name} attempt {attempt + 1} failed for " f"{entity_name or '<unknown>'} after {elapsed:.2f}s: {e}, retrying..." ) if retry_delay > 0: await asyncio.sleep(retry_delay) def parse_optional_float(raw: str | None) -> float | None: """Decode env strings (or any text) into ``float | None``. Empty string and the literal ``"None"`` (case-insensitive) collapse to ``None`` so users can leave a knob un-set in ``.env`` and have the consuming code fall back to its own default. Any other non-numeric value raises :class:`ValueError` so misconfigured envs fail loudly at parse time rather than silently downstream. Non-finite values (``nan`` / ``inf``) are also rejected: they parse as floats but corrupt semantic-chunker thresholds. """ if raw is None: return None stripped = raw.strip() if not stripped or stripped.lower() == "none": return None value = float(stripped) if not math.isfinite(value): raise ValueError(f"expected a finite float, got {raw!r}") return value def validate_file_path_security(file_path_str: str, base_dir: Path) -> Optional[Path]: """ Validate file path security to prevent Path Traversal attacks. Args: file_path_str: The file path string to validate base_dir: The base directory that the file must be within Returns: Path: Safe file path (resolved, contained to ``base_dir``) if valid; None if unsafe, malformed, or unresolvable. Does NOT check existence — a returned path is guaranteed inside ``base_dir`` but may not exist; the caller decides what "inside but absent" means. """ if not file_path_str or not file_path_str.strip(): return None try: # Clean the file path string clean_path_str = file_path_str.strip() # Check for obvious path traversal patterns before processing # This catches both Unix (..) and Windows (..\) style traversals if ".." in clean_path_str: # Additional check for Windows-style backslash traversal if ( "\\..\\" in clean_path_str or clean_path_str.startswith("..\\") or clean_path_str.endswith("\\..") ): return None # Normalize path separators (convert backslashes to forward slashes) # This helps handle Windows-style paths on Unix systems normalized_path = clean_path_str.replace("\\", "/") # Create path object and resolve it (handles symlinks and relative paths) candidate_path = (base_dir / normalized_path).resolve() base_dir_resolved = base_dir.resolve() # Check if the resolved path is within the base directory if not candidate_path.is_relative_to(base_dir_resolved): return None return candidate_path except Exception as e: logger.warning(f"Invalid file path detected: {file_path_str} - {str(e)}") return None def get_env_value( env_key: str, default: any, value_type: type = str, special_none: bool = False ) -> any: """ Get value from environment variable with type conversion Args: env_key (str): Environment variable key default (any): Default value if env variable is not set value_type (type): Type to convert the value to special_none (bool): If True, return None when value is "None" Returns: any: Converted value from environment or default """ value = os.getenv(env_key) if value is None: return default # Handle special case for "None" string if special_none and value == "None": return None if value_type is bool: return value.strip().lower() in ("true", "1", "yes", "t", "on") # Handle list type with JSON parsing if value_type is list: try: import json parsed_value = json.loads(value) # Ensure the parsed value is actually a list if isinstance(parsed_value, list): return parsed_value else: logger.warning( f"Environment variable {env_key} is not a valid JSON list, using default" ) return default except (json.JSONDecodeError, ValueError) as e: logger.warning( f"Failed to parse {env_key} as JSON list: {e}, using default" ) return default try: converted = value_type(value) except (ValueError, TypeError): return default # Reject non-finite floats so env-backed thresholds cannot become NaN/Inf. if ( value_type is float and isinstance(converted, float) and not math.isfinite(converted) ): logger.warning( "Environment variable %s=%r is not a finite float, using default", env_key, value, ) return default return converted class EffectiveUserPrompt(NamedTuple): """The instruction text a query actually sends, and how it renders.""" text: str """Composed instructions, ``""`` when there are none. Cache-key material.""" slot: str """Value for the ``{user_prompt}`` placeholder; ``"n/a"`` when there are none.""" def resolve_user_prompt( user_prompt: str | None, prefix: str | None, disable_prefix: bool = False, ) -> EffectiveUserPrompt: """Compose the server-side prompt prefix with a request's ``user_prompt``. ``prefix`` is operator configuration (``USER_PROMPT_PREFIX`` or ``USER_PROMPT_PREFIX_FILE``); ``user_prompt`` is what the caller sent. Neither side is normalized: no stripping, no separator inserted. The operator owns the formatting and ends the prefix with its own ``\n\n`` when it should not run into the caller's text — stripping here would eat exactly that. Not stripping ``user_prompt`` matters too: it keeps ``text`` byte-identical to the pre-prefix ``user_prompt or ""`` whenever no prefix is configured, so existing answer-cache entries keep hitting (see :func:`compute_args_hash` call sites in ``operate.py``). **An empty ``user_prompt`` does not disable the prefix.** When the caller sends nothing, the prefix alone becomes the instructions sent to the LLM -- it is indistinguishable downstream from a caller having sent exactly that text, which is also why the prompt templates need no second placeholder. Only ``disable_prefix`` suppresses it. ``"n/a"`` is reached solely when BOTH sides are empty. """ resolved_prefix = "" if disable_prefix else (prefix or "") text = resolved_prefix + (user_prompt or "") return EffectiveUserPrompt(text=text, slot=f"\n\n{text}" if text else "n/a") # Use TYPE_CHECKING to avoid circular imports if TYPE_CHECKING: from lightrag.base import BaseKVStorage, BaseVectorStorage, QueryParam # use the .env that is inside the current folder # allows to use different .env file for each lightrag instance # the OS environment variables take precedence over the .env file load_dotenv(dotenv_path=".env", override=False) VERBOSE_DEBUG = os.getenv("VERBOSE", "false").lower() == "true" PERFORMANCE_TIMING_LOGS = ( os.getenv("LIGHTRAG_PERFORMANCE_TIMING_LOGS", "false").lower() == "true" ) def verbose_debug(msg: str, *args, **kwargs): """Function for outputting detailed debug information. When VERBOSE_DEBUG=True, outputs the complete message. When VERBOSE_DEBUG=False, outputs only the first 50 characters. Args: msg: The message format string *args: Arguments to be formatted into the message **kwargs: Keyword arguments passed to logger.debug() """ if VERBOSE_DEBUG: logger.debug(msg, *args, **kwargs) else: # Format the message with args first if args: formatted_msg = msg % args else: formatted_msg = msg # Then truncate the formatted message truncated_msg = ( formatted_msg[:150] + "..." if len(formatted_msg) > 150 else formatted_msg ) # Remove consecutive newlines truncated_msg = re.sub(r"\n+", "\n", truncated_msg) logger.debug(truncated_msg, **kwargs) def set_verbose_debug(enabled: bool): """Enable or disable verbose debug output""" global VERBOSE_DEBUG VERBOSE_DEBUG = enabled def performance_timing_log(msg: str, *args, **kwargs): """Emit targeted performance timing logs only when explicitly enabled.""" if PERFORMANCE_TIMING_LOGS: logger.info(msg, *args, **kwargs) def safe_log_value(value: str, max_length: int = 200) -> str: """Make a caller-supplied value safe to embed in a log line. Replaces non-printable characters -- notably CR/LF, which could otherwise be used to forge extra log lines (log injection) -- with '?' and truncates over-long values. Only the *logged* form is sanitized; the caller keeps the original value for its own logic. Used wherever untrusted input reaches the log: rate-limit keys built from a submitted username, and operator-supplied identifiers in audited admin actions (source-conflict repair). """ sanitized = "".join(ch if ch.isprintable() else "?" for ch in value) if len(sanitized) > max_length: sanitized = sanitized[:max_length] + "…(truncated)" return sanitized statistic_data = {"llm_call": 0, "llm_cache": 0, "embed_call": 0} class LightragPathFilter(logging.Filter): """Filter for lightrag logger to filter out frequent path access logs""" def __init__(self): super().__init__() # Define paths to be filtered self.filtered_paths = [ "/documents/paginated", "/health", "/webui/", "/documents/pipeline_status", ] # self.filtered_paths = ["/health", "/webui/"] def filter(self, record): try: # Check if record has the required attributes for an access log if not hasattr(record, "args") or not isinstance(record.args, tuple): return True if len(record.args) < 5: return True # Extract method, path and status from the record args method = record.args[1] path = record.args[2] status = record.args[4] # Filter out successful GET/POST requests to filtered paths if ( (method == "GET" or method == "POST") and (status == 200 or status == 304) and path in self.filtered_paths ): return False return True except Exception: # In case of any error, let the message through return True def setup_logger( logger_name: str, level: str = "INFO", add_filter: bool = False, log_file_path: str | None = None, enable_file_logging: bool = True, ): """Set up a logger with console and optionally file handlers Args: logger_name: Name of the logger to set up level: Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL) add_filter: Whether to add LightragPathFilter to the logger log_file_path: Path to the log file. If None and file logging is enabled, defaults to lightrag.log in LOG_DIR or cwd enable_file_logging: Whether to enable logging to a file (defaults to True) """ # Configure formatters detailed_formatter = logging.Formatter( "%(asctime)s - %(name)s - %(levelname)s - %(message)s" ) simple_formatter = logging.Formatter("%(levelname)s: %(message)s") logger_instance = logging.getLogger(logger_name) logger_instance.setLevel(level) logger_instance.handlers = [] # Clear existing handlers logger_instance.propagate = False # Add console handler with safe stream handling console_handler = SafeStreamHandler() console_handler.setFormatter(simple_formatter) console_handler.setLevel(level) logger_instance.addHandler(console_handler) # Add file handler by default unless explicitly disabled if enable_file_logging: # Get log file path if log_file_path is None: log_dir = os.getenv("LOG_DIR", os.getcwd()) log_file_path = os.path.abspath(os.path.join(log_dir, DEFAULT_LOG_FILENAME)) # Ensure log directory exists os.makedirs(os.path.dirname(log_file_path), exist_ok=True) # Get log file max size and backup count from environment variables log_max_bytes = get_env_value("LOG_MAX_BYTES", DEFAULT_LOG_MAX_BYTES, int) log_backup_count = get_env_value( "LOG_BACKUP_COUNT", DEFAULT_LOG_BACKUP_COUNT, int ) try: # Add file handler file_handler = logging.handlers.RotatingFileHandler( filename=log_file_path, maxBytes=log_max_bytes, backupCount=log_backup_count, encoding="utf-8", ) file_handler.setFormatter(detailed_formatter) file_handler.setLevel(level) logger_instance.addHandler(file_handler) except PermissionError as e: logger.warning(f"Could not create log file at {log_file_path}: {str(e)}") logger.warning("Continuing with console logging only") # Add path filter if requested if add_filter: path_filter = LightragPathFilter() logger_instance.addFilter(path_filter) class UnlimitedSemaphore: """A context manager that allows unlimited access.""" async def __aenter__(self): pass async def __aexit__(self, exc_type, exc, tb): pass @dataclass class TaskState: """Task state tracking for priority queue management""" future: asyncio.Future start_time: float execution_start_time: float = None worker_started: bool = False cancellation_requested: bool = False cleanup_done: bool = False @dataclass class EmbeddingFunc: """Embedding function wrapper with dimension validation This class wraps an embedding function to ensure that the output embeddings have the correct dimension. If wrapped multiple times, the inner wrappers will be automatically unwrapped to prevent configuration conflicts where inner wrapper settings would override outer wrapper settings. Using functools.partial for parameter binding: A common pattern is to use functools.partial to pre-bind model and host parameters to an embedding function. When the base embedding function is already decorated with @wrap_embedding_func_with_attrs (e.g., ollama_embed), use `.func` to access the original unwrapped function to avoid double wrapping: Example: from functools import partial # ❌ Wrong - causes double wrapping (inner EmbeddingFunc still executes) func=partial(ollama_embed, embed_model="bge-m3:latest", host="http://localhost:11434") # ✅ Correct - access the unwrapped function via .func func=partial(ollama_embed.func, embed_model="bge-m3:latest", host="http://localhost:11434") Context-aware embedding: The wrapper supports passing a 'context' parameter to distinguish between query and document embeddings. This allows wrapped functions to apply different processing (e.g., prefixes, different models) based on the context: Example: embeddings = await embed_func(texts, context="document") # For indexing embeddings = await embed_func([query], context="query") # For search Return shape contract: The wrapped function MUST return a 2D numpy array of shape ``(len(texts), embedding_dim)`` -- exactly one row per input text, in input order. Every storage backend consumes the result positionally (``embeddings[i]`` belongs to ``texts[i]``), so any other shape is rejected with a ValueError; the wrapper never reshapes, slices or pads the result, because the row-to-input mapping cannot be recovered once it is wrong. Rank and dimension are always checked. The row count is checked against the input batch, resolved from the first positional argument or from the kwarg named after the wrapped function's first parameter; when neither applies the row count is left unverified rather than guessed at. In particular: - A single input still returns ``(1, embedding_dim)``, not ``(embedding_dim,)``. - A provider that returns more vectors than inputs (e.g. one vector per internally split segment of an over-long text) needs its token limit declared via ``max_token_size``, or a dedicated adapter that normalizes the output to one vector per input. - An empty input list may return any empty array, including a bare ``np.array([])``. Args: embedding_dim: Expected dimension of the embeddings(For dimension checking and workspace data isolation in vector DB) func: The actual embedding function to wrap max_token_size: Enable embedding token limit checking for description summarization(Set embedding_token_limit in LightRAG) send_dimensions: Whether to inject embedding_dim argument to underlying function model_name: Model name for implementing workspace data isolation in vector DB supports_asymmetric: Whether the underlying function supports context parameter so it can be injected """ embedding_dim: int func: callable max_token_size: int | None = None send_dimensions: bool = False model_name: str | None = ( None # Model name for implementing workspace data isolation in vector DB ) supports_asymmetric: bool = ( False # Whether underlying function accepts context parameter ) def __post_init__(self): """Unwrap nested EmbeddingFunc to prevent double wrapping issues. When an EmbeddingFunc wraps another EmbeddingFunc, the inner wrapper's __call__ preprocessing would override the outer wrapper's settings. This method detects and unwraps nested EmbeddingFunc instances to ensure that only the outermost wrapper's configuration is applied. """ # Check if func is already an EmbeddingFunc instance and unwrap it max_unwrap_depth = 3 # Safety limit to prevent infinite loops unwrap_count = 0 while isinstance(self.func, EmbeddingFunc): unwrap_count += 1 if unwrap_count > max_unwrap_depth: raise ValueError( f"EmbeddingFunc unwrap depth exceeded {max_unwrap_depth}. " "Possible circular reference detected." ) # Unwrap to get the original function self.func = self.func.func if unwrap_count > 0: logger.warning( f"Detected nested EmbeddingFunc wrapping (depth: {unwrap_count}), " "auto-unwrapped to prevent configuration conflicts. " "Consider using .func to access the unwrapped function directly." ) def _resolve_input_batch(self, args: tuple, kwargs: dict) -> Any: """Return the sequence of texts the caller passed, or None if unknown. The vector count can only be checked against something. Positional is the overwhelmingly common path and costs nothing to read. A keyword call needs the wrapped function's first parameter name to know which kwarg holds the texts, so the signature is inspected only on that path -- and only the parameter name is read, never a full bind(), which would raise on the extra kwargs this wrapper and its priority decorator pass through (``_priority``, ``context``, ...). Returning None means "not resolvable", which downgrades the vector count check to unverifiable rather than guessing at a mapping. """ if args: return args[0] if not kwargs: return None try: params = inspect.signature(self.func).parameters except (TypeError, ValueError): # Builtins and C-implemented callables expose no signature. return None for param in params.values(): if param.kind in ( inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY, ): return kwargs.get(param.name) # A positional-only or *args first parameter cannot be addressed # by keyword at all, so the batch stays unknown. return None return None async def __call__(self, *args, **kwargs) -> np.ndarray: # Only inject embedding_dim when send_dimensions is True if self.send_dimensions: # Check if user provided embedding_dim parameter if "embedding_dim" in kwargs: user_provided_dim = kwargs["embedding_dim"] # If user's value differs from class attribute, output warning if ( user_provided_dim is not None and user_provided_dim != self.embedding_dim ): logger.warning( f"Ignoring user-provided embedding_dim={user_provided_dim}, " f"using declared embedding_dim={self.embedding_dim} from decorator" ) # Inject embedding_dim from decorator kwargs["embedding_dim"] = self.embedding_dim # Remove context parameter if underlying function does not support asymmetric embedding if "context" in kwargs and not self.supports_asymmetric: # Log when a user-provided context is ignored due to lack of support logger.debug( "Context parameter was provided but supports_asymmetric=False. The context value has been ignored." ) kwargs.pop("context") # Check if underlying function supports max_token_size and inject if not provided if self.max_token_size is not None and "max_token_size" not in kwargs: sig = inspect.signature(self.func) if "max_token_size" in sig.parameters: kwargs["max_token_size"] = self.max_token_size # Call the actual embedding function result = await self.func(*args, **kwargs) # Validate the result shape directly rather than inferring it from a # total-element count. A total-element check cannot tell a genuine # vector-count mismatch (2N, D) apart from a dimension mismatch # (N, 2D) -- both divide out to the same "actual vectors" number, so # the wrong one of the two gets reported. Checking ndim/shape[0]/ # shape[1] directly identifies which one actually happened. # # Every mismatch is fatal: the wrapper never reshapes, slices or pads # the result. Each raise is preceded by a logger.error carrying the # likely cause and the fix, so the short exception message stays # readable while the diagnosis is still available in the logs. expected_dim = self.embedding_dim # None means the input batch could not be resolved from the call, so # the vector count is unverifiable -- see _resolve_input_batch. input_batch = self._resolve_input_batch(args, kwargs) expected_vectors = ( len(input_batch) if isinstance(input_batch, (list, tuple)) else None ) # An empty batch carries no vectors and no dimension to validate. A # provider that short-circuits with `if not texts: return np.array([])` # yields a 1D (0,) array, which is a correct answer to a zero-input # request and must not be rejected by the 2D check below. An empty # input paired with a non-empty result still falls through and fails. if expected_vectors == 0 and result.size == 0: return result if result.ndim != 2: logger.error( f"Embedding result has unexpected shape {result.shape}: this " f"wrapper requires a 2D array of exactly one row per input " f"text, i.e. (len(texts), {expected_dim}). A single input " f"must still come back as (1, {expected_dim}), not " f"({expected_dim},); a flattened or nested array is rejected " f"rather than reshaped, because the row-to-input mapping " f"cannot be recovered from it." ) raise ValueError( f"Embedding result has unexpected shape: expected a 2D " f"array (vectors, dimension) but got ndim={result.ndim} " f"(shape={result.shape})." ) if expected_vectors is not None and result.shape[0] != expected_vectors: logger.error( f"Embedding vector count mismatch: {expected_vectors} text(s) " f"in, {result.shape[0]} vector(s) out (shape={result.shape}). " f"The usual cause is an embedding service that splits an " f"over-long input internally and returns one vector per " f"segment: declare the model's real token limit so texts are " f"truncated before the call -- EMBEDDING_TOKEN_LIMIT on the " f"API server, or max_token_size on " f"@wrap_embedding_func_with_attrs for a custom embedding " f"function. A provider that legitimately returns multiple " f"vectors per input needs a dedicated adapter that normalizes " f"its output to one vector per input before it reaches this " f"wrapper; nothing is reshaped or truncated here, since there " f"is no general contract for which rows correspond to which " f"inputs." ) raise ValueError( f"Vector count mismatch: expected {expected_vectors} vectors " f"(one per input text) but got {result.shape[0]} " f"(shape={result.shape})." ) if result.shape[1] != expected_dim: logger.error( f"Embedding dimension mismatch: the model returned " f"{result.shape[1]}-dimensional vectors but this embedding " f"function declares {expected_dim} (shape={result.shape}). " f"Check that EMBEDDING_DIM (or the embedding_dim passed to " f"@wrap_embedding_func_with_attrs / EmbeddingFunc) matches " f"the model actually being called, and that the endpoint " f"honours the requested output dimension. Vectors already " f"stored under the previously declared dimension do not match " f"the corrected one, so clear the data directory too unless " f"nothing has been indexed yet." ) raise ValueError( f"Embedding dimension mismatch: expected dimension " f"{expected_dim} but got {result.shape[1]} (shape={result.shape})." ) return result def compute_args_hash(*args: Any) -> str: """Compute a hash for the given arguments with safe Unicode handling. Args: *args: Arguments to hash Returns: str: Hash string Note: - **Single-argument calls preserve the original algorithm** (plain ``str(arg)``) so that ``compute_mdhash_id(content)`` keeps producing the same document IDs across upgrades. Existing persisted IDs stay valid. - **Two or more arguments** use length-prefixed encoding (``"{len}:{str(arg)}"`` joined without separators) so that adjacent field boundaries are unambiguously recoverable. This prevents the cache-key collisions that the delimiter-less join could produce (e.g. ``("abc","x")`` vs ``("ab","cx")`` both hashed to the same MD5). Length-prefixing is used instead of a sentinel character because query text and free-form fields can contain arbitrary characters, so any fixed delimiter could still be constructed to collide. """ if len(args) <= 1: # Single-arg (or no-arg) path: keep the legacy behavior unchanged so # compute_mdhash_id(content) document IDs remain stable. args_str = "".join([str(arg) for arg in args]) else: # Multi-arg path: length-prefix each field to make boundaries unique. # Format: "3:abc1:x2:10" — the leading length makes the field extent # unambiguous regardless of the field content. args_str = "".join(f"{len(s)}:{s}" for s in (str(arg) for arg in args)) # Use 'replace' error handling to safely encode problematic Unicode characters # This replaces invalid characters with Unicode replacement character (U+FFFD) try: return md5(args_str.encode("utf-8")).hexdigest() except UnicodeEncodeError: # Handle surrogate characters and other encoding issues safe_bytes = args_str.encode("utf-8", errors="replace") return md5(safe_bytes).hexdigest() def _serialize_cache_variant(value: Any) -> str: """Serialize cache-affecting options to a stable string for hash inputs.""" if value is None: return "" if hasattr(value, "model_dump") and callable(value.model_dump): try: value = value.model_dump(mode="json") except TypeError: value = value.model_dump() if hasattr(value, "model_json_schema") and callable(value.model_json_schema): value = value.model_json_schema() try: return json.dumps( value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=repr, ) except (TypeError, ValueError): return repr(value) def get_llm_cache_identity( global_config: dict[str, Any] | None, role: str, ) -> dict[str, Any]: """Get the non-secret LLM identity used to partition LLM cache keys. Includes ``role``, ``binding``, ``model``, and ``host``. Deliberately excludes ``api_key`` and ``provider_options`` so cache keys remain non-secret and safe to persist. """ config = global_config or {} identities = config.get("llm_cache_identities") if isinstance(identities, dict): identity = identities.get(role) if isinstance(identity, dict): return dict(identity) return { "role": role, "binding": None, "model": config.get("llm_model_name"), "host": None, } def serialize_llm_cache_identity(identity: Any) -> str: """Serialize an LLM cache identity for inclusion in hash inputs.""" return _serialize_cache_variant(identity) def _validate_cached_response_format(response_format: Any | None) -> None: """Reject structured-output modes that the cache wrapper does not support.""" if response_format is None: return if ( isinstance(response_format, dict) and response_format.get("type") == "json_object" ): return raise ValueError( "use_llm_func_with_cache only supports response_format={'type': 'json_object'}; " "json_schema and typed response_format values must not be passed through the cache wrapper." ) def compute_mdhash_id(content: str, prefix: str = "") -> str: """ Compute a unique ID for a given content string. The ID is a combination of the given prefix and the MD5 hash of the content string. """ return prefix + compute_args_hash(content) def get_unique_filename_in_parsed(target_dir: Path, original_name: str) -> str: """Generate a unique filename in target_dir, adding numeric suffixes on conflict. Tries the original name first, then `{stem}_001{ext}` ... `{stem}_999{ext}`, falling back to a timestamp-suffixed name if all numeric slots are taken. """ original_path = Path(original_name) base_name = original_path.stem extension = original_path.suffix if not (target_dir / original_name).exists(): return original_name for i in range(1, 1000): new_name = f"{base_name}_{i:03d}{extension}" if not (target_dir / new_name).exists(): return new_name return f"{base_name}_{int(time.time())}{extension}" async def move_file_to_parsed_dir( file_path: Path, *, skip_if_already_parsed: bool = False, ) -> Path | None: """Move a processed source file into its sibling __parsed__ directory. Returns the new path on success, the input path if `skip_if_already_parsed` is set and the file already lives in a `__parsed__` directory, or None if the source no longer exists. """ if not file_path.exists() or not file_path.is_file(): return None if skip_if_already_parsed and file_path.parent.name == PARSED_DIR_NAME: return file_path parsed_dir = file_path.parent / PARSED_DIR_NAME await asyncio.to_thread(parsed_dir.mkdir, parents=True, exist_ok=True) unique_filename = get_unique_filename_in_parsed(parsed_dir, file_path.name) target_path = parsed_dir / unique_filename await asyncio.to_thread(file_path.rename, target_path) logger.debug( f"Moved file to parsed directory: {file_path.name} -> {unique_filename}" ) return target_path def make_relation_vdb_ids(src_entity: str, tgt_entity: str) -> list[str]: """Return candidate relation VDB IDs for an undirected edge. The normalized ID is returned first for all new writes. The reverse-order ID is kept as a compatibility fallback for historical custom-KG imports that hashed the relation using the original endpoint order. """ normalized_src, normalized_tgt = sorted((src_entity, tgt_entity)) relation_ids = [compute_mdhash_id(normalized_src + normalized_tgt, prefix="rel-")] reverse_relation_id = compute_mdhash_id( normalized_tgt + normalized_src, prefix="rel-" ) if reverse_relation_id not in relation_ids: relation_ids.append(reverse_relation_id) return relation_ids def generate_cache_key(mode: str, cache_type: str, hash_value: str) -> str: """Generate a flattened cache key in the format {mode}:{cache_type}:{hash} Args: mode: Cache mode (e.g., 'default', 'local', 'global') cache_type: Type of cache (e.g., 'extract', 'query', 'keywords') hash_value: Hash value from compute_args_hash Returns: str: Flattened cache key """ return f"{mode}:{cache_type}:{hash_value}" def parse_cache_key(cache_key: str) -> tuple[str, str, str] | None: """Parse a flattened cache key back into its components Args: cache_key: Flattened cache key in format {mode}:{cache_type}:{hash} Returns: tuple[str, str, str] | None: (mode, cache_type, hash) or None if invalid format """ parts = cache_key.split(":", 2) if len(parts) == 3: return parts[0], parts[1], parts[2] return None # Custom exception classes class QueueFullError(Exception): """Raised when the queue is full and the wait times out""" pass class VectorStorageConsistencyError(Exception): """Raised when a step AFTER a durable graph update fails. The knowledge graph (plus the text_chunks KV store) is the authoritative data source, so no data is lost — but something that mirrors or annotates it did not complete: * a **vector storage** write, so the vector records no longer mirror the graph and query results may be incomplete until they are rebuilt. Stop the LightRAG server and run the offline rebuild tool (``lightrag-rebuild-vdb``) to restore consistency. * a **chunk-tracking** retirement, so rows survive for graph objects a merge or rename has removed. Those are dead bookkeeping until the same key recurs; the message names the keys. What every case shares is the reason this type exists at all: the graph mutation IS durable, so the failure must not be reported as one that did not happen. ``_edit_entity_impl``'s ``allow_merge`` handler re-raises exactly this type and folds every other exception into a partial-success summary answering HTTP 200 — which for a landed merge would name the source entity it just deleted as the surviving one. """ pass class WorkerTimeoutError(Exception): """Worker-level timeout exception with specific timeout information""" def __init__(self, timeout_value: float, timeout_type: str = "execution"): self.timeout_value = timeout_value self.timeout_type = timeout_type super().__init__(f"Worker {timeout_type} timeout after {timeout_value}s") class HealthCheckTimeoutError(Exception): """Health Check-level timeout exception""" def __init__(self, timeout_value: float, execution_duration: float): self.timeout_value = timeout_value self.execution_duration = execution_duration super().__init__( f"Task forcefully terminated due to execution timeout (>{timeout_value}s, actual: {execution_duration:.1f}s)" ) class TokenBudgetError(Exception): """Raised by :class:`Tokenizer`'s split/truncate contract when not even the first complete Unicode code point of a candidate fits within ``max_tokens``. Carries the fields needed to diagnose the failure without re-deriving them: the budget that was violated, the independently-encoded token count of that single code point under the same tokenizer, and a bounded preview of the offending text. """ def __init__(self, max_tokens: int, code_point_token_count: int, preview: str): self.max_tokens = max_tokens self.code_point_token_count = code_point_token_count self.preview = preview super().__init__( f"Cannot fit content within max_tokens={max_tokens}: the first " f"complete Unicode code point alone encodes to " f"{code_point_token_count} token(s). Preview: {preview!r}" ) def priority_limit_async_func_call( max_size: int, llm_timeout: float = None, max_execution_timeout: float = None, max_task_duration: float = None, max_queue_size: int = 1000, cleanup_timeout: float = 2.0, queue_name: str = "limit_async", concurrency_group: str | None = None, ): """ Enhanced priority-limited asynchronous function call decorator with robust timeout handling This decorator provides a comprehensive solution for managing concurrent LLM requests with: - Multi-layer timeout protection (LLM -> Worker -> Health Check -> User) - Task state tracking to prevent race conditions - Enhanced health check system with stuck task detection - Proper resource cleanup and error recovery - Optional cross-process global concurrency gating (gunicorn multi-worker) Args: max_size: Maximum number of concurrent calls max_queue_size: Maximum queue capacity to prevent memory overflow llm_timeout: LLM provider timeout (from global config), used to calculate other timeouts max_execution_timeout: Maximum time for worker to execute function (defaults to llm_timeout + 30s) max_task_duration: Maximum time before health check intervenes (defaults to llm_timeout + 60s) cleanup_timeout: Maximum time to wait for cleanup operations (defaults to 2.0s) queue_name: Optional queue name for logging identification (defaults to "limit_async") concurrency_group: Optional cross-process concurrency group name (e.g. "llm:extract", "embedding", "rerank"). When shared storage was initialized with a global limit for this group, workers acquire a cross-worker slot (lease with heartbeat self-healing) before executing, capping total in-flight calls across all gunicorn workers; the group's queue stats are also published for /health aggregation. With no global limit configured for the group (single-process / embedded usage) the slot gate is bypassed — execution behavior matches the original per-process decorator — but queue stats are still published to shared storage so the aggregated /health view works; in single-process mode that is a cheap local-dict write (no IPC, no slot acquisition). Only concurrency_group=None is fully self-contained: shared storage is never touched at all (no slot gate AND no stats publishing). Returns: Decorator function """ def final_dec(func): # Ensure func is callable if not callable(func): raise TypeError(f"Expected a callable object, got {type(func)}") # Calculate timeout hierarchy if llm_timeout is provided (Dynamic Timeout Calculation) if llm_timeout is not None: nonlocal max_execution_timeout, max_task_duration if max_execution_timeout is None: max_execution_timeout = ( llm_timeout * 2 ) # Reserved timeout buffer for low-level retry if max_task_duration is None: max_task_duration = ( llm_timeout * 2 + 15 ) # Reserved timeout buffer for health check phase # The queue is created lazily in ensure_workers(): the default path # keeps the bounded queue, while global-limit mode needs an unbounded # physical queue (admission is enforced logically via live_queued so # cancelled-but-not-yet-drained tuples can never wedge the queue). queue: asyncio.PriorityQueue | None = None tasks = set() initialization_lock = asyncio.Lock() counter = 0 shutdown_event = asyncio.Event() initialized = False accepting_new_tasks = True worker_health_check_task = None # Enhanced task state management task_states = {} # task_id -> TaskState task_states_lock = asyncio.Lock() active_futures = weakref.WeakSet() reinit_count = 0 submitted_total = 0 completed_total = 0 failed_total = 0 cancelled_total = 0 rejected_total = 0 # --- Cross-worker global concurrency gate state (global-limit mode) --- # Tri-state: None until resolved on first ensure_workers() (which runs # after initialize_share_data() in every supported flow). use_global_limit: bool | None = None publish_stats = False shared = None # lazily imported lightrag.kg.shared_storage module work_available = asyncio.Event() admission_cond = asyncio.Condition() # Logical queued count: live tasks waiting in the queue (excludes # running tasks and cancelled zombies) — same capacity semantics as # the bounded queue's maxsize in the default path. live_queued = 0 held_leases: set[str] = set() pending_release: set[str] = set() global_slot_waits = 0 zombie_compact_threshold = max( DEFAULT_ZOMBIE_COMPACT_THRESHOLD, max_queue_size if max_queue_size > 0 else 0, ) # Slot pump machinery (global-limit mode): ONE coroutine per process # acquires global slots and hands (lease, task) pairs to executor # workers through dispatch_queue. executing counts tasks picked up # by workers; worker_free wakes the pump when one finishes. # NOTE: dispatch_queue deliberately never gets task_done()/join() — # the join()-based graceful drain tracks the PHYSICAL queue only (a # dispatched item's physical-queue task_done() is deferred to the # worker), and shutdown empties any undelivered dispatch entries with # a get_nowait() loop. So dispatch_queue.unfinished_tasks grows # unbounded by design; it is never read. Don't add a join() here # without also adding matching task_done() calls. dispatch_queue: asyncio.Queue | None = None pump_task: asyncio.Task | None = None executing = 0 worker_free = asyncio.Event() last_publish_time = 0.0 last_release_warn_time = 0.0 last_renew_warn_time = 0.0 def _resolve_mode() -> bool: """Resolve global-limit / stats-publishing mode from shared storage. Returns True when the resolution is final. Never imports or touches shared storage when concurrency_group is None (standalone decorator usage stays fully self-contained). """ nonlocal use_global_limit, publish_stats, shared if use_global_limit is not None: return True if concurrency_group is None: use_global_limit = False publish_stats = False return True if shared is None: from lightrag.kg import shared_storage as shared_module shared = shared_module if not shared.is_share_data_initialized(): return False # not final yet — caller decides how to commit use_global_limit = shared.is_global_concurrency_limited(concurrency_group) publish_stats = True return True def _snapshot() -> dict: """Synchronous snapshot of local state for cross-worker publishing. Reads counters without locks: all mutations happen on the event loop between awaits, so a synchronous read is always consistent. """ running = sum( 1 for task_state in task_states.values() if task_state.worker_started and not task_state.future.done() ) physical_queued = queue.qsize() if queue is not None else 0 return { "queue_name": queue_name, "max_async": max_size, "max_queue_size": max_queue_size, "queued": live_queued if use_global_limit else physical_queued, "physical_queued": physical_queued, "running": running, "in_flight": len(task_states), "worker_count": len([task for task in tasks if not task.done()]), "initialized": initialized, "submitted_total": submitted_total, "completed_total": completed_total, "failed_total": failed_total, "cancelled_total": cancelled_total, "rejected_total": rejected_total, "global_slot_waits": global_slot_waits, "pid": os.getpid(), "updated_at": time.time(), } async def _publish_stats(force: bool = False) -> None: """Best-effort, debounced publish of the local stats snapshot. Called from counter-update points (debounced to the min publish interval) and force-flushed by the 5s maintenance pass, which also propagates any counter change that happened between debounced publishes and keeps the snapshot ahead of the aggregation stale TTL. """ nonlocal last_publish_time if not publish_stats: return now = time.time() if ( not force and now - last_publish_time < DEFAULT_QUEUE_STATS_MIN_PUBLISH_INTERVAL ): return try: await shared.publish_queue_stats(queue_name, _snapshot()) last_publish_time = now except Exception as e: logger.debug(f"{queue_name}: queue stats publish failed: {e}") async def _notify_admission() -> None: async with admission_cond: admission_cond.notify_all() async def _try_acquire_slot() -> tuple[str | None, bool]: """Non-blocking global slot acquisition (fail-closed on errors). Returns ``(lease_id, is_priority_waiter)``: on failure the second element reports whether this process is the longest-waiting live poller of the group, which drives the adaptive poll backoff below. """ try: lease_id, is_priority = await shared.try_acquire_global_slot_tracked( concurrency_group ) except Exception as e: # try_acquire_global_slot_tracked is fail-closed internally; # this guard keeps the worker alive even if it ever raises. logger.debug(f"{queue_name}: global slot acquisition error: {e}") return None, False if lease_id is not None: held_leases.add(lease_id) return lease_id, is_priority async def _release_lease_safely(lease_id: str) -> None: """Release a global slot without raising (safe in finally blocks). A failed release is parked in pending_release: it is no longer renewed, the health check retries it, and even if every retry fails the heartbeat TTL guarantees any process eventually reclaims the slot — capacity never leaks permanently. """ nonlocal last_release_warn_time held_leases.discard(lease_id) try: await shared.release_global_slot(concurrency_group, lease_id) pending_release.discard(lease_id) except asyncio.CancelledError: pending_release.add(lease_id) raise except Exception as e: pending_release.add(lease_id) now = time.time() if now - last_release_warn_time >= 30.0: last_release_warn_time = now logger.warning( f"{queue_name}: failed to release global slot lease " f"(queued for retry; heartbeat expiry guarantees " f"reclamation): {e}" ) async def _compact_physical_queue() -> None: """Drain zombie tuples that accumulate while no slot is available. Without this, a long fail-closed period (shared storage errors) or externally saturated slots would let cancelled tasks pile up in the unbounded physical queue with no consumer. Bounded batches keep the event loop responsive; every popped tuple gets exactly one task_done() (live tuples are re-queued first, adding a fresh unfinished count) so queue.join() in shutdown never wedges. """ nonlocal live_queued if queue is None or not use_global_limit: return if queue.qsize() - live_queued <= zombie_compact_threshold: return survivors = [] scanned = 0 notify_needed = False while scanned < DEFAULT_COMPACT_BATCH_LIMIT: try: item = queue.get_nowait() except asyncio.QueueEmpty: break scanned += 1 task_id = item[2] is_zombie = False # Classify under task_states_lock so we serialize with the # wait_func timeout cleanup path (never judge by a stale # snapshot taken outside the lock). async with task_states_lock: task_state = task_states.get(task_id) if ( task_state is None or task_state.cancellation_requested or task_state.future.cancelled() or task_state.future.done() ): is_zombie = True if task_state is not None: task_states.pop(task_id, None) if not task_state.worker_started: live_queued -= 1 notify_needed = True if is_zombie: queue.task_done() else: survivors.append(item) for item in survivors: queue.put_nowait(item) queue.task_done() if survivors: work_available.set() if notify_needed: await _notify_admission() async def _run_maintenance() -> None: """One heartbeat pass of cross-worker upkeep (never raises). Runs every health-check tick: lease renewal (correctness path — failures get a rate-limited WARNING, the suspect grace absorbs short outages), pending-release retries, lease reaping, zombie compaction, and a forced stats flush (which also keeps this worker's snapshot from going stale in the aggregation view). """ nonlocal last_renew_warn_time if use_global_limit: try: await shared.renew_global_slots( concurrency_group, tuple(held_leases) ) except Exception as e: now = time.time() if now - last_renew_warn_time >= 30.0: last_renew_warn_time = now logger.warning( f"{queue_name}: global slot lease renewal failed " f"(leases may be reclaimed after the suspect " f"grace if this persists): {e}" ) for lease_id in tuple(pending_release): try: await shared.release_global_slot(concurrency_group, lease_id) pending_release.discard(lease_id) except Exception as e: logger.debug( f"{queue_name}: pending lease release retry failed: {e}" ) break # shared area still unhealthy; retry next pass try: await shared.reconcile_global_slots(concurrency_group) except Exception as e: logger.debug(f"{queue_name}: global slot reconcile failed: {e}") try: await _compact_physical_queue() except Exception as e: logger.warning(f"{queue_name}: queue compaction failed: {e}") if publish_stats: await _publish_stats(force=True) async def worker(): """Enhanced worker that processes tasks with proper timeout and state management""" try: while not shutdown_event.is_set(): try: # Get task from queue with timeout for shutdown checking try: ( priority, count, task_id, args, kwargs, ctx, ) = await asyncio.wait_for(queue.get(), timeout=1.0) except asyncio.TimeoutError: continue # Get task state and mark worker as started async with task_states_lock: if task_id not in task_states: queue.task_done() continue task_state = task_states[task_id] task_state.worker_started = True # Record execution start time when worker actually begins processing task_state.execution_start_time = ( asyncio.get_running_loop().time() ) # Check if task was cancelled before worker started if ( task_state.cancellation_requested or task_state.future.cancelled() ): async with task_states_lock: task_states.pop(task_id, None) queue.task_done() continue try: # Execute the call under the enqueuer's captured # context, not the worker's creation-time context # (asyncio.Task snapshots contextvars once, at # create_task() time, and this worker task is # long-lived — reused across many unrelated # calls). ctx.run(asyncio.ensure_future, ...) # schedules the coroutine as a fresh Task whose # own context is copied from ctx (works on # Python 3.10, unlike create_task's context= # kwarg which needs 3.11+). exec_task = ctx.run( asyncio.ensure_future, func(*args, **kwargs) ) if max_execution_timeout is not None: result = await asyncio.wait_for( exec_task, timeout=max_execution_timeout ) else: result = await exec_task # Set result if future is still valid if not task_state.future.done(): task_state.future.set_result(result) except asyncio.TimeoutError: # Worker-level timeout (max_execution_timeout exceeded) logger.warning( f"{queue_name}: Worker timeout for task {task_id} after {max_execution_timeout}s" ) if not task_state.future.done(): task_state.future.set_exception( WorkerTimeoutError( max_execution_timeout, "execution" ) ) except asyncio.CancelledError: # Task was cancelled during execution if not task_state.future.done(): task_state.future.cancel() logger.debug( f"{queue_name}: Task {task_id} cancelled during execution" ) except Exception as e: # Function execution error logger.error( f"{queue_name}: Error in decorated function for task {task_id}: {str(e)}" ) if not task_state.future.done(): task_state.future.set_exception(e) finally: # Clean up task state async with task_states_lock: task_states.pop(task_id, None) queue.task_done() except Exception as e: # Critical error in worker loop logger.error( f"{queue_name}: Critical error in worker: {str(e)}" ) await asyncio.sleep(0.1) finally: logger.debug(f"{queue_name}: Worker exiting") async def slot_pump(): """Single per-process slot acquirer for global-limit mode. Slot-first, drain-second: the pump acquires a cross-process slot BEFORE consuming the local queue, so while slots are saturated tasks stay queued (cancellable, never misjudged as running) and local priority order is preserved — the queue head is committed only once a slot is held, so a later high-priority arrival can still overtake. Centralizing acquisition in ONE coroutine (instead of max_size polling workers) divides the cross-process poll/IPC rate by max_size, and a slot is requested only when there is BOTH physically queued work and an idle worker to run it immediately — the worker herd can no longer grab slots it cannot use (which inflated global_in_use and reset this process's waiter seniority on every no-op acquire). Residual churn: queued items may all turn out to be zombies after the slot is acquired (drained bounded, slot returned right away). """ nonlocal live_queued, global_slot_waits poll_delay = DEFAULT_GLOBAL_SLOT_POLL_MIN try: while not shutdown_event.is_set(): try: # Idle wait on an event instead of qsize polling. The # clear-then-recheck ordering has no await in between, # so a concurrent put+set can never be lost; the 1.0s # timeout only preserves the shutdown check. if queue.qsize() == 0: work_available.clear() if queue.qsize() == 0: try: await asyncio.wait_for( work_available.wait(), timeout=1.0 ) except asyncio.TimeoutError: pass continue # Never hold a slot no local worker could service # immediately: undelivered dispatches plus running # executions already saturate max_size. if dispatch_queue.qsize() + executing >= max_size: worker_free.clear() if dispatch_queue.qsize() + executing >= max_size: try: await asyncio.wait_for( worker_free.wait(), timeout=1.0 ) except asyncio.TimeoutError: pass continue # Acquire a global slot before touching the queue — # tasks must remain queued (and cancellable) while # all slots are busy. Fail-closed errors land here # too, as a None lease. lease_id, is_priority_waiter = await _try_acquire_slot() if lease_id is None: global_slot_waits += 1 # Soft FIFO across processes: the longest-waiting # live process keeps the fastest poll rate so it # usually claims the next freed slot; everyone # else backs off, bounded by the deferred cap so # a freed slot is never left idle for long when # the favored waiter is gone (promotion lag). if is_priority_waiter: poll_delay = DEFAULT_GLOBAL_SLOT_POLL_MIN else: poll_delay = min( poll_delay * 2, DEFAULT_GLOBAL_SLOT_POLL_DEFERRED_MAX, ) await asyncio.sleep(poll_delay) continue poll_delay = DEFAULT_GLOBAL_SLOT_POLL_MIN live_task = None dispatched = False try: # Take the queue head, draining zombies (bounded # by the drain limit so a zombie-heavy process # doesn't hog a scarce slot for local cleanup). zombies_drained = 0 notify_needed = False while live_task is None: try: item = queue.get_nowait() except asyncio.QueueEmpty: # All queued items were zombies (or # compaction holds them): return the # slot immediately. break task_id, args, kwargs, ctx = ( item[2], item[3], item[4], item[5], ) is_zombie = False async with task_states_lock: task_state = task_states.get(task_id) if ( task_state is None or task_state.cancellation_requested or task_state.future.cancelled() or task_state.future.done() ): is_zombie = True if task_state is not None: task_states.pop(task_id, None) if not task_state.worker_started: live_queued -= 1 notify_needed = True else: task_state.worker_started = True task_state.execution_start_time = ( asyncio.get_running_loop().time() ) live_queued -= 1 notify_needed = True live_task = ( task_id, task_state, args, kwargs, ctx, ) if is_zombie: # Never call the provider for a zombie. queue.task_done() zombies_drained += 1 if ( zombies_drained >= DEFAULT_GLOBAL_SLOT_DRAIN_LIMIT ): break if live_task is not None: # No suspension points between claiming the # live task above and this put_nowait, so a # claimed task is always dispatched (the # admission check guaranteed a free worker). dispatch_queue.put_nowait((lease_id, *live_task)) dispatched = True if notify_needed: await _notify_admission() await _publish_stats() finally: if not dispatched: await _release_lease_safely(lease_id) except Exception as e: logger.error( f"{queue_name}: Critical error in slot pump: {str(e)}" ) await asyncio.sleep(0.1) finally: logger.debug(f"{queue_name}: Slot pump exiting") async def limited_worker(): """Executor worker for global-limit mode. Runs tasks handed over by the slot pump together with their already-held global slot; execution/timeout/exception semantics match the default worker. The lease travels with the task and is always released here (or by the shutdown drain for undelivered dispatch entries). """ nonlocal executing try: while not shutdown_event.is_set(): try: try: ( lease_id, task_id, task_state, args, kwargs, ctx, ) = await asyncio.wait_for( dispatch_queue.get(), timeout=1.0 ) except asyncio.TimeoutError: continue executing += 1 try: # Re-check: the task may have been cancelled in # the (tiny) window between dispatch and pickup. if ( task_state.cancellation_requested or task_state.future.cancelled() or task_state.future.done() ): continue # finally cleans up + returns slot # Run under the enqueuer's captured context, not # this long-lived worker's creation-time context # — see the matching comment in worker(). exec_task = ctx.run( asyncio.ensure_future, func(*args, **kwargs) ) if max_execution_timeout is not None: result = await asyncio.wait_for( exec_task, timeout=max_execution_timeout, ) else: result = await exec_task if not task_state.future.done(): task_state.future.set_result(result) except asyncio.TimeoutError: logger.warning( f"{queue_name}: Worker timeout for task {task_id} after {max_execution_timeout}s" ) if not task_state.future.done(): task_state.future.set_exception( WorkerTimeoutError( max_execution_timeout, "execution" ) ) except asyncio.CancelledError: if not task_state.future.done(): task_state.future.cancel() logger.debug( f"{queue_name}: Task {task_id} cancelled during execution" ) except Exception as e: logger.error( f"{queue_name}: Error in decorated function for task {task_id}: {str(e)}" ) if not task_state.future.done(): task_state.future.set_exception(e) finally: executing -= 1 worker_free.set() async with task_states_lock: task_states.pop(task_id, None) queue.task_done() await _release_lease_safely(lease_id) await _publish_stats() except Exception as e: logger.error( f"{queue_name}: Critical error in worker: {str(e)}" ) await asyncio.sleep(0.1) finally: logger.debug(f"{queue_name}: Worker exiting") def _create_worker_task() -> asyncio.Task: return asyncio.create_task( limited_worker() if use_global_limit else worker() ) async def enhanced_health_check(): """Enhanced health check with stuck task detection and recovery""" nonlocal initialized, pump_task try: while not shutdown_event.is_set(): await asyncio.sleep(5) # Check every 5 seconds current_time = asyncio.get_running_loop().time() # Detect and handle stuck tasks based on execution start time if max_task_duration is not None: stuck_tasks = [] async with task_states_lock: for task_id, task_state in list(task_states.items()): # Only check tasks that have started execution if ( task_state.worker_started and task_state.execution_start_time is not None and current_time - task_state.execution_start_time > max_task_duration ): stuck_tasks.append( ( task_id, current_time - task_state.execution_start_time, ) ) # Force cleanup of stuck tasks for task_id, execution_duration in stuck_tasks: logger.warning( f"{queue_name}: Detected stuck task {task_id} (execution time: {execution_duration:.1f}s), forcing cleanup" ) async with task_states_lock: if task_id in task_states: task_state = task_states[task_id] if not task_state.future.done(): task_state.future.set_exception( HealthCheckTimeoutError( max_task_duration, execution_duration ) ) task_states.pop(task_id, None) # Worker recovery logic current_tasks = set(tasks) done_tasks = {t for t in current_tasks if t.done()} tasks.difference_update(done_tasks) active_tasks_count = len(tasks) workers_needed = max_size - active_tasks_count if workers_needed > 0: logger.info( f"{queue_name}: Creating {workers_needed} new workers" ) new_tasks = set() for _ in range(workers_needed): task = _create_worker_task() new_tasks.add(task) task.add_done_callback(tasks.discard) tasks.update(new_tasks) # Pump recovery: without it no slot is ever acquired and # the whole limited queue stalls. if use_global_limit and (pump_task is None or pump_task.done()): logger.warning(f"{queue_name}: Recreating dead slot pump") pump_task = asyncio.create_task(slot_pump()) # Cross-worker upkeep: lease heartbeat / reaping, zombie # compaction, stats flush. Internally best-effort — each # step isolates its own failures so the health check # loop never exits because of shared-storage errors. await _run_maintenance() except Exception as e: logger.error(f"{queue_name}: Error in enhanced health check: {str(e)}") finally: logger.debug(f"{queue_name}: Enhanced health check task exiting") initialized = False async def ensure_workers(): """Ensure worker system is initialized with enhanced error handling""" nonlocal initialized, worker_health_check_task, tasks, reinit_count nonlocal queue, use_global_limit, dispatch_queue, pump_task if initialized: return async with initialization_lock: if initialized: return # Resolve the concurrency mode once (cached for the lifetime # of the wrapper) and lazily create the matching queue. When # shared storage is not initialized at this point (standalone # usage), commit to the default unlimited path. if use_global_limit is None and not _resolve_mode(): use_global_limit = False if queue is None: if use_global_limit: queue = asyncio.PriorityQueue() dispatch_queue = asyncio.Queue() else: queue = asyncio.PriorityQueue(maxsize=max_queue_size) if reinit_count > 0: reinit_count += 1 logger.warning( f"{queue_name}: Reinitializing system (count: {reinit_count})" ) else: reinit_count = 1 # Clean up completed tasks current_tasks = set(tasks) done_tasks = {t for t in current_tasks if t.done()} tasks.difference_update(done_tasks) active_tasks_count = len(tasks) if active_tasks_count > 0 and reinit_count > 1: logger.warning( f"{queue_name}: {active_tasks_count} tasks still running during reinitialization" ) # Create worker tasks workers_needed = max_size - active_tasks_count for _ in range(workers_needed): task = _create_worker_task() tasks.add(task) task.add_done_callback(tasks.discard) # Start the slot pump (kept out of `tasks` so the worker # recovery count never mistakes it for an executor worker). if use_global_limit and (pump_task is None or pump_task.done()): pump_task = asyncio.create_task(slot_pump()) # Start enhanced health check worker_health_check_task = asyncio.create_task(enhanced_health_check()) initialized = True # Log dynamic timeout configuration timeout_info = [] if llm_timeout is not None: timeout_info.append(f"Func: {llm_timeout}s") if max_execution_timeout is not None: timeout_info.append(f"Worker: {max_execution_timeout}s") if max_task_duration is not None: timeout_info.append(f"Health Check: {max_task_duration}s") timeout_str = ( f"(Timeouts: {', '.join(timeout_info)})" if timeout_info else "" ) logger.info( f"{queue_name}: {workers_needed} new workers initialized {timeout_str}" ) async def get_queue_stats(): """Return a best-effort snapshot of queue and worker state.""" async with task_states_lock: running = sum( 1 for task_state in task_states.values() if task_state.worker_started and not task_state.future.done() ) in_flight = len(task_states) active_workers = len([task for task in tasks if not task.done()]) physical_queued = queue.qsize() if queue is not None else 0 stats = { "queue_name": queue_name, "max_async": max_size, "max_queue_size": max_queue_size, # Global-limit mode reports the logical queued count (live # tasks only — cancelled zombies still physically present in # the unbounded queue are excluded). "queued": live_queued if use_global_limit else physical_queued, "running": running, "in_flight": in_flight, "worker_count": active_workers, "initialized": initialized, "submitted_total": submitted_total, "completed_total": completed_total, "failed_total": failed_total, "cancelled_total": cancelled_total, "rejected_total": rejected_total, } if use_global_limit: stats["physical_queued"] = physical_queued stats["global_slot_waits"] = global_slot_waits return stats async def get_aggregated_queue_stats(): """Local stats merged with every worker process's published snapshot. Publishes this process's fresh snapshot first, then sums the flat counter fields across all live snapshots (schema-compatible with get_queue_stats so /health consumers and the webui need no changes), adding ``reporting_workers`` / ``per_worker`` and — in global-limit mode — ``global_limit`` / ``global_in_use``. Any shared-storage failure falls back to the local snapshot. """ local = await get_queue_stats() if not _resolve_mode() or not publish_stats: return local try: await shared.publish_queue_stats(queue_name, _snapshot()) aggregated = await shared.aggregate_queue_stats(queue_name) result = dict(local) for field_name in shared.QUEUE_STATS_SUM_FIELDS: if field_name in aggregated: result[field_name] = aggregated[field_name] result["reporting_workers"] = aggregated["reporting_workers"] result["per_worker"] = aggregated["per_worker"] if use_global_limit: result["global_limit"] = shared.get_global_concurrency_limit( concurrency_group ) result["global_in_use"] = await shared.global_concurrency_in_use( concurrency_group ) waiters = await shared.global_slot_waiters(concurrency_group) result["global_waiting_workers"] = len(waiters) result["global_longest_wait"] = ( round(waiters[0]["waited"], 3) if waiters else 0.0 ) return result except Exception as e: logger.debug( f"{queue_name}: queue stats aggregation failed, " f"falling back to local snapshot: {e}" ) return local async def shutdown(graceful: bool = True, timeout: float | None = None): """Shut down workers and cleanup resources. Graceful mode stops new submissions and drains queued/running work; if the drain exceeds ``timeout`` (defaulting to ``max_task_duration`` or 30s), it falls through to forced cancellation so shutdown never blocks indefinitely. """ nonlocal accepting_new_tasks, initialized, worker_health_check_task nonlocal pump_task logger.info(f"{queue_name}: Shutting down priority queue workers") if use_global_limit: # Stop accepting and wake admission waiters inside the same # Condition critical section: a request sleeping on admission # (no _queue_timeout) must observe the flag flip and raise # the shutdown rejection instead of sleeping forever. async with admission_cond: accepting_new_tasks = False admission_cond.notify_all() else: accepting_new_tasks = False drain_timed_out = False if graceful and queue is not None: effective_timeout = timeout if effective_timeout is None: effective_timeout = ( max_task_duration if max_task_duration is not None else 30.0 ) try: await asyncio.wait_for(queue.join(), timeout=effective_timeout) except asyncio.TimeoutError: drain_timed_out = True logger.warning( f"{queue_name}: Graceful drain timed out after " f"{effective_timeout}s; cancelling pending work" ) if not graceful or drain_timed_out: # Cancel all active futures for future in list(active_futures): if not future.done(): future.cancel() # Cancel all pending tasks async with task_states_lock: for task_id, task_state in list(task_states.items()): if not task_state.future.done(): task_state.future.cancel() task_states.clear() while queue is not None: try: queue.get_nowait() queue.task_done() except asyncio.QueueEmpty: break shutdown_event.set() # Cancel the slot pump first so no new dispatch entries appear # while workers drain below. if pump_task is not None and not pump_task.done(): pump_task.cancel() try: await pump_task except asyncio.CancelledError: pass pump_task = None # Cancel worker tasks for task in list(tasks): if not task.done(): task.cancel() # Wait for all tasks to complete if tasks: await asyncio.gather(*tasks, return_exceptions=True) # Drain undelivered dispatch entries: each one was already # popped from the physical queue and carries a held lease. while dispatch_queue is not None: try: ( lease_id, task_id, task_state, _args, _kwargs, _ctx, ) = dispatch_queue.get_nowait() except asyncio.QueueEmpty: break if not task_state.future.done(): task_state.future.cancel() async with task_states_lock: task_states.pop(task_id, None) queue.task_done() await _release_lease_safely(lease_id) # Cancel health check task if worker_health_check_task and not worker_health_check_task.done(): worker_health_check_task.cancel() try: await worker_health_check_task except asyncio.CancelledError: pass worker_health_check_task = None initialized = False # Return any global slots still held (worker cancellation may # have interrupted a release) and retract our published stats. # Best-effort: heartbeat expiry reclaims anything left behind. if use_global_limit: for lease_id in list(held_leases | pending_release): held_leases.discard(lease_id) pending_release.discard(lease_id) try: await shared.release_global_slot(concurrency_group, lease_id) except Exception: pass # Our workers stop polling now: drop the waiter record so # this process never lingers in the longest-waiter seat # (the stale TTL covers crashes where this never runs). try: await shared.clear_slot_waiter(concurrency_group) except Exception: pass if publish_stats: try: await shared.unpublish_queue_stats(queue_name) except Exception: pass logger.info(f"{queue_name}: Priority queue workers shutdown complete") async def _limited_wait(args, kwargs, _priority, _timeout, _queue_timeout): """wait_func body for global-limit mode (logical admission). Admission reserves logical capacity (live_queued) BEFORE the task state is registered, with the same semantics as the bounded queue in the default path: only live queued tasks count toward max_queue_size (running tasks and cancelled zombies do not), _queue_timeout bounds the wait with QueueFullError, and max_queue_size <= 0 means unlimited admission. The reservation is released exactly once — by the worker when the task turns running (worker_started flip), or by the cleanup below when the task dies while still queued. """ nonlocal counter, submitted_total, completed_total, cancelled_total nonlocal failed_total, rejected_total, live_queued task_id = ( f"{id(asyncio.current_task())}_{asyncio.get_running_loop().time()}" ) future = asyncio.Future() task_state = TaskState( future=future, start_time=asyncio.get_running_loop().time() ) def _admission_open() -> bool: return live_queued < max_queue_size or not accepting_new_tasks # --- Admission: reserve capacity before registering --- async with admission_cond: if not accepting_new_tasks: rejected_total += 1 raise RuntimeError(f"{queue_name}: Queue is shutting down") if max_queue_size > 0 and live_queued >= max_queue_size: try: if _queue_timeout is not None: await asyncio.wait_for( admission_cond.wait_for(_admission_open), timeout=_queue_timeout, ) else: await admission_cond.wait_for(_admission_open) except asyncio.TimeoutError: raise QueueFullError( f"{queue_name}: Queue full, timeout after {_queue_timeout} seconds" ) if not accepting_new_tasks: # Woken by shutdown's notify_all. rejected_total += 1 raise RuntimeError(f"{queue_name}: Queue is shutting down") live_queued += 1 # Reservation window: until the task state is registered, any # exception/cancellation must hand the reservation back or this # slot of logical capacity would be occupied forever. try: async with task_states_lock: task_states[task_id] = task_state except BaseException: async with admission_cond: live_queued -= 1 admission_cond.notify_all() raise # From here the reservation belongs to the exactly-once rule # (worker_started transfer, or the finally cleanup below). try: active_futures.add(future) # Get counter for FIFO ordering async with initialization_lock: current_count = counter counter += 1 # Capture the enqueuer's context (e.g. an active OpenTelemetry # span) so the worker executes func() under it instead of the # long-lived worker task's own creation-time context. It rides # as the 6th tuple element; keep the unpack sites in sync # (worker / slot_pump / limited_worker / shutdown drain). ctx = contextvars.copy_context() # Unbounded physical queue: put_nowait never blocks, and the # (priority, count, ...) tuple keeps heap ordering intact. queue.put_nowait((_priority, current_count, task_id, args, kwargs, ctx)) submitted_total += 1 work_available.set() await _publish_stats() # Wait for result with the same semantics as the default path try: if _timeout is not None: result = await asyncio.wait_for(future, _timeout) else: result = await future completed_total += 1 await _publish_stats() return result except asyncio.TimeoutError: # User-level timeout: the task may still be queued (e.g. # waiting for a global slot) — mark it cancelled so no # worker ever calls the provider for it. async with task_states_lock: if task_id in task_states: task_states[task_id].cancellation_requested = True if not future.done(): future.cancel() cleanup_start = asyncio.get_running_loop().time() while ( task_id in task_states and asyncio.get_running_loop().time() - cleanup_start < cleanup_timeout ): await asyncio.sleep(0.1) cancelled_total += 1 raise TimeoutError( f"{queue_name}: User timeout after {_timeout} seconds" ) except WorkerTimeoutError as e: failed_total += 1 raise TimeoutError(f"{queue_name}: {str(e)}") except HealthCheckTimeoutError as e: failed_total += 1 raise TimeoutError(f"{queue_name}: {str(e)}") except asyncio.CancelledError: cancelled_total += 1 raise except Exception: failed_total += 1 raise finally: active_futures.discard(future) notify_needed = False async with task_states_lock: popped = task_states.pop(task_id, None) if popped is not None and not popped.worker_started: # Died while still queued: release the reservation # here — the worker never will (exactly-once). live_queued -= 1 notify_needed = True if notify_needed: async with admission_cond: admission_cond.notify_all() @wraps(func) async def wait_func( *args, _priority=DEFAULT_PROCESSING_PRIORITY, _timeout=None, _queue_timeout=None, **kwargs, ): """ Execute function with enhanced priority-based concurrency control and timeout handling Args: *args: Positional arguments passed to the function _priority: Call priority (lower values have higher priority) _timeout: Maximum time to wait for completion (in seconds, none means determinded by max_execution_timeout of the queue) _queue_timeout: Maximum time to wait for entering the queue (in seconds) **kwargs: Keyword arguments passed to the function Returns: The result of the function call Raises: TimeoutError: If the function call times out at any level QueueFullError: If the queue is full and waiting times out Any exception raised by the decorated function """ nonlocal submitted_total, completed_total, cancelled_total, failed_total nonlocal rejected_total if not accepting_new_tasks: rejected_total += 1 raise RuntimeError(f"{queue_name}: Queue is shutting down") await ensure_workers() if use_global_limit: return await _limited_wait( args, kwargs, _priority, _timeout, _queue_timeout ) # Generate unique task ID task_id = ( f"{id(asyncio.current_task())}_{asyncio.get_running_loop().time()}" ) future = asyncio.Future() # Create task state task_state = TaskState( future=future, start_time=asyncio.get_running_loop().time() ) try: # Register task state async with task_states_lock: task_states[task_id] = task_state active_futures.add(future) # Get counter for FIFO ordering nonlocal counter async with initialization_lock: current_count = counter counter += 1 # Capture the enqueuer's context (e.g. an active # OpenTelemetry span) so the worker executes func() under # it instead of the long-lived worker task's own # creation-time context. It rides as the 6th tuple element; # keep the unpack sites in sync (worker / slot_pump / # limited_worker / shutdown drain). ctx = contextvars.copy_context() # Queue the task with timeout handling try: if not accepting_new_tasks: rejected_total += 1 raise RuntimeError(f"{queue_name}: Queue is shutting down") if _queue_timeout is not None: await asyncio.wait_for( queue.put( (_priority, current_count, task_id, args, kwargs, ctx) ), timeout=_queue_timeout, ) else: await queue.put( (_priority, current_count, task_id, args, kwargs, ctx) ) submitted_total += 1 await _publish_stats() except asyncio.TimeoutError: raise QueueFullError( f"{queue_name}: Queue full, timeout after {_queue_timeout} seconds" ) except Exception as e: # Clean up on queue error if not future.done(): future.set_exception(e) raise # Wait for result with timeout handling try: if _timeout is not None: result = await asyncio.wait_for(future, _timeout) else: result = await future completed_total += 1 await _publish_stats() return result except asyncio.TimeoutError: # This is user-level timeout (asyncio.wait_for caused) # Mark cancellation request async with task_states_lock: if task_id in task_states: task_states[task_id].cancellation_requested = True # Cancel future if not future.done(): future.cancel() # Wait for worker cleanup with timeout cleanup_start = asyncio.get_running_loop().time() while ( task_id in task_states and asyncio.get_running_loop().time() - cleanup_start < cleanup_timeout ): await asyncio.sleep(0.1) cancelled_total += 1 raise TimeoutError( f"{queue_name}: User timeout after {_timeout} seconds" ) except WorkerTimeoutError as e: # This is Worker-level timeout, directly propagate exception information failed_total += 1 raise TimeoutError(f"{queue_name}: {str(e)}") except HealthCheckTimeoutError as e: # This is Health Check-level timeout, directly propagate exception information failed_total += 1 raise TimeoutError(f"{queue_name}: {str(e)}") except asyncio.CancelledError: cancelled_total += 1 raise except Exception: failed_total += 1 raise finally: # Ensure cleanup active_futures.discard(future) async with task_states_lock: task_states.pop(task_id, None) # Add shutdown method to decorated function wait_func.shutdown = shutdown wait_func.get_queue_stats = get_queue_stats wait_func.get_aggregated_queue_stats = get_aggregated_queue_stats # One upkeep pass (lease renewal / pending releases / reaping / # compaction / stats flush). The health check runs it every 5s; # exposed for tests and operational tooling. wait_func.run_maintenance = _run_maintenance return wait_func return final_dec def wrap_embedding_func_with_attrs(**kwargs): """Decorator to add embedding dimension and token limit attributes to embedding functions. This decorator wraps an async embedding function and returns an EmbeddingFunc instance that automatically handles dimension parameter injection and attribute management. WARNING: DO NOT apply this decorator to wrapper functions that call other decorated embedding functions. This will cause double decoration and parameter injection conflicts. Correct usage patterns: 1. Direct decoration: ```python @wrap_embedding_func_with_attrs(embedding_dim=1536, max_token_size=8192, model_name="my_embedding_model") async def my_embed(texts, embedding_dim=None): # Direct implementation return embeddings ``` 2. Double decoration: ```python @wrap_embedding_func_with_attrs(embedding_dim=1536, max_token_size=8192, model_name="my_embedding_model") @retry(...) async def my_embed(texts, ...): # Base implementation pass @wrap_embedding_func_with_attrs(embedding_dim=1024, max_token_size=4096, model_name="another_embedding_model") # Note: No @retry here! async def my_new_embed(texts, ...): # CRITICAL: Call .func to access unwrapped function return await my_embed.func(texts, ...) # ✅ Correct # return await my_embed(texts, ...) # ❌ Wrong - double decoration! ``` 3. Context-aware decoration: ```python @wrap_embedding_func_with_attrs( embedding_dim=1536, model_name="my_embedding_model", supports_asymmetric=True ) async def my_embed(texts, context="document"): # Apply different prefixes based on context if context == "query": texts = ["search_query: " + t for t in texts] elif context == "document": texts = ["search_document: " + t for t in texts] return embeddings ``` The decorated function becomes an EmbeddingFunc instance with: - embedding_dim: The embedding dimension - max_token_size: Maximum token limit (optional) - model_name: Model name (optional) - supports_asymmetric: Whether context parameter is supported (optional) - func: The original unwrapped function (access via .func) - __call__: Wrapper that injects embedding_dim parameter and context Args: embedding_dim: The dimension of embedding vectors max_token_size: Maximum number of tokens (optional) send_dimensions: Whether to pass embedding_dim as a keyword argument (for models with configurable embedding dimensions). supports_asymmetric: Whether the function supports context parameter (optional). If omitted, this is auto-detected from the wrapped function's signature (set to True iff the function accepts a ``context`` parameter). Returns: A decorator that wraps the function as an EmbeddingFunc instance """ def final_dec(func) -> EmbeddingFunc: embedding_kwargs = dict(kwargs) # Auto-detect supports_asymmetric from the wrapped function's signature # if the caller did not declare it explicitly. Without this, any user or # third-party embed function that accepts a `context` parameter but # forgets to set ``supports_asymmetric=True`` would have its `context` # silently dropped by ``EmbeddingFunc.__call__``, defeating the # task-aware embedding feature. if "supports_asymmetric" not in embedding_kwargs: try: sig = inspect.signature(func) embedding_kwargs["supports_asymmetric"] = "context" in sig.parameters except (TypeError, ValueError): # inspect.signature can fail for builtins; fall back to False. embedding_kwargs["supports_asymmetric"] = False new_func = EmbeddingFunc(**embedding_kwargs, func=func) return new_func return final_dec def load_json(file_name): if not os.path.exists(file_name): return None with open(file_name, encoding="utf-8-sig") as f: content = f.read() # Empty/whitespace existing file: same contract as missing (callers use or {}). if not content.strip(): logger.warning("Empty JSON file treated as missing: %s", file_name) return None return json.loads(content) def _sanitize_string_for_json(text: str) -> str: """Remove characters that cannot be encoded in UTF-8 for JSON serialization. Uses regex for optimal performance with zero-copy optimization for clean strings. Fast detection path for clean strings (99% of cases) with efficient removal for dirty strings. Args: text: String to sanitize Returns: Original string if clean (zero-copy), sanitized string if dirty """ if not text: return text # Fast path: Check if sanitization is needed using C-level regex search if not _SURROGATE_PATTERN.search(text): return text # Zero-copy for clean strings - most common case # Slow path: Remove problematic characters using C-level regex substitution return _SURROGATE_PATTERN.sub("", text) class SanitizingJSONEncoder(json.JSONEncoder): """ Custom JSON encoder that sanitizes data during serialization. This encoder cleans strings during the encoding process without creating a full copy of the data structure, making it memory-efficient for large datasets. """ def encode(self, o): """Override encode method to handle simple string cases""" if isinstance(o, str): return json.encoder.encode_basestring(_sanitize_string_for_json(o)) return super().encode(o) def iterencode(self, o, _one_shot=False): """ Override iterencode to sanitize strings during serialization. This is the core method that handles complex nested structures. """ # Preprocess: sanitize all strings in the object sanitized = self._sanitize_for_encoding(o) # Call parent's iterencode with sanitized data for chunk in super().iterencode(sanitized, _one_shot): yield chunk def _sanitize_for_encoding(self, obj): """ Recursively sanitize strings in an object. Creates new objects only when necessary to avoid deep copies. Args: obj: Object to sanitize Returns: Sanitized object with cleaned strings """ if isinstance(obj, str): return _sanitize_string_for_json(obj) elif isinstance(obj, dict): # Create new dict with sanitized keys and values new_dict = {} for k, v in obj.items(): clean_k = _sanitize_string_for_json(k) if isinstance(k, str) else k clean_v = self._sanitize_for_encoding(v) new_dict[clean_k] = clean_v return new_dict elif isinstance(obj, (list, tuple)): # Sanitize list/tuple elements cleaned = [self._sanitize_for_encoding(item) for item in obj] return type(obj)(cleaned) if isinstance(obj, tuple) else cleaned else: # Numbers, booleans, None, etc. remain unchanged return obj def write_json(json_obj, file_name): """ Write JSON data to file with optimized sanitization strategy. This function uses a two-stage approach: 1. Fast path: Try direct serialization (works for clean data ~99% of time) 2. Slow path: Use custom encoder that sanitizes during serialization The custom encoder approach avoids creating a deep copy of the data, making it memory-efficient. When sanitization occurs, the caller should reload the cleaned data from the file to update shared memory. Writes are atomic: both the fast path and the sanitizing fallback land in the same per-writer tmp sibling, and only the final ``os.replace`` publishes the file. A crash mid-write leaves the prior snapshot intact. Args: json_obj: Object to serialize (may be a shallow copy from shared memory) file_name: Output file path Returns: bool: True if sanitization was applied (caller should reload data), False if direct write succeeded (no reload needed) """ from lightrag.file_atomic import atomic_write sanitized = False def _do_write(tmp_path: str) -> None: nonlocal sanitized try: # Strategy 1: Fast path - try direct serialization. with open(tmp_path, "w", encoding="utf-8") as f: json.dump(json_obj, f, indent=2, ensure_ascii=False) except (UnicodeEncodeError, UnicodeDecodeError) as e: logger.debug(f"Direct JSON write failed, using sanitizing encoder: {e}") # Strategy 2: Use sanitizing encoder (zero-copy). Reusing the # same tmp path keeps the operation single-rename even on the # slow path. with open(tmp_path, "w", encoding="utf-8") as f: json.dump( json_obj, f, indent=2, ensure_ascii=False, cls=SanitizingJSONEncoder, ) sanitized = True atomic_write(file_name, _do_write) if sanitized: logger.info(f"JSON sanitization applied during write: {file_name}") return sanitized @dataclass(frozen=True, slots=True) class TokenSpan: """A character-offset span into an original string, plus its own token count. ``content[start:end]`` is always a verbatim substring of the string that produced this span — never a decoded reconstruction — and independently re-encoding it under the same :class:`Tokenizer` yields exactly ``token_count`` tokens, which is guaranteed to be ``<= max_tokens`` for whatever budget produced the span. See the ``Tokenizer.split_by_token_limit`` / ``truncate_by_token_limit`` docstrings for the full contract. """ start: int end: int token_count: int class TokenizerInterface(Protocol): """ Defines the interface for a tokenizer, requiring encode and decode methods. Implementations MUST be safe to call concurrently from multiple OS threads — LightRAG runs token counting in worker threads to keep it off the asyncio event loop, and does not serialize calls on the implementation's behalf. See :class:`Tokenizer` for why serializing here would be the wrong layer. """ def encode(self, content: str) -> List[int]: """Encodes a string into a list of tokens. May be called concurrently from multiple threads. """ ... def decode(self, tokens: List[int]) -> str: """Decodes a list of tokens into a string. May be called concurrently from multiple threads. """ ... # --------------------------------------------------------------------------- # Bounded submission to a thread pool # --------------------------------------------------------------------------- # # Moving CPU-bound work off the event loop frees the loop to keep accepting # requests, which means more work can be in flight at once than before. A # ``ThreadPoolExecutor``'s wait queue is unbounded, so submissions need their own # ceiling. # # The subtle part is WHO holds the permit. Writing ``async with sem: await # run_in_executor(...)`` is wrong: cancelling the awaiting coroutine returns the # permit immediately, but the thread-pool task is not cancelled — an already # running ``concurrent.futures.Future.cancel()`` just returns False and the thread # runs to completion. A client that repeatedly submits and cancels would hand back # permits it is still consuming and rebuild an unbounded queue. The permit # therefore belongs to the executor future and is released from its done callback, # so permit occupancy tracks actual thread occupancy exactly. def _consume_future_exception(fut: "asyncio.Future") -> None: """Mark a shielded future's exception as retrieved. When ``shield`` is cancelled nobody awaits the inner future any more, so an exception raised by the thread would surface as an "exception was never retrieved" traceback at GC time. Retrieving it here only clears that log flag; a caller that is still awaiting still sees the exception. """ if not fut.cancelled(): fut.exception() async def _wait_deferring_cancellation( future: "asyncio.Future", pending_cancel: Optional[asyncio.CancelledError], ) -> Optional[asyncio.CancelledError]: """Await ``future`` to completion, deferring cancellation of the caller. Loop + shield, not a single shield: a lone ``await asyncio.shield(...)`` returns immediately on RE-cancellation, so a second cancel would let the caller escape while the work is still running. Same shape and same reason as ``_KeyedLockContext.__aexit__`` in ``lightrag/kg/shared_storage.py``. Returns the cancellation to re-raise once every step is done (the first one seen, if any). Cancelling ``future`` ITSELF still propagates immediately: it did not run to completion and we must not pretend it did. """ while not future.done(): try: await asyncio.shield(future) except asyncio.CancelledError as exc: if future.cancelled(): raise pending_cancel = pending_cancel or exc except BaseException: # The awaited work failed. Do NOT let that exception out from here: a # cancellation recorded on an earlier round has to take precedence, # and deciding that is the caller's job. The exception stays on # ``future`` for the caller to re-raise or log. Anything raised while # the future is still pending came from elsewhere and propagates. if not future.done(): raise break return pending_cancel async def _bounded_submit_impl( executor: ThreadPoolExecutor, semaphore: asyncio.Semaphore, fn: Callable[..., Any], args: tuple[Any, ...], kwargs: dict[str, Any], *, wait_for_completion: bool, on_committed: Optional[Callable[[], Awaitable[Any]]] = None, ) -> Any: """Shared body of :func:`bounded_submit` and :func:`run_in_storage_io`. Private, and takes ``args`` / ``kwargs`` as explicit containers rather than ``*args, **kwargs``, so the control flag can never collide with an argument meant for ``fn``. ``bounded_submit`` forwards its ``**kwargs`` verbatim, so putting ``wait_for_completion`` on its own signature would either make it positional-only (before the ``/``, where passing it by keyword would silently forward it to ``fn`` instead of setting it) or steal a legitimate argument of ``fn`` (after ``*args``). ``wait_for_completion`` selects what happens AFTER the work is submitted: * ``False`` — cancelling the caller returns immediately. The thread still runs to completion and still owns the permit; the caller just stops waiting for it. * ``True`` — the caller refuses to return until the worker is done, because the worker may be halfway through mutating state the caller's lock is protecting (see :func:`run_in_storage_io`). Either way the permit wait itself stays cancellable: nothing has been submitted at that point, so a cancelled caller leaves no work in flight and can release whatever lock it holds immediately. ``on_committed`` (requires ``wait_for_completion``) is the bookkeeping that must not be separated from a write that already landed — flipping other processes' reload flags, retiring a redo log. It runs inside the SAME uncancellable region as the write, and only when the write actually succeeded: * write succeeded → ``on_committed`` runs to completion, and only then is a deferred cancellation re-raised; * ``fn`` raised → ``on_committed`` does NOT run and the exception propagates; the write did not land, so its bookkeeping must not either; * cancelled while waiting for a permit → ``on_committed`` does NOT run, because nothing was ever submitted. That is load-bearing, not tidiness: Nano's bookkeeping retires the redo log, and running it without a write would discard rows that were never persisted. * ``on_committed`` raised → the write is already durable, so the failure is re-raised as :class:`CommitBookkeepingError`, which says exactly that. The hook's own exception must NOT reach the caller unwrapped: it is then indistinguishable from a write that never landed, and every call site's ``except Exception`` handles it by reasoning that only holds for that case. """ await semaphore.acquire() try: concurrent_future: ConcurrentFuture = executor.submit( contextvars.copy_context().run, partial(fn, *args, **kwargs) ) except BaseException: semaphore.release() raise loop = asyncio.get_running_loop() def _release(_done: ConcurrentFuture) -> None: # Runs on the worker thread, so the release has to be posted back to the # loop that owns the semaphore. A closed loop means the semaphore died # with it and there is nothing left to release. try: loop.call_soon_threadsafe(semaphore.release) except RuntimeError: pass concurrent_future.add_done_callback(_release) async_future = asyncio.wrap_future(concurrent_future) async_future.add_done_callback(_consume_future_exception) if not wait_for_completion: if on_committed is not None: raise ValueError("on_committed requires wait_for_completion") return await asyncio.shield(async_future) pending_cancel = await _wait_deferring_cancellation(async_future, None) write_exc = None if async_future.cancelled() else async_future.exception() commit_exc: Optional[BaseException] = None if write_exc is None and on_committed is not None: # The write landed, so its bookkeeping is no longer optional: a cancel # here would leave the file published with the other processes never told # to reload it. Deferred through this step too — without the second wait # the gap simply moves one layer down, onto whatever this awaits. commit_future = asyncio.ensure_future(on_committed()) commit_future.add_done_callback(_consume_future_exception) pending_cancel = await _wait_deferring_cancellation( commit_future, pending_cancel ) if not commit_future.cancelled(): commit_exc = commit_future.exception() if pending_cancel is not None: # The caller gets CancelledError, so nobody will ever see these. # ``_consume_future_exception`` already marked them retrieved (no # "exception was never retrieved" noise at GC time); log them so the # failure is not lost entirely. for label, exc in (("Offloaded work", write_exc), ("Commit hook", commit_exc)): if exc is not None: logger.error(f"{label} failed while its caller was cancelled: {exc}") raise pending_cancel if commit_exc is not None: if isinstance(commit_exc, Exception): # Typed, because the two failures this function can report want # opposite handling and used to look identical: ``fn`` raising means # nothing was persisted, while reaching HERE means the write is on # disk and only its publication failed. A caller that cannot tell # them apart rolls back in-memory state the file already has, and # reports a durable mutation as one that never happened. raise CommitBookkeepingError( f"The offloaded write landed, but its commit bookkeeping " f"failed: {commit_exc}", result=async_future.result(), ) from commit_exc # A BaseException that is not an Exception (SystemExit, KeyboardInterrupt) # is interpreter-level control flow, not a bookkeeping failure, and # wrapping it would demote a shutdown request into a storage error. A # cancelled hook never arrives here — ``commit_future.cancelled()`` is # tested above, and that case is the caller's cancellation path. raise commit_exc return async_future.result() async def bounded_submit( executor: ThreadPoolExecutor, semaphore: asyncio.Semaphore, fn: Callable[..., Any], /, *args: Any, **kwargs: Any, ) -> Any: """Run ``fn`` in ``executor`` under a permit owned by the executor future. Args: executor: destination pool. semaphore: submission ceiling for this pool, from :func:`get_loop_semaphore`. fn: synchronous callable to run in the pool. *args, **kwargs: forwarded to ``fn`` verbatim. This function declares no keyword arguments of its own, so ``fn`` may use any parameter name. Returns: Whatever ``fn`` returns. Cancelling the caller does not cancel the thread: the work runs to completion and only then is the permit returned. That is deliberate — see the module comment above. Callers that must not RETURN before the thread finishes use :func:`run_in_storage_io` instead. """ return await _bounded_submit_impl( executor, semaphore, fn, args, kwargs, wait_for_completion=False ) # Semaphores are per event loop, executors are per process. ``asyncio.Semaphore`` # binds to the first loop that has to wait on it and raises from any other loop # (``asyncio.mixins._LoopBoundMixin``), and the test suite runs one fresh loop per # case, so a module-level singleton would fail. Storing it ON the loop makes its # lifetime exactly the loop's; a container keyed by the loop would not, because # once contended the semaphore holds a strong reference back to its loop and a # ``WeakKeyDictionary`` entry whose value references its key never expires. _LOOP_SEMAPHORE_ATTR_PREFIX = "_lightrag_submit_semaphore_" _LOOP_SEMAPHORE_FALLBACK: dict[int, tuple[Any, dict[str, asyncio.Semaphore]]] = {} def _fallback_semaphore(loop: Any, name: str, capacity: int) -> asyncio.Semaphore: """Per-loop semaphore for loops that reject attribute assignment. The entry keeps a strong reference to the loop on purpose. Deriving the loop from the semaphore does not work: ``Semaphore.acquire()`` only binds ``_loop`` when it actually has to wait, so an uncontended semaphore never learns which loop it belongs to, leaving nothing to test ``is_closed()`` on and no way to detect ``id()`` reuse. Closed loops are swept on the next call, so a closed loop outlives itself by at most one lookup. """ for stale_key, (stale_loop, _) in list(_LOOP_SEMAPHORE_FALLBACK.items()): if stale_loop.is_closed(): _LOOP_SEMAPHORE_FALLBACK.pop(stale_key, None) key = id(loop) entry = _LOOP_SEMAPHORE_FALLBACK.get(key) if entry is None or entry[0] is not loop: entry = (loop, {}) _LOOP_SEMAPHORE_FALLBACK[key] = entry semaphores = entry[1] semaphore = semaphores.get(name) if semaphore is None: semaphore = asyncio.Semaphore(capacity) semaphores[name] = semaphore return semaphore def get_loop_semaphore(name: str, capacity: int) -> asyncio.Semaphore: """Return the running loop's semaphore called ``name``, creating it once. ``capacity`` is only used on creation. It is intentionally a fixed constant rather than a per-``LightRAG`` setting: this helper is module level and is called from places that hold no instance, and several instances sharing a loop would otherwise each build their own semaphore and lose the global ceiling. """ loop = asyncio.get_running_loop() attr = _LOOP_SEMAPHORE_ATTR_PREFIX + name semaphore = getattr(loop, attr, None) if semaphore is not None: return semaphore semaphore = asyncio.Semaphore(capacity) try: setattr(loop, attr, semaphore) except (AttributeError, TypeError): return _fallback_semaphore(loop, name, capacity) return semaphore # --------------------------------------------------------------------------- # Chunking off the event loop # --------------------------------------------------------------------------- # # Chunking is CPU-bound in the document the caller supplied and, for the # recursive strategy, in the separator cascade supplied with it. Run inline in an # ``async def`` — which is what every branch except V did — it holds the only # thread serving HTTP for its whole duration, so one 414 KiB document made an # unrelated ``GET /health`` take 63 seconds (GHSA-26pm-px5v-8c4w). # # One worker. Chunking is serialized by the event loop as things stand — while # one document chunks, no other document's coroutine can run — so a single worker # preserves that concurrency exactly while freeing the loop, rather than # introducing parallelism this change has no reason to introduce. # # Invariants: # * code running in here calls the SYNCHRONOUS tokenizer API. Awaiting an async # helper from a worker thread would park it on the loop while the loop waits # for the thread. # * no nested submissions. One worker plus a held permit means a task that # submits again deadlocks. Where a chunker calls another chunker (V's # post-processing calls R), it calls it DIRECTLY — it is already in the pool. _CHUNKING_EXECUTOR: Optional[ThreadPoolExecutor] = None _CHUNKING_EXECUTOR_GUARD = threading.Lock() def get_chunking_executor() -> ThreadPoolExecutor: """The process-wide single-worker pool used for chunking.""" global _CHUNKING_EXECUTOR if _CHUNKING_EXECUTOR is None: with _CHUNKING_EXECUTOR_GUARD: if _CHUNKING_EXECUTOR is None: _CHUNKING_EXECUTOR = ThreadPoolExecutor( max_workers=1, thread_name_prefix="lightrag-chunking" ) return _CHUNKING_EXECUTOR async def run_in_chunking_executor(fn: Callable[..., Any], *args: Any, **kwargs) -> Any: """Run a synchronous, chunking-bound callable off the event loop. Bounded like any other submission: the submitters include public SDK entry points (``ainsert_custom_kg``, ``ainsert_custom_chunks``) gated by neither ``max_parallel_insert`` nor the pipeline's busy flag, so "the pipeline limits concurrency" is not a ceiling that actually holds here. """ from lightrag.constants import CHUNKING_SUBMIT_LIMIT semaphore = get_loop_semaphore("chunking", CHUNKING_SUBMIT_LIMIT) return await bounded_submit( get_chunking_executor(), semaphore, partial(fn, *args, **kwargs) ) # --------------------------------------------------------------------------- # File-backend persistence off the event loop # --------------------------------------------------------------------------- # # The file backends rewrite their whole file on every commit, synchronously, on # the event loop: ``write_json`` for the JSON KV / doc-status stores, # ``nx.write_graphml`` for NetworkX, ``NanoVectorDB.save`` for nano. A single # ``_insert_done`` flushes every storage, and a document purge issues several of # them, so on a large corpus the loop stops serving HTTP for seconds at a time — # the same failure mode as GHSA-26pm-px5v-8c4w, in a different place. # # One worker. Those commits are ALREADY serialized today: ``_flush_storages`` # gathers them, but each synchronous write runs to completion without yielding, # so they happen one after another anyway. A single worker preserves that # concurrency exactly while freeing the loop. More workers would only make the # pure-Python encoders (``json.dump`` with ``indent``, GraphML's per-element # loop) contend for the GIL with each other AND with the loop — turning "the # loop is blocked for N seconds" into "the loop is starved for N seconds", # which is worse. # # Not the default executor: that one is shared with # ``UnifiedLock._acquire_mp_lock_in_executor``, password hashing and ``stat`` # calls, and parking it behind a multi-second file write would delay work that # has to run promptly. # # Invariants for anything submitted here: # * no nested submission to this pool — one worker plus a held permit # deadlocks; # * no calling back into the storage layer (``_get_client`` / ``_get_graph``): # they take ``NamespaceLock``, which the caller already holds and which is # not reentrant; # * therefore the pool holds no locks, so it cannot take part in a cycle. _STORAGE_IO_EXECUTOR: Optional[ThreadPoolExecutor] = None _STORAGE_IO_EXECUTOR_GUARD = threading.Lock() def get_storage_io_executor() -> ThreadPoolExecutor: """The process-wide single-worker pool used for file-backend persistence.""" global _STORAGE_IO_EXECUTOR if _STORAGE_IO_EXECUTOR is None: with _STORAGE_IO_EXECUTOR_GUARD: if _STORAGE_IO_EXECUTOR is None: _STORAGE_IO_EXECUTOR = ThreadPoolExecutor( max_workers=1, thread_name_prefix="lightrag-storage-io" ) return _STORAGE_IO_EXECUTOR async def run_in_storage_io(fn: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: """Run a synchronous storage-persistence callable off the event loop. Uncancellable once submitted, unlike :func:`bounded_submit`. Callers hold their storage lock across this call and the worker may be halfway through mutating shared state — ``NanoVectorDBStorage`` points its client's ``storage_file`` at the temporary sibling for the duration of the save. Returning early would release the lock with that state still swapped, a window the synchronous write being replaced here did not have, because it could not be cancelled at all. The wait for a submission permit stays cancellable: at that point nothing has been submitted, so a cancelled caller leaves no half-written file behind and releases its lock immediately. """ from lightrag.constants import STORAGE_IO_SUBMIT_LIMIT semaphore = get_loop_semaphore("storage_io", STORAGE_IO_SUBMIT_LIMIT) return await _bounded_submit_impl( get_storage_io_executor(), semaphore, fn, args, kwargs, wait_for_completion=True, ) def log_without_raising(emit: Callable[[str], Any], message: str) -> None: """Emit a log line that must never propagate a sink failure to its caller. For code past a point of no return, where the caller's status no longer has the right to change. The file backends' ``drop`` commit hooks are the motivating case: once the file is gone the destruction happened, and a broken handler, formatter or output target must not turn it into ``{"status": "error"}`` — the exact misreport those hooks exist to prevent. Their outer ``except`` needs it for the mirror reason: a sink failure there must not swallow the ``"error"`` a real destructive failure has to report. Pass the log method itself (``log_without_raising(logger.info, msg)``) rather than wrapping the call in a lambda: an ``except X as e`` name is unbound at the end of its block, so a closure over it reads as undefined to static analysis even though it resolves at call time. The failure is deliberately **swallowed, not re-reported**: any report would travel the same broken sink. This is the narrow exception to *fail loud* — it applies to logging alone, never to the work being logged about. """ try: emit(message) except Exception: pass async def commit_in_storage_io( fn: Callable[[], Any], on_committed: Callable[[], Awaitable[Any]], ) -> Any: """``run_in_storage_io`` for a write whose bookkeeping must not be orphaned. Backends that publish a file and then tell the other processes to reload it (``set_all_update_flags``) have two steps that must not come apart. A cancel landing between them leaves the file published while every other worker keeps reading the previous snapshot, until some later commit happens to fix it. The synchronous writes this replaces had the same gap — with the loop frozen for the whole write, cancellation could only be delivered at the first suspension point after it, which in multiprocess mode is the lock acquire inside ``set_all_update_flags``, before any flag is flipped. Freeing the event loop makes the window far easier to hit, so it is closed here rather than left to grow. ``fn`` takes no arguments (wrap it in a lambda or ``partial``), which keeps this signature clear of the keyword-forwarding hazard ``run_in_storage_io`` has to live with. ``on_committed`` runs ONLY if ``fn`` succeeded — see ``_bounded_submit_impl`` for why running it otherwise would lose data. Raises: CommitBookkeepingError: ``fn`` succeeded and ``on_committed`` did not. The write is DURABLE; only its publication failed. Every call site must handle this separately from a write failure and report the commit as landed — see the exception's own docstring for the contract, and ``NetworkXStorage.index_done_callback`` for the reference handler. Exception: whatever ``fn`` raised. Nothing was persisted. """ from lightrag.constants import STORAGE_IO_SUBMIT_LIMIT semaphore = get_loop_semaphore("storage_io", STORAGE_IO_SUBMIT_LIMIT) return await _bounded_submit_impl( get_storage_io_executor(), semaphore, fn, (), {}, wait_for_completion=True, on_committed=on_committed, ) class Tokenizer: """ A wrapper around a tokenizer to provide a consistent interface for encoding and decoding. Thread-safety contract ---------------------- ``encode`` and ``decode`` MAY be called concurrently from several OS threads: token counting is CPU-bound and runs in worker threads so it does not block the asyncio event loop. **An injected tokenizer is responsible for its own thread safety** — through immutable state, an internal lock, thread-local state, or whatever its implementation requires. LightRAG does not serialize it, and adding a lock here would be actively harmful: the event loop would then wait on a worker thread holding it, recreating the very freeze that moving this work off the loop exists to remove. The built-in :class:`TiktokenTokenizer` satisfies this: ``tiktoken``'s own ``encode_batch`` / ``decode_batch`` fan ``Encoding.encode`` / ``.decode`` of a single instance across a ``ThreadPoolExecutor``, so concurrent use is a supported capability of the library rather than an assumption made here. The dependency floor in ``pyproject.toml`` is pinned to the oldest release for which that holds. Note that sharing is the norm, not the exception: ``tiktoken`` caches encodings in a process-wide registry, and ``Encoding.__setstate__`` rebinds a copy's ``__dict__`` to the registered instance's — so every ``TiktokenTokenizer`` in the process, including ones produced by ``copy.deepcopy``, funnels into a single ``CoreBPE``. An injected tokenizer that is not thread-safe cannot be made safe by copying it. Deep-copy requirement --------------------- An injected tokenizer must additionally survive ``copy.deepcopy``: ``LightRAG`` is a dataclass and ``_build_global_config`` runs ``dataclasses.asdict`` over it, which deep-copies non-dataclass fields. An implementation that achieves thread safety with an internal ``threading.Lock`` would otherwise fail at construction with ``TypeError: cannot pickle '_thread.lock' object``. Such an implementation should define ``__deepcopy__`` returning ``self`` — correct precisely because being thread-safe is what makes it shareable. ``_build_global_config`` then restores this object over the copy ``asdict`` made, so every consumer reading ``global_config["tokenizer"]`` holds the same instance as ``LightRAG.tokenizer``. That is an identity guarantee, not a saving: ``asdict`` copies the field before the restore can intervene, which is exactly why the deep-copy requirement above still applies. """ def __init__(self, model_name: str, tokenizer: TokenizerInterface): """ Initializes the Tokenizer with a tokenizer model name and a tokenizer instance. Args: model_name: The associated model name for the tokenizer. tokenizer: An instance of a class implementing the TokenizerInterface, which must be safe to call concurrently from multiple threads. """ self.model_name: str = model_name self.tokenizer: TokenizerInterface = tokenizer def encode(self, content: str) -> List[int]: """ Encodes a string into a list of tokens using the underlying tokenizer. May be called concurrently from multiple threads; see the class docstring. Args: content: The string to encode. Returns: A list of integer tokens. """ try: return self.tokenizer.encode(content) except ValueError as e: # tiktoken (and some other tokenizers) raise ValueError when the # content contains literal special-token strings such as # "<|endoftext|>", because by default disallowed_special is the # full set of special tokens. This crashes document indexing on # any user content that happens to contain those strings — common # in documentation, notes, or model output captured in source # corpora. Retry with disallowed_special=() so the tokens are # encoded as ordinary text. Tokenizers that don't accept the # kwarg fall through and re-raise the original error. if "special token" not in str(e): raise try: return self.tokenizer.encode(content, disallowed_special=()) except TypeError: raise e def decode(self, tokens: List[int]) -> str: """ Decodes a list of tokens into a string using the underlying tokenizer. May be called concurrently from multiple threads; see the class docstring. Args: tokens: A list of integer tokens to decode. Returns: The decoded string. """ return self.tokenizer.decode(tokens) # ----------------------------------------------------------------------- # Safe split/truncate contract # ----------------------------------------------------------------------- # # These default implementations rely ONLY on ``encode()`` — never on # ``decode()`` — because ``decode(tokens[:k])`` is not guaranteed to be a # character-exact prefix of the original string for an arbitrary # third-party tokenizer (only tiktoken's ``decode_with_offsets`` gives a # reliable token-to-character mapping; see ``TiktokenTokenizer`` below for # the optimized override). Every candidate here is instead a literal # character-offset slice of the input, so there is no alignment risk, at # the cost of a handful of extra ``encode()`` calls per span compared to # the tiktoken fast path. # # BPE token count is not monotonic in character-prefix length, so no # candidate is ever trusted without encoding it. The search brackets the # answer between a length verified to fit and one verified to overflow -- # reached by doubling or halving from the ratio estimate -- and then # bisects that bracket. The bracket only ever narrows, which rules out # oscillation, and both phases are logarithmic in the remaining character # count, so the number of re-encodes stays bounded however wrong the # initial char/token ratio estimate is for a given region of text. # # Bracketing is an assumption about the sampled lengths, not a proof about # every length between them: the span returned is always verified to fit, # but under a sufficiently non-monotonic tokenizer a longer one may exist # that the bisection never sampled. # # No cross-call state: nothing here is written back onto ``self``. An # injected tokenizer must remain safe under concurrent calls from # multiple threads and under ``copy.deepcopy`` (see the class docstring), # and any mutable "observed retreat" state would violate both. def _bounded_prefix_span( self, content: str, start: int, max_tokens: int, chars_per_token: float ) -> TokenSpan: """Find a safe ``TokenSpan`` starting at character offset ``start``. ``chars_per_token`` is a rough estimate (chars per token, from a prior whole-content encode) used only to pick a starting candidate length; the result is always independently verified by encoding the exact candidate substring. That estimate is a whole-content average, so wherever token density varies it is wrong in one of two directions at ``start``: a region sparser than the average makes the seed too short, a denser one makes it too long. Accepting the first candidate that happens to fit leaves most of the budget unspent in the first case; stepping down from an over-long seed by exponentially growing amounts overshoots the whole remainder in the second, which is why that step-down had to clamp at one character. Both are avoided by bracketing the answer between a length verified to fit and one verified to overflow, then bisecting. """ remaining_len = len(content) - start seed = min(remaining_len, max(1, int(max_tokens * chars_per_token))) # ``good`` is the longest length verified to fit, ``bad`` the shortest # verified not to; ``bad`` starts one past the end as a sentinel for # "nothing has overflowed yet". good, good_count = 0, 0 bad = remaining_len + 1 count = len(self.encode(content[start : start + seed])) if count <= max_tokens: good, good_count = seed, count if seed == remaining_len: return TokenSpan(start, start + good, good_count) candidate = seed while True: # double until a candidate overflows candidate = min(remaining_len, candidate * 2) probe_count = len(self.encode(content[start : start + candidate])) if probe_count > max_tokens: bad = candidate break good, good_count = candidate, probe_count if candidate == remaining_len: return TokenSpan(start, start + good, good_count) else: bad = seed candidate = seed while candidate > 1: # halve until a candidate fits candidate //= 2 probe_count = len(self.encode(content[start : start + candidate])) if probe_count <= max_tokens: good, good_count = candidate, probe_count break bad = candidate while bad - good > 1: # bisect the bracket midpoint = (good + bad) // 2 probe_count = len(self.encode(content[start : start + midpoint])) if probe_count <= max_tokens: good, good_count = midpoint, probe_count else: bad = midpoint if good > 0: return TokenSpan(start, start + good, good_count) first_cp = content[start : start + 1] first_cp_tokens = len(self.encode(first_cp)) if first_cp_tokens > max_tokens: raise TokenBudgetError( max_tokens, first_cp_tokens, content[start : start + 80] ) return TokenSpan(start, start + 1, first_cp_tokens) def truncate_by_token_limit(self, content: str, max_tokens: int) -> TokenSpan: """Return a verified safe prefix of ``content`` that fits ``max_tokens``. The result always starts at character offset 0. If the whole string already fits, the returned span covers it entirely. Otherwise the bracketed search aims to use the available budget efficiently, but a non-monotonic tokenizer may have a longer fitting prefix that was not sampled. Raises ``ValueError`` for a non-positive budget, and ``TokenBudgetError`` if not even the first complete Unicode code point fits. """ if max_tokens <= 0: raise ValueError(f"max_tokens must be positive, got {max_tokens}") if not content: return TokenSpan(0, 0, 0) full_tokens = len(self.encode(content)) if full_tokens <= max_tokens: return TokenSpan(0, len(content), full_tokens) chars_per_token = len(content) / full_tokens return self._bounded_prefix_span(content, 0, max_tokens, chars_per_token) def split_by_token_limit( self, content: str, max_tokens: int, overlap_tokens: int = 0 ) -> List[TokenSpan]: """Split ``content`` into safe, contiguous, optionally-overlapping spans. Covers the whole string with no gaps and real forward progress: every non-final span strictly extends the covered range (see the module contract notes above the class). ``overlap_tokens`` is a best-effort target, not a guarantee of the mathematically closest boundary — BPE token count is not monotonic in character length, so this generic implementation only guarantees a safe, progress-making boundary near the target, retreating the target itself (by the same exponential steps) whenever the estimated overlap would fail to make progress. """ if max_tokens <= 0: raise ValueError(f"max_tokens must be positive, got {max_tokens}") if overlap_tokens < 0: raise ValueError( f"overlap_tokens must be non-negative, got {overlap_tokens}" ) if not content: return [] total_tokens = len(self.encode(content)) if total_tokens <= max_tokens: return [TokenSpan(0, len(content), total_tokens)] # One global char/token ratio estimate, reused as a local performance # hint for every window and overlap guess in this call only — never # persisted on `self` (see the note above). chars_per_token = len(content) / total_tokens spans: List[TokenSpan] = [] covered_end = 0 total_len = len(content) while covered_end < total_len: if covered_end > 0: # Clamp to what the previous window can actually lend before # even trying: an overlap target that dwarfs the previous # window's own token count would otherwise retreat from that # huge starting point step by step, wasting retries on values # that could never have worked. target_overlap = min(overlap_tokens, max(0, spans[-1].token_count - 1)) else: target_overlap = 0 step = 1 span: Optional[TokenSpan] = None while True: overlap_chars = ( int(round(target_overlap * chars_per_token)) if target_overlap > 0 else 0 ) start = max(0, covered_end - overlap_chars) try: candidate_span = self._bounded_prefix_span( content, start, max_tokens, chars_per_token ) except TokenBudgetError: if target_overlap == 0: raise candidate_span = None if candidate_span is not None and ( candidate_span.end > covered_end or target_overlap == 0 ): span = candidate_span break target_overlap = max(0, target_overlap - step) step *= 2 spans.append(span) covered_end = span.end return spans class TiktokenTokenizer(Tokenizer): """ A Tokenizer implementation using the tiktoken library. """ def __init__(self, model_name: str = "gpt-4o-mini"): """ Initializes the TiktokenTokenizer with a specified model name. Args: model_name: The model name for the tiktoken tokenizer to use. Defaults to "gpt-4o-mini". Raises: ImportError: If tiktoken is not installed. ValueError: If the model_name is invalid. """ try: import tiktoken except ImportError: raise ImportError( "tiktoken is not installed. Please install it with `pip install tiktoken` or pass a custom `tokenizer`." ) try: tokenizer = tiktoken.encoding_for_model(model_name) super().__init__(model_name=model_name, tokenizer=tokenizer) except KeyError: raise ValueError(f"Invalid model_name: {model_name}.") # ----------------------------------------------------------------------- # Optimized split/truncate: decode_with_offsets fast path # ----------------------------------------------------------------------- # # `tiktoken.Encoding.decode_with_offsets` (available since the repo's # declared `tiktoken>=0.7.0` floor) maps each token index to the character # offset where it starts, so a candidate "keep the first k tokens" prefix # can be located in O(1) instead of guessing a char/token ratio. The # candidate substring is still always independently re-encoded via # `self.encode` (never `self.tokenizer.encode` directly, to preserve the # base class's disallowed_special=() fallback) before being trusted, # because BPE re-tokenization of a truncated substring is not guaranteed # to reproduce the same token count. # # Retreat here happens in TOKEN space (1, 2, 4, 8, ... tokens) rather than # character space, which is both more precise and, since consecutive token # counts can map to the same character offset when a multi-byte UTF-8 # character's bytes are split across tokens, deduplicated so the same # substring is never re-encoded twice in a row. def _prefix_span_via_boundary( self, content: str, start_char: int, start_token: int, max_tokens: int, total_tokens: int, boundary: Callable[[int], int], ) -> tuple[TokenSpan, int]: """Find a safe span starting at ``start_char``/``start_token``. ``boundary(k)`` returns the character offset where absolute token index ``k`` starts (``boundary(total_tokens)`` is ``len(content)``). Returns ``(span, end_token)`` — the absolute token index the span's ``end`` corresponds to, so callers can resume from it without re-deriving it via a char-offset search. """ remaining_tokens = total_tokens - start_token k = min(remaining_tokens, max_tokens) step = 1 last_end: Optional[int] = None while True: end = boundary(start_token + k) if end != last_end: # A token boundary can coincide with start_char (a # continuation-byte-only token reports no character advance — # see decode_with_offsets' docstring); an empty candidate # would trivially satisfy count <= max_tokens without # covering anything, so it must never be accepted here. if end > start_char: candidate = content[start_char:end] count = len(self.encode(candidate)) if count <= max_tokens: return TokenSpan(start_char, end, count), start_token + k last_end = end if k <= 1: break k = max(1, k - step) step *= 2 # Floor: fall back to a direct check of the first complete Unicode # code point, bypassing token-boundary ambiguity entirely (a single # code point's bytes can themselves be split across tokens). end_char = start_char + 1 first_cp = content[start_char:end_char] first_cp_tokens = len(self.encode(first_cp)) if first_cp_tokens > max_tokens: raise TokenBudgetError( max_tokens, first_cp_tokens, content[start_char : start_char + 80] ) end_token = start_token while end_token < total_tokens and boundary(end_token) < end_char: end_token += 1 return TokenSpan(start_char, end_char, first_cp_tokens), end_token def truncate_by_token_limit(self, content: str, max_tokens: int) -> TokenSpan: if max_tokens <= 0: raise ValueError(f"max_tokens must be positive, got {max_tokens}") if not content: return TokenSpan(0, 0, 0) tokens = self.encode(content) total_tokens = len(tokens) if total_tokens <= max_tokens: return TokenSpan(0, len(content), total_tokens) try: decoded_text, offsets = self.tokenizer.decode_with_offsets(tokens) except Exception: return super().truncate_by_token_limit(content, max_tokens) if decoded_text != content: # Lossy roundtrip (shouldn't normally happen once special tokens # are handled via self.encode's fallback) — fall back to the # generic, alignment-safe implementation rather than risk an # off-by-however-many char offset. return super().truncate_by_token_limit(content, max_tokens) def boundary(k: int) -> int: return offsets[k] if k < total_tokens else len(content) span, _ = self._prefix_span_via_boundary( content, 0, 0, max_tokens, total_tokens, boundary ) return span def split_by_token_limit( self, content: str, max_tokens: int, overlap_tokens: int = 0 ) -> List[TokenSpan]: if max_tokens <= 0: raise ValueError(f"max_tokens must be positive, got {max_tokens}") if overlap_tokens < 0: raise ValueError( f"overlap_tokens must be non-negative, got {overlap_tokens}" ) if not content: return [] tokens = self.encode(content) total_tokens = len(tokens) if total_tokens <= max_tokens: return [TokenSpan(0, len(content), total_tokens)] try: decoded_text, offsets = self.tokenizer.decode_with_offsets(tokens) except Exception: return super().split_by_token_limit(content, max_tokens, overlap_tokens) if decoded_text != content: return super().split_by_token_limit(content, max_tokens, overlap_tokens) def boundary(k: int) -> int: return offsets[k] if k < total_tokens else len(content) spans: List[TokenSpan] = [] covered_end_char = 0 covered_end_token = 0 total_len = len(content) while covered_end_char < total_len: if covered_end_token > 0: # Clamp before trying: an overlap target that dwarfs the # previous window's own token count would otherwise retreat # from that huge starting point step by step, wasting # retries on values that could never have worked. target_overlap = min(overlap_tokens, max(0, spans[-1].token_count - 1)) else: target_overlap = 0 step = 1 accepted: Optional[tuple[TokenSpan, int]] = None while True: start_token = max(0, covered_end_token - target_overlap) start_char = boundary(start_token) try: candidate = self._prefix_span_via_boundary( content, start_char, start_token, max_tokens, total_tokens, boundary, ) except TokenBudgetError: if target_overlap == 0: raise candidate = None if candidate is not None and ( candidate[0].end > covered_end_char or target_overlap == 0 ): accepted = candidate break target_overlap = max(0, target_overlap - step) step *= 2 span, covered_end_token = accepted spans.append(span) covered_end_char = span.end return spans def pack_user_ass_to_openai_messages(*args: str): roles = ["user", "assistant"] return [ {"role": roles[i % 2], "content": content} for i, content in enumerate(args) ] def split_string_by_multi_markers(content: str, markers: list[str]) -> list[str]: """Split a string by multiple markers""" if not markers: return [content] content = content if content is not None else "" results = re.split("|".join(re.escape(marker) for marker in markers), content) return [r.strip() for r in results if r.strip()] # --------------------------------------------------------------------------- # Token counting off the event loop # --------------------------------------------------------------------------- # # Tokenizing is CPU-bound and roughly linear in the input the caller chose — # about half a second per MiB — so doing it inline in an ``async def`` stops the # process from answering anything at all for the duration, /health included # (GHSA-r8jh-295g-vv42). These helpers move it to a thread. # # One worker, deliberately. Query-side tokenizing is serialized by the event loop # today, so a single worker preserves that concurrency exactly rather than # introducing parallelism this change has no reason to introduce. Thread-safety # is NOT what the single worker is for — that is the injected tokenizer's own # responsibility (see ``Tokenizer``), and it does not depend on how many pools # exist. # # Invariant: code running inside this executor must call the SYNCHRONOUS # tokenizer API. Awaiting these helpers from a worker thread would park that # thread on the loop while the loop waits for the thread. _TOKENIZER_EXECUTOR: Optional[ThreadPoolExecutor] = None _TOKENIZER_EXECUTOR_GUARD = threading.Lock() def get_tokenizer_executor() -> ThreadPoolExecutor: """The process-wide single-worker pool used for token counting.""" global _TOKENIZER_EXECUTOR if _TOKENIZER_EXECUTOR is None: with _TOKENIZER_EXECUTOR_GUARD: if _TOKENIZER_EXECUTOR is None: _TOKENIZER_EXECUTOR = ThreadPoolExecutor( max_workers=1, thread_name_prefix="lightrag-tokenizer" ) return _TOKENIZER_EXECUTOR async def run_in_tokenizer_executor(fn: Callable[..., Any], *args: Any) -> Any: """Run a synchronous, tokenizer-bound callable off the event loop. Callers that already have a whole loop of encodes should hand over the loop, not each encode: submissions must stay O(1) per request or the executor's queue depth scales with the work instead of with the concurrency. """ from lightrag.constants import TOKENIZER_SUBMIT_LIMIT semaphore = get_loop_semaphore("tokenizer", TOKENIZER_SUBMIT_LIMIT) return await bounded_submit(get_tokenizer_executor(), semaphore, fn, *args) async def aencode(tokenizer: Tokenizer, content: str) -> List[int]: """Encode ``content`` without occupying the event loop.""" return await run_in_tokenizer_executor(tokenizer.encode, content) async def acount_tokens(tokenizer: Tokenizer, content: str) -> int: """Token count of ``content``, computed off the event loop.""" return await run_in_tokenizer_executor(_count_tokens_sync, tokenizer, content) def _count_tokens_sync(tokenizer: Tokenizer, content: str) -> int: return len(tokenizer.encode(content)) def _rendered_prefix_item_count( rendered: list[str], separator: str, safe_end: int ) -> int: """Largest ``k`` such that ``separator.join(rendered[:k])`` fits in ``safe_end`` chars.""" cumulative = 0 sep_len = len(separator) for i, text in enumerate(rendered): cumulative += (sep_len if i > 0 else 0) + len(text) if cumulative > safe_end: return i return len(rendered) def truncate_list_by_token_size( list_data: list[Any], key: Callable[[Any], str], separator: str, max_token_size: int, tokenizer: Tokenizer, ) -> list[Any]: """Truncate a list of data by token size, keeping only whole items. Counts the real serialized text — every item's ``key(item)`` joined by ``separator`` — so the separator's own tokens are part of the budget (the previous per-item-only count silently missed them; see #3559). Never partially truncates an item: the result is always "keep the first K complete items, drop the rest", never a half-rendered item. ``key``/``separator`` must match exactly what the caller will actually render downstream — a mismatch (e.g. truncating on a fuller dict than the one that ends up serialized) reintroduces the same class of undercount. """ if max_token_size <= 0 or not list_data: return [] rendered = [key(data) for data in list_data] full_text = separator.join(rendered) try: safe_span = tokenizer.truncate_by_token_limit(full_text, max_token_size) except TokenBudgetError: return [] k = _rendered_prefix_item_count(rendered, separator, safe_span.end) # BPE token count is not monotonic in text length, so the safe prefix # above is not proof that the first k items' OWN serialization (which is # shorter, since it excludes whatever partial item/separator was cut off) # is itself safe — independently re-verify and shrink k if needed. while k > 0: candidate_text = separator.join(rendered[:k]) if len(tokenizer.encode(candidate_text)) <= max_token_size: break k -= 1 return list_data[:k] async def atruncate_list_by_token_size( list_data: list[Any], key: Callable[[Any], str], separator: str, max_token_size: int, tokenizer: Tokenizer, ) -> list[Any]: """Async :func:`truncate_list_by_token_size`. The WHOLE loop is one submission, never one per element. Submitting per element would make the executor's queue depth scale with the list length times the number of in-flight requests, turning a bounded queue into an amplifier — and this loop routinely runs over every retrieved chunk, which is the largest single block of synchronous tokenizing on the query path. """ return await run_in_tokenizer_executor( truncate_list_by_token_size, list_data, key, separator, max_token_size, tokenizer, ) def normalize_string_list(raw_values: Any, context: str = "") -> list[str]: """Return a list of non-empty strings from raw_values. Non-string elements are dropped and logged as warnings. If raw_values is not a list, an empty list is returned. """ if not isinstance(raw_values, list): return [] result = [] for i, value in enumerate(raw_values): if isinstance(value, str) and value: result.append(value) else: logger.warning( "Non-string element dropped from list%s at index %d: %r", f" ({context})" if context else "", i, value, ) return result def split_text_by_token_limit( text: str, tokenizer: Tokenizer, max_tokens: int ) -> list[str]: """Deprecated: use ``tokenizer.split_by_token_limit`` directly instead. Thin compatibility wrapper around the safe ``TokenSpan``-based split contract (zero overlap, no sentence-boundary preservation — overlap is what now carries the "don't lose context at a cut" job the old sentence-first packer used to do; see ``enforce_chunk_token_limit_before_embedding`` for the overlap-bearing caller). Stays permissive on a non-positive ``max_tokens`` (returns ``[]`` rather than raising) to match this function's pre-existing contract for any remaining direct callers. """ if not text or max_tokens <= 0: return [] spans = tokenizer.split_by_token_limit(text, max_tokens, overlap_tokens=0) return [text[span.start : span.end] for span in spans] def _parent_to_source_projection( source_content: str, parent_start: int, parent_end: int, parent_content: str ) -> list[int] | None: """Monotonic char-offset map from ``parent_content`` positions to ``source_content``. Built once per parent chunk and reused for every hard-split child it produces (an ``O(parent length)`` build shared across ``N`` children, instead of redone per child). ``proj[i]`` for ``0 <= i <= len(parent_content)`` is the absolute ``source_content`` offset corresponding to position ``i`` in ``parent_content``. Needed because a parent's own ``content`` is not always a byte-verbatim slice of the document text it was extracted from — e.g. the V strategy's ``SemanticChunker`` rejoins sentences with a single space, which can differ from the original whitespace run by more or fewer characters. Returns ``None`` if the two cannot be reconciled even after removing all whitespace (content diverged for some other reason, e.g. a summarizing rewrite) — the caller must not guess in that case. """ source_slice = source_content[parent_start:parent_end] if source_slice == parent_content: return list(range(parent_start, parent_end + 1)) src_non_ws_positions = [i for i, ch in enumerate(source_slice) if not ch.isspace()] parent_non_ws_positions = [ i for i, ch in enumerate(parent_content) if not ch.isspace() ] if len(src_non_ws_positions) != len(parent_non_ws_positions) or [ source_slice[i] for i in src_non_ws_positions ] != [parent_content[i] for i in parent_non_ws_positions]: return None proj: list[Optional[int]] = [None] * (len(parent_content) + 1) for p_idx, s_idx in zip(parent_non_ws_positions, src_non_ws_positions): proj[p_idx] = parent_start + s_idx # Whitespace runs (and the trailing end position) inherit the offset of # the next known non-whitespace position, so every index resolves to a # definite, still-monotonic offset. next_val = parent_end for i in range(len(proj) - 1, -1, -1): if proj[i] is None: proj[i] = next_val else: next_val = proj[i] return proj # type: ignore[return-value] def _map_child_span( local_start: int, local_end: int, parent_start: int, parent_end: int, projection: list[int] | None, ) -> dict[str, int] | None: """Map a child's ``[local_start, local_end)`` offset within the parent's own ``content`` to an absolute ``{start, end}`` span into the document text. ``projection`` is ``None`` for the direct-arithmetic fast path (parent ``content`` is a verbatim slice of the document, so ``parent_start +`` is exact); otherwise it is the per-parent map from :func:`_parent_to_source_projection`. """ if projection is None: abs_start, abs_end = parent_start + local_start, parent_start + local_end else: abs_start, abs_end = projection[local_start], projection[local_end] if abs_start < parent_start or abs_end > parent_end or abs_end <= abs_start: return None return {"start": abs_start, "end": abs_end} def enforce_chunk_token_limit_before_embedding( chunking_result: list[dict[str, Any]] | tuple[dict[str, Any], ...], tokenizer: Tokenizer, max_tokens: int, overlap_tokens: int = 0, source_content: str | None = None, ) -> list[dict[str, Any]]: """Hard fallback split before embedding while preserving heading hierarchy. Splits any chunk still over ``max_tokens`` via the safe ``Tokenizer.split_by_token_limit`` contract (see ``lightrag.utils.Tokenizer``), carrying ``overlap_tokens`` of context between consecutive pieces. This no longer preserves sentence/paragraph boundaries — overlap is what now carries the "don't lose context at a cut" job that used to be the sentence-first packer's. ``source_content`` is the merged document text the chunker itself received (the same string :mod:`lightrag.sidecar.backfill` reconstructs from ``blocks.jsonl``) — needed to correctly relocate a hard-split child's ``_source_span`` when a parent chunk's ``content`` is not a byte-verbatim slice of it (see :func:`_parent_to_source_projection`). Pass ``None`` only when the caller genuinely has no such text (e.g. ``ainsert_custom_chunks``); a parent that would need the projection in that case just loses its children's ``_source_span`` instead of raising, matching this function's pre-existing behavior for callers with no provenance concept at all. When ``source_content`` IS supplied but a parent's content has diverged from it beyond whitespace, the projection cannot be trusted at all, and this raises ``ChunkBlockMatchError`` rather than silently emitting a wrong span — the same failure sidecar backfill itself would eventually surface, just earlier and with better context. Known limitation -- ``sidecar`` is not re-scoped for pre-sidecar chunks: this function runs BEFORE :func:`lightrag.sidecar.backfill.backfill_chunk_sidecars` in the pipeline (see ``pipeline.py``'s call order around the "Final hard guard before embedding" comment). For the F/R/V strategies that is exactly right: at this point they carry only ``_source_span`` (no ``sidecar`` yet), this function recomputes a correctly-shrunk ``_source_span`` per child, and backfill then derives each child's own ``sidecar`` from that shrunk span afterward. But the P (paragraph_semantic) strategy attaches its ``sidecar`` field during chunking itself, upstream of this function, and does not route through backfill at all. If a P chunk is large enough to still need a hard split here, every resulting child is a shallow copy of the parent (see the loop below) and so inherits the *exact same* ``sidecar`` value unmodified -- all children end up pointing at the whole parent's block range instead of each getting a sidecar scoped to its own, smaller slice. ``_source_span`` itself never reaches storage either way (stripped in ``build_chunks_dict_from_chunking_result``); this note is only about the accuracy of the ``sidecar`` field that IS persisted to ``text_chunks``. Follow-up if this needs fixing: shrink/re-derive ``sidecar`` per child for the P case too, analogous to what backfill already does for F/R/V. """ if max_tokens <= 0: return list(chunking_result) normalized: list[dict[str, Any]] = [] for dp in chunking_result: if not isinstance(dp, dict): continue content = dp.get("content", "") if not isinstance(content, str) or not content.strip(): continue # A single call does both the "is this already within budget" check # AND the split: split_by_token_limit's own fast path returns one # span covering the whole content when it already fits, so there is # no separate pre-check encode here to duplicate the full-content # encode it does internally regardless. TokenBudgetError (content # cannot be safely split at all — not even its first Unicode code # point fits max_tokens) is intentionally NOT caught: swallowing it # here would mean silently re-emitting the original, still-oversized # chunk instead of failing the document — exactly the unsafe # fallback this contract exists to remove. spans = tokenizer.split_by_token_limit( content, max_tokens, overlap_tokens=overlap_tokens ) if len(spans) == 1: ndp = dict(dp) ndp["tokens"] = spans[0].token_count normalized.append(ndp) continue if overlap_tokens > 0 and any( spans[i].start >= spans[i - 1].end for i in range(1, len(spans)) ): logger.warning( "Requested embedding_chunk_overlap_token_size=%d could not be " "honored for at least one window of chunk %r (retreated to 0 " "to preserve forward progress)", overlap_tokens, dp.get("chunk_id", dp.get("chunk_order_index")), ) base_chunk_id = dp.get("chunk_id") parent_span = dp.get("_source_span") # Built once per parent, reused for every child below — never redone # per child (that would be O(children x parent length) instead of # O(parent length)). projection: list[int] | None = None parent_start = parent_end = None if isinstance(parent_span, dict): try: parent_start = int(parent_span["start"]) parent_end = int(parent_span["end"]) except (KeyError, TypeError, ValueError): parent_start = parent_end = None if parent_start is not None and ( parent_start < 0 or parent_end < parent_start ): parent_start = parent_end = None if parent_start is not None and parent_end - parent_start != len(content): if source_content: projection = _parent_to_source_projection( source_content, parent_start, parent_end, content ) if projection is None: raise ChunkBlockMatchError( chunk_order_index=int(dp.get("chunk_order_index", -1)), chunk_preview=content, blocks_path=None, ) else: # No provenance text available: children below fall back # to dropping _source_span rather than emit a naive, # possibly-wrong offset (parent_start is cleared so # _map_child_span is never reached with an untrustworthy # direct-arithmetic assumption). parent_start = parent_end = None total_parts = len(spans) for i, span in enumerate(spans, 1): piece = content[span.start : span.end] new_dp = dict(dp) new_dp["content"] = piece new_dp["tokens"] = span.token_count # Shallow-copy preserves the nested heading dict and sidecar # block from the source chunk; only the payload (content/tokens # /chunk_id) is rewritten per split slice. For a P-strategy # parent that already carries a "sidecar" field, this means # every child gets an unmodified copy of the PARENT's sidecar, # not one re-scoped to its own smaller slice -- see the "Known # limitation" paragraph in this function's docstring. if isinstance(base_chunk_id, str) and base_chunk_id.strip(): new_dp["chunk_id"] = f"{base_chunk_id}-s{i:02d}" child_span = None if parent_start is not None: child_span = _map_child_span( span.start, span.end, parent_start, parent_end, projection ) if child_span is not None: new_dp["_source_span"] = child_span elif "_source_span" in new_dp: new_dp.pop("_source_span", None) new_dp["split_type"] = "hard_fallback" new_dp["split_part"] = i new_dp["split_total"] = total_parts normalized.append(new_dp) # Rebuild order index to keep continuity after splitting. for idx, item in enumerate(normalized): item["chunk_order_index"] = idx return normalized def cosine_similarity(v1, v2): """Calculate cosine similarity between two vectors""" dot_product = np.dot(v1, v2) norm1 = np.linalg.norm(v1) norm2 = np.linalg.norm(v2) denom = norm1 * norm2 # Zero or non-finite vectors are orthogonal to everything in ranking use; avoid NaN/Inf. if denom == 0 or not np.isfinite(denom) or not np.isfinite(dot_product): return 0.0 return float(dot_product / denom) async def handle_cache( hashing_kv, args_hash, prompt, mode="default", cache_type="unknown", ) -> tuple[str, int] | None: """Generic cache handling function with flattened cache keys Returns: tuple[str, int] | None: (content, create_time) if cache hit, None if cache miss """ if hashing_kv is None: return None if mode != "default": # handle cache for all type of query if not hashing_kv.global_config.get("enable_llm_cache"): return None else: # handle cache for entity extraction if not hashing_kv.global_config.get("enable_llm_cache_for_entity_extract"): return None # Use flattened cache key format: {mode}:{cache_type}:{hash} flattened_key = generate_cache_key(mode, cache_type, args_hash) cache_entry = await hashing_kv.get_by_id(flattened_key) if cache_entry: logger.debug(f"Flattened cache hit(key:{flattened_key})") content = cache_entry["return"] timestamp = cache_entry.get("create_time", 0) return content, timestamp logger.debug(f"Cache missed(mode:{mode} type:{cache_type})") return None @dataclass class CacheData: args_hash: str content: str prompt: str mode: str = "default" cache_type: str = "query" chunk_id: str | None = None queryparam: dict | None = None async def save_to_cache(hashing_kv, cache_data: CacheData): """Save data to cache using flattened key structure. Args: hashing_kv: The key-value storage for caching cache_data: The cache data to save """ # Skip if storage is None or content is a streaming response if hashing_kv is None or not cache_data.content: return # If content is a streaming response, don't cache it if hasattr(cache_data.content, "__aiter__"): logger.debug("Streaming response detected, skipping cache") return # Use flattened cache key format: {mode}:{cache_type}:{hash} flattened_key = generate_cache_key( cache_data.mode, cache_data.cache_type, cache_data.args_hash ) # Check if we already have identical content cached existing_cache = await hashing_kv.get_by_id(flattened_key) if existing_cache: existing_content = existing_cache.get("return") if existing_content == cache_data.content: logger.warning( f"Cache duplication detected for {flattened_key}, skipping update" ) return # Create cache entry with flattened structure cache_entry = { "return": cache_data.content, "cache_type": cache_data.cache_type, "chunk_id": cache_data.chunk_id if cache_data.chunk_id is not None else None, "original_prompt": cache_data.prompt, "queryparam": cache_data.queryparam if cache_data.queryparam is not None else None, } logger.info(f" == LLM cache == saving: {flattened_key}") # Save using flattened key await hashing_kv.upsert({flattened_key: cache_entry}) def safe_unicode_decode(content): # Regular expression to find all Unicode escape sequences of the form \uXXXX unicode_escape_pattern = re.compile(r"\\u([0-9a-fA-F]{4})") # Function to replace the Unicode escape with the actual character def replace_unicode_escape(match): # Convert the matched hexadecimal value into the actual Unicode character return chr(int(match.group(1), 16)) # Perform the substitution decoded_content = unicode_escape_pattern.sub( replace_unicode_escape, content.decode("utf-8") ) return decoded_content def exists_func(obj, func_name: str) -> bool: """Check if a function exists in an object or not. :param obj: :param func_name: :return: True / False """ if callable(getattr(obj, func_name, None)): return True else: return False async def _cooperative_yield(iteration: int, every: int = 64) -> None: """Periodically yield control to the event loop during CPU-heavy async loops. Call inside long synchronous-style loops to prevent event loop starvation in single-worker deployments. Yields every `every` iterations. """ if iteration > 0 and iteration % every == 0: await asyncio.sleep(0) async def wait_tasks_with_drain( tasks: list[asyncio.Task], *, context: str = "", task_labels: dict[asyncio.Task, str] | None = None, ) -> list[Any]: """Await *tasks*, guaranteeing none is left running when this returns/raises. Concurrent multi-store writers (entity/relation merge, rebuild) must never leave a sibling task writing in the background after a failure — a failed ``gather``/``wait`` does not by itself imply the other write tasks stopped (issue #3400, "incomplete async failure coordination"). Behavior: - All tasks succeed: returns their results (completion order). - Any task fails: cancels every still-pending sibling, awaits (drains) them ALL so no background write survives, logs every additional exception, then re-raises the FIRST exception observed. - This coroutine itself is cancelled: cancels and drains all tasks before propagating ``CancelledError``. Args: tasks: ``asyncio.Task`` objects (already scheduled). context: Optional short label used in log messages. task_labels: Optional per-task display labels for pending-task logging. """ if not tasks: return [] ctx = f" [{context}]" if context else "" results: list[Any] = [] first_exception: BaseException | None = None try: done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_EXCEPTION) for i, task in enumerate(done, start=1): try: results.append(task.result()) except BaseException as e: if first_exception is None: first_exception = e elif not isinstance(e, asyncio.CancelledError): logger.error(f"Additional task failure{ctx}: {e}") # NOTE: an await point — external cancellation delivered here is # handled by the enclosing except so still-pending siblings never # keep writing in the background. await _cooperative_yield(i, every=32) if pending: if task_labels: pending_labels = [ task_labels.get(task, "<unknown>") for task in pending ] preview = ", ".join(pending_labels[:10]) if len(pending_labels) > 10: preview += f", ... (+{len(pending_labels) - 10} more)" logger.warning(f"Draining pending tasks{ctx}: {preview or '<none>'}") for task in pending: task.cancel() pending_results = await asyncio.gather(*pending, return_exceptions=True) for result in pending_results: if isinstance(result, BaseException): if first_exception is None: first_exception = result elif not isinstance(result, asyncio.CancelledError): logger.error(f"Additional task failure{ctx}: {result}") else: results.append(result) except asyncio.CancelledError: # External cancellation of THIS waiter, delivered at ANY await point # above — asyncio.wait itself, a cooperative yield while collecting # done results (with siblings still pending after FIRST_EXCEPTION), # or the drain gather. Cancel and drain everything before # propagating so no task keeps writing in the background. Per-task # CancelledError stored in a task's own result is caught by the # inner handler and does NOT take this path. for task in tasks: task.cancel() await asyncio.gather(*tasks, return_exceptions=True) raise if first_exception is not None: raise first_exception return results def always_get_an_event_loop() -> asyncio.AbstractEventLoop: """ Ensure that there is always an event loop available. Reuses the loop running on (or installed on) the current thread so that repeated synchronous calls share a single loop; if none exists or it is closed, creates a new one and installs it as the current loop. Returns: asyncio.AbstractEventLoop: The current or newly created event loop. """ # Reuse a loop actively running on this thread. try: return asyncio.get_running_loop() except RuntimeError: pass # Reuse a loop already installed on this thread, but never let # asyncio.get_event_loop() lazily auto-create one — on Python 3.12+ that # emits a DeprecationWarning. Promote that warning to an error so the # "no current loop" case falls through to explicit creation below, while a # genuinely installed (open) loop is still returned and reused. with warnings.catch_warnings(): warnings.simplefilter("error", DeprecationWarning) try: current_loop = asyncio.get_event_loop() if not current_loop.is_closed(): return current_loop except (RuntimeError, DeprecationWarning): pass # No usable loop on this thread — create one and install it. logger.info("Creating a new event loop in main thread.") new_loop = asyncio.new_event_loop() asyncio.set_event_loop(new_loop) return new_loop async def aexport_data( chunk_entity_relation_graph, entities_vdb, relationships_vdb, output_path: str, file_format: str = "csv", include_vector_data: bool = False, ) -> None: """ Asynchronously exports all entities, relations, and relationships to various formats. Args: chunk_entity_relation_graph: Graph storage instance for entities and relations entities_vdb: Vector database storage for entities relationships_vdb: Vector database storage for relationships output_path: The path to the output file (including extension). file_format: Output format - "csv", "excel", "md", "txt". - csv: Comma-separated values file - excel: Microsoft Excel file with multiple sheets - md: Markdown tables - txt: Plain text formatted output include_vector_data: Whether to include data from the vector database. """ # Collect data entities_data = [] relations_data = [] relationships_data = [] # --- Entities --- all_entities = await chunk_entity_relation_graph.get_all_labels() for entity_name in all_entities: # Get entity information from graph node_data = await chunk_entity_relation_graph.get_node(entity_name) source_id = node_data.get("source_id") if node_data else None entity_info = { "graph_data": node_data, "source_id": source_id, } # Optional: Get vector database information if include_vector_data: entity_id = compute_mdhash_id(entity_name, prefix="ent-") vector_data = await entities_vdb.get_by_id(entity_id) entity_info["vector_data"] = vector_data entity_row = { "entity_name": entity_name, "source_id": source_id, "graph_data": str( entity_info["graph_data"] ), # Convert to string to ensure compatibility } if include_vector_data and "vector_data" in entity_info: entity_row["vector_data"] = str(entity_info["vector_data"]) entities_data.append(entity_row) # --- Relations --- for src_entity in all_entities: for tgt_entity in all_entities: if src_entity == tgt_entity: continue edge_exists = await chunk_entity_relation_graph.has_edge( src_entity, tgt_entity ) if edge_exists: # Get edge information from graph edge_data = await chunk_entity_relation_graph.get_edge( src_entity, tgt_entity ) source_id = edge_data.get("source_id") if edge_data else None relation_info = { "graph_data": edge_data, "source_id": source_id, } # Optional: Get vector database information if include_vector_data: vector_data = None for rel_id in make_relation_vdb_ids(src_entity, tgt_entity): vector_data = await relationships_vdb.get_by_id(rel_id) if vector_data is not None: break relation_info["vector_data"] = vector_data relation_row = { "src_entity": src_entity, "tgt_entity": tgt_entity, "source_id": relation_info["source_id"], "graph_data": str(relation_info["graph_data"]), # Convert to string } if include_vector_data and "vector_data" in relation_info: relation_row["vector_data"] = str(relation_info["vector_data"]) relations_data.append(relation_row) # --- Relationships (from VectorDB) --- # ``client_storage`` is a debug/snapshot escape hatch, not part of the # BaseVectorStorage contract: only NanoVectorDBStorage (async property) # and FaissVectorDBStorage (sync property) implement it. Every other # backend (Milvus, Qdrant, Postgres, Redis, MongoDB, OpenSearch) has no # such attribute at all. This section is a supplementary raw-record # dump; the entity/relation data above already carries the authoritative # graph content, so skip it rather than fail the whole export on a # backend that doesn't support it. # # Check the class, not the instance, for attribute presence: # ``hasattr(relationships_vdb, ...)`` would call the getter to answer # the question, and for NanoVectorDB's *async* property that getter # returns a coroutine that ``hasattr`` immediately discards -- # "coroutine was never awaited" plus doing the (wasted) work twice. # Attribute access on the class itself returns the property descriptor # without invoking it. if hasattr(type(relationships_vdb), "client_storage"): # The two implementations disagree on sync vs. async (NanoVectorDB's # is async, Faiss's is a plain sync property), so the value itself # decides whether to await -- a fixed isinstance/backend-name check # would silently break the moment either changes. client_storage = relationships_vdb.client_storage if inspect.isawaitable(client_storage): # Freshness-aware shape (NanoVectorDBStorage): awaiting the # property funnels through the backend's reload-if-stale path, # so the snapshot picks up whatever another process committed. client_storage = await client_storage else: # Synchronous shape (FaissVectorDBStorage), which is documented # as deliberately NOT going through `_get_index`: in a reader # process it can hand back a snapshot older than the latest # committed one until some other call triggers a reload. That # is fine in the single-process case a local file-backed store # usually runs in, and this section is a supplementary raw dump # -- the authoritative entity/relation content above comes from # the graph storage -- so the export proceeds. But it must not # pass silently: exports get used for backups, and a section # that quietly omits recent records is worse than one whose # limits are stated. logger.warning( f"{type(relationships_vdb).__name__}.client_storage is a " "synchronous snapshot that does not check for a newer " "on-disk commit; in a multi-process deployment the raw " "VectorDB relationships section of this export may lag " "records committed by another process." ) for rel in client_storage["data"]: # Faiss materializes the embedding INTO its metadata rows -- at # flush time, and again by reconstruction when loading a # persisted index -- so a raw `str(rel)` would carry the full # vector even for the default vector-free export, inflating the # file by orders of magnitude and emitting embeddings the caller # asked to leave out. Faiss's own read paths filter the key for # exactly this reason. NanoVectorDB is unaffected either way: it # keeps vectors in a separate `matrix`, not in these rows. # # Build a new dict rather than popping: these rows are live # references into the backend's own metadata store. if not include_vector_data: rel = {k: v for k, v in rel.items() if k != "__vector__"} relationships_data.append( { "relationship_id": rel["__id__"], "data": str(rel), # Convert to string for compatibility } ) else: logger.debug( f"{type(relationships_vdb).__name__} does not expose client_storage; " "skipping the raw VectorDB relationships dump in the export." ) # Export based on format if file_format == "csv": # CSV export with open(output_path, "w", newline="", encoding="utf-8") as csvfile: # Entities if entities_data: csvfile.write("# ENTITIES\n") writer = csv.DictWriter(csvfile, fieldnames=entities_data[0].keys()) writer.writeheader() writer.writerows(entities_data) csvfile.write("\n\n") # Relations if relations_data: csvfile.write("# RELATIONS\n") writer = csv.DictWriter(csvfile, fieldnames=relations_data[0].keys()) writer.writeheader() writer.writerows(relations_data) csvfile.write("\n\n") # Relationships if relationships_data: csvfile.write("# RELATIONSHIPS\n") writer = csv.DictWriter( csvfile, fieldnames=relationships_data[0].keys() ) writer.writeheader() writer.writerows(relationships_data) elif file_format == "excel": # Excel export import pandas as pd entities_df = pd.DataFrame(entities_data) if entities_data else pd.DataFrame() relations_df = ( pd.DataFrame(relations_data) if relations_data else pd.DataFrame() ) relationships_df = ( pd.DataFrame(relationships_data) if relationships_data else pd.DataFrame() ) with pd.ExcelWriter(output_path, engine="xlsxwriter") as writer: if not entities_df.empty: entities_df.to_excel(writer, sheet_name="Entities", index=False) if not relations_df.empty: relations_df.to_excel(writer, sheet_name="Relations", index=False) if not relationships_df.empty: relationships_df.to_excel( writer, sheet_name="Relationships", index=False ) elif file_format == "md": # Markdown export with open(output_path, "w", encoding="utf-8") as mdfile: mdfile.write("# LightRAG Data Export\n\n") # Entities mdfile.write("## Entities\n\n") if entities_data: # Write header mdfile.write("| " + " | ".join(entities_data[0].keys()) + " |\n") mdfile.write( "| " + " | ".join(["---"] * len(entities_data[0].keys())) + " |\n" ) # Write rows for entity in entities_data: mdfile.write( "| " + " | ".join(str(v) for v in entity.values()) + " |\n" ) mdfile.write("\n\n") else: mdfile.write("*No entity data available*\n\n") # Relations mdfile.write("## Relations\n\n") if relations_data: # Write header mdfile.write("| " + " | ".join(relations_data[0].keys()) + " |\n") mdfile.write( "| " + " | ".join(["---"] * len(relations_data[0].keys())) + " |\n" ) # Write rows for relation in relations_data: mdfile.write( "| " + " | ".join(str(v) for v in relation.values()) + " |\n" ) mdfile.write("\n\n") else: mdfile.write("*No relation data available*\n\n") # Relationships mdfile.write("## Relationships\n\n") if relationships_data: # Write header mdfile.write("| " + " | ".join(relationships_data[0].keys()) + " |\n") mdfile.write( "| " + " | ".join(["---"] * len(relationships_data[0].keys())) + " |\n" ) # Write rows for relationship in relationships_data: mdfile.write( "| " + " | ".join(str(v) for v in relationship.values()) + " |\n" ) else: mdfile.write("*No relationship data available*\n\n") elif file_format == "txt": # Plain text export with open(output_path, "w", encoding="utf-8") as txtfile: txtfile.write("LIGHTRAG DATA EXPORT\n") txtfile.write("=" * 80 + "\n\n") # Entities txtfile.write("ENTITIES\n") txtfile.write("-" * 80 + "\n") if entities_data: # Create fixed width columns col_widths = { k: max(len(k), max(len(str(e[k])) for e in entities_data)) for k in entities_data[0] } header = " ".join(k.ljust(col_widths[k]) for k in entities_data[0]) txtfile.write(header + "\n") txtfile.write("-" * len(header) + "\n") # Write rows for entity in entities_data: row = " ".join( str(v).ljust(col_widths[k]) for k, v in entity.items() ) txtfile.write(row + "\n") txtfile.write("\n\n") else: txtfile.write("No entity data available\n\n") # Relations txtfile.write("RELATIONS\n") txtfile.write("-" * 80 + "\n") if relations_data: # Create fixed width columns col_widths = { k: max(len(k), max(len(str(r[k])) for r in relations_data)) for k in relations_data[0] } header = " ".join(k.ljust(col_widths[k]) for k in relations_data[0]) txtfile.write(header + "\n") txtfile.write("-" * len(header) + "\n") # Write rows for relation in relations_data: row = " ".join( str(v).ljust(col_widths[k]) for k, v in relation.items() ) txtfile.write(row + "\n") txtfile.write("\n\n") else: txtfile.write("No relation data available\n\n") # Relationships txtfile.write("RELATIONSHIPS\n") txtfile.write("-" * 80 + "\n") if relationships_data: # Create fixed width columns col_widths = { k: max(len(k), max(len(str(r[k])) for r in relationships_data)) for k in relationships_data[0] } header = " ".join( k.ljust(col_widths[k]) for k in relationships_data[0] ) txtfile.write(header + "\n") txtfile.write("-" * len(header) + "\n") # Write rows for relationship in relationships_data: row = " ".join( str(v).ljust(col_widths[k]) for k, v in relationship.items() ) txtfile.write(row + "\n") else: txtfile.write("No relationship data available\n\n") else: raise ValueError( f"Unsupported file format: {file_format}. Choose from: csv, excel, md, txt" ) if file_format is not None: print(f"Data exported to: {output_path} with format: {file_format}") else: print("Data displayed as table format") def export_data( chunk_entity_relation_graph, entities_vdb, relationships_vdb, output_path: str, file_format: str = "csv", include_vector_data: bool = False, ) -> None: """ Synchronously exports all entities, relations, and relationships to various formats. Args: chunk_entity_relation_graph: Graph storage instance for entities and relations entities_vdb: Vector database storage for entities relationships_vdb: Vector database storage for relationships output_path: The path to the output file (including extension). file_format: Output format - "csv", "excel", "md", "txt". - csv: Comma-separated values file - excel: Microsoft Excel file with multiple sheets - md: Markdown tables - txt: Plain text formatted output include_vector_data: Whether to include data from the vector database. """ try: loop = asyncio.get_event_loop() except RuntimeError: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) loop.run_until_complete( aexport_data( chunk_entity_relation_graph, entities_vdb, relationships_vdb, output_path, file_format, include_vector_data, ) ) def lazy_external_import(module_name: str, class_name: str) -> Callable[..., Any]: """Lazily import a class from an external module based on the package of the caller.""" # Get the caller's module and package import inspect caller_frame = inspect.currentframe().f_back module = inspect.getmodule(caller_frame) package = module.__package__ if module else None def import_class(*args: Any, **kwargs: Any): import importlib module = importlib.import_module(module_name, package=package) cls = getattr(module, class_name) return cls(*args, **kwargs) return import_class async def update_chunk_cache_list( chunk_id: str, text_chunks_storage: "BaseKVStorage", cache_keys: list[str], cache_scenario: str = "batch_update", ) -> None: """Update chunk's llm_cache_list with the given cache keys Args: chunk_id: Chunk identifier text_chunks_storage: Text chunks storage instance cache_keys: List of cache keys to add to the list cache_scenario: Description of the cache scenario for logging """ if not cache_keys: return try: chunk_data = await text_chunks_storage.get_by_id(chunk_id) if chunk_data: # Ensure llm_cache_list exists if "llm_cache_list" not in chunk_data: chunk_data["llm_cache_list"] = [] # Add cache keys to the list if not already present existing_keys = set(chunk_data["llm_cache_list"]) new_keys = [key for key in cache_keys if key not in existing_keys] if new_keys: chunk_data["llm_cache_list"].extend(new_keys) # Update the chunk in storage await text_chunks_storage.upsert({chunk_id: chunk_data}) logger.debug( f"Updated chunk {chunk_id} with {len(new_keys)} cache keys ({cache_scenario})" ) except Exception as e: logger.warning( f"Failed to update chunk {chunk_id} with cache references on {cache_scenario}: {e}" ) class TruncatedResponse(str): """An LLM response that was cut off by the model's output token limit. Behaves exactly like ``str`` so every downstream consumer — tolerant JSON parsing, text sanitization, entity/relation extraction — keeps working on the partial content unchanged. The only added meaning is a signal to the LLM cache layer: a truncated response must NOT be persisted, because a cached partial payload (e.g. cut-off extraction JSON) would be replayed on every later run, even when a larger token budget would have produced the complete output. See ``is_truncated_response`` and the cache-write guards in ``use_llm_func_with_cache`` and ``lightrag.operate``. """ __slots__ = () def is_truncated_response(value: Any) -> bool: """Return True if ``value`` is an LLM response flagged as token-limit truncated. Safe for any input: non-string values (e.g. streaming async iterators) and plain ``str`` responses from providers that do not emit the marker return False, so callers degrade to the pre-existing "cache everything" behavior. """ return isinstance(value, TruncatedResponse) def format_response_diagnostics(**fields: Any) -> str: """Render provider response diagnostics as ``key=value`` pairs. ``None`` becomes ``n/a`` so a field the provider did not report is visibly absent rather than looking like a zero. """ return ", ".join( f"{key}={'n/a' if value is None else value}" for key, value in fields.items() ) def empty_length_truncated_hint( budget_hint: str, *, reasoning_consumed_budget: bool = False ) -> str: """Explain an EMPTY response whose finish reason is the output token limit. Shared by every binding so the four providers describe the same failure identically. This is the structurally-broken case, not "ran a bit long": generation stopped before producing a single content token, so there is nothing to salvage and nothing to cache — the caller raises rather than returning "" and letting the document be indexed as an empty graph (issue #3601 gap 4). ``budget_hint`` names the provider's own output-budget knob, since that is the actionable part and only the binding knows it. """ cause = "generation hit the token limit before emitting any content" if reasoning_consumed_budget: cause += " (budget consumed by reasoning)" return f"{cause}; {budget_hint}" # doc_status.metadata key holding the per-document truncation summary. LLM_TRUNCATION_METADATA_KEY = "llm_truncation" # Cap on the number of affected subjects (chunk ids / entity names) echoed into # that metadata entry. The doc_status row is serialized into every documents # listing response, so the summary must stay O(1) in document size; the exact # per-chunk detail lives in the server log, which is unbounded by design. TRUNCATION_METADATA_SAMPLE_LIMIT = 10 class TokenLimitTruncationTally: """Accumulator for token-limit truncation events over one scope. A document whose output budget is too small does not truncate once, it truncates on EVERY chunk: publishing one ``pipeline_status`` line per event would push the rest of the run out of the bounded ``history_messages`` ring and still leave the operator counting lines. So each producer stage keeps a tally, publishes its FIRST event immediately (the condition must not stay hidden until a long run ends) plus ONE aggregated line when it finishes, and folds its counts into the document-scoped tally the pipeline stamps into ``doc_status.metadata`` — the durable, machine-readable record that outlives the status ring and reaches the API. Not thread-safe, and does not need to be: every mutation is a plain, await-free update made from tasks on a single event loop. """ __slots__ = ("_events", "_stages", "_subjects") def __init__(self) -> None: self._events = 0 self._stages: dict[str, int] = {} # Ordered set: preserves first-seen order for the metadata sample while # de-duplicating a subject that truncated at more than one stage. self._subjects: dict[str, None] = {} def __bool__(self) -> bool: return self._events > 0 @property def events(self) -> int: """Total truncated responses, counting a subject once per stage.""" return self._events @property def affected(self) -> int: """Distinct subjects (chunk ids / entity names) with >= 1 truncation.""" return len(self._subjects) def record(self, stage: str, subject: str) -> bool: """Record one truncated response; True when it is this tally's first.""" first = self._events == 0 self._events += 1 self._stages[stage] = self._stages.get(stage, 0) + 1 if subject: self._subjects.setdefault(subject, None) return first def absorb(self, other: "TokenLimitTruncationTally | None") -> None: """Fold a stage-scoped tally into this (document-scoped) one.""" if not other: return self._events += other._events for stage, count in other._stages.items(): self._stages[stage] = self._stages.get(stage, 0) + count for subject in other._subjects: self._subjects.setdefault(subject, None) def stage_breakdown(self) -> str: """Human-readable per-stage counts, e.g. ``initial: 12, gleaning: 3``.""" return ", ".join(f"{stage}: {count}" for stage, count in self._stages.items()) def as_metadata(self) -> dict[str, Any] | None: """Summary payload for ``doc_status.metadata``; None when nothing hit.""" if not self._events: return None subjects = list(self._subjects) payload: dict[str, Any] = { "events": self._events, "affected": len(subjects), "stages": dict(self._stages), "samples": subjects[:TRUNCATION_METADATA_SAMPLE_LIMIT], } omitted = len(subjects) - TRUNCATION_METADATA_SAMPLE_LIMIT if omitted > 0: payload["samples_omitted"] = omitted return payload def as_metadata_extra(self) -> dict[str, Any]: """``metadata_extra`` fragment for a doc_status transition upsert. Empty on a clean run, so the key is simply absent rather than persisting a zeroed record. Because ``LLM_TRUNCATION_METADATA_KEY`` is deliberately NOT in ``_DOC_STATUS_METADATA_CARRY_OVER_KEYS``, that absence also CLEARS a previous attempt's summary instead of resurrecting it — a re-run with a larger token budget must not keep reporting the old truncation. """ payload = self.as_metadata() return {LLM_TRUNCATION_METADATA_KEY: payload} if payload else {} def merge_truncation_metadata( base: dict[str, Any] | None, extra: dict[str, Any] | None ) -> dict[str, Any] | None: """Combine two persisted ``llm_truncation`` payloads into one. Exists for paths that ADD to a surviving record instead of replacing it: the custom-chunk patch (its truncations must not overwrite the base run's record, nor a truncated base be blanked by a clean patch), the same operation's resumed attempts (a failed attempt's partial graph writes stay, so its record must survive a clean resume), and the analyze→process hand-off. The pipeline's whole-document reprocess never needs this — there replace-or-clear is the correct semantics. Merging live tallies would be exact, but only the persisted payloads survive (the base run's tally object is long gone), and those are lossy: ``samples`` is capped at :data:`TRUNCATION_METADATA_SAMPLE_LIMIT`, so subjects beyond the cap cannot be de-duplicated. ``events`` and ``stages`` are exact sums regardless. ``affected`` is exact whenever both sample lists are complete (no ``samples_omitted``); otherwise it deduplicates what the samples do show and over-counts a subject that truncated on both sides but is visible in neither. Base-vs-patch merges rarely collide (different chunk id schemes; only repeat ``summary`` subjects overlap), but attempt-over-attempt merges re-run the SAME chunk ids, so collision is the normal case there — still exact up to the sample cap, over-counted past it. """ if not base: return dict(extra) if extra else None if not extra: return dict(base) base_samples = [s for s in (base.get("samples") or []) if isinstance(s, str)] extra_samples = [s for s in (extra.get("samples") or []) if isinstance(s, str)] # Ordered union, base first — mirrors the tally's first-seen ordering. union_samples = list(dict.fromkeys(base_samples + extra_samples)) both_complete = not base.get("samples_omitted") and not extra.get("samples_omitted") if both_complete: affected = len(union_samples) else: overlap = len(set(base_samples) & set(extra_samples)) affected = int(base.get("affected") or 0) + int(extra.get("affected") or 0) affected -= overlap stages: dict[str, int] = dict(base.get("stages") or {}) for stage, count in (extra.get("stages") or {}).items(): stages[stage] = stages.get(stage, 0) + int(count) merged: dict[str, Any] = { "events": int(base.get("events") or 0) + int(extra.get("events") or 0), "affected": affected, "stages": stages, "samples": union_samples[:TRUNCATION_METADATA_SAMPLE_LIMIT], } omitted = affected - len(merged["samples"]) if omitted > 0: merged["samples_omitted"] = omitted return merged def remove_think_tags(text: str) -> str: """Remove <think>...</think> tags and their content from the text. Preserves the :class:`TruncatedResponse` marker so downstream consumers can still distinguish partial model output after sanitization. Handles two cases: 1. Complete <think>...</think> blocks anywhere in the text. 2. Orphaned </think> at the very start (e.g., from streaming that begins mid-think-block), removing everything before and including it. """ was_truncated = is_truncated_response(text) # First, remove orphaned </think> prefix (content before first </think> # when there is no preceding <think> tag) text = re.sub(r"^((?!<think>).)*?</think>", "", text, flags=re.DOTALL) # Then remove all complete <think>...</think> blocks text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL) cleaned = text.strip() return TruncatedResponse(cleaned) if was_truncated else cleaned def _reject_empty_truncated_response( cleaned: str, *, raw_len: int, cache_type: str, chunk_id: str | None ) -> None: """Fail a truncated response that think-tag removal left with nothing. The provider bindings each escalate their own empty + token-limit response, but they cannot see this one: ``<think>reasoning</think>`` with no answer after it is a NON-empty payload, so it passes every binding's check and only becomes visibly empty here. This function is the shared throat that every KG-building LLM call passes through, so the rule lands once for all four providers. Parse-stage callers (``cache_type="smartheading"``) already treat an empty answer as an error — ``_parse_llm_json`` raises ``TitleBlockLLMError`` on it — so this only replaces "answer carries no JSON object" with the actual cause and its remedy. """ if not is_truncated_response(cleaned) or cleaned.strip(): return diagnostics = format_response_diagnostics( chunk_id=chunk_id, # Everything the model produced was reasoning, by construction: the # payload was non-empty before think-tag removal and empty after. reasoning_content_len=raw_len, ) hint = empty_length_truncated_hint( "consider raising the LLM's output token limit or disabling thinking " "mode for this role", reasoning_consumed_budget=True, ) message = ( f"Received empty {cache_type} content after think-tag removal " f"({diagnostics}): {hint}" ) logger.error(message) raise EmptyTruncatedResponseError(message) async def use_llm_func_with_cache( user_prompt: str, use_llm_func: callable, llm_response_cache: "BaseKVStorage | None" = None, system_prompt: str | None = None, max_tokens: int = None, history_messages: list[dict[str, str]] = None, cache_type: str = "extract", chunk_id: str | None = None, cache_keys_collector: list = None, response_format: Any | None = None, entity_extraction: bool = False, llm_cache_identity: Any | None = None, ) -> tuple[str, int]: """Call LLM function with cache support and text sanitization If cache is available and enabled (determined by handle_cache based on mode), retrieve result from cache; otherwise call LLM function and save result to cache. This function applies text sanitization to prevent UTF-8 encoding errors for all LLM providers. Args: user_prompt: Input text to send to LLM use_llm_func: LLM function with higher priority llm_response_cache: Cache storage instance max_tokens: Maximum tokens for generation history_messages: History messages list cache_type: Type of cache chunk_id: Chunk identifier to store in cache system_prompt: Optional system prompt sent alongside the user prompt cache_keys_collector: Optional list to collect cache keys for batch processing response_format: Structured output control forwarded to the LLM provider. Providers translate this to their native structured-output surface (OpenAI response_format, Ollama format, Gemini response_mime_type/schema). ``{"type": "json_object"}`` requests JSON output; typed/schema payloads trigger schema-constrained output where supported; ``None`` leaves output unconstrained. Providers that do not support structured output safely strip this argument. entity_extraction: Deprecated. When True and ``response_format`` is not provided, maps to ``{"type": "json_object"}``. Prefer passing ``response_format`` directly. llm_cache_identity: Non-secret model/provider identity used to partition cache entries across role model, binding, or host changes. Returns: tuple[str, int]: (LLM response text, timestamp) - For cache hits: (content, cache_create_time) - For cache misses: (content, current_timestamp) """ if entity_extraction and response_format is None: warnings.warn( "use_llm_func_with_cache(entity_extraction=True) is deprecated; " "pass response_format={'type': 'json_object'} instead.", DeprecationWarning, stacklevel=2, ) response_format = {"type": "json_object"} _validate_cached_response_format(response_format) # Sanitize input text to prevent UTF-8 encoding errors for all LLM providers safe_user_prompt = sanitize_text_for_encoding(user_prompt) safe_system_prompt = ( sanitize_text_for_encoding(system_prompt) if system_prompt else None ) # Sanitize history messages if provided safe_history_messages = None if history_messages: safe_history_messages = [] for i, msg in enumerate(history_messages): safe_msg = msg.copy() if "content" in safe_msg: safe_msg["content"] = sanitize_text_for_encoding(safe_msg["content"]) safe_history_messages.append(safe_msg) history = json.dumps(safe_history_messages, ensure_ascii=False) else: history = None if llm_response_cache: prompt_parts = [] if safe_user_prompt: prompt_parts.append(safe_user_prompt) if safe_system_prompt: prompt_parts.append(safe_system_prompt) if history: prompt_parts.append(history) _prompt = "\n".join(prompt_parts) response_format_key = _serialize_cache_variant(response_format) llm_identity_key = serialize_llm_cache_identity(llm_cache_identity) arg_hash = compute_args_hash( _prompt, "\n<response_format>\n", response_format_key, "\n<llm_identity>\n", llm_identity_key, ) # Generate cache key for this LLM call cache_key = generate_cache_key("default", cache_type, arg_hash) cached_result = await handle_cache( llm_response_cache, arg_hash, _prompt, "default", cache_type=cache_type, ) if cached_result: content, timestamp = cached_result logger.debug(f"Found cache for {arg_hash}") statistic_data["llm_cache"] += 1 # Add cache key to collector if provided if cache_keys_collector is not None: cache_keys_collector.append(cache_key) return content, timestamp statistic_data["llm_call"] += 1 # Call LLM with sanitized input kwargs = {} if safe_history_messages: kwargs["history_messages"] = safe_history_messages if max_tokens is not None: kwargs["max_tokens"] = max_tokens if response_format is not None: kwargs["response_format"] = response_format res: str = await use_llm_func( safe_user_prompt, system_prompt=safe_system_prompt, **kwargs ) # ``remove_think_tags`` re-wraps its result, so the marker survives # sanitization and both this cache guard and the caller (which reports # the truncation to pipeline status) read the same flag off ``res``. raw_len = len(res) res = remove_think_tags(res) _reject_empty_truncated_response( res, raw_len=raw_len, cache_type=cache_type, chunk_id=chunk_id ) res_truncated = is_truncated_response(res) # Generate timestamp for cache miss (LLM call completion time) current_timestamp = int(time.time()) if llm_response_cache.global_config.get("enable_llm_cache_for_entity_extract"): if res_truncated: # Do not persist truncated extraction output: a cached partial # payload would be replayed on every later run, even when a # larger token budget would have completed the extraction. logger.warning( f"Skipping LLM cache write for truncated {cache_type} response " f"(finish_reason=length, chunk_id={chunk_id})" ) else: await save_to_cache( llm_response_cache, CacheData( args_hash=arg_hash, content=res, prompt=_prompt, cache_type=cache_type, chunk_id=chunk_id, ), ) # Add cache key to collector if provided if cache_keys_collector is not None: cache_keys_collector.append(cache_key) return res, current_timestamp # When cache is disabled, directly call LLM with sanitized input kwargs = {} if safe_history_messages: kwargs["history_messages"] = safe_history_messages if max_tokens is not None: kwargs["max_tokens"] = max_tokens if response_format is not None: kwargs["response_format"] = response_format try: res = await use_llm_func( safe_user_prompt, system_prompt=safe_system_prompt, **kwargs ) except Exception as e: # Add [LLM func] prefix to error message error_msg = f"[LLM func] {str(e)}" # Re-raise with the same exception type but modified message raise type(e)(error_msg) from e # Generate timestamp for non-cached LLM call current_timestamp = int(time.time()) raw_len = len(res) cleaned = remove_think_tags(res) _reject_empty_truncated_response( cleaned, raw_len=raw_len, cache_type=cache_type, chunk_id=chunk_id ) return cleaned, current_timestamp def get_content_summary(content: str, max_length: int = 250) -> str: """Get summary of document content Args: content: Original document content max_length: Maximum length of summary Returns: Truncated content with ellipsis if needed """ content = content.strip() if len(content) <= max_length: return content return content[:max_length] + "..." def sanitize_and_normalize_extracted_text( input_text: str, remove_inner_quotes=False ) -> str: """Sanitize and normalize extracted text Args: input_text: text string to be processed remove_inner_quotes: whether to remove Chinese quotation marks and English quotation marks adjacent to Chinese characters, and to normalize non-breaking spaces. Matching outer quotation marks are removed independently of this option only when the enclosed text contains no corresponding quote characters. Returns: Sanitized and normalized text string """ safe_input_text = sanitize_text_for_encoding(input_text) if safe_input_text: normalized_text = normalize_extracted_info( safe_input_text, remove_inner_quotes=remove_inner_quotes ) return normalized_text return "" def normalize_entity_name(input_text: str) -> str: """Normalize an entity identifier using the extraction naming contract.""" return sanitize_and_normalize_extracted_text(input_text, remove_inner_quotes=True) def normalize_extracted_info(name: str, remove_inner_quotes=False) -> str: """Normalize entity/relation names and description with the following rules: - Clean HTML tags (paragraph and line break tags) - Convert Chinese symbols to English symbols - Remove spaces between Chinese characters - Remove spaces between Chinese characters and English letters/numbers - Preserve spaces within English text and numbers - Replace Chinese parentheses with English parentheses - Replace Chinese dash with English dash - Remove a matching outer English/Chinese quote or book title mark pair only when the enclosed text contains no corresponding mark characters - Filter out short numeric-only text (length < 3 and only digits/dots) - remove_inner_quotes = True - Remove Chinese quotation marks - Remove English quotation marks adjacent to Chinese characters - Convert non-breaking spaces to regular spaces - Convert narrow non-breaking spaces after non-digits to regular spaces Args: name: Entity name to normalize remove_inner_quotes: whether to apply the optional quote and non-breaking-space rules above. Matching outer quotation marks are removed independently of this option only when the enclosed text contains no corresponding quote characters. Returns: Normalized entity name """ # Clean HTML tags - remove paragraph and line break tags name = re.sub(r"</p\s*>|<p\s*>|<p/>", "", name, flags=re.IGNORECASE) name = re.sub(r"</br\s*>|<br\s*>|<br/>", "", name, flags=re.IGNORECASE) # Chinese full-width letters to half-width (A-Z, a-z) name = name.translate( str.maketrans( "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", ) ) # Chinese full-width numbers to half-width name = name.translate(str.maketrans("0123456789", "0123456789")) # Chinese full-width symbols to half-width name = name.replace("-", "-") # Chinese minus name = name.replace("+", "+") # Chinese plus name = name.replace("/", "/") # Chinese slash name = name.replace("*", "*") # Chinese asterisk # Replace Chinese parentheses with English parentheses name = name.replace("(", "(").replace(")", ")") # Replace Chinese dash with English dash (additional patterns) name = name.replace("—", "-").replace("-", "-") # Chinese full-width space to regular space (after other replacements) name = name.replace(" ", " ") # Use regex to remove spaces between Chinese characters # Regex explanation: # (?<=[\u4e00-\u9fa5]): Positive lookbehind for Chinese character # \s+: One or more whitespace characters # (?=[\u4e00-\u9fa5]): Positive lookahead for Chinese character name = re.sub(r"(?<=[\u4e00-\u9fa5])\s+(?=[\u4e00-\u9fa5])", "", name) # Remove spaces between Chinese and English/numbers/symbols name = re.sub( r"(?<=[\u4e00-\u9fa5])\s+(?=[a-zA-Z0-9\(\)\[\]@#$%!&\*\-=+_])", "", name ) name = re.sub( r"(?<=[a-zA-Z0-9\(\)\[\]@#$%!&\*\-=+_])\s+(?=[\u4e00-\u9fa5])", "", name ) # Remove outer quotes if len(name) >= 2: # Handle double quotes if name.startswith('"') and name.endswith('"'): inner_content = name[1:-1] if '"' not in inner_content: # No double quotes inside name = inner_content # Handle single quotes if name.startswith("'") and name.endswith("'"): inner_content = name[1:-1] if "'" not in inner_content: # No single quotes inside name = inner_content # Handle Chinese-style double quotes if name.startswith("“") and name.endswith("”"): inner_content = name[1:-1] if "“" not in inner_content and "”" not in inner_content: name = inner_content if name.startswith("‘") and name.endswith("’"): inner_content = name[1:-1] if "‘" not in inner_content and "’" not in inner_content: name = inner_content # Handle Chinese-style book title mark if name.startswith("《") and name.endswith("》"): inner_content = name[1:-1] if "《" not in inner_content and "》" not in inner_content: name = inner_content if remove_inner_quotes: # Remove Chinese quotes name = name.replace("“", "").replace("”", "").replace("‘", "").replace("’", "") # Remove English queotes in and around chinese name = re.sub(r"['\"]+(?=[\u4e00-\u9fa5])", "", name) name = re.sub(r"(?<=[\u4e00-\u9fa5])['\"]+", "", name) # Convert non-breaking space to regular space name = name.replace("\u00a0", " ") # Convert narrow non-breaking space to regular space when after non-digits name = re.sub(r"(?<=[^\d])\u202F", " ", name) # Remove spaces from the beginning and end of the text name = name.strip() # Filter out pure numeric content with length < 3 if len(name) < 3 and re.match(r"^[0-9]+$", name): return "" def should_filter_by_dots(text): """ Check if the string consists only of dots and digits, with at least one dot Filter cases include: 1.2.3, 12.3, .123, 123., 12.3., .1.23 etc. """ return all(c.isdigit() or c == "." for c in text) and "." in text if len(name) < 6 and should_filter_by_dots(name): # Filter out mixed numeric and dot content with length < 6, requiring at least one dot return "" return name def sanitize_text_for_encoding( text: str, replacement_char: str = "", *, strip: bool = True ) -> str: """Sanitize text to ensure safe UTF-8 encoding by removing or replacing problematic characters. This function handles: - Surrogate characters (the main cause of encoding errors) - Other invalid Unicode sequences - Control characters that might cause issues - Unescape HTML escapes - Remove control characters - Whitespace trimming (see ``strip``) Args: text: Input text to sanitize replacement_char: Character to use for replacing invalid sequences strip: Trim leading/trailing whitespace. Default ``True``, which is what every caller sanitizing a whole payload wants. Pass ``False`` when the text is a FRAGMENT that will sit inside a larger sanitized string: its boundary whitespace is interior to that string and survives there, so trimming the fragment in isolation would produce something the consumer of the larger string never saw. ``kg_extraction_validator`` is the case in point — the chunk sits inside a fenced ``---Input Text---`` section of the prompt. Returns: Sanitized text that can be safely encoded as UTF-8 """ if not text: return text # First, strip whitespace if strip: text = text.strip() # Early return if text is empty after basic cleaning if not text: return text # 1. html.unescape first to catch entities that might become surrogates or control chars text = html.unescape(text) # 2. Use pre-compiled regex to clean surrogates and non-characters in one pass # This replaces the slow manual loop and initial .encode() check text = _SURROGATE_PATTERN.sub(replacement_char, text) # 3. Remove control characters but preserve common whitespace (\t, \n, \r) text = _CONTROL_CHAR_PATTERN_ALL.sub(replacement_char, text) return text.strip() if strip else text def strip_control_characters(text: str, replacement_char: str = "") -> str: """Remove control/separator chars and surrogates while preserving text. Strips the same character classes as :func:`sanitize_text_for_encoding` (surrogates via ``_SURROGATE_PATTERN`` and control chars via ``_CONTROL_CHAR_PATTERN_ALL`` — including the C0 separators ``\\x1c``-``\\x1f`` FS/GS/RS/US, while keeping ``\\t``/``\\n``/``\\r``) but deliberately does *not* ``html.unescape`` or ``.strip()`` the result. This makes it safe for text that carries intentional markup (e.g. sidecar block content with ``<table>``/``<drawing>``/``<equation>`` tags, where unescaping ``&lt;`` would corrupt the markup) or significant leading/ trailing whitespace. For control-char-free input it returns the string unchanged, so it does not perturb existing content hashes or snapshots. """ if not text: return text text = _SURROGATE_PATTERN.sub(replacement_char, text) return _CONTROL_CHAR_PATTERN_ALL.sub(replacement_char, text) # LLMs emitting LaTeX inside JSON strings routinely under-escape backslashes: # "\frac" is *valid* JSON meaning form feed + "rac", so JSON parsers # (including json_repair) silently decode it and the LaTeX command is # destroyed. Form feed (\x0c) and backspace (\x08) followed by a letter have # no legitimate use in LLM-generated prose, so restoring the backslash is # unconditionally safe. The other three decodable escapes (\t, \n, \r) map to # legitimate whitespace and cannot be restored without guessing; they are only # *detected* (see _WS_LATEX_SUSPECT_PATTERN) so real-world frequency can be # observed before deciding on heuristic restoration. _FORMFEED_LATEX_PATTERN = re.compile(r"\x0c(?=[A-Za-z])") _BACKSPACE_LATEX_PATTERN = re.compile(r"\x08(?=[A-Za-z])") # Whitespace + residue spelling that completes a common LaTeX command whose # remainder collides with no English word ("eq"/"o"/"exists" are deliberately # absent: "eq." abbreviations, the word "o"/"exists" would false-positive). _WS_LATEX_SUSPECT_PATTERN = re.compile( r"\t(?=(?:au|heta|imes|ext|ilde|herefore|riangle)\b)" r"|\r(?=(?:ho|ight|angle|ceil)\b)" r"|\n(?=(?:abla|otin)\b)" ) def repair_vlm_json_escape_damage(text: str, *, context: str = "") -> str: """Restore LaTeX backslashes destroyed by JSON escape decoding. Applied to string values parsed out of VLM/LLM JSON responses, where an un-doubled LaTeX command like ``"\\frac"`` arrives as ``\\x0c`` + ``rac``. Only the two zero-risk cases are repaired: - form feed + letter -> ``\\f`` + letter (``\\frac``, ``\\forall``, ...) - backspace + letter -> ``\\b`` + letter (``\\beta``, ``\\bar``, ...) Isolated control characters (not followed by a letter) are left alone for downstream sanitization to drop. Whitespace-class damage (``\\tau`` -> tab + ``au`` etc.) is ambiguous with legitimate whitespace and is only logged at WARNING level, never rewritten. Args: text: Parsed string value to repair. context: Optional label (e.g. ``"table/t1.description"``) included in the detection log line. """ if not text: return text repaired = _FORMFEED_LATEX_PATTERN.sub(r"\\f", text) repaired = _BACKSPACE_LATEX_PATTERN.sub(r"\\b", repaired) if repaired != text: logger.warning( "Repaired LaTeX escape damage (\\f/\\b decoded by JSON parser)%s", f" in {context}" if context else "", ) suspect = _WS_LATEX_SUSPECT_PATTERN.search(repaired) if suspect: snippet = repaired[max(0, suspect.start() - 30) : suspect.start() + 30] logger.warning( "Suspected whitespace-class LaTeX escape damage%s (not auto-repaired): %r", f" in {context}" if context else "", snippet, ) return repaired def repair_vlm_json_escape_damage_nested(obj: Any, *, context: str = "") -> Any: """Apply :func:`repair_vlm_json_escape_damage` to every string inside a parsed JSON structure (dicts / lists nested arbitrarily). Used on the output of ``json_repair.loads`` for LLM responses that may quote LaTeX — multimodal analysis objects and entity-extraction results (``{"entities": [{...}], "relationships": [{...}]}``). Non-string leaves are returned untouched. """ if isinstance(obj, str): return repair_vlm_json_escape_damage(obj, context=context) if isinstance(obj, dict): return { key: repair_vlm_json_escape_damage_nested( value, context=f"{context}.{key}" if context else str(key) ) for key, value in obj.items() } if isinstance(obj, list): return [ repair_vlm_json_escape_damage_nested(item, context=context) for item in obj ] return obj _CODE_FENCE_PATTERN = re.compile( r"^\s*```(?:json|JSON)?\s*\n?(.*?)\n?\s*```\s*$", re.DOTALL ) def strip_markdown_code_fence(text: str) -> str: """Strip a surrounding markdown code fence (```json ... ``` or ``` ... ```). Why: LLM training priors strongly associate "JSON output" with fenced code blocks, so providers routinely wrap responses despite explicit instructions to the contrary. Stripping here avoids relying on ``json_repair`` and the noisy warning it emits. The ``\\n?`` make the interior newlines optional so single-line fences (```` ```json {...}``` ````) are handled too. """ match = _CODE_FENCE_PATTERN.match(text) return match.group(1) if match else text def _first_structural_opener(text: str) -> tuple[str | None, int]: """Return the first ``[`` or ``{`` that sits outside a double-quoted string. Only ``"`` is treated as a string delimiter: prose apostrophes (``Here's``) are far too common to treat ``'`` as a quote, and leading prose that wraps a ``[`` inside single quotes is rare enough to accept as graceful degradation. The result drives the top-level array-vs-object decision — a leading ``[`` means a top-level array (rejected by the caller), a leading ``{`` marks where object recovery starts. """ quote_open = False escaped = False for index, char in enumerate(text): if quote_open: if escaped: escaped = False elif char == "\\": escaped = True elif char == '"': quote_open = False continue if char == '"': quote_open = True elif char in "[{": return char, index return None, -1 def _first_balanced_object_slice(text: str) -> str: """Return the first brace-balanced ``{...}`` slice; ``text`` starts at ``{``. ``"`` always opens a string. ``'`` opens a string only in a value/key position — right after ``{``, ``[``, ``,`` or ``:`` (spaces skipped) — so a bare apostrophe inside an unquoted token (``O'Reilly``, ``it's``) is not mistaken for a string start. Without that guard the scan would run past the object's closing ``}`` and json_repair would fold trailing prose into the last value — a silent, schema-valid but corrupted result. Single quotes are tracked at all because weaker models emit single-quoted JSON strings that can legitimately contain stray braces. Falls back to the whole string when no matching close brace exists so an incomplete object stays repairable. """ depth = 0 quote: str | None = None escaped = False previous_non_space = "" for index, char in enumerate(text): if quote is not None: if escaped: escaped = False elif char == "\\": escaped = True elif char == quote: quote = None continue if char == '"' or (char == "'" and previous_non_space in "{[,:"): quote = char elif char == "{": depth += 1 elif char == "}": depth -= 1 if depth == 0: return text[: index + 1] if not char.isspace(): previous_non_space = char return text def tolerant_load_json_dict(text: str) -> dict[str, Any]: """Recover a single JSON object from noisy LLM/VLM text. Returns the first genuine JSON *object*, or ``{}`` when the text does not yield exactly one object. ``{}`` is the signal callers act on: multimodal analysis retries once, entity extraction falls back to delimiter parsing. Recovered — returns the object (the malformed shapes weak models emit): * a clean object, optionally inside a markdown ``json`` code fence, with or without interior newlines: ``{"a": 1}`` * leading prose before the object — a label, a ``#`` or ``//`` comment lead-in, a URL, or an apostrophe: ``Here is the result: {"a":1}``, ``Result #1: {"a":1}``, ``Source: http://x//y#z {"a":1}``, ``Here's the note: {"a":1}`` * trailing prose after the object, even when it contains braces — the bug this helper exists for, where a greedy ``{...}`` slice over-extends and drops everything: ``{"facts":[...]} trailing {brace}`` * object-level slips ``json_repair`` fixes — trailing comma, single quotes, unquoted keys, truncation: ``{"a":1,}``, ``{'a':1}``, ``{a:1}``, ``{"a":1`` * a genuine (possibly malformed) object followed by a bracket citation: ``{"a":1,} [1]`` Rejected — returns ``{}`` so callers retry / fall back rather than accept a non-object: * any top-level array, so one element is never mistaken for the whole answer — including prose-prefixed, truncated, single-quoted, and commented arrays: ``[{"a":1},{"b":2}]``, ``Here is the result: [{...}]``, ``['note', {...}]``, ``[/* ] */ {...}]`` * a top-level array reached past a broken array shell, an unclosed quote, or pseudo-object prose — never scavenged for an inner object: ``[}{"a":1}]``, ``"oops [}{"a":1}]``, ``{note} [{"a":1}]`` * an object behind bracketed prose — indistinguishable from a real array without heuristics and not seen in practice: ``analysis: [draft] {"a":1}`` Two edge rules worth knowing when extending this: * the top-level shape is decided by the first structural opener outside a double-quoted string; ``json_repair`` runs only on the first balanced ``{...}`` slice, never the whole string (it would scavenge a dict across structural boundaries and defeat array rejection). * a single quote is a string delimiter only in a value/key position, so a bare apostrophe in an unquoted token (``O'Reilly``, ``it's``) does not swallow the object's closing brace. Not handled here: LaTeX-escape damage in string values — callers apply ``repair_vlm_json_escape_damage_nested`` themselves, keeping that choke point out of this helper. """ if not text: return {} candidate = strip_markdown_code_fence(text).strip() # Decide the top-level shape from the raw structure FIRST: the first opener # outside a double-quoted string. json_repair is never run over the whole # candidate — it scavenges a dict across structural boundaries (out of a # broken array shell '[}{...}]', or past pseudo-object prose '{note} [..]'), # which would bypass the top-level-array rejection contract. opener, index = _first_structural_opener(candidate) if opener != "{": # '[' is a top-level array. None means the structure is untrusted — e.g. # an unclosed double quote swallows the openers ('"oops [}{"a":1}]'). # Reject either way so callers retry (multimodal) / fall back (entity). return {} suffix = candidate[index:] # 1) Strict decode of the first object from the opener. A clean object, # optionally followed by trailing prose, is the answer — raw_decode stops # at the end of the first value, so "{...} trailing {brace}" is handled. try: obj, _end = json.JSONDecoder().raw_decode(suffix) if isinstance(obj, dict): return obj except Exception: pass # 2) The opener did not start a clean object: a malformed leading object # (trailing comma / single quotes / unquoted keys), possibly followed by # trailing text such as a "[1]" citation, must still be recovered. Repair # the first balanced slice and accept it only if it is itself an object. # json_repair returns a *list* (not a dict) for pseudo-object prose like # '{note}', so a real payload of the form '{note} [ {...} ]' falls through # to {} here — a trailing top-level array is never scavenged for an # element (repairing only the slice, never the whole candidate, is what # keeps the array out of reach). slice_ = _first_balanced_object_slice(suffix) try: repaired = json_repair.loads(slice_) if isinstance(repaired, dict): return repaired except Exception: pass return {} def check_storage_env_vars(storage_name: str) -> None: """Check if all required environment variables for storage implementation exist Args: storage_name: Storage implementation name Raises: ValueError: If required environment variables are missing """ from lightrag.kg import STORAGE_ENV_REQUIREMENTS required_vars = STORAGE_ENV_REQUIREMENTS.get(storage_name, []) missing_vars = [var for var in required_vars if var not in os.environ] if missing_vars: raise ValueError( f"Storage implementation '{storage_name}' requires the following " f"environment variables: {', '.join(missing_vars)}" ) def pick_by_weighted_polling( entities_or_relations: list[dict], max_related_chunks: int, min_related_chunks: int = 1, ) -> list[str]: """ Linear gradient weighted polling algorithm for text chunk selection. This algorithm ensures that entities/relations with higher importance get more text chunks, forming a linear decreasing allocation pattern. Args: entities_or_relations: List of entities or relations sorted by importance (high to low) max_related_chunks: Expected number of text chunks for the highest importance entity/relation min_related_chunks: Expected number of text chunks for the lowest importance entity/relation Returns: List of selected text chunk IDs """ if not entities_or_relations: return [] if max_related_chunks <= 0: return [] n = len(entities_or_relations) if n == 1: # Only one entity/relation, return its first max_related_chunks text chunks entity_chunks = entities_or_relations[0].get("sorted_chunks", []) return entity_chunks[:max_related_chunks] # Calculate expected text chunk count for each position (linear decrease) expected_counts = [] for i in range(n): # Linear interpolation: from max_related_chunks to min_related_chunks ratio = i / (n - 1) if n > 1 else 0 expected = max_related_chunks - ratio * ( max_related_chunks - min_related_chunks ) expected_counts.append(int(round(expected))) # First round allocation: allocate by expected values selected_chunks = [] used_counts = [] # Track number of chunks used by each entity total_remaining = 0 # Accumulate remaining quotas for i, entity_rel in enumerate(entities_or_relations): entity_chunks = entity_rel.get("sorted_chunks", []) expected = expected_counts[i] # Actual allocatable count actual = min(expected, len(entity_chunks)) selected_chunks.extend(entity_chunks[:actual]) used_counts.append(actual) # Accumulate remaining quota remaining = expected - actual if remaining > 0: total_remaining += remaining # Second round allocation: multi-round scanning to allocate remaining quotas for _ in range(total_remaining): allocated = False # Scan entities one by one, allocate one chunk when finding unused chunks for i, entity_rel in enumerate(entities_or_relations): entity_chunks = entity_rel.get("sorted_chunks", []) # Check if there are still unused chunks if used_counts[i] < len(entity_chunks): # Allocate one chunk selected_chunks.append(entity_chunks[used_counts[i]]) used_counts[i] += 1 allocated = True break # If no chunks were allocated in this round, all entities are exhausted if not allocated: break return selected_chunks async def pick_by_vector_similarity( query: str, text_chunks_storage: "BaseKVStorage", chunks_vdb: "BaseVectorStorage", num_of_chunks: int, entity_info: list[dict[str, Any]], embedding_func: callable, query_embedding=None, ) -> list[str]: """ Vector similarity-based text chunk selection algorithm. This algorithm selects text chunks based on cosine similarity between the query embedding and text chunk embeddings. Args: query: User's original query string text_chunks_storage: Text chunks storage instance chunks_vdb: Vector database storage for chunks num_of_chunks: Number of chunks to select entity_info: List of entity information containing chunk IDs embedding_func: Embedding function to compute query embedding Returns: List of selected text chunk IDs sorted by similarity (highest first) """ logger.debug( f"Vector similarity chunk selection: num_of_chunks={num_of_chunks}, entity_info_count={len(entity_info) if entity_info else 0}" ) if not entity_info or num_of_chunks <= 0: return [] # Preserve first occurrence order for similarity ties and fallback selection. all_chunk_ids = dict.fromkeys( chunk_id for entity in entity_info for chunk_id in entity.get("sorted_chunks", []) ) if not all_chunk_ids: logger.warning( "Vector similarity chunk selection: no chunk IDs found in entity_info" ) return [] logger.debug( f"Vector similarity chunk selection: {len(all_chunk_ids)} unique chunk IDs collected" ) all_chunk_ids = list(all_chunk_ids) try: # Use pre-computed query embedding if provided, otherwise compute it if query_embedding is None: query_embedding = await embedding_func([query], context="query") query_embedding = query_embedding[ 0 ] # Extract first embedding from batch result logger.debug( "Computed query embedding for vector similarity chunk selection" ) else: logger.debug( "Using pre-computed query embedding for vector similarity chunk selection" ) # Get chunk embeddings from vector database chunk_vectors = await chunks_vdb.get_vectors_by_ids(all_chunk_ids) logger.debug( f"Vector similarity chunk selection: {len(chunk_vectors)} chunk vectors Retrieved" ) if not chunk_vectors: logger.warning( "Vector similarity chunk selection: no vectors retrieved from chunks_vdb" ) return [] if len(chunk_vectors) != len(all_chunk_ids): # A referenced chunk with no stored vector signals an inconsistency # between the graph/text stores and the vector store. Keep the # existing safety behavior: abort vector ranking so callers can fall # back to the WEIGHT method, but make the mismatch easier to observe # and repair by logging counts and a bounded sample of missing IDs. expected = len(all_chunk_ids) retrieved = len(chunk_vectors) missing_ids = [ chunk_id for chunk_id in all_chunk_ids if chunk_id not in chunk_vectors ] sample = missing_ids[:5] logger.warning( "Vector similarity chunk selection: data inconsistency detected " "(expected %s, retrieved %s, missing %s). Falling back to WEIGHT method. " "Missing chunk IDs (sample): %s. " "Vector/text storages may be out of sync; consider re-embedding or repairing.", expected, retrieved, expected - retrieved, sample, ) return [] # Calculate cosine similarities similarities = [] valid_vectors = 0 for chunk_id in all_chunk_ids: if chunk_id in chunk_vectors: chunk_embedding = chunk_vectors[chunk_id] try: # Calculate cosine similarity similarity = cosine_similarity(query_embedding, chunk_embedding) similarities.append((chunk_id, similarity)) valid_vectors += 1 except Exception as e: logger.warning( f"Vector similarity chunk selection: failed to calculate similarity for chunk {chunk_id}: {e}" ) else: logger.warning( f"Vector similarity chunk selection: no vector found for chunk {chunk_id}" ) # Sort by similarity (highest first) and select top num_of_chunks similarities.sort(key=lambda x: x[1], reverse=True) selected_chunks = [chunk_id for chunk_id, _ in similarities[:num_of_chunks]] logger.debug( f"Vector similarity chunk selection: {len(selected_chunks)} chunks from {len(all_chunk_ids)} candidates" ) return selected_chunks except Exception as e: logger.error(f"[VECTOR_SIMILARITY] Error in vector similarity sorting: {e}") import traceback logger.error(f"[VECTOR_SIMILARITY] Traceback: {traceback.format_exc()}") # Fallback to simple truncation logger.debug("[VECTOR_SIMILARITY] Falling back to simple truncation") return all_chunk_ids[:num_of_chunks] class TokenTracker: """Track token usage for LLM calls.""" def __init__(self): self.reset() def __enter__(self): self.reset() return self def __exit__(self, exc_type, exc_val, exc_tb): print(self) def reset(self): self.prompt_tokens = 0 self.completion_tokens = 0 self.total_tokens = 0 self.call_count = 0 def add_usage(self, token_counts): """Add token usage from one LLM call. Args: token_counts: A dictionary containing prompt_tokens, completion_tokens, total_tokens """ self.prompt_tokens += token_counts.get("prompt_tokens", 0) self.completion_tokens += token_counts.get("completion_tokens", 0) # If total_tokens is provided, use it directly; otherwise calculate the sum if "total_tokens" in token_counts: self.total_tokens += token_counts["total_tokens"] else: self.total_tokens += token_counts.get( "prompt_tokens", 0 ) + token_counts.get("completion_tokens", 0) self.call_count += 1 def get_usage(self): """Get current usage statistics.""" return { "prompt_tokens": self.prompt_tokens, "completion_tokens": self.completion_tokens, "total_tokens": self.total_tokens, "call_count": self.call_count, } def __str__(self): usage = self.get_usage() return ( f"LLM call count: {usage['call_count']}, " f"Prompt tokens: {usage['prompt_tokens']}, " f"Completion tokens: {usage['completion_tokens']}, " f"Total tokens: {usage['total_tokens']}" ) async def apply_rerank_if_enabled( query: str, retrieved_docs: list[dict], global_config: dict, enable_rerank: bool = True, top_n: int = None, ) -> list[dict]: """ Apply reranking to retrieved documents if rerank is enabled. Args: query: The search query retrieved_docs: List of retrieved documents global_config: Global configuration containing rerank settings enable_rerank: Whether to enable reranking from query parameter top_n: Number of top documents to return after reranking Returns: Reranked documents if rerank is enabled, otherwise original documents """ if not enable_rerank or not retrieved_docs: return retrieved_docs rerank_func = global_config.get("rerank_model_func") if not rerank_func: logger.warning( "Rerank is enabled but no rerank model is configured. Please set up a rerank model or set enable_rerank=False in query parameters." ) return retrieved_docs try: # Extract document content for reranking document_texts = [] for doc in retrieved_docs: # Try multiple possible content fields content = ( doc.get("content") or doc.get("text") or doc.get("chunk_content") or doc.get("document") or str(doc) ) document_texts.append(content) # Call the new rerank function that returns index-based results rerank_results = await rerank_func( query=query, documents=document_texts, top_n=top_n, ) # Process rerank results based on return format if rerank_results and len(rerank_results) > 0: first_object = next( (result for result in rerank_results if isinstance(result, dict)), None, ) # Check if results are in the new index-based format if first_object is not None and "index" in first_object: # New format: [{"index": 0, "relevance_score": 0.85}, ...] reranked_docs = [] for result in rerank_results: normalized_result, _ = normalize_rerank_result( result, len(retrieved_docs) ) if normalized_result is None: continue index = normalized_result["index"] relevance_score = normalized_result["relevance_score"] # Get original document and add rerank score doc = retrieved_docs[index].copy() doc["rerank_score"] = relevance_score reranked_docs.append(doc) if not reranked_docs: logger.warning( "Rerank returned no usable results, using original chunks" ) return retrieved_docs logger.info( f"Successfully reranked: {len(reranked_docs)} chunks from {len(retrieved_docs)} original chunks" ) return reranked_docs else: # Legacy format: assume it's already reranked documents logger.info(f"Using legacy rerank format: {len(rerank_results)} chunks") return rerank_results[:top_n] if top_n else rerank_results else: logger.warning("Rerank returned empty results, using original chunks") return retrieved_docs except Exception as e: logger.error(f"Error during reranking: {e}, using original chunks") return retrieved_docs def normalize_rerank_result( result: Any, max_index: int ) -> tuple[dict[str, int | float] | None, str | None]: """Validate one provider result and return the public rerank shape. Providers, compatible proxies, and custom rerank functions can return mixed result lists. Centralizing validation keeps provider adapters, chunk score aggregation, and the final query boundary consistent. """ if not isinstance(result, dict): return None, "not an object" index = result.get("index") if isinstance(index, bool) or not isinstance(index, int): return None, "invalid index" if not 0 <= index < max_index: return None, "index out of range" score_value = result.get("relevance_score") if isinstance(score_value, bool): return None, "invalid relevance score" try: score = float(score_value) except (TypeError, ValueError, OverflowError): return None, "invalid relevance score" if not math.isfinite(score): return None, "non-finite relevance score" return {"index": index, "relevance_score": score}, None async def process_chunks_unified( query: str, unique_chunks: list[dict], query_param: "QueryParam", global_config: dict, source_type: str = "mixed", chunk_token_limit: int = None, # Add parameter for dynamic token limit progress_callback=None, ) -> list[dict]: """ Unified processing for text chunks: deduplication, chunk_top_k limiting, reranking, and token truncation. Args: query: Search query for reranking unique_chunks: List of deduplicated text chunks to process query_param: Query parameters containing configuration global_config: Global configuration dictionary source_type: Source type for logging ("vector", "entity", "relationship", "mixed") chunk_token_limit: Dynamic token limit for chunks (if None, uses default) Returns: Processed and filtered list of text chunks """ if not unique_chunks: return [] origin_count = len(unique_chunks) # 1. Apply reranking if enabled and query is provided if query_param.enable_rerank and query and unique_chunks: if progress_callback: await progress_callback("reranking") rerank_top_k = query_param.chunk_top_k or len(unique_chunks) unique_chunks = await apply_rerank_if_enabled( query=query, retrieved_docs=unique_chunks, global_config=global_config, enable_rerank=query_param.enable_rerank, top_n=rerank_top_k, ) # 2. Filter by minimum rerank score if reranking is enabled if query_param.enable_rerank and unique_chunks: min_rerank_score = global_config.get("min_rerank_score", 0.5) if min_rerank_score > 0.0: original_count = len(unique_chunks) # Filter chunks with score below threshold filtered_chunks = [] for chunk in unique_chunks: rerank_score = chunk.get( "rerank_score", 1.0 ) # Default to 1.0 if no score if rerank_score >= min_rerank_score: filtered_chunks.append(chunk) unique_chunks = filtered_chunks filtered_count = original_count - len(unique_chunks) if filtered_count > 0: logger.info( f"Rerank filtering: {len(unique_chunks)} chunks remained (min rerank score: {min_rerank_score})" ) if not unique_chunks: return [] # 3. Apply chunk_top_k limiting if specified if query_param.chunk_top_k is not None and query_param.chunk_top_k > 0: if len(unique_chunks) > query_param.chunk_top_k: unique_chunks = unique_chunks[: query_param.chunk_top_k] logger.debug( f"Kept chunk_top-k: {len(unique_chunks)} chunks (deduplicated original: {origin_count})" ) # 4. Token-based final truncation tokenizer = global_config.get("tokenizer") if tokenizer and unique_chunks: # Set default chunk_token_limit if not provided if chunk_token_limit is None: # Get default from query_param or global_config chunk_token_limit = getattr( query_param, "max_total_tokens", global_config.get("MAX_TOTAL_TOKENS", DEFAULT_MAX_TOTAL_TOKENS), ) original_count = len(unique_chunks) unique_chunks = await run_in_tokenizer_executor( _truncate_chunks_for_unified_context, unique_chunks, chunk_token_limit, tokenizer, ) logger.debug( f"Token truncation: {len(unique_chunks)} chunks from {original_count} " f"(chunk available tokens: {chunk_token_limit}, source: {source_type})" ) # 5. add id field to each chunk final_chunks = [] for i, chunk in enumerate(unique_chunks): chunk_with_id = chunk.copy() chunk_with_id["id"] = f"DC{i + 1}" final_chunks.append(chunk_with_id) return final_chunks def normalize_source_ids_limit_method(method: str | None) -> str: """Normalize the source ID limiting strategy and fall back to default when invalid.""" if not method: return DEFAULT_SOURCE_IDS_LIMIT_METHOD normalized = method.upper() if normalized not in VALID_SOURCE_IDS_LIMIT_METHODS: logger.warning( "Unknown SOURCE_IDS_LIMIT_METHOD '%s', falling back to %s", method, DEFAULT_SOURCE_IDS_LIMIT_METHOD, ) return DEFAULT_SOURCE_IDS_LIMIT_METHOD return normalized def merge_source_ids( existing_ids: Iterable[str] | None, new_ids: Iterable[str] | None ) -> list[str]: """Merge two iterables of source IDs into one flat, ordered, deduplicated list. Every element is split on ``GRAPH_FIELD_SEP``, stripped, and deduplicated by first-seen order. The split is what makes this the normalization boundary for chunk-id lists: an element that is itself a joined string (``"chunk-a<SEP>chunk-b"``) would otherwise be stored as a single id, and such an id matches no key in ``text_chunks``. It would then corrupt the ``chunk_ids``/``count`` rows of ``entity_chunks_storage`` / ``relation_chunks_storage``, miscount ``apply_source_ids_limit``'s truncation (two chunks counted as one), and miss every downstream ``get_by_ids`` lookup. No in-tree producer passes a joined element today — every caller splits its graph ``source_id`` first, and extraction emits one ``chunk_key`` per record — so the split is a guard against a future upstream regression rather than a fix for a live path. A joined element arriving here therefore means an upstream bug and is logged at debug level so the silent repair does not hide the root cause. Despite the name, this also merges the sibling union fields that share the same ``GRAPH_FIELD_SEP`` encoding: ``file_path`` and ``description`` in the edge-dedup merges of ``mongo_impl`` / ``opensearch_impl``. Stripping applies to those fragments too, so ``" foo"`` and ``"foo"`` collapse into one entry. """ merged: list[str] = [] seen: set[str] = set() for sequence in (existing_ids, new_ids): if not sequence: continue for source_id in sequence: if not source_id: continue if not isinstance(source_id, str): # Coerce rather than skip: dropping the value would silently # lose provenance, which is the failure mode this function # exists to prevent. The warning surfaces the type bug. logger.warning( f"merge_source_ids received a non-string id " f"{source_id!r} ({type(source_id).__name__}); coercing to str" ) source_id = str(source_id) if GRAPH_FIELD_SEP in source_id: logger.debug( f"merge_source_ids splitting a GRAPH_FIELD_SEP-joined value " f"{source_id!r}; callers are expected to pass individual ids" ) # split_string_by_multi_markers already strips each fragment and # drops the empty ones, so a whitespace-only element yields nothing. for sid in split_string_by_multi_markers(source_id, [GRAPH_FIELD_SEP]): if sid not in seen: seen.add(sid) merged.append(sid) return merged def apply_source_ids_limit( source_ids: Sequence[str], limit: int, method: str, *, identifier: str | None = None, ) -> list[str]: """Apply a limit strategy to a sequence of source IDs.""" if limit <= 0: return [] source_ids_list = list(source_ids) if len(source_ids_list) <= limit: return source_ids_list normalized_method = normalize_source_ids_limit_method(method) if normalized_method == SOURCE_IDS_LIMIT_METHOD_FIFO: truncated = source_ids_list[-limit:] else: # IGNORE_NEW truncated = source_ids_list[:limit] if identifier and len(truncated) < len(source_ids_list): logger.debug( "Source_id truncated: %s | %s keeping %s of %s entries", identifier, normalized_method, len(truncated), len(source_ids_list), ) return truncated def compute_incremental_chunk_ids( existing_full_chunk_ids: list[str], old_chunk_ids: list[str], new_chunk_ids: list[str], ) -> list[str]: """ Compute incrementally updated chunk IDs based on changes. This function applies delta changes (additions and removals) to an existing list of chunk IDs while maintaining order and ensuring deduplication. Delta additions from new_chunk_ids are placed at the end. Empty IDs are dropped from both inputs. Authority model: ``existing_full_chunk_ids`` — the entity/relation chunk-tracking row — is AUTHORITATIVE. A graph node's ``source_id`` is only a truncated view of it (see ``apply_source_ids_limit``), and it may legitimately retain STALE chunk IDs that tracking has already pruned: the purge path reads tracking first and falls back to ``source_id`` only when the tracking row is absent, and its ``graph_references_deleted_chunks`` branch exists precisely to handle a graph that still references chunks tracking has dropped. Consequently an ID present in BOTH ``old_chunk_ids`` and ``new_chunk_ids`` but absent from ``existing_full_chunk_ids`` is treated as intentionally pruned and is NOT restored — only genuine additions (``new - old``) are appended. Widening Step 2 to append every entry of ``new_chunk_ids`` would resurrect stale attribution into the authoritative store, which a later purge would then use to rebuild or retain KG objects sourced from chunks that no longer exist. Repairing genuinely missing attribution is the job of ``audit_kg_integrity(..., apply=True)``, never of this function. Args: existing_full_chunk_ids: Complete list of existing chunk IDs from storage old_chunk_ids: Previous chunk IDs from source_id (chunks being replaced) new_chunk_ids: New chunk IDs from updated source_id (chunks being added) Returns: Updated list of chunk IDs with deduplication Example: >>> existing = ['chunk-1', 'chunk-2', 'chunk-3'] >>> old = ['chunk-1', 'chunk-2'] >>> new = ['chunk-2', 'chunk-4'] >>> compute_incremental_chunk_ids(existing, old, new) ['chunk-2', 'chunk-3', 'chunk-4'] """ # Calculate changes chunks_to_remove = set(old_chunk_ids) - set(new_chunk_ids) chunks_to_add = set(new_chunk_ids) - set(old_chunk_ids) # Step 1: Remove chunks that are no longer needed updated_chunk_ids = [ cid for cid in existing_full_chunk_ids if cid and cid not in chunks_to_remove ] seen = set(updated_chunk_ids) # Step 2: Append genuine additions only (preserving order from new_chunk_ids). # The `seen` check is not redundant with `chunks_to_add`: tracking may already # hold an ID that is in new_chunk_ids but not in old_chunk_ids, because # source_id is a truncated view of the tracking row. for cid in new_chunk_ids: if cid and cid in chunks_to_add and cid not in seen: seen.add(cid) updated_chunk_ids.append(cid) return updated_chunk_ids def subtract_source_ids( source_ids: Iterable[str], ids_to_remove: Collection[str], ) -> list[str]: """Remove a collection of IDs from an ordered iterable while preserving order.""" removal_set = set(ids_to_remove) if not removal_set: return [source_id for source_id in source_ids if source_id] return [ source_id for source_id in source_ids if source_id and source_id not in removal_set ] def has_chunk_tracking_row(stored_data: Any) -> bool: """Return True when a chunk-tracking KV row is present and structurally usable. Presence is defined by schema — a dict carrying a ``chunk_ids`` list — never by list truthiness. A persisted row of ``{"chunk_ids": [], "count": 0}`` is authoritative ("this object tracks no chunks") and must be distinguished from an absent row or a legacy/partial shape (``{}``, ``{"count": 0}``, ``{"chunk_ids": null}``), which mean UNKNOWN. Only in the UNKNOWN case may a caller fall back to the graph object's ``source_id`` — a truncated view that can still name chunks a previous purge already pruned (see ``compute_incremental_chunk_ids``); reseeding a present-but-empty row from it resurrects stale attribution (issue #3609). """ return isinstance(stored_data, dict) and isinstance( stored_data.get("chunk_ids"), list ) def make_relation_chunk_key(src: str, tgt: str) -> str: """Create a deterministic storage key for relation chunk tracking.""" return GRAPH_FIELD_SEP.join(sorted((src, tgt))) def parse_relation_chunk_key(key: str) -> tuple[str, str]: """Parse a relation chunk storage key back into its entity pair.""" parts = key.split(GRAPH_FIELD_SEP) if len(parts) != 2: raise ValueError(f"Invalid relation chunk key: {key}") return parts[0], parts[1] def generate_track_id(prefix: str = "upload") -> str: """Generate a unique tracking ID with timestamp and UUID Args: prefix: Prefix for the track ID (e.g., 'upload', 'insert') Returns: str: Unique tracking ID in format: {prefix}_{timestamp}_{uuid} """ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") unique_id = str(uuid.uuid4())[:8] # Use first 8 characters of UUID return f"{prefix}_{timestamp}_{unique_id}" def get_pinyin_sort_key(text: str) -> str: """Generate sort key for Chinese pinyin sorting This function uses pypinyin for true Chinese pinyin sorting. If pypinyin is not available, it falls back to simple lowercase string sorting. Args: text: Text to generate sort key for Returns: str: Sort key that can be used for comparison and sorting """ if not text: return "" if _PYPINYIN_AVAILABLE: try: # Convert Chinese characters to pinyin, keep non-Chinese as-is pinyin_list = pypinyin.lazy_pinyin(text, style=pypinyin.Style.NORMAL) return "".join(pinyin_list).lower() except Exception: # Silently fall back to simple string sorting on any error return text.lower() else: # pypinyin not available, use simple string sorting return text.lower() def fix_tuple_delimiter_corruption( record: str, delimiter_core: str, tuple_delimiter: str ) -> str: """ Fix various forms of tuple_delimiter corruption from LLM output. This function handles missing or replaced characters around the core delimiter. It fixes common corruption patterns where the LLM output doesn't match the expected tuple_delimiter format. Args: record: The text record to fix delimiter_core: The core delimiter (e.g., "S" from "<|#|>") tuple_delimiter: The complete tuple delimiter (e.g., "<|#|>") Returns: The corrected record with proper tuple_delimiter format """ if not record or not delimiter_core or not tuple_delimiter: return record # Escape the delimiter core for regex use escaped_delimiter_core = re.escape(delimiter_core) # Fix: <|##|> -> <|#|>, <|#||#|> -> <|#|>, <|#|||#|> -> <|#|> record = re.sub( rf"<\|{escaped_delimiter_core}\|*?{escaped_delimiter_core}\|>", tuple_delimiter, record, ) # Fix: <|\#|> -> <|#|> record = re.sub( rf"<\|\\{escaped_delimiter_core}\|>", tuple_delimiter, record, ) # Fix: <|> -> <|#|>, <||> -> <|#|> (glued only; keep free-text "a <|> b") record = re.sub( r"(?<=\S)<\|+>(?=\S)", tuple_delimiter, record, ) # Fix: <X|#|> -> <|#|>, <|#|Y> -> <|#|>, <X|#|Y> -> <|#|>, <||#||> -> <|#|> (one extra characters outside pipes) record = re.sub( rf"<.?\|{escaped_delimiter_core}\|.?>", tuple_delimiter, record, ) # Fix: <#>, <#|>, <|#> -> <|#|> (missing one or both pipes) record = re.sub( rf"<\|?{escaped_delimiter_core}\|?>", tuple_delimiter, record, ) # Fix: <X#|> -> <|#|>, <|#X> -> <|#|> (one pipe is replaced by other character) record = re.sub( rf"<[^|]{escaped_delimiter_core}\|>|<\|{escaped_delimiter_core}[^|]>", tuple_delimiter, record, ) # Fix: <|#| -> <|#|>, <|#|| -> <|#|> (missing closing >) record = re.sub( rf"<\|{escaped_delimiter_core}\|+(?!>)", tuple_delimiter, record, ) # Fix <|#: -> <|#|> (missing closing >) record = re.sub( rf"<\|{escaped_delimiter_core}:(?!>)", tuple_delimiter, record, ) # Fix: <||#> -> <|#|> (double pipe at start, missing pipe at end) record = re.sub( rf"<\|+{escaped_delimiter_core}>", tuple_delimiter, record, ) # Fix: <|| -> <|#|> (glued only; keep free-text/code "x <|| y") # Anchor the left side only: "<||" is an unterminated separator whose right # side is the next field's raw content, so a right-hand \S anchor would add # nothing while risking a boundary miss. The left \S is what distinguishes a # glued corruption (name<||type) from a spaced free-text mention (x <|| y). record = re.sub( r"(?<=\S)<\|\|(?!>)", tuple_delimiter, record, ) # Fix: |#|> -> <|#|> (missing opening <) record = re.sub( rf"(?<!<)\|{escaped_delimiter_core}\|>", tuple_delimiter, record, ) # Fix: <|#|>| -> <|#|> ( this is a fix for: <|#|| -> <|#|> ) record = re.sub( rf"<\|{escaped_delimiter_core}\|>\|", tuple_delimiter, record, ) # Fix: ||#|| -> <|#|> (double pipes on both sides without angle brackets) record = re.sub( rf"\|\|{escaped_delimiter_core}\|\|", tuple_delimiter, record, ) return record def create_prefixed_exception(original_exception: Exception, prefix: str) -> Exception: """ Safely create a prefixed exception that adapts to all error types. Args: original_exception: The original exception. prefix: The prefix to add. Returns: A new exception with the prefix, maintaining the original exception type if possible. """ try: # Method 1: Try to reconstruct using original arguments. if hasattr(original_exception, "args") and original_exception.args: args = list(original_exception.args) # Find the first string argument and prefix it. This is safer for # exceptions like OSError where the first arg is an integer (errno). found_str = False for i, arg in enumerate(args): if isinstance(arg, str): args[i] = f"{prefix}: {arg}" found_str = True break # If no string argument is found, prefix the first argument's string representation. if not found_str: args[0] = f"{prefix}: {args[0]}" return type(original_exception)(*args) else: # Method 2: If no args, try single parameter construction. return type(original_exception)(f"{prefix}: {str(original_exception)}") except Exception: # Method 3: If reconstruction fails for any reason, wrap it in a # RuntimeError preserving the original type name and message. This is a # defensive catch-all: most known failures already surface as TypeError # (e.g. json.JSONDecodeError needs (msg, doc, pos) and # openai.APIStatusError/BadRequestError need keyword-only # (response, body), so rebuilding from args alone raises TypeError), but # an exotic constructor could raise something else (KeyError, a # validation error, ...). Catching `Exception` guarantees this helper # never raises while prefixing — `KeyboardInterrupt`/`SystemExit` are # BaseException and still propagate. The original exception and its full # traceback are preserved by the caller's `raise ... from original`. return RuntimeError( f"{prefix}: {type(original_exception).__name__}: {str(original_exception)}" ) def convert_to_user_format( entities_context: list[dict], relations_context: list[dict], chunks: list[dict], references: list[dict], query_mode: str, entity_id_to_original: dict = None, relation_id_to_original: dict = None, ) -> dict[str, Any]: """Convert internal data format to user-friendly format using original database data""" # Convert entities format using original data when available formatted_entities = [] for entity in entities_context: entity_name = entity.get("entity", "") # Try to get original data first original_entity = None if entity_id_to_original and entity_name in entity_id_to_original: original_entity = entity_id_to_original[entity_name] if original_entity: # Use original database data formatted_entities.append( { "entity_name": original_entity.get("entity_name", entity_name), "entity_type": original_entity.get("entity_type", "UNKNOWN"), "description": original_entity.get("description", ""), "source_id": original_entity.get("source_id", ""), "file_path": original_entity.get("file_path", "unknown_source"), "created_at": original_entity.get("created_at", ""), } ) else: # Fallback to LLM context data (for backward compatibility) formatted_entities.append( { "entity_name": entity_name, "entity_type": entity.get("type", "UNKNOWN"), "description": entity.get("description", ""), "source_id": entity.get("source_id", ""), "file_path": entity.get("file_path", "unknown_source"), "created_at": entity.get("created_at", ""), } ) # Convert relationships format using original data when available formatted_relationships = [] for relation in relations_context: entity1 = relation.get("entity1", "") entity2 = relation.get("entity2", "") relation_key = (entity1, entity2) # Try to get original data first original_relation = None if relation_id_to_original and relation_key in relation_id_to_original: original_relation = relation_id_to_original[relation_key] if original_relation: # Use original database data formatted_relationships.append( { "src_id": original_relation.get("src_id", entity1), "tgt_id": original_relation.get("tgt_id", entity2), "description": original_relation.get("description", ""), "keywords": original_relation.get("keywords", ""), "weight": original_relation.get("weight", 1.0), "source_id": original_relation.get("source_id", ""), "file_path": original_relation.get("file_path", "unknown_source"), "created_at": original_relation.get("created_at", ""), } ) else: # Fallback to LLM context data (for backward compatibility) formatted_relationships.append( { "src_id": entity1, "tgt_id": entity2, "description": relation.get("description", ""), "keywords": relation.get("keywords", ""), "weight": relation.get("weight", 1.0), "source_id": relation.get("source_id", ""), "file_path": relation.get("file_path", "unknown_source"), "created_at": relation.get("created_at", ""), } ) # Convert chunks format (chunks already contain complete data) formatted_chunks = [] for i, chunk in enumerate(chunks): chunk_data = { "reference_id": chunk.get("reference_id", ""), "content": chunk.get("content", ""), "file_path": chunk.get("file_path", "unknown_source"), "chunk_id": chunk.get("chunk_id", ""), } formatted_chunks.append(chunk_data) logger.debug( f"[convert_to_user_format] Formatted {len(formatted_chunks)}/{len(chunks)} chunks" ) # Build basic metadata (metadata details will be added by calling functions) metadata = { "query_mode": query_mode, "keywords": { "high_level": [], "low_level": [], }, # Placeholder, will be set by calling functions } return { "status": "success", "message": "Query processed successfully", "data": { "entities": formatted_entities, "relationships": formatted_relationships, "chunks": formatted_chunks, "references": references, }, "metadata": metadata, } def generate_reference_list_from_chunks( chunks: list[dict], ) -> tuple[list[dict], list[dict]]: """ Generate reference list from chunks, prioritizing by occurrence frequency. This function extracts file_paths from chunks, counts their occurrences, sorts by frequency and first appearance order, creates reference_id mappings, and builds a reference_list structure. Args: chunks: List of chunk dictionaries with file_path information Returns: tuple: (reference_list, updated_chunks_with_reference_ids) - reference_list: List of dicts with reference_id and file_path - updated_chunks_with_reference_ids: Original chunks with reference_id field added """ if not chunks: return [], [] # 1. Extract all valid file_paths and count their occurrences file_path_counts = {} for chunk in chunks: file_path = chunk.get("file_path", "") if file_path and file_path != "unknown_source": file_path_counts[file_path] = file_path_counts.get(file_path, 0) + 1 # 2. Sort file paths by frequency (descending), then by first appearance order # Create a list of (file_path, count, first_index) tuples file_path_with_indices = [] seen_paths = set() for i, chunk in enumerate(chunks): file_path = chunk.get("file_path", "") if file_path and file_path != "unknown_source" and file_path not in seen_paths: file_path_with_indices.append((file_path, file_path_counts[file_path], i)) seen_paths.add(file_path) # Sort by count (descending), then by first appearance index (ascending) sorted_file_paths = sorted(file_path_with_indices, key=lambda x: (-x[1], x[2])) unique_file_paths = [item[0] for item in sorted_file_paths] # 3. Create mapping from file_path to reference_id (prioritized by frequency) file_path_to_ref_id = {} for i, file_path in enumerate(unique_file_paths): file_path_to_ref_id[file_path] = str(i + 1) # 4. Add reference_id field to each chunk updated_chunks = [] for chunk in chunks: chunk_copy = chunk.copy() file_path = chunk_copy.get("file_path", "") if file_path and file_path != "unknown_source": chunk_copy["reference_id"] = file_path_to_ref_id[file_path] else: chunk_copy["reference_id"] = "" updated_chunks.append(chunk_copy) # 5. Build reference_list reference_list = [] for i, file_path in enumerate(unique_file_paths): reference_list.append({"reference_id": str(i + 1), "file_path": file_path}) return reference_list, updated_chunks def render_chunks_context_text(chunks_with_reference_ids: list[dict]) -> str: """Render the exact chunk-context text sent to the LLM. ``chunks_with_reference_ids`` must already carry ``reference_id`` — the second return value of :func:`generate_reference_list_from_chunks`. This is the single place that projects a chunk down to ``{reference_id, content, content_headings?}`` and serializes it, one JSON object per line, so that any token-budget check done against this exact call sequence matches what callers go on to send downstream verbatim. """ chunks_context = [] for chunk in chunks_with_reference_ids: entry = { "reference_id": chunk["reference_id"], "content": chunk["content"], } if chunk.get("content_headings"): entry["content_headings"] = chunk["content_headings"] chunks_context.append(entry) return "\n".join( json.dumps(text_unit, ensure_ascii=False) for text_unit in chunks_context ) def _truncate_chunks_for_unified_context( chunks: list[dict], max_token_size: int, tokenizer: "Tokenizer" ) -> list[dict]: """Two-stage truncation used by :func:`process_chunks_unified`. Counting a chunk list's tokens against the chunk dicts themselves (as the single-stage version used to) undercounts: what actually reaches the LLM is the ``{reference_id, content, content_headings?}`` projection built by :func:`generate_reference_list_from_chunks` / :func:`render_chunks_context_text`, and ``reference_id`` itself is recomputed from each survivor's ``file_path`` frequency — which changes with the exact set of chunks kept, not just by a token or two. Stage 1 approximates a safe count K from ``{content, content_headings}`` alone (``reference_id`` isn't assigned yet, and can't be until the survivor set is known — a chicken-and-egg the real renderer resolves by running after truncation, not before). Stage 2 re-renders that exact candidate list through the real renderer and independently re-verifies (shrinking K if needed), so the K this function returns is guaranteed safe under the SAME rendering the caller will perform afterward on the same list. Both stages run in this one synchronous call — it must always be submitted as a single ``run_in_tokenizer_executor`` job, never split across two round-trips through the event loop. """ if max_token_size <= 0 or not chunks: return [] def _approx_key(chunk: dict) -> str: payload = {"content": chunk.get("content")} if chunk.get("content_headings"): payload["content_headings"] = chunk["content_headings"] return json.dumps(payload, ensure_ascii=False) approx = truncate_list_by_token_size( chunks, key=_approx_key, separator="\n", max_token_size=max_token_size, tokenizer=tokenizer, ) k = len(approx) while k > 0: _, rendered_chunks = generate_reference_list_from_chunks(approx[:k]) text = render_chunks_context_text(rendered_chunks) if len(tokenizer.encode(text)) <= max_token_size: break k -= 1 return approx[:k] def validate_workspace(workspace: str) -> str: """Validate a workspace name used to build per-workspace directories. File-based storages place their data in a subdirectory named after the workspace under ``working_dir`` (``os.path.join(working_dir, workspace)``). To prevent path traversal, the workspace must be a single path component: it may not contain a path separator nor be a relative path reference. Unlike a sanitizing approach, this validator does not rewrite the name. Legitimate names containing dots (e.g. ``"v1.0"``) are accepted unchanged, while unsafe names are rejected so the caller fails fast instead of silently reading or writing outside the intended directory. Args: workspace: Workspace name from configuration or environment variables. Returns: The workspace name unchanged when it is valid. Raises: ValueError: If the workspace contains ``/`` or ``\\``, or is ``"."`` or ``".."``. Examples: >>> validate_workspace("my_workspace") 'my_workspace' >>> validate_workspace("v1.0") 'v1.0' >>> validate_workspace("../../../etc") Traceback (most recent call last): ... ValueError: Invalid workspace name '../../../etc': must not contain path separators ('/', '\\') or be a relative path reference ('.', '..') """ if "/" in workspace or "\\" in workspace or workspace in (".", ".."): raise ValueError( f"Invalid workspace name {workspace!r}: must not contain path " "separators ('/', '\\') or be a relative path reference ('.', '..')" ) return workspace # Complement of the XML 1.0 ``Char`` production. GraphML is XML, so a string # holding any other code point cannot be serialized at all -- tab, newline and # carriage return are the only C0 controls XML admits, and descriptions # legitimately carry those. _XML_INCOMPATIBLE_CHAR_PATTERN = re.compile( r"[^\u0009\u000a\u000d\u0020-\ud7ff\ue000-\ufffd\U00010000-\U0010ffff]" ) # Attribute names a backend would read as something other than a name. Kept to # exactly that: MongoGraphStorage passes names straight into a ``$set`` # document, where a dot is a *path* separator (``source_ids.0`` rewrites an # element of the chunk-attribution array rather than creating a field) and a # leading ``$`` is an update operator. # # Deliberately NOT "must be an identifier". Names like ``display-name`` or # ``has space`` carry no interpretation hazard on any backend, and the manual # edit API accepted them before the field allowlist landed -- so the document # backends that persisted them hold such names today. Refusing them here would # make those entities unmodifiable, because every rewrite path (entity edit, # rename, merge, extraction rebuild) spreads the *stored* attributes back into # the upsert payload. _GRAPH_ATTRIBUTE_NAME_DISALLOWED_SUBSTRING = "." _GRAPH_ATTRIBUTE_NAME_DISALLOWED_PREFIX = "$" # Integer attributes must fit in a signed 64-bit integer. Verified against the # installed driver rather than assumed: neo4j's packstream Packer raises # ``OverflowError("Integer ... out of range")`` outside ``[-2**63, 2**63)`` # (``neo4j/_codec/packstream/v1/__init__.py``), and GraphML declares an int # attribute as ``attr.type="long"``, which is ``xsd:long`` -- also int64. So a # larger int is outside the intersection this module defines, even though # networkx will happily write it and PostgreSQL's arbitrary-precision ``jsonb`` # numeric will happily store it. _GRAPH_ATTRIBUTE_INT_MIN = -(2**63) _GRAPH_ATTRIBUTE_INT_MAX = 2**63 - 1 def _graphml_encodable_types() -> frozenset[type]: """The exact types networkx's GraphML writer knows how to encode. Asked of networkx rather than enumerated here, because ``add_data`` resolves a value by looking its type up in ``self.xml_type`` -- so that table *is* the definition, and reproducing it by hand would drift. Sourcing it also makes the guard's justification true by construction: every type this module refuses is a type the write would have failed on. An ``isinstance`` shortcut over ``np.integer`` / ``np.floating`` would be wrong, not merely loose: ``np.longdouble`` is an ``np.floating`` and is *not* in the table, so it would be accepted here and rejected at write time -- the poisoning this exists to prevent. The four builtins are unioned in as a floor, and a failure to read the table degrades to exactly that floor. ``networkx`` carries no version bound in pyproject.toml, so its internals could move; degrading to the floor is stricter, never looser, so it cannot open the hole it guards. """ builtin_scalars = frozenset({str, int, float, bool}) try: from networkx.readwrite.graphml import GraphML probe = GraphML() probe.construct_types() return frozenset(probe.xml_type) | builtin_scalars except Exception: # networkx internals moved: fall back to the strict floor return builtin_scalars _GRAPHML_ENCODABLE_TYPES = _graphml_encodable_types() # The intersection every registered backend can carry. Narrower than # ``_GRAPHML_ENCODABLE_TYPES``, which also holds the numpy scalar types GraphML # writes: those are not portable, and not marginally so -- ``json.dumps`` # (PGTableGraphStorage's jsonb column) and ``bson`` (MongoGraphStorage) both # refuse ``np.float32`` / ``np.int64`` / ``np.uint32`` / ``np.bool_`` outright. _PORTABLE_SCALAR_TYPES = frozenset({str, int, float, bool}) def xml_attribute_name_rejection(name: Any) -> str | None: """Explain why *name* cannot be used as an XML-serialized attribute name. GraphML writes attribute names into the XML ``attr.name`` field, so a name is subject to the same character rule as a value -- a claim checked against networkx rather than assumed: a name holding a control character, a lone surrogate or U+FFFE makes ``write_graphml`` raise, which on ``NetworkXStorage`` is the instance-wide persistence outage the storage guard exists to prevent. Everything else is allowed, including names a *portable* rule would refuse (``a.b``, ``$set``, ``display-name``, ``has space``, ``""``). Two reasons: GraphML round-trips all of those, and -- the load-bearing one -- a name this function rejects can never have been persisted in an existing GraphML file, because the write that would have stored it failed. That is what makes the check safe to apply to a rewrite payload, which spreads a fetched object's stored names back in. Args: name: Candidate attribute name. Returns: A reason fragment, or ``None`` when XML can carry the name. """ if not isinstance(name, str): # networkx raises TypeError("keywords must be strings") before mutating, # so this cannot poison the graph -- it is folded in so the caller gets a # validation error rather than an unhandled TypeError. return f"must be a string, got {type(name).__name__}" match = _XML_INCOMPATIBLE_CHAR_PATTERN.search(name) if match: return ( "must not contain the character " f"U+{ord(match.group()):04X}, which XML cannot encode" ) return None def xml_attribute_value_rejection(value: Any) -> str | None: """Explain why *value* cannot be encoded in XML at all. The narrower of the two value rules, and the one a GraphML-backed store should enforce: it rejects only what cannot be *serialized*, not what is merely unportable. ``NaN``, ``inf`` and an integer past int64 all round-trip through GraphML unchanged, so they are accepted here even though ``graph_attribute_value_rejection`` refuses them. "Serialized" is the operative word, and it is not the same as "is a scalar": an integer with more digits than ``sys.get_int_max_str_digits()`` cannot be rendered as text at all, so GraphML cannot hold it however ordinary it looks. That difference is deliberate, and it is what keeps a backend guard from stranding data. A workspace can already hold such a value -- the manual edit API accepted anything before its field allowlist landed -- and every rewrite path (entity edit, extraction rebuild) spreads a fetched object's stored attributes back into the upsert payload. Enforcing the *portable* rule at the storage layer would therefore make those objects unmodifiable, while enforcing it where caller input enters costs nothing. Args: value: Candidate attribute value. Returns: A reason fragment suitable for appending to ``"attribute 'x' "``, or ``None`` when XML can carry the value. """ # Exact type, not ``isinstance``. networkx's GraphML writer resolves a value # by looking its type up in a dict (``add_data``: ``if element_type not in # self.xml_type``), so a *subclass* of a supported scalar is rejected at write # time -- ``enum.StrEnum``, ``enum.IntEnum``, ``np.str_``, ``Decimal`` and any # user ``class X(str)`` all satisfy ``isinstance`` and all make # ``write_graphml`` raise. Read from the installed networkx, not assumed. # # The set comes from that same table (see ``_graphml_encodable_types``), so # it includes the numpy scalar types GraphML writes -- and so that every type # refused here is one the write would genuinely have failed on. value_type = type(value) if value_type not in _GRAPHML_ENCODABLE_TYPES: if isinstance(value, (str, int, float, bool)): # A subclass: worth saying so, because the caller reasonably expects # a `str` subclass to be a string. return ( f"must be a type GraphML can encode, got {value_type.__name__} " "(GraphML resolves value types exactly, so a subclass of a " "supported scalar is still refused)" ) return f"must be a string, number or boolean, got {value_type.__name__}" if value_type is not str and value_type is not int: # bool, float and the numpy numeric scalars need nothing further: the # only remaining checks are the digit limit (int) and the character rule # (str). A numpy int is width-bounded, so it cannot hit the digit limit. return None if value_type is int: # GraphML stores values as text, and networkx stringifies them on write. # CPython refuses to stringify an integer with more digits than # ``sys.get_int_max_str_digits()`` (4300 by default since 3.11, and # settable at runtime), so a large enough int makes ``write_graphml`` # raise even though the int itself is a perfectly ordinary scalar. # Attempt the same conversion the writer will make rather than comparing # against the limit, so this cannot drift from it if the limit is changed. try: str(value) except ValueError: # `get_int_max_str_digits` arrived in 3.11 and was backported to # 3.10.7, so it can be missing on the `requires-python = ">=3.10"` # floor. It is unreachable here when missing -- no limit means # `str()` cannot raise -- but read it defensively rather than relying # on that. read_limit = getattr(sys, "get_int_max_str_digits", None) limit = f" (limit {read_limit()} digits)" if read_limit else "" return f"has more digits than Python will render as text{limit}" return None match = _XML_INCOMPATIBLE_CHAR_PATTERN.search(value) if match: return ( "must not contain the character " f"U+{ord(match.group()):04X}, which XML cannot encode" ) return None def graph_attribute_value_rejection(value: Any) -> str | None: """Explain why *value* cannot be stored as a graph node/edge attribute. The wider of the two value rules: the intersection of what *every* registered backend can carry, meant for the point where caller input enters (see ``xml_attribute_value_rejection`` for the narrower serialization rule a storage backend should enforce instead, and why the two differ). The rules are the intersection of what the seven registered ``GRAPH_STORAGE`` backends can carry, so the same payload behaves the same way on all of them: * ``bool`` -- accepted everywhere. * ``int`` -- must fit in int64. A larger int is written happily by networkx and stored happily by PostgreSQL's arbitrary-precision ``jsonb`` numeric, but the Neo4j driver refuses to pack it and GraphML mislabels it as ``xsd:long``. See ``_GRAPH_ATTRIBUTE_INT_MIN``. * ``float`` -- must be finite. ``NaN`` / ``inf`` survive GraphML but ``json.dumps`` renders them as bare ``NaN`` / ``Infinity``, which is not valid JSON and is rejected by the ``jsonb`` column PGTableGraphStorage writes to. * ``str`` -- must be XML-compatible, because GraphML cannot encode the other code points at all (see ``_XML_INCOMPATIBLE_CHAR_PATTERN``). * anything else (``dict``, ``list``, ``None``, ``bytes``, ...) -- rejected. ``None`` is called out because it is the one non-scalar that reads as harmless: GraphML refuses it outright, while on the Cypher backends ``SET n += {k: null}`` silently *deletes* the property. Args: value: Candidate attribute value. Returns: A reason fragment suitable for appending to ``"attribute 'x' "``, or ``None`` when the value is storable. """ rejection = xml_attribute_value_rejection(value) if rejection is not None: return rejection # The XML rule accepts what GraphML can write, which includes numpy scalars; # portability is narrower. ``json.dumps`` (PGTableGraphStorage's jsonb # column) and ``bson`` (MongoGraphStorage) both refuse ``np.float32`` / # ``np.int64`` / ``np.uint32`` / ``np.bool_``, so a numpy value that persists # fine on NetworkX cannot cross to another backend. value_type = type(value) if value_type not in _PORTABLE_SCALAR_TYPES: return ( f"must be a str, int, float or bool to be portable across graph " f"backends, got {value_type.__name__}" ) # The type is pinned exactly now, so ``is int`` rather than ``isinstance`` -- # a bool cannot reach the range check. if value_type is int: if not _GRAPH_ATTRIBUTE_INT_MIN <= value <= _GRAPH_ATTRIBUTE_INT_MAX: return ( "must be a 64-bit integer " f"({_GRAPH_ATTRIBUTE_INT_MIN} to {_GRAPH_ATTRIBUTE_INT_MAX})" ) return None if value_type is float and not math.isfinite(value): return "must be a finite number" return None def validate_graph_attributes(attributes: dict[str, Any], *, context: str) -> None: """Reject a node/edge attribute mapping no graph backend could store. Args: attributes: Attribute mapping about to be written to graph storage. context: Prefix identifying the object being written, used in the error message (e.g. ``"entity 'Tesla'"``). Raises: ValueError: On the first unusable attribute name or value. """ # The portable contract is the intersection, so it owns both name rules: a # name XML cannot encode is unstorable on NetworkX, and a name MongoDB would # read as a field path is unstorable there. Neither backend needs the other's # rule (see each rejection helper), but caller input has to satisfy both. validate_xml_attributes(attributes, context=context) validate_interpreted_attribute_names(attributes, context=context) validate_graph_attribute_values(attributes, context=context) def validate_interpreted_attribute_names( attributes: dict[str, Any], *, context: str ) -> None: """Reject attribute names a backend would read as something other than a name. The *interpretation* rule, not the serialization one: see ``xml_attribute_name_rejection`` for the character rule a GraphML-backed store needs. A backend should enforce the hazard it actually has, and the two hazards live in different backends: * Names are dangerous only where they are *interpreted*. MongoDB passes them into a ``$set`` document, so a dot is a path (``source_ids.0`` rewrites an element of the chunk-attribution array instead of creating a field) and a leading ``$`` is an update operator. * Names are inert in GraphML, which is why NetworkXStorage validates values only. Enforcing names there would additionally reject re-ingesting a node whose *stored* attributes predate this validation, which is a migration failure with no safety benefit. The rule is the interpretation hazard and nothing beyond it -- see ``_GRAPH_ATTRIBUTE_NAME_DISALLOWED_SUBSTRING`` for why "must be an identifier" would be the wrong rule. A name that a backend *stores* rather than interprets is legal here even if it is unusual, because rewrite paths feed stored attributes back into the upsert payload and a stricter rule would strand existing data. Args: attributes: Attribute mapping about to be written to graph storage. context: Prefix identifying the object being written. Raises: ValueError: On the first unusable attribute name. """ for key in attributes: if not isinstance(key, str) or not key: raise ValueError( f"{context}: invalid attribute name {key!r}: must be a non-empty string" ) if _GRAPH_ATTRIBUTE_NAME_DISALLOWED_SUBSTRING in key: raise ValueError( f"{context}: invalid attribute name {key!r}: must not contain " f"{_GRAPH_ATTRIBUTE_NAME_DISALLOWED_SUBSTRING!r}, which is read " "as a field path rather than part of the name" ) if key.startswith(_GRAPH_ATTRIBUTE_NAME_DISALLOWED_PREFIX): raise ValueError( f"{context}: invalid attribute name {key!r}: must not start with " f"{_GRAPH_ATTRIBUTE_NAME_DISALLOWED_PREFIX!r}, which is read as " "an update operator" ) def validate_xml_attributes(attributes: dict[str, Any], *, context: str) -> None: """Reject attribute names or values XML cannot encode. For a GraphML-backed store. Names are checked as well as values because GraphML serializes them into the XML ``attr.name`` field, so an unencodable name breaks the same write. See ``xml_attribute_name_rejection`` and ``xml_attribute_value_rejection`` for why both rules are narrower than the portable contract. Args: attributes: Attribute mapping about to be written to graph storage. context: Prefix identifying the object being written. Raises: ValueError: On the first name or value XML cannot carry. """ for key, value in attributes.items(): rejection = xml_attribute_name_rejection(key) if rejection is not None: raise ValueError(f"{context}: attribute name {key!r} {rejection}") rejection = xml_attribute_value_rejection(value) if rejection is not None: raise ValueError(f"{context}: attribute {key!r} {rejection}") def validate_graph_attribute_values( attributes: dict[str, Any], *, context: str ) -> None: """Reject attribute values no graph backend can store. See ``graph_attribute_value_rejection`` for the rules and ``validate_interpreted_attribute_names`` for why the halves are separate. Args: attributes: Attribute mapping about to be written to graph storage. context: Prefix identifying the object being written. Raises: ValueError: On the first unstorable value. """ for key, value in attributes.items(): rejection = graph_attribute_value_rejection(value) if rejection is not None: raise ValueError(f"{context}: attribute {key!r} {rejection}")
from __future__ import annotations
import weakref
import sys
import asyncio
import contextvars
import html
import csv
import inspect
import json
import logging
import logging.handlers
import math
import os
import re
import time
import uuid
import threading
import warnings
from concurrent.futures import Future as ConcurrentFuture, ThreadPoolExecutor
from dataclasses import dataclass
from datetime import datetime
from functools import partial, wraps
from hashlib import md5
from pathlib import Path
from typing import (
Any,
Awaitable,
Protocol,
Callable,
TYPE_CHECKING,
List,
NamedTuple,
Optional,
Iterable,
Sequence,
Collection,
)
import numpy as np
from dotenv import load_dotenv
import json_repair
from lightrag.exceptions import (
ChunkBlockMatchError,
CommitBookkeepingError,
EmptyTruncatedResponseError,
)
from lightrag.constants import (
DEFAULT_LOG_MAX_BYTES,
DEFAULT_LOG_BACKUP_COUNT,
DEFAULT_LOG_FILENAME,
GRAPH_FIELD_SEP,
DEFAULT_MAX_TOTAL_TOKENS,
DEFAULT_PROCESSING_PRIORITY,
DEFAULT_SOURCE_IDS_LIMIT_METHOD,
VALID_SOURCE_IDS_LIMIT_METHODS,
SOURCE_IDS_LIMIT_METHOD_FIFO,
PARSED_DIR_NAME,
DEFAULT_GLOBAL_SLOT_POLL_MIN,
DEFAULT_GLOBAL_SLOT_POLL_DEFERRED_MAX,
DEFAULT_GLOBAL_SLOT_DRAIN_LIMIT,
DEFAULT_ZOMBIE_COMPACT_THRESHOLD,
DEFAULT_COMPACT_BATCH_LIMIT,
DEFAULT_QUEUE_STATS_MIN_PUBLISH_INTERVAL,
)
# Precompile regex pattern for JSON sanitization (module-level, compiled once)
_SURROGATE_PATTERN = re.compile(r"[\uD800-\uDFFF\uFFFE\uFFFF]")
_CONTROL_CHAR_PATTERN_ALL = re.compile(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]")
class SafeStreamHandler(logging.StreamHandler):
"""StreamHandler that gracefully handles closed streams during shutdown.
This handler prevents "ValueError: I/O operation on closed file" errors
that can occur when pytest or other test frameworks close stdout/stderr
before Python's logging cleanup runs.
"""
def flush(self):
"""Flush the stream, ignoring errors if the stream is closed."""
try:
super().flush()
except (ValueError, OSError):
# Stream is closed or otherwise unavailable, silently ignore
pass
def close(self):
"""Close the handler, ignoring errors if the stream is already closed."""
try:
super().close()
except (ValueError, OSError):
# Stream is closed or otherwise unavailable, silently ignore
pass
# Initialize logger with basic configuration
logger = logging.getLogger("lightrag")
logger.propagate = False # prevent log message send to root logger

Xet Storage Details

Size:
331 kB
·
Xet hash:
876488390cc79a3c5e9dc673be80641fba2a2e67aef4e6129988dbf09d3e55b0

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.