Buckets:
from __future__ import annotations
import time
import asyncio
from typing import Any, Awaitable, Callable, cast
from .base import DeletionResult
from .exceptions import CommitBookkeepingError
from .kg.shared_storage import get_storage_keyed_lock
from .constants import GRAPH_FIELD_SEP, RELATION_NO_EVIDENCE_SOURCE_IDS
from .operate import _truncate_vdb_content
from .utils import (
VectorStorageConsistencyError,
_consume_future_exception,
_wait_deferring_cancellation,
compute_mdhash_id,
graph_attribute_value_rejection,
log_without_raising,
logger,
make_relation_vdb_ids,
normalize_entity_name,
safe_vdb_operation_with_exception,
)
from .base import StorageNameSpace
# Field specs for the manual entity/relation mutation APIs. The value is the
# expected shape: "text" for a string attribute, "number" for a numeric one.
#
# These are an allowlist, not documentation. `updated_data` used to be merged
# into the stored object wholesale (`{**node_data, **updated_data}`), so any key
# a caller invented was written with whatever value it carried -- including
# values no graph backend can store. See `_sanitize_graph_fields`.
_TEXT_FIELD = "text"
_NUMBER_FIELD = "number"
# `entity_name` is the rename target, which `aedit_entity` resolves and writes
# back into `updated_data` before the merge; it is a legal edit field but not a
# legal *create* field (create takes the name as its own argument).
_EDITABLE_ENTITY_FIELDS: dict[str, str] = {
"entity_name": _TEXT_FIELD,
"entity_type": _TEXT_FIELD,
"description": _TEXT_FIELD,
"source_id": _TEXT_FIELD,
"file_path": _TEXT_FIELD,
}
_ENTITY_DATA_FIELDS: dict[str, str] = {
key: kind for key, kind in _EDITABLE_ENTITY_FIELDS.items() if key != "entity_name"
}
_RELATION_DATA_FIELDS: dict[str, str] = {
"description": _TEXT_FIELD,
"keywords": _TEXT_FIELD,
"source_id": _TEXT_FIELD,
"file_path": _TEXT_FIELD,
"weight": _NUMBER_FIELD,
}
def _sanitize_graph_fields(
data: dict[str, Any],
*,
allowed_fields: dict[str, str],
object_type: str,
reject_unknown: bool,
) -> dict[str, Any]:
"""Return *data* reduced to well-typed, storable graph attributes.
Two call shapes, because the two families of caller differ in what they do
with a key they do not recognise:
* ``reject_unknown=True`` (the edit/merge paths) -- these merge the caller's
mapping into the stored object, so an unrecognised key *is written*. It
has to be refused, and refused loudly: silently dropping it on an edit
endpoint would report success for a change that never happened.
* ``reject_unknown=False`` (the create paths) -- these already copy a fixed
set of named fields out of the mapping, so an unrecognised key is
structurally incapable of reaching storage. Only the values of the
recognised keys need checking, and refusing extra keys here would break
payloads that work today for no security gain.
Per-field types matter as much as the allowlist. A value can be a perfectly
storable scalar and still be wrong: ``{"source_id": 123}`` serializes fine
(GraphML long, JSON number), reaches disk, survives a restart, and then
breaks every later reader that splits ``source_id`` on ``GRAPH_FIELD_SEP``
-- a durable failure, unlike the transient one a non-scalar causes.
Args:
data: Caller-supplied attribute mapping.
allowed_fields: Field name to expected shape (``_TEXT_FIELD`` /
``_NUMBER_FIELD``).
object_type: ``"entity"`` or ``"relation"``, for error messages.
reject_unknown: Whether an unrecognised key is an error (see above).
Returns:
A new mapping holding only recognised fields, with numeric fields
coerced to ``float``.
Raises:
ValueError: On an unknown field (when ``reject_unknown``), a field whose
value has the wrong shape, or a value no graph backend can store.
"""
sanitized: dict[str, Any] = {}
for key, value in data.items():
kind = allowed_fields.get(key)
if kind is None:
if reject_unknown:
raise ValueError(
f"Unknown {object_type} field '{key}'. Allowed fields: "
f"{', '.join(sorted(allowed_fields))}"
)
continue
rejection = graph_attribute_value_rejection(value)
if rejection is not None:
raise ValueError(f"{object_type.capitalize()} field '{key}' {rejection}")
if kind == _NUMBER_FIELD:
# bool is an int subclass, and `float(True)` would quietly become
# 1.0 -- a boolean weight is a caller mistake, not a number.
if isinstance(value, bool):
raise ValueError(
f"{object_type.capitalize()} field '{key}' must be a number, "
"got bool"
)
try:
# A numeric string is accepted because the create paths have
# always run the value through `float()`; normalizing here means
# the *stored* attribute is a float on the edit paths too,
# instead of a string that only the VDB payload converted.
coerced = float(value)
except (TypeError, ValueError, OverflowError):
# OverflowError is what `float()` raises for an int too large to
# convert. The int64 bound in `graph_attribute_value_rejection`
# already refuses those above, so this is belt-and-braces for a
# future caller passing some other type whose `__float__`
# overflows -- without it the endpoint returns 500 for what is a
# validation failure.
raise ValueError(
f"{object_type.capitalize()} field '{key}' must be a number, "
f"got {value!r}"
) from None
# Re-check the *coerced* value. The check above ran on what the
# caller sent, and for a numeric string that is a perfectly storable
# `str` -- it is this conversion that can produce the non-scalar the
# contract forbids ("nan" -> NaN, "1e999" -> inf). Skipping it would
# accept the request and then fail in storage: PGTableGraphStorage's
# jsonb column rejects the bare `NaN` json.dumps emits, so a 400
# would arrive as a 500, while permissive backends keep the value.
rejection = graph_attribute_value_rejection(coerced)
if rejection is not None:
raise ValueError(
f"{object_type.capitalize()} field '{key}' {rejection}"
)
sanitized[key] = coerced
elif not isinstance(value, str):
raise ValueError(
f"{object_type.capitalize()} field '{key}' must be a string, got "
f"{type(value).__name__}"
)
else:
sanitized[key] = value
return sanitized
def relation_evidence_source_ids(source_id: str) -> list[str]:
"""Return ordered distinct source IDs that count as relation evidence.
Empty values and historical no-source placeholders are intentionally
excluded. New manual relations store an omitted source as an empty string;
the placeholders remain here only so legacy rows keep the same meaning.
"""
if not isinstance(source_id, str):
raise ValueError(
"Relation field 'source_id' must be a string, got "
f"{type(source_id).__name__}"
)
return list(
dict.fromkeys(
source
for source in source_id.split(GRAPH_FIELD_SEP)
if source and source not in RELATION_NO_EVIDENCE_SOURCE_IDS
)
)
def relation_evidence_count(source_id: str) -> int:
"""Return the number of distinct real evidence sources on a relation."""
return len(relation_evidence_source_ids(source_id))
def validate_relation_weight(
weight: Any,
source_id: str,
*,
context: str = "Relation",
) -> float:
"""Normalize a relation weight and enforce its evidence-count floor.
A relation weight is its evidence count plus an optional manual boost. A
caller may choose a fractional weight only when the relation has no real
source IDs; negative weights are therefore always invalid.
"""
normalized = _sanitize_graph_fields(
{"weight": weight, "source_id": source_id},
allowed_fields=_RELATION_DATA_FIELDS,
object_type="relation",
reject_unknown=True,
)
normalized_weight = normalized["weight"]
evidence_count = relation_evidence_count(normalized["source_id"])
if normalized_weight < evidence_count:
raise ValueError(
f"{context} weight {normalized_weight} cannot be less than its "
f"distinct-source evidence count {evidence_count}"
)
return normalized_weight
def apply_relation_weight_floor(weight: Any, source_id: str) -> float:
"""Normalize a stored weight and repair it to the evidence-count floor."""
normalized = _sanitize_graph_fields(
{"weight": weight, "source_id": source_id},
allowed_fields=_RELATION_DATA_FIELDS,
object_type="relation",
reject_unknown=True,
)
return max(
normalized["weight"],
float(relation_evidence_count(normalized["source_id"])),
)
def _require_non_empty_description(
description: Any, *, operation: str, object_type: str
) -> None:
if description is None or not str(description).strip():
raise ValueError(
f"{object_type.capitalize()} description cannot be empty for {operation} operation"
)
def _reject_self_loop_relation(
source_entity: Any, target_entity: Any, *, operation: str
) -> None:
"""Refuse a relation whose two endpoints are the same entity.
LightRAG's own extraction never emits a self-loop: ``operate.py`` drops
``source == target`` in both record parsers and again in
``_merge_edges_then_upsert``, and ``amerge_entities`` skips any endpoint
pair that would collapse into one. Manual creation is therefore the only
way such an edge can reach the graph, and a self-loop carries no
connectivity for graph retrieval — it only widens the surface where the
rest of the code has to reason about ``src == tgt``.
Rejected at the same layer as an empty description: before the keyed lock,
so a refusal writes nothing.
"""
if source_entity == target_entity:
raise ValueError(
f"Cannot {operation} a self-loop relation on '{source_entity}': "
"source and target must be different entities"
)
def _normalize_manual_entity_name(entity_name: Any) -> str:
"""Apply the extraction naming contract to a manual entity identifier."""
if not isinstance(entity_name, str):
raise ValueError("Entity name must be a string")
return normalize_entity_name(entity_name)
async def _persist_graph_updates(
entities_vdb=None,
relationships_vdb=None,
chunk_entity_relation_graph=None,
entity_chunks_storage=None,
relation_chunks_storage=None,
) -> None:
"""Unified callback to persist updates after graph operations.
Ensures all relevant storage instances are properly persisted after
operations like delete, edit, create, or merge.
A ``CommitBookkeepingError`` from any of them is logged, not propagated: it
says that store's write is durable and only its publication failed, so
raising it would fold a flush that DID land into the caller's failure
handling — reporting a completed deletion as ``fail``/500, and stranding the
tracking rows the retry would then believe it still owed. Every other
exception still propagates: those mean the flush did not happen.
Args:
entities_vdb: Entity vector database storage (optional)
relationships_vdb: Relationship vector database storage (optional)
chunk_entity_relation_graph: Graph storage instance (optional)
entity_chunks_storage: Entity-chunk tracking storage (optional)
relation_chunks_storage: Relation-chunk tracking storage (optional)
"""
storages = []
# Collect all non-None storage instances
if entities_vdb is not None:
storages.append(entities_vdb)
if relationships_vdb is not None:
storages.append(relationships_vdb)
if chunk_entity_relation_graph is not None:
storages.append(chunk_entity_relation_graph)
if entity_chunks_storage is not None:
storages.append(entity_chunks_storage)
if relation_chunks_storage is not None:
storages.append(relation_chunks_storage)
async def _flush(storage_inst) -> None:
try:
await cast(StorageNameSpace, storage_inst).index_done_callback()
except CommitBookkeepingError as e:
log_without_raising(
logger.error,
f"Persisting {getattr(storage_inst, 'namespace', storage_inst)} "
f"landed, but publishing it failed: {e.__cause__}. The data is "
"durable; cross-process visibility is deferred to the next "
"commit.",
)
# Persist all storage instances in parallel
if storages:
await asyncio.gather(*[_flush(storage_inst) for storage_inst in storages])
async def _finish_deferring_cancellation(coro, description: str) -> None:
"""Run ``coro`` to completion even if this task is cancelled meanwhile.
Cancellation can arrive *after* the graph commit has already landed:
``commit_in_storage_io`` deliberately finishes the GraphML write and its
commit hook before re-raising ``CancelledError``, and ``CancelledError`` is a
``BaseException``, so the deletion helpers' ``except Exception`` does not see
it either. Returning there would leave the object durably gone with its
tracking rows intact — and for an entity the incident relation rows are then
unreachable by the ``not_found`` sweep, so a recreated relation would read
them back as authoritative provenance: exactly the defect this module exists
to prevent, and with no audit line, since the code that logs the stranded
keys never runs.
Loop-and-shield rather than a bare ``asyncio.shield``: a second cancel would
let the caller escape while the cleanup is still running. Same idiom, and
same reason, as ``_bounded_submit_impl``'s own deferred commit hook.
"""
future = asyncio.ensure_future(coro)
future.add_done_callback(_consume_future_exception)
pending_cancel = await _wait_deferring_cancellation(future, None)
error = None if future.cancelled() else future.exception()
if pending_cancel is not None:
if error is not None:
# The caller is about to get CancelledError, so this is the only
# place this failure can still be seen.
logger.error(
f"{description} failed while its caller was cancelled: {error}"
)
raise pending_cancel
if error is not None:
raise error
async def _commit_graph_or_raise(chunk_entity_relation_graph, context: str) -> None:
"""Commit the graph, and refuse to continue unless it actually persisted.
``NetworkXStorage.index_done_callback`` does not raise when it declines to
write: if another process committed since this one last read the graph, it
reloads from disk, DISCARDS the in-memory mutation and returns ``False``.
Treating a normal return as proof of a commit would let a deletion whose node
is still live go on to drop that node's tracking rows and report success —
precisely the state the staging in :func:`adelete_by_entity` exists to
prevent, reached with no crash and no error surfaced anywhere.
Only an explicit ``False`` counts as "did not commit". The base signature is
``-> None``, so backends that simply return nothing are unaffected.
``CommitBookkeepingError`` is the opposite answer and is caught here: it
means the write DID land and only its publication failed. Letting it out
would reach the caller's generic ``except Exception`` and be reported as a
deletion that did not happen, so the tracking rows of an object that is
durably gone would be left behind — and for an entity the incident relation
rows are then unreachable, since the edges that named them are gone with the
node. ``NetworkXStorage`` already absorbs its own publication failure, so no
shipped backend raises this today; this is the standing answer for any graph
backend that hands the decision to its caller instead, because the default
(treat it as a declined commit) is the one that loses data.
"""
try:
committed = await chunk_entity_relation_graph.index_done_callback()
except CommitBookkeepingError as e:
log_without_raising(
logger.error,
f"{context}: the graph write is durable, but publishing it failed: "
f"{e.__cause__}. Continuing with the tracking cleanup this removal "
"owes; cross-process visibility is deferred to the next commit.",
)
return
if committed is False:
raise RuntimeError(
f"{context}: the graph commit was skipped because another process "
"updated the graph, so the in-memory deletion was discarded"
)
async def _sweep_orphan_tracking_row(
tracking_storage, storage_key: str, description: str
) -> bool:
"""Drop a chunk-tracking row whose graph object no longer exists.
Returns True when a row was found and deleted. The caller must flush
regardless of that answer — see the call sites: an in-memory row is not the
same question as pending durable state. Presence is tested with a plain
``is not None`` rather than :func:`has_chunk_tracking_row`: a legacy or
partial row (``{}``, ``{"count": 0}``) is still stored attribution for an
object that is gone, and sweeping it is exactly as correct as sweeping a
well-formed one. A backend error is NOT swallowed: it reaches the caller's
generic handler, so a missing object whose tracking backend is down reports
``fail``/500 rather than ``not_found``/404. That is deliberate — a 404 over a
row this call failed to sweep would be a silent failure, while a 500 tells
the caller to retry, which is what makes the ordering converge.
"""
if tracking_storage is None:
return False
if await tracking_storage.get_by_id(storage_key) is None:
return False
await tracking_storage.delete([storage_key])
logger.info(f"Delete: swept orphan chunk tracking row for {description}")
return True
async def adelete_by_entity(
chunk_entity_relation_graph,
entities_vdb,
relationships_vdb,
entity_name: str,
entity_chunks_storage=None,
relation_chunks_storage=None,
) -> DeletionResult:
"""Asynchronously delete an entity and all its relationships.
Also cleans up entity_chunks_storage and relation_chunks_storage to remove chunk tracking.
Tracking rows are removed only AFTER the graph node is gone, matching the
ordering of the document purge path. The reverse order is forbidden by the
purge recovery contract: a tracking row is the authoritative attribution
carrier, so dropping it while the object survives (a transient vector or
graph failure below) leaves a live entity whose provenance has silently
degraded to the truncated graph ``source_id`` — from which a later purge can
conclude "no remaining sources" and delete an entity other documents still
reference. The residual window of the ordering used here is the inverse and
strictly milder: an orphan tracking row whose object is already gone, which
the ``not_found`` sweep below converges on retry.
The ordering is enforced at the DURABLE level, not just the call level, and
holds for any mix of backends. Storage families differ in when a mutation
becomes durable — a deferred graph (NetworkX/JSON) commits only in
``index_done_callback``, an immediate-write tracking store (Redis/PG) commits
inside ``delete()`` — so neither ordering the calls nor ordering the flushes
is sufficient alone. The work is therefore staged: (1) commit the graph
object's removal by itself and verify it actually happened — a graph backend
may decline to write and report that by return value rather than by raising
(see :func:`_commit_graph_or_raise`), (2) only then delete and commit the tracking
rows, (3) flush the vector storages last, so a vector failure cannot abort a
deletion whose node is already durably gone and strand rows the sweep cannot
reach. A stale vector record is the recoverable, rebuildable residue this
codebase already accepts elsewhere.
A commit whose WRITE landed and whose PUBLICATION failed is not a failed
commit, and this function does not treat it as one: the file backends absorb
it themselves, and :func:`_commit_graph_or_raise` /
:func:`_persist_graph_updates` absorb a ``CommitBookkeepingError`` from any
backend that hands it up instead. The cleanup is attempted either way. If it
then fails for a real reason, the handling is unchanged — the orphaned
relation keys are named in the log and the call answers ``fail``/500. If it
succeeds, the deletion answers ``success``; what the publication failure cost
is cross-process visibility, deferred to the next commit and logged there,
not the removal itself.
Args:
chunk_entity_relation_graph: Graph storage instance
entities_vdb: Vector database storage for entities
relationships_vdb: Vector database storage for relationships
entity_name: Name of the entity to delete
entity_chunks_storage: Optional KV storage for tracking chunks that reference this entity
relation_chunks_storage: Optional KV storage for tracking chunks that reference relations
"""
# Use keyed lock for entity to ensure atomic graph and vector db operations.
# The doc-ingest pipeline locks edges under sorted([src, tgt]) in this same
# namespace, so [entity_name] already mutually excludes any concurrent edge
# write that touches this entity — get_storage_keyed_lock acquires one
# mutex per key, and identical key strings share the same mutex.
workspace = entities_vdb.global_config.get("workspace", "")
namespace = f"{workspace}:GraphDB" if workspace else "GraphDB"
async with get_storage_keyed_lock(
[entity_name], namespace=namespace, enable_logging=False
):
try:
# Check if the entity exists
if not await chunk_entity_relation_graph.has_node(entity_name):
logger.warning(f"Entity '{entity_name}' not found.")
# An absent node with a surviving tracking row is by definition
# an orphan: a previous deletion removed the node but failed
# before its tracking row, or an older release never cleaned up
# at all. Sweeping it here is what makes the staging documented
# in the docstring converge on retry. Only this entity's own row
# can be reached — the node is gone, so its incident edges are
# unknowable.
await _sweep_orphan_tracking_row(
entity_chunks_storage, entity_name, f"entity `{entity_name}`"
)
# Flush whether or not a row was visible. A deferred backend
# whose commit failed during an earlier attempt still holds that
# delete in memory while the stale row sits on disk, so keying
# the flush off in-memory presence would make the failure
# permanent: the retry would see nothing and skip the commit
# that is exactly what is owed. Both stores are flushed because
# the failed attempt may have been an entity deletion that got
# as far as its incident relation rows. These callbacks are
# dirty-gated, so with nothing pending this costs nothing.
await _persist_graph_updates(
entity_chunks_storage=entity_chunks_storage,
relation_chunks_storage=relation_chunks_storage,
)
return DeletionResult(
status="not_found",
doc_id=entity_name,
message=f"Entity '{entity_name}' not found.",
status_code=404,
)
# Retrieve related relationships before deleting the node
edges = await chunk_entity_relation_graph.get_node_edges(entity_name)
related_relations_count = len(edges) if edges else 0
# Resolve the incident relation keys while the edges are still
# readable; the rows themselves are deleted after the node below.
relation_keys_to_delete: list[str] = []
if relation_chunks_storage is not None and edges:
from .utils import make_relation_chunk_key
for src, tgt in edges:
# Normalize entity order for consistent key generation
normalized_src, normalized_tgt = sorted([src, tgt])
relation_keys_to_delete.append(
make_relation_chunk_key(normalized_src, normalized_tgt)
)
await entities_vdb.delete_entity(entity_name)
await relationships_vdb.delete_entity_relation(entity_name)
# PHASE 2 — the node is gone for good; drop its tracking rows.
# Relation rows first: an entity row left behind is reachable by the
# not_found sweep, while an orphaned relation row is not once the
# node is deleted, so the unrecoverable step takes the earliest slot
# and names its keys in the log if it fails.
async def _commit_graph_and_drop_entity_tracking() -> None:
await chunk_entity_relation_graph.delete_node(entity_name)
# PHASE 1 — make the node's removal durable, and nothing else.
# Only a confirmed graph commit is a safe point to touch the
# authoritative tracking rows.
# A `CancelledError` out of this await is NOT caught, and must
# not be. It carries no information about what landed, and the
# two cases it covers want opposite handling:
#
# * cancelled before the write was submitted (waiting for a
# storage-IO permit, e.g. at loop shutdown) -- nothing is
# durable, so deleting the tracking rows would put an
# immediate-write store's row in the grave while the node
# survives on disk: the one state the purge recovery
# contract forbids;
# * cancelled while the write was in flight -- durable, so the
# cleanup is owed.
#
# Reaching the line below is the only reliable evidence that the
# commit landed, so that is what the cleanup is predicated on.
# The caller's own cancellation never surfaces here -- it is
# deferred by the shield in `_finish_deferring_cancellation` and
# re-raised after this whole region returns -- so the case being
# given up is a DIRECT cancellation of this task mid-write. Its
# residue (node gone, relation rows stale) is the documented,
# recoverable one; the alternative is the forbidden one.
await _commit_graph_or_raise(
chunk_entity_relation_graph, f"Entity Delete: '{entity_name}'"
)
if relation_keys_to_delete:
try:
await relation_chunks_storage.delete(relation_keys_to_delete)
except Exception:
logger.error(
"Entity Delete: failed to remove relation chunk tracking; "
f"these rows are now orphaned: {relation_keys_to_delete}"
)
raise
logger.info(
f"Entity Delete: removed chunk tracking for {len(relation_keys_to_delete)} relations"
)
if entity_chunks_storage is not None:
# Delete entity's entry from entity_chunks_storage
await entity_chunks_storage.delete([entity_name])
logger.info(
f"Entity Delete: removed chunk tracking for `{entity_name}`"
)
await _persist_graph_updates(
entity_chunks_storage=entity_chunks_storage,
relation_chunks_storage=relation_chunks_storage,
)
# The graph mutation, its durable commit and the tracking cleanup
# are one cancellation-deferring region. commit_in_storage_io can
# raise a deferred CancelledError from the commit await itself; if
# only phase 2 were protected, execution would never reach it.
await _finish_deferring_cancellation(
_commit_graph_and_drop_entity_tracking(),
f"Entity Delete: '{entity_name}' graph and tracking cleanup",
)
# PHASE 3 — vector stores last. Bundling their flush with the graph
# would let a vector failure abort a deletion whose node is already
# durably gone, stranding tracking rows the sweep cannot reach.
# Their own residue is the window this codebase already accepts
# elsewhere: graph updated, vector storage stale and rebuildable.
await _persist_graph_updates(
entities_vdb=entities_vdb,
relationships_vdb=relationships_vdb,
)
message = f"Entity Delete: remove '{entity_name}' and its {related_relations_count} relations"
logger.info(message)
return DeletionResult(
status="success",
doc_id=entity_name,
message=message,
status_code=200,
)
except Exception as e:
error_message = f"Error while deleting entity '{entity_name}': {e}"
logger.error(error_message)
return DeletionResult(
status="fail",
doc_id=entity_name,
message=error_message,
status_code=500,
)
async def adelete_by_relation(
chunk_entity_relation_graph,
relationships_vdb,
source_entity: str,
target_entity: str,
relation_chunks_storage=None,
) -> DeletionResult:
"""Asynchronously delete a relation between two entities.
Also cleans up relation_chunks_storage to remove chunk tracking.
As in :func:`adelete_by_entity`, the tracking row is removed only after the
edge itself is gone, so a failure can never strand a live relation without
its authoritative provenance; the inverse residue (an orphan row) is swept
by the ``not_found`` branch on the next attempt.
Args:
chunk_entity_relation_graph: Graph storage instance
relationships_vdb: Vector database storage for relationships
source_entity: Name of the source entity
target_entity: Name of the target entity
relation_chunks_storage: Optional KV storage for tracking chunks that reference this relation
"""
relation_str = f"{source_entity} -> {target_entity}"
# Normalize entity order for undirected graph (ensures consistent key generation)
if source_entity > target_entity:
source_entity, target_entity = target_entity, source_entity
# Use keyed lock for relation to ensure atomic graph and vector db operations
workspace = relationships_vdb.global_config.get("workspace", "")
namespace = f"{workspace}:GraphDB" if workspace else "GraphDB"
sorted_edge_key = sorted([source_entity, target_entity])
async with get_storage_keyed_lock(
sorted_edge_key, namespace=namespace, enable_logging=False
):
try:
# Check if the relation exists
edge_exists = await chunk_entity_relation_graph.has_edge(
source_entity, target_entity
)
from .utils import make_relation_chunk_key
# Normalize entity order for consistent key generation
normalized_src, normalized_tgt = sorted([source_entity, target_entity])
storage_key = make_relation_chunk_key(normalized_src, normalized_tgt)
if not edge_exists:
message = f"Relation from '{source_entity}' to '{target_entity}' does not exist"
logger.warning(message)
await _sweep_orphan_tracking_row(
relation_chunks_storage,
storage_key,
f"relation `{normalized_src}`~`{normalized_tgt}`",
)
# Unconditional for the same reason as in adelete_by_entity: a
# retry must be able to commit a delete an earlier attempt left
# pending in memory.
await _persist_graph_updates(
relation_chunks_storage=relation_chunks_storage
)
return DeletionResult(
status="not_found",
doc_id=relation_str,
message=message,
status_code=404,
)
# Delete relation from vector database
rel_ids_to_delete = [
compute_mdhash_id(source_entity + target_entity, prefix="rel-"),
compute_mdhash_id(target_entity + source_entity, prefix="rel-"),
]
await relationships_vdb.delete(rel_ids_to_delete)
async def _commit_graph_and_drop_relation_tracking() -> None:
await chunk_entity_relation_graph.remove_edges(
[(source_entity, target_entity)]
)
# Same ordered phases as adelete_by_entity — see there for why
# the graph commit must be confirmed before tracking is touched.
# Not wrapped in a cancellation catch -- see the entity path
# for why reaching the next line is the only sound evidence that
# the commit landed.
await _commit_graph_or_raise(
chunk_entity_relation_graph,
f"Relation Delete: `{source_entity}`~`{target_entity}`",
)
if relation_chunks_storage is not None:
await relation_chunks_storage.delete([storage_key])
logger.info(
f"Relation Delete: removed chunk tracking for `{source_entity}`~`{target_entity}`"
)
await _persist_graph_updates(
relation_chunks_storage=relation_chunks_storage,
)
# Protect the graph mutation and commit as well as the owed cleanup;
# the commit await itself is where deferred cancellation reappears.
await _finish_deferring_cancellation(
_commit_graph_and_drop_relation_tracking(),
f"Relation Delete: `{source_entity}`~`{target_entity}` graph and tracking cleanup",
)
await _persist_graph_updates(relationships_vdb=relationships_vdb)
message = f"Relation Delete: `{source_entity}`~`{target_entity}` deleted successfully"
logger.info(message)
return DeletionResult(
status="success",
doc_id=relation_str,
message=message,
status_code=200,
)
except Exception as e:
error_message = f"Error while deleting relation from '{source_entity}' to '{target_entity}': {e}"
logger.error(error_message)
return DeletionResult(
status="fail",
doc_id=relation_str,
message=error_message,
status_code=500,
)
async def _edit_entity_impl(
chunk_entity_relation_graph,
entities_vdb,
relationships_vdb,
entity_name: str,
updated_data: dict[str, str],
*,
entity_chunks_storage=None,
relation_chunks_storage=None,
) -> dict[str, Any]:
"""Internal helper that edits an entity without acquiring storage locks.
This function performs the actual entity edit operations without lock management.
It should only be called by public APIs that have already acquired necessary locks.
Args:
chunk_entity_relation_graph: Graph storage instance
entities_vdb: Vector database storage for entities
relationships_vdb: Vector database storage for relationships
entity_name: Name of the entity to edit
updated_data: Dictionary containing updated attributes (including optional entity_name for renaming)
entity_chunks_storage: Optional KV storage for tracking chunks
relation_chunks_storage: Optional KV storage for tracking relation chunks
Returns:
Dictionary containing updated entity information
Note:
Caller must acquire appropriate locks before calling this function.
If renaming (entity_name in updated_data), this function will check if the new name exists.
"""
new_entity_name = updated_data.get("entity_name", entity_name)
is_renaming = new_entity_name != entity_name
original_entity_name = entity_name
node_exists = await chunk_entity_relation_graph.has_node(entity_name)
if not node_exists:
raise ValueError(f"Entity '{entity_name}' does not exist")
node_data = await chunk_entity_relation_graph.get_node(entity_name)
if is_renaming:
existing_node = await chunk_entity_relation_graph.has_node(new_entity_name)
if existing_node:
raise ValueError(
f"Entity name '{new_entity_name}' already exists, cannot rename"
)
new_node_data = {**node_data, **updated_data}
new_node_data["entity_id"] = new_entity_name
if "entity_name" in new_node_data:
del new_node_data[
"entity_name"
] # Node data should not contain entity_name field
if is_renaming:
logger.info(f"Entity Edit: renaming `{entity_name}` to `{new_entity_name}`")
await chunk_entity_relation_graph.upsert_node(new_entity_name, new_node_data)
relations_to_update = []
relations_to_delete = []
edges = await chunk_entity_relation_graph.get_node_edges(entity_name)
if edges:
for source, target in edges:
edge_data = await chunk_entity_relation_graph.get_edge(source, target)
if edge_data:
edge_data = dict(edge_data)
edge_source_id = edge_data.get("source_id") or ""
edge_data["source_id"] = edge_source_id
edge_data["weight"] = apply_relation_weight_floor(
edge_data.get("weight", 1.0), edge_source_id
)
relations_to_delete.append(
compute_mdhash_id(source + target, prefix="rel-")
)
relations_to_delete.append(
compute_mdhash_id(target + source, prefix="rel-")
)
# Rename every matching endpoint. A self-loop has the old
# entity on both sides, so changing only the first match
# would create (new, old), which is then removed together
# with the old node below.
renamed_source = (
new_entity_name if source == entity_name else source
)
renamed_target = (
new_entity_name if target == entity_name else target
)
await chunk_entity_relation_graph.upsert_edge(
renamed_source, renamed_target, edge_data
)
relations_to_update.append(
(renamed_source, renamed_target, edge_data)
)
# The old node is removed inside the cancellation-deferring region
# below, not here. On an immediate-write graph backend (Neo4j, Memgraph,
# MongoDB, PostgreSQL -- their graph `index_done_callback` is a no-op)
# `delete_node` is durable the moment it returns, so removing it at this
# point would leave every await between here and the retirement able to
# exit with the old node gone and its tracking rows still on disk.
# adelete_by_entity stages its own `delete_node` the same way.
old_entity_id = compute_mdhash_id(entity_name, prefix="ent-")
await entities_vdb.delete([old_entity_id])
await relationships_vdb.delete(relations_to_delete)
for src, tgt, edge_data in relations_to_update:
normalized_src, normalized_tgt = sorted([src, tgt])
description = edge_data.get("description", "")
keywords = edge_data.get("keywords", "")
source_id = edge_data.get("source_id", "")
weight = float(edge_data.get("weight", 1.0))
relation_id = compute_mdhash_id(
normalized_src + normalized_tgt, prefix="rel-"
)
# The graph rename cascade above already mutated this edge; a
# truncation failure here is the same "graph updated, VDB
# payload could not be completed" class of failure.
try:
content = _truncate_vdb_content(
f"{normalized_src}\t{normalized_tgt}\n{keywords}\n{description}",
relationships_vdb.global_config,
f"relation:{normalized_src}-{normalized_tgt}",
)
relation_data = {
relation_id: {
"content": content,
"src_id": normalized_src,
"tgt_id": normalized_tgt,
"source_id": source_id,
"description": description,
"keywords": keywords,
"weight": weight,
}
}
await relationships_vdb.upsert(relation_data)
except Exception as e:
raise VectorStorageConsistencyError(
f"Vector storage upsert failed for relation `{normalized_src}`~`{normalized_tgt}` "
f"while renaming entity `{original_entity_name}` to `{new_entity_name}`: {e}. "
"The knowledge graph was already updated, so it may now be inconsistent "
"with the vector storage. No data is lost (the graph is the authoritative "
"source). Stop the LightRAG server and run the offline rebuild tool "
"(lightrag-rebuild-vdb) to restore consistency."
) from e
entity_name = new_entity_name
else:
await chunk_entity_relation_graph.upsert_node(entity_name, new_node_data)
description = new_node_data.get("description", "")
source_id = new_node_data.get("source_id", "")
entity_type = new_node_data.get("entity_type", "")
entity_id = compute_mdhash_id(entity_name, prefix="ent-")
# The graph node was already updated above; a truncation failure here is
# the same "graph updated, VDB payload could not be completed" class of
# failure as an upsert failure.
try:
content = _truncate_vdb_content(
entity_name + "\n" + description,
entities_vdb.global_config,
f"entity:{entity_name}",
)
entity_data = {
entity_id: {
"content": content,
"entity_name": entity_name,
"source_id": source_id,
"description": description,
"entity_type": entity_type,
}
}
await entities_vdb.upsert(entity_data)
except Exception as e:
raise VectorStorageConsistencyError(
f"Vector storage upsert failed for entity `{entity_name}` during entity edit: "
f"{e}. The knowledge graph was already updated, so it may now be inconsistent "
"with the vector storage. No data is lost (the graph is the authoritative "
"source). Stop the LightRAG server and run the offline rebuild tool "
"(lightrag-rebuild-vdb) to restore consistency."
) from e
# Old keys whose rows may only be retired once the graph state that
# replaced them is durable -- see the retirement block below.
tracking_keys_to_retire: list[tuple[Any, str]] = []
if entity_chunks_storage is not None or relation_chunks_storage is not None:
from .utils import (
make_relation_chunk_key,
compute_incremental_chunk_ids,
has_chunk_tracking_row,
)
if entity_chunks_storage is not None:
storage_key = original_entity_name if is_renaming else entity_name
stored_data = await entity_chunks_storage.get_by_id(storage_key)
# Row presence by schema, never list truthiness — see
# has_chunk_tracking_row for the authority model.
has_stored_row = has_chunk_tracking_row(stored_data)
old_source_id = node_data.get("source_id", "")
old_chunk_ids = [cid for cid in old_source_id.split(GRAPH_FIELD_SEP) if cid]
new_source_id = new_node_data.get("source_id", "")
new_chunk_ids = [cid for cid in new_source_id.split(GRAPH_FIELD_SEP) if cid]
source_id_changed = set(new_chunk_ids) != set(old_chunk_ids)
if source_id_changed or not has_stored_row or is_renaming:
existing_full_chunk_ids = []
if has_stored_row:
existing_full_chunk_ids = [
cid for cid in stored_data.get("chunk_ids", []) if cid
]
# Reseed from the graph's source_id only when no tracking row exists at
# all; a present-but-empty row must not be repopulated from a possibly
# stale source_id.
if not has_stored_row:
existing_full_chunk_ids = old_chunk_ids.copy()
updated_chunk_ids = compute_incremental_chunk_ids(
existing_full_chunk_ids, old_chunk_ids, new_chunk_ids
)
# On rename, write the new key BEFORE deleting the old one: on
# RPC-backed KV storages each call commits independently, so the
# reverse order opens a crash window in which the row exists under
# neither key — turning a curated row absent and re-arming the
# stale reseed. An orphaned old-key row is dead bookkeeping; a
# lost row is the bug.
await entity_chunks_storage.upsert(
{
entity_name: {
"chunk_ids": updated_chunk_ids,
"count": len(updated_chunk_ids),
}
}
)
if is_renaming:
# Retired below, after the graph commit that actually
# removes the old node. Deleting it here would strip the
# provenance of an entity still on disk.
tracking_keys_to_retire.append(
(entity_chunks_storage, original_entity_name)
)
logger.info(
f"Entity Edit: find {len(updated_chunk_ids)} chunks related to `{entity_name}`"
)
if is_renaming and relation_chunks_storage is not None and relations_to_update:
for src, tgt, edge_data in relations_to_update:
old_src = original_entity_name if src == entity_name else src
old_tgt = original_entity_name if tgt == entity_name else tgt
old_normalized_src, old_normalized_tgt = sorted([old_src, old_tgt])
new_normalized_src, new_normalized_tgt = sorted([src, tgt])
old_storage_key = make_relation_chunk_key(
old_normalized_src, old_normalized_tgt
)
new_storage_key = make_relation_chunk_key(
new_normalized_src, new_normalized_tgt
)
if old_storage_key != new_storage_key:
old_stored_data = await relation_chunks_storage.get_by_id(
old_storage_key
)
# Row presence by schema (see has_chunk_tracking_row): a
# curated row — including an empty one — is authoritative and
# must be migrated as-is; an absent or legacy/partial shape is
# unknown and reseeded from the edge's source_id.
old_row_present = has_chunk_tracking_row(old_stored_data)
relation_chunk_ids = []
if old_row_present:
relation_chunk_ids = [
cid
for cid in old_stored_data.get("chunk_ids", [])
if cid and cid not in RELATION_NO_EVIDENCE_SOURCE_IDS
]
else:
relation_source_id = edge_data.get("source_id", "")
relation_chunk_ids = relation_evidence_source_ids(
relation_source_id
)
# Migrate the row to the new key. A present-but-empty row is
# authoritative ("this relation tracks no chunks") and must
# survive the rename; dropping it would make the row absent and
# cause the next relation edit to reseed it from a possibly-stale
# source_id — the exact bug this PR fixes, re-armed at rename.
# Only skip the write when the row was absent AND source_id
# yielded nothing to seed from. Upsert the new key BEFORE
# deleting the old one: on RPC-backed KV storages each call
# commits independently, and the reverse order opens a crash
# window in which the row exists under neither key — losing the
# curated row this migration exists to preserve.
if old_row_present or relation_chunk_ids:
await relation_chunks_storage.upsert(
{
new_storage_key: {
"chunk_ids": relation_chunk_ids,
"count": len(relation_chunk_ids),
}
}
)
tracking_keys_to_retire.append(
(relation_chunks_storage, old_storage_key)
)
logger.info(
f"Entity Edit: migrate {len(relations_to_update)} relations after rename"
)
# Commit the graph on its own, and confirm it, before retiring any old
# tracking key. A rename removes the old node and republishes its edges
# under new keys; until that is durable the OLD objects are what is on disk,
# and their rows are the only authoritative provenance those objects have.
# A declined commit discards the rename entirely, so it must not be read as
# success either -- same staging, and same reason, as adelete_by_entity.
#
# The migrated rows are written before this point on purpose: that ordering
# is the fix for the opposite failure (#3609), where a crash left the row
# under neither key. An orphaned new-key row is dead bookkeeping a retry
# overwrites; a live object with no row is the bug.
# Same staging as the merge: the migrated rows have to be on disk before the
# commit that removes the objects the old rows describe. A deferred KV
# backend keeps them in shared memory until this flush, so a process exit
# between the commit and it would restart with the renamed graph and only
# the old keys' rows -- the surviving node and its relations with no
# authoritative tracking at all.
if is_renaming:
try:
await _persist_graph_updates(
entity_chunks_storage=entity_chunks_storage,
relation_chunks_storage=relation_chunks_storage,
)
except Exception as e:
raise VectorStorageConsistencyError(
f"Persisting the migrated chunk tracking failed while renaming "
f"`{original_entity_name}` to `{new_entity_name}`: {e}. The old "
"node was NOT removed and still carries its chunk tracking, so "
"nothing has lost its provenance. Retry the rename."
) from e
async def _commit_rename_and_retire_tracking() -> None:
# `entity_name` has been rebound to the new name by now, so the removal
# names the original explicitly.
if is_renaming:
await chunk_entity_relation_graph.delete_node(original_entity_name)
await _commit_graph_or_raise(
chunk_entity_relation_graph, f"Entity Edit: `{original_entity_name}`"
)
for storage, key in tracking_keys_to_retire:
try:
await storage.delete([key])
except Exception as e:
orphaned = [k for _, k in tracking_keys_to_retire]
logger.error(
"Entity Edit: failed to retire chunk tracking after renaming "
f"`{original_entity_name}`; these rows are now orphaned: "
f"{orphaned}"
)
# Typed for the same reason as the merge path: the rename is
# already durable here, so a failure that reads as "the rename
# did not happen" would be a lie about the graph.
raise VectorStorageConsistencyError(
f"Retiring the chunk tracking of `{original_entity_name}` failed "
f"after renaming it to `{new_entity_name}`: {e}. The rename "
"itself is durable -- the old node is gone and its relations "
f"were republished -- but these rows survive as orphans: "
f"{orphaned}. They describe objects that no longer exist and are "
"dead bookkeeping unless the same keys reappear later, at which "
"point extraction would read them back as authoritative "
"provenance."
) from e
# Flushed inside the region: on a deferred KV backend the deletes above
# only touch memory, so a cancellation delivered before this flush would
# leave the retired rows on disk -- the orphan the region prevents.
await _persist_graph_updates(
entity_chunks_storage=entity_chunks_storage,
relation_chunks_storage=relation_chunks_storage,
)
# One cancellation-deferring region, as in adelete_by_entity: after the
# commit the old node is gone for good, so its rows describe nothing and the
# cleanup is owed; a cancellation delivered mid-cleanup would strand them.
# The region starts before the commit because that await is itself where a
# deferred cancellation reappears.
await _finish_deferring_cancellation(
_commit_rename_and_retire_tracking(),
f"Entity Edit: `{original_entity_name}` graph and tracking cleanup",
)
# Vector stores last: their residue is the rebuildable window this codebase
# accepts elsewhere, and bundling them earlier would let a vector failure
# abort an edit whose graph state is already durable.
await _persist_graph_updates(
entities_vdb=entities_vdb,
relationships_vdb=relationships_vdb,
)
logger.info(f"Entity Edit: `{entity_name}` successfully updated")
return await get_entity_info(
chunk_entity_relation_graph,
entities_vdb,
entity_name,
include_vector_data=False,
)
async def aedit_entity(
chunk_entity_relation_graph,
entities_vdb,
relationships_vdb,
entity_name: str,
updated_data: dict[str, str],
allow_rename: bool = True,
allow_merge: bool = False,
entity_chunks_storage=None,
relation_chunks_storage=None,
) -> dict[str, Any]:
"""Asynchronously edit entity information.
Updates entity information in the knowledge graph and re-embeds the entity in the vector database.
Also synchronizes entity_chunks_storage and relation_chunks_storage to track chunk references.
Args:
chunk_entity_relation_graph: Graph storage instance
entities_vdb: Vector database storage for entities
relationships_vdb: Vector database storage for relationships
entity_name: Name of the entity to edit
updated_data: Attributes to update. Allowed fields: ``entity_name``
(rename target), ``entity_type``, ``description``,
``source_id``, ``file_path`` -- all strings. Any other key,
or a value that is not a storable string, is rejected with
``ValueError``.
allow_rename: Whether to allow entity renaming, defaults to True
allow_merge: Whether to merge into an existing entity when renaming to an existing name, defaults to False
entity_chunks_storage: Optional KV storage for tracking chunks that reference this entity
relation_chunks_storage: Optional KV storage for tracking chunks that reference relations
Returns:
Dictionary containing updated entity information and operation summary with the following structure:
{
"entity_name": str, # Name of the entity
"description": str, # Entity description
"entity_type": str, # Entity type
"source_id": str, # Source chunk IDs
... # Other entity properties
"operation_summary": {
"merged": bool, # Whether entity was merged
"merge_status": str, # "success" | "failed" | "not_attempted"
"merge_error": str | None, # Error message if merge failed
"operation_status": str, # "success" | "partial_success" | "failure"
"target_entity": str | None, # Target entity name if renaming/merging
"final_entity": str, # Final entity name after operation
"renamed": bool # Whether entity was renamed
}
}
operation_status values:
- "success": Operation completed successfully (update/rename/merge all succeeded)
- "partial_success": Non-name updates succeeded but merge failed
- "failure": Operation failed completely
merge_status values:
- "success": Entity successfully merged into target
- "failed": Merge operation failed
- "not_attempted": No merge was attempted (normal update/rename)
"""
# Order matters: the empty-description check runs first so a `None`
# description keeps reporting itself as empty (which is what it means to a
# caller) rather than as a type error. Both refuse before any storage is
# touched, so the ordering is about the message, not about safety.
if "description" in updated_data:
_require_non_empty_description(
updated_data.get("description"), operation="edit", object_type="entity"
)
# Reduce to allowed, well-typed fields before anything else: this runs
# outside the storage lock and before the first graph read, so a rejected
# payload has touched nothing at all.
updated_data = _sanitize_graph_fields(
updated_data,
allowed_fields=_EDITABLE_ENTITY_FIELDS,
object_type="entity",
reject_unknown=True,
)
requested_entity_name = entity_name
normalized_entity_name = _normalize_manual_entity_name(requested_entity_name)
has_new_entity_name = "entity_name" in updated_data
requested_new_entity_name = updated_data.get("entity_name")
normalized_new_entity_name = (
_normalize_manual_entity_name(requested_new_entity_name)
if has_new_entity_name
else None
)
# Lock every exact/canonical source and target candidate before resolving
# legacy manual names. The doc-ingest pipeline acquires
# edge locks as sorted([src, tgt]) in the same namespace, and
# get_storage_keyed_lock takes one mutex per key — so locking the
# entity name already mutually excludes any concurrent edge write that
# touches it, no need to enumerate incident edges here.
lock_keys = {requested_entity_name}
if normalized_entity_name:
lock_keys.add(normalized_entity_name)
if has_new_entity_name:
lock_keys.add(requested_new_entity_name)
if normalized_new_entity_name:
lock_keys.add(normalized_new_entity_name)
workspace = entities_vdb.global_config.get("workspace", "")
namespace = f"{workspace}:GraphDB" if workspace else "GraphDB"
async with get_storage_keyed_lock(
sorted(lock_keys), namespace=namespace, enable_logging=False
):
try:
# Prefer an exact legacy key when it exists. Otherwise resolve the
# caller's spelling to the extraction-normalized identifier.
if (
requested_entity_name != normalized_entity_name
and await chunk_entity_relation_graph.has_node(requested_entity_name)
):
entity_name = requested_entity_name
elif normalized_entity_name:
entity_name = normalized_entity_name
else:
raise ValueError("Entity name cannot be empty after normalization")
if has_new_entity_name:
if (
requested_new_entity_name != normalized_new_entity_name
and await chunk_entity_relation_graph.has_node(
requested_new_entity_name
)
):
new_entity_name = requested_new_entity_name
elif normalized_new_entity_name:
new_entity_name = normalized_new_entity_name
else:
raise ValueError("Entity name cannot be empty after normalization")
updated_data["entity_name"] = new_entity_name
else:
new_entity_name = entity_name
is_renaming = new_entity_name != entity_name
operation_summary: dict[str, Any] = {
"merged": False,
"merge_status": "not_attempted",
"merge_error": None,
"operation_status": "success",
"target_entity": None,
"final_entity": new_entity_name if is_renaming else entity_name,
"renamed": is_renaming,
}
if is_renaming and not allow_rename:
raise ValueError(
"Entity renaming is not allowed. Set allow_rename=True to enable this feature"
)
if is_renaming:
target_exists = await chunk_entity_relation_graph.has_node(
new_entity_name
)
if target_exists:
if not allow_merge:
raise ValueError(
f"Entity name '{new_entity_name}' already exists, cannot rename"
)
logger.info(
f"Entity Edit: `{entity_name}` will be merged into `{new_entity_name}`"
)
# Track whether non-name updates were applied
non_name_updates_applied = False
non_name_updates = {
key: value
for key, value in updated_data.items()
if key != "entity_name"
}
# Apply non-name updates first
if non_name_updates:
try:
logger.info(
"Entity Edit: applying non-name updates before merge"
)
await _edit_entity_impl(
chunk_entity_relation_graph,
entities_vdb,
relationships_vdb,
entity_name,
non_name_updates,
entity_chunks_storage=entity_chunks_storage,
relation_chunks_storage=relation_chunks_storage,
)
non_name_updates_applied = True
except Exception as update_error:
# If update fails, re-raise immediately
logger.error(
f"Entity Edit: non-name updates failed: {update_error}"
)
raise
# Attempt to merge entities
try:
merge_result = await _merge_entities_impl(
chunk_entity_relation_graph,
entities_vdb,
relationships_vdb,
[entity_name],
new_entity_name,
merge_strategy=None,
target_entity_data=None,
entity_chunks_storage=entity_chunks_storage,
relation_chunks_storage=relation_chunks_storage,
)
# Merge succeeded
operation_summary.update(
{
"merged": True,
"merge_status": "success",
"merge_error": None,
"operation_status": "success",
"target_entity": new_entity_name,
"final_entity": new_entity_name,
}
)
return {**merge_result, "operation_summary": operation_summary}
except VectorStorageConsistencyError:
# Fail-loud: the merge is durably applied to the graph and
# a step after it -- the vector storage write, or the
# chunk-tracking retirement -- did not complete. This must
# reach the caller (mapped to a 500 by the route), NOT be
# folded into the partial-success summary below, which
# answers HTTP 200 and names `entity_name` as the surviving
# entity: the very source this merge has just removed.
logger.error(
f"Entity Edit: merge of '{entity_name}' into "
f"'{new_entity_name}' is durable but a following step "
"failed; re-raising VectorStorageConsistencyError"
)
raise
except Exception as merge_error:
# Merge failed, but update may have succeeded
logger.error(f"Entity Edit: merge failed: {merge_error}")
# Return partial success status (update succeeded but merge failed)
operation_summary.update(
{
"merged": False,
"merge_status": "failed",
"merge_error": str(merge_error),
"operation_status": "partial_success"
if non_name_updates_applied
else "failure",
"target_entity": new_entity_name,
"final_entity": entity_name, # Keep source entity name
}
)
# Get current entity info (with applied updates if any)
entity_info = await get_entity_info(
chunk_entity_relation_graph,
entities_vdb,
entity_name,
include_vector_data=False,
)
return {**entity_info, "operation_summary": operation_summary}
# Normal edit flow (no merge involved)
edit_result = await _edit_entity_impl(
chunk_entity_relation_graph,
entities_vdb,
relationships_vdb,
entity_name,
updated_data,
entity_chunks_storage=entity_chunks_storage,
relation_chunks_storage=relation_chunks_storage,
)
operation_summary["operation_status"] = "success"
return {**edit_result, "operation_summary": operation_summary}
except Exception as e:
logger.error(f"Error while editing entity '{entity_name}': {e}")
raise
async def aedit_relation(
chunk_entity_relation_graph,
entities_vdb,
relationships_vdb,
source_entity: str,
target_entity: str,
updated_data: dict[str, Any],
relation_chunks_storage=None,
) -> dict[str, Any]:
"""Asynchronously edit relation information.
Updates relation (edge) information in the knowledge graph and re-embeds the relation in the vector database.
Also synchronizes the relation_chunks_storage to track which chunks reference this relation.
Args:
chunk_entity_relation_graph: Graph storage instance
entities_vdb: Vector database storage for entities
relationships_vdb: Vector database storage for relationships
source_entity: Name of the source entity
target_entity: Name of the target entity
updated_data: Attributes to update. Allowed fields:
``description``, ``keywords``, ``source_id``, ``file_path``
(strings) and ``weight`` (number). Any other key, or a value
of the wrong shape, is rejected with ``ValueError``. The complete
updated relation must satisfy ``weight >=`` its number of distinct
real ``source_id`` values. Set ``source_id`` to an empty string in
the same edit when assigning a fractional weight below the
previous evidence count.
relation_chunks_storage: Optional KV storage for tracking chunks that reference this relation
Returns:
Dictionary containing updated relation information
"""
# See `aedit_entity` for the ordering rationale.
if "description" in updated_data:
_require_non_empty_description(
updated_data.get("description"), operation="edit", object_type="relation"
)
updated_data = _sanitize_graph_fields(
updated_data,
allowed_fields=_RELATION_DATA_FIELDS,
object_type="relation",
reject_unknown=True,
)
# Normalize entity order for undirected graph (ensures consistent key generation)
if source_entity > target_entity:
source_entity, target_entity = target_entity, source_entity
# Use keyed lock for relation to ensure atomic graph and vector db operations
workspace = relationships_vdb.global_config.get("workspace", "")
namespace = f"{workspace}:GraphDB" if workspace else "GraphDB"
sorted_edge_key = sorted([source_entity, target_entity])
async with get_storage_keyed_lock(
sorted_edge_key, namespace=namespace, enable_logging=False
):
try:
# 1. Get current relation information
edge_exists = await chunk_entity_relation_graph.has_edge(
source_entity, target_entity
)
if not edge_exists:
raise ValueError(
f"Relation from '{source_entity}' to '{target_entity}' does not exist"
)
edge_data = await chunk_entity_relation_graph.get_edge(
source_entity, target_entity
)
# 2. Recalculate relation's vector representation. Construct and
# verify the VDB payload BEFORE any mutation below: if truncation
# fails (a deterministic, non-retryable content-shape problem),
# nothing has been touched yet. The actual VDB delete/upsert I/O
# calls happen strictly after the graph write below, so a
# transient VDB I/O failure (unlike a truncation failure) still
# leaves the normal recoverable (graph-updated, VDB-stale)
# window -- the new relation content is never lost.
new_edge_data = {**edge_data, **updated_data}
description = new_edge_data.get("description", "")
keywords = new_edge_data.get("keywords", "")
source_id = new_edge_data.get("source_id") or ""
if "weight" in updated_data or "source_id" in updated_data:
weight = validate_relation_weight(
new_edge_data.get("weight", 1.0),
source_id,
context=f"Relation `{source_entity}`~`{target_entity}`",
)
else:
# An unrelated edit is also a safe repair point for a legacy
# row that predates the evidence-floor contract.
weight = apply_relation_weight_floor(
new_edge_data.get("weight", 1.0), source_id
)
new_edge_data["source_id"] = source_id
new_edge_data["weight"] = weight
content = _truncate_vdb_content(
f"{source_entity}\t{target_entity}\n{keywords}\n{description}",
relationships_vdb.global_config,
f"relation:{source_entity}-{target_entity}",
)
relation_id = compute_mdhash_id(
source_entity + target_entity, prefix="rel-"
)
relation_data = {
relation_id: {
"content": content,
"src_id": source_entity,
"tgt_id": target_entity,
"source_id": source_id,
"description": description,
"keywords": keywords,
"weight": weight,
}
}
# 3. Update relation information in the graph
await chunk_entity_relation_graph.upsert_edge(
source_entity, target_entity, new_edge_data
)
# Delete the old relation record from the vector database.
# Delete both permutations to handle relationships created
# before normalization.
rel_ids_to_delete = [
compute_mdhash_id(source_entity + target_entity, prefix="rel-"),
compute_mdhash_id(target_entity + source_entity, prefix="rel-"),
]
await relationships_vdb.delete(rel_ids_to_delete)
logger.debug(
f"Relation Delete: delete vdb for `{source_entity}`~`{target_entity}`"
)
# Update vector database
await relationships_vdb.upsert(relation_data)
# 4. Synchronize relation chunk tracking after source edits, when
# tracking is missing, or when a legacy row still contains a
# historical no-evidence placeholder.
if relation_chunks_storage is not None:
from .utils import (
make_relation_chunk_key,
compute_incremental_chunk_ids,
has_chunk_tracking_row,
)
storage_key = make_relation_chunk_key(source_entity, target_entity)
# Check if storage has existing data
stored_data = await relation_chunks_storage.get_by_id(storage_key)
# Row presence by schema, never list truthiness — see
# has_chunk_tracking_row for the authority model.
has_stored_row = has_chunk_tracking_row(stored_data)
stored_chunk_ids = (
[cid for cid in stored_data["chunk_ids"] if cid]
if has_stored_row
else []
)
tracking_has_legacy_placeholder = any(
cid in RELATION_NO_EVIDENCE_SOURCE_IDS for cid in stored_chunk_ids
)
# Get old and new source_id
old_source_id = edge_data.get("source_id", "")
old_chunk_ids = relation_evidence_source_ids(old_source_id)
new_source_id = new_edge_data.get("source_id", "")
new_chunk_ids = relation_evidence_source_ids(new_source_id)
source_id_changed = set(new_chunk_ids) != set(old_chunk_ids)
raw_source_id_changed = new_source_id != old_source_id
# Update if: the evidence set or the raw source_id changed, no
# tracking row exists, or a legacy row still carries a
# no-evidence placeholder this edit can repair.
if (
source_id_changed
or raw_source_id_changed
or tracking_has_legacy_placeholder
or not has_stored_row
):
# Get existing full chunk_ids from storage
existing_full_chunk_ids = []
if has_stored_row:
existing_full_chunk_ids = [
cid
for cid in stored_chunk_ids
if cid and cid not in RELATION_NO_EVIDENCE_SOURCE_IDS
]
# Reseed from the graph's source_id only when no tracking row exists
# at all; a present-but-empty row must not be repopulated from a
# possibly stale source_id.
if not has_stored_row:
existing_full_chunk_ids = old_chunk_ids.copy()
# Use utility function to compute incremental updates
updated_chunk_ids = compute_incremental_chunk_ids(
existing_full_chunk_ids, old_chunk_ids, new_chunk_ids
)
# Update storage (Update even if updated_chunk_ids is empty)
await relation_chunks_storage.upsert(
{
storage_key: {
"chunk_ids": updated_chunk_ids,
"count": len(updated_chunk_ids),
}
}
)
logger.info(
f"Relation Delete: update chunk tracking for `{source_entity}`~`{target_entity}`"
)
# 5. Save changes
await _persist_graph_updates(
relationships_vdb=relationships_vdb,
chunk_entity_relation_graph=chunk_entity_relation_graph,
relation_chunks_storage=relation_chunks_storage,
)
logger.info(
f"Relation Delete: `{source_entity}`~`{target_entity}`' successfully updated"
)
return await get_relation_info(
chunk_entity_relation_graph,
relationships_vdb,
source_entity,
target_entity,
include_vector_data=False,
)
except Exception as e:
logger.error(
f"Error while editing relation from '{source_entity}' to '{target_entity}': {e}"
)
raise
async def acreate_entity(
chunk_entity_relation_graph,
entities_vdb,
relationships_vdb,
entity_name: str,
entity_data: dict[str, Any],
entity_chunks_storage=None,
relation_chunks_storage=None,
*,
before_create: Callable[[], Awaitable[None]] | None = None,
) -> dict[str, Any]:
"""Asynchronously create a new entity.
Creates a new entity in the knowledge graph and adds it to the vector database.
Replaces entity chunk tracking with the distinct real source IDs supplied
for this creation, or an explicit empty row when there are none. Historical
no-source placeholders do not count as evidence.
Manual creation does not protect the entity or its description from deletion:
once documents mention it, purging its last real source may delete it entirely.
Args:
chunk_entity_relation_graph: Graph storage instance
entities_vdb: Vector database storage for entities
relationships_vdb: Vector database storage for relationships
entity_name: Name of the new entity
entity_data: Dictionary containing entity attributes, e.g. {"description": "description", "entity_type": "type"}
entity_chunks_storage: Optional KV storage for tracking chunks that reference this entity
relation_chunks_storage: Optional KV storage for tracking chunks that reference relations
before_create: Optional migration hook, awaited after validation and before writes
Returns:
Dictionary containing created entity information
"""
_require_non_empty_description(
entity_data.get("description"), operation="create", object_type="entity"
)
# The named-field copy below stops an unknown *key* from reaching storage,
# but not an unstorable *value* on a known one: a non-scalar `entity_type`
# reaches `upsert_node` unexamined. Type-check the recognised fields (extra
# keys stay ignored, as they always were -- see `_sanitize_graph_fields`).
entity_data = _sanitize_graph_fields(
entity_data,
allowed_fields=_ENTITY_DATA_FIELDS,
object_type="entity",
reject_unknown=False,
)
requested_entity_name = entity_name
entity_name = _normalize_manual_entity_name(requested_entity_name)
if not entity_name:
raise ValueError("Entity name cannot be empty after normalization")
# Lock both spellings so an existing pre-normalization manual entity cannot
# race a canonical create and become a semantic duplicate.
workspace = entities_vdb.global_config.get("workspace", "")
namespace = f"{workspace}:GraphDB" if workspace else "GraphDB"
async with get_storage_keyed_lock(
sorted({requested_entity_name, entity_name}),
namespace=namespace,
enable_logging=False,
):
try:
if (
requested_entity_name != entity_name
and await chunk_entity_relation_graph.has_node(requested_entity_name)
):
raise ValueError(f"Entity '{requested_entity_name}' already exists")
# Check if the normalized entity already exists.
existing_node = await chunk_entity_relation_graph.has_node(entity_name)
if existing_node:
raise ValueError(f"Entity '{entity_name}' already exists")
# Prepare node data with defaults if missing
node_data = {
"entity_id": entity_name,
"entity_type": entity_data.get("entity_type", "UNKNOWN"),
"description": entity_data.get("description", ""),
"source_id": entity_data.get("source_id", "manual_creation"),
"file_path": entity_data.get("file_path", "manual_creation"),
"created_at": int(time.time()),
}
# Prepare content for entity. Construct and verify the VDB
# payload BEFORE the first graph mutation below: if truncation
# fails (a deterministic, non-retryable content-shape problem
# that a rebuild would hit identically), nothing has been
# written yet. The actual VDB I/O call still happens after the
# graph write below -- only this cheap verification step is
# front-loaded, so a transient VDB I/O failure still leaves a
# recoverable (graph updated, VDB stale) window.
description = node_data.get("description", "")
source_id = node_data.get("source_id", "")
entity_type = node_data.get("entity_type", "")
content = _truncate_vdb_content(
entity_name + "\n" + description,
entities_vdb.global_config,
f"entity:{entity_name}",
)
# Calculate entity ID
entity_id = compute_mdhash_id(entity_name, prefix="ent-")
# Prepare data for vector database update
entity_data_for_vdb = {
entity_id: {
"content": content,
"entity_name": entity_name,
"source_id": source_id,
"description": description,
"entity_type": entity_type,
"file_path": entity_data.get("file_path", "manual_creation"),
}
}
if before_create is not None:
await before_create()
# Add entity to knowledge graph
await chunk_entity_relation_graph.upsert_node(entity_name, node_data)
# Update vector database
await entities_vdb.upsert(entity_data_for_vdb)
# Update entity_chunks_storage to track chunk references
if entity_chunks_storage is not None:
source_id = node_data.get("source_id", "")
chunk_ids = list(
dict.fromkeys(
cid
for cid in source_id.split(GRAPH_FIELD_SEP)
if cid and cid not in RELATION_NO_EVIDENCE_SOURCE_IDS
)
)
# Explicit creation starts new attribution, never inherits an
# orphan row from a previously deleted object (issue #3838).
await entity_chunks_storage.upsert(
{
entity_name: {
"chunk_ids": chunk_ids,
"count": len(chunk_ids),
}
}
)
logger.info(
f"Entity Create: tracked {len(chunk_ids)} chunks for `{entity_name}`"
)
# Save changes
await _persist_graph_updates(
entities_vdb=entities_vdb,
relationships_vdb=relationships_vdb,
chunk_entity_relation_graph=chunk_entity_relation_graph,
entity_chunks_storage=entity_chunks_storage,
relation_chunks_storage=relation_chunks_storage,
)
logger.info(f"Entity Create: '{entity_name}' successfully created")
return await get_entity_info(
chunk_entity_relation_graph,
entities_vdb,
entity_name,
include_vector_data=False,
)
except Exception as e:
logger.error(f"Error while creating entity '{entity_name}': {e}")
raise
async def acreate_relation(
chunk_entity_relation_graph,
entities_vdb,
relationships_vdb,
source_entity: str,
target_entity: str,
relation_data: dict[str, Any],
relation_chunks_storage=None,
*,
before_create: Callable[[], Awaitable[None]] | None = None,
) -> dict[str, Any]:
"""Asynchronously create a new relation between entities.
Creates a new relation (edge) in the knowledge graph and adds it to the vector database.
Replaces relation chunk tracking with the distinct real source IDs supplied
for this creation, or an explicit empty row when there are none. Orphan rows
from a previous relation are never inherited.
Manual creation does not protect the relation or its description from deletion:
once documents mention it, purging its last real source may delete it entirely.
Args:
chunk_entity_relation_graph: Graph storage instance
entities_vdb: Vector database storage for entities
relationships_vdb: Vector database storage for relationships
source_entity: Name of the source entity
target_entity: Name of the target entity
relation_data: Dictionary containing relation attributes. ``weight``
defaults to 1.0 and must be at least the number of distinct real
values in ``source_id``. An omitted ``source_id`` is stored as an
empty string and permits a non-negative fractional weight.
relation_chunks_storage: Optional KV storage for tracking chunks that reference this relation
before_create: Optional migration hook, awaited after validation and before writes
Returns:
Dictionary containing created relation information
"""
_require_non_empty_description(
relation_data.get("description"), operation="create", object_type="relation"
)
# The graph edge below is written with these exact strings, so comparing
# them raw is precisely the condition that would produce a self-loop.
_reject_self_loop_relation(source_entity, target_entity, operation="create")
# Same reasoning as `acreate_entity`: values of the recognised fields are
# type-checked, unrecognised keys stay ignored.
relation_data = _sanitize_graph_fields(
relation_data,
allowed_fields=_RELATION_DATA_FIELDS,
object_type="relation",
reject_unknown=False,
)
# Use keyed lock for relation to ensure atomic graph and vector db operations
workspace = relationships_vdb.global_config.get("workspace", "")
namespace = f"{workspace}:GraphDB" if workspace else "GraphDB"
sorted_edge_key = sorted([source_entity, target_entity])
async with get_storage_keyed_lock(
sorted_edge_key, namespace=namespace, enable_logging=False
):
try:
# Check if both entities exist
source_exists = await chunk_entity_relation_graph.has_node(source_entity)
target_exists = await chunk_entity_relation_graph.has_node(target_entity)
if not source_exists:
raise ValueError(f"Source entity '{source_entity}' does not exist")
if not target_exists:
raise ValueError(f"Target entity '{target_entity}' does not exist")
# Check if relation already exists
existing_edge = await chunk_entity_relation_graph.has_edge(
source_entity, target_entity
)
if existing_edge:
raise ValueError(
f"Relation from '{source_entity}' to '{target_entity}' already exists"
)
source_id = relation_data.get("source_id", "")
weight = validate_relation_weight(
relation_data.get("weight", 1.0),
source_id,
context=f"Relation `{source_entity}`~`{target_entity}`",
)
# Prepare edge data with defaults if missing. An omitted source is
# genuinely source-less; do not synthesize an evidence ID.
edge_data = {
"description": relation_data.get("description", ""),
"keywords": relation_data.get("keywords", ""),
"source_id": source_id,
"weight": weight,
"file_path": relation_data.get("file_path", "manual_creation"),
"created_at": int(time.time()),
}
# Normalize entity order for the VDB record's identity — kept
# separate from source_entity/target_entity used for the graph
# edge write below, which must use the caller's original
# direction.
vdb_src, vdb_tgt = (
(target_entity, source_entity)
if source_entity > target_entity
else (source_entity, target_entity)
)
# Prepare content for embedding
description = edge_data.get("description", "")
keywords = edge_data.get("keywords", "")
source_id = edge_data["source_id"]
weight = edge_data["weight"]
# Construct and verify the VDB payload BEFORE the first graph
# mutation below: if truncation fails (a deterministic,
# non-retryable content-shape problem), nothing has been
# written yet. The actual VDB I/O call still happens after the
# graph write below.
content = _truncate_vdb_content(
f"{keywords}\t{vdb_src}\n{vdb_tgt}\n{description}",
relationships_vdb.global_config,
f"relation:{vdb_src}-{vdb_tgt}",
)
# Calculate relation ID
relation_id = compute_mdhash_id(vdb_src + vdb_tgt, prefix="rel-")
# Prepare data for vector database update
relation_data_for_vdb = {
relation_id: {
"content": content,
"src_id": vdb_src,
"tgt_id": vdb_tgt,
"source_id": source_id,
"description": description,
"keywords": keywords,
"weight": weight,
"file_path": relation_data.get("file_path", "manual_creation"),
}
}
if before_create is not None:
await before_create()
# Add relation to knowledge graph
await chunk_entity_relation_graph.upsert_edge(
source_entity, target_entity, edge_data
)
# Update vector database
await relationships_vdb.upsert(relation_data_for_vdb)
# Update relation_chunks_storage to track chunk references
if relation_chunks_storage is not None:
from .utils import make_relation_chunk_key
storage_key = make_relation_chunk_key(vdb_src, vdb_tgt)
source_id = edge_data.get("source_id", "")
chunk_ids = relation_evidence_source_ids(source_id)
# Explicit creation starts new attribution, never inherits an
# orphan row from a previously deleted object (issue #3838).
await relation_chunks_storage.upsert(
{
storage_key: {
"chunk_ids": chunk_ids,
"count": len(chunk_ids),
}
}
)
logger.info(
f"Relation Create: tracked {len(chunk_ids)} chunks for `{vdb_src}`~`{vdb_tgt}`"
)
# Save changes
await _persist_graph_updates(
relationships_vdb=relationships_vdb,
chunk_entity_relation_graph=chunk_entity_relation_graph,
relation_chunks_storage=relation_chunks_storage,
)
logger.info(
f"Relation Create: `{vdb_src}`~`{vdb_tgt}` successfully created"
)
return await get_relation_info(
chunk_entity_relation_graph,
relationships_vdb,
vdb_src,
vdb_tgt,
include_vector_data=False,
)
except Exception as e:
logger.error(
f"Error while creating relation from '{source_entity}' to '{target_entity}': {e}"
)
raise
async def _merge_entities_impl(
chunk_entity_relation_graph,
entities_vdb,
relationships_vdb,
source_entities: list[str],
target_entity: str,
*,
merge_strategy: dict[str, str] = None,
target_entity_data: dict[str, Any] = None,
entity_chunks_storage=None,
relation_chunks_storage=None,
) -> dict[str, Any]:
"""Internal helper that merges entities without acquiring storage locks.
This function performs the actual entity merge operations without lock management.
It should only be called by public APIs that have already acquired necessary locks.
Args:
chunk_entity_relation_graph: Graph storage instance
entities_vdb: Vector database storage for entities
relationships_vdb: Vector database storage for relationships
source_entities: List of source entity names to merge
target_entity: Name of the target entity after merging
merge_strategy: Deprecated. Merge strategy for each field (optional)
target_entity_data: Dictionary of specific values to set for target entity (optional)
entity_chunks_storage: Optional KV storage for tracking chunks
relation_chunks_storage: Optional KV storage for tracking relation chunks
Returns:
Dictionary containing the merged entity information
Note:
Caller must acquire appropriate locks before calling this function.
All source entities and the target entity should be locked together.
When redirected relations collapse onto the same endpoint, their
weight is ``max(input weights, distinct merged evidence sources)``.
Failure semantics:
The knowledge graph is the authoritative data source. If a vector
storage upsert fails after retries (steps 7/8), this function raises
VectorStorageConsistencyError instead of attempting any rollback: the
graph already holds the merged state, no data is lost, and the source
entities have NOT been deleted yet (step 11 is never reached). The
vector storage may then lag behind the graph; running the offline
rebuild tool (``lightrag-rebuild-vdb``) restores full consistency.
"""
# Default merge strategy for entities
default_entity_merge_strategy = {
"description": "concatenate",
"entity_type": "keep_first",
"source_id": "join_unique",
"file_path": "join_unique",
}
effective_entity_merge_strategy = default_entity_merge_strategy
if merge_strategy:
logger.warning(
"Entity Merge: merge_strategy parameter is deprecated and will be ignored in a future release."
)
effective_entity_merge_strategy = {
**default_entity_merge_strategy,
**merge_strategy,
}
target_entity_data = {} if target_entity_data is None else target_entity_data
# 1. Check if all source entities exist
source_entities_data = {}
for entity_name in source_entities:
node_exists = await chunk_entity_relation_graph.has_node(entity_name)
if not node_exists:
raise ValueError(f"Source entity '{entity_name}' does not exist")
node_data = await chunk_entity_relation_graph.get_node(entity_name)
source_entities_data[entity_name] = node_data
# 2. Check if target entity exists and get its data if it does
target_exists = await chunk_entity_relation_graph.has_node(target_entity)
existing_target_entity_data = {}
if target_exists:
existing_target_entity_data = await chunk_entity_relation_graph.get_node(
target_entity
)
# 3. Merge entity data
merged_entity_data = _merge_attributes(
list(source_entities_data.values())
+ ([existing_target_entity_data] if target_exists else []),
effective_entity_merge_strategy,
filter_none_only=False, # Use entity behavior: filter falsy values
)
# Apply any explicitly provided target entity data (overrides merged data)
for key, value in target_entity_data.items():
merged_entity_data[key] = value
# 4. Get all relationships of the source entities and target entity (if exists)
all_relations = []
entities_to_collect = source_entities.copy()
# If target entity exists and not already in source_entities, add it
if target_exists and target_entity not in source_entities:
entities_to_collect.append(target_entity)
for entity_name in entities_to_collect:
# Get all relationships of the entities
edges = await chunk_entity_relation_graph.get_node_edges(entity_name)
if edges:
for src, tgt in edges:
# Ensure src is the current entity
if src == entity_name:
edge_data = await chunk_entity_relation_graph.get_edge(src, tgt)
all_relations.append((src, tgt, edge_data))
# 5. Create or update the target entity. A missing target is intentional:
# spelling-repair merges may consolidate sources into a new canonical name.
merged_entity_data["entity_id"] = target_entity
await chunk_entity_relation_graph.upsert_node(target_entity, merged_entity_data)
target_action = "updated" if target_exists else "created"
logger.info(f"Entity Merge: {target_action} target '{target_entity}'")
# 6. Recreate all relations pointing to the target entity in KG
# Also collect chunk tracking information in the same loop
relation_updates = {} # Track relationships that need to be merged
relations_to_delete = []
# Initialize chunk tracking variables
relation_chunk_tracking = {} # key: storage_key, value: list of chunk_ids
old_relation_keys_to_delete = []
for src, tgt, edge_data in all_relations:
relations_to_delete.append(compute_mdhash_id(src + tgt, prefix="rel-"))
relations_to_delete.append(compute_mdhash_id(tgt + src, prefix="rel-"))
# Collect old chunk tracking key for deletion
if relation_chunks_storage is not None:
from .utils import make_relation_chunk_key, has_chunk_tracking_row
old_storage_key = make_relation_chunk_key(src, tgt)
old_relation_keys_to_delete.append(old_storage_key)
new_src = target_entity if src in source_entities else src
new_tgt = target_entity if tgt in source_entities else tgt
# Skip relationships between source entities to avoid self-loops
if new_src == new_tgt:
logger.info(f"Entity Merge: skipping `{src}`~`{tgt}` to avoid self-loop")
continue
# Normalize entity order for consistent duplicate detection (undirected relationships)
normalized_src, normalized_tgt = sorted([new_src, new_tgt])
relation_key = f"{normalized_src}|{normalized_tgt}"
# Process chunk tracking for this relation
if relation_chunks_storage is not None:
storage_key = make_relation_chunk_key(normalized_src, normalized_tgt)
# Get chunk_ids from storage for this original relation
stored = await relation_chunks_storage.get_by_id(old_storage_key)
# Row presence by schema, consistent with the edit/rename paths —
# see has_chunk_tracking_row for the authority model. Legacy
# no-evidence placeholders are dropped so the merged row keeps only
# real chunk attribution.
if has_chunk_tracking_row(stored):
chunk_ids = [
cid
for cid in stored["chunk_ids"]
if cid and cid not in RELATION_NO_EVIDENCE_SOURCE_IDS
]
else:
# Fallback to source_id from graph
source_id = edge_data.get("source_id", "")
chunk_ids = relation_evidence_source_ids(source_id)
# Accumulate chunk_ids with ordered deduplication
if storage_key not in relation_chunk_tracking:
relation_chunk_tracking[storage_key] = []
existing_chunks = set(relation_chunk_tracking[storage_key])
for chunk_id in chunk_ids:
if chunk_id not in existing_chunks:
existing_chunks.add(chunk_id)
relation_chunk_tracking[storage_key].append(chunk_id)
if relation_key in relation_updates:
# Merge relationship data
existing_data = relation_updates[relation_key]["data"]
updated_relation = _merge_attributes(
[existing_data, edge_data],
{
"description": "concatenate",
"keywords": "join_unique_comma",
"source_id": "join_unique",
"file_path": "join_unique",
"weight": "max",
},
filter_none_only=True, # Use relation behavior: only filter None
)
logger.debug(
f"Entity Merge: deduplicating relation `{normalized_src}`~`{normalized_tgt}`"
)
else:
updated_relation = edge_data.copy()
relation_updates[relation_key] = {
"graph_src": new_src,
"graph_tgt": new_tgt,
"norm_src": normalized_src,
"norm_tgt": normalized_tgt,
"data": updated_relation,
}
# Every distinct real source contributes a baseline evidence count of
# 1. Apply the floor to the first redirected relation as well as later
# duplicates so every relation rewritten by the merge repairs legacy
# undersized weights. The "max" merge strategy still preserves the
# strongest input boost and FIFO-accumulated weights.
updated_relation["weight"] = apply_relation_weight_floor(
updated_relation.get("weight", 1.0),
updated_relation.get("source_id") or "",
)
relation_updates[relation_key]["data"] = updated_relation
# Apply relationship updates
logger.info(f"Entity Merge: updating {len(relation_updates)} relations")
for rel_data in relation_updates.values():
await chunk_entity_relation_graph.upsert_edge(
rel_data["graph_src"], rel_data["graph_tgt"], rel_data["data"]
)
logger.info(
f"Entity Merge: updating relation `{rel_data['graph_src']}`~`{rel_data['graph_tgt']}`"
)
# Update relation chunk tracking storage. Upsert the migrated rows BEFORE
# deleting the old keys: on RPC-backed KV storages each call commits
# independently, so the reverse order opened a crash window in which every
# migrated relation row was lost — turning curated rows absent and
# re-arming the stale source_id reseed (#3609). A key can be both old and
# new (relations already attached to an existing target keep their key),
# so only keys outside the new key set are deleted; deleting them after
# the upsert would drop the rows just written.
#
# Bound before the guard, not inside it: step 11b reads this list
# unconditionally, and merging entities that carry no incident edges
# leaves `all_relations` empty. Assigning only in the branch below made
# that case raise UnboundLocalError AFTER the graph commit had already
# removed the source entities -- reporting failure for a merge that had
# in fact landed.
stale_relation_keys: list[str] = []
if relation_chunks_storage is not None and all_relations:
if relation_chunk_tracking:
updates = {}
for storage_key, chunk_ids in relation_chunk_tracking.items():
updates[storage_key] = {
"chunk_ids": chunk_ids,
"count": len(chunk_ids),
}
await relation_chunks_storage.upsert(updates)
logger.info(
f"Entity Merge: {len(updates)} relation chunk tracking records updated"
)
# Deleted only in step 11b, once the graph state that replaced these
# keys is durable. Deleting here would retire the authoritative rows of
# relations that are still on disk -- either because the redirected
# edges have not been committed yet, or because the source node whose
# removal takes the old edges with it has not been either.
stale_relation_keys = [
key
for key in dict.fromkeys(old_relation_keys_to_delete)
if key not in relation_chunk_tracking
]
# 7. Update relationship vector representations
logger.debug(
f"Entity Merge: deleting {len(relations_to_delete)} relations from vdb"
)
if relations_to_delete:
try:
await safe_vdb_operation_with_exception(
operation=lambda ids=relations_to_delete: relationships_vdb.delete(ids),
operation_name="merge_relation_delete",
entity_name=target_entity,
max_retries=3,
retry_delay=0.2,
)
except Exception as e:
raise VectorStorageConsistencyError(
f"Vector storage delete of {len(relations_to_delete)} stale relation "
f"record(s) failed while merging entities into '{target_entity}': {e}. "
"The knowledge graph was updated but the vector storage was not, so they "
"may now be inconsistent. No data is lost (the graph is the authoritative "
"source and the source entities were not deleted). Stop the LightRAG server "
"and run the offline rebuild tool (lightrag-rebuild-vdb) to restore "
"consistency."
) from e
for rel_data in relation_updates.values():
edge_data = rel_data["data"]
normalized_src = rel_data["norm_src"]
normalized_tgt = rel_data["norm_tgt"]
description = edge_data.get("description", "")
keywords = edge_data.get("keywords", "")
source_id = edge_data.get("source_id", "")
weight = float(edge_data.get("weight", 1.0))
relation_id = compute_mdhash_id(normalized_src + normalized_tgt, prefix="rel-")
# The graph was already updated above (step 6); a truncation failure
# here is the same class of "graph updated, VDB payload could not be
# completed" failure as a VDB-operation failure below, so it gets the
# same VectorStorageConsistencyError treatment.
try:
# Use normalized order for content and relation ID
content = _truncate_vdb_content(
f"{keywords}\t{normalized_src}\n{normalized_tgt}\n{description}",
entities_vdb.global_config,
f"relation:{normalized_src}-{normalized_tgt}",
)
relation_data_for_vdb = {
relation_id: {
"content": content,
"src_id": normalized_src,
"tgt_id": normalized_tgt,
"source_id": source_id,
"description": description,
"keywords": keywords,
"weight": weight,
"file_path": edge_data.get("file_path", ""),
}
}
await safe_vdb_operation_with_exception(
operation=lambda payload=relation_data_for_vdb: (
relationships_vdb.upsert(payload)
),
operation_name="merge_relation_upsert",
entity_name=f"{normalized_src}-{normalized_tgt}",
max_retries=3,
retry_delay=0.2,
)
except Exception as e:
raise VectorStorageConsistencyError(
f"Vector storage upsert failed for relation `{normalized_src}`~`{normalized_tgt}` "
f"while merging entities into '{target_entity}': {e}. "
"The knowledge graph was updated but the vector storage was not, so they may "
"now be inconsistent. No data is lost (the graph is the authoritative source "
"and the source entities were not deleted). Stop the LightRAG server and run "
"the offline rebuild tool (lightrag-rebuild-vdb) to restore consistency."
) from e
logger.debug(
f"Entity Merge: updating vdb `{normalized_src}`~`{normalized_tgt}`"
)
logger.info(f"Entity Merge: {len(relation_updates)} relations in vdb updated")
# 8. Update entity vector representation
description = merged_entity_data.get("description", "")
source_id = merged_entity_data.get("source_id", "")
entity_type = merged_entity_data.get("entity_type", "")
entity_id = compute_mdhash_id(target_entity, prefix="ent-")
# The graph was already updated above (step 5); a truncation failure here
# is the same class of "graph updated, VDB payload could not be
# completed" failure as a VDB-operation failure below, so it gets the
# same VectorStorageConsistencyError treatment.
try:
content = _truncate_vdb_content(
target_entity + "\n" + description,
entities_vdb.global_config,
f"entity:{target_entity}",
)
entity_data_for_vdb = {
entity_id: {
"content": content,
"entity_name": target_entity,
"source_id": source_id,
"description": description,
"entity_type": entity_type,
"file_path": merged_entity_data.get("file_path", ""),
}
}
await safe_vdb_operation_with_exception(
operation=lambda payload=entity_data_for_vdb: entities_vdb.upsert(payload),
operation_name="merge_entity_upsert",
entity_name=target_entity,
max_retries=3,
retry_delay=0.2,
)
except Exception as e:
raise VectorStorageConsistencyError(
f"Vector storage upsert failed for entity '{target_entity}' during entity merge: {e}. "
"The knowledge graph was updated but the vector storage was not, so they may "
"now be inconsistent. No data is lost (the graph is the authoritative source "
"and the source entities were not deleted). Stop the LightRAG server and run "
"the offline rebuild tool (lightrag-rebuild-vdb) to restore consistency."
) from e
logger.info(f"Entity Merge: updating vdb `{target_entity}`")
# The sources that step 11 actually removes, and whose tracking rows step
# 11b retires once that removal is durable -- never before it: a row retired
# ahead of the object it describes is the state the purge recovery contract
# forbids. Bound here, ahead of every consumer, so the retirement never
# depends on a branch having run -- the shape of the `stale_relation_keys`
# defect.
entities_to_remove = [e for e in source_entities if e != target_entity]
# 9. Merge entity chunk tracking (source entities first, then target entity)
if entity_chunks_storage is not None:
from .utils import has_chunk_tracking_row
all_chunk_id_lists = []
any_row_present = False
# Build list of entities to process (source entities first, then target entity)
entities_to_process = []
# Add source entities first (excluding target if it's already in source list)
for entity_name in source_entities:
if entity_name != target_entity:
entities_to_process.append(entity_name)
# Add target entity last (if it exists)
if target_exists:
entities_to_process.append(target_entity)
# Pre-merge node data captured in steps 1-2, BEFORE step 5 overwrote
# the target node with the merged payload (whose source_id unions
# every input's contribution and must not be attributed to any single
# input entity).
pre_merge_node_data = dict(source_entities_data)
if target_exists:
pre_merge_node_data[target_entity] = existing_target_entity_data
# Process all entities in order with unified logic. Row presence by
# schema, never list truthiness (see has_chunk_tracking_row) — and
# decided PER INPUT ENTITY: one entity's curated-empty row asserts
# only that THAT entity tracks no chunks, never that another input's
# unknown (absent/malformed) row is empty too. An input without a
# usable row falls back to its own pre-merge graph source_id, exactly
# as a single-entity edit would; folding such an input into an
# authoritative empty target row would silently discard its
# attribution with no later recovery path (the empty row is
# authoritative from then on).
for entity_name in entities_to_process:
stored = await entity_chunks_storage.get_by_id(entity_name)
if has_chunk_tracking_row(stored):
any_row_present = True
chunk_ids = [cid for cid in stored.get("chunk_ids", []) if cid]
else:
node_source_id = (pre_merge_node_data.get(entity_name) or {}).get(
"source_id"
) or ""
chunk_ids = [
cid for cid in node_source_id.split(GRAPH_FIELD_SEP) if cid
]
if chunk_ids:
all_chunk_id_lists.append(chunk_ids)
# Merge chunk_ids with ordered deduplication (preserves order, source entities first)
merged_chunk_ids = []
seen = set()
for chunk_id_list in all_chunk_id_lists:
for chunk_id in chunk_id_list:
if chunk_id not in seen:
seen.add(chunk_id)
merged_chunk_ids.append(chunk_id)
# Update target entity's chunk tracking BEFORE deleting the source
# rows: on RPC-backed KV storages each call commits independently, so
# the reverse order opens a crash window in which the merged
# attribution exists under no key at all. Write only when at least
# one input had a usable row: a present row is what licenses an
# authoritative target row (the source_id fallbacks above only fill
# in the other inputs' contributions within that write). With no row
# present the merged attribution is entirely unknown, and the target
# row stays absent — fabricating one would promote UNKNOWN to
# authoritative.
if any_row_present:
await entity_chunks_storage.upsert(
{
target_entity: {
"chunk_ids": merged_chunk_ids,
"count": len(merged_chunk_ids),
}
}
)
logger.info(
f"Entity Merge: find {len(merged_chunk_ids)} chunks related to '{target_entity}'"
)
# 9b. Make every migrated row durable BEFORE the first commit that publishes
# the objects they describe. On a deferred KV backend (the default
# JsonKVStorage) an upsert only touches shared memory, so a commit ahead of
# this flush publishes new graph objects whose authoritative tracking exists
# nowhere on disk -- and a process exit in between comes back to a graph
# that can only fall back to the KEEP-truncated `source_id` a purge misreads.
#
# The invariant is stated over EVERY commit in this function, not just the
# source removal: a graph commit may only publish objects whose tracking
# rows are already on disk. That is why the tracking construction above was
# moved ahead of step 8b as well -- covering only the removal commit left
# the first publication (the merged target and its redirected relations)
# outside it.
#
# Flushing first leaves the opposite residue: rows on disk for objects that
# do not exist yet, which is dead bookkeeping a retry overwrites -- the same
# trade f86ef93c settled for the upsert-before-delete ordering.
try:
await _persist_graph_updates(
entity_chunks_storage=entity_chunks_storage,
relation_chunks_storage=relation_chunks_storage,
)
except Exception as e:
raise VectorStorageConsistencyError(
f"Persisting the migrated chunk tracking failed while merging into "
f"'{target_entity}': {e}. Nothing has been published yet -- the "
"source entities are intact and still carry their chunk tracking, "
"so no provenance is lost. Retry the merge."
) from e
# 10. Persist the graph and vector storages now — before any source-entity
# deletion (step 11). Deferred-embedding backends (e.g. nano/faiss) do NOT
# call the embedder inside upsert(); they embed and persist in
# index_done_callback, so an embedder outage surfaces only at flush time,
# outside the upsert try/except above. Flushing here, while the source
# entities are still intact, keeps the fail-loud guarantee true for those
# backends: on failure we raise VectorStorageConsistencyError before
# deleting anything, and the error message ("source entities not deleted")
# remains accurate. The graph is flushed first so it is the authoritative
# on-disk source the offline rebuild tool can recover from.
#
# Checked, not bare: a declined commit RELOADS the graph from disk and
# discards the in-memory merge (see _commit_graph_or_raise). Continuing past
# that would run step 11's node deletions against the reloaded state, so a
# later successful commit would publish the source entities' removal without
# the relation redirection that is supposed to preserve them.
await _commit_graph_or_raise(
chunk_entity_relation_graph, f"Entity Merge: into '{target_entity}'"
)
try:
await safe_vdb_operation_with_exception(
operation=relationships_vdb.index_done_callback,
operation_name="merge_relation_flush",
entity_name=target_entity,
max_retries=3,
retry_delay=0.2,
)
await safe_vdb_operation_with_exception(
operation=entities_vdb.index_done_callback,
operation_name="merge_entity_flush",
entity_name=target_entity,
max_retries=3,
retry_delay=0.2,
)
except Exception as e:
raise VectorStorageConsistencyError(
f"Vector storage flush failed after merging entities into '{target_entity}': {e}. "
"The knowledge graph was updated but the vector storage embeddings could not be "
"persisted, so they may now be inconsistent. No data is lost (the graph is the "
"authoritative source and the source entities were not deleted). Stop the LightRAG "
"server and run the offline rebuild tool (lightrag-rebuild-vdb) to restore consistency."
) from e
# 11. Delete the source entities' vector records, BEFORE the node removals
# below. Same order, and same reason, as adelete_by_entity: on an
# immediate-write graph backend (Neo4j, Memgraph, MongoDB, PostgreSQL --
# their graph `index_done_callback` is a no-op) `delete_node` is durable the
# moment it returns, so a vector failure after it would exit with the source
# already gone and its tracking rows unretired. Doing the vector work first
# leaves the inverse residue -- a live node whose vector record is missing --
# which is the rebuildable window this codebase already accepts.
if target_entity in source_entities:
logger.warning(
f"Entity Merge: source entity'{target_entity}' is same as target entity"
)
for entity_name in entities_to_remove:
logger.info(f"Entity Merge: deleting '{entity_name}' from vdb")
entity_id = compute_mdhash_id(entity_name, prefix="ent-")
try:
await safe_vdb_operation_with_exception(
operation=lambda eid=entity_id: entities_vdb.delete([eid]),
operation_name="merge_source_entity_delete",
entity_name=entity_name,
max_retries=3,
retry_delay=0.2,
)
except Exception as e:
raise VectorStorageConsistencyError(
f"Vector storage delete of merged-away source entity '{entity_name}' "
f"failed while finalizing the merge into '{target_entity}': {e}. "
"The source entities were NOT removed from the knowledge graph (the "
"authoritative source) and still carry their chunk tracking, so no "
"data is lost and nothing has lost its provenance; retrying the "
"merge is safe. Vector records deleted before this failure leave "
"those entities with no embedding until the merge is retried or the "
"offline rebuild tool (lightrag-rebuild-vdb) is run."
) from e
# 11b. Make the source entities' removal durable, and only then retire the
# tracking rows they and their old relation keys owned. Same staging as
# adelete_by_entity, and for the same reason: an authoritative row must
# never predecease the object it describes, because a later
# _purge_kg_contributions then falls back to the KEEP-truncated graph
# source_id and can conclude "no remaining sources".
#
# The upserts above deliberately stay BEFORE this commit -- that ordering is
# f86ef93c's fix for the opposite failure (#3609), where the row existed
# under neither key. An orphaned new-key row is dead bookkeeping and is
# overwritten by a retry; a row that is absent while its object lives is the
# bug. Both invariants hold with the upserts before the commit and the
# deletes after it.
async def _commit_source_removal_and_retire_tracking() -> None:
# The node removals belong INSIDE the region, not before it: on an
# immediate-write graph backend they are durable as they run, so a
# cancellation between them and the commit would strand the tracking
# rows of objects that are already gone. adelete_by_entity stages its
# own `delete_node` the same way.
for entity_name in entities_to_remove:
logger.info(f"Entity Merge: deleting '{entity_name}' from KG")
await chunk_entity_relation_graph.delete_node(entity_name)
try:
await _commit_graph_or_raise(
chunk_entity_relation_graph,
f"Entity Merge: removing sources merged into '{target_entity}'",
)
except Exception as e:
raise VectorStorageConsistencyError(
f"Persisting the source-entity removal failed while finalizing the merge "
f"into '{target_entity}': {e}. The merged relations are already on disk, "
"but the source entities were NOT removed and still carry their chunk "
"tracking, so nothing has lost its provenance. Retry the merge; it is "
"idempotent from this state."
) from e
# A failure below is raised as VectorStorageConsistencyError, not bare:
# the merge is already durable at this point, and only that type is
# re-raised by the `allow_merge` handler in `_edit_entity_impl`.
# Anything else is folded there into a partial-success summary that
# answers HTTP 200 with `final_entity` set to the source entity -- a
# source this commit has just removed. Reporting a landed merge as one
# that did not happen is worse than the orphan rows themselves.
if relation_chunks_storage is not None and stale_relation_keys:
try:
await relation_chunks_storage.delete(stale_relation_keys)
except Exception as e:
logger.error(
"Entity Merge: failed to retire relation chunk tracking; "
f"these rows are now orphaned: {stale_relation_keys}"
)
raise VectorStorageConsistencyError(
f"Retiring the relation chunk tracking failed while finalizing "
f"the merge into '{target_entity}': {e}. The merge itself is "
"durable -- the source entities are removed and their relations "
"redirected -- but the tracking rows of the replaced relation "
f"keys survive as orphans: {stale_relation_keys}. They describe "
"objects that no longer exist and are dead bookkeeping unless "
"the same relation endpoints reappear later, at which point "
"extraction would read them back as authoritative provenance."
) from e
if entity_chunks_storage is not None and entities_to_remove:
try:
await entity_chunks_storage.delete(entities_to_remove)
except Exception as e:
logger.error(
"Entity Merge: failed to retire entity chunk tracking; "
f"these rows are now orphaned: {entities_to_remove}"
)
raise VectorStorageConsistencyError(
f"Retiring the entity chunk tracking failed while finalizing the "
f"merge into '{target_entity}': {e}. The merge itself is durable "
"-- the source entities are removed and their relations "
"redirected -- but their tracking rows survive as orphans: "
f"{entities_to_remove}. They describe entities that no longer "
"exist and are dead bookkeeping unless the same names are "
"extracted again, at which point extraction would read them back "
"as authoritative provenance."
) from e
# Flushed inside the region, not after it: on a deferred KV backend the
# deletes above only touch memory, so a cancellation delivered between
# them and this flush would leave the retired rows on disk -- exactly
# the orphan the region exists to prevent.
try:
await _persist_graph_updates(
entity_chunks_storage=entity_chunks_storage,
relation_chunks_storage=relation_chunks_storage,
)
except Exception as e:
raise VectorStorageConsistencyError(
f"Persisting the retired chunk tracking failed while finalizing the "
f"merge into '{target_entity}': {e}. The merge has been applied to the "
"knowledge graph (the authoritative source) and the source entities "
"were removed, but their chunk tracking rows may survive as orphans. "
"No data is lost; the stale rows are dead bookkeeping."
) from e
# The source removal, its durable commit and the tracking retirement are one
# cancellation-deferring region, for the same reason as adelete_by_entity:
# once the commit lands, the tracking rows describe objects that no longer
# exist, and a cancellation delivered mid-cleanup would strand them. The
# commit await is itself where a deferred cancellation reappears, so the
# region has to start before it -- protecting only the deletes would mean
# never reaching them.
await _finish_deferring_cancellation(
_commit_source_removal_and_retire_tracking(),
f"Entity Merge: source removal and tracking cleanup for '{target_entity}'",
)
# 12. Save the vector storages. The graph is already committed above, so
# flushing it again here would rewrite the whole file for nothing.
try:
await _persist_graph_updates(
entities_vdb=entities_vdb,
relationships_vdb=relationships_vdb,
)
except Exception as e:
raise VectorStorageConsistencyError(
f"Persisting the merged state failed while finalizing the merge into "
f"'{target_entity}': {e}. The merge has been applied to the knowledge graph "
"(the authoritative source) and the source entities were removed, but the "
"vector storage may not be fully persisted, so they may now be "
"inconsistent. No data is lost. Stop the LightRAG server and run the "
"offline rebuild tool (lightrag-rebuild-vdb) to restore consistency."
) from e
logger.info(
f"Entity Merge: successfully merged {len(source_entities)} entities into '{target_entity}'"
)
return await get_entity_info(
chunk_entity_relation_graph,
entities_vdb,
target_entity,
include_vector_data=False,
)
async def amerge_entities(
chunk_entity_relation_graph,
entities_vdb,
relationships_vdb,
source_entities: list[str],
target_entity: str,
merge_strategy: dict[str, str] = None,
target_entity_data: dict[str, Any] = None,
entity_chunks_storage=None,
relation_chunks_storage=None,
) -> dict[str, Any]:
"""Asynchronously merge multiple entities into one entity.
Merges multiple source entities into a target entity, handling all relationships,
and updating both the knowledge graph and vector database.
Also merges chunk tracking information from entity_chunks_storage and relation_chunks_storage.
Args:
chunk_entity_relation_graph: Graph storage instance
entities_vdb: Vector database storage for entities
relationships_vdb: Vector database storage for relationships
source_entities: List of source entity names to merge
target_entity: Name of the target entity after merging
merge_strategy: Deprecated (Each field uses its own default strategy). If provided,
customizations are applied but a warning is logged.
target_entity_data: Dictionary of specific values to set for the target entity,
overriding any merged values, e.g. {"description": "custom description", "entity_type": "PERSON"}
entity_chunks_storage: Optional KV storage for tracking chunks that reference entities
relation_chunks_storage: Optional KV storage for tracking chunks that reference relations
Returns:
Dictionary containing the merged entity information
"""
if not source_entities:
raise ValueError("At least one source entity is required for merge")
# `_merge_entities_impl` copies these keys onto the merged node verbatim
# (`merged_entity_data[key] = value`), which is the same wholesale merge the
# edit paths do. Not reachable from the HTTP route today -- the merge
# endpoint does not pass it -- but it is part of the public Python API.
if target_entity_data:
target_entity_data = _sanitize_graph_fields(
target_entity_data,
allowed_fields=_ENTITY_DATA_FIELDS,
object_type="entity",
reject_unknown=True,
)
requested_source_entities = list(source_entities)
normalized_source_entities = [
_normalize_manual_entity_name(entity_name)
for entity_name in requested_source_entities
]
requested_target_entity = target_entity
normalized_target_entity = _normalize_manual_entity_name(requested_target_entity)
# Lock every exact/canonical candidate before resolving legacy keys. This
# shares the extraction pipeline's canonical locks while preserving access
# to historical manually-created names.
lock_key_set = set(requested_source_entities)
lock_key_set.update(name for name in normalized_source_entities if name)
lock_key_set.add(requested_target_entity)
if normalized_target_entity:
lock_key_set.add(normalized_target_entity)
lock_keys = sorted(lock_key_set)
workspace = entities_vdb.global_config.get("workspace", "")
namespace = f"{workspace}:GraphDB" if workspace else "GraphDB"
async with get_storage_keyed_lock(
lock_keys, namespace=namespace, enable_logging=False
):
try:
resolved_source_entities: list[str] = []
seen_source_entities: set[str] = set()
for requested_name, normalized_name in zip(
requested_source_entities,
normalized_source_entities,
strict=True,
):
if (
requested_name != normalized_name
and await chunk_entity_relation_graph.has_node(requested_name)
):
resolved_name = requested_name
elif normalized_name:
resolved_name = normalized_name
else:
raise ValueError(
"Source entity name cannot be empty after normalization"
)
# Multiple caller spellings may resolve to one canonical node.
# Merge it once so relations, vectors, and deletion are not
# processed repeatedly.
if resolved_name not in seen_source_entities:
seen_source_entities.add(resolved_name)
resolved_source_entities.append(resolved_name)
if (
requested_target_entity != normalized_target_entity
and await chunk_entity_relation_graph.has_node(requested_target_entity)
):
target_entity = requested_target_entity
elif normalized_target_entity:
target_entity = normalized_target_entity
else:
raise ValueError(
"Target entity name cannot be empty after normalization"
)
return await _merge_entities_impl(
chunk_entity_relation_graph,
entities_vdb,
relationships_vdb,
resolved_source_entities,
target_entity,
merge_strategy=merge_strategy,
target_entity_data=target_entity_data,
entity_chunks_storage=entity_chunks_storage,
relation_chunks_storage=relation_chunks_storage,
)
except Exception as e:
logger.error(f"Error merging entities: {e}")
raise
def _merge_attributes(
data_list: list[dict[str, Any]],
merge_strategy: dict[str, str],
filter_none_only: bool = False,
) -> dict[str, Any]:
"""Merge attributes from multiple entities or relationships.
This unified function handles merging of both entity and relationship attributes,
applying different merge strategies per field.
Args:
data_list: List of dictionaries containing entity or relationship data
merge_strategy: Merge strategy for each field. Supported strategies:
- "concatenate": Join all values with GRAPH_FIELD_SEP
- "keep_first": Keep the first non-empty value
- "keep_last": Keep the last non-empty value
- "join_unique": Join unique items separated by GRAPH_FIELD_SEP
- "join_unique_comma": Join unique items separated by comma and space
- "max": Keep the maximum numeric value (for numeric fields)
filter_none_only: If True, only filter None values (keep empty strings, 0, etc.).
If False, filter all falsy values. Default is False for backward compatibility.
Returns:
Dictionary containing merged data
"""
merged_data = {}
# Collect all possible keys
all_keys = set()
for data in data_list:
all_keys.update(data.keys())
# Merge values for each key
for key in all_keys:
# Get all values for this key based on filtering mode
if filter_none_only:
values = [data.get(key) for data in data_list if data.get(key) is not None]
else:
values = [data.get(key) for data in data_list if data.get(key)]
if not values:
continue
# Merge values according to strategy
strategy = merge_strategy.get(key, "keep_first")
if strategy == "concatenate":
# Convert all values to strings and join with GRAPH_FIELD_SEP
merged_data[key] = GRAPH_FIELD_SEP.join(str(v) for v in values)
elif strategy == "keep_first":
merged_data[key] = values[0]
elif strategy == "keep_last":
merged_data[key] = values[-1]
elif strategy == "join_unique":
# Handle fields separated by GRAPH_FIELD_SEP
unique_items = set()
for value in values:
items = str(value).split(GRAPH_FIELD_SEP)
unique_items.update(items)
merged_data[key] = GRAPH_FIELD_SEP.join(unique_items)
elif strategy == "join_unique_comma":
# Handle fields separated by comma, join unique items with comma
unique_items = set()
for value in values:
items = str(value).split(",")
unique_items.update(item.strip() for item in items if item.strip())
merged_data[key] = ",".join(sorted(unique_items))
elif strategy == "max":
# For numeric fields like weight
try:
merged_data[key] = max(float(v) for v in values)
except (ValueError, TypeError):
# Fallback to first value if conversion fails
merged_data[key] = values[0]
else:
# Default strategy: keep first value
merged_data[key] = values[0]
return merged_data
async def get_entity_info(
chunk_entity_relation_graph,
entities_vdb,
entity_name: str,
include_vector_data: bool = False,
) -> dict[str, str | None | dict[str, str]]:
"""Get detailed information of an entity"""
# Get information from the graph
node_data = await chunk_entity_relation_graph.get_node(entity_name)
source_id = node_data.get("source_id") if node_data else None
result: dict[str, str | None | dict[str, str]] = {
"entity_name": entity_name,
"source_id": source_id,
"graph_data": node_data,
}
# Optional: Get vector database information
if include_vector_data:
entity_id = compute_mdhash_id(entity_name, prefix="ent-")
vector_data = await entities_vdb.get_by_id(entity_id)
result["vector_data"] = vector_data
return result
async def get_relation_info(
chunk_entity_relation_graph,
relationships_vdb,
src_entity: str,
tgt_entity: str,
include_vector_data: bool = False,
) -> dict[str, str | None | dict[str, str]]:
"""
Get detailed information of a relationship between two entities.
Relationship is unidirectional, swap src_entity and tgt_entity does not change the relationship.
Args:
src_entity: Source entity name
tgt_entity: Target entity name
include_vector_data: Whether to include vector database information
Returns:
Dictionary containing relationship information
"""
# Get information from the graph
edge_data = await chunk_entity_relation_graph.get_edge(src_entity, tgt_entity)
source_id = edge_data.get("source_id") if edge_data else None
result: dict[str, str | None | dict[str, str]] = {
"src_entity": src_entity,
"tgt_entity": tgt_entity,
"source_id": source_id,
"graph_data": edge_data,
}
# Optional: Get vector database information
if include_vector_data:
vector_data = None
for rel_id in make_relation_vdb_ids(src_entity, tgt_entity):
vector_data = await relationships_vdb.get_by_id(rel_id)
if vector_data is not None:
break
result["vector_data"] = vector_data
return result
from __future__ import annotations
import time
import asyncio
from typing import Any, Awaitable, Callable, cast
from .base import DeletionResult
from .exceptions import CommitBookkeepingError
from .kg.shared_storage import get_storage_keyed_lock
from .constants import GRAPH_FIELD_SEP, RELATION_NO_EVIDENCE_SOURCE_IDS
from .operate import _truncate_vdb_content
from .utils import (
VectorStorageConsistencyError,
_consume_future_exception,
_wait_deferring_cancellation,
compute_mdhash_id,
graph_attribute_value_rejection,
log_without_raising,
logger,
make_relation_vdb_ids,
normalize_entity_name,
safe_vdb_operation_with_exception,
)
from .base import StorageNameSpace
# Field specs for the manual entity/relation mutation APIs. The value is the
# expected shape: "text" for a string attribute, "number" for a numeric one.
#
# These are an allowlist, not documentation. `updated_data` used to be merged
# into the stored object wholesale (`{**node_data, **updated_data}`), so any key
# a caller invented was written with whatever value it carried -- including
# values no graph backend can store. See `_sanitize_graph_fields`.
_TEXT_FIELD = "text"
_NUMBER_FIELD = "number"
# `entity_name` is the rename target, which `aedit_entity` resolves and writes
# back into `updated_data` before the merge; it is a legal edit field but not a
# legal *create* field (create takes the name as its own argument).
_EDITABLE_ENTITY_FIELDS: dict[str, str] = {
"entity_name": _TEXT_FIELD,
"entity_type": _TEXT_FIELD,
"description": _TEXT_FIELD,
"source_id": _TEXT_FIELD,
"file_path": _TEXT_FIELD,
}
_ENTITY_DATA_FIELDS: dict[str, str] = {
key: kind for key, kind in _EDITABLE_ENTITY_FIELDS.items() if key != "entity_name"
}
_RELATION_DATA_FIELDS: dict[str, str] = {
"description": _TEXT_FIELD,
"keywords": _TEXT_FIELD,
"source_id": _TEXT_FIELD,
"file_path": _TEXT_FIELD,
"weight": _NUMBER_FIELD,
}
def _sanitize_graph_fields(
data: dict[str, Any],
*,
allowed_fields: dict[str, str],
object_type: str,
reject_unknown: bool,
) -> dict[str, Any]:
"""Return *data* reduced to well-typed, storable graph attributes.
Two call shapes, because the two families of caller differ in what they do
with a key they do not recognise:
* ``reject_unknown=True`` (the edit/merge paths) -- these merge the caller's
mapping into the stored object, so an unrecognised key *is written*. It
has to be refused, and refused loudly: silently dropping it on an edit
endpoint would report success for a change that never happened.
* ``reject_unknown=False`` (the create paths) -- these already copy a fixed
set of named fields out of the mapping, so an unrecognised key is
structurally incapable of reaching storage. Only the values of the
recognised keys need checking, and refusing extra keys here would break
payloads that work today for no security gain.
Per-field types matter as much as the allowlist. A value can be a perfectly
storable scalar and still be wrong: ``{"source_id": 123}`` serializes fine
(GraphML long, JSON number), reaches disk, survives a restart, and then
breaks every later reader that splits ``source_id`` on ``GRAPH_FIELD_SEP``
-- a durable failure, unlike the transient one a non-scalar causes.
Args:
data: Caller-supplied attribute mapping.
allowed_fields: Field name to expected shape (``_TEXT_FIELD`` /
``_NUMBER_FIELD``).
object_type: ``"entity"`` or ``"relation"``, for error messages.
reject_unknown: Whether an unrecognised key is an error (see above).
Returns:
A new mapping holding only recognised fields, with numeric fields
coerced to ``float``.
Raises:
ValueError: On an unknown field (when ``reject_unknown``), a field whose
value has the wrong shape, or a value no graph backend can store.
"""
Xet Storage Details
- Size:
- 145 kB
- Xet hash:
- b2bdcb8b2dc26eb94730544135df0b2c6c0f298987da1f476f8b32a6247f280d
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.