"""HotpotQA as a real natural-language multi-hop graph reasoning workload. Why this construction --------------------- The upstream GFM-RAG pipeline turns paragraphs into a knowledge graph with an LLM (NER + OpenIE). That needs an LLM budget and hours of preprocessing, and it introduces the extractor's errors into the measurement. This module instead builds an **entity graph** directly from the data HotpotQA already ships, with no model in the loop: * nodes are the paragraph titles, which in HotpotQA are Wikipedia article titles and therefore entities; * an edge ``A -mentions-> B`` exists when B's title occurs in A's text; * the start entity is the title mentioned in the question; * the gold evidence is the ``supporting_facts`` titles. This is the entity-graph formulation used by multi-hop GNN work on HotpotQA. It is lossier than full OpenIE, and the honest consequence is stated in the metric: the primary measure is **supporting-title retrieval** -- did the returned path cover the gold supporting paragraphs -- because that is what a graph retriever is responsible for. Answer-span matching is reported only when the answer is itself a title, since a span answer is the language model's job, not the graph's. Unlike the KG workload, the bridge entity is *not* named in the question, so this does measure intent inference, on real text with real distractors. """ from __future__ import annotations import json import re from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from pathlib import Path from graph.graph import EdgeRecord, GraphStore from graph.types import GraphQuery, ReasoningStatus MENTIONS = "mentions" #: Titles shorter than this are too generic to match reliably in free text #: ("It", "US"), and matching them creates spurious edges. MIN_TITLE_LENGTH = 4 @dataclass(frozen=True) class HotpotGraphSpec: """Controls for entity-graph construction.""" #: Include an edge when a title appears in another paragraph's text. link_on_mention: bool = True #: Also link paragraphs that share a rare capitalised token, which recovers #: bridges that the exact-title test misses. link_on_shared_entity: bool = False min_title_length: int = MIN_TITLE_LENGTH @dataclass(frozen=True) class HotpotExample: """One HotpotQA question as a graph reasoning problem.""" example_id: str graph: GraphStore query: GraphQuery question: str answer: str #: Gold supporting paragraph titles. supporting_titles: tuple[str, ...] #: Titles mentioned by the question, used as search anchors. start_titles: tuple[str, ...] level: str question_type: str status: ReasoningStatus #: True when the answer string is itself a paragraph title, so entity-level #: answer matching is meaningful for this question. answer_is_title: bool def _normalise(text: str) -> str: return re.sub(r"\s+", " ", text).strip().lower() def _mentions(text: str, title: str, *, min_length: int) -> bool: if len(title) < min_length: return False return _normalise(title) in _normalise(text) def _shared_entity_tokens(text: str) -> set[str]: """Return conservative capitalized entity tokens for bridge recovery.""" stopwords = { "After", "Before", "During", "From", "Later", "Most", "Other", "That", "The", "This", "When", "Where", "Which", "While", } return { token.lower() for token in re.findall(r"\b[A-Z][A-Za-zÀ-ÖØ-öø-ÿ]{3,}\b", text) if token not in stopwords } def _build_graph( titles: Sequence[str], paragraphs: Mapping[str, str], spec: HotpotGraphSpec, version: str, ) -> GraphStore: nodes = {title: "Paragraph" for title in titles} edges: list[EdgeRecord] = [] seen: set[tuple[str, str]] = set() title_tokens = {title: _shared_entity_tokens(title) for title in titles} for source in titles: text = paragraphs.get(source, "") source_tokens = _shared_entity_tokens(text) for target in titles: if source == target: continue if not spec.link_on_mention: continue mentioned = _mentions(text, target, min_length=spec.min_title_length) shared = spec.link_on_shared_entity and bool( source_tokens & title_tokens.get(target, set()) ) if not mentioned and not shared: continue if (source, target) in seen: continue seen.add((source, target)) edges.append( EdgeRecord( source_id=source, relation_id=MENTIONS, target_id=target, source_type="Paragraph", target_type="Paragraph", relation_name=MENTIONS, attributes={}, ) ) return GraphStore( nodes, edges, graph_version=version, # ``mentions`` is many-valued: a paragraph legitimately mentions several # others, so it must never be read as a contradiction. relation_attributes={MENTIONS: {"functional": "false"}}, ) def load_hotpotqa( path: str | Path, *, limit: int | None = None, spec: HotpotGraphSpec | None = None, require_start: bool = True, max_hops: int = 3, ) -> Iterator[HotpotExample]: """Stream HotpotQA JSON or Parquet examples as graph problems. ``require_start`` skips questions where no paragraph title occurs in the question text: without an anchor there is no graph-local starting point, and silently scoring those as failures would confuse entity linking with reasoning. The count of skipped questions is the caller's to report. """ spec = spec or HotpotGraphSpec() input_path = Path(path) if input_path.suffix == ".parquet": try: import pyarrow.parquet as pq except ImportError as error: # pragma: no cover - optional dependency raise RuntimeError("reading HotpotQA Parquet requires pyarrow") from error rows = pq.read_table(input_path).to_pylist() else: payload = json.loads(input_path.read_text(encoding="utf-8")) if not isinstance(payload, list): raise ValueError("HotpotQA JSON must contain a top-level list") rows = payload produced = 0 for row in rows: if not isinstance(row, Mapping): continue if limit is not None and produced >= limit: return context = row["context"] if isinstance(context, Mapping): titles = list(context["title"]) sentences = context["sentences"] paragraphs = { title: " ".join(sentence_list) for title, sentence_list in zip(titles, sentences, strict=True) } else: titles = [str(item[0]) for item in context] paragraphs = { str(item[0]): " ".join(str(sentence) for sentence in item[1]) for item in context } question = str(row["question"]) starts = tuple( title for title in titles if _mentions(question, title, min_length=spec.min_title_length) ) if require_start and not starts: continue supporting_facts = row["supporting_facts"] if isinstance(supporting_facts, Mapping): supporting = tuple(dict.fromkeys(supporting_facts["title"])) else: supporting = tuple(dict.fromkeys(str(item[0]) for item in supporting_facts)) answer = str(row["answer"]) example_id = str(row.get("_id", row.get("id", ""))) graph = _build_graph(titles, paragraphs, spec, f"hotpotqa@{example_id}") query = GraphQuery( text=question, start_entities=starts, start_entity_ids=starts, min_hops=1, max_hops=max_hops, return_paths=True, verify=True, operation="find_entities", answer_mode="entity", ) yield HotpotExample( example_id=example_id, graph=graph, query=query, question=question, answer=answer, supporting_titles=supporting, start_titles=starts, level=str(row.get("level", "")), question_type=str(row.get("type", "")), status=ReasoningStatus.SUPPORTED, answer_is_title=answer in titles, ) produced += 1 def supporting_title_metrics( predicted_titles: Sequence[str], gold_titles: Sequence[str] ) -> dict[str, float]: """Precision, recall, F1 and exact match over supporting paragraph titles.""" predicted = set(predicted_titles) gold = set(gold_titles) if not gold: empty = float(not predicted) return {"precision": empty, "recall": empty, "f1": empty, "exact_match": empty} if not predicted: return {"precision": 0.0, "recall": 0.0, "f1": 0.0, "exact_match": 0.0} hits = len(predicted & gold) precision = hits / len(predicted) recall = hits / len(gold) f1 = ( 0.0 if precision + recall == 0 else 2 * precision * recall / (precision + recall) ) return { "precision": precision, "recall": recall, "f1": f1, "exact_match": float(predicted == gold), }