gabfon's picture
Upload folder using huggingface_hub
3e363e2 verified
Raw History Blame Contribute Delete
9.65 kB
"""HotpotQA as a real natural-language multi-hop graph reasoning workload.
Why this construction
---------------------
The upstream GFM-RAG pipeline turns paragraphs into a knowledge graph with an
LLM (NER + OpenIE). That needs an LLM budget and hours of preprocessing, and it
introduces the extractor's errors into the measurement. This module instead
builds an **entity graph** directly from the data HotpotQA already ships, with no
model in the loop:
* nodes are the paragraph titles, which in HotpotQA are Wikipedia article titles
and therefore entities;
* an edge ``A -mentions-> B`` exists when B's title occurs in A's text;
* the start entity is the title mentioned in the question;
* the gold evidence is the ``supporting_facts`` titles.
This is the entity-graph formulation used by multi-hop GNN work on HotpotQA. It
is lossier than full OpenIE, and the honest consequence is stated in the metric:
the primary measure is **supporting-title retrieval** -- did the returned path
cover the gold supporting paragraphs -- because that is what a graph retriever is
responsible for. Answer-span matching is reported only when the answer is itself
a title, since a span answer is the language model's job, not the graph's.
Unlike the KG workload, the bridge entity is *not* named in the question, so this
does measure intent inference, on real text with real distractors.
"""
from __future__ import annotations
import json
import re
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from graph.graph import EdgeRecord, GraphStore
from graph.types import GraphQuery, ReasoningStatus
MENTIONS = "mentions"
#: Titles shorter than this are too generic to match reliably in free text
#: ("It", "US"), and matching them creates spurious edges.
MIN_TITLE_LENGTH = 4
@dataclass(frozen=True)
class HotpotGraphSpec:
"""Controls for entity-graph construction."""
#: Include an edge when a title appears in another paragraph's text.
link_on_mention: bool = True
#: Also link paragraphs that share a rare capitalised token, which recovers
#: bridges that the exact-title test misses.
link_on_shared_entity: bool = False
min_title_length: int = MIN_TITLE_LENGTH
@dataclass(frozen=True)
class HotpotExample:
"""One HotpotQA question as a graph reasoning problem."""
example_id: str
graph: GraphStore
query: GraphQuery
question: str
answer: str
#: Gold supporting paragraph titles.
supporting_titles: tuple[str, ...]
#: Titles mentioned by the question, used as search anchors.
start_titles: tuple[str, ...]
level: str
question_type: str
status: ReasoningStatus
#: True when the answer string is itself a paragraph title, so entity-level
#: answer matching is meaningful for this question.
answer_is_title: bool
def _normalise(text: str) -> str:
return re.sub(r"\s+", " ", text).strip().lower()
def _mentions(text: str, title: str, *, min_length: int) -> bool:
if len(title) < min_length:
return False
return _normalise(title) in _normalise(text)
def _shared_entity_tokens(text: str) -> set[str]:
"""Return conservative capitalized entity tokens for bridge recovery."""
stopwords = {
"After",
"Before",
"During",
"From",
"Later",
"Most",
"Other",
"That",
"The",
"This",
"When",
"Where",
"Which",
"While",
}
return {
token.lower()
for token in re.findall(r"\b[A-Z][A-Za-zÀ-ÖØ-öø-ÿ]{3,}\b", text)
if token not in stopwords
}
def _build_graph(
titles: Sequence[str],
paragraphs: Mapping[str, str],
spec: HotpotGraphSpec,
version: str,
) -> GraphStore:
nodes = {title: "Paragraph" for title in titles}
edges: list[EdgeRecord] = []
seen: set[tuple[str, str]] = set()
title_tokens = {title: _shared_entity_tokens(title) for title in titles}
for source in titles:
text = paragraphs.get(source, "")
source_tokens = _shared_entity_tokens(text)
for target in titles:
if source == target:
continue
if not spec.link_on_mention:
continue
mentioned = _mentions(text, target, min_length=spec.min_title_length)
shared = spec.link_on_shared_entity and bool(
source_tokens & title_tokens.get(target, set())
)
if not mentioned and not shared:
continue
if (source, target) in seen:
continue
seen.add((source, target))
edges.append(
EdgeRecord(
source_id=source,
relation_id=MENTIONS,
target_id=target,
source_type="Paragraph",
target_type="Paragraph",
relation_name=MENTIONS,
attributes={},
)
)
return GraphStore(
nodes,
edges,
graph_version=version,
# ``mentions`` is many-valued: a paragraph legitimately mentions several
# others, so it must never be read as a contradiction.
relation_attributes={MENTIONS: {"functional": "false"}},
)
def load_hotpotqa(
path: str | Path,
*,
limit: int | None = None,
spec: HotpotGraphSpec | None = None,
require_start: bool = True,
max_hops: int = 3,
) -> Iterator[HotpotExample]:
"""Stream HotpotQA JSON or Parquet examples as graph problems.
``require_start`` skips questions where no paragraph title occurs in the
question text: without an anchor there is no graph-local starting point, and
silently scoring those as failures would confuse entity linking with
reasoning. The count of skipped questions is the caller's to report.
"""
spec = spec or HotpotGraphSpec()
input_path = Path(path)
if input_path.suffix == ".parquet":
try:
import pyarrow.parquet as pq
except ImportError as error: # pragma: no cover - optional dependency
raise RuntimeError("reading HotpotQA Parquet requires pyarrow") from error
rows = pq.read_table(input_path).to_pylist()
else:
payload = json.loads(input_path.read_text(encoding="utf-8"))
if not isinstance(payload, list):
raise ValueError("HotpotQA JSON must contain a top-level list")
rows = payload
produced = 0
for row in rows:
if not isinstance(row, Mapping):
continue
if limit is not None and produced >= limit:
return
context = row["context"]
if isinstance(context, Mapping):
titles = list(context["title"])
sentences = context["sentences"]
paragraphs = {
title: " ".join(sentence_list)
for title, sentence_list in zip(titles, sentences, strict=True)
}
else:
titles = [str(item[0]) for item in context]
paragraphs = {
str(item[0]): " ".join(str(sentence) for sentence in item[1])
for item in context
}
question = str(row["question"])
starts = tuple(
title
for title in titles
if _mentions(question, title, min_length=spec.min_title_length)
)
if require_start and not starts:
continue
supporting_facts = row["supporting_facts"]
if isinstance(supporting_facts, Mapping):
supporting = tuple(dict.fromkeys(supporting_facts["title"]))
else:
supporting = tuple(dict.fromkeys(str(item[0]) for item in supporting_facts))
answer = str(row["answer"])
example_id = str(row.get("_id", row.get("id", "")))
graph = _build_graph(titles, paragraphs, spec, f"hotpotqa@{example_id}")
query = GraphQuery(
text=question,
start_entities=starts,
start_entity_ids=starts,
min_hops=1,
max_hops=max_hops,
return_paths=True,
verify=True,
operation="find_entities",
answer_mode="entity",
)
yield HotpotExample(
example_id=example_id,
graph=graph,
query=query,
question=question,
answer=answer,
supporting_titles=supporting,
start_titles=starts,
level=str(row.get("level", "")),
question_type=str(row.get("type", "")),
status=ReasoningStatus.SUPPORTED,
answer_is_title=answer in titles,
)
produced += 1
def supporting_title_metrics(
predicted_titles: Sequence[str], gold_titles: Sequence[str]
) -> dict[str, float]:
"""Precision, recall, F1 and exact match over supporting paragraph titles."""
predicted = set(predicted_titles)
gold = set(gold_titles)
if not gold:
empty = float(not predicted)
return {"precision": empty, "recall": empty, "f1": empty, "exact_match": empty}
if not predicted:
return {"precision": 0.0, "recall": 0.0, "f1": 0.0, "exact_match": 0.0}
hits = len(predicted & gold)
precision = hits / len(predicted)
recall = hits / len(gold)
f1 = (
0.0
if precision + recall == 0
else 2 * precision * recall / (precision + recall)
)
return {
"precision": precision,
"recall": recall,
"f1": f1,
"exact_match": float(predicted == gold),
}