Download code/graph/real/hotpotqa.py from frontal-labs/graph-reasoner-typed-compositional: direct link, hf CLI and curl.
- Browser
- Download file 9.65 kB
-
https://huggingface.co/frontal-labs/graph-reasoner-typed-compositional/resolve/main/code/graph/real/hotpotqa.py
- Command line
-
hf download hf://frontal-labs/graph-reasoner-typed-compositional/code/graph/real/hotpotqa.py
-
curl -L -o hotpotqa.py https://huggingface.co/frontal-labs/graph-reasoner-typed-compositional/resolve/main/code/graph/real/hotpotqa.py
9.65 kB
| """HotpotQA as a real natural-language multi-hop graph reasoning workload. | |
| Why this construction | |
| --------------------- | |
| The upstream GFM-RAG pipeline turns paragraphs into a knowledge graph with an | |
| LLM (NER + OpenIE). That needs an LLM budget and hours of preprocessing, and it | |
| introduces the extractor's errors into the measurement. This module instead | |
| builds an **entity graph** directly from the data HotpotQA already ships, with no | |
| model in the loop: | |
| * nodes are the paragraph titles, which in HotpotQA are Wikipedia article titles | |
| and therefore entities; | |
| * an edge ``A -mentions-> B`` exists when B's title occurs in A's text; | |
| * the start entity is the title mentioned in the question; | |
| * the gold evidence is the ``supporting_facts`` titles. | |
| This is the entity-graph formulation used by multi-hop GNN work on HotpotQA. It | |
| is lossier than full OpenIE, and the honest consequence is stated in the metric: | |
| the primary measure is **supporting-title retrieval** -- did the returned path | |
| cover the gold supporting paragraphs -- because that is what a graph retriever is | |
| responsible for. Answer-span matching is reported only when the answer is itself | |
| a title, since a span answer is the language model's job, not the graph's. | |
| Unlike the KG workload, the bridge entity is *not* named in the question, so this | |
| does measure intent inference, on real text with real distractors. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import re | |
| from collections.abc import Iterator, Mapping, Sequence | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from graph.graph import EdgeRecord, GraphStore | |
| from graph.types import GraphQuery, ReasoningStatus | |
| MENTIONS = "mentions" | |
| #: Titles shorter than this are too generic to match reliably in free text | |
| #: ("It", "US"), and matching them creates spurious edges. | |
| MIN_TITLE_LENGTH = 4 | |
| class HotpotGraphSpec: | |
| """Controls for entity-graph construction.""" | |
| #: Include an edge when a title appears in another paragraph's text. | |
| link_on_mention: bool = True | |
| #: Also link paragraphs that share a rare capitalised token, which recovers | |
| #: bridges that the exact-title test misses. | |
| link_on_shared_entity: bool = False | |
| min_title_length: int = MIN_TITLE_LENGTH | |
| class HotpotExample: | |
| """One HotpotQA question as a graph reasoning problem.""" | |
| example_id: str | |
| graph: GraphStore | |
| query: GraphQuery | |
| question: str | |
| answer: str | |
| #: Gold supporting paragraph titles. | |
| supporting_titles: tuple[str, ...] | |
| #: Titles mentioned by the question, used as search anchors. | |
| start_titles: tuple[str, ...] | |
| level: str | |
| question_type: str | |
| status: ReasoningStatus | |
| #: True when the answer string is itself a paragraph title, so entity-level | |
| #: answer matching is meaningful for this question. | |
| answer_is_title: bool | |
| def _normalise(text: str) -> str: | |
| return re.sub(r"\s+", " ", text).strip().lower() | |
| def _mentions(text: str, title: str, *, min_length: int) -> bool: | |
| if len(title) < min_length: | |
| return False | |
| return _normalise(title) in _normalise(text) | |
| def _shared_entity_tokens(text: str) -> set[str]: | |
| """Return conservative capitalized entity tokens for bridge recovery.""" | |
| stopwords = { | |
| "After", | |
| "Before", | |
| "During", | |
| "From", | |
| "Later", | |
| "Most", | |
| "Other", | |
| "That", | |
| "The", | |
| "This", | |
| "When", | |
| "Where", | |
| "Which", | |
| "While", | |
| } | |
| return { | |
| token.lower() | |
| for token in re.findall(r"\b[A-Z][A-Za-zÀ-ÖØ-öø-ÿ]{3,}\b", text) | |
| if token not in stopwords | |
| } | |
| def _build_graph( | |
| titles: Sequence[str], | |
| paragraphs: Mapping[str, str], | |
| spec: HotpotGraphSpec, | |
| version: str, | |
| ) -> GraphStore: | |
| nodes = {title: "Paragraph" for title in titles} | |
| edges: list[EdgeRecord] = [] | |
| seen: set[tuple[str, str]] = set() | |
| title_tokens = {title: _shared_entity_tokens(title) for title in titles} | |
| for source in titles: | |
| text = paragraphs.get(source, "") | |
| source_tokens = _shared_entity_tokens(text) | |
| for target in titles: | |
| if source == target: | |
| continue | |
| if not spec.link_on_mention: | |
| continue | |
| mentioned = _mentions(text, target, min_length=spec.min_title_length) | |
| shared = spec.link_on_shared_entity and bool( | |
| source_tokens & title_tokens.get(target, set()) | |
| ) | |
| if not mentioned and not shared: | |
| continue | |
| if (source, target) in seen: | |
| continue | |
| seen.add((source, target)) | |
| edges.append( | |
| EdgeRecord( | |
| source_id=source, | |
| relation_id=MENTIONS, | |
| target_id=target, | |
| source_type="Paragraph", | |
| target_type="Paragraph", | |
| relation_name=MENTIONS, | |
| attributes={}, | |
| ) | |
| ) | |
| return GraphStore( | |
| nodes, | |
| edges, | |
| graph_version=version, | |
| # ``mentions`` is many-valued: a paragraph legitimately mentions several | |
| # others, so it must never be read as a contradiction. | |
| relation_attributes={MENTIONS: {"functional": "false"}}, | |
| ) | |
| def load_hotpotqa( | |
| path: str | Path, | |
| *, | |
| limit: int | None = None, | |
| spec: HotpotGraphSpec | None = None, | |
| require_start: bool = True, | |
| max_hops: int = 3, | |
| ) -> Iterator[HotpotExample]: | |
| """Stream HotpotQA JSON or Parquet examples as graph problems. | |
| ``require_start`` skips questions where no paragraph title occurs in the | |
| question text: without an anchor there is no graph-local starting point, and | |
| silently scoring those as failures would confuse entity linking with | |
| reasoning. The count of skipped questions is the caller's to report. | |
| """ | |
| spec = spec or HotpotGraphSpec() | |
| input_path = Path(path) | |
| if input_path.suffix == ".parquet": | |
| try: | |
| import pyarrow.parquet as pq | |
| except ImportError as error: # pragma: no cover - optional dependency | |
| raise RuntimeError("reading HotpotQA Parquet requires pyarrow") from error | |
| rows = pq.read_table(input_path).to_pylist() | |
| else: | |
| payload = json.loads(input_path.read_text(encoding="utf-8")) | |
| if not isinstance(payload, list): | |
| raise ValueError("HotpotQA JSON must contain a top-level list") | |
| rows = payload | |
| produced = 0 | |
| for row in rows: | |
| if not isinstance(row, Mapping): | |
| continue | |
| if limit is not None and produced >= limit: | |
| return | |
| context = row["context"] | |
| if isinstance(context, Mapping): | |
| titles = list(context["title"]) | |
| sentences = context["sentences"] | |
| paragraphs = { | |
| title: " ".join(sentence_list) | |
| for title, sentence_list in zip(titles, sentences, strict=True) | |
| } | |
| else: | |
| titles = [str(item[0]) for item in context] | |
| paragraphs = { | |
| str(item[0]): " ".join(str(sentence) for sentence in item[1]) | |
| for item in context | |
| } | |
| question = str(row["question"]) | |
| starts = tuple( | |
| title | |
| for title in titles | |
| if _mentions(question, title, min_length=spec.min_title_length) | |
| ) | |
| if require_start and not starts: | |
| continue | |
| supporting_facts = row["supporting_facts"] | |
| if isinstance(supporting_facts, Mapping): | |
| supporting = tuple(dict.fromkeys(supporting_facts["title"])) | |
| else: | |
| supporting = tuple(dict.fromkeys(str(item[0]) for item in supporting_facts)) | |
| answer = str(row["answer"]) | |
| example_id = str(row.get("_id", row.get("id", ""))) | |
| graph = _build_graph(titles, paragraphs, spec, f"hotpotqa@{example_id}") | |
| query = GraphQuery( | |
| text=question, | |
| start_entities=starts, | |
| start_entity_ids=starts, | |
| min_hops=1, | |
| max_hops=max_hops, | |
| return_paths=True, | |
| verify=True, | |
| operation="find_entities", | |
| answer_mode="entity", | |
| ) | |
| yield HotpotExample( | |
| example_id=example_id, | |
| graph=graph, | |
| query=query, | |
| question=question, | |
| answer=answer, | |
| supporting_titles=supporting, | |
| start_titles=starts, | |
| level=str(row.get("level", "")), | |
| question_type=str(row.get("type", "")), | |
| status=ReasoningStatus.SUPPORTED, | |
| answer_is_title=answer in titles, | |
| ) | |
| produced += 1 | |
| def supporting_title_metrics( | |
| predicted_titles: Sequence[str], gold_titles: Sequence[str] | |
| ) -> dict[str, float]: | |
| """Precision, recall, F1 and exact match over supporting paragraph titles.""" | |
| predicted = set(predicted_titles) | |
| gold = set(gold_titles) | |
| if not gold: | |
| empty = float(not predicted) | |
| return {"precision": empty, "recall": empty, "f1": empty, "exact_match": empty} | |
| if not predicted: | |
| return {"precision": 0.0, "recall": 0.0, "f1": 0.0, "exact_match": 0.0} | |
| hits = len(predicted & gold) | |
| precision = hits / len(predicted) | |
| recall = hits / len(gold) | |
| f1 = ( | |
| 0.0 | |
| if precision + recall == 0 | |
| else 2 * precision * recall / (precision + recall) | |
| ) | |
| return { | |
| "precision": precision, | |
| "recall": recall, | |
| "f1": f1, | |
| "exact_match": float(predicted == gold), | |
| } | |