Spaces:
Running on Zero
Running on Zero
| """Verbalising graph facts and scoring how well their evidence supports them (NLI). | |
| Papers describe their own work in the first person ("We propose X"), and NLI models do not | |
| treat "This paper proposes X" as entailed by that (measured with | |
| cross-encoder/nli-deberta-v3-small: 0.04 vs 1.00 for "We propose X."). Paper-subject facts are | |
| therefore verbalised in the first person, and an edge's support score is the maximum over its | |
| phrasings. Known weakness: metric names are poorly distinguished ("We report BLEU" entails | |
| "We report ROUGE." at 0.97), so verbatim quote anchoring remains the primary guard. | |
| """ | |
| from __future__ import annotations | |
| from .graph import KG | |
| from .schemas import Edge | |
| TEMPLATES = { | |
| "proposes": "{s} proposes {o}.", | |
| "uses": "{s} uses {o}.", | |
| "evaluates_on": "{s} is evaluated on {o}.", | |
| "addresses": "{s} addresses the task of {o}.", | |
| "reports_metric": "{s} reports results in terms of {o}.", | |
| "compares_with": "{s} is compared with {o}.", | |
| } | |
| PAPER_TEMPLATES = { | |
| "proposes": ["We propose {o}."], | |
| "uses": ["We use {o}."], | |
| "evaluates_on": ["We evaluate on {o}."], | |
| "addresses": ["This paper addresses the task of {o}.", "We address {o}."], | |
| "reports_metric": ["We report {o}."], | |
| } | |
| def verbalise(subject_name: str, relation: str, object_name: str, subject_is_paper: bool = False) -> str: | |
| """Primary phrasing (used in prompts and displays).""" | |
| return hypotheses(subject_name, relation, object_name, subject_is_paper)[0] | |
| def hypotheses(subject_name: str, relation: str, object_name: str, subject_is_paper: bool = False) -> list[str]: | |
| if subject_is_paper and relation in PAPER_TEMPLATES: | |
| return [t.format(o=object_name) for t in PAPER_TEMPLATES[relation]] | |
| return [TEMPLATES[relation].format(s=subject_name, o=object_name)] | |
| def edge_hypotheses(kg: KG, e: Edge) -> list[str]: | |
| subj = kg.nodes.get(e.subject) | |
| return hypotheses(kg.name(e.subject), e.relation, kg.name(e.object), subject_is_paper=bool(subj and subj.type == "Paper")) | |
| def verbalise_edge(kg: KG, e: Edge) -> str: | |
| return edge_hypotheses(kg, e)[0] | |
| def score_edge_list(kg: KG, edges: list[Edge], nli) -> dict[str, float]: | |
| """Max entailment probability over each edge's phrasings, given its own evidence quote. | |
| ``nli`` needs ``score_pairs(list[(premise, hypothesis)]) -> array[n, 2]`` (entail, contradict). | |
| """ | |
| pairs: list[tuple[str, str]] = [] | |
| owner: list[str] = [] | |
| for e in edges: | |
| for h in edge_hypotheses(kg, e): | |
| pairs.append((e.quote, h)) | |
| owner.append(e.edge_id) | |
| if not pairs: | |
| return {} | |
| scores = nli.score_pairs(pairs) | |
| out: dict[str, float] = {} | |
| for eid, s in zip(owner, scores): | |
| out[eid] = max(out.get(eid, 0.0), round(float(s[0]), 4)) | |
| return out | |
| def score_edges(kg: KG, nli) -> dict[str, float]: | |
| return score_edge_list(kg, list(kg.edges.values()), nli) | |