Simon Clematide
Add tree resize handle, pedagogical guides, and UI/comparison enhancements
115fba1
Raw History Blame Contribute Delete
5.49 kB
"""
Utilities for parsing, analyzing, and formatting CoNLL-U strings.
Extracts token tables, compares trees, and identifies syntactic differences.
"""
from collections import OrderedDict
from typing import List, Dict, Any, Tuple, Optional
import pandas as pd
class BoundedParseCache:
"""
In-memory LRU cache with a configurable maximum item capacity.
Stores CoNLL-U parsed outputs keyed by (model_id, sentence).
"""
def __init__(self, maxsize: int = 128):
self.maxsize = maxsize
self._data: OrderedDict[Tuple[str, str], str] = OrderedDict()
def get(self, model: str, sentence: str, strip_morphology: bool = True) -> Optional[str]:
key = (model, sentence, strip_morphology)
if key in self._data:
self._data.move_to_end(key)
return self._data[key]
return None
def set(self, model: str, sentence: str, conllu: str, strip_morphology: bool = True) -> None:
key = (model, sentence, strip_morphology)
if key in self._data:
self._data.move_to_end(key)
self._data[key] = conllu
if len(self._data) > self.maxsize:
self._data.popitem(last=False)
def __len__(self) -> int:
return len(self._data)
def clear(self) -> None:
self._data.clear()
def conllu_to_table(conllu_text: str) -> pd.DataFrame:
"""
Parses a single sentence CoNLL-U string into a clean Pandas DataFrame.
Includes ID, Form, Lemma, UPOS, FEATS, Head, and Deprel.
"""
rows = []
for line in conllu_text.splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
cols = line.split("\t")
if len(cols) >= 8:
rows.append({
"ID": cols[0],
"Form": cols[1],
"Lemma": cols[2],
"UPOS": cols[3],
"FEATS": cols[5] if len(cols) > 5 else "_",
"Head": cols[6],
"Deprel": cols[7],
})
if not rows:
return pd.DataFrame(columns=["ID", "Form", "Lemma", "UPOS", "FEATS", "Head", "Deprel"])
return pd.DataFrame(rows)
def extract_tree_signature(conllu_text: str) -> List[Tuple[str, str, str, str, str]]:
"""
Extracts a list of (ID, Form, UPOS, Head, Deprel) tokens for exact difference comparison.
"""
tokens = []
for line in conllu_text.splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
cols = line.split("\t")
if len(cols) >= 8 and not ("-" in cols[0] or "." in cols[0]):
tokens.append((cols[0], cols[1], cols[3], cols[6], cols[7]))
return tokens
def compare_trees(
conllu_a: str,
name_a: str,
conllu_b: str,
name_b: str
) -> Dict[str, Any]:
"""
Compares two parsed trees token-by-token.
Returns:
{
"identical": bool,
"diff_summary": str,
"diff_table": pd.DataFrame,
"discrepancy_count": int
}
"""
sig_a = extract_tree_signature(conllu_a)
sig_b = extract_tree_signature(conllu_b)
if not sig_a or not sig_b:
return {
"identical": False,
"diff_summary": "Insufficient tree data to compare.",
"diff_table": pd.DataFrame(),
"discrepancy_count": 0,
}
# Head IDs only refer to corresponding words when tokenization aligns.
if [(token[0], token[1]) for token in sig_a] != [(token[0], token[1]) for token in sig_b]:
count_mismatch = len(sig_a) != len(sig_b)
return {
"identical": False,
"diff_summary": "Tokenization differs; POS tags and dependency arcs cannot be compared by position.",
"diff_table": pd.DataFrame([{
"Token": "Tokenization",
f"{name_a} (Tokens)": " | ".join(f"{token[0]}: {token[1]}" for token in sig_a),
f"{name_b} (Tokens)": " | ".join(f"{token[0]}: {token[1]}" for token in sig_b),
"Difference": "Token Count Mismatch" if count_mismatch else "Token Alignment Mismatch",
}]),
"discrepancy_count": 1,
}
differences = []
for i in range(len(sig_a)):
id_a, form_a, upos_a, head_a, rel_a = sig_a[i]
id_b, form_b, upos_b, head_b, rel_b = sig_b[i]
pos_diff = upos_a != upos_b
dep_diff = (head_a != head_b) or (rel_a != rel_b)
if pos_diff or dep_diff:
differences.append({
"Token": f"{id_a}: {form_a}",
f"{name_a} (UPOS)": upos_a,
f"{name_b} (UPOS)": upos_b,
f"{name_a} (Head → Deprel)": f"{head_a} → {rel_a}",
f"{name_b} (Head → Deprel)": f"{head_b} → {rel_b}",
"Difference": "POS & Syntax" if (pos_diff and dep_diff) else ("POS" if pos_diff else "Syntax Arc"),
})
is_identical = len(differences) == 0
if is_identical:
summary = f"Both parsers produced identical POS tags and dependency trees ({len(sig_a)} tokens)."
else:
summary = f"Detected {len(differences)} syntactic / POS difference(s) between **{name_a}** and **{name_b}**."
df_diff = pd.DataFrame(differences) if differences else pd.DataFrame()
return {
"identical": is_identical,
"diff_summary": summary,
"diff_table": df_diff,
"discrepancy_count": len(differences),
}