""" Utilities for parsing, analyzing, and formatting CoNLL-U strings. Extracts token tables, compares trees, and identifies syntactic differences. """ from collections import OrderedDict from typing import List, Dict, Any, Tuple, Optional import pandas as pd class BoundedParseCache: """ In-memory LRU cache with a configurable maximum item capacity. Stores CoNLL-U parsed outputs keyed by (model_id, sentence). """ def __init__(self, maxsize: int = 128): self.maxsize = maxsize self._data: OrderedDict[Tuple[str, str], str] = OrderedDict() def get(self, model: str, sentence: str, strip_morphology: bool = True) -> Optional[str]: key = (model, sentence, strip_morphology) if key in self._data: self._data.move_to_end(key) return self._data[key] return None def set(self, model: str, sentence: str, conllu: str, strip_morphology: bool = True) -> None: key = (model, sentence, strip_morphology) if key in self._data: self._data.move_to_end(key) self._data[key] = conllu if len(self._data) > self.maxsize: self._data.popitem(last=False) def __len__(self) -> int: return len(self._data) def clear(self) -> None: self._data.clear() def conllu_to_table(conllu_text: str) -> pd.DataFrame: """ Parses a single sentence CoNLL-U string into a clean Pandas DataFrame. Includes ID, Form, Lemma, UPOS, FEATS, Head, and Deprel. """ rows = [] for line in conllu_text.splitlines(): line = line.strip() if not line or line.startswith("#"): continue cols = line.split("\t") if len(cols) >= 8: rows.append({ "ID": cols[0], "Form": cols[1], "Lemma": cols[2], "UPOS": cols[3], "FEATS": cols[5] if len(cols) > 5 else "_", "Head": cols[6], "Deprel": cols[7], }) if not rows: return pd.DataFrame(columns=["ID", "Form", "Lemma", "UPOS", "FEATS", "Head", "Deprel"]) return pd.DataFrame(rows) def extract_tree_signature(conllu_text: str) -> List[Tuple[str, str, str, str, str]]: """ Extracts a list of (ID, Form, UPOS, Head, Deprel) tokens for exact difference comparison. """ tokens = [] for line in conllu_text.splitlines(): line = line.strip() if not line or line.startswith("#"): continue cols = line.split("\t") if len(cols) >= 8 and not ("-" in cols[0] or "." in cols[0]): tokens.append((cols[0], cols[1], cols[3], cols[6], cols[7])) return tokens def compare_trees( conllu_a: str, name_a: str, conllu_b: str, name_b: str ) -> Dict[str, Any]: """ Compares two parsed trees token-by-token. Returns: { "identical": bool, "diff_summary": str, "diff_table": pd.DataFrame, "discrepancy_count": int } """ sig_a = extract_tree_signature(conllu_a) sig_b = extract_tree_signature(conllu_b) if not sig_a or not sig_b: return { "identical": False, "diff_summary": "Insufficient tree data to compare.", "diff_table": pd.DataFrame(), "discrepancy_count": 0, } # Head IDs only refer to corresponding words when tokenization aligns. if [(token[0], token[1]) for token in sig_a] != [(token[0], token[1]) for token in sig_b]: count_mismatch = len(sig_a) != len(sig_b) return { "identical": False, "diff_summary": "Tokenization differs; POS tags and dependency arcs cannot be compared by position.", "diff_table": pd.DataFrame([{ "Token": "Tokenization", f"{name_a} (Tokens)": " | ".join(f"{token[0]}: {token[1]}" for token in sig_a), f"{name_b} (Tokens)": " | ".join(f"{token[0]}: {token[1]}" for token in sig_b), "Difference": "Token Count Mismatch" if count_mismatch else "Token Alignment Mismatch", }]), "discrepancy_count": 1, } differences = [] for i in range(len(sig_a)): id_a, form_a, upos_a, head_a, rel_a = sig_a[i] id_b, form_b, upos_b, head_b, rel_b = sig_b[i] pos_diff = upos_a != upos_b dep_diff = (head_a != head_b) or (rel_a != rel_b) if pos_diff or dep_diff: differences.append({ "Token": f"{id_a}: {form_a}", f"{name_a} (UPOS)": upos_a, f"{name_b} (UPOS)": upos_b, f"{name_a} (Head → Deprel)": f"{head_a} → {rel_a}", f"{name_b} (Head → Deprel)": f"{head_b} → {rel_b}", "Difference": "POS & Syntax" if (pos_diff and dep_diff) else ("POS" if pos_diff else "Syntax Arc"), }) is_identical = len(differences) == 0 if is_identical: summary = f"Both parsers produced identical POS tags and dependency trees ({len(sig_a)} tokens)." else: summary = f"Detected {len(differences)} syntactic / POS difference(s) between **{name_a}** and **{name_b}**." df_diff = pd.DataFrame(differences) if differences else pd.DataFrame() return { "identical": is_identical, "diff_summary": summary, "diff_table": df_diff, "discrepancy_count": len(differences), }