Download core/conllu_utils.py from simon-clmtd/universal-dependencies-visualizer: direct link, hf CLI and curl.
- Browser
- Download file 5.49 kB
-
https://huggingface.co/spaces/simon-clmtd/universal-dependencies-visualizer/resolve/main/core/conllu_utils.py
- Command line
-
hf download hf://spaces/simon-clmtd/universal-dependencies-visualizer/core/conllu_utils.py
-
curl -L -o conllu_utils.py https://huggingface.co/spaces/simon-clmtd/universal-dependencies-visualizer/resolve/main/core/conllu_utils.py
5.49 kB
| """ | |
| Utilities for parsing, analyzing, and formatting CoNLL-U strings. | |
| Extracts token tables, compares trees, and identifies syntactic differences. | |
| """ | |
| from collections import OrderedDict | |
| from typing import List, Dict, Any, Tuple, Optional | |
| import pandas as pd | |
| class BoundedParseCache: | |
| """ | |
| In-memory LRU cache with a configurable maximum item capacity. | |
| Stores CoNLL-U parsed outputs keyed by (model_id, sentence). | |
| """ | |
| def __init__(self, maxsize: int = 128): | |
| self.maxsize = maxsize | |
| self._data: OrderedDict[Tuple[str, str], str] = OrderedDict() | |
| def get(self, model: str, sentence: str, strip_morphology: bool = True) -> Optional[str]: | |
| key = (model, sentence, strip_morphology) | |
| if key in self._data: | |
| self._data.move_to_end(key) | |
| return self._data[key] | |
| return None | |
| def set(self, model: str, sentence: str, conllu: str, strip_morphology: bool = True) -> None: | |
| key = (model, sentence, strip_morphology) | |
| if key in self._data: | |
| self._data.move_to_end(key) | |
| self._data[key] = conllu | |
| if len(self._data) > self.maxsize: | |
| self._data.popitem(last=False) | |
| def __len__(self) -> int: | |
| return len(self._data) | |
| def clear(self) -> None: | |
| self._data.clear() | |
| def conllu_to_table(conllu_text: str) -> pd.DataFrame: | |
| """ | |
| Parses a single sentence CoNLL-U string into a clean Pandas DataFrame. | |
| Includes ID, Form, Lemma, UPOS, FEATS, Head, and Deprel. | |
| """ | |
| rows = [] | |
| for line in conllu_text.splitlines(): | |
| line = line.strip() | |
| if not line or line.startswith("#"): | |
| continue | |
| cols = line.split("\t") | |
| if len(cols) >= 8: | |
| rows.append({ | |
| "ID": cols[0], | |
| "Form": cols[1], | |
| "Lemma": cols[2], | |
| "UPOS": cols[3], | |
| "FEATS": cols[5] if len(cols) > 5 else "_", | |
| "Head": cols[6], | |
| "Deprel": cols[7], | |
| }) | |
| if not rows: | |
| return pd.DataFrame(columns=["ID", "Form", "Lemma", "UPOS", "FEATS", "Head", "Deprel"]) | |
| return pd.DataFrame(rows) | |
| def extract_tree_signature(conllu_text: str) -> List[Tuple[str, str, str, str, str]]: | |
| """ | |
| Extracts a list of (ID, Form, UPOS, Head, Deprel) tokens for exact difference comparison. | |
| """ | |
| tokens = [] | |
| for line in conllu_text.splitlines(): | |
| line = line.strip() | |
| if not line or line.startswith("#"): | |
| continue | |
| cols = line.split("\t") | |
| if len(cols) >= 8 and not ("-" in cols[0] or "." in cols[0]): | |
| tokens.append((cols[0], cols[1], cols[3], cols[6], cols[7])) | |
| return tokens | |
| def compare_trees( | |
| conllu_a: str, | |
| name_a: str, | |
| conllu_b: str, | |
| name_b: str | |
| ) -> Dict[str, Any]: | |
| """ | |
| Compares two parsed trees token-by-token. | |
| Returns: | |
| { | |
| "identical": bool, | |
| "diff_summary": str, | |
| "diff_table": pd.DataFrame, | |
| "discrepancy_count": int | |
| } | |
| """ | |
| sig_a = extract_tree_signature(conllu_a) | |
| sig_b = extract_tree_signature(conllu_b) | |
| if not sig_a or not sig_b: | |
| return { | |
| "identical": False, | |
| "diff_summary": "Insufficient tree data to compare.", | |
| "diff_table": pd.DataFrame(), | |
| "discrepancy_count": 0, | |
| } | |
| # Head IDs only refer to corresponding words when tokenization aligns. | |
| if [(token[0], token[1]) for token in sig_a] != [(token[0], token[1]) for token in sig_b]: | |
| count_mismatch = len(sig_a) != len(sig_b) | |
| return { | |
| "identical": False, | |
| "diff_summary": "Tokenization differs; POS tags and dependency arcs cannot be compared by position.", | |
| "diff_table": pd.DataFrame([{ | |
| "Token": "Tokenization", | |
| f"{name_a} (Tokens)": " | ".join(f"{token[0]}: {token[1]}" for token in sig_a), | |
| f"{name_b} (Tokens)": " | ".join(f"{token[0]}: {token[1]}" for token in sig_b), | |
| "Difference": "Token Count Mismatch" if count_mismatch else "Token Alignment Mismatch", | |
| }]), | |
| "discrepancy_count": 1, | |
| } | |
| differences = [] | |
| for i in range(len(sig_a)): | |
| id_a, form_a, upos_a, head_a, rel_a = sig_a[i] | |
| id_b, form_b, upos_b, head_b, rel_b = sig_b[i] | |
| pos_diff = upos_a != upos_b | |
| dep_diff = (head_a != head_b) or (rel_a != rel_b) | |
| if pos_diff or dep_diff: | |
| differences.append({ | |
| "Token": f"{id_a}: {form_a}", | |
| f"{name_a} (UPOS)": upos_a, | |
| f"{name_b} (UPOS)": upos_b, | |
| f"{name_a} (Head → Deprel)": f"{head_a} → {rel_a}", | |
| f"{name_b} (Head → Deprel)": f"{head_b} → {rel_b}", | |
| "Difference": "POS & Syntax" if (pos_diff and dep_diff) else ("POS" if pos_diff else "Syntax Arc"), | |
| }) | |
| is_identical = len(differences) == 0 | |
| if is_identical: | |
| summary = f"Both parsers produced identical POS tags and dependency trees ({len(sig_a)} tokens)." | |
| else: | |
| summary = f"Detected {len(differences)} syntactic / POS difference(s) between **{name_a}** and **{name_b}**." | |
| df_diff = pd.DataFrame(differences) if differences else pd.DataFrame() | |
| return { | |
| "identical": is_identical, | |
| "diff_summary": summary, | |
| "diff_table": df_diff, | |
| "discrepancy_count": len(differences), | |
| } | |