Spaces:
Running
Running
Download backend/pg2/validation_import.py from thebickersteth/voxpreference: direct link, hf CLI and curl.
- Browser
- Download file 4.38 kB
-
https://huggingface.co/spaces/thebickersteth/voxpreference/resolve/main/backend/pg2/validation_import.py
- Command line
-
hf download hf://spaces/thebickersteth/voxpreference/backend/pg2/validation_import.py
-
curl -L -o validation_import.py https://huggingface.co/spaces/thebickersteth/voxpreference/resolve/main/backend/pg2/validation_import.py
4.38 kB
| from __future__ import annotations | |
| import csv | |
| import math | |
| from collections import defaultdict | |
| from io import TextIOBase | |
| from typing import Any | |
| MAX_ANNOTATION_ROWS = 5000 | |
| MAX_CSV_IPA_CHARS = 64 | |
| TRUTHY = {"1", "true", "yes", "y", "validated"} | |
| def _clean(value: Any) -> str: | |
| return str(value or "").strip() | |
| def _to_float(value: Any, default: float) -> float: | |
| try: | |
| number = float(value) | |
| except (TypeError, ValueError): | |
| return default | |
| if not math.isfinite(number): | |
| return default | |
| return number | |
| def _to_bool(value: Any) -> bool: | |
| return _clean(value).lower() in TRUTHY | |
| def rows_from_csv(file_obj: TextIOBase) -> list[dict[str, str]]: | |
| reader = csv.DictReader(file_obj) | |
| return [dict(row) for row in reader] | |
| def aggregate_annotation_rows(rows: list[dict[str, Any]]) -> dict[str, Any]: | |
| if not rows: | |
| return {"success": False, "errors": ["CSV file has no annotation rows."]} | |
| if len(rows) > MAX_ANNOTATION_ROWS: | |
| return { | |
| "success": False, | |
| "errors": [f"CSV has too many rows. Limit is {MAX_ANNOTATION_ROWS} rows."], | |
| } | |
| first = rows[0] | |
| target_word = _clean(first.get("target_word") or first.get("word")) | |
| reference_ipa = _clean(first.get("reference_ipa") or first.get("standard_ipa")) | |
| grouped: dict[str, dict[str, Any]] = {} | |
| reviewers_by_ipa: dict[str, set[str]] = defaultdict(set) | |
| notes_by_ipa: dict[str, list[str]] = defaultdict(list) | |
| labels_by_ipa: dict[str, set[str]] = defaultdict(set) | |
| errors = [] | |
| for index, row in enumerate(rows, start=2): | |
| ipa = _clean(row.get("ipa") or row.get("variant_ipa")) | |
| if not ipa: | |
| errors.append(f"Row {index}: ipa is required.") | |
| continue | |
| if len(ipa) > MAX_CSV_IPA_CHARS: | |
| errors.append(f"Row {index}: ipa must be {MAX_CSV_IPA_CHARS} characters or fewer.") | |
| continue | |
| row_word = _clean(row.get("target_word") or row.get("word")) | |
| if row_word and not target_word: | |
| target_word = row_word | |
| if row_word and target_word and row_word != target_word: | |
| errors.append(f"Row {index}: target_word differs from {target_word}.") | |
| count = max(0.0, _to_float(row.get("count"), 1.0)) | |
| confidence = max(0.0, min(1.0, _to_float(row.get("confidence"), 1.0))) | |
| reviewer = _clean(row.get("reviewer") or row.get("validated_by")) | |
| source_type = _clean(row.get("source_type") or row.get("source")) or "expert" | |
| validated = _to_bool(row.get("validated")) or bool(reviewer and source_type in {"expert", "human"}) | |
| if ipa not in grouped: | |
| grouped[ipa] = { | |
| "ipa": ipa, | |
| "count": 0.0, | |
| "label": "", | |
| "notes": "", | |
| "source_type": source_type, | |
| "reviewer": "", | |
| "confidence": confidence, | |
| "validated": validated, | |
| } | |
| grouped[ipa]["count"] += count | |
| grouped[ipa]["confidence"] = min(grouped[ipa]["confidence"], confidence) | |
| grouped[ipa]["validated"] = bool(grouped[ipa]["validated"] or validated) | |
| if reviewer: | |
| reviewers_by_ipa[ipa].add(reviewer) | |
| if _clean(row.get("notes")): | |
| notes_by_ipa[ipa].append(_clean(row.get("notes"))) | |
| if _clean(row.get("label")): | |
| labels_by_ipa[ipa].add(_clean(row.get("label"))) | |
| if not target_word: | |
| errors.append("target_word is required in the CSV.") | |
| if errors: | |
| return {"success": False, "errors": errors} | |
| variants = [] | |
| for ipa, variant in grouped.items(): | |
| reviewers = sorted(reviewers_by_ipa[ipa]) | |
| variant["reviewer"] = ", ".join(reviewers) or None | |
| variant["label"] = ", ".join(sorted(labels_by_ipa[ipa])) or None | |
| variant["notes"] = " | ".join(notes_by_ipa[ipa]) or None | |
| variants.append(variant) | |
| variants.sort(key=lambda item: item["count"], reverse=True) | |
| return { | |
| "success": True, | |
| "target_word": target_word, | |
| "reference_ipa": reference_ipa, | |
| "variants": variants, | |
| "annotation_summary": { | |
| "row_count": len(rows), | |
| "variant_count": len(variants), | |
| "reviewers": sorted({reviewer for reviewers in reviewers_by_ipa.values() for reviewer in reviewers}), | |
| }, | |
| } | |