voxpreference / backend /pg2 /validation_import.py
system's picture
system HF Staff
Update VoxPreference
20fb2bc verified
Raw History Blame Contribute Delete
4.38 kB
from __future__ import annotations
import csv
import math
from collections import defaultdict
from io import TextIOBase
from typing import Any
MAX_ANNOTATION_ROWS = 5000
MAX_CSV_IPA_CHARS = 64
TRUTHY = {"1", "true", "yes", "y", "validated"}
def _clean(value: Any) -> str:
return str(value or "").strip()
def _to_float(value: Any, default: float) -> float:
try:
number = float(value)
except (TypeError, ValueError):
return default
if not math.isfinite(number):
return default
return number
def _to_bool(value: Any) -> bool:
return _clean(value).lower() in TRUTHY
def rows_from_csv(file_obj: TextIOBase) -> list[dict[str, str]]:
reader = csv.DictReader(file_obj)
return [dict(row) for row in reader]
def aggregate_annotation_rows(rows: list[dict[str, Any]]) -> dict[str, Any]:
if not rows:
return {"success": False, "errors": ["CSV file has no annotation rows."]}
if len(rows) > MAX_ANNOTATION_ROWS:
return {
"success": False,
"errors": [f"CSV has too many rows. Limit is {MAX_ANNOTATION_ROWS} rows."],
}
first = rows[0]
target_word = _clean(first.get("target_word") or first.get("word"))
reference_ipa = _clean(first.get("reference_ipa") or first.get("standard_ipa"))
grouped: dict[str, dict[str, Any]] = {}
reviewers_by_ipa: dict[str, set[str]] = defaultdict(set)
notes_by_ipa: dict[str, list[str]] = defaultdict(list)
labels_by_ipa: dict[str, set[str]] = defaultdict(set)
errors = []
for index, row in enumerate(rows, start=2):
ipa = _clean(row.get("ipa") or row.get("variant_ipa"))
if not ipa:
errors.append(f"Row {index}: ipa is required.")
continue
if len(ipa) > MAX_CSV_IPA_CHARS:
errors.append(f"Row {index}: ipa must be {MAX_CSV_IPA_CHARS} characters or fewer.")
continue
row_word = _clean(row.get("target_word") or row.get("word"))
if row_word and not target_word:
target_word = row_word
if row_word and target_word and row_word != target_word:
errors.append(f"Row {index}: target_word differs from {target_word}.")
count = max(0.0, _to_float(row.get("count"), 1.0))
confidence = max(0.0, min(1.0, _to_float(row.get("confidence"), 1.0)))
reviewer = _clean(row.get("reviewer") or row.get("validated_by"))
source_type = _clean(row.get("source_type") or row.get("source")) or "expert"
validated = _to_bool(row.get("validated")) or bool(reviewer and source_type in {"expert", "human"})
if ipa not in grouped:
grouped[ipa] = {
"ipa": ipa,
"count": 0.0,
"label": "",
"notes": "",
"source_type": source_type,
"reviewer": "",
"confidence": confidence,
"validated": validated,
}
grouped[ipa]["count"] += count
grouped[ipa]["confidence"] = min(grouped[ipa]["confidence"], confidence)
grouped[ipa]["validated"] = bool(grouped[ipa]["validated"] or validated)
if reviewer:
reviewers_by_ipa[ipa].add(reviewer)
if _clean(row.get("notes")):
notes_by_ipa[ipa].append(_clean(row.get("notes")))
if _clean(row.get("label")):
labels_by_ipa[ipa].add(_clean(row.get("label")))
if not target_word:
errors.append("target_word is required in the CSV.")
if errors:
return {"success": False, "errors": errors}
variants = []
for ipa, variant in grouped.items():
reviewers = sorted(reviewers_by_ipa[ipa])
variant["reviewer"] = ", ".join(reviewers) or None
variant["label"] = ", ".join(sorted(labels_by_ipa[ipa])) or None
variant["notes"] = " | ".join(notes_by_ipa[ipa]) or None
variants.append(variant)
variants.sort(key=lambda item: item["count"], reverse=True)
return {
"success": True,
"target_word": target_word,
"reference_ipa": reference_ipa,
"variants": variants,
"annotation_summary": {
"row_count": len(rows),
"variant_count": len(variants),
"reviewers": sorted({reviewer for reviewers in reviewers_by_ipa.values() for reviewer in reviewers}),
},
}