File size: 4,378 Bytes
20fb2bc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
from __future__ import annotations

import csv
import math
from collections import defaultdict
from io import TextIOBase
from typing import Any

MAX_ANNOTATION_ROWS = 5000
MAX_CSV_IPA_CHARS = 64

TRUTHY = {"1", "true", "yes", "y", "validated"}


def _clean(value: Any) -> str:
    return str(value or "").strip()


def _to_float(value: Any, default: float) -> float:
    try:
        number = float(value)
    except (TypeError, ValueError):
        return default
    if not math.isfinite(number):
        return default
    return number


def _to_bool(value: Any) -> bool:
    return _clean(value).lower() in TRUTHY


def rows_from_csv(file_obj: TextIOBase) -> list[dict[str, str]]:
    reader = csv.DictReader(file_obj)
    return [dict(row) for row in reader]


def aggregate_annotation_rows(rows: list[dict[str, Any]]) -> dict[str, Any]:
    if not rows:
        return {"success": False, "errors": ["CSV file has no annotation rows."]}
    if len(rows) > MAX_ANNOTATION_ROWS:
        return {
            "success": False,
            "errors": [f"CSV has too many rows. Limit is {MAX_ANNOTATION_ROWS} rows."],
        }

    first = rows[0]
    target_word = _clean(first.get("target_word") or first.get("word"))
    reference_ipa = _clean(first.get("reference_ipa") or first.get("standard_ipa"))
    grouped: dict[str, dict[str, Any]] = {}
    reviewers_by_ipa: dict[str, set[str]] = defaultdict(set)
    notes_by_ipa: dict[str, list[str]] = defaultdict(list)
    labels_by_ipa: dict[str, set[str]] = defaultdict(set)
    errors = []

    for index, row in enumerate(rows, start=2):
        ipa = _clean(row.get("ipa") or row.get("variant_ipa"))
        if not ipa:
            errors.append(f"Row {index}: ipa is required.")
            continue
        if len(ipa) > MAX_CSV_IPA_CHARS:
            errors.append(f"Row {index}: ipa must be {MAX_CSV_IPA_CHARS} characters or fewer.")
            continue

        row_word = _clean(row.get("target_word") or row.get("word"))
        if row_word and not target_word:
            target_word = row_word
        if row_word and target_word and row_word != target_word:
            errors.append(f"Row {index}: target_word differs from {target_word}.")

        count = max(0.0, _to_float(row.get("count"), 1.0))
        confidence = max(0.0, min(1.0, _to_float(row.get("confidence"), 1.0)))
        reviewer = _clean(row.get("reviewer") or row.get("validated_by"))
        source_type = _clean(row.get("source_type") or row.get("source")) or "expert"
        validated = _to_bool(row.get("validated")) or bool(reviewer and source_type in {"expert", "human"})

        if ipa not in grouped:
            grouped[ipa] = {
                "ipa": ipa,
                "count": 0.0,
                "label": "",
                "notes": "",
                "source_type": source_type,
                "reviewer": "",
                "confidence": confidence,
                "validated": validated,
            }

        grouped[ipa]["count"] += count
        grouped[ipa]["confidence"] = min(grouped[ipa]["confidence"], confidence)
        grouped[ipa]["validated"] = bool(grouped[ipa]["validated"] or validated)
        if reviewer:
            reviewers_by_ipa[ipa].add(reviewer)
        if _clean(row.get("notes")):
            notes_by_ipa[ipa].append(_clean(row.get("notes")))
        if _clean(row.get("label")):
            labels_by_ipa[ipa].add(_clean(row.get("label")))

    if not target_word:
        errors.append("target_word is required in the CSV.")
    if errors:
        return {"success": False, "errors": errors}

    variants = []
    for ipa, variant in grouped.items():
        reviewers = sorted(reviewers_by_ipa[ipa])
        variant["reviewer"] = ", ".join(reviewers) or None
        variant["label"] = ", ".join(sorted(labels_by_ipa[ipa])) or None
        variant["notes"] = " | ".join(notes_by_ipa[ipa]) or None
        variants.append(variant)

    variants.sort(key=lambda item: item["count"], reverse=True)
    return {
        "success": True,
        "target_word": target_word,
        "reference_ipa": reference_ipa,
        "variants": variants,
        "annotation_summary": {
            "row_count": len(rows),
            "variant_count": len(variants),
            "reviewers": sorted({reviewer for reviewers in reviewers_by_ipa.values() for reviewer in reviewers}),
        },
    }