File size: 4,026 Bytes
dffbc1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
#!/usr/bin/env python
"""Build the frozen T2 taxonomy from the v1.1 annotations.

Reads ``<root>/annotations/vX/sY.json`` for S1-S8 (all annotated volunteers),
applies the decided rules in ``pulse/taxonomy.py`` and writes the frozen JSON::

    python scripts/build_taxonomy.py --root $PULSE_ROOT \
        --pool splits/pool27.json --pool splits/pool33.json \
        --out taxonomy/taxonomy_v1.1.json
"""
from __future__ import annotations

import argparse
import json
import re
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from pulse import DATA_TAG  # noqa: E402
from pulse.data.annotations import load_segments_file  # noqa: E402
from pulse.paths import DataRoot, make_rid, vol_number  # noqa: E402
from pulse.splits import load_split  # noqa: E402
from pulse.taxonomy import NOUN_THRESHOLD, build_taxonomy, file_md5  # noqa: E402


def main(argv=None) -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--root", default=None)
    ap.add_argument("--pool", action="append", default=[], help="split file(s) for per-pool counts")
    ap.add_argument("--threshold", type=int, default=NOUN_THRESHOLD)
    ap.add_argument("--out", required=True)
    args = ap.parse_args(argv)

    dr = DataRoot.from_arg(args.root)
    files = []
    for vdir in sorted(dr.annotations_dir.iterdir(), key=lambda p: p.name):
        if not (vdir.is_dir() and re.match(r"^v\d+$", vdir.name)):
            continue
        for f in sorted(vdir.glob("s*.json")):
            if re.match(r"^s[1-8]\.json$", f.name):
                files.append((make_rid(vdir.name, f.stem), f))
    files.sort(key=lambda x: (vol_number(x[0].split("s")[0]), int(x[0].split("s")[1])))
    if not files:
        raise SystemExit(f"no annotation files under {dr.annotations_dir}")
    segs = []
    for rid, f in files:
        for s in load_segments_file(f, rid):
            segs.append((rid, s.verb, s.noun, s.hand))
    pools = {}
    for p in args.pool:
        sp = load_split(p)
        pools[sp.name] = sp.volunteers
    tax = build_taxonomy(segs, args.threshold, pools, n_files=len(files))
    tax["data_tag"] = DATA_TAG
    tax["n_annotated_volunteers"] = len({rid.split("s")[0] for rid, _ in files})
    tax["generated_by"] = "scripts/build_taxonomy.py"
    Path(args.out).parent.mkdir(parents=True, exist_ok=True)
    with open(args.out, "w", encoding="utf-8", newline="\n") as f:
        json.dump(tax, f, indent=2, ensure_ascii=False)
        f.write("\n")

    c = tax["counts"]
    print(f"{len(files)} files, {tax['n_annotated_volunteers']} volunteers, "
          f"{tax['n_segments_total']} segments; kept {tax['n_segments_kept']} "
          f"(dropped: verb {tax['n_segments_dropped_verb']}, rare noun "
          f"{tax['n_segments_dropped_noun']}, incomplete {tax['n_segments_incomplete']})")
    print(f"raw verbs: {c['verb_raw_all']}")
    print(f"raw hands: {c['hand_raw_all']}")
    print(f"{len(tax['verbs'])} fine verbs: " + ", ".join(f"{v} {n}" for v, n in c['verb_fine'].items()))
    print("composite: " + ", ".join(f"{v} {n}" for v, n in c['verb_composite'].items()))
    print(f"{len(tax['nouns'])} nouns (>= {args.threshold}): "
          + ", ".join(f"{n} {k}" for n, k in c['noun'].items()))
    print("below threshold: " + ", ".join(f"{n} {k}" for n, k in c['nouns_below_threshold'].items()))
    print(f"aliases applied: {c['noun_aliases_applied']}")
    print("chance (uniform): " + ", ".join(f"{k} {v:.4f}" for k, v in tax['chance']['uniform'].items()))
    print(f"majority: {json.dumps(tax['chance']['majority_all_volunteers'])}")
    for p, d in tax["pools"].items():
        print(f"pool {p}: {d['n_segments_kept']}/{d['n_segments_all']} segments kept; "
              f"nouns reaching {args.threshold} within pool: {d['nouns_reaching_threshold_within_pool']}")
    print(f"wrote {args.out} (md5 {file_md5(args.out)})")
    return 0


if __name__ == "__main__":
    sys.exit(main())