File size: 4,026 Bytes
dffbc1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | #!/usr/bin/env python
"""Build the frozen T2 taxonomy from the v1.1 annotations.
Reads ``<root>/annotations/vX/sY.json`` for S1-S8 (all annotated volunteers),
applies the decided rules in ``pulse/taxonomy.py`` and writes the frozen JSON::
python scripts/build_taxonomy.py --root $PULSE_ROOT \
--pool splits/pool27.json --pool splits/pool33.json \
--out taxonomy/taxonomy_v1.1.json
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from pulse import DATA_TAG # noqa: E402
from pulse.data.annotations import load_segments_file # noqa: E402
from pulse.paths import DataRoot, make_rid, vol_number # noqa: E402
from pulse.splits import load_split # noqa: E402
from pulse.taxonomy import NOUN_THRESHOLD, build_taxonomy, file_md5 # noqa: E402
def main(argv=None) -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--root", default=None)
ap.add_argument("--pool", action="append", default=[], help="split file(s) for per-pool counts")
ap.add_argument("--threshold", type=int, default=NOUN_THRESHOLD)
ap.add_argument("--out", required=True)
args = ap.parse_args(argv)
dr = DataRoot.from_arg(args.root)
files = []
for vdir in sorted(dr.annotations_dir.iterdir(), key=lambda p: p.name):
if not (vdir.is_dir() and re.match(r"^v\d+$", vdir.name)):
continue
for f in sorted(vdir.glob("s*.json")):
if re.match(r"^s[1-8]\.json$", f.name):
files.append((make_rid(vdir.name, f.stem), f))
files.sort(key=lambda x: (vol_number(x[0].split("s")[0]), int(x[0].split("s")[1])))
if not files:
raise SystemExit(f"no annotation files under {dr.annotations_dir}")
segs = []
for rid, f in files:
for s in load_segments_file(f, rid):
segs.append((rid, s.verb, s.noun, s.hand))
pools = {}
for p in args.pool:
sp = load_split(p)
pools[sp.name] = sp.volunteers
tax = build_taxonomy(segs, args.threshold, pools, n_files=len(files))
tax["data_tag"] = DATA_TAG
tax["n_annotated_volunteers"] = len({rid.split("s")[0] for rid, _ in files})
tax["generated_by"] = "scripts/build_taxonomy.py"
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
with open(args.out, "w", encoding="utf-8", newline="\n") as f:
json.dump(tax, f, indent=2, ensure_ascii=False)
f.write("\n")
c = tax["counts"]
print(f"{len(files)} files, {tax['n_annotated_volunteers']} volunteers, "
f"{tax['n_segments_total']} segments; kept {tax['n_segments_kept']} "
f"(dropped: verb {tax['n_segments_dropped_verb']}, rare noun "
f"{tax['n_segments_dropped_noun']}, incomplete {tax['n_segments_incomplete']})")
print(f"raw verbs: {c['verb_raw_all']}")
print(f"raw hands: {c['hand_raw_all']}")
print(f"{len(tax['verbs'])} fine verbs: " + ", ".join(f"{v} {n}" for v, n in c['verb_fine'].items()))
print("composite: " + ", ".join(f"{v} {n}" for v, n in c['verb_composite'].items()))
print(f"{len(tax['nouns'])} nouns (>= {args.threshold}): "
+ ", ".join(f"{n} {k}" for n, k in c['noun'].items()))
print("below threshold: " + ", ".join(f"{n} {k}" for n, k in c['nouns_below_threshold'].items()))
print(f"aliases applied: {c['noun_aliases_applied']}")
print("chance (uniform): " + ", ".join(f"{k} {v:.4f}" for k, v in tax['chance']['uniform'].items()))
print(f"majority: {json.dumps(tax['chance']['majority_all_volunteers'])}")
for p, d in tax["pools"].items():
print(f"pool {p}: {d['n_segments_kept']}/{d['n_segments_all']} segments kept; "
f"nouns reaching {args.threshold} within pool: {d['nouns_reaching_threshold_within_pool']}")
print(f"wrote {args.out} (md5 {file_md5(args.out)})")
return 0
if __name__ == "__main__":
sys.exit(main())
|