#!/usr/bin/env python """Build the frozen T2 taxonomy from the v1.1 annotations. Reads ``/annotations/vX/sY.json`` for S1-S8 (all annotated volunteers), applies the decided rules in ``pulse/taxonomy.py`` and writes the frozen JSON:: python scripts/build_taxonomy.py --root $PULSE_ROOT \ --pool splits/pool27.json --pool splits/pool33.json \ --out taxonomy/taxonomy_v1.1.json """ from __future__ import annotations import argparse import json import re import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from pulse import DATA_TAG # noqa: E402 from pulse.data.annotations import load_segments_file # noqa: E402 from pulse.paths import DataRoot, make_rid, vol_number # noqa: E402 from pulse.splits import load_split # noqa: E402 from pulse.taxonomy import NOUN_THRESHOLD, build_taxonomy, file_md5 # noqa: E402 def main(argv=None) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--root", default=None) ap.add_argument("--pool", action="append", default=[], help="split file(s) for per-pool counts") ap.add_argument("--threshold", type=int, default=NOUN_THRESHOLD) ap.add_argument("--out", required=True) args = ap.parse_args(argv) dr = DataRoot.from_arg(args.root) files = [] for vdir in sorted(dr.annotations_dir.iterdir(), key=lambda p: p.name): if not (vdir.is_dir() and re.match(r"^v\d+$", vdir.name)): continue for f in sorted(vdir.glob("s*.json")): if re.match(r"^s[1-8]\.json$", f.name): files.append((make_rid(vdir.name, f.stem), f)) files.sort(key=lambda x: (vol_number(x[0].split("s")[0]), int(x[0].split("s")[1]))) if not files: raise SystemExit(f"no annotation files under {dr.annotations_dir}") segs = [] for rid, f in files: for s in load_segments_file(f, rid): segs.append((rid, s.verb, s.noun, s.hand)) pools = {} for p in args.pool: sp = load_split(p) pools[sp.name] = sp.volunteers tax = build_taxonomy(segs, args.threshold, pools, n_files=len(files)) tax["data_tag"] = DATA_TAG tax["n_annotated_volunteers"] = len({rid.split("s")[0] for rid, _ in files}) tax["generated_by"] = "scripts/build_taxonomy.py" Path(args.out).parent.mkdir(parents=True, exist_ok=True) with open(args.out, "w", encoding="utf-8", newline="\n") as f: json.dump(tax, f, indent=2, ensure_ascii=False) f.write("\n") c = tax["counts"] print(f"{len(files)} files, {tax['n_annotated_volunteers']} volunteers, " f"{tax['n_segments_total']} segments; kept {tax['n_segments_kept']} " f"(dropped: verb {tax['n_segments_dropped_verb']}, rare noun " f"{tax['n_segments_dropped_noun']}, incomplete {tax['n_segments_incomplete']})") print(f"raw verbs: {c['verb_raw_all']}") print(f"raw hands: {c['hand_raw_all']}") print(f"{len(tax['verbs'])} fine verbs: " + ", ".join(f"{v} {n}" for v, n in c['verb_fine'].items())) print("composite: " + ", ".join(f"{v} {n}" for v, n in c['verb_composite'].items())) print(f"{len(tax['nouns'])} nouns (>= {args.threshold}): " + ", ".join(f"{n} {k}" for n, k in c['noun'].items())) print("below threshold: " + ", ".join(f"{n} {k}" for n, k in c['nouns_below_threshold'].items())) print(f"aliases applied: {c['noun_aliases_applied']}") print("chance (uniform): " + ", ".join(f"{k} {v:.4f}" for k, v in tax['chance']['uniform'].items())) print(f"majority: {json.dumps(tax['chance']['majority_all_volunteers'])}") for p, d in tax["pools"].items(): print(f"pool {p}: {d['n_segments_kept']}/{d['n_segments_all']} segments kept; " f"nouns reaching {args.threshold} within pool: {d['nouns_reaching_threshold_within_pool']}") print(f"wrote {args.out} (md5 {file_md5(args.out)})") return 0 if __name__ == "__main__": sys.exit(main())