Download scripts/build_taxonomy.py from velvet-pine-22/PULSE-code: direct link, hf CLI and curl.
- Browser
- Download file 4.03 kB
-
https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/scripts/build_taxonomy.py
- Command line
-
hf download hf://velvet-pine-22/PULSE-code/scripts/build_taxonomy.py
-
curl -L -o build_taxonomy.py https://huggingface.co/velvet-pine-22/PULSE-code/resolve/main/scripts/build_taxonomy.py
4.03 kB
| #!/usr/bin/env python | |
| """Build the frozen T2 taxonomy from the v1.1 annotations. | |
| Reads ``<root>/annotations/vX/sY.json`` for S1-S8 (all annotated volunteers), | |
| applies the decided rules in ``pulse/taxonomy.py`` and writes the frozen JSON:: | |
| python scripts/build_taxonomy.py --root $PULSE_ROOT \ | |
| --pool splits/pool27.json --pool splits/pool33.json \ | |
| --out taxonomy/taxonomy_v1.1.json | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import re | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from pulse import DATA_TAG # noqa: E402 | |
| from pulse.data.annotations import load_segments_file # noqa: E402 | |
| from pulse.paths import DataRoot, make_rid, vol_number # noqa: E402 | |
| from pulse.splits import load_split # noqa: E402 | |
| from pulse.taxonomy import NOUN_THRESHOLD, build_taxonomy, file_md5 # noqa: E402 | |
| def main(argv=None) -> int: | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("--root", default=None) | |
| ap.add_argument("--pool", action="append", default=[], help="split file(s) for per-pool counts") | |
| ap.add_argument("--threshold", type=int, default=NOUN_THRESHOLD) | |
| ap.add_argument("--out", required=True) | |
| args = ap.parse_args(argv) | |
| dr = DataRoot.from_arg(args.root) | |
| files = [] | |
| for vdir in sorted(dr.annotations_dir.iterdir(), key=lambda p: p.name): | |
| if not (vdir.is_dir() and re.match(r"^v\d+$", vdir.name)): | |
| continue | |
| for f in sorted(vdir.glob("s*.json")): | |
| if re.match(r"^s[1-8]\.json$", f.name): | |
| files.append((make_rid(vdir.name, f.stem), f)) | |
| files.sort(key=lambda x: (vol_number(x[0].split("s")[0]), int(x[0].split("s")[1]))) | |
| if not files: | |
| raise SystemExit(f"no annotation files under {dr.annotations_dir}") | |
| segs = [] | |
| for rid, f in files: | |
| for s in load_segments_file(f, rid): | |
| segs.append((rid, s.verb, s.noun, s.hand)) | |
| pools = {} | |
| for p in args.pool: | |
| sp = load_split(p) | |
| pools[sp.name] = sp.volunteers | |
| tax = build_taxonomy(segs, args.threshold, pools, n_files=len(files)) | |
| tax["data_tag"] = DATA_TAG | |
| tax["n_annotated_volunteers"] = len({rid.split("s")[0] for rid, _ in files}) | |
| tax["generated_by"] = "scripts/build_taxonomy.py" | |
| Path(args.out).parent.mkdir(parents=True, exist_ok=True) | |
| with open(args.out, "w", encoding="utf-8", newline="\n") as f: | |
| json.dump(tax, f, indent=2, ensure_ascii=False) | |
| f.write("\n") | |
| c = tax["counts"] | |
| print(f"{len(files)} files, {tax['n_annotated_volunteers']} volunteers, " | |
| f"{tax['n_segments_total']} segments; kept {tax['n_segments_kept']} " | |
| f"(dropped: verb {tax['n_segments_dropped_verb']}, rare noun " | |
| f"{tax['n_segments_dropped_noun']}, incomplete {tax['n_segments_incomplete']})") | |
| print(f"raw verbs: {c['verb_raw_all']}") | |
| print(f"raw hands: {c['hand_raw_all']}") | |
| print(f"{len(tax['verbs'])} fine verbs: " + ", ".join(f"{v} {n}" for v, n in c['verb_fine'].items())) | |
| print("composite: " + ", ".join(f"{v} {n}" for v, n in c['verb_composite'].items())) | |
| print(f"{len(tax['nouns'])} nouns (>= {args.threshold}): " | |
| + ", ".join(f"{n} {k}" for n, k in c['noun'].items())) | |
| print("below threshold: " + ", ".join(f"{n} {k}" for n, k in c['nouns_below_threshold'].items())) | |
| print(f"aliases applied: {c['noun_aliases_applied']}") | |
| print("chance (uniform): " + ", ".join(f"{k} {v:.4f}" for k, v in tax['chance']['uniform'].items())) | |
| print(f"majority: {json.dumps(tax['chance']['majority_all_volunteers'])}") | |
| for p, d in tax["pools"].items(): | |
| print(f"pool {p}: {d['n_segments_kept']}/{d['n_segments_all']} segments kept; " | |
| f"nouns reaching {args.threshold} within pool: {d['nouns_reaching_threshold_within_pool']}") | |
| print(f"wrote {args.out} (md5 {file_md5(args.out)})") | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |