Buckets:
| """cwm3/data/materialise_can_r2.py — CAN evidence dirs for the round-2 manifest via the H5 server's own path | |
| (cwm2/annotation_human/active_select.materialise_can + cwm2/data/build_evidence.build, whitelist-only canspec reader, DMS-free): | |
| writes cwm2/annotation_human/can_r2/<idx>.{prefix,outcome}.json for every clip of clips_r2.json (can/ of round 1 untouched).""" | |
| import json, sys, time, collections | |
| from pathlib import Path | |
| EXPLORE = Path("/home/USER/explore"); AH = EXPLORE / "cwm2" / "annotation_human"; ROOT = EXPLORE / "cwm3" | |
| sys.path.insert(0, str(AH)); sys.path.insert(0, str(EXPLORE / "cwm2" / "data")); sys.path.insert(0, str(ROOT / "common")) | |
| import active_select as AS # module import only (main guarded) | |
| import build_evidence as BE | |
| from runmeta import write_run_json | |
| tag = sys.argv[1] if len(sys.argv) > 1 else "r2" | |
| t0 = time.time(); man = json.load(open(AH / f"clips_{tag}.json")); can_dir = AH / f"can_{tag}"; can_dir.mkdir(exist_ok=True) | |
| BE.cs.load_can = BE.load_can_cached | |
| how = collections.Counter(); missing = []; modes = collections.Counter() | |
| for c in man["clips"]: | |
| c = dict(c); c.setdefault("mode_design", c.get("ctrl")) | |
| h = AS.materialise_can(c, can_dir, BE.build, print); how[h.split(":")[0]] += 1 | |
| if h == "missing": missing.append(c["idx"]) | |
| m, sp = AS.mode_at_anchor(can_dir, c["idx"]); modes[m] += 1 | |
| n_files = len(list(can_dir.glob("*.json"))) | |
| summ = dict(manifest=str(AH / f"clips_{tag}.json"), can_dir=str(can_dir), n_clips=len(man["clips"]), how=dict(how), missing_idx=missing, n_json_files=n_files, mode_at_anchor=dict(modes)) | |
| write_run_json(ROOT / "results", f"materialise_can_{tag}", [Path(__file__), AH / "active_select.py", EXPLORE / "cwm2" / "data" / "build_evidence.py"], [AH / f"clips_{tag}.json"], None, t0, extra={"summary": summ}, input_listing=False) | |
| print("CAN_DONE " + json.dumps(summ)); sys.exit(1 if missing else 0) | |
Xet Storage Details
- Size:
- 1.9 kB
- Xet hash:
- ba49ec79ad82be72116156143102599cc9cf21ffdfd2c53e67df402946a9b336
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.