Buckets:
| # Harness recorded in each Trackio logbook dataset's trackio/index.json (+ tool names as a hint). | |
| import json, sys, time, collections as C, concurrent.futures as cf | |
| from huggingface_hub import hf_hub_download, HfApi | |
| H = sys.argv[1]; api = HfApi() | |
| ids = [json.loads(l)["id"] for l in open(f"{H}/inventory.jsonl") if json.loads(l)["harness"] == "repro-bot"] | |
| def one(rid): | |
| for a in range(4): | |
| try: | |
| files = [f for f in api.list_repo_files(rid, repo_type="dataset")] | |
| idx = next((f for f in files if f.endswith("trackio/index.json")), None) | |
| if not idx: return rid, ["no-index"], 0 | |
| d = json.load(open(hf_hub_download(rid, idx, repo_type="dataset", cache_dir=f"{H}/idx_cache"))) | |
| ss = d.get("sessions", d if isinstance(d, list) else []) | |
| prov = [str(s.get("provider")) for s in ss] | |
| ev = sum(int(s.get("event_count") or 0) for s in ss) | |
| return rid, prov or ["empty"], ev | |
| except Exception as e: | |
| time.sleep(3 * (a + 1)); err = type(e).__name__ | |
| return rid, ["error:" + err], 0 | |
| with cf.ThreadPoolExecutor(4) as ex: | |
| rows = list(ex.map(one, ids)) | |
| json.dump(rows, open(f"{H}/trackio_harness.json", "w")) | |
| c = C.Counter(); stub = C.Counter() | |
| for rid, prov, ev in rows: | |
| p = C.Counter(prov).most_common(1)[0][0]; c[p] += 1; stub[p] += ev < 20 | |
| print(len(rows), "datasets"); print("dataset main provider:", c.most_common()); print("of which <20 events (stubs):", dict(stub)) | |
Xet Storage Details
- Size:
- 1.49 kB
- Xet hash:
- 65c1c2f86d617b235eb349beb76633bb02ec5fd5a2f2a1c55c4ac5beb5fd3bc2
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.