Buckets:
| # Session stats for paper-reproduction bot datasets (Trackio Logbook format). | |
| import json, random, sys, time, collections, concurrent.futures as cf | |
| from huggingface_hub import HfApi, hf_hub_download | |
| S = sys.argv[1]; api = HfApi(); CACHE = f"{S}/hub/repro_cache" | |
| inv = [json.loads(l) for l in open(f"{S}/hub/inventory.jsonl")] | |
| ids = sorted(r["id"] for r in inv if r.get("harness") == "repro-bot") | |
| random.Random(0).shuffle(ids); ids = ids[:60] | |
| def dl(rid, f): | |
| for a in range(4): | |
| try: return hf_hub_download(rid, f, repo_type="dataset", cache_dir=CACHE) | |
| except Exception: | |
| if a == 3: raise | |
| time.sleep(5 * (a + 1)) | |
| def one(rid): | |
| out = [] | |
| try: | |
| tree = {f.path: f.size for f in api.list_repo_tree(rid, repo_type="dataset", recursive=True) if getattr(f, "size", None)} | |
| idx = next((p for p in tree if p.endswith("trackio/index.json")), None) | |
| if not idx: return rid, [], "no-index" | |
| root = idx.rsplit("index.json", 1)[0] | |
| top = json.load(open(dl(rid, idx))) | |
| for s in top.get("sessions", []): | |
| row = {"dataset": rid, "paper_slug": rid.split("/repro-", 1)[-1].removesuffix("-traces"), **{k: s.get(k) for k in | |
| ("id", "title", "provider", "model", "started_at", "ended_at", "duration_ms", "event_count", "turn_count")}} | |
| kinds, tools, errs, usage, first_user = collections.Counter(), collections.Counter(), 0, collections.Counter(), None | |
| sidx = root + (s.get("index_file") or "") | |
| files = [] | |
| if sidx in tree: | |
| files = [root + c["file"] for c in json.load(open(dl(rid, sidx))).get("chunks", [])] | |
| if sum(tree.get(f, 0) for f in files) < 20_000_000: | |
| for f in files: | |
| for e in json.load(open(dl(rid, f))).get("events", []): | |
| k = e.get("kind"); kinds[k] += 1 | |
| if k == "tool_call": tools[e.get("tool_name") or e.get("title")] += 1 | |
| if k == "tool_result" and e.get("status") not in (None, "success", "completed", "ok"): errs += 1 | |
| for uk, uv in (e.get("usage") or {}).items(): | |
| if isinstance(uv, (int, float)): usage[uk] += uv | |
| if k in ("user", "user_message") and first_user is None: first_user = (e.get("text") or "")[:400] | |
| row.update(event_kinds=dict(kinds), tools=dict(tools), tool_calls=sum(tools.values()), tool_errors=errs, | |
| usage=dict(usage), first_user=first_user, events_parsed=True) | |
| else: | |
| row["events_parsed"] = False | |
| out.append(row) | |
| return rid, out, None | |
| except Exception as e: | |
| return rid, out, type(e).__name__ | |
| with cf.ThreadPoolExecutor(5) as ex, open(f"{S}/hub/repro_sessions.jsonl", "w") as fo: | |
| for rid, rows, err in ex.map(one, ids): | |
| for r in rows: fo.write(json.dumps(r, ensure_ascii=False) + "\n") | |
| print(rid[:70], len(rows), err or "", flush=True) | |
Xet Storage Details
- Size:
- 3.05 kB
- Xet hash:
- a4de90d9eae44b353a3e609a5410f3a4f3bbe26ba719ff8155d90af50455946d
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.