Buckets:
| # Dataset-level inventory of agent-trace datasets: harness, size, dates, popularity, copies, card models. | |
| import json, re, sys, time, concurrent.futures as cf | |
| from huggingface_hub import HfApi | |
| S = sys.argv[1]; api = HfApi() | |
| probe = {l.split("\t")[0]: l.rstrip("\n").split("\t") for l in open(f"{S}/probe.tsv")} | |
| sniff = {l.split("\t")[0]: (l.rstrip("\n").split("\t") + ["", ""])[2] for l in open(f"{S}/hub/sniff_other.tsv")} | |
| MODELS = re.compile(r"\b(claude[- ]?(?:opus|sonnet|haiku|fable)?[- ]?[\d.]*|fable[- ]?\d[\d.]*|opus[- ]?\d[\d.]*|sonnet[- ]?\d[\d.]*|gpt-?\d[\w.-]*|o\d-?(?:mini|pro)?|codex[\w.-]*|gemini[- ]?[\d.]+[\w-]*|qwen[\w.-]*|kimi[\w.-]*|glm-?[\d.]+[\w-]*|deepseek[\w.-]*|minimax[\w.-]*|gemma[\w.-]*|llama[\w.-]*|mistral[\w.-]*|grok[\w.-]*|mimo[\w.-]*)", re.I) | |
| def harness(rid): | |
| kind = probe.get(rid, [rid, "err"])[1] | |
| if kind in ("pi", "claude-code"): return kind | |
| s = sniff.get(rid, "") | |
| if "session_meta" in s or "'payload'" in s: return "codex" | |
| if "Agent repro run" in s or "Reproduction session" in s or ("schema_version" in s and "sessions" in s): return "repro-bot" | |
| if '"source": "cli"' in s or "hermes-agent" in s: return "hermes" | |
| if "sessionTitle" in s or "hostId" in s: return "factory" | |
| if "claude_code_version" in s: return "claude-code" | |
| return "other" if kind != "err" else "err" | |
| def info(rid): | |
| for a in range(5): | |
| try: | |
| d = api.dataset_info(rid) | |
| card = d.card_data.to_dict() if d.card_data else {} | |
| txt = " ".join(map(str, [card.get("pretty_name", ""), card.get("tags", ""), d.tags, rid])) | |
| try: | |
| readme = api.hf_hub_download(rid, "README.md", repo_type="dataset", cache_dir=f"{S}/hub/inv_cache") if any(s.rfilename == "README.md" for s in (d.siblings or [])) else None | |
| if readme: txt += " " + open(readme, errors="replace").read()[:20000] | |
| except Exception: pass | |
| models = sorted({m.lower().strip(" -") for m in MODELS.findall(txt) if len(m) > 3})[:15] | |
| p = probe.get(rid, [rid, "err", "0", "0"]) | |
| return {"id": rid, "owner": rid.split("/")[0], "harness": harness(rid), "nfiles": int(p[2] or 0), "bytes": int(p[3] or 0), | |
| "created_at": str(d.created_at), "last_modified": str(d.last_modified), "downloads": d.downloads, "likes": d.likes, | |
| "tags": d.tags, "license": card.get("license"), "card_models": models, "gated": d.gated, "private": d.private} | |
| except Exception as e: | |
| if "ConnectError" in type(e).__name__ or "Timeout" in type(e).__name__ or "429" in str(e): time.sleep(5 * (a + 1)); continue | |
| p = probe.get(rid, [rid, "err", "0", "0"]) | |
| return {"id": rid, "owner": rid.split("/")[0], "harness": harness(rid), "nfiles": int(p[2] or 0), "bytes": int(p[3] or 0), "error": type(e).__name__} | |
| return {"id": rid, "error": "retries"} | |
| ids = list(probe) | |
| with cf.ThreadPoolExecutor(5) as ex: rows = list(ex.map(info, ids)) | |
| rows.sort(key=lambda r: r.get("created_at") or "9999") | |
| first = {} | |
| for r in rows: | |
| sig = (r.get("nfiles"), r.get("bytes")) | |
| r["copy_of"] = first.get(sig) if r.get("bytes") else None | |
| if r.get("bytes") and sig not in first: first[sig] = r["id"] | |
| with open(f"{S}/hub/inventory.jsonl", "w") as fo: | |
| for r in rows: fo.write(json.dumps(r) + "\n") | |
| print(len(rows), sum("error" in r for r in rows), "errors") | |
Xet Storage Details
- Size:
- 3.41 kB
- Xet hash:
- 6aeaa3df864d2d67624dd09dae80f2e47e4a710262d31f117146e10837159daf
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.