Buckets:
| # Aggregate hub/sessions.jsonl by dataset category x harness. | |
| # python -I agg_sessions.py <scratchpad> -> prints tables, writes hub/sessions_tagged.jsonl | |
| import collections as C, json, re, statistics as st, sys | |
| S = sys.argv[1] | |
| organic = set(json.load(open(f"{S}/hub/organic2.json"))) | |
| labels = [json.loads(l) for l in open(f"{S}/hub/labels.jsonl")] | |
| auto = C.defaultdict(list) | |
| for r in labels: | |
| auto[r["dataset"]].append(r.get("author") == "automated") | |
| automated = {d for d, v in auto.items() if sum(v) / len(v) > 0.5} | |
| def category(ds): | |
| if ds.startswith("AletheiaResearch/"): # generated "build X" sessions with scripted follow-ups | |
| return "synthetic-dump" | |
| if ds in organic: | |
| return "real" | |
| if "/repro-" in ds: | |
| return "paper-repro-bot" | |
| if ds in automated or re.search(r"hf-coding-tools|kernelbench|mentionsanalysis|benchmark|eval", ds, re.I): | |
| return "benchmark/harness" | |
| return "synthetic-dump" | |
| AGGREGATES = {"owenqwenllmwine/bigpi-1"} # re-uploads of other datasets | |
| rows, seen = [], set() | |
| lines = [json.loads(l) for l in open(f"{S}/hub/sessions.jsonl")] | |
| lines.sort(key=lambda r: r["dataset"] in AGGREGATES) # originals first | |
| for r in lines: | |
| key = (r["harness"], r["user_prompts"], r["assistant_msgs"], r["tool_calls"], r["tok_out"], r["tok_in"]) | |
| if key in seen: | |
| continue | |
| seen.add(key) | |
| r["category"] = category(r["dataset"]) | |
| r["model"] = max(r["models"], key=r["models"].get) if r["models"] else None | |
| rows.append(r) | |
| with open(f"{S}/hub/sessions_tagged.jsonl", "w") as fo: | |
| for r in rows: | |
| fo.write(json.dumps(r) + "\n") | |
| def q(v, p): | |
| v = sorted(v) | |
| return v[min(len(v) - 1, int(p * len(v)))] if v else 0 | |
| def fam(m): | |
| m = (m or "?").lower().split("/")[-1] | |
| for k in ("claude", "gpt", "glm", "kimi", "qwen", "deepseek", "gemini", "minimax", "grok", "mimo", "fable", "llama", "mistral", "gemma", "nemotron", "devstral", "o3", "o4"): | |
| if k in m: | |
| return k | |
| return m[:20] | |
| print(f"sessions after dedupe: {len(rows)}") | |
| groups = C.defaultdict(list) | |
| for r in rows: | |
| groups[(r["category"], r["harness"])].append(r) | |
| hdr = "| category | harness | sessions | datasets | user msgs p50/p90 | tool calls p50/p90 | active min p50/p90 | out tok p50 | tool err % | compaction % |" | |
| print(hdr) | |
| print("|" + "---|" * hdr.count("|")[:-1] if False else "|---|---|---|---|---|---|---|---|---|---|") | |
| for (cat, h), rs in sorted(groups.items(), key=lambda x: (x[0][0], -len(x[1]))): | |
| up = [r["user_prompts"] for r in rs]; tc = [r["tool_calls"] for r in rs]; am = [r["active_s"] / 60 for r in rs] | |
| ot = [r["tok_out"] for r in rs] | |
| err = 100 * sum(r["tool_errors"] for r in rs) / max(1, sum(r["tool_calls"] for r in rs)) | |
| comp = 100 * sum(r["compactions"] > 0 for r in rs) / len(rs) | |
| print(f"| {cat} | {h} | {len(rs)} | {len({r['dataset'] for r in rs})} | {q(up,.5)}/{q(up,.9)} | {q(tc,.5)}/{q(tc,.9)} | " | |
| f"{q(am,.5):.0f}/{q(am,.9):.0f} | {q(ot,.5)} | {err:.1f} | {comp:.1f} |") | |
| for cat in ("real", "synthetic-dump", "benchmark/harness", "paper-repro-bot"): | |
| rs = [r for r in rows if r["category"] == cat] | |
| if not rs: | |
| continue | |
| print(f"\n## {cat}: {len(rs)} sessions") | |
| print("model families:", C.Counter(fam(r["model"]) for r in rs).most_common(12)) | |
| print("top models:", C.Counter(r["model"] for r in rs).most_common(15)) | |
| print("providers:", C.Counter(p for r in rs for p in r["providers"]).most_common(10)) | |
| tools = C.Counter() | |
| for r in rs: | |
| tools.update(r["tools"]) | |
| tot = sum(tools.values()) | |
| print("top tools:", [(k, round(100 * v / tot, 1)) for k, v in tools.most_common(15)]) | |
| print("sessions with subagents %:", round(100 * sum(r["subagent_msgs"] > 0 for r in rs) / len(rs), 1), | |
| "| multi-model sessions %:", round(100 * sum(len(r["models"]) > 1 for r in rs) / len(rs), 1), | |
| "| aborted %:", round(100 * sum(r["aborts"] > 0 for r in rs) / len(rs), 1)) | |
| cost = [r["cost"] for r in rs if r["cost"]] | |
| if cost: | |
| print("pi cost/session $ p50/p90/sum:", round(q(cost, .5), 3), round(q(cost, .9), 2), round(sum(cost), 0)) | |
| wall = [r["wall_s"] / 3600 for r in rs] | |
| print("wall-clock hours p50/p90:", round(q(wall, .5), 2), round(q(wall, .9), 1)) | |
| buckets = C.Counter("1" if r["user_prompts"] <= 1 else "2-5" if r["user_prompts"] <= 5 else "6-20" if r["user_prompts"] <= 20 else ">20" for r in rs) | |
| print("user msgs buckets:", dict(buckets)) | |
| tcb = C.Counter("0" if r["tool_calls"] == 0 else "1-10" if r["tool_calls"] <= 10 else "11-50" if r["tool_calls"] <= 50 else "51-200" if r["tool_calls"] <= 200 else ">200" for r in rs) | |
| print("tool call buckets:", dict(tcb)) | |
Xet Storage Details
- Size:
- 4.71 kB
- Xet hash:
- eafacacfdddab082e51d10a6530016b28c84d3c68ccd6537bdf3428fbdd541c9
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.