Buckets:
| # Pull human prompts out of agent-trace datasets on the Hub (Claude Code, Codex, Hermes). | |
| # Copies of the same dataset (same file count + byte size) are collapsed to one. | |
| # Per dataset: up to FILES files (smallest-first random mix, <20MB), all user prompts per file. | |
| # python -I extract_prompts.py probe.tsv sniff_other.tsv out.jsonl cache_dir | |
| import collections, json, random, re, sys, time, concurrent.futures as cf | |
| from huggingface_hub import HfApi, hf_hub_download | |
| probe, sniff, out, cache = sys.argv[1:5] | |
| ONLY = set(open(sys.argv[5]).read().split()) if len(sys.argv) > 5 else None # retry these datasets, append | |
| FILES, MAXB = 40, 20_000_000 | |
| api = HfApi() | |
| kinds = {} | |
| for l in open(probe): | |
| rid, kind, n, size = l.rstrip("\n").split("\t")[:4] | |
| if kind == "claude-code": | |
| kinds[rid] = ("claude-code", int(n), int(size)) | |
| for l in open(sniff): | |
| rid, _, s = (l.rstrip("\n").split("\t") + ["", ""])[:3] | |
| if "session_meta" in s or "'payload'" in s: | |
| kinds[rid] = ("codex", 0, 0) | |
| elif '"source": "cli"' in s or '"source": "hermes-agent"' in s: | |
| kinds[rid] = ("hermes", 0, 0) | |
| # collapse copies (same n files + bytes) | |
| seen, todo = {}, [] | |
| for rid, (k, n, size) in sorted(kinds.items()): | |
| sig = (k, n, size) if n else (k, rid) | |
| if sig in seen: | |
| continue | |
| seen[sig] = rid | |
| if ONLY is None or rid in ONLY: | |
| todo.append((rid, k)) | |
| NOISE = re.compile(r"^\s*(<(command-|local-command|system-reminder|environment_context|user_instructions|user_shell_command|turn_aborted)|" | |
| r"Caveat:|\[Request interrupted|# AGENTS\.md|This session is being continued)") | |
| def text_of(c): | |
| if isinstance(c, str): | |
| return c | |
| if isinstance(c, list): | |
| if any(isinstance(x, dict) and x.get("type") == "tool_result" for x in c): | |
| return "" | |
| return "\n".join(x.get("text", "") for x in c if isinstance(x, dict) and x.get("type") in ("text", "input_text")) | |
| return "" | |
| def prompts(path, kind): | |
| out = [] | |
| with open(path, errors="replace") as fh: | |
| for line in fh: | |
| try: | |
| d = json.loads(line) | |
| except ValueError: | |
| continue | |
| if not isinstance(d, dict): | |
| continue | |
| t = "" | |
| if kind == "claude-code": | |
| if d.get("type") == "user" and not d.get("isMeta") and not d.get("isSidechain"): | |
| m = d.get("message") or {} | |
| if m.get("role") == "user": | |
| t = text_of(m.get("content")) | |
| elif kind == "codex": | |
| p = d.get("payload") or {} | |
| if d.get("type") == "event_msg" and p.get("type") == "user_message": | |
| t = p.get("message", "") | |
| elif kind == "hermes": | |
| for m in d.get("messages") or d.get("conversation") or []: | |
| if isinstance(m, dict) and m.get("role") == "user": | |
| tt = text_of(m.get("content")) | |
| if tt.strip() and not NOISE.match(tt): | |
| out.append(tt.strip()) | |
| continue | |
| if t.strip() and not NOISE.match(t): | |
| out.append(t.strip()) | |
| return out | |
| def one(item): | |
| rid, kind = item | |
| try: | |
| files = [f for f in api.list_repo_tree(rid, repo_type="dataset", recursive=True) | |
| if getattr(f, "size", None) and f.path.endswith(".jsonl") and 200 < f.size < MAXB] | |
| random.Random(rid).shuffle(files) | |
| rows = [] | |
| for f in files[:FILES]: | |
| for attempt in range(4): | |
| try: | |
| p = hf_hub_download(rid, f.path, repo_type="dataset", cache_dir=cache) | |
| break | |
| except Exception: # noqa: BLE001 | |
| if attempt == 3: | |
| raise | |
| time.sleep(5 * (attempt + 1)) | |
| for i, t in enumerate(prompts(p, kind)): | |
| rows.append({"dataset": rid, "format": kind, "file": f.path, "turn": i, "prompt": t[:3000]}) | |
| return rid, kind, len(files), rows, None | |
| except Exception as e: # noqa: BLE001 | |
| return rid, kind, 0, [], type(e).__name__ | |
| n = 0 | |
| with cf.ThreadPoolExecutor(2 if ONLY else 12) as ex, open(out, "a" if ONLY else "w") as fo: | |
| for rid, kind, nf, rows, err in ex.map(one, todo): | |
| for r in rows: | |
| fo.write(json.dumps(r, ensure_ascii=False) + "\n") | |
| n += len(rows) | |
| print(f"{kind:12s} {rid:60s} files={nf:5d} prompts={len(rows):5d} {err or ''}", flush=True) | |
| print("datasets", len(todo), "prompts", n) | |
Xet Storage Details
- Size:
- 4.61 kB
- Xet hash:
- 0dbbca6f5e6550f0299764f5147b053834415e0388876721c093dd162997da0e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.