Hugging Face's logo

Spaces:
FineEnvs
/
RL-Explorer
Running on CPU Upgrade

App Files Community
1
RL-Explorer / app /catalog.py
AdithyaSK's picture
AdithyaSK HF Staff
Deploy HF RL Explorer
6abb7be verified
Raw History Blame Contribute Delete
102 kB
"""Harbor datasets on the Hub: which exist, where their tasks are, what each task asks and how it is graded.
A Harbor task is a folder holding `task.toml`. Datasets put those folders in different places (`tasks/<task>/`,
`tasks/<group>/<task>/`, `<task>/` at the root, `<benchmark>/tasks/<task>/`), so a task is found by where its
`task.toml` is, never by a fixed path. A dataset is indexed on first open, in the background:
listing one recursive listing of the repository: every task folder, its files and their sizes
reading one filtered download of the files an index needs (task.toml, instruction.md, tests/test.sh)
parsing one row per task: title, facets, environment, how it is graded
and the index is kept (STORAGE_DIR/indexes) until the dataset's revision changes.
Public datasets are read anonymously, with `token=False`: the server's own token can see its owner's private and
gated datasets, and a visitor naming one must not get it shown. A private or gated dataset is read with the
signed-in visitor's own token, and every request for it checks that token's access again before anything cached
is served. Nothing that holds an answer is shown: metadata keys like `gold_answer` are withheld by name, and
`solution/` files and data files beside the grader (rubrics, expected outputs) are listed but never sent.
Each indexed dataset also gets a pack: every task's instruction, task.toml, tests and Dockerfile as text, and
its file list. Task pages read the pack, so they never wait on the Hub; packs and indexes can live in a bucket.
"""
from __future__ import annotations
import gzip
import hashlib
import json
import os
import posixpath
import re
import threading
import time
import tomllib
from collections import Counter, OrderedDict
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Any
from . import config
# ── collections: featured environments, grouped ──────────────────────────────
# Admins edit these (app/settings.py); these are the defaults. An entry is a dataset id, or `space:<id>` for a Space.
# `color` names one of the page's validated identity colours (--c-<color> in the stylesheets).
DEFAULT_COLLECTIONS: list[dict[str, Any]] = [
{"id": "fineenvs", "group": "FineEnvs on OpenEnv", "icon": "globe", "color": "amber", "about": "FineEnvs' live environment servers: tasks, tools, rewards",
"ids": [f"space:FineEnvs/{n}" for n in ("geoguesser-env", "latex-ocr-env", "watercolour-env", "nayana-ocr-env", "fleurs-asr-env",
"smoldataenv-multi-harness-harbor", "smoldataenv-multi-harness-whitebox", "wordle-openenv")]},
{"id": "mimo", "group": "MiMo-V2.6 RL", "icon": "grid", "color": "webdev", "about": "Xiaomi's RL release, raw and in Harbor",
"ids": ["XiaomiMiMo/MiMo-V2.6-RL-oss"] + [f"FineEnvs/MiMo-V2.6-RL-harbor-{d}" for d in ("code", "cyber", "general", "terminal", "webdev", "music")]},
{"id": "terminal", "group": "Terminal-Bench", "icon": "terminal", "color": "cyber", "about": "terminal tasks, hand-written tests",
"ids": ["harborframework/terminal-bench-2.1", "harborframework/terminal-bench-3.0", "harborframework/terminal-bench-2.0",
"harborframework/terminal-bench-science", "IntelligenceLab/Long-Horizon-Terminal-Bench"]},
{"id": "data", "group": "Data analysis", "icon": "table", "color": "general", "about": "questions over real data, checked answers",
"ids": ["FineEnvs/data-agent-harbor-eval", "FineEnvs/data-agent-harbor-test", "FineEnvs/SmolDataEnvs-harbor-train",
"AdithyaSK/dabstep-harbor", "FineEnvs/SmolDataEnvs"]},
{"id": "swe", "group": "Software engineering", "icon": "code", "color": "code", "about": "repository tasks, graded by tests",
"ids": ["FineEnvs/repo2rlenv-swe-flow", "FineEnvs/repo2rlenv-swe-smith", "FineEnvs/repo2rlenv-r2e", "FineEnvs/repo2rlenv-cli-gym",
"FineEnvs/HF_ML_Tasksmith", "Lego-X/LegoFlow-SWE"]},
{"id": "mixed", "group": "Mixed and other", "icon": "flask", "color": "music", "about": "agentic benchmarks across domains",
"ids": ["internlm/WildClawBench-Harbor", "armin-aptura/skilltrainbench-public", "space:FineEnvs/wordle-openenv", "space:openenv/coding_env"]},
]
COLORS = ("code", "webdev", "cyber", "music", "general", "amber")
ICONS = ("grid", "terminal", "table", "code", "image", "flask", "globe", "brain", "music", "briefcase", "database", "layout", "target", "box")
def collections() -> list[dict[str, Any]]:
"""The collections, as admins last saved them, else the defaults."""
from . import settings
return settings.get("collections") or DEFAULT_COLLECTIONS
def collection_of(key: str) -> str | None:
"""The collection an environment is in: `key` is a dataset id or `space:<id>`."""
return next((g["id"] for g in collections() if key in g["ids"]), None)
def featured_datasets() -> list[str]:
return [k for g in collections() for k in g["ids"] if not k.startswith("space:")]
def collection(spec: str) -> dict[str, Any] | None:
"""The collection an environment is featured in (its id, name, icon, colour), or None."""
cid = collection_of(spec)
return next(({k: v for k, v in g.items() if k != "ids"} for g in collections() if g["id"] == cid), None)
def hidden() -> set[str]:
"""Environments admins took off the listings (spam, broken), by key."""
from . import settings
return set(settings.get("hidden", []))
# Editorial picks, shown separately from the metric-based trending order.
DEFAULT_PINNED = ["space:FineEnvs/geoguesser-env", "space:FineEnvs/latex-ocr-env"]
def pinned() -> list[str]:
from . import settings
return list(settings.get("pinned", DEFAULT_PINNED))
HUB_ID = re.compile(r"^[A-Za-z0-9][\w.-]*/[\w.-]+$")
# Metadata that is an answer. Harbor puts no rules on `[metadata]`, and datasets keep gold answers there.
ANSWER_KEY = re.compile(r"(gold|answer|solution|expected|reference_output|ground_truth|oracle)", re.I)
CONTACT_KEY = re.compile(r"(email|e_mail|phone)", re.I) # an author's contact details: not metadata to browse by
PACKED = re.compile(r"\.(parquet|tar|tar\.gz|tgz|zip|jsonl)$", re.I)
MAX_DEPTH = 8
# Bumped when what an index holds or how it is read changes, so older indexes are rebuilt rather than misread.
INDEX_VERSION = 13
# Benchmarks open their instructions with a canary string, there to catch training sets that leak them.
CANARY = re.compile(r"BENCHMARK DATA SHOULD NEVER APPEAR|canary GUID|canary string", re.I)
TEXT_LIMIT = 512 * 1024
# Metadata keys that mean the same thing across datasets, read in this order.
DIFFICULTY_KEYS = ("difficulty", "difficulty_tier", "difficulty_level", "task_complexity", "complexity", "level")
CATEGORY_KEYS = ("category", "domain", "task_type", "skill_type", "type", "subcategory")
TAG_KEYS = ("tags", "keywords")
def _api(token: str | None = None):
from huggingface_hub import HfApi
return HfApi(token=token or False)
def _who(token: str | None) -> str:
"""A token's stand-in in cache keys: never the token itself."""
return hashlib.sha256(token.encode()).hexdigest()[:16] if token else ""
def check_spec(spec: str) -> str:
spec = (spec or "").strip()
if not HUB_ID.match(spec) or ".." in spec:
raise ValueError("a Hub dataset id, like org/name")
return spec
def _slug(spec: str) -> str:
return spec.replace("/", "__")
# ── Hub search and dataset info ──────────────────────────────────────────────
_memo: dict[tuple, tuple[float, Any]] = {}
_big: "OrderedDict[tuple, tuple[float, Any]]" = OrderedDict() # whole files, every row of a split: a few at a time
_memo_lock = threading.Lock()
_inflight: dict[tuple, threading.Lock] = {}
STALE_FOR = 24 * 3600 # a value whose refresh fails is served this much longer (the Hub down shouldn't take pages down)
BIG_ENTRIES = 3
def _cached(key: tuple, ttl: float, fn, big: bool = False):
"""A value kept `ttl` seconds. Concurrent misses fetch it once; a failed refresh serves the last good value (up to a
day old) instead of failing; `big` values (whole files) are kept a few at a time, not 2,000."""
store_ = _big if big else _memo
now = time.time()
with _memo_lock:
hit = store_.get(key)
if hit and now - hit[0] < ttl:
return hit[1]
lock = _inflight.setdefault(key, threading.Lock())
with lock:
with _memo_lock: # someone else may have fetched it while we waited
hit = store_.get(key)
if hit and time.time() - hit[0] < ttl:
return hit[1]
try:
value = fn()
except Exception:
if hit and time.time() - hit[0] < STALE_FOR:
return hit[1]
raise
finally:
with _memo_lock:
_inflight.pop(key, None)
with _memo_lock:
store_[key] = (time.time(), value)
if big:
_big.move_to_end(key)
while len(_big) > BIG_ENTRIES:
_big.popitem(last=False)
elif len(_memo) > 2000:
for k in sorted(_memo, key=lambda k: _memo[k][0])[:500]:
_memo.pop(k, None)
return value
# What a dataset is, as the Hub's own badges say it.
BADGES = (("rl-environment", "RL Environment"), ("benchmark", "Benchmark"))
SORTS = {"trending": "trending_score", "downloads": "downloads", "updated": "last_modified", "likes": "likes", "new": "created_at"}
_EXPAND = ["downloads", "likes", "lastModified", "tags", "trendingScore", "gated", "private", "createdAt", "description"]
# card paragraphs that say where a dataset comes from rather than what it is
_NOT_A_BRIEF = re.compile(r"^(\(|warning|note\b|disclaimer|⚠|important)|mirror|primary source|open issues|pull requests|leaderboard above|"
r"how to run|https?://", re.I)
# a card's first line that names a section, not the dataset ("Dataset Description:", "Overview")
_GENERIC_HEADING = re.compile(r"^(dataset\s+(card|description|summary|overview|details|information)|description|overview|summary|"
r"introduction|about(\s+this\s+dataset)?|abstract|readme|contents?|table\s+of\s+contents)\s*:?$", re.I)
def _card_text(text: str | None) -> tuple[str | None, str]:
"""The dataset card's heading and its first paragraph that describes the dataset, from the Hub's plain-text
`description` (the card's text, cut short by the Hub)."""
text = re.sub(r"…\s*See the full description on the dataset page:.*$", "…", text or "", flags=re.S)
paras = []
for block in re.split(r"\n[ \t]*\n", text):
lines = [ln.strip() for ln in block.splitlines() if ln.strip()]
while lines and len(lines[0]) < 40 and ("·" in lines[0] or "|" in lines[0]): # "Paper · Blog · Leaderboard"
lines.pop(0)
if lines:
paras.append(re.sub(r"\s+", " ", " ".join(lines)))
heading = paras[0] if paras and len(paras[0]) <= 80 else None
named = re.sub(r"^dataset card for\s+", "", heading or "", flags=re.I).strip()
if heading and (not named or _GENERIC_HEADING.match(named) or named.endswith(":")):
named = ""
brief = next((p for p in paras[1 if heading else 0:] if len(p) >= 60 and not _NOT_A_BRIEF.search(p) and not CANARY.search(p)), "")
return named or None, (brief if len(brief) <= 280 else brief[:279].rsplit(" ", 1)[0] + "…")
def framework_of(tags: list[str]) -> str:
"""Which kind of environment a dataset is, from its Hub tags (the Hub's own convention: `rl-environment` plus a
framework tag): Harbor task folders, NeMo Gym or Verifiers data, OpenEnv data, verl rows, or other RL rows. A
dataset whose index finds Harbor task folders is Harbor whatever its tags say (environments() corrects it)."""
t = set(tags)
if t & {"harbor", "library:harbor"}:
return "harbor"
if t & {"nemo-gym", "library:nemo-gym"}:
return "nemo-gym"
if t & {"verifiers", "library:verifiers"}:
return "verifiers"
if t & {"openenv", "library:openenv"}:
return "openenv-data"
if t & {"verl", "skyrl"}:
return "verl"
return "rows"
def _summary(d: Any) -> dict[str, Any]:
raw = d.tags or []
tags = [t for t in raw if ":" not in t]
heading, brief = _card_text(getattr(d, "description", None))
return {"id": d.id, "key": d.id, "kind": "dataset", "collection": collection_of(d.id), "heading": heading, "brief": brief,
"downloads": d.downloads or 0, "likes": d.likes or 0,
"updated": d.last_modified.isoformat() if getattr(d, "last_modified", None) else None,
"created": d.created_at.isoformat() if getattr(d, "created_at", None) else None,
"trending": getattr(d, "trending_score", None) or 0,
"badges": [label for tag, label in BADGES if tag in raw],
"tags": [t for t in tags if t not in ("harbor",) + tuple(tag for tag, _ in BADGES)][:12],
"gated": bool(d.gated), "private": bool(d.private), "framework": framework_of(raw)}
def _listing_fetch(full: bool) -> list[dict[str, Any]]:
found = _api().list_datasets(filter="harbor", sort="trending_score", limit=None if full else 2000, expand=_EXPAND)
rows = {d.id: _summary(d) for d in found if not d.private and not d.gated}
# other RL environment formats: rows a processor reads (app/envs)
for tag in ("rl-environment", "library:verifiers", "library:nemo-gym", "verifiers", "nemo-gym", "library:openenv", "verl", "skyrl"):
try:
for d in _api().list_datasets(filter=tag, sort="trending_score", limit=None if full else 1000, expand=_EXPAND):
if d.id not in rows and not d.private and not d.gated:
rows[d.id] = _summary(d)
except Exception: # noqa: BLE001 - only the quick bootstrap tolerates incomplete discovery
if full:
raise
continue
missing = [s for s in featured_datasets() if s not in rows]
with ThreadPoolExecutor(max_workers=12) as pool: # one Hub call each: in parallel
for spec, row in zip(missing, pool.map(_info_or_none, missing)):
if row:
rows[spec] = row
for sp in spaces(full=full):
rows[sp["key"]] = sp
return list(rows.values())
LISTING_TTL = 900 # seconds a listing is served before it's refreshed (in the background)
# On the Spaces the indexer Job (app/indexer.py) writes listing.json.gz every hour: re-read it when it changes and
# don't fetch the Hub's listing in every process, unless it's older than LISTING_STALE (the Job stopped).
LISTING_FROM_STORE = os.environ.get("RLX_LISTING_FROM_STORE", "0") == "1"
LISTING_STALE = 3 * 3600
_env_list: dict[str, Any] = {"rows": None, "at": 0.0, "full": False, "refreshing": False, "mtime": None, "looked": 0.0}
_listing_lock = threading.Lock()
def _listing_path() -> Path:
return config.STORAGE_DIR / "listing.json.gz"
def _listing_rows() -> list[dict[str, Any]]:
"""Every environment on the Hub, served at once: from memory, else from the last listing written to the store
(so a restart doesn't wait on the Hub), else a quick listing (the 1,000 most trending per tag); a full one (every
Space with its files, minutes) refreshes it in the background whenever it's older than LISTING_TTL."""
with _listing_lock:
rows, at = _env_list["rows"], _env_list["at"]
if LISTING_FROM_STORE:
if time.time() - _env_list["looked"] > 30: # a stat every 30 s, a read when the file changed
_env_list["looked"] = time.time()
try:
mtime = _listing_path().stat().st_mtime
if mtime != _env_list["mtime"]:
doc = json.loads(gzip.decompress(_listing_path().read_bytes()))
rows, at = doc["rows"], doc["at"]
with _listing_lock:
_env_list.update(rows=rows, at=at, full=doc.get("full", False), mtime=mtime)
except (OSError, ValueError, KeyError):
pass
if rows is not None and time.time() - at < LISTING_STALE:
return rows # the Job's listing: no fetch here
if rows is None:
try:
doc = json.loads(gzip.decompress(_listing_path().read_bytes()))
rows, at = doc["rows"], doc["at"]
with _listing_lock:
_env_list.update(rows=rows, at=at, full=doc.get("full", False))
except (OSError, ValueError, KeyError):
rows = None
if rows is None:
rows = _listing_fetch(full=False)
at = time.time()
with _listing_lock:
_env_list.update(rows=rows, at=at, full=False)
_refresh_listing(force=True)
elif time.time() - at > LISTING_TTL or not _env_list.get("full"):
_refresh_listing()
return rows
def _refresh_listing(force: bool = False) -> None:
with _listing_lock:
if _env_list["refreshing"] or (not force and time.time() - _env_list["at"] < LISTING_TTL and _env_list.get("full")):
return
_env_list["refreshing"] = True
def run() -> None:
try:
rows = _listing_fetch(full=True)
if rows:
with _listing_lock:
_env_list.update(rows=rows, at=time.time(), full=True)
atomic_write(_listing_path(), gzip.compress(json.dumps({"at": time.time(), "full": True, "rows": rows}, default=str).encode()))
except Exception: # noqa: BLE001 - the last listing stays; the next request tries again
pass
finally:
with _listing_lock:
_env_list["refreshing"] = False
threading.Thread(target=run, daemon=True, name="listing").start()
def environments(include_hidden: bool = False) -> list[dict[str, Any]]:
"""Every public RL environment on the Hub: datasets tagged `harbor` or as RL environments of other formats,
environment Spaces (OpenEnv, ORS, others), and every featured one, with its counters and, for an indexed dataset,
its task count. The page sorts and filters these itself."""
gone, pins = (set() if include_hidden else hidden()), pinned()
out = []
for r in _listing_rows():
if r["key"] in gone:
continue
indexed = headline(r["id"]) if r["kind"] == "dataset" else None
fw = "harbor" if indexed and indexed.get("tasks") else r.get("framework") # its index found task folders
coll = collection_of(r["key"])
out.append({**r, "framework": fw, "collection": coll, "pinned": r["key"] in pins, "indexed": indexed})
return out
# ── OpenEnv and other environment Spaces ─────────────────────────────────────
_SPACE_EXPAND = ["likes", "trendingScore", "sdk", "runtime", "tags", "lastModified", "createdAt", "cardData", "private"]
def _space_summary(s: Any) -> dict[str, Any]:
card = (s.card_data.to_dict() if s.card_data else {}) or {}
raw = [str(t).lower() for t in (s.tags or [])]
stage = getattr(s.runtime, "stage", None) if s.runtime is not None else None
version = next((t[8:] for t in raw if re.match(r"^openenv-\d", t)), None)
files = [f.rfilename for f in (getattr(s, "siblings", None) or [])]
manifest = min((f for f in files if _MANIFEST.match(f) and not _VENDORED.match(f)), key=lambda f: f.count("/"), default=None)
openenv = bool({"openenv", "library:openenv"} & set(raw)) or bool(version) or manifest is not None
ors = bool({"ors", "openreward"} & set(raw))
return {"id": s.id, "key": f"space:{s.id}", "kind": "space", "openenv": openenv, "manifest": manifest,
"framework": "openenv" if openenv else "ors" if ors else "space", "files": len(files) or None,
"openenv_version": version, "mcp": any(t in ("mcp-server", "mcp") for t in raw),
"hardware": getattr(s.runtime, "hardware", None) if s.runtime is not None else None,
"heading": card.get("title"), "brief": (card.get("short_description") or "")[:280],
"downloads": 0, "likes": s.likes or 0, "trending": getattr(s, "trending_score", None) or 0,
"updated": s.last_modified.isoformat() if getattr(s, "last_modified", None) else None,
"created": s.created_at.isoformat() if getattr(s, "created_at", None) else None,
"sdk": s.sdk, "stage": str(stage) if stage else None,
"badges": ["OpenEnv"] if openenv else ["ORS"] if ors else [],
"tags": [t for t in raw if ":" not in t and t not in ("docker", "openenv", "rl-environment", "mcp-server", "ors", "openreward") and not t.startswith("openenv-")][:12],
"private": bool(s.private), "gated": False}
SPACE_TAGS = ("openenv", "library:openenv", "rl-environment", "ors", "openreward")
# a manifest a Space's own environment declares (not one vendored from OpenEnv's source, or its templates)
_MANIFEST = re.compile(r"^(?:[^/]+/)?openenv\.ya?ml$")
_VENDORED = re.compile(r"^(src/|build/|dist/|templates?/|examples?/|tests?/|vendor/|\.venv/|.*\.egg-info/|envs/)")
def spaces(full: bool = True) -> list[dict[str, Any]]:
"""Environment Spaces: OpenEnv servers, ORS servers and others tagged as RL environments (Docker Spaces, not
articles or demos). `full` lists every one with its files (thousands; a minute or two, so done in the background)
and tells OpenEnv apart by its manifest; otherwise the most trending 1,000 per tag, by tags alone."""
featured = [k[6:] for g in collections() for k in g["ids"] if k.startswith("space:")]
rows: dict[str, dict[str, Any]] = {}
expand = _SPACE_EXPAND + (["siblings"] if full else [])
for tag in SPACE_TAGS:
try:
for sp in _api().list_spaces(filter=tag, sort="trending_score", limit=None if full else 1000, expand=expand):
tags = sp.tags or []
if sp.private or sp.sdk != "docker" or "research-article-template" in tags:
continue
rows.setdefault(sp.id, _space_summary(sp))
except Exception: # noqa: BLE001 - a full census must not silently publish partial counts
if full:
raise
continue
for sid in featured:
if sid not in rows:
try:
sp = _api().space_info(sid, expand=expand)
if not sp.private and sp.sdk == "docker" and "research-article-template" not in (sp.tags or []):
rows[sid] = _space_summary(sp)
except Exception: # noqa: BLE001
continue
return list(rows.values())
def space(spec: str) -> dict[str, Any]:
"""A Space's page: its card and its README. What its server offers (and its stage now) is spaces_live.probe."""
spec = check_spec(spec)
def fetch():
sp = _api().space_info(spec, expand=_SPACE_EXPAND + ["subdomain", "siblings"]) # its files: its openenv.yaml
if sp.private:
raise PermissionError("this Space is private")
row = _space_summary(sp)
readme = ""
try:
from huggingface_hub import hf_hub_download
readme = _read(Path(hf_hub_download(spec, "README.md", repo_type="space", token=False,
local_dir=str(config.CACHE_DIR / "spaces" / _slug(spec)))), 256 * 1024)
readme = re.sub(r"\A---\n.*?\n---\n", "", readme, flags=re.S) # the YAML header
except Exception: # noqa: BLE001
pass
sub = getattr(sp, "subdomain", None) or ""
host = f"https://{sub}.hf.space" if sub else ""
if not re.match(r"^https://[a-z0-9-]+\.hf\.space$", host or ""): # only ever call the Space's own hf.space host
host = ""
server: dict[str, Any] = {} # what the server offers comes live from /api/spaces/<id>/live (spaces_live.probe)
return {**row, "readme": readme[:200_000], "host": host, "server": server, "collection": collection_of(row["key"]),
"license": (sp.card_data.to_dict() if sp.card_data else {}).get("license")}
return _cached(("space", spec), 120, fetch)
def _info_or_none(spec: str) -> dict[str, Any] | None:
try:
return info(spec)
except Exception: # noqa: BLE001 - a deleted or gated dataset drops out, it doesn't break the page
return None
_INFO_EXPAND = ["sha", "downloads", "likes", "lastModified", "createdAt", "tags", "cardData", "gated", "private", "trendingScore",
"description"]
def _fetch_info(spec: str, token: str | None) -> dict[str, Any]:
# only these fields: by default the Hub also returns every file, seconds for a big dataset
d = _api(token).dataset_info(spec, expand=_INFO_EXPAND)
restricted = bool(d.private or d.gated)
if restricted and not token:
raise PermissionError("this dataset is private or gated: sign in with an account that can read it")
card = (d.card_data.to_dict() if d.card_data else {}) or {}
return {**_summary(d), "sha": d.sha, "pretty_name": card.get("pretty_name"), "license": card.get("license"),
"harbor": "harbor" in (d.tags or []), "restricted": restricted, "all_tags": list(d.tags or [])}
def info(spec: str, token: str | None = None) -> dict[str, Any]:
"""A dataset's revision, card fields and counters. A public one is read anonymously (and cached for everyone);
a private or gated one only with a token that can read it, and cached per token. Raises otherwise."""
from huggingface_hub.errors import RepositoryNotFoundError
spec = check_spec(spec)
try:
return _cached(("info", spec), 300, lambda: _fetch_info(spec, None))
except (RepositoryNotFoundError, PermissionError):
if not token:
raise
return _cached(("info", spec, _who(token)), 120, lambda: _fetch_info(spec, token))
def mine(token: str) -> list[dict[str, Any]]:
"""The visitor's own Harbor datasets and their organizations', private ones included, read with their token."""
from huggingface_hub import whoami
def fetch():
me = whoami(token=token)
owners = [me["name"]] + [o["name"] for o in (me.get("orgs") or [])][:10]
def one(owner: str) -> list[dict[str, Any]]:
try:
return [_summary(d) for d in _api(token).list_datasets(author=owner, filter="harbor", limit=300, expand=_EXPAND)]
except Exception: # noqa: BLE001 - one organization failing leaves the rest
return []
with ThreadPoolExecutor(max_workers=6) as pool:
rows = {r["id"]: r for batch in pool.map(one, owners) for r in batch}
return list(rows.values())
return [{**r, "mine": True, "indexed": headline(r["id"])} for r in _cached(("mine", _who(token)), 120, fetch)]
def random_task(spec: str | None = None, token: str | None = None) -> dict[str, str]:
"""A task to open: from `spec`, or from a featured dataset already indexed (each dataset equally likely, so the
biggest doesn't crowd out the rest)."""
import random
if spec:
info(spec, token) # may this visitor read it?
pool = [check_spec(spec)] if spec else featured_datasets()
random.shuffle(pool)
for s in pool:
idx = _read_index(s)
if idx and idx.get("tasks"):
return {"dataset": s, "path": random.choice(idx["tasks"])["path"]}
raise LookupError("no indexed tasks yet" if not spec else "this dataset isn't indexed yet")
def warm() -> None:
"""Index the featured datasets in the background, so their task counts are ready. One from each collection in
turn, so every collection has some early rather than the first all done before the second."""
from itertools import chain, zip_longest
try:
environments() # the home page's one fetch, ready before the first visitor asks
except Exception: # noqa: BLE001 - the page fetches it again
pass
order = [s for s in chain.from_iterable(zip_longest(*([k for k in g["ids"] if not k.startswith("space:")] for g in collections()))) if s]
for spec in order:
try:
meta = info(spec)
if _read_index(spec, meta["sha"]):
continue
job = {"state": "listing", "done": 0, "total": 0, "sha": meta["sha"], "at": time.time()}
with _jobs_lock:
if spec in _jobs and _jobs[spec].get("state") not in ("done", "error"):
continue
_jobs[spec] = job
_build(spec, meta, job)
except Exception: # noqa: BLE001 - warming is best effort; opening the dataset retries
continue
# ── indexing ─────────────────────────────────────────────────────────────────
_jobs: dict[str, dict[str, Any]] = {}
_jobs_lock = threading.Lock()
def atomic_write(p: Path, data: bytes) -> None:
"""Write a file so a reader sees the old one or the new one, never half of either, even with two processes (the
explorer, its admin, an indexer job) writing it: a temp name of this process's own, then a rename."""
p.parent.mkdir(parents=True, exist_ok=True)
tmp = p.with_name(f".{p.name}.{os.getpid()}.{threading.get_ident()}.tmp")
tmp.write_bytes(data)
tmp.replace(p)
def _index_path(spec: str) -> Path:
return config.INDEX_DIR / f"{_slug(spec)}.json.gz"
_indexes: "OrderedDict[tuple, tuple[int, dict]]" = OrderedDict() # (path, mtime) -> (bytes, parsed index)
_indexes_lock = threading.Lock()
INDEX_CACHE_BYTES = int(os.environ.get("RLX_INDEX_CACHE_MB", 400)) * 1024 * 1024
def _read_index(spec: str, sha: str | None = None) -> dict[str, Any] | None:
"""A dataset's index, parsed once per version of its file (a big one takes a quarter second to parse), within a
memory budget. Callers must not change what it returns."""
p = _index_path(spec)
try:
key = (str(p), p.stat().st_mtime)
except OSError:
return None
with _indexes_lock:
hit = _indexes.get(key)
if hit:
_indexes.move_to_end(key)
if hit:
idx = hit[1]
else:
try:
raw = gzip.decompress(p.read_bytes())
idx = json.loads(raw)
except (OSError, ValueError):
return None
with _indexes_lock:
for k in [k for k in _indexes if k[0] == key[0]]: # an older version of the same file
_indexes.pop(k, None)
_indexes[key] = (len(raw), idx)
while len(_indexes) > 1 and sum(n for n, _ in _indexes.values()) > INDEX_CACHE_BYTES:
_indexes.popitem(last=False)
if (sha and idx.get("sha") != sha) or idx.get("version") != INDEX_VERSION:
return None
return idx
def _write_index(idx: dict[str, Any]) -> None:
atomic_write(_index_path(idx["spec"]), gzip.compress(json.dumps(idx, separators=(",", ":")).encode()))
# a few numbers beside it, cheap to read for every card in a listing
atomic_write(_head_path(idx["spec"]), json.dumps({**_headline(idx), "sha": idx["sha"], "version": idx.get("version")}).encode())
_heads.pop(idx["spec"], None)
def _head_path(spec: str) -> Path:
return config.INDEX_DIR / f"{_slug(spec)}.head.json"
_heads: dict[str, tuple[float, dict[str, Any] | None]] = {}
def headline(spec: str) -> dict[str, Any] | None:
"""What a listing shows of an indexed dataset (its task count and how it is graded), or None. Re-read from the
store at most once a minute per dataset: a listing asks for hundreds."""
hit = _heads.get(spec)
if hit and time.time() - hit[0] < 60:
return hit[1]
try:
h = json.loads(_head_path(spec).read_text())
h = h if h.get("version") == INDEX_VERSION else None
except (OSError, ValueError):
h = None
_heads[spec] = (time.time(), h)
return h
def _headline(idx: dict[str, Any]) -> dict[str, Any]:
s = idx.get("summary") or {}
return {"tasks": len(idx.get("tasks") or []), "packed": idx.get("packed"), "graded": s.get("verifier"),
"image": s.get("image"), "built": idx.get("built")}
def index_status(spec: str, token: str | None = None) -> dict[str, Any]:
"""The index if it is current, else the indexing job's progress (starting one if none runs)."""
spec = check_spec(spec)
meta = info(spec, token)
idx = _read_index(spec, meta["sha"])
if idx and not idx.get("partial"):
return {"state": "done", "index": idx}
if idx: # the tasks are known, their files are being read: the page opens now and refreshes when it's whole
with _jobs_lock:
job = _jobs.get(spec)
alive = job and job.get("sha") == meta["sha"] and job["state"] not in ("done", "error")
running = sum(1 for j in _jobs.values() if j.get("state") not in ("done", "error"))
if not alive and running < config.MAX_INDEX_JOBS and not (job and job["state"] == "error" and time.time() - job.get("at", 0) < 300):
job = {"state": "listing", "done": 0, "total": 0, "sha": meta["sha"], "at": time.time(), "t": {"listing": time.time()}}
_jobs[spec] = job
threading.Thread(target=_build, args=(spec, meta, job, token if meta.get("restricted") else None), daemon=True,
name=f"index-{spec}").start()
prog = {k: job.get(k) for k in ("state", "done", "total", "error")} if job else None
return {"state": "done", "index": idx, "progress": prog}
with _jobs_lock:
job = _jobs.get(spec)
if job and job.get("sha") == meta["sha"] and job["state"] not in ("error",):
return {**{k: v for k, v in job.items() if k != "sha"}, "now": time.time()}
if job and job["state"] == "error" and time.time() - job.get("at", 0) < 60:
return {k: v for k, v in job.items() if k != "sha"}
running = sum(1 for j in _jobs.values() if j.get("state") not in ("done", "error"))
if running >= config.MAX_INDEX_JOBS:
raise RuntimeError(f"{running} datasets are being indexed right now; try again in a minute")
job = {"state": "listing", "done": 0, "total": 0, "sha": meta["sha"], "at": time.time(), "t": {"listing": time.time()}}
_jobs[spec] = job
threading.Thread(target=_build, args=(spec, meta, job, token if meta.get("restricted") else None), daemon=True,
name=f"index-{spec}").start()
return {**{k: v for k, v in job.items() if k != "sha"}, "now": time.time()}
def _build(spec: str, meta: dict[str, Any], job: dict[str, Any], token: str | None = None) -> None:
try:
idx, pack = build_index(spec, meta, job, token, on_partial=lambda early: (_write_index(early), job.update(partial=True)))
job.setdefault("t", {})["saving"] = time.time()
job["state"] = "saving"
if pack is not None:
_write_pack(spec, meta["sha"], pack)
_write_index(idx)
job.update(state="done")
except Exception as exc: # noqa: BLE001 - reported to the page
job.update(state="error", error=f"{type(exc).__name__}: {str(exc)[:300]}", at=time.time())
def _task_dirs(paths: list[str]) -> list[str]:
"""Folders holding a `task.toml`, shallowest first; a task folder inside another task's folder is not a task."""
found = sorted({posixpath.dirname(p) for p in paths if posixpath.basename(p) == "task.toml"
and p.count("/") <= MAX_DEPTH and not any(seg.startswith(".") for seg in p.split("/"))},
key=lambda d: (d.count("/"), d))
kept: list[str] = []
for d in found:
if not any(d == k or d.startswith(k + "/") for k in kept if k):
kept.append(d)
return sorted(kept)
# what a task's page shows, fetched for every task when the dataset is indexed
CORE_FILES = ("task.toml", "instruction.md", "tests/test.sh", "environment/Dockerfile", "tests/Dockerfile")
PACK_BUDGET = 512 * 1024 * 1024 # bytes to fetch for a dataset's pack before falling back to CORE_FILES
SMALL = 64 * 1024
# a multi-step task's own files: steps/<name>/instruction.md, tests/, solution/, workdir/ (Harbor's TaskPaths)
STEP_FILE = re.compile(r"^steps/([^/]+)/(.+)$")
MAX_STEPS = 20
def _in_step(rel: str) -> str:
"""A step's file as the task-level file it stands for (`steps/s1/tests/x.py` is `tests/x.py`), else `rel`."""
m = STEP_FILE.match(rel)
return m.group(2) if m else rel
def _wanted(rel: str, size: int) -> bool:
if rel in ("task.toml", "instruction.md") or (STEP_FILE.match(rel) and _in_step(rel) == "instruction.md"):
return size <= TEXT_LIMIT
if size > SMALL or _binary_name(rel):
return False
return (rel.startswith("tests/") or (rel.startswith("steps/") and _in_step(rel).startswith("tests/"))
or bool(re.match(r"environment/(Dockerfile|(docker-)?compose\.ya?ml)$", rel)))
def _pick(files: dict[str, int], budget_ok: bool = True) -> list[str]:
"""A task's files worth keeping as text: its toml and instruction, the verifier's scripts, the environment, and
each step's instruction and tests (a few per step)."""
picked = [r for r, n in files.items() if _wanted(r, n) and (budget_ok or r in CORE_FILES or _in_step(r) in ("instruction.md", "tests/test.sh"))]
tests = sorted((r for r in picked if r.startswith("tests/")), key=lambda r: (r != "tests/test.sh", r.count("/"), r))[:24]
steps: dict[str, list[str]] = {}
for r in sorted((r for r in picked if r.startswith("steps/")), key=lambda r: (_in_step(r) not in ("instruction.md", "tests/test.sh"), r.count("/"), r)):
own = steps.setdefault(STEP_FILE.match(r).group(1), [])
if len(own) < 7 and (len(steps) <= MAX_STEPS or own):
own.append(r)
return [r for r in picked if not r.startswith(("tests/", "steps/"))] + tests + [r for own in steps.values() for r in own]
def build_index(spec: str, meta: dict[str, Any], job: dict[str, Any] | None = None,
token: str | None = None, on_partial=None) -> tuple[dict[str, Any], dict[str, Any] | None]:
"""The dataset's index (one row per task) and its pack (every task's text and file list). A dataset walked
folder by folder gets a first index as soon as its tasks are found (`on_partial`), with their names only, so
its page opens while the tasks' files are read."""
from huggingface_hub.hf_api import RepoFile
job = job if job is not None else {}
job.setdefault("t", {})["listing"] = time.time()
lister = token or (config.INDEX_TOKEN if not meta.get("restricted") else None)
files, walked = _listing(spec, meta["sha"], lister, job)
dirs = walked["tasks"] if walked else _task_dirs(list(files))
job["tasks"] = len(dirs)
base = {"spec": spec, "sha": meta["sha"], "built": time.time(), "info": meta, "version": INDEX_VERSION}
if not dirs:
packed = sorted(p for p in files if PACKED.search(p))[:20]
return {**base, "tasks": [], "packed": packed or None, "summary": {},
"note": "Tasks are packed into archives or tables, which this explorer can't open yet."
if packed else "No task.toml anywhere in this repository: it isn't in Harbor's format."}, None
if len(dirs) > config.MAX_INDEX_TASKS:
raise ValueError(f"{len(dirs):,} tasks: more than the {config.MAX_INDEX_TASKS:,} this explorer indexes")
# every file under each task folder, in one pass over the listing
if walked:
per_task = walked["files"]
else:
per_task = {d: {} for d in dirs}
for path, size in files.items():
parent = posixpath.dirname(path)
while True: # up the folders until one is a task's (a few steps, not a scan of every task)
if parent in per_task:
per_task[parent][path[len(parent) + 1:] if parent else path] = size
break
if not parent:
break
parent = posixpath.dirname(parent)
if walked and on_partial:
early = [_row(i, d, Path("/nonexistent"), per_task[d], [], {})[0] for i, d in enumerate(dirs)]
for r in early:
r.update(title=r["name"].replace("__", " · ").replace("_", " ").replace("-", " ").strip() or r["name"], pending=True, invalid=False)
_group(early)
on_partial({**base, "tasks": early, "summary": summarize(early), "packed": None, "walked": True, "partial": True})
picks = {d: _pick(per_task[d]) for d in dirs}
if sum(per_task[d][r] for d in dirs for r in picks[d]) > PACK_BUDGET:
picks = {d: _pick(per_task[d], budget_ok=False) for d in dirs}
paths = [f"{d}/{r}" if d else r for d in dirs for r in picks[d]]
job["t"]["reading"] = time.time()
job.update(state="reading", done=0, total=len(paths))
paths.sort(key=lambda p: (posixpath.basename(p) not in ("task.toml", "instruction.md"), p)) # what a row needs most, first
local = config.CACHE_DIR / "index-files" / f"{_slug(spec)}@{meta['sha'][:12]}"
# a public dataset may be fetched with the operator's own token (higher rate limits), never a private one
_download(spec, meta["sha"], token or (config.INDEX_TOKEN if not meta.get("restricted") else None), paths, local, job)
job["t"]["parsing"] = time.time()
job.update(state="parsing", done=0, total=len(dirs))
with ThreadPoolExecutor(max_workers=config.INDEX_WORKERS) as pool:
out = list(pool.map(lambda pair: _row(pair[0], pair[1], local, per_task[pair[1]], picks[pair[1]], job), enumerate(dirs)))
rows = [r for r, _ in out]
_group(rows)
_briefs(rows)
# texts by content: the grader's helpers are often the same file in every task, kept once
blobs: dict[str, str] = {}
tasks: dict[str, Any] = {}
for d, (_, texts) in zip(dirs, out):
refs = {}
for rel, text in texts.items():
h = hashlib.sha1(text.encode()).hexdigest()[:16]
blobs.setdefault(h, text)
refs[rel] = h
tasks[d] = {"files": dict(sorted(per_task[d].items())[:2000]), "more": max(0, len(per_task[d]) - 2000), "texts": refs,
**({"lazy": walked["lazy"].get(d, [])} if walked else {})}
return ({**base, "tasks": rows, "summary": summarize(rows), "packed": None, **({"walked": True} if walked else {})},
{"blobs": blobs, "tasks": tasks})
def looks_harbor(spec: str, sha: str, token: str | None = None) -> bool:
"""Whether a dataset not tagged `harbor` is laid out as Harbor tasks anyway (PrimeIntellect/Terminal-Lego-15k is
tagged `verifiers`): a task.toml one, two or three folders down, looking at the first few folders of each level.
At most eight Hub calls, cached."""
from huggingface_hub.hf_api import RepoFile, RepoFolder
def folders(path: str | None, n: int) -> list[str]:
out = []
for e in api.list_repo_tree(spec, repo_type="dataset", revision=sha, path_in_repo=path):
if isinstance(e, RepoFolder) and not e.path.rsplit("/", 1)[-1].startswith("."):
out.append(e.path)
if len(out) >= n:
break
return out
def found(cand: list[str]) -> bool:
return any(isinstance(e, RepoFile) for e in api.get_paths_info(spec, cand[:500], repo_type="dataset", revision=sha))
def check() -> bool:
try:
level = folders(None, 40)
if found(["task.toml"] + [f"{d}/task.toml" for d in level]):
return True
for depth in (2, 3): # tasks/<task>/, then tasks/<split>/<task>/
level = [c for d in level[:3] for c in folders(d, 10)]
if not level:
return False
if found([f"{d}/task.toml" for d in level]):
return True
return False
except Exception: # noqa: BLE001 - can't tell: read it as rows
return False
api = _api(token)
return _cached(("looks-harbor", spec, sha), 3600, check)
LIST_BUDGET = float(os.environ.get("RLX_LIST_BUDGET", 25)) # seconds for one recursive listing before walking instead
# a task's files read for the index when it is walked folder by folder: checked in batches, never listed
KNOWN_FILES = ("task.toml", "instruction.md", "README.md", "tests/test.sh", "environment/Dockerfile",
"environment/docker-compose.yaml", "environment/docker-compose.yml", "environment/compose.yaml", "solution/solve.sh",
"tests/Dockerfile", "trajectory.json") # a separate grader's build; a prior conversation
class _TooBig(Exception):
pass
def _listing(spec: str, sha: str, token: str | None, job: dict[str, Any]) -> tuple[dict[str, int], dict[str, Any] | None]:
"""Every file, from one recursive listing; or, when that is too slow or too big (tasks that each ship a whole
app or repository), the tasks found folder by folder (see _walk). Returns (files, None) or ({}, walked)."""
from huggingface_hub.hf_api import RepoFile
files: dict[str, int] = {}
t0 = time.time()
try:
for n, entry in enumerate(_api(token).list_repo_tree(spec, repo_type="dataset", revision=sha, recursive=True)):
if isinstance(entry, RepoFile):
files[entry.path] = entry.size
if n % 500 == 0:
job["files"] = len(files)
if time.time() - t0 > LIST_BUDGET or n > config.MAX_LISTING_FILES:
raise _TooBig
except _TooBig:
job.update(files=0, mode="folders")
return {}, _walk(spec, sha, token, job)
job["files"] = len(files)
return files, None
def _walk(spec: str, sha: str, token: str | None, job: dict[str, Any]) -> dict[str, Any]:
"""Find the tasks folder by folder: list a level, ask which of its folders hold a task.toml (one batched call per
thousand folders), descend into the rest. Inside a task only the files the index reads are looked up (batched
too); the task's folder is listed when its page opens, and each subfolder when someone opens it (`task_folder`)."""
from huggingface_hub.hf_api import RepoFile, RepoFolder
api = _api(token)
def ls(path: str) -> list[Any]:
return list(api.list_repo_tree(spec, repo_type="dataset", revision=sha, path_in_repo=path or None, recursive=False))
def info_of(paths: list[str]) -> list[Any]:
out: list[Any] = []
chunks = [paths[i:i + 500] for i in range(0, len(paths), 500)] # the Hub takes up to ~1,000 form fields a call
with ThreadPoolExecutor(max_workers=4) as pool:
for got in pool.map(lambda c: api.get_paths_info(spec, c, repo_type="dataset", revision=sha), chunks):
out += got
return out
tasks: list[str] = []
found: dict[str, int] = {}
root = ls("")
calls, depth = 1, 0
if any(isinstance(e, RepoFile) and e.path == "task.toml" for e in root):
tasks, found, frontier = [""], {"task.toml": next(e.size for e in root if e.path == "task.toml")}, []
else:
frontier = [e.path for e in root if isinstance(e, RepoFolder) and not posixpath.basename(e.path).startswith(".")]
while frontier and depth < MAX_DEPTH:
hits = {}
for e in info_of([f"{d}/task.toml" for d in frontier]):
if isinstance(e, RepoFile):
hits[posixpath.dirname(e.path)] = e.size
calls += (len(frontier) + 499) // 500
tasks += sorted(hits)
found.update({f"{d}/task.toml": n for d, n in hits.items()})
if len(tasks) > config.MAX_INDEX_TASKS:
raise ValueError(f"over {config.MAX_INDEX_TASKS:,} tasks: more than this explorer indexes")
rest = [d for d in frontier if d not in hits]
if len(rest) > 3000:
raise ValueError(f"{len(rest):,} folders without a task.toml at one level: not a Harbor layout this explorer can walk")
job.update(tasks=len(tasks), files=calls)
nxt: list[str] = []
with ThreadPoolExecutor(max_workers=8) as pool:
for entries in pool.map(ls, rest):
nxt += [e.path for e in entries if isinstance(e, RepoFolder) and not posixpath.basename(e.path).startswith(".")]
calls += len(rest)
frontier, depth = nxt, depth + 1
tasks.sort(key=lambda d: (d.count("/"), d))
# the files the index reads, looked up for every task at once (five hundred paths a call); each task's own folder
# is listed when its page opens
files: dict[str, dict[str, int]] = {d: {"task.toml": found.get(f"{d}/task.toml".lstrip("/"), 0)} for d in tasks}
job.update(state="listing", done=0, total=len(tasks))
for e in info_of([f"{d}/{k}".lstrip("/") for d in tasks for k in KNOWN_FILES if k != "task.toml"]):
if not isinstance(e, RepoFile):
continue
for d in (posixpath.dirname(e.path), posixpath.dirname(posixpath.dirname(e.path))):
if d in files:
files[d][e.path[len(d) + 1:] if d else e.path] = e.size
break
job["files"] = sum(len(f) for f in files.values())
return {"tasks": tasks, "files": files, "lazy": {d: ["*"] for d in tasks}}
def _folder(spec: str, sha: str, path: str, rel: str, token: str | None) -> list[dict[str, Any]]:
"""One folder of a task (rel "" is the task's own), its entries relative to the task, cached for an hour."""
from huggingface_hub.hf_api import RepoFile
target = "/".join(x for x in (path, rel) if x)
def fetch():
out = []
for e in _api(token).list_repo_tree(spec, repo_type="dataset", revision=sha, path_in_repo=target or None, recursive=False):
sub = e.path[len(path) + 1:] if path else e.path
if isinstance(e, RepoFile):
out.append({"path": sub, "size": e.size, "withheld": _withheld(sub)})
elif not posixpath.basename(sub).startswith("."):
out.append({"path": sub + "/", "dir": True, "lazy": True, "withheld": _withheld(sub + "/")})
if len(out) >= 3000:
break
return out
return _cached(("folder", spec, sha, target), 3600, fetch)
def _lister(idx: dict[str, Any], token: str | None) -> str | None:
return token or (config.INDEX_TOKEN if not idx["info"].get("restricted") else None)
def task_folder(spec: str, path: str, rel: str, token: str | None = None) -> dict[str, Any]:
"""One folder inside a task, listed when someone opens it in the file viewer: its files and subfolders."""
spec = check_spec(spec)
_, idx = _task_row(spec, path, token)
rel = posixpath.normpath(rel or "").strip("/")
if not rel or rel.startswith("..") or rel == ".":
raise ValueError("not a folder in this task")
entries = _folder(spec, idx["sha"], path, rel, _lister(idx, token))
return {"entries": entries, "truncated": len(entries) >= 3000}
def _walked_root(spec: str, idx: dict[str, Any], path: str, token: str | None) -> tuple[dict[str, int], list[str]]:
"""A walked task's own folder: its files, and its folders (listed when opened). Empty for a listed task."""
pack = _read_pack(spec, idx["sha"])
entry = pack and pack["tasks"].get(path)
if not idx.get("walked") and (not entry or "*" not in (entry.get("lazy") or [])):
return {}, sorted(entry.get("lazy") or []) if entry else []
entry = entry or {"files": {}}
try:
entries = _folder(spec, idx["sha"], path, "", _lister(idx, token))
except Exception: # noqa: BLE001 - the index's own files still show
return {}, sorted({k.split("/")[0] for k in entry["files"] if "/" in k})
return ({e["path"]: e["size"] for e in entries if not e.get("dir")},
sorted(e["path"].rstrip("/") for e in entries if e.get("dir")))
def _download(spec: str, sha: str, token: str | None, paths: list[str], local: Path, job: dict[str, Any]) -> None:
"""The files, into a folder for this revision, many at once: one GET each (no metadata round trip), so a
dataset of thousands of tasks reads in minutes. A file already there isn't fetched again."""
import httpx
from urllib.parse import quote
headers = {"User-Agent": "hf-rl-explorer"} | ({"Authorization": f"Bearer {token}"} if token else {})
base = f"https://huggingface.co/datasets/{spec}/resolve/{sha}/"
with httpx.Client(headers=headers, follow_redirects=True, timeout=httpx.Timeout(60, connect=15),
limits=httpx.Limits(max_connections=48, max_keepalive_connections=48)) as client:
def one(path: str) -> None:
dst = local / path
try:
if not dst.exists():
for attempt in range(4):
r = client.get(base + quote(path))
if r.status_code == 429 or r.status_code >= 500:
time.sleep(1.5 * (attempt + 1))
continue
if r.status_code == 200:
dst.parent.mkdir(parents=True, exist_ok=True)
tmp = dst.with_name(dst.name + ".part")
tmp.write_bytes(r.content)
tmp.replace(dst)
break
except Exception: # noqa: BLE001 - a file that won't download is a gap on one task's page, not a failed index
pass
job["done"] = job.get("done", 0) + 1
with ThreadPoolExecutor(max_workers=48) as pool:
list(pool.map(one, paths))
# ── packs: every task's text, one file per dataset ───────────────────────────
_packs: OrderedDict[tuple[str, str], dict[str, Any]] = OrderedDict()
_pack_bytes: dict[tuple[str, str], int] = {}
_packs_lock = threading.Lock()
PACK_CACHE_BYTES = int(os.environ.get("RLX_PACK_CACHE_MB", 700)) * 1024 * 1024 # about one big pack (LegoFlow) and a few small
def _pack_path(spec: str) -> Path:
return config.PACK_DIR / f"{_slug(spec)}.json.gz"
def _write_pack(spec: str, sha: str, pack: dict[str, Any]) -> None:
atomic_write(_pack_path(spec), gzip.compress(json.dumps({"spec": spec, "sha": sha, "version": INDEX_VERSION, **pack},
separators=(",", ":")).encode(), compresslevel=6))
def _read_pack(spec: str, sha: str) -> dict[str, Any] | None:
"""A dataset's pack, kept in memory for the few datasets read most recently."""
key = (spec, sha)
with _packs_lock:
if key in _packs:
_packs.move_to_end(key)
return _packs[key]
try:
raw = gzip.decompress(_pack_path(spec).read_bytes())
pack = json.loads(raw)
except (OSError, ValueError):
return None
if pack.get("sha") != sha or pack.get("version") != INDEX_VERSION or "blobs" not in pack:
return None
size = len(raw) * 2 # parsed, it takes about twice its text
with _packs_lock:
_packs[key] = pack
_pack_bytes[key] = size
while len(_packs) > 1 and sum(_pack_bytes.get(k, 0) for k in _packs) > PACK_CACHE_BYTES:
old, _ = _packs.popitem(last=False)
_pack_bytes.pop(old, None)
return pack
# what may hold an answer: the reference solution, files named for answers, data beside the grader
ANSWER_FILE = re.compile(r"(^|[/_.\-])(gold|answers?|solutions?|expected|ground_truth|oracle|rubrics?|verifier_meta|references?)"
r"([/_.\-]|$)", re.I)
ANSWER_TEXT = re.compile(r"\b(pass_anchor|gold_answer|expected_answer|ground_truth|reference_answer)\b")
CODE_FILE = re.compile(r"\.(sh|bash|py|js|mjs|ts|rb|go|rs|java|pl|r)$", re.I)
ANSWER_NAME = r"[A-Za-z_]*(?:expected|answer|gold|solution|ground_truth|oracle|reference)[A-Za-z_0-9]*"
_HEREDOC = re.compile(r"(<<-?\s*['\"]?(\w+)['\"]?[^\n]*\n)(.*?)(\n[ \t]*\2\b)", re.S)
_ASSIGN = re.compile(rf"^(\s*(?:export\s+|local\s+|readonly\s+)?({ANSWER_NAME})\s*[:]?=\s*)(['\"0-9\[{{-].*)$", re.I | re.M)
def _mask_answers(rel: str, text: str) -> str:
"""A grader script with the answer it compares against taken out: a heredoc whose name or target looks like an
answer (`expected=$(cat <<'ANSWER_EOF' ... ANSWER_EOF)`), and literal values given to answer-like names
(`EXPECTED="42"`). The logic stays readable; the answer doesn't. A step's tests (`steps/<name>/tests/`) too."""
if not _in_step(rel).startswith("tests/"):
return text
def heredoc(m: re.Match) -> str:
line_start = text.rfind("\n", 0, m.start()) + 1
head = text[line_start:m.start()]
if re.search(ANSWER_NAME, m.group(2), re.I) or re.search(ANSWER_NAME, head, re.I):
return f"{m.group(1)}‹withheld: the expected answer›{m.group(4)}"
return m.group(0)
text = _HEREDOC.sub(heredoc, text)
return _ASSIGN.sub(lambda m: f"{m.group(1)}\"‹withheld›\"", text)
# keys of a data file that hold an answer or the reference command (e.g. a manifest's "original_bash")
ANSWER_DATA_KEY = re.compile(r"(gold|answers?$|solutions?|expected|ground_?truth|oracle|reference|_bash$|^bash$|_cmd$|_command$|"
r"_commands$|solve|_output$|^output$|target_)", re.I)
def _answer_data(rel: str, text: str) -> bool:
"""A JSON or YAML data file whose keys hold an answer: a task manifest naming the reference command, say."""
if not re.search(r"\.(json|jsonl|ya?ml)$", rel, re.I) or len(text) > 2_000_000:
return False
keys: list[str] = []
try:
docs = [json.loads(text)] if rel.lower().endswith(".json") else [json.loads(x) for x in text.splitlines()[:200] if x.strip()] if rel.lower().endswith(".jsonl") else []
except ValueError:
docs = []
if not docs and re.search(r"\.ya?ml$", rel, re.I):
keys = re.findall(r"^\s*-?\s*([\w.\-]+)\s*:", text, re.M)
def walk(v: Any, depth: int = 0) -> None:
if depth > 6:
return
if isinstance(v, dict):
for k, x in v.items():
keys.append(str(k))
walk(x, depth + 1)
elif isinstance(v, list):
for x in v[:200]:
walk(x, depth + 1)
for d in docs:
walk(d)
return any(ANSWER_DATA_KEY.search(k) and not k.endswith(("_path", "_file", "_dir")) for k in keys)
def _withheld(rel: str, text: str | None = None) -> bool:
"""Whether a task file is kept off the page. Graders' scripts are shown; what they compare against isn't, nor
a data file whose keys hold an answer or the reference command. A multi-step task's steps (`steps/<name>/`) hold
the same folders, read by the same rules: each step's `solution/` is withheld, its `tests/` like the task's."""
base = _in_step(rel)
if rel.startswith("solution/") or base.startswith("solution/") or ANSWER_FILE.search(rel):
return True
if base.startswith("tests/") and base != "tests/test.sh" and not CODE_FILE.search(base):
return True
if text and _answer_data(rel, text):
return True
return bool(text and (rel.startswith(("tests/", "environment/")) or base.startswith(("tests/", "workdir/"))) and ANSWER_TEXT.search(text))
def _briefs(rows: list[dict[str, Any]]) -> None:
"""Each task's brief: its first paragraph that most of the dataset's tasks don't share. Datasets open every
instruction with the same preamble ("You are an agent working in a sandbox..."), which tells two tasks apart
no better than nothing."""
seen = Counter(p for r in rows for p in set(r["_paras"]))
common = max(2, int(0.3 * len(rows)))
titles = Counter(r["title"] for r in rows)
for r in rows:
own = [p for p in r.pop("_paras") if seen[p] <= common and not p.startswith(r["title"][:60])]
r["brief"] = own[0] if own else ""
# a heading every task shares ("Development Requirements Document") names none of them: its own words do
if titles[r["title"]] > common and own:
r["title"] = own[0] if len(own[0]) <= 140 else own[0][:139].rsplit(" ", 1)[0] + "…"
r["brief"] = own[1] if len(own) > 1 else ""
# ── one task ─────────────────────────────────────────────────────────────────
def _read(path: Path, limit: int = TEXT_LIMIT) -> str:
try:
with path.open("rb") as fh:
return fh.read(limit).decode("utf-8", errors="replace")
except OSError:
return ""
def _first_line(text: str, limit: int = 160) -> str:
"""The instruction's first line of prose: not a code fence or what is inside one, not markup or data."""
fenced = False
for line in text.splitlines():
t = line.strip()
if t.startswith("```"):
fenced = not fenced
continue
t = t.lstrip("#").strip()
if fenced or not t or t.startswith(("<", "|", "---", "{", "[", '"', "'", "$ ")) or CANARY.search(t):
continue
if sum(c.isalpha() or c == " " for c in t) < 0.6 * len(t): # mostly symbols: code or data, not a sentence
continue
return t[:limit]
return ""
def _paragraphs(text: str, limit: int = 320, most: int = 5) -> list[str]:
"""The instruction's first few prose paragraphs, flattened, for a task's brief (chosen per dataset later)."""
out = []
for p in re.split(r"\n\s*\n", text):
p = p.strip()
if not p or p.startswith(("#", "```", "|", "<")) or len(p) < 30 or CANARY.search(p):
continue
if (p.endswith(":") and len(p) < 120) or re.match(r"([-*+]|\d+[.)])\s", p): # a list, or its lead-in
continue
flat = re.sub(r"\s+", " ", p)
out.append(flat if len(flat) <= limit else flat[: limit - 1].rsplit(" ", 1)[0] + "…")
if len(out) >= most:
break
return out
def _scalar(v: Any, limit: int = 80) -> str | None:
if isinstance(v, bool):
return "yes" if v else "no"
if isinstance(v, (int, float, str)) and str(v).strip():
t = " ".join(str(v).split())
return t if len(t) <= limit else t[: limit - 1].rstrip() + "…"
return None
def _strings(v: Any) -> list[str]:
if isinstance(v, str):
return [x.strip() for x in v.split(",") if x.strip()] if "," in v and len(v) < 200 else [v.strip()]
if isinstance(v, (list, tuple)):
return [str(x).strip()[:60] for x in v if isinstance(x, (str, int, float)) and str(x).strip()]
return []
def _first(meta: dict[str, Any], keys: tuple[str, ...]) -> str | None:
for k in keys:
if k in meta and not ANSWER_KEY.search(k):
v = _scalar(meta[k])
if v:
return v
return None
# ── task.toml, read the way Harbor reads it ──────────────────────────────────
# the sections Harbor's TaskConfig knows (harbor/models/task/config.py); anything else at the top level (LegoFlow's
# [scoring]) is the dataset's own, kept as metadata
HARBOR_KEYS = ("schema_version", "version", "task", "metadata", "verifier", "agent", "environment", "solution", "source",
"multi_step_reward_strategy", "steps", "artifacts")
_quiet = threading.Event()
def _dict(v: Any) -> dict[str, Any]:
return v if isinstance(v, dict) else {}
def _num(v: Any) -> int | float | None:
"""A number from task.toml, or None (a file Harbor rejects may say `gpus = "1"`)."""
return v if isinstance(v, (int, float)) and not isinstance(v, bool) else None
def _size_mb(v: Any) -> int | None:
"""A legacy size ("4G", "512M", "1024K") in MB, as Harbor's EnvironmentConfig converts `memory` and `storage`."""
m = re.fullmatch(r"\s*(\d+(?:\.\d+)?)\s*([GMK])\s*", v, re.I) if isinstance(v, str) else None
if not m:
return None
n, unit = float(m.group(1)), m.group(2).upper()
return int(n * 1024) if unit == "G" else int(n) if unit == "M" else int(n / 1024)
def _as_written(doc: dict[str, Any]) -> dict[str, Any]:
"""task.toml as written, with Harbor's legacy fields converted the way TaskConfig converts them (for a file it
rejects, still worth reading)."""
def env_of(e: Any) -> dict[str, Any]:
e = dict(_dict(e))
for old, new in (("memory", "memory_mb"), ("storage", "storage_mb")):
if old in e:
mb = _size_mb(e.pop(old))
if mb is not None:
e.setdefault(new, mb)
if e.get("allow_internet") is not None and "network_mode" not in e and e.get("allowed_hosts") is None:
e["network_mode"] = "public" if e["allow_internet"] else "no-network"
e.pop("allow_internet", None)
e.setdefault("network_mode", "public")
if isinstance(e.get("os"), str):
e["os"] = e["os"].lower()
return e
def ver_of(v: Any) -> dict[str, Any]:
v = dict(_dict(v))
if isinstance(v.get("environment"), dict):
v["environment"] = env_of(v["environment"])
return v
cfg = {k: v for k, v in doc.items() if k != "_invalid"}
if "version" in cfg:
cfg.setdefault("schema_version", cfg.pop("version"))
cfg.update(environment=env_of(doc.get("environment")), verifier=ver_of(doc.get("verifier")), agent=dict(_dict(doc.get("agent"))))
steps = doc.get("steps")
cfg["steps"] = [dict(s, verifier=ver_of(s.get("verifier"))) for s in steps if isinstance(s, dict)] if isinstance(steps, list) else None
return cfg
def _task_config(doc: dict[str, Any]) -> tuple[dict[str, Any], str | None]:
"""task.toml as Harbor's own TaskConfig reads it, as plain JSON: legacy `memory = "4G"`, `storage`, `allow_internet`
and `version` converted, Harbor's defaults filled in. A file TaskConfig rejects (so Harbor won't run it) is read as
written, legacy fields converted the same way, with TaskConfig's reason."""
import copy
import warnings
if not doc or doc.get("_invalid"):
return {}, None
try:
from harbor.models.task.config import TaskConfig
except ImportError: # no Harbor here: as written
return _as_written(doc), None
if not _quiet.is_set(): # one per legacy field per task: thousands while indexing
warnings.filterwarnings("ignore", message=r"The '\w+' field is deprecated", category=DeprecationWarning)
_quiet.set()
try:
return TaskConfig.model_validate(copy.deepcopy(doc)).model_dump(mode="json"), None
except Exception as exc: # noqa: BLE001 - pydantic's ValidationError, or a validator's own error
why = f"{type(exc).__name__}: {exc}"
try:
e = exc.errors()[0] # type: ignore[attr-defined]
where = ".".join(str(x) for x in e.get("loc") or ())
msg = re.sub(r"^(Value|Assertion) error, ", "", str(e.get("msg") or ""))
why = f"`{where}`: {msg}" if where else msg
except Exception: # noqa: BLE001 - not a ValidationError: its own words
pass
return _as_written(doc), " ".join(why.split())[:240]
def _verifier_mode(ver: Any) -> str | None:
"""Harbor's rule for one [verifier]: its `environment_mode`, else "separate" when it defines an environment, else
unset (a step inherits the task's; the task's default is "shared")."""
ver = _dict(ver)
if ver.get("environment_mode") in ("shared", "separate"):
return ver["environment_mode"]
return "separate" if isinstance(ver.get("environment"), dict) else None
def _phases(cfg: dict[str, Any]) -> dict[str, str]:
"""The network each phase sets for itself (Harbor's [agent] and [verifier] `network_mode` overrides, a separate
grader's own container, each step's), by phase. The sandbox's own is the row's env["network"]."""
out: dict[str, str] = {}
def put(key: str, sec: Any) -> None:
mode = _dict(sec).get("network_mode")
if isinstance(mode, str) and mode:
out[key] = mode
ver = _dict(cfg.get("verifier"))
put("agent", cfg.get("agent"))
put("verifier", ver)
if _verifier_mode(ver) == "separate":
put("verifier.environment", ver.get("environment"))
for s in (cfg.get("steps") or [])[:MAX_STEPS]:
s = _dict(s)
name = str(s.get("name") or "step")
put(f"steps.{name}.agent", s.get("agent"))
put(f"steps.{name}.verifier", s.get("verifier"))
if _verifier_mode(s.get("verifier")) == "separate":
put(f"steps.{name}.verifier.environment", _dict(s.get("verifier")).get("environment"))
return out
def phase_label(key: str) -> str:
"""A phase key from `_phases` in words: "verifier" is "the grader", "steps.s1.agent" is "step `s1`'s agent"."""
m = re.match(r"steps\.(.+)\.(agent|verifier|verifier\.environment)$", key)
own = {"agent": "agent", "verifier": "grader", "verifier.environment": "grader's own container"}
return f"step `{m.group(1)}`'s {own[m.group(2)]}" if m else f"the {own.get(key, key)}"
def _hosts(cfg: dict[str, Any]) -> list[str]:
"""Every host a network allowlist names, in any phase."""
secs = [cfg.get("environment"), cfg.get("agent"), cfg.get("verifier"), _dict(cfg.get("verifier")).get("environment")]
for s in cfg.get("steps") or []:
secs += [_dict(s).get("agent"), _dict(s).get("verifier"), _dict(_dict(s).get("verifier")).get("environment")]
out: list[str] = []
for sec in secs:
for h in _dict(sec).get("allowed_hosts") or []:
if isinstance(h, str) and h not in out:
out.append(h)
return out
def _separate(cfg: dict[str, Any], names: Any, texts: dict[str, str]) -> dict[str, Any] | None:
"""A grader that runs in a container of its own (`[verifier] environment_mode = "separate"`, or a
`[verifier.environment]`), as Harbor builds it: from `[verifier.environment]` (else a copy of `[environment]`),
its image, else `tests/Dockerfile`; and whether an HF Sandbox can start it (an image, or a Dockerfile this
explorer can replay)."""
from . import dockerfile
ver = _dict(cfg.get("verifier"))
task_mode = _verifier_mode(ver) or "shared"
steps = [_dict(s) for s in cfg.get("steps") or []]
modes = [(_verifier_mode(s.get("verifier")) or task_mode) for s in steps] if steps else [task_mode]
if "separate" not in modes:
return None
own = isinstance(ver.get("environment"), dict)
venv = _dict(ver.get("environment")) if own else _dict(cfg.get("environment"))
image = _scalar(venv.get("docker_image"), 300)
df = "tests/Dockerfile" in names
if image:
ok, why = True, ""
elif df:
text = texts.get("tests/Dockerfile")
ok, why = dockerfile.check(text) if text else (False, "couldn't be read")
why = f"its `tests/Dockerfile` can't be replayed here ({why})" if why else ""
else:
ok, why = False, "it gives that container no image or Dockerfile of its own"
return {k: v for k, v in {"own": own, "image": image, "dockerfile": df, "ok": ok, "why": why, "gpus": _num(venv.get("gpus")) or 0,
"steps": modes.count("separate") if steps else None}.items() if v not in (None, "", 0, False) or k == "ok"}
def _flat(table: Any, prefix: str = "", depth: int = 0) -> list[tuple[str, str]]:
"""A metadata table's short values by dotted key (`repo2env.recipe`), nested tables two levels down; answers and
contact details left out."""
out: list[tuple[str, str]] = []
for k, v in _dict(table).items():
if ANSWER_KEY.search(str(k)) or CONTACT_KEY.search(str(k)):
continue
if isinstance(v, dict):
if depth < 2:
out += _flat(v, f"{prefix}{k}.", depth + 1)
else:
s = _scalar(v)
if s is not None and len(s) <= 60:
out.append((f"{prefix}{k}", s))
return out
def _answer_keys(v: Any, prefix: str = "", depth: int = 0) -> list[str]:
"""Every answer-like key in a table, at any depth (and in arrays of tables), by dotted path."""
out: list[str] = []
if depth > 8:
return out
if isinstance(v, dict):
for k, x in v.items():
if ANSWER_KEY.search(str(k)):
out.append(f"{prefix}{k}")
else:
out += _answer_keys(x, f"{prefix}{k}.", depth + 1)
elif isinstance(v, list):
for x in v[:200]:
out += [k for k in _answer_keys(x, prefix, depth + 1) if k not in out]
return out
def _clean(v: Any, depth: int = 0) -> Any:
"""A table as JSON with its answer-like and contact keys left out at every depth (dates as text)."""
if depth > 8:
return None
if isinstance(v, dict):
return {str(k): _clean(x, depth + 1) for k, x in v.items() if not ANSWER_KEY.search(str(k)) and not CONTACT_KEY.search(str(k))}
if isinstance(v, list):
return [_clean(x, depth + 1) for x in v[:200]]
return v if v is None or isinstance(v, (str, int, float, bool)) else str(v)
# a grader file that calls a model itself: an LLM SDK imported, and its completion / messages / responses API called
_SDK = re.compile(r"\bfrom\s+(?:openai|anthropic|litellm)(?:\.[\w.]+)?\s+import\b|\bimport\s+(?:openai|anthropic|litellm)\b|"
r"\brequire\(\s*['\"](?:openai|@anthropic-ai/sdk)['\"]\s*\)|\bfrom\s+['\"](?:openai|@anthropic-ai/sdk)['\"]")
_CALL = re.compile(r"\.(?:chat\.completions|completions|messages|responses)\.(?:create|parse|stream)\s*\(|\blitellm\.a?completion\s*\(")
def _calls_model(text: str) -> bool:
"""Whether a grader file calls a model: it imports openai, anthropic or litellm AND calls a completion, messages or
responses API. Mentioning a judge (a `judge: bool` parameter, a key's name) isn't enough."""
if not text or not _SDK.search(text):
return False
if _CALL.search(text):
return True
return bool(re.search(r"\bfrom\s+litellm\s+import\s+[^\n]*\ba?completion\b", text)
and re.search(r"^(?!\s*(?:async\s+)?def\s)[^\n#]*(?<![\w.])a?completion\s*\(", text, re.M))
def verifier_kind(test_sh: str, extra: str = "", env_keys: list[str] | None = None, *, toml_text: str | None = None,
sources: dict[str, str] | None = None) -> dict[str, Any]:
"""How a task is graded, read from its verifier script: one score (`reward.txt`) or named scores
(`reward.json`, with the keys a literal in the script names), tests (pytest), and whether a model grades it: its
task.toml hands the grader a model's key or URL (app/judge.py's plan), or a grader file imports an LLM SDK and calls
it (`sources`: the grader's other files by path; without them, `extra` is read as one). `env_keys` is accepted for
old callers and no longer counts: a key's name alone isn't a judge."""
text = test_sh + "\n" + extra
kind = "json" if "reward.json" in text else "txt" if "reward.txt" in text else None
keys: list[str] = []
if kind == "json":
for line in text.splitlines():
if "reward" in line and "{" in line:
keys += re.findall(r"""["'](\w{2,40})["']\s*:""", line)
judged = False
if toml_text:
from . import judge as judges
judged = judges.plan(toml_text).get("judge") is not None
files = [test_sh, *sources.values()] if sources is not None else [test_sh, extra]
return {"kind": kind, "keys": sorted(set(keys))[:12], "pytest": "pytest" in text,
"judge": judged or any(_calls_model(t) for t in files)}
def _compact(d: dict[str, Any]) -> dict[str, Any]:
return {k: v for k, v in d.items() if v not in (None, "", [], {}, False, 0) or k == "ok"}
def _facts(cfg: dict[str, Any], error: str | None, doc: dict[str, Any], names: Any,
texts: dict[str, str]) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
"""What the index keeps of a task.toml beyond the basics, compactly (empty values left out): for env, verifier,
and the row's `spec`."""
env, ver, agent = _dict(cfg.get("environment")), _dict(cfg.get("verifier")), _dict(cfg.get("agent"))
steps = [_dict(s) for s in cfg.get("steps") or []]
tpu = _dict(env.get("tpu"))
env_more = _compact({
"storage_mb": _num(env.get("storage_mb")), "os": env.get("os") if env.get("os") not in (None, "linux") else None,
"gpu_types": [str(g) for g in env.get("gpu_types") or []][:8], "tpu": " ".join(str(tpu[k]) for k in ("type", "topology") if tpu.get(k)),
"workdir": _scalar(env.get("workdir"), 200), "skills_dir": _scalar(env.get("skills_dir"), 200),
"healthcheck": isinstance(env.get("healthcheck"), dict) or any(isinstance(s.get("healthcheck"), dict) for s in steps),
"mcp": [str(_dict(m).get("name")) for m in env.get("mcp_servers") or [] if _dict(m).get("name")][:10],
"hosts": _hosts(cfg)[:20], "phases": _phases(cfg)})
ver_more = _compact({
"separate": _separate(cfg, names, texts), "user": _scalar(ver.get("user")), "collect": len(ver.get("collect") or []),
"bat": "tests/test.bat" in names})
if steps:
mins = {str(s.get("name")): s.get("min_reward") for s in steps if s.get("min_reward") is not None}
multi = _compact({"strategy": cfg.get("multi_step_reward_strategy") or "mean", "min_reward": dict(list(mins.items())[:MAX_STEPS])})
else:
multi = None
pkg = _dict(cfg.get("task"))
arts = [a if isinstance(a, str) else _dict(a).get("source") for a in cfg.get("artifacts") or []]
spec = _compact({
"schema": _scalar(cfg.get("schema_version")), "harbor_error": error,
"package": _compact({"name": _scalar(pkg.get("name"), 120), "version": _scalar(pkg.get("version"), 40)}),
"authors": [str(_dict(a).get("name"))[:80] for a in pkg.get("authors") or [] if _dict(a).get("name")][:8],
"agent_user": _scalar(agent.get("user")), "artifacts": [str(a)[:160] for a in arts if a][:8], "multi": multi,
"step_names": [str(s.get("name")) for s in steps][:MAX_STEPS],
"trajectory": "trajectory.json" in names or any(_in_step(n) == "trajectory.json" for n in names if n.startswith("steps/")),
"tables": [k for k, v in doc.items() if k not in HARBOR_KEYS and k != "_invalid" and isinstance(v, dict)][:8]})
return env_more, ver_more, spec
def _row(i: int, d: str, local: Path, files: dict[str, int], picks: list[str],
job: dict[str, Any]) -> tuple[dict[str, Any], dict[str, str]]:
"""One task's index row, and its texts for the pack (answers left out, task.toml masked). task.toml is read by
Harbor's own TaskConfig (`_task_config`); a multi-step task's instructions come from steps/<name>/instruction.md."""
folder = local / d if d else local
texts = {r: _read(folder / r, TEXT_LIMIT if _in_step(r) in ("task.toml", "instruction.md") else SMALL) for r in picks}
toml_text = texts.get("task.toml", "")
try:
doc = tomllib.loads(toml_text) if toml_text else {}
except tomllib.TOMLDecodeError:
doc = {"_invalid": True}
cfg, error = _task_config(doc)
task, meta = _dict(cfg.get("task")), _dict(doc.get("metadata"))
raw_env, env, ver, agent = _dict(doc.get("environment")), _dict(cfg.get("environment")), _dict(cfg.get("verifier")), _dict(cfg.get("agent"))
steps = cfg.get("steps") if isinstance(cfg.get("steps"), list) else None
first = str(_dict(steps[0]).get("name")) if steps else None
instruction = texts.get("instruction.md", "") or (texts.get(f"steps/{first}/instruction.md", "") if first else "")
test_sh = texts.get("tests/test.sh", "")
name = posixpath.basename(d) or "task"
title = (_scalar(meta.get("title"), 200) or _scalar(task.get("description"), 200) or _first_line(instruction)
or name).strip()
tags: list[str] = []
for k in TAG_KEYS:
for t in _strings(meta.get(k)) + (_strings(task.get(k)) if k == "keywords" else []):
if t not in tags and len(tags) < 12:
tags.append(t)
names = list(files)
job["done"] = job.get("done", 0) + 1
# the scripts test.sh hands the grading to (`python3 /tests/grade.py`) say which reward file it writes; a step's
# tests too
sources = {r: t for r, t in texts.items() if _in_step(r).startswith("tests/") and r != "tests/test.sh"}
tables = {k: v for k, v in doc.items() if k not in HARBOR_KEYS and isinstance(v, dict)}
flat = _flat(meta)[:48] + [kv for k, v in tables.items() for kv in _flat(v, f"{k}.", 1)][:16]
env_more, ver_more, spec = _facts(cfg, error, doc, set(names), texts)
row = {
"i": i, "path": d, "name": name, "title": title[:240],
"_paras": _paragraphs(instruction),
"difficulty": _first(meta, DIFFICULTY_KEYS), "category": _first(meta, CATEGORY_KEYS), "tags": tags,
"meta": dict(flat),
"withheld": sorted(set(_answer_keys(meta) + [k for t, v in tables.items() for k in _answer_keys(v, f"{t}.")])),
"env": {"image": _scalar(env.get("docker_image"), 300), "dockerfile": "environment/Dockerfile" in files,
"compose": any(re.match(r"environment/(docker-)?compose\.ya?ml$", n) for n in names),
"cpus": _num(env.get("cpus")), "memory_mb": _num(env.get("memory_mb")), "gpus": _num(env.get("gpus")) or 0,
"internet": False if _network(raw_env) in ("no-network", "allowlist") else raw_env.get("allow_internet"),
"network": _network(raw_env), "build_timeout": env.get("build_timeout_sec"), **env_more},
"verifier": {**verifier_kind(test_sh, "\n".join(sources.values()), toml_text=toml_text, sources=sources),
"timeout": ver.get("timeout_sec"), "files": sorted(n for n in names if n.startswith("tests/"))[:40],
"env": sorted(_dict(ver.get("env")).keys()), **ver_more},
"agent_timeout": agent.get("timeout_sec"),
"steps": len(steps) if isinstance(steps, list) else None,
"solution": any(_in_step(n).startswith("solution/") for n in names),
"reads_env": "${" in toml_text,
"files": len(files), "bytes": sum(files.values()),
"invalid": bool(doc.get("_invalid")) or not toml_text,
"spec": spec,
}
row["run"] = runnable(row, texts)["how"] # how it runs on an HF Sandbox: "image", "dockerfile", or not at all
kept = {r: (_masked_toml(t) if r == "task.toml" else _mask_answers(r, t)) for r, t in texts.items() if t and not _withheld(r, t)}
return row, kept
def _group(rows: list[dict[str, Any]]) -> None:
"""A task's group: the folders between the dataset's common root and the task, when there are any."""
parents = [posixpath.dirname(r["path"]) for r in rows]
common = posixpath.commonpath(parents) if parents and all(parents) else ""
for r, p in zip(rows, parents):
rel = p[len(common):].strip("/") if common else p
rel = re.sub(r"(^|/)tasks$", "", rel).strip("/") # `hle/tasks` is the group `hle`
r["group"] = rel or None
def summarize(rows: list[dict[str, Any]]) -> dict[str, Any]:
"""A dataset at a glance: environments, how it is graded, and the facets worth filtering by."""
n = len(rows) or 1
kinds = Counter(r["verifier"]["kind"] or "unknown" for r in rows)
keys = Counter(k for r in rows for k in r["verifier"]["keys"])
meta_keys = Counter(k for r in rows for k in r["meta"])
facets = {}
for name in ("difficulty", "category", "group"):
c = Counter(r.get(name) for r in rows if r.get(name))
if 1 < len(c) <= 60 or (len(c) == 1 and sum(c.values()) < n):
facets[name] = c.most_common(40)
tags = Counter(t for r in rows for t in r["tags"])
if tags:
facets["tags"] = tags.most_common(40)
# other metadata worth filtering by: a few distinct values, on enough of the tasks
for k, seen in meta_keys.most_common(30):
if k in DIFFICULTY_KEYS + CATEGORY_KEYS + TAG_KEYS or seen < 0.2 * n:
continue
c = Counter(r["meta"].get(k) for r in rows if r["meta"].get(k))
repeated = sum(v for v in c.values() if v > 1)
if 2 <= len(c) <= 25 and repeated >= 0.5 * sum(c.values()): # categories, not one-off names
facets[f"meta:{k}"] = c.most_common(25)
return {
"tasks": len(rows),
"image": sum(1 for r in rows if r["env"]["image"]),
"dockerfile_only": sum(1 for r in rows if not r["env"]["image"] and r["env"]["dockerfile"]),
"compose": sum(1 for r in rows if r["env"]["compose"]),
"gpu": sum(1 for r in rows if r["env"]["gpus"]),
"solution": sum(1 for r in rows if r["solution"]),
"runnable": sum(1 for r in rows if r.get("run")),
"replayed": sum(1 for r in rows if r.get("run") == "dockerfile"),
"multi_step": sum(1 for r in rows if r["steps"]),
"reads_env": sum(1 for r in rows if r["reads_env"]),
"invalid": sum(1 for r in rows if r["invalid"]),
"verifier": dict(kinds), "reward_keys": keys.most_common(20),
"pytest": sum(1 for r in rows if r["verifier"]["pytest"]),
"judge": sum(1 for r in rows if r["verifier"]["judge"]),
"withheld": sorted({k for r in rows for k in r["withheld"]}),
"facets": facets,
"meta_keys": [k for k, _ in meta_keys.most_common(40)],
}
# ── a task's page ────────────────────────────────────────────────────────────
def _task_row(spec: str, path: str, token: str | None = None) -> tuple[dict[str, Any], dict[str, Any]]:
meta = info(spec, token) # checks this visitor may read it, every time
idx = _read_index(spec, meta["sha"])
if not idx:
raise LookupError("this dataset is not indexed yet")
for r in idx["tasks"]:
if r["path"] == path:
return r, idx
raise LookupError("no such task")
def _task_files(spec: str, sha: str, path: str, token: str | None) -> dict[str, int]:
from huggingface_hub.hf_api import RepoFile
def fetch():
out = {}
for e in _api(token).list_repo_tree(spec, repo_type="dataset", revision=sha, path_in_repo=path or None, recursive=True):
if isinstance(e, RepoFile):
out[e.path[len(path) + 1:] if path else e.path] = e.size
return out
return _cached(("files", spec, sha, path), 3600, fetch)
def _fetch(spec: str, sha: str, path: str, rel: str, token: str | None) -> Path:
from huggingface_hub import hf_hub_download
target = f"{path}/{rel}".lstrip("/")
return Path(hf_hub_download(spec, target, repo_type="dataset", revision=sha, token=token or False,
local_dir=str(config.CACHE_DIR / _slug(spec))))
_TOML_KEY = re.compile(r"^(\s*)([\w.\-\"']+)(\s*=\s*)(.*)$")
_TOML_TABLE = re.compile(r"^\s*\[\[?\s*([^\[\]]+?)\s*\]\]?\s*(#.*)?$")
def _value_end(lines: list[str], i: int, value: str) -> int:
"""The last line of the TOML value that starts on line `i` (`value`: its text there): line `i` itself, unless it
opens an array, an inline table or a multi-line string that closes further down."""
if not re.match(r"\s*(\[|\{|\"\"\"|''')", value):
return i
acc = value
for j in range(i, min(len(lines), i + 5000)):
if j > i:
acc += "\n" + lines[j]
if re.search(r"(\]|\}|\"\"\"|''')\s*(#.*)?$", lines[j] if j > i else value): # it may close here: try it
try:
tomllib.loads("x = " + acc)
return j
except tomllib.TOMLDecodeError:
pass
return i
def _masked_toml(text: str) -> str:
"""task.toml as written, with any answer-like metadata value replaced: a key named like an answer (its whole
value, over every line it spans), and every key of a table named like one (`[metadata.gold]`)."""
lines = text.splitlines()
out: list[str] = []
hidden, i = False, 0
while i < len(lines):
line = lines[i]
m = _TOML_KEY.match(line)
t = None if m else _TOML_TABLE.match(line)
if t: # a table: its keys are hidden when its name is an answer's (not Harbor's own [solution.env])
name = t.group(1)
hidden = bool(ANSWER_KEY.search(name)) and not re.fullmatch(r"solution(\.env)?", name.strip())
out.append(line)
i += 1
continue
if m:
end = _value_end(lines, i, m.group(4))
if hidden or ANSWER_KEY.search(m.group(2)):
out.append(f"{m.group(1)}{m.group(2)}{m.group(3)}\"‹withheld›\"")
else:
out += lines[i:end + 1]
i = end + 1
continue
out.append(line)
i += 1
return "\n".join(out)
def _task_texts(spec: str, sha: str, path: str, token: str | None) -> tuple[dict[str, int], dict[str, str], set[str]]:
"""A task's file list, its texts (answers left out) and which files were read: from the dataset's pack when it
has one, else from the Hub."""
pack = _read_pack(spec, sha)
entry = pack and pack["tasks"].get(path)
if entry:
files = entry["files"]
return files, {rel: pack["blobs"][h] for rel, h in entry["texts"].items()}, set(_pick(files))
idx = _read_index(spec, sha)
if idx and idx.get("walked"): # a task too big to list: its own folder and the files a page reads
from huggingface_hub.hf_api import RepoFile
files = {e["path"]: e["size"] for e in _folder(spec, sha, path, "", _lister(idx, token)) if not e.get("dir")}
want = [f"{path}/{k}".lstrip("/") for k in KNOWN_FILES if "/" in k]
for e in _api(_lister(idx, token)).get_paths_info(spec, want, repo_type="dataset", revision=sha):
if isinstance(e, RepoFile):
files[e.path[len(path) + 1:] if path else e.path] = e.size
else:
files = _task_files(spec, sha, path, token)
picks = _pick(files)[:16]
texts: dict[str, str] = {}
def one(rel: str) -> None:
try:
text = _read(_fetch(spec, sha, path, rel, token), TEXT_LIMIT if rel in ("task.toml", "instruction.md") else SMALL)
except Exception: # noqa: BLE001 - one missing file doesn't sink the page
return
if not _withheld(rel, text):
texts[rel] = _masked_toml(text) if rel == "task.toml" else _mask_answers(rel, text)
with ThreadPoolExecutor(max_workers=8) as pool:
list(pool.map(one, picks))
return files, texts, set(picks)
def _steps_detail(cfg: dict[str, Any], doc: dict[str, Any], files: dict[str, int], texts: dict[str, str]) -> list[dict[str, Any]]:
"""A multi-step task's steps as Harbor runs them: each one's instruction (steps/<name>/instruction.md), its own
tests, the files staged before it (workdir/), whether it has a solution (never shown) and its settings."""
raw = {str(_dict(s).get("name")): _dict(s) for s in doc.get("steps") or [] if isinstance(s, dict)}
out = []
for s in (cfg.get("steps") or [])[:MAX_STEPS]:
s = _dict(s)
name = str(s.get("name") or "")
own = f"steps/{name}/"
ins = own + "instruction.md"
inline = raw.get(name, {}).get("instruction")
out.append({
"name": name,
# Harbor reads the file; an older task.toml wrote it inline (Harbor ignores that, and the step can't run)
"instruction": texts.get(ins) if ins in texts else None, "inline": inline[:20000] if isinstance(inline, str) else None,
"found": ins in files, "withheld": ins in files and _withheld(ins),
"tests": sorted(k[len(own):] for k in files if k.startswith(own + "tests/"))[:40],
"texts": {k[len(own):]: v for k, v in texts.items() if k.startswith(own + "tests/")},
"workdir": sorted(k[len(own) + 8:] for k in files if k.startswith(own + "workdir/"))[:40],
"solution": any(k.startswith(own + "solution/") for k in files),
"config": {k: v for k, v in s.items() if k != "name"},
})
return out
def task(spec: str, path: str, token: str | None = None) -> dict[str, Any]:
"""What the task page shows: the row, the instruction in full (each step's, for a multi-step task), the settings as
Harbor reads them, the verifier's own files, the environment's recipe and the file list. Answers stay out (see the
module docstring)."""
spec = check_spec(spec)
row, idx = _task_row(spec, path, token)
sha = idx["sha"]
files, texts, read = _task_texts(spec, sha, path, token)
root_files, folders = _walked_root(spec, idx, path, token)
files = {**root_files, **files}
toml_text = texts.get("task.toml", "")
try:
doc = tomllib.loads(toml_text) if toml_text else {}
except tomllib.TOMLDecodeError:
doc = {}
cfg, _ = _task_config(doc) # from the masked task.toml: answer-like values are already out
test_files = {k: v for k, v in texts.items() if k.startswith("tests/")}
sources = {k: v for k, v in texts.items() if _in_step(k).startswith("tests/") and k != "tests/test.sh"}
verifier = {**row["verifier"], **verifier_kind(texts.get("tests/test.sh", ""), "\n".join(sources.values()),
toml_text=toml_text, sources=sources)}
if row["verifier"]["kind"] and not verifier["kind"]: # the index read files the page withholds
verifier.update(kind=row["verifier"]["kind"], keys=row["verifier"]["keys"])
verifier["judge"] = bool(row["verifier"].get("judge") or verifier["judge"])
return {
**row, "dataset": spec, "sha": sha, "verifier": verifier, "collection": collection(spec), "runnable": runnable(row, texts),
"restricted": bool(idx["info"].get("restricted")),
"instruction": texts.get("instruction.md", ""),
"toml": toml_text,
"config": {k: v for k, v in cfg.items() if k != "metadata"},
"metadata": _clean(_dict(doc.get("metadata"))),
"tables": _clean({k: v for k, v in doc.items() if k not in HARBOR_KEYS}),
"steps_detail": _steps_detail(cfg, doc, files, texts),
"dockerfile": texts.get("environment/Dockerfile"),
"compose": next((v for k, v in texts.items() if re.match(r"environment/(docker-)?compose\.ya?ml$", k)), None),
"tests": test_files,
"tree": [{"path": k, "size": v, "withheld": _withheld(k, texts.get(k)) or (k in read and k not in texts)}
for k, v in sorted(files.items())][:2000]
+ [{"path": f"{d}/", "dir": True, "lazy": True, "withheld": _withheld(f"{d}/")} for d in folders],
"walked": bool(folders),
"tree_truncated": len(files) > 2000,
}
def _network(env: dict[str, Any]) -> str | None:
"""The network the task asks for: Harbor's `network_mode`, or the older `allow_internet`."""
mode = env.get("network_mode")
if isinstance(mode, str):
return mode
return "no-network" if env.get("allow_internet") is False else None
def _spec_row(row: dict[str, Any], texts: dict[str, str]) -> dict[str, Any]:
"""A row read outside an index (a Harbor task packed into a dataset row) completed from its task.toml with what
runnable() checks."""
try:
doc = tomllib.loads(texts["task.toml"])
except tomllib.TOMLDecodeError:
return {**row, "spec": {}}
cfg, error = _task_config(doc)
env_more, ver_more, spec = _facts(cfg, error, doc, set(texts), texts)
return {**row, "env": {"gpus": _num(_dict(cfg.get("environment")).get("gpus")) or 0, **(row.get("env") or {}), **env_more},
"verifier": {**(row.get("verifier") or {}), **ver_more}, "spec": spec}
def runnable(row: dict[str, Any], texts: dict[str, str]) -> dict[str, Any]:
"""Whether the task can run on an HF Sandbox: from its prebuilt image, or by replaying its Dockerfile. Not when
Harbor itself rejects its task.toml, nor when it asks for what Harbor's HF Sandbox environment can't give or
enforce: a Windows container, a GPU or a TPU, or no internet or a network allowlist for the sandbox or for any one
phase (Harbor refuses rather than run it more open than its author meant). A grader in a container of its own needs
an image, or a `tests/Dockerfile` this explorer can replay. (`${VAR}`s nothing here fills: app/judge.py.)"""
from . import dockerfile
if "spec" not in row and texts.get("task.toml"): # not from an index: its task.toml says the rest
row = _spec_row(row, texts)
e, spec = row.get("env") or {}, row.get("spec") or {}
sep = (row.get("verifier") or {}).get("separate") or {}
def no(why: str) -> dict[str, Any]:
return {"ok": False, "how": None, "why": why}
if spec.get("harbor_error"):
return no(f"Harbor itself rejects its task.toml ({spec['harbor_error']})")
if e.get("os") == "windows":
return no("it targets a Windows container, and HF Sandboxes run Linux")
gpus = _num(e.get("gpus")) or _num(sep.get("gpus")) or 0
if gpus > 0:
return no(f"it asks for {'a GPU' if gpus == 1 else f'{gpus} GPUs'}, which Harbor can't attach to an HF Sandbox")
if e.get("tpu"):
return no(f"it asks for a TPU slice ({e['tpu']}), which Harbor can't attach to an HF Sandbox")
net = e.get("network")
if net in ("no-network", "allowlist"):
return no("it asks for a sandbox without internet access, which HF Sandboxes can't enforce"
if net == "no-network" else "it asks for a network allowlist, which HF Sandboxes can't enforce")
for phase, mode in (e.get("phases") or {}).items():
if mode in ("no-network", "allowlist"):
return no(f"{phase_label(phase)} asks for {'no internet access' if mode == 'no-network' else 'a network allowlist'}, "
"which HF Sandboxes can't enforce")
if sep and not sep.get("ok"):
return no(f"its grader runs in a container of its own, and {sep.get('why') or 'that container cannot start here'}")
if row["env"].get("image"):
return {"ok": True, "how": "image"}
if row["env"].get("compose"):
return {"ok": False, "how": None, "why": "it runs several containers from a compose file"}
text = texts.get("environment/Dockerfile")
ok, why = dockerfile.check(text)
return {"ok": ok, "how": "dockerfile" if ok else None, "why": why,
"base": dockerfile.plan(text).base if ok else None}
def task_dir(spec: str, path: str, token: str | None = None) -> Path:
"""A task's folder on local disk, every file but the reference solution, for the runner."""
from huggingface_hub import snapshot_download
meta = info(spec, token)
local = config.CACHE_DIR / _slug(spec)
prefix = f"{path}/" if path else ""
snapshot_download(spec, repo_type="dataset", revision=meta["sha"], local_dir=str(local),
token=(token if meta.get("restricted") else None) or False,
allow_patterns=[f"{prefix}*"], ignore_patterns=[f"{prefix}solution/*", f"{prefix}steps/*/solution/*"], max_workers=8)
return local / path if path else local
def _binary_name(name: str) -> bool:
return bool(re.search(r"\.(png|jpe?g|gif|webp|pdf|zip|tar|gz|tgz|bz2|xz|parquet|arrow|bin|pt|pkl|npy|npz|so|o|exe|"
r"whl|mp3|wav|mp4|db|sqlite|xlsx?|docx?|pptx?)$", name, re.I))
def task_file(spec: str, path: str, rel: str, token: str | None = None) -> dict[str, Any]:
"""One file from a task, for the file viewer: a text file inside the task's folder that holds no answer."""
spec = check_spec(spec)
row, idx = _task_row(spec, path, token)
files, texts, _ = _task_texts(spec, idx["sha"], path, token)
rel = posixpath.normpath(rel or "").lstrip("/")
if rel.startswith("..") or rel == ".":
return {"path": rel, "error": "not a file in this task"}
root_files, folders = _walked_root(spec, idx, path, token)
files = {**root_files, **files}
if rel not in files: # in a folder the index didn't list: look it up in that folder's listing
if "/" not in rel or rel.split("/")[0] not in folders:
return {"path": rel, "error": "not a file in this task"}
hit = next((e for e in task_folder(spec, path, posixpath.dirname(rel), token)["entries"] if e["path"] == rel and not e.get("dir")), None)
if hit is None:
return {"path": rel, "error": "not a file in this task"}
files = {**files, rel: hit["size"]}
if _withheld(rel):
return {"path": rel, "size": files[rel], "withheld": True}
if rel in texts:
if _withheld(rel, texts[rel]): # packs built before a rule was added are checked again as they're served
return {"path": rel, "size": files[rel], "withheld": True}
return {"path": rel, "size": files[rel], "text": texts[rel], "truncated": files[rel] > len(texts[rel].encode())}
if _binary_name(rel):
return {"path": rel, "size": files[rel], "binary": True}
text = _read(_fetch(spec, idx["sha"], path, rel, token))
if "\x00" in text[:8192]:
return {"path": rel, "size": files[rel], "binary": True}
if _withheld(rel, text):
return {"path": rel, "size": files[rel], "withheld": True}
if rel == "task.toml":
text = _masked_toml(text)
return {"path": rel, "size": files[rel], "text": _mask_answers(rel, text), "truncated": files[rel] > TEXT_LIMIT}