Aryan Sethi
Claude Opus 5 (1M context)
Drop barrel, add held-out stop-sign distance, add the GPU runbook
5d449ff | """Publish the unified dataset to the Hugging Face Hub. | |
| The local dataset is symlinks into four different dataset roots, which is right | |
| for building it and useless for moving it. This packs the real bytes into tar | |
| shards, adds the metadata phase 2 and phase 3 need, and uploads the lot as a | |
| dataset repo, so a fresh Colab runtime is one download away from training | |
| instead of one 30 GB nuScenes pull away. | |
| What lands on the Hub: | |
| manifest.parquet the source of truth -- previewable in the Hub viewer | |
| class_priors.yaml per-class physical size, for the known-size estimator | |
| calib/<sensor_id>.yaml K, camera height, ego<-cam rotation | |
| data.yaml portable (path: "."); pull.py rewrites it | |
| shards/{train,val}_NNN.tar | |
| README.md dataset card, generated from the manifest | |
| Images and labels are sharded rather than uploaded as individual files because | |
| the Hub handles a few large files far better than 40,000 small ones, and a | |
| ~1 GB shard is a resumable unit if the upload drops. | |
| Usage: | |
| huggingface-cli login # or pass --token | |
| python -m src.hub.push --repo-id <user>/cone-distance-v1 | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import os | |
| import shutil | |
| import tarfile | |
| import tempfile | |
| from pathlib import Path | |
| import pandas as pd | |
| import yaml | |
| from src.common import paths, schema | |
| DEFAULT_SHARD_BYTES = 1_000_000_000 # ~1 GB | |
| def pair_files(yolo_root: Path, split: str) -> list[tuple[Path, Path]]: | |
| """(image, label) pairs for one split, in a stable order.""" | |
| image_dir = yolo_root / split / "images" | |
| label_dir = yolo_root / split / "labels" | |
| pairs = [] | |
| for image_path in sorted(image_dir.iterdir()): | |
| label_path = label_dir / f"{image_path.stem}.txt" | |
| if not label_path.exists(): | |
| raise SystemExit(f"{image_path} has no label file; re-run src.data.to_yolo") | |
| pairs.append((image_path, label_path)) | |
| return pairs | |
| def write_shards(yolo_root: Path, staging: Path, split: str, | |
| shard_bytes: int) -> int: | |
| """Pack one split into tar shards under staging/shards/. | |
| Members keep the local layout (`train/images/x.jpg`, `train/labels/x.txt`) | |
| so extracting every shard into one directory reproduces it exactly. | |
| Symlinks are dereferenced -- the tar holds real bytes. | |
| """ | |
| shard_dir = staging / "shards" | |
| shard_dir.mkdir(parents=True, exist_ok=True) | |
| pairs = pair_files(yolo_root, split) | |
| index, written, archive = 0, 0, None | |
| for image_path, label_path in pairs: | |
| if archive is None: | |
| archive = tarfile.open(shard_dir / f"{split}_{index:03d}.tar", "w") | |
| written = 0 | |
| # resolve() follows the symlink to the real image in the source dataset. | |
| archive.add(image_path.resolve(), arcname=f"{split}/images/{image_path.name}") | |
| archive.add(label_path, arcname=f"{split}/labels/{label_path.name}") | |
| written += image_path.resolve().stat().st_size | |
| if written >= shard_bytes: | |
| archive.close() | |
| archive, index = None, index + 1 | |
| if archive is not None: | |
| archive.close() | |
| index += 1 | |
| print(f" {split}: {len(pairs)} images in {index} shard(s)") | |
| return index | |
| def write_data_yaml(staging: Path) -> None: | |
| """A portable data.yaml. pull.py rewrites `path` to wherever it lands.""" | |
| with open(staging / "data.yaml", "w") as handle: | |
| yaml.safe_dump( | |
| { | |
| "path": ".", | |
| "train": "train/images", | |
| "val": "val/images", | |
| "names": {index: name for index, name in enumerate(schema.CLASSES)}, | |
| }, | |
| handle, | |
| sort_keys=False, | |
| ) | |
| def write_dataset_card(manifest: pd.DataFrame, staging: Path, repo_id: str) -> None: | |
| """Generate the card from the manifest, so it cannot drift from the data.""" | |
| objects = schema.objects_only(manifest) | |
| counts = pd.crosstab(objects["source"], objects["class"]) | |
| distances = ( | |
| objects[objects["gt_distance_m"].notna()] | |
| .groupby("class")["gt_distance_m"] | |
| .describe(percentiles=[0.5])[["count", "min", "50%", "max"]] | |
| ) | |
| card = f"""--- | |
| license: other | |
| task_categories: | |
| - object-detection | |
| tags: | |
| - autonomous-driving | |
| - monocular-depth | |
| - yolo | |
| --- | |
| # {repo_id} | |
| Cones, barriers and stop signs from nuScenes, Argoverse 2, COCO and | |
| BDD100K, merged into one schema with camera intrinsics and ground-truth distance | |
| attached. Built for monocular geometric distance estimation: distance comes from | |
| focal length and pixel geometry at inference time, and the LiDAR-derived | |
| `gt_distance_m` here is scoring ground truth only. | |
| ## Contents | |
| | File | What it is | | |
| | --- | --- | | |
| | `manifest.parquet` | one row per object; the source of truth | | |
| | `calib/<sensor_id>.yaml` | `K`, `cam_height_m`, `R_ego_from_cam` per camera | | |
| | `class_priors.yaml` | per-class physical size, derived from the 3D cuboids | | |
| | `data.yaml` | Ultralytics dataset config (`path` is relative; rewrite on download) | | |
| | `shards/*.tar` | images and YOLO labels, `{{split}}/images/` and `{{split}}/labels/` | | |
| ```python | |
| from huggingface_hub import snapshot_download | |
| snapshot_download("{repo_id}", repo_type="dataset", local_dir="data/hub") | |
| # then extract every shards/*.tar into data/hub/ and point Ultralytics at data.yaml | |
| ``` | |
| ## Instances per source and class | |
| ``` | |
| {counts.to_string()} | |
| ``` | |
| ## Ground-truth distance | |
| `gt_distance_m` is **forward depth along the camera optical axis** — the Z | |
| component of the object centre in the camera frame — not Euclidean range. | |
| Null for COCO and BDD100K, which have no depth ground truth. | |
| ``` | |
| {distances.to_string()} | |
| ``` | |
| ## Frames per split | |
| ``` | |
| {manifest.groupby("split")["image_path"].nunique().to_string()} | |
| ``` | |
| Split by **scene, not by frame**: adjacent video frames are near-duplicates, so a | |
| frame-level split leaks the val set into training and inflates val mAP a long | |
| way. Scenes are split independently within each source so every source reaches | |
| val. | |
| ## Notes | |
| - Argoverse 2's `CONSTRUCTION_BARREL` is not kept. An orange drum is a | |
| different object from a jersey barrier and is not an evaluation class. | |
| - Boxes shorter than 15 px, nuScenes visibility below 0.4, and boxes more than | |
| 50% truncated at the frame edge are filtered out. | |
| - BDD100K contributes negative frames only, excluding any frame containing a | |
| traffic sign — its generic sign class hides real stop signs. | |
| - Source datasets keep their own licences (nuScenes and Argoverse 2 are | |
| non-commercial research licences; COCO is CC BY 4.0). This repo redistributes | |
| derived crops and labels under those terms. | |
| """ | |
| (staging / "README.md").write_text(card) | |
| def push(root: Path, repo_id: str, private: bool, token: str | None, | |
| shard_bytes: int, staging_dir: Path | None) -> None: | |
| from huggingface_hub import HfApi | |
| manifest_path = root / "manifest.parquet" | |
| yolo_root = root / "yolo" | |
| if not yolo_root.exists(): | |
| raise SystemExit(f"{yolo_root} does not exist; run src.data.to_yolo first") | |
| manifest = schema.read_manifest(manifest_path) | |
| staging = Path(staging_dir) if staging_dir else Path(tempfile.mkdtemp(prefix="hub_")) | |
| staging.mkdir(parents=True, exist_ok=True) | |
| print(f"staging in {staging}") | |
| try: | |
| print("packing shards") | |
| for split in ("train", "val"): | |
| write_shards(yolo_root, staging, split, shard_bytes) | |
| shutil.copy(manifest_path, staging / "manifest.parquet") | |
| priors = root / "class_priors.yaml" | |
| if priors.exists(): | |
| shutil.copy(priors, staging / "class_priors.yaml") | |
| if (root / "calib").exists(): | |
| shutil.copytree(root / "calib", staging / "calib", dirs_exist_ok=True) | |
| write_data_yaml(staging) | |
| write_dataset_card(manifest, staging, repo_id) | |
| total = sum(p.stat().st_size for p in staging.rglob("*") if p.is_file()) | |
| print(f"\nuploading {total / 1e9:.2f} GB to {repo_id}") | |
| api = HfApi(token=token) | |
| api.create_repo(repo_id, repo_type="dataset", private=private, exist_ok=True) | |
| api.upload_folder(repo_id=repo_id, repo_type="dataset", folder_path=str(staging)) | |
| print(f"\nhttps://huggingface.co/datasets/{repo_id}") | |
| finally: | |
| if staging_dir is None: | |
| shutil.rmtree(staging, ignore_errors=True) | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__, | |
| formatter_class=argparse.RawDescriptionHelpFormatter) | |
| parser.add_argument("--repo-id", required=True, help="e.g. <user>/cone-distance-v1") | |
| parser.add_argument("--unified-root", type=Path, default=None) | |
| parser.add_argument("--private", action="store_true") | |
| parser.add_argument("--token", default=os.environ.get("HF_TOKEN"), | |
| help="defaults to $HF_TOKEN, then the cached " | |
| "huggingface-cli login. Prefer the environment: a " | |
| "token passed on the command line is visible to " | |
| "anyone who can run ps.") | |
| parser.add_argument("--shard-bytes", type=int, default=DEFAULT_SHARD_BYTES) | |
| parser.add_argument("--staging-dir", type=Path, default=None, | |
| help="keep the staging directory here instead of a temp dir") | |
| args = parser.parse_args() | |
| push( | |
| root=paths.unified_root(args.unified_root), | |
| repo_id=args.repo_id, | |
| private=args.private, | |
| token=args.token, | |
| shard_bytes=args.shard_bytes, | |
| staging_dir=args.staging_dir, | |
| ) | |
| if __name__ == "__main__": | |
| main() | |