Buckets:
| """Fetch pinned public model/kernel and dataset assets into explicit locations.""" | |
| import argparse | |
| import json | |
| import os | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parents[1] | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--model-path", type=Path, required=True) | |
| parser.add_argument("--download-model", action="store_true") | |
| args = parser.parse_args() | |
| os.environ["HF_HOME"] = str(ROOT / ".cache/huggingface") | |
| os.environ["OPENPROBLEMS_CACHE_DIR"] = str(ROOT / ".cache/openproblems") | |
| os.environ.pop("HF_HUB_OFFLINE", None) | |
| os.environ.pop("TRANSFORMERS_OFFLINE", None) | |
| from huggingface_hub import snapshot_download | |
| if args.download_model: | |
| snapshot_download("openai/gpt-oss-120b", revision="b5c939de8f754692c1647ca79fbf85e8c1e70f8a", local_dir=args.model_path) | |
| if not (args.model_path / "model.safetensors.index.json").is_file(): | |
| raise RuntimeError("Supply a complete model directory, or use --download-model") | |
| from kernels import get_kernel | |
| get_kernel("kernels-community/vllm-flash-attn3", revision="9a3cd57fdb1a2c01ef72f1ef48ceca4991134ce6") | |
| import openproblems.data | |
| openproblems.data.no_cleanup() | |
| from openproblems.data.pancreas import load_pancreas | |
| from openproblems.tasks.denoising.datasets.utils import split_data | |
| data = split_data(load_pancreas(test=False, keep_techs=["inDrop1"]), seed=42) | |
| result = dict(dataset="pancreas/inDrop1", seed=42, shape=list(data.shape), | |
| model_path=str(args.model_path.resolve()), kernel_revision="9a3cd57fdb1a2c01ef72f1ef48ceca4991134ce6") | |
| (ROOT / ".cache/data_ready.json").write_text(json.dumps(result, indent=2) + "\n") | |
| print(json.dumps(result, indent=2)) | |
Xet Storage Details
- Size:
- 1.64 kB
- Xet hash:
- 1e1e1794f3fdbeb769b2a7740f265b7da2e4f507911de1391ad138e971efa0d3
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.