zefancainlp's picture
download
raw
1.64 kB
"""Fetch pinned public model/kernel and dataset assets into explicit locations."""
import argparse
import json
import os
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
parser = argparse.ArgumentParser()
parser.add_argument("--model-path", type=Path, required=True)
parser.add_argument("--download-model", action="store_true")
args = parser.parse_args()
os.environ["HF_HOME"] = str(ROOT / ".cache/huggingface")
os.environ["OPENPROBLEMS_CACHE_DIR"] = str(ROOT / ".cache/openproblems")
os.environ.pop("HF_HUB_OFFLINE", None)
os.environ.pop("TRANSFORMERS_OFFLINE", None)
from huggingface_hub import snapshot_download
if args.download_model:
snapshot_download("openai/gpt-oss-120b", revision="b5c939de8f754692c1647ca79fbf85e8c1e70f8a", local_dir=args.model_path)
if not (args.model_path / "model.safetensors.index.json").is_file():
raise RuntimeError("Supply a complete model directory, or use --download-model")
from kernels import get_kernel
get_kernel("kernels-community/vllm-flash-attn3", revision="9a3cd57fdb1a2c01ef72f1ef48ceca4991134ce6")
import openproblems.data
openproblems.data.no_cleanup()
from openproblems.data.pancreas import load_pancreas
from openproblems.tasks.denoising.datasets.utils import split_data
data = split_data(load_pancreas(test=False, keep_techs=["inDrop1"]), seed=42)
result = dict(dataset="pancreas/inDrop1", seed=42, shape=list(data.shape),
model_path=str(args.model_path.resolve()), kernel_revision="9a3cd57fdb1a2c01ef72f1ef48ceca4991134ce6")
(ROOT / ".cache/data_ready.json").write_text(json.dumps(result, indent=2) + "\n")
print(json.dumps(result, indent=2))

Xet Storage Details

Size:
1.64 kB
·
Xet hash:
1e1e1794f3fdbeb769b2a7740f265b7da2e4f507911de1391ad138e971efa0d3

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.