File size: 888 Bytes
4a393d1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 | from datasets import load_dataset
from huggingface_hub import hf_hub_download
import os
D="data"
for name, path, cfg in [
("smol_smoltalk","HuggingFaceTB/smol-smoltalk",None),
("everyday","HuggingFaceTB/smoltalk","everyday-conversations"),
("apigen","argilla/apigen-function-calling",None),
("hermes_fc","NousResearch/hermes-function-calling-v1","func_calling"),
("hermes_fc_single","NousResearch/hermes-function-calling-v1","func_calling_singleturn"),
("hermes_glaive","NousResearch/hermes-function-calling-v1","glaive_func_calling"),
]:
out=os.path.join(D,name)
if os.path.exists(out): print("skip",name); continue
ds=load_dataset(path,cfg) if cfg else load_dataset(path)
ds.save_to_disk(out); print("saved",name,ds)
hf_hub_download("nvidia/Nemotron-Agentic-v1","data/tool_calling.jsonl",repo_type="dataset",local_dir=os.path.join(D,"nemotron"))
|