| import json |
| import os |
| from datasets import load_dataset |
| from tqdm import tqdm |
|
|
| |
| |
| |
| |
| OUTPUT_DIR = "shell/playground/data/hf_data_0330" |
| |
| os.makedirs(OUTPUT_DIR, exist_ok=True) |
|
|
| |
| |
| |
| def process_direct_conversations(conversations): |
| """处理已经是 from/value 格式的 conversations""" |
| valid_convs = [] |
| for c in conversations: |
| val = c.get("value", "") |
| if val is not None and val != "": |
| valid_convs.append({ |
| "from": c.get("from"), |
| "value": val |
| }) |
| return valid_convs |
|
|
| |
| |
| |
| def process_and_save(dataset_iterator, output_filename, desc, convert_func): |
| """统一的数据遍历、转换与保存逻辑""" |
| output_path = os.path.join(OUTPUT_DIR, output_filename) |
| print(f"\n🚀 开始处理: {desc}") |
| |
| ix = 0 |
| with open(output_path, 'w', encoding='utf-8') as f: |
| for item in tqdm(dataset_iterator, desc=desc): |
| conversations = convert_func(item) |
| |
| |
| if not conversations: |
| continue |
| |
| conv = { |
| 'id': ix, |
| 'conversations': conversations |
| } |
| |
| |
| if 'source' in item: |
| conv['source'] = item['source'] |
| |
| f.write(json.dumps(conv, ensure_ascii=False) + '\n') |
| ix += 1 |
| |
| print(f"✅ [{desc}] 转换完成! 总计写入 {ix} 条，保存至 {output_path}") |
|
|
| |
| |
| |
| def main(): |
| |
| |
| process_and_save( |
| dataset_iterator=load_dataset("OpenDCAI/dataflow-instruct-10k", split="train"), |
| output_filename="dataflow-instruct-10k_train.jsonl", |
| desc="OpenDCAI/dataflow-instruct-10k", |
| convert_func=lambda x: process_direct_conversations(x['conversations']) |
| ) |
| |
| print("\n🎉 数据集下载与转换任务已全部完成！") |
|
|
| if __name__ == "__main__": |
| main() |
|
|
