File size: 836 Bytes
c617d00 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 | import os
import json
from datasets import load_dataset
def load_data():
os.makedirs('data/raw', exist_ok=True)
out_path = 'data/raw/samanantar_kn_50k.json'
print("Downloading ai4bharat/samanantar kn subset...")
ds = load_dataset('ai4bharat/samanantar', 'kn', split='train', streaming=True)
samples = []
print("Fetching 50k samples...")
for i, item in enumerate(ds):
if i >= 50000:
break
en = item.get('src', item.get('en', item.get('english', '')))
kn = item.get('tgt', item.get('kn', item.get('kannada', '')))
samples.append({'en': en, 'kn': kn})
with open(out_path, 'w', encoding='utf-8') as f:
json.dump(samples, f, ensure_ascii=False, indent=2)
print(f"Saved 50k samples to {out_path}")
if __name__ == '__main__':
load_data()
|