File size: 2,079 Bytes
547c0e3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | """Download and tokenise the Bengali news benchmark data used in the PursuitLoRA / TensorPEFT experiments.
python prepare_bengali_data.py --out data # writes data/train.bin, data/val.bin
Source: Bengali news articles (Anandabazar, Ebela, Zee News) from
github.com/soham96/Bengali_news_classifier (pinned commit). Articles are shuffled with a fixed seed;
the first 400 articles form the validation split. Tokens are GPT-2 BPE ids stored as uint16.
"""
import argparse
import io
import json
import os
import random
import re
import unicodedata
import urllib.request
import zipfile
import numpy as np
from transformers import AutoTokenizer
URL = ("https://raw.githubusercontent.com/soham96/Bengali_news_classifier/"
"4c2a51b51fc56208426ef19368bfcb08acc9d0c4/data/data.zip")
p = argparse.ArgumentParser()
p.add_argument("--out", default="data")
p.add_argument("--tokenizer", default="openai-community/gpt2")
p.add_argument("--zip", default=None, help="use a local copy of data.zip instead of downloading")
a = p.parse_args()
raw = open(a.zip, "rb").read() if a.zip else urllib.request.urlopen(URL).read()
docs = []
with zipfile.ZipFile(io.BytesIO(raw)) as z:
for name in sorted(n for n in z.namelist() if n.endswith("_articles.txt") and "__MACOSX" not in n):
for art in json.loads(z.read(name).decode("utf-8"))["articles"]:
t = unicodedata.normalize("NFC", (art.get("title", "") + "। " + art.get("body", "")).strip())
t = re.sub(r"\s+", " ", t)
if len(t) > 200:
docs.append(t)
random.Random(0).shuffle(docs)
tok = AutoTokenizer.from_pretrained(a.tokenizer)
def enc(ds, cap):
ids = []
for d in ds:
ids += tok(d).input_ids + [tok.eos_token_id]
if len(ids) >= cap:
break
return np.array(ids[:cap], dtype=np.uint16)
os.makedirs(a.out, exist_ok=True)
enc(docs[:400], 200_000).tofile(os.path.join(a.out, "val.bin"))
enc(docs[400:], 3_000_000).tofile(os.path.join(a.out, "train.bin"))
print(f"{len(docs)} articles -> {a.out}/train.bin, {a.out}/val.bin")
|