File size: 5,295 Bytes
2bff5be | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 | """Prepare disjoint, progressively available shards of open English and Turkish web text."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
from huggingface_hub import HfFileSystem, hf_hub_download
from tokenizers import Tokenizer
PARENT = "db3e8a10bd3fcefde40acdc25ba7c6dc59cde32b"
ROOT = Path("work/third/data")
SOURCES = {
"en": (
"HuggingFaceFW/fineweb-edu",
"87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
"sample/10BT/000_00000.parquet",
),
"tr": (
"HuggingFaceFW/fineweb-2",
"af9c13333eb981300149d5ca60a8e9d659b276b9",
"data/tur_Latn/train/000_00000.parquet",
),
}
STORIES = {
"en": (
"roneneldan/TinyStories",
"f54c09fd23315a6f9c86f9dc80f725de7d8f9c64",
"data/train-00000-of-00004-2d5a1467fff1081b.parquet",
),
"tr": (
"esat-krky/TinyStories_Turkish",
"f039de0f75a000ccc62a6eff43e30eb5d8576c34",
"train/data-00000-of-00001.arrow",
),
}
def tokenizer():
path = os.environ.get("FRANOK_TOKENIZER") or hf_hub_download(
"yunusemrejr/Franok", "tokenizer.json", revision=PARENT
)
return Tokenizer.from_file(path)
def prepare(lang, limit, shard_tokens, stream, collection="web"):
root = ROOT / lang
root.mkdir(exist_ok=True, parents=True)
base_receipt = root / "receipt.json"
suffix = "" if collection == "web" else collection + "-"
receipt = base_receipt if not suffix else root / (collection + ".json")
skip = 0
if collection == "web-more":
skip = sum(json.loads(base_receipt.read_text())["docs"].values())
if receipt.exists() and json.loads(receipt.read_text())["target"] == limit:
return
repo, rev, fname = (STORIES if collection == "stories" else SOURCES)[lang]
tok = tokenizer()
counts = {"train": 0, "val": 0}
docs = {"train": 0, "val": 0}
buffers = {"train": [], "val": []}
shards = {"train": 0, "val": 0}
seen = set()
def flush(split):
if not buffers[split]:
return
file = root / f"{split}-{suffix}{shards[split]:05d}.bin"
tmp = file.with_suffix(".part")
array = np.asarray(buffers[split], dtype="<u2")
array.tofile(tmp)
tmp.replace(file)
counts[split] += len(array)
buffers[split].clear()
shards[split] += 1
if stream:
path = f"datasets/{repo}@{rev}/{fname}"
source = HfFileSystem().open(path, "rb", block_size=16 << 20)
else:
source = open(
hf_hub_download(repo, fname, repo_type="dataset", revision=rev), "rb"
)
with source:
columns = (
["text"] if collection == "stories" else ["text", "url", "language_score"]
)
batches = (
pa.ipc.open_stream(source)
if fname.endswith(".arrow")
else pq.ParquetFile(source).iter_batches(batch_size=128, columns=columns)
)
for batch in batches:
texts, splits = [], []
for row in batch.to_pylist():
text = row["text"].strip()
if len(text) < 180 or row.get("language_score", 1.0) < 0.75:
continue
digest = hashlib.blake2b(text.encode(), digest_size=12).digest()
if digest in seen:
continue
seen.add(digest)
if len(seen) <= skip:
continue
group = hashlib.blake2b(
(row.get("url") or digest.hex()).encode(), digest_size=8
).digest()
splits.append(
"val" if int.from_bytes(group, "little") % 200 == 0 else "train"
)
texts.append(text[:24000])
for split, encoded in zip(splits, tok.encode_batch(texts)):
buffers[split].extend([1, 5 if lang == "en" else 6, *encoded.ids, 2])
docs[split] += 1
if len(buffers[split]) >= (shard_tokens if split == "train" else 8192):
flush(split)
if counts["train"] + len(buffers["train"]) >= limit:
break
for split in buffers:
flush(split)
if not counts["val"] or counts["train"] < min(limit, 20000):
raise RuntimeError(f"Insufficient {lang} corpus: {counts}")
receipt.write_text(
json.dumps(
dict(
source=(repo, rev, fname),
skipped_unique_documents=skip,
docs=docs,
tokens=counts,
target=limit,
shards=shards,
),
indent=2,
)
+ "\n"
)
print(lang, docs, counts, flush=True)
if __name__ == "__main__":
p = argparse.ArgumentParser()
p.add_argument("language", choices=SOURCES)
p.add_argument("--tokens", type=int, default=180000000)
p.add_argument("--shard-tokens", type=int, default=1000000)
p.add_argument("--stream", action="store_true")
p.add_argument(
"--collection", choices=("web", "web-more", "stories"), default="web"
)
a = p.parse_args()
prepare(a.language, a.tokens, a.shard_tokens, a.stream, a.collection)
|