File size: 5,518 Bytes
6e802a4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | # jsonl_to_parquet.py: Lossless corpus JSONL <-> parquet (to_row() rebuilds the exact training row).
import argparse
import hashlib
import json
import os
import sys
from multiprocessing import Pool
STR_COLS = ["prompt", "reasoning", "emotion_label", "pace", "pitch", "cot_form", "source", "speaker", "id", "file",
"caption_source"]
CHUNK = 20000
def _schema():
import pyarrow as pa
return pa.schema([(c, pa.string()) for c in STR_COLS] + [
("speaker_id", pa.int32()),
("speech_tokens", pa.list_(pa.int16())),
("residual_codes", pa.list_(pa.list_(pa.int16()))),
("plan", pa.string()),
("meta", pa.string()),
])
def _is_int(x):
return isinstance(x, int) and not isinstance(x, bool)
def from_row(r):
out = {c: None for c in STR_COLS + ["speaker_id", "speech_tokens", "residual_codes", "plan", "meta"]}
meta = {}
for k, v in r.items():
if k in STR_COLS and isinstance(v, str):
out[k] = v
elif k == "speaker_id" and _is_int(v) and -2 ** 31 <= v < 2 ** 31:
out[k] = v
elif k == "speech_tokens" and isinstance(v, list) and all(_is_int(x) and -32768 <= x < 32768 for x in v):
out[k] = v
elif (k == "residual_codes" and isinstance(v, list) and all(isinstance(lv, list) for lv in v)
and all(_is_int(x) and -32768 <= x < 32768 for lv in v for x in lv)):
out[k] = v
elif k == "plan" and isinstance(v, dict):
out[k] = json.dumps(v, sort_keys=True, separators=(",", ":"))
else:
meta[k] = v
out["meta"] = json.dumps(meta, sort_keys=True, separators=(",", ":")) if meta else None
return out
def to_row(c):
r = json.loads(c["meta"]) if c.get("meta") else {}
for k in STR_COLS + ["speaker_id", "speech_tokens", "residual_codes"]:
if c.get(k) is not None:
r[k] = c[k]
if c.get("plan") is not None:
r["plan"] = json.loads(c["plan"])
return r
def row_hash(r):
return hashlib.sha1(json.dumps(r, sort_keys=True, separators=(",", ":")).encode()).digest()
def _shard(task):
import pyarrow as pa
import pyarrow.parquet as pq
src, offset, n_rows, path = task
schema = _schema()
hashes, buf = [], []
tmp = path + ".tmp"
w = pq.ParquetWriter(tmp, schema, compression="zstd")
def flush():
cols = {name: [] for name in schema.names}
for r in buf:
for k, v in from_row(r).items():
cols[k].append(v)
w.write_table(pa.table(cols, schema=schema))
buf.clear()
with open(src, "rb") as f:
f.seek(offset)
for _ in range(n_rows):
line = f.readline()
if not line:
raise SystemExit(f"{src}: unexpected EOF in shard {path}")
r = json.loads(line)
hashes.append(row_hash(r))
buf.append(r)
if len(buf) >= CHUNK:
flush()
if buf:
flush()
w.close()
i = 0
for b in pq.ParquetFile(tmp).iter_batches(batch_size=CHUNK):
for c in b.to_pylist():
if i >= len(hashes) or row_hash(to_row(c)) != hashes[i]:
raise SystemExit(f"{path}: row {i} does not round-trip")
i += 1
if i != len(hashes):
raise SystemExit(f"{path}: {i} rows read back, {len(hashes)} written")
os.replace(tmp, path)
h = hashlib.sha256()
with open(path, "rb") as f:
for blk in iter(lambda: f.read(1 << 24), b""):
h.update(blk)
return {"file": os.path.basename(path), "rows": n_rows, "bytes": os.path.getsize(path), "sha256": h.hexdigest()}
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--src", required=True)
ap.add_argument("--out", required=True)
ap.add_argument("--split", default="train")
ap.add_argument("--shard_rows", type=int, default=100000)
ap.add_argument("--workers", type=int, default=8)
args = ap.parse_args()
os.makedirs(args.out, exist_ok=True)
offsets, n = [], 0
with open(args.src, "rb") as f:
pos = 0
for line in f:
if n % args.shard_rows == 0:
offsets.append(pos)
pos += len(line)
n += 1
k = len(offsets)
tasks = [(args.src, off, min(args.shard_rows, n - i * args.shard_rows),
os.path.join(args.out, f"{args.split}-{i:05d}-of-{k:05d}.parquet")) for i, off in enumerate(offsets)]
print(f"[parquet] {args.src}: {n} rows -> {k} shards of <= {args.shard_rows}", flush=True)
with Pool(args.workers) as pool:
res = []
for r in pool.imap(_shard, tasks):
print(f"[parquet] ok {r['file']}: {r['rows']} rows, {r['bytes'] / 1e9:.2f} GB, round-trip verified",
flush=True)
res.append(r)
assert sum(r["rows"] for r in res) == n
man = {"source": os.path.basename(args.src), "rows": n, "shards": res,
"source_bytes": os.path.getsize(args.src), "parquet_bytes": sum(r["bytes"] for r in res),
"lossless_round_trip_verified": True}
with open(os.path.join(args.out, f"{args.split}_manifest.json"), "w") as f:
json.dump(man, f, indent=1)
print(f"[parquet] DONE {args.src}: {n} rows, {man['source_bytes'] / 1e9:.2f} GB JSONL -> "
f"{man['parquet_bytes'] / 1e9:.2f} GB parquet", flush=True)
if __name__ == "__main__":
sys.exit(main())
|