columnist / joint /prep_dataset.py
Quenlla's picture
Upload 13 files
cb179a8 verified
Raw History Blame Contribute Delete
6.3 kB
"""Turn charts.jsonl.gz into the arrays the model trains on, plus split flags and simple chart statistics.
python scripts/foundation/prep_dataset.py --tokens data/foundation/charts.jsonl.gz \\
--listing data/foundation/charts_4k.csv --bench dataset/om-mania-labels/charts.csv \\
--pairs data/foundation/round_pairs.csv --out data/foundation/store
Writes tok.npy (uint8 row token), gap.npy (ms to previous row), sv.npy (scroll multiplier), offsets.npy,
meta.csv (one line per chart: ids, split flags, statistics, star rating).
Pretraining uses only charts that are not copies, not in a benchmark song, not in a held-out tournament and
not in a held-out retrieval set.
"""
import argparse
import gzip
import hashlib
import json
import re
from pathlib import Path
import numpy as np
import pandas as pd
STAT_COLS = ["n_notes", "n_rows", "length_s", "ln_share", "nps_mean", "nps_peak", "chord_mean", "chord3_share",
"gap_logstd", "gap_logmean", "jack_share", "n_sv", "sv_std", "hold_len_mean"]
def h(s, mod):
return int(hashlib.md5(str(s).encode()).hexdigest(), 16) % mod
def song_key(artist, title):
return re.sub(r"[^a-z0-9 ]+", "", f"{artist} {title}".lower()).strip()
def chart_rows(notes, sv):
"""notes [(col, t, end|None)], sv [(t, mult)] -> tok, gap, svrow, stats."""
cols = np.array([n[0] for n in notes]); t = np.array([n[1] for n in notes], dtype=np.int64)
ends = np.array([-1 if n[2] is None else n[2] for n in notes], dtype=np.int64)
hold = ends >= 0
et = np.concatenate([t, ends[hold]]); ec = np.concatenate([cols, cols[hold]])
es = np.concatenate([np.where(hold, 2, 1), np.full(hold.sum(), 3)])
times, inv = np.unique(et, return_inverse=True)
states = np.zeros((len(times), 4), np.int64)
np.maximum.at(states, (inv, ec), es)
tok = (states * np.array([1, 4, 16, 64])).sum(1)
gap = np.diff(times, prepend=times[0])
if sv:
st = np.array([s[0] for s in sv]); sm = np.array([s[1] for s in sv], dtype=np.float32)
o = np.argsort(st, kind="stable"); st, sm = st[o], sm[o]
j = np.searchsorted(st, times, side="right") - 1
svrow = np.where(j >= 0, sm[np.clip(j, 0, None)], 1.0).astype(np.float32)
else:
svrow, sm = np.ones(len(times), np.float32), np.ones(1, np.float32)
length = max(1.0, (times[-1] - times[0]) / 1000)
taps = (states > 0).sum(1)
heads = (states == 1) | (states == 2)
pt = np.where(heads.any(1), 1, 0)
sec = np.bincount(((t - t[0]) // 1000).astype(int))
lg = np.log1p(gap[1:].clip(min=0)) if len(gap) > 1 else np.zeros(1)
prev_heads = np.vstack([np.zeros((1, 4), bool), heads[:-1]])
jack = ((heads & prev_heads) .any(1) & (gap < 200)).mean()
stats = [len(notes), len(times), length, hold.mean(), len(notes) / length, sec.max(),
(taps[pt == 1].mean() if pt.any() else 0), (taps >= 3).mean(), lg.std(), lg.mean(), jack,
len(sv), float(np.log(np.clip(sm, 0.1, 10)).std()), (ends[hold] - t[hold]).mean() if hold.any() else 0]
return tok.astype(np.uint8), gap.clip(0, 60000).astype(np.float16), svrow.astype(np.float16), stats
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--tokens", required=True); ap.add_argument("--listing", required=True)
ap.add_argument("--bench", required=True); ap.add_argument("--pairs", required=True)
ap.add_argument("--out", required=True); ap.add_argument("--max-charts", type=int, default=0)
a = ap.parse_args()
out = Path(a.out); out.mkdir(parents=True, exist_ok=True)
toks, gaps, svs, offs, meta, stats = [], [], [], [0], [], []
with gzip.open(a.tokens, "rt") as f:
for line in f:
c = json.loads(line)
tk, gp, sv, st = chart_rows(c["notes"], c["sv"])
if len(tk) < 32:
continue
toks.append(tk); gaps.append(gp); svs.append(sv); stats.append(st)
offs.append(offs[-1] + len(tk)); meta.append((c["beatmap_id"], c["fingerprint"], c["dup_of"]))
if a.max_charts and len(meta) >= a.max_charts:
break
np.save(out / "tok.npy", np.concatenate(toks)); np.save(out / "gap.npy", np.concatenate(gaps))
np.save(out / "sv.npy", np.concatenate(svs)); np.save(out / "offsets.npy", np.array(offs, np.int64))
m = pd.DataFrame(meta, columns=["beatmap_id", "fingerprint", "dup_of"])
m = pd.concat([m, pd.DataFrame(stats, columns=STAT_COLS)], axis=1)
lst = pd.read_csv(a.listing, usecols=["beatmap_id", "beatmapset_id", "artist", "title", "stars", "mapper_id"])
lst["song"] = [song_key(x, y) for x, y in zip(lst.artist, lst.title)]
m = m.merge(lst[["beatmap_id", "beatmapset_id", "song", "stars", "mapper_id"]], on="beatmap_id", how="left")
m["set_id"] = m.beatmapset_id.fillna(-m.beatmap_id).astype(int)
m["song"] = m.song.fillna("id" + m.beatmap_id.astype(str))
bench = pd.read_csv(a.bench)
bench = bench[bench["keys"] == 4][["beatmap_id", "fold"]].rename(columns={"fold": "bench_fold"})
m = m.merge(bench, on="beatmap_id", how="left")
bench_sets = set(m.loc[m.bench_fold.notna(), "set_id"]); bench_songs = set(m.loc[m.bench_fold.notna(), "song"])
m["in_bench_group"] = m.set_id.isin(bench_sets) | m.song.isin(bench_songs)
p = pd.read_csv(a.pairs)
held = {t for t in p.tournament.unique() if h(t, 5) == 0}
test_ids = set(p.loc[p.tournament.isin(held), ["easy_id", "hard_id"]].to_numpy().ravel())
m["tournament_test"] = m.beatmap_id.isin(test_ids)
sizes = m[m.dup_of.isna()].groupby("set_id").size()
multi = set(sizes[sizes >= 2].index)
m["retrieval_test"] = m.set_id.isin(multi) & (m.set_id.map(lambda s: h(s, 33)) == 0) & ~m.in_bench_group
m["pretrain_ok"] = (m.dup_of.isna() & ~m.in_bench_group & ~m.tournament_test & ~m.retrieval_test)
m.to_csv(out / "meta.csv", index=False)
print(len(m), "charts;", int(m.pretrain_ok.sum()), "for pretraining;", int(m.bench_fold.notna().sum()), "benchmark;",
int(m.tournament_test.sum()), "in held-out tournaments;", int(m.retrieval_test.sum()), "in retrieval test sets;",
f"{offs[-1]} rows")
if __name__ == "__main__":
main()