"""Turn charts.jsonl.gz into the arrays the model trains on, plus split flags and simple chart statistics. python scripts/foundation/prep_dataset.py --tokens data/foundation/charts.jsonl.gz \\ --listing data/foundation/charts_4k.csv --bench dataset/om-mania-labels/charts.csv \\ --pairs data/foundation/round_pairs.csv --out data/foundation/store Writes tok.npy (uint8 row token), gap.npy (ms to previous row), sv.npy (scroll multiplier), offsets.npy, meta.csv (one line per chart: ids, split flags, statistics, star rating). Pretraining uses only charts that are not copies, not in a benchmark song, not in a held-out tournament and not in a held-out retrieval set. """ import argparse import gzip import hashlib import json import re from pathlib import Path import numpy as np import pandas as pd STAT_COLS = ["n_notes", "n_rows", "length_s", "ln_share", "nps_mean", "nps_peak", "chord_mean", "chord3_share", "gap_logstd", "gap_logmean", "jack_share", "n_sv", "sv_std", "hold_len_mean"] def h(s, mod): return int(hashlib.md5(str(s).encode()).hexdigest(), 16) % mod def song_key(artist, title): return re.sub(r"[^a-z0-9 ]+", "", f"{artist} {title}".lower()).strip() def chart_rows(notes, sv): """notes [(col, t, end|None)], sv [(t, mult)] -> tok, gap, svrow, stats.""" cols = np.array([n[0] for n in notes]); t = np.array([n[1] for n in notes], dtype=np.int64) ends = np.array([-1 if n[2] is None else n[2] for n in notes], dtype=np.int64) hold = ends >= 0 et = np.concatenate([t, ends[hold]]); ec = np.concatenate([cols, cols[hold]]) es = np.concatenate([np.where(hold, 2, 1), np.full(hold.sum(), 3)]) times, inv = np.unique(et, return_inverse=True) states = np.zeros((len(times), 4), np.int64) np.maximum.at(states, (inv, ec), es) tok = (states * np.array([1, 4, 16, 64])).sum(1) gap = np.diff(times, prepend=times[0]) if sv: st = np.array([s[0] for s in sv]); sm = np.array([s[1] for s in sv], dtype=np.float32) o = np.argsort(st, kind="stable"); st, sm = st[o], sm[o] j = np.searchsorted(st, times, side="right") - 1 svrow = np.where(j >= 0, sm[np.clip(j, 0, None)], 1.0).astype(np.float32) else: svrow, sm = np.ones(len(times), np.float32), np.ones(1, np.float32) length = max(1.0, (times[-1] - times[0]) / 1000) taps = (states > 0).sum(1) heads = (states == 1) | (states == 2) pt = np.where(heads.any(1), 1, 0) sec = np.bincount(((t - t[0]) // 1000).astype(int)) lg = np.log1p(gap[1:].clip(min=0)) if len(gap) > 1 else np.zeros(1) prev_heads = np.vstack([np.zeros((1, 4), bool), heads[:-1]]) jack = ((heads & prev_heads) .any(1) & (gap < 200)).mean() stats = [len(notes), len(times), length, hold.mean(), len(notes) / length, sec.max(), (taps[pt == 1].mean() if pt.any() else 0), (taps >= 3).mean(), lg.std(), lg.mean(), jack, len(sv), float(np.log(np.clip(sm, 0.1, 10)).std()), (ends[hold] - t[hold]).mean() if hold.any() else 0] return tok.astype(np.uint8), gap.clip(0, 60000).astype(np.float16), svrow.astype(np.float16), stats def main(): ap = argparse.ArgumentParser() ap.add_argument("--tokens", required=True); ap.add_argument("--listing", required=True) ap.add_argument("--bench", required=True); ap.add_argument("--pairs", required=True) ap.add_argument("--out", required=True); ap.add_argument("--max-charts", type=int, default=0) a = ap.parse_args() out = Path(a.out); out.mkdir(parents=True, exist_ok=True) toks, gaps, svs, offs, meta, stats = [], [], [], [0], [], [] with gzip.open(a.tokens, "rt") as f: for line in f: c = json.loads(line) tk, gp, sv, st = chart_rows(c["notes"], c["sv"]) if len(tk) < 32: continue toks.append(tk); gaps.append(gp); svs.append(sv); stats.append(st) offs.append(offs[-1] + len(tk)); meta.append((c["beatmap_id"], c["fingerprint"], c["dup_of"])) if a.max_charts and len(meta) >= a.max_charts: break np.save(out / "tok.npy", np.concatenate(toks)); np.save(out / "gap.npy", np.concatenate(gaps)) np.save(out / "sv.npy", np.concatenate(svs)); np.save(out / "offsets.npy", np.array(offs, np.int64)) m = pd.DataFrame(meta, columns=["beatmap_id", "fingerprint", "dup_of"]) m = pd.concat([m, pd.DataFrame(stats, columns=STAT_COLS)], axis=1) lst = pd.read_csv(a.listing, usecols=["beatmap_id", "beatmapset_id", "artist", "title", "stars", "mapper_id"]) lst["song"] = [song_key(x, y) for x, y in zip(lst.artist, lst.title)] m = m.merge(lst[["beatmap_id", "beatmapset_id", "song", "stars", "mapper_id"]], on="beatmap_id", how="left") m["set_id"] = m.beatmapset_id.fillna(-m.beatmap_id).astype(int) m["song"] = m.song.fillna("id" + m.beatmap_id.astype(str)) bench = pd.read_csv(a.bench) bench = bench[bench["keys"] == 4][["beatmap_id", "fold"]].rename(columns={"fold": "bench_fold"}) m = m.merge(bench, on="beatmap_id", how="left") bench_sets = set(m.loc[m.bench_fold.notna(), "set_id"]); bench_songs = set(m.loc[m.bench_fold.notna(), "song"]) m["in_bench_group"] = m.set_id.isin(bench_sets) | m.song.isin(bench_songs) p = pd.read_csv(a.pairs) held = {t for t in p.tournament.unique() if h(t, 5) == 0} test_ids = set(p.loc[p.tournament.isin(held), ["easy_id", "hard_id"]].to_numpy().ravel()) m["tournament_test"] = m.beatmap_id.isin(test_ids) sizes = m[m.dup_of.isna()].groupby("set_id").size() multi = set(sizes[sizes >= 2].index) m["retrieval_test"] = m.set_id.isin(multi) & (m.set_id.map(lambda s: h(s, 33)) == 0) & ~m.in_bench_group m["pretrain_ok"] = (m.dup_of.isna() & ~m.in_bench_group & ~m.tournament_test & ~m.retrieval_test) m.to_csv(out / "meta.csv", index=False) print(len(m), "charts;", int(m.pretrain_ok.sum()), "for pretraining;", int(m.bench_fold.notna().sum()), "benchmark;", int(m.tournament_test.sum()), "in held-out tournaments;", int(m.retrieval_test.sum()), "in retrieval test sets;", f"{offs[-1]} rows") if __name__ == "__main__": main()