Download joint/prep_dataset.py from Quenlla/columnist: direct link, hf CLI and curl.
- Browser
- Download file 6.3 kB
-
https://huggingface.co/Quenlla/columnist/resolve/main/joint/prep_dataset.py
- Command line
-
hf download hf://Quenlla/columnist/joint/prep_dataset.py
-
curl -L -o prep_dataset.py https://huggingface.co/Quenlla/columnist/resolve/main/joint/prep_dataset.py
6.3 kB
| """Turn charts.jsonl.gz into the arrays the model trains on, plus split flags and simple chart statistics. | |
| python scripts/foundation/prep_dataset.py --tokens data/foundation/charts.jsonl.gz \\ | |
| --listing data/foundation/charts_4k.csv --bench dataset/om-mania-labels/charts.csv \\ | |
| --pairs data/foundation/round_pairs.csv --out data/foundation/store | |
| Writes tok.npy (uint8 row token), gap.npy (ms to previous row), sv.npy (scroll multiplier), offsets.npy, | |
| meta.csv (one line per chart: ids, split flags, statistics, star rating). | |
| Pretraining uses only charts that are not copies, not in a benchmark song, not in a held-out tournament and | |
| not in a held-out retrieval set. | |
| """ | |
| import argparse | |
| import gzip | |
| import hashlib | |
| import json | |
| import re | |
| from pathlib import Path | |
| import numpy as np | |
| import pandas as pd | |
| STAT_COLS = ["n_notes", "n_rows", "length_s", "ln_share", "nps_mean", "nps_peak", "chord_mean", "chord3_share", | |
| "gap_logstd", "gap_logmean", "jack_share", "n_sv", "sv_std", "hold_len_mean"] | |
| def h(s, mod): | |
| return int(hashlib.md5(str(s).encode()).hexdigest(), 16) % mod | |
| def song_key(artist, title): | |
| return re.sub(r"[^a-z0-9 ]+", "", f"{artist} {title}".lower()).strip() | |
| def chart_rows(notes, sv): | |
| """notes [(col, t, end|None)], sv [(t, mult)] -> tok, gap, svrow, stats.""" | |
| cols = np.array([n[0] for n in notes]); t = np.array([n[1] for n in notes], dtype=np.int64) | |
| ends = np.array([-1 if n[2] is None else n[2] for n in notes], dtype=np.int64) | |
| hold = ends >= 0 | |
| et = np.concatenate([t, ends[hold]]); ec = np.concatenate([cols, cols[hold]]) | |
| es = np.concatenate([np.where(hold, 2, 1), np.full(hold.sum(), 3)]) | |
| times, inv = np.unique(et, return_inverse=True) | |
| states = np.zeros((len(times), 4), np.int64) | |
| np.maximum.at(states, (inv, ec), es) | |
| tok = (states * np.array([1, 4, 16, 64])).sum(1) | |
| gap = np.diff(times, prepend=times[0]) | |
| if sv: | |
| st = np.array([s[0] for s in sv]); sm = np.array([s[1] for s in sv], dtype=np.float32) | |
| o = np.argsort(st, kind="stable"); st, sm = st[o], sm[o] | |
| j = np.searchsorted(st, times, side="right") - 1 | |
| svrow = np.where(j >= 0, sm[np.clip(j, 0, None)], 1.0).astype(np.float32) | |
| else: | |
| svrow, sm = np.ones(len(times), np.float32), np.ones(1, np.float32) | |
| length = max(1.0, (times[-1] - times[0]) / 1000) | |
| taps = (states > 0).sum(1) | |
| heads = (states == 1) | (states == 2) | |
| pt = np.where(heads.any(1), 1, 0) | |
| sec = np.bincount(((t - t[0]) // 1000).astype(int)) | |
| lg = np.log1p(gap[1:].clip(min=0)) if len(gap) > 1 else np.zeros(1) | |
| prev_heads = np.vstack([np.zeros((1, 4), bool), heads[:-1]]) | |
| jack = ((heads & prev_heads) .any(1) & (gap < 200)).mean() | |
| stats = [len(notes), len(times), length, hold.mean(), len(notes) / length, sec.max(), | |
| (taps[pt == 1].mean() if pt.any() else 0), (taps >= 3).mean(), lg.std(), lg.mean(), jack, | |
| len(sv), float(np.log(np.clip(sm, 0.1, 10)).std()), (ends[hold] - t[hold]).mean() if hold.any() else 0] | |
| return tok.astype(np.uint8), gap.clip(0, 60000).astype(np.float16), svrow.astype(np.float16), stats | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--tokens", required=True); ap.add_argument("--listing", required=True) | |
| ap.add_argument("--bench", required=True); ap.add_argument("--pairs", required=True) | |
| ap.add_argument("--out", required=True); ap.add_argument("--max-charts", type=int, default=0) | |
| a = ap.parse_args() | |
| out = Path(a.out); out.mkdir(parents=True, exist_ok=True) | |
| toks, gaps, svs, offs, meta, stats = [], [], [], [0], [], [] | |
| with gzip.open(a.tokens, "rt") as f: | |
| for line in f: | |
| c = json.loads(line) | |
| tk, gp, sv, st = chart_rows(c["notes"], c["sv"]) | |
| if len(tk) < 32: | |
| continue | |
| toks.append(tk); gaps.append(gp); svs.append(sv); stats.append(st) | |
| offs.append(offs[-1] + len(tk)); meta.append((c["beatmap_id"], c["fingerprint"], c["dup_of"])) | |
| if a.max_charts and len(meta) >= a.max_charts: | |
| break | |
| np.save(out / "tok.npy", np.concatenate(toks)); np.save(out / "gap.npy", np.concatenate(gaps)) | |
| np.save(out / "sv.npy", np.concatenate(svs)); np.save(out / "offsets.npy", np.array(offs, np.int64)) | |
| m = pd.DataFrame(meta, columns=["beatmap_id", "fingerprint", "dup_of"]) | |
| m = pd.concat([m, pd.DataFrame(stats, columns=STAT_COLS)], axis=1) | |
| lst = pd.read_csv(a.listing, usecols=["beatmap_id", "beatmapset_id", "artist", "title", "stars", "mapper_id"]) | |
| lst["song"] = [song_key(x, y) for x, y in zip(lst.artist, lst.title)] | |
| m = m.merge(lst[["beatmap_id", "beatmapset_id", "song", "stars", "mapper_id"]], on="beatmap_id", how="left") | |
| m["set_id"] = m.beatmapset_id.fillna(-m.beatmap_id).astype(int) | |
| m["song"] = m.song.fillna("id" + m.beatmap_id.astype(str)) | |
| bench = pd.read_csv(a.bench) | |
| bench = bench[bench["keys"] == 4][["beatmap_id", "fold"]].rename(columns={"fold": "bench_fold"}) | |
| m = m.merge(bench, on="beatmap_id", how="left") | |
| bench_sets = set(m.loc[m.bench_fold.notna(), "set_id"]); bench_songs = set(m.loc[m.bench_fold.notna(), "song"]) | |
| m["in_bench_group"] = m.set_id.isin(bench_sets) | m.song.isin(bench_songs) | |
| p = pd.read_csv(a.pairs) | |
| held = {t for t in p.tournament.unique() if h(t, 5) == 0} | |
| test_ids = set(p.loc[p.tournament.isin(held), ["easy_id", "hard_id"]].to_numpy().ravel()) | |
| m["tournament_test"] = m.beatmap_id.isin(test_ids) | |
| sizes = m[m.dup_of.isna()].groupby("set_id").size() | |
| multi = set(sizes[sizes >= 2].index) | |
| m["retrieval_test"] = m.set_id.isin(multi) & (m.set_id.map(lambda s: h(s, 33)) == 0) & ~m.in_bench_group | |
| m["pretrain_ok"] = (m.dup_of.isna() & ~m.in_bench_group & ~m.tournament_test & ~m.retrieval_test) | |
| m.to_csv(out / "meta.csv", index=False) | |
| print(len(m), "charts;", int(m.pretrain_ok.sum()), "for pretraining;", int(m.bench_fold.notna().sum()), "benchmark;", | |
| int(m.tournament_test.sum()), "in held-out tournaments;", int(m.retrieval_test.sum()), "in retrieval test sets;", | |
| f"{offs[-1]} rows") | |
| if __name__ == "__main__": | |
| main() | |