File size: 6,295 Bytes
b3308b3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 | """Turn charts.jsonl.gz into the arrays the model trains on, plus split flags and simple chart statistics.
python scripts/foundation/prep_dataset.py --tokens data/foundation/charts.jsonl.gz \\
--listing data/foundation/charts_4k.csv --bench dataset/om-mania-labels/charts.csv \\
--pairs data/foundation/round_pairs.csv --out data/foundation/store
Writes tok.npy (uint8 row token), gap.npy (ms to previous row), sv.npy (scroll multiplier), offsets.npy,
meta.csv (one line per chart: ids, split flags, statistics, star rating).
Pretraining uses only charts that are not copies, not in a benchmark song, not in a held-out tournament and
not in a held-out retrieval set.
"""
import argparse
import gzip
import hashlib
import json
import re
from pathlib import Path
import numpy as np
import pandas as pd
STAT_COLS = ["n_notes", "n_rows", "length_s", "ln_share", "nps_mean", "nps_peak", "chord_mean", "chord3_share",
"gap_logstd", "gap_logmean", "jack_share", "n_sv", "sv_std", "hold_len_mean"]
def h(s, mod):
return int(hashlib.md5(str(s).encode()).hexdigest(), 16) % mod
def song_key(artist, title):
return re.sub(r"[^a-z0-9 ]+", "", f"{artist} {title}".lower()).strip()
def chart_rows(notes, sv):
"""notes [(col, t, end|None)], sv [(t, mult)] -> tok, gap, svrow, stats."""
cols = np.array([n[0] for n in notes]); t = np.array([n[1] for n in notes], dtype=np.int64)
ends = np.array([-1 if n[2] is None else n[2] for n in notes], dtype=np.int64)
hold = ends >= 0
et = np.concatenate([t, ends[hold]]); ec = np.concatenate([cols, cols[hold]])
es = np.concatenate([np.where(hold, 2, 1), np.full(hold.sum(), 3)])
times, inv = np.unique(et, return_inverse=True)
states = np.zeros((len(times), 4), np.int64)
np.maximum.at(states, (inv, ec), es)
tok = (states * np.array([1, 4, 16, 64])).sum(1)
gap = np.diff(times, prepend=times[0])
if sv:
st = np.array([s[0] for s in sv]); sm = np.array([s[1] for s in sv], dtype=np.float32)
o = np.argsort(st, kind="stable"); st, sm = st[o], sm[o]
j = np.searchsorted(st, times, side="right") - 1
svrow = np.where(j >= 0, sm[np.clip(j, 0, None)], 1.0).astype(np.float32)
else:
svrow, sm = np.ones(len(times), np.float32), np.ones(1, np.float32)
length = max(1.0, (times[-1] - times[0]) / 1000)
taps = (states > 0).sum(1)
heads = (states == 1) | (states == 2)
pt = np.where(heads.any(1), 1, 0)
sec = np.bincount(((t - t[0]) // 1000).astype(int))
lg = np.log1p(gap[1:].clip(min=0)) if len(gap) > 1 else np.zeros(1)
prev_heads = np.vstack([np.zeros((1, 4), bool), heads[:-1]])
jack = ((heads & prev_heads) .any(1) & (gap < 200)).mean()
stats = [len(notes), len(times), length, hold.mean(), len(notes) / length, sec.max(),
(taps[pt == 1].mean() if pt.any() else 0), (taps >= 3).mean(), lg.std(), lg.mean(), jack,
len(sv), float(np.log(np.clip(sm, 0.1, 10)).std()), (ends[hold] - t[hold]).mean() if hold.any() else 0]
return tok.astype(np.uint8), gap.clip(0, 60000).astype(np.float16), svrow.astype(np.float16), stats
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--tokens", required=True); ap.add_argument("--listing", required=True)
ap.add_argument("--bench", required=True); ap.add_argument("--pairs", required=True)
ap.add_argument("--out", required=True); ap.add_argument("--max-charts", type=int, default=0)
a = ap.parse_args()
out = Path(a.out); out.mkdir(parents=True, exist_ok=True)
toks, gaps, svs, offs, meta, stats = [], [], [], [0], [], []
with gzip.open(a.tokens, "rt") as f:
for line in f:
c = json.loads(line)
tk, gp, sv, st = chart_rows(c["notes"], c["sv"])
if len(tk) < 32:
continue
toks.append(tk); gaps.append(gp); svs.append(sv); stats.append(st)
offs.append(offs[-1] + len(tk)); meta.append((c["beatmap_id"], c["fingerprint"], c["dup_of"]))
if a.max_charts and len(meta) >= a.max_charts:
break
np.save(out / "tok.npy", np.concatenate(toks)); np.save(out / "gap.npy", np.concatenate(gaps))
np.save(out / "sv.npy", np.concatenate(svs)); np.save(out / "offsets.npy", np.array(offs, np.int64))
m = pd.DataFrame(meta, columns=["beatmap_id", "fingerprint", "dup_of"])
m = pd.concat([m, pd.DataFrame(stats, columns=STAT_COLS)], axis=1)
lst = pd.read_csv(a.listing, usecols=["beatmap_id", "beatmapset_id", "artist", "title", "stars", "mapper_id"])
lst["song"] = [song_key(x, y) for x, y in zip(lst.artist, lst.title)]
m = m.merge(lst[["beatmap_id", "beatmapset_id", "song", "stars", "mapper_id"]], on="beatmap_id", how="left")
m["set_id"] = m.beatmapset_id.fillna(-m.beatmap_id).astype(int)
m["song"] = m.song.fillna("id" + m.beatmap_id.astype(str))
bench = pd.read_csv(a.bench)
bench = bench[bench["keys"] == 4][["beatmap_id", "fold"]].rename(columns={"fold": "bench_fold"})
m = m.merge(bench, on="beatmap_id", how="left")
bench_sets = set(m.loc[m.bench_fold.notna(), "set_id"]); bench_songs = set(m.loc[m.bench_fold.notna(), "song"])
m["in_bench_group"] = m.set_id.isin(bench_sets) | m.song.isin(bench_songs)
p = pd.read_csv(a.pairs)
held = {t for t in p.tournament.unique() if h(t, 5) == 0}
test_ids = set(p.loc[p.tournament.isin(held), ["easy_id", "hard_id"]].to_numpy().ravel())
m["tournament_test"] = m.beatmap_id.isin(test_ids)
sizes = m[m.dup_of.isna()].groupby("set_id").size()
multi = set(sizes[sizes >= 2].index)
m["retrieval_test"] = m.set_id.isin(multi) & (m.set_id.map(lambda s: h(s, 33)) == 0) & ~m.in_bench_group
m["pretrain_ok"] = (m.dup_of.isna() & ~m.in_bench_group & ~m.tournament_test & ~m.retrieval_test)
m.to_csv(out / "meta.csv", index=False)
print(len(m), "charts;", int(m.pretrain_ok.sum()), "for pretraining;", int(m.bench_fold.notna().sum()), "benchmark;",
int(m.tournament_test.sum()), "in held-out tournaments;", int(m.retrieval_test.sum()), "in retrieval test sets;",
f"{offs[-1]} rows")
if __name__ == "__main__":
main()
|