File size: 6,295 Bytes
b3308b3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
"""Turn charts.jsonl.gz into the arrays the model trains on, plus split flags and simple chart statistics.



    python scripts/foundation/prep_dataset.py --tokens data/foundation/charts.jsonl.gz \\

        --listing data/foundation/charts_4k.csv --bench dataset/om-mania-labels/charts.csv \\

        --pairs data/foundation/round_pairs.csv --out data/foundation/store



Writes tok.npy (uint8 row token), gap.npy (ms to previous row), sv.npy (scroll multiplier), offsets.npy,

meta.csv (one line per chart: ids, split flags, statistics, star rating).

Pretraining uses only charts that are not copies, not in a benchmark song, not in a held-out tournament and

not in a held-out retrieval set.

"""
import argparse
import gzip
import hashlib
import json
import re
from pathlib import Path

import numpy as np
import pandas as pd

STAT_COLS = ["n_notes", "n_rows", "length_s", "ln_share", "nps_mean", "nps_peak", "chord_mean", "chord3_share",
             "gap_logstd", "gap_logmean", "jack_share", "n_sv", "sv_std", "hold_len_mean"]


def h(s, mod):
    return int(hashlib.md5(str(s).encode()).hexdigest(), 16) % mod


def song_key(artist, title):
    return re.sub(r"[^a-z0-9 ]+", "", f"{artist} {title}".lower()).strip()


def chart_rows(notes, sv):
    """notes [(col, t, end|None)], sv [(t, mult)] -> tok, gap, svrow, stats."""
    cols = np.array([n[0] for n in notes]); t = np.array([n[1] for n in notes], dtype=np.int64)
    ends = np.array([-1 if n[2] is None else n[2] for n in notes], dtype=np.int64)
    hold = ends >= 0
    et = np.concatenate([t, ends[hold]]); ec = np.concatenate([cols, cols[hold]])
    es = np.concatenate([np.where(hold, 2, 1), np.full(hold.sum(), 3)])
    times, inv = np.unique(et, return_inverse=True)
    states = np.zeros((len(times), 4), np.int64)
    np.maximum.at(states, (inv, ec), es)
    tok = (states * np.array([1, 4, 16, 64])).sum(1)
    gap = np.diff(times, prepend=times[0])
    if sv:
        st = np.array([s[0] for s in sv]); sm = np.array([s[1] for s in sv], dtype=np.float32)
        o = np.argsort(st, kind="stable"); st, sm = st[o], sm[o]
        j = np.searchsorted(st, times, side="right") - 1
        svrow = np.where(j >= 0, sm[np.clip(j, 0, None)], 1.0).astype(np.float32)
    else:
        svrow, sm = np.ones(len(times), np.float32), np.ones(1, np.float32)
    length = max(1.0, (times[-1] - times[0]) / 1000)
    taps = (states > 0).sum(1)
    heads = (states == 1) | (states == 2)
    pt = np.where(heads.any(1), 1, 0)
    sec = np.bincount(((t - t[0]) // 1000).astype(int))
    lg = np.log1p(gap[1:].clip(min=0)) if len(gap) > 1 else np.zeros(1)
    prev_heads = np.vstack([np.zeros((1, 4), bool), heads[:-1]])
    jack = ((heads & prev_heads) .any(1) & (gap < 200)).mean()
    stats = [len(notes), len(times), length, hold.mean(), len(notes) / length, sec.max(),
             (taps[pt == 1].mean() if pt.any() else 0), (taps >= 3).mean(), lg.std(), lg.mean(), jack,
             len(sv), float(np.log(np.clip(sm, 0.1, 10)).std()), (ends[hold] - t[hold]).mean() if hold.any() else 0]
    return tok.astype(np.uint8), gap.clip(0, 60000).astype(np.float16), svrow.astype(np.float16), stats


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--tokens", required=True); ap.add_argument("--listing", required=True)
    ap.add_argument("--bench", required=True); ap.add_argument("--pairs", required=True)
    ap.add_argument("--out", required=True); ap.add_argument("--max-charts", type=int, default=0)
    a = ap.parse_args()
    out = Path(a.out); out.mkdir(parents=True, exist_ok=True)
    toks, gaps, svs, offs, meta, stats = [], [], [], [0], [], []
    with gzip.open(a.tokens, "rt") as f:
        for line in f:
            c = json.loads(line)
            tk, gp, sv, st = chart_rows(c["notes"], c["sv"])
            if len(tk) < 32:
                continue
            toks.append(tk); gaps.append(gp); svs.append(sv); stats.append(st)
            offs.append(offs[-1] + len(tk)); meta.append((c["beatmap_id"], c["fingerprint"], c["dup_of"]))
            if a.max_charts and len(meta) >= a.max_charts:
                break
    np.save(out / "tok.npy", np.concatenate(toks)); np.save(out / "gap.npy", np.concatenate(gaps))
    np.save(out / "sv.npy", np.concatenate(svs)); np.save(out / "offsets.npy", np.array(offs, np.int64))
    m = pd.DataFrame(meta, columns=["beatmap_id", "fingerprint", "dup_of"])
    m = pd.concat([m, pd.DataFrame(stats, columns=STAT_COLS)], axis=1)

    lst = pd.read_csv(a.listing, usecols=["beatmap_id", "beatmapset_id", "artist", "title", "stars", "mapper_id"])
    lst["song"] = [song_key(x, y) for x, y in zip(lst.artist, lst.title)]
    m = m.merge(lst[["beatmap_id", "beatmapset_id", "song", "stars", "mapper_id"]], on="beatmap_id", how="left")
    m["set_id"] = m.beatmapset_id.fillna(-m.beatmap_id).astype(int)
    m["song"] = m.song.fillna("id" + m.beatmap_id.astype(str))
    bench = pd.read_csv(a.bench)
    bench = bench[bench["keys"] == 4][["beatmap_id", "fold"]].rename(columns={"fold": "bench_fold"})
    m = m.merge(bench, on="beatmap_id", how="left")
    bench_sets = set(m.loc[m.bench_fold.notna(), "set_id"]); bench_songs = set(m.loc[m.bench_fold.notna(), "song"])
    m["in_bench_group"] = m.set_id.isin(bench_sets) | m.song.isin(bench_songs)
    p = pd.read_csv(a.pairs)
    held = {t for t in p.tournament.unique() if h(t, 5) == 0}
    test_ids = set(p.loc[p.tournament.isin(held), ["easy_id", "hard_id"]].to_numpy().ravel())
    m["tournament_test"] = m.beatmap_id.isin(test_ids)
    sizes = m[m.dup_of.isna()].groupby("set_id").size()
    multi = set(sizes[sizes >= 2].index)
    m["retrieval_test"] = m.set_id.isin(multi) & (m.set_id.map(lambda s: h(s, 33)) == 0) & ~m.in_bench_group
    m["pretrain_ok"] = (m.dup_of.isna() & ~m.in_bench_group & ~m.tournament_test & ~m.retrieval_test)
    m.to_csv(out / "meta.csv", index=False)
    print(len(m), "charts;", int(m.pretrain_ok.sum()), "for pretraining;", int(m.bench_fold.notna().sum()), "benchmark;",
          int(m.tournament_test.sum()), "in held-out tournaments;", int(m.retrieval_test.sum()), "in retrieval test sets;",
          f"{offs[-1]} rows")


if __name__ == "__main__":
    main()