columnist / columnist.py
Quenlla's picture
Upload 10 files
b3308b3 verified
Raw History Blame Contribute Delete
13.7 kB
"""Columnist: a small sequence model for 4K osu!mania charts (data helpers + network).
A chart is a sequence of ROWS (notes that fall at the same millisecond). Each row has a token for the state of
the four columns (none / tap / hold head / hold tail -> 4**4 = 256 tokens), the gap in ms to the previous
row, and the visible scroll speed. No BPM, snap or grid information is used anywhere.
Network: row embedding + numeric features -> causal conv stem -> transformer layers with local attention
(window of rows) and a per-head time penalty (rows far apart in seconds attend less). The same network can run
bidirectionally (masked-row and contrastive objectives) or causally (next-row objective, per-note use).
"""
from __future__ import annotations
import math
import os
import numpy as np
import torch
import torch.nn.functional as F
from torch import nn
N_TOK = 256
MASK_ID = 256
N_GAPB = 24
WINDOW = 64 # rows to each side
L_CROP = 512
def gap_bucket(gap_ms: torch.Tensor) -> torch.Tensor:
return (torch.log2(gap_ms.float() + 1) * 2).floor().clamp(0, N_GAPB - 1).long()
def mirror_lut() -> torch.Tensor:
"""token -> token with the four columns reversed. The table covers tokens with the optional held bits (bits 16..19, one per
column, set by Store when extras are on) and MASK_ID, so lut[tok] works for every kind of token."""
v = np.arange(1 << 20, dtype=np.int64)
base, held = v & 0xFFFF, v >> 16
mb = np.where(base < N_TOK, sum((((base >> (2 * c)) & 3) << (2 * (3 - c))) for c in range(4)), base)
mh = sum((((held >> c) & 1) << (3 - c)) for c in range(4))
return torch.from_numpy((mb | (mh << 16)).astype(np.int64))
N_LOOK = 6
VIS_H = (400.0, 800.0, 1600.0) # scroll distance (ms of travel at multiplier 1) that counts as "on screen"
def look_features(g_ms, sv, valid, causal):
"""Extra inputs (B,L,6): time to the next row, its log ratio to the previous gap, scroll distance since the previous
row, and how many following rows are within VIS_H of scroll travel (a proxy for what is on screen). Everything is
computed from row times and the scroll multiplier; no snap, beat or BPM-grid information. The columns that look ahead
(0, 1, 3, 4, 5) are zeroed when causal=True so the next-row objective cannot see the answer. g_ms must already be
zero for masked rows (so a masked row's gap never leaks through its neighbours)."""
B, L = g_ms.shape
nxt = torch.cat([g_ms[:, 1:], torch.zeros_like(g_ms[:, :1])], 1) * torch.cat([valid[:, 1:], torch.zeros_like(valid[:, :1])], 1)
f_next = torch.log1p(nxt) / 6.9
ratio = (torch.log1p(nxt) - torch.log1p(g_ms)).clamp(-6, 6) / 3
svc = torch.nan_to_num(sv.float(), nan=1.0, posinf=10.0, neginf=0.1).clamp(0.1, 10.0)
dx = g_ms * svc * valid
xv = dx.cumsum(1)
idx = torch.arange(L, device=g_ms.device)
x = torch.where(valid, xv, xv.max(1, keepdim=True).values + 1e6 * (1 + idx)[None])
nval = valid.sum(1, keepdim=True)
cols = [f_next, ratio, torch.log1p(dx) / 9]
vis = []
for H in VIS_H:
j = torch.searchsorted(x.contiguous(), (x + H).contiguous(), right=True).minimum(nval)
vis.append(torch.log1p(((j - idx[None] - 1).clamp(min=0) * valid).float()) / 4)
out = torch.stack(cols + vis, -1)
if causal:
keep = torch.tensor([0, 0, 1, 0, 0, 0], dtype=out.dtype, device=out.device)
out = out * keep
return out
N_V3 = 7
V3_H = 600.0 # ms of scroll travel that counts as "on screen" (as osu_patterns.deep.tokens.H_MS)
V3_W = 64 # rows looked back when measuring the speed swing and the number of speed changes on screen
def v3_features(g_ms, sv, valid, held):
"""The inputs of the old v3 network that Columnist lacks (no snap, beat or BPM-grid information): the explicit per-column
'held' state (a hold passes through this row; computed from the whole chart by make_held.py, bits 16..19 of the token),
how long the row has been on screen, and the speed swing and the number of speed changes among the rows on screen.
g_ms is zero for masked rows. Returns (B,L,7)."""
B, L = g_ms.shape
svc = torch.nan_to_num(sv.float(), nan=1.0, posinf=10.0, neginf=0.1).clamp(0.1, 10.0)
lsp = torch.log(svc)
x = (g_ms * svc * valid).cumsum(1)
t = (g_ms * valid).cumsum(1)
ar = torch.arange(L, device=g_ms.device)
j0 = torch.searchsorted(x.contiguous(), (x - V3_H).contiguous(), right=False)
j0 = torch.minimum(j0, ar[None].expand(B, L))
vis = torch.log(((t - t.gather(1, j0)) / V3_H).clamp(0.05, 20.0))
k = torch.arange(V3_W, device=g_ms.device)
idx = ar[:, None] - k[None, :] # (L,W) rows i-k
ok = (idx >= 0)[None] & (idx[None] >= j0[:, :, None]) # rows still on screen
ls = lsp[:, idx.clamp(min=0)] # (B,L,W)
mx = ls.masked_fill(~ok, -1e4).max(-1).values
mn = ls.masked_fill(~ok, 1e4).min(-1).values
swing = (mx - mn).clamp(0, 6) / 2
chg = (((ls[..., :-1] - ls[..., 1:]).abs() > 0.02) & ok[..., 1:]).sum(-1).float()
held_f = torch.stack([((held >> c) & 1).float() for c in range(4)], -1)
return torch.cat([held_f, torch.stack([vis, swing, torch.log1p(chg) / 3], -1)], -1)
def row_features(tok, gap, sv, valid, look=False, causal=False, v3=False):
"""tok (B,L) long (MASK_ID allowed), gap (B,L) ms, sv (B,L) scroll multiplier -> (B,L,14 or 14+N_LOOK) floats, times (B,L) s."""
held_bits = (tok >> 16) & 15
tok = tok & 0xFFFF
masked = tok == MASK_ID
t = tok.clamp(max=N_TOK - 1)
states = torch.stack([(t >> (2 * c)) & 3 for c in range(4)], -1) # (B,L,4)
states = torch.where(masked[..., None], torch.zeros_like(states), states)
delta = (states == 2).long() - (states == 3).long()
held_before = (delta.cumsum(1) - delta).clamp(min=0).float().clamp(max=1)
gap = torch.nan_to_num(gap.float(), nan=0.0, posinf=60000.0)
g = torch.log1p(gap.clamp(min=0)) / 6.9
g = torch.where(masked, torch.zeros_like(g), g)
four = [f(g * (2.0 ** k) * math.pi) for k in range(4) for f in (torch.sin, torch.cos)]
svf = torch.log(torch.nan_to_num(sv.float(), nan=1.0, posinf=10.0, neginf=0.1).clamp(0.1, 10.0))
feats = torch.cat([g[..., None], torch.stack(four, -1), svf[..., None], held_before], -1)
if look:
gm = torch.where(masked, torch.zeros_like(gap), gap.clamp(min=0))
feats = torch.cat([feats, look_features(gm, sv, valid, causal)], -1)
if v3:
gm3 = torch.where(masked, torch.zeros_like(gap), gap.clamp(min=0))
feats = torch.cat([feats, v3_features(gm3, sv, valid, held_bits)], -1)
time = (gap.float().clamp(min=0) * (~masked) * valid).cumsum(1) / 1000.0
return feats * valid[..., None], time
class Block(nn.Module):
def __init__(self, d, heads, p):
super().__init__()
self.h = heads
self.n1, self.n2 = nn.LayerNorm(d), nn.LayerNorm(d)
self.qkv = nn.Linear(d, 3 * d)
self.o = nn.Linear(d, d)
self.ff = nn.Sequential(nn.Linear(d, 4 * d), nn.GELU(), nn.Linear(4 * d, d))
self.drop = nn.Dropout(p)
def forward(self, x, bias):
B, L, D = x.shape
q, k, v = self.qkv(self.n1(x)).view(B, L, 3, self.h, D // self.h).permute(2, 0, 3, 1, 4)
a = F.scaled_dot_product_attention(q, k, v, attn_mask=bias.to(q.dtype))
x = x + self.drop(self.o(a.transpose(1, 2).reshape(B, L, D)))
return x + self.drop(self.ff(self.n2(x)))
class Columnist(nn.Module):
def __init__(self, d=128, layers=4, heads=4, p=0.1, look=False, v3=False):
super().__init__()
self.d, self.heads, self.look, self.v3 = d, heads, look, v3
self.tok = nn.Embedding(N_TOK + 1, d)
self.num = nn.Linear(14 + (N_LOOK if look else 0) + (N_V3 if v3 else 0), d)
self.stem = nn.ModuleList([nn.Conv1d(d, 2 * d, 5) for _ in range(2)]) # left-padded: causal
self.blocks = nn.ModuleList(Block(d, heads, p) for _ in range(layers))
self.norm = nn.LayerNorm(d)
self.register_buffer("slopes", torch.tensor([0.05 * 2 ** (i * 5 / max(1, heads - 1)) for i in range(heads)]))
self.tok_head = nn.Linear(d, N_TOK)
self.gap_head = nn.Linear(d, N_GAPB)
self.proj = nn.Sequential(nn.Linear(d, d), nn.GELU(), nn.Linear(d, 64))
self.rank_head = nn.Linear(d, 1) # difficulty-order objectives (ladder, rounds)
self.order_head = nn.Linear(2 * d, 1) # which of two windows comes first
self.stat_head = nn.Linear(d, 4) # window statistics
self.rate_head = nn.Linear(2 * d, 1) # log rate ratio between two views of one window
def attn_bias(self, time, valid, causal):
B, L = valid.shape
idx = torch.arange(L, device=valid.device)
rel = idx[None, :] - idx[:, None] # j - i
allowed = rel.abs() <= WINDOW
if causal:
allowed = allowed & (rel <= 0)
allowed = allowed[None] & valid[:, None, :]
allowed = allowed | torch.eye(L, dtype=torch.bool, device=valid.device)[None]
dt = (time[:, None, :] - time[:, :, None]).abs() # (B,L,L) seconds
bias = -self.slopes[None, :, None, None] * dt[:, None]
return bias.masked_fill(~allowed[:, None], float("-inf"))
def forward(self, tok, gap, sv, valid, causal=False):
feats, time = row_features(tok, gap, sv, valid, self.look, causal, self.v3)
x = self.tok(tok & 0xFFFF) + self.num(feats)
h = x.transpose(1, 2)
for conv in self.stem:
h = h + F.glu(conv(F.pad(h, (4, 0))), dim=1)
x = h.transpose(1, 2)
bias = self.attn_bias(time, valid, causal)
for blk in self.blocks:
x = blk(x, bias)
return self.norm(x) * valid[..., None]
@staticmethod
def pool(h, valid):
return h.sum(1) / valid.sum(1, keepdim=True).clamp(min=1)
class Store:
"""Memory-mapped chart store written by prep_dataset.py."""
def __init__(self, d):
self.tok = np.load(f"{d}/tok.npy", mmap_mode="r")
self.gap = np.load(f"{d}/gap.npy", mmap_mode="r")
self.sv = np.load(f"{d}/sv.npy", mmap_mode="r")
self.off = np.load(f"{d}/offsets.npy")
self.held = np.load(f"{d}/held.npy", mmap_mode="r") if os.path.exists(f"{d}/held.npy") else None # per-column held bits (make_held.py)
self.extras = False # when True, crop() adds the held bits as bits 16..19 of every token
def n(self, i):
return int(self.off[i + 1] - self.off[i])
def crop(self, i, start, L=L_CROP):
a = int(self.off[i]) + start
b = min(a + L, int(self.off[i + 1]))
n = b - a
tok = np.zeros(L, np.int64); gap = np.zeros(L, np.float32); sv = np.ones(L, np.float32)
tok[:n], gap[:n], sv[:n] = self.tok[a:b], self.gap[a:b], self.sv[a:b]
if self.extras and self.held is not None:
tok[:n] |= self.held[a:b].astype(np.int64) << 16
valid = np.zeros(L, bool); valid[:n] = True
return tok, gap, sv, valid
def random_crop(self, i, rng, L=L_CROP):
return self.crop(i, int(rng.integers(0, max(1, self.n(i) - L + 1))), L)
@torch.no_grad()
def embed_charts(model, store, ids, device, L=L_CROP, bs=64):
"""Chart embedding = mean of window embeddings over non-overlapping windows of L rows. Returns (len(ids), d)."""
model.eval()
store.extras = bool(getattr(model, "v3", False))
wins = [(k, s) for k, i in enumerate(ids) for s in range(0, max(1, store.n(i)), L)]
sums = torch.zeros(len(ids), model.d); cnt = torch.zeros(len(ids))
for b in range(0, len(wins), bs):
part = wins[b:b + bs]
arr = [store.crop(ids[k], s, L) for k, s in part]
tok, gap, sv, valid = (torch.from_numpy(np.stack(x)).to(device) for x in zip(*arr))
with torch.autocast(device_type=device.type, enabled=device.type == "cuda"):
h = model(tok, gap, sv, valid)
e = Columnist.pool(h.float(), valid).cpu()
w = valid.sum(1).cpu().float()
for (k, _), ek, wk in zip(part, e, w):
sums[k] += ek * wk; cnt[k] += wk
return (sums / cnt.clamp(min=1)[:, None]).numpy()
def window_stats(tok, gap, valid):
"""Free regression targets from a window: log rows/s, hold share, mean chord size, mean log gap. (B,4)"""
t = (tok & 0xFFFF).clamp(max=N_TOK - 1)
states = torch.stack([(t >> (2 * c)) & 3 for c in range(4)], -1)
v = valid.float()
n = v.sum(1).clamp(min=1)
secs = (torch.nan_to_num(gap.float()) * v).sum(1).clamp(min=1) / 1000
chord = ((states > 0).sum(-1).float() * v).sum(1) / n
hold = (((states == 2) | (states == 3)).any(-1).float() * v).sum(1) / n
lg = (torch.log1p(torch.nan_to_num(gap.float()).clamp(min=0)) * v).sum(1) / n
return torch.stack([torch.log(n / secs), hold, chord, lg], 1)
def diff_level(name: str) -> int:
"""Mapper's own difficulty name -> coarse level (1 easy .. 5 expert); 0 if unknown."""
import re
n = re.sub(r"[^a-z ]+", " ", str(name).lower())
for words, lvl in ((("beginner", "easy", "novice"), 1), (("normal", "basic"), 2), (("hard", "advanced"), 3),
(("insane", "another"), 4), (("expert", "extra", "extreme", "ex", "master", "lunatic"), 5)):
if any(w in n.split() for w in words):
return lvl
return 0