Download scrawler/text_model.py from levossadtchi/Scrawler-open: direct link, hf CLI and curl.
- Browser
- Download file 11.1 kB
-
https://huggingface.co/levossadtchi/Scrawler-open/resolve/main/scrawler/text_model.py
- Command line
-
hf download hf://levossadtchi/Scrawler-open/scrawler/text_model.py
-
curl -L -o text_model.py https://huggingface.co/levossadtchi/Scrawler-open/resolve/main/scrawler/text_model.py
11.1 kB
| """The text model: a diffusion denoiser over the pen points of a whole line, and its duration planner. | |
| The planner says how many pen points each character gets. The denoiser then turns noise into points, knowing for every point | |
| which character it belongs to and how far through that character it is. It is not autoregressive: all points are denoised at once. | |
| """ | |
| import math | |
| import numpy as np | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from .common import AMP, CKPT_PLANNER, CKPT_TEXT, DEVICE, load_checkpoint | |
| MAX_POINTS = 1200 | |
| NOISE_STEPS = 100 | |
| SAMPLE_STEPS = 20 | |
| GUIDANCE = 1.5 | |
| PHASES = 8 | |
| HIDDEN = 384 | |
| GRU_HIDDEN = 512 | |
| BLOCKS_BEFORE = 6 | |
| BLOCKS_AFTER = 4 | |
| ADVANCE = 0.017 # mean x advance per point; the net predicts the residual | |
| SCALE_X = 0.6 | |
| SCALE_Y = 0.25 | |
| def alpha_bar(steps=NOISE_STEPS, s=0.008): | |
| t = np.arange(1, steps + 1) / steps | |
| f = np.cos((t + s) / (1 + s) * math.pi / 2) ** 2 / math.cos(s / (1 + s) * math.pi / 2) ** 2 | |
| return np.clip(f, 1e-4, 0.9999).astype(np.float32) | |
| ALPHA_BAR = torch.tensor(alpha_bar(), device=DEVICE) | |
| def time_embedding(t, dim=128): | |
| half = dim // 2 | |
| freqs = torch.exp(-math.log(10000) * torch.arange(half, device=t.device) / half) | |
| a = t.float()[:, None] * freqs[None] * (1000.0 / NOISE_STEPS) | |
| return torch.cat([a.sin(), a.cos()], -1) | |
| class FiLM(nn.Module): | |
| def __init__(self, cond_dim, dim): | |
| super().__init__() | |
| self.l = nn.Linear(cond_dim, 2 * dim) | |
| nn.init.zeros_(self.l.weight) | |
| nn.init.zeros_(self.l.bias) | |
| def forward(self, x, g): | |
| a, b = self.l(g)[:, None].chunk(2, -1) | |
| return x * (1 + a) + b | |
| class ConvBlock(nn.Module): | |
| """A ConvNeXt-style 1D block.""" | |
| def __init__(self, dim, cond_dim, dilation): | |
| super().__init__() | |
| self.dw = nn.Conv1d(dim, dim, 7, padding=3 * dilation, dilation=dilation, groups=dim) | |
| self.ln = nn.LayerNorm(dim) | |
| self.film = FiLM(cond_dim, dim) | |
| self.f1 = nn.Linear(dim, 4 * dim) | |
| self.f2 = nn.Linear(4 * dim, dim) | |
| self.gam = nn.Parameter(torch.full((dim,), 0.1)) | |
| def forward(self, x, mask, g): | |
| h = self.dw((x * mask).transpose(1, 2)).transpose(1, 2) | |
| h = self.film(self.ln(h), g) | |
| h = self.f2(F.gelu(self.f1(h))) | |
| return (x + self.gam * h) * mask | |
| class Denoiser(nn.Module): | |
| def __init__(self, vocab_size): | |
| super().__init__() | |
| D, G = HIDDEN, 256 | |
| self.ec = nn.Embedding(vocab_size + 1, D) # current, previous and next character; the last row = "dropped" for guidance | |
| self.ep = nn.Embedding(vocab_size + 1, D) | |
| self.en = nn.Embedding(vocab_size + 1, D) | |
| self.eph = nn.Embedding(PHASES, D) | |
| self.inp = nn.Linear(3 + 3 + 3 + 2, D) | |
| self.tm = nn.Sequential(nn.Linear(128, G), nn.SiLU(), nn.Linear(G, G)) | |
| self.sp = nn.Linear(256, G) | |
| self.null_style = nn.Parameter(torch.zeros(256)) | |
| self.semb_proj = nn.Linear(256, 256) | |
| self.sin = nn.Linear(256, D) | |
| dilations = [1, 2, 4, 8, 16, 1, 2, 4, 8, 16] | |
| self.pre = nn.ModuleList([ConvBlock(D, G, dilations[k]) for k in range(BLOCKS_BEFORE)]) | |
| self.gln = nn.LayerNorm(D) | |
| self.gfilm = FiLM(G, D) | |
| self.gru = nn.GRU(D, GRU_HIDDEN, num_layers=2, batch_first=True, bidirectional=True) | |
| self.gout = nn.Linear(2 * GRU_HIDDEN, D) | |
| self.post = nn.ModuleList([ConvBlock(D, G, dilations[k]) for k in range(BLOCKS_AFTER)]) | |
| self.oln = nn.LayerNorm(D) | |
| self.head = nn.Linear(D, 3) | |
| def forward(self, z, t, cond, style, self_cond): | |
| mask_ = cond["mask"] | |
| mask = mask_[..., None] | |
| length = z.size(1) | |
| pos = torch.arange(length, device=z.device, dtype=torch.float32)[None, :, None].expand(z.size(0), -1, -1) / 600.0 | |
| phase = cond["progress"][..., None] | |
| feats = torch.cat( | |
| [z, self_cond, phase, torch.sin(2 * math.pi * phase), torch.cos(2 * math.pi * phase), torch.log1p(cond["duration"])[..., None] / 3.0, pos], | |
| -1, | |
| ) | |
| x = (self.inp(feats) + self.ec(cond["char"]) + self.ep(cond["prev"]) + self.en(cond["next"]) + self.eph(cond["phase"]) + self.sin(style)[:, None]) * mask | |
| g = F.silu(self.tm(time_embedding(t)) + self.sp(style)) | |
| for block in self.pre: | |
| x = block(x, mask, g) | |
| h = self.gfilm(self.gln(x), g) | |
| lengths = mask_.sum(1).long().clamp(min=1) | |
| with torch.autocast("cuda", enabled=False): | |
| packed = nn.utils.rnn.pack_padded_sequence(h.float(), lengths.cpu(), batch_first=True, enforce_sorted=False) | |
| out, _ = self.gru(packed) | |
| out, _ = nn.utils.rnn.pad_packed_sequence(out, batch_first=True, total_length=length) | |
| x = (x + self.gout(out.to(x.dtype))) * mask | |
| for block in self.post: | |
| x = block(x, mask, g) | |
| out = self.head(self.oln(x)).float() | |
| return out[..., :2] * mask, out[..., 2] | |
| class Planner(nn.Module): | |
| """Characters + style -> log1p of the number of pen points of each character.""" | |
| MAX_TOKENS = 200 | |
| def __init__(self, vocab_size, d=256, layers=4): | |
| super().__init__() | |
| self.emb = nn.Embedding(vocab_size, d) | |
| self.pos = nn.Embedding(self.MAX_TOKENS + 2, d) | |
| self.sty = nn.Linear(256, d) | |
| self.null = nn.Parameter(torch.zeros(256)) | |
| self.enc = nn.TransformerEncoder(nn.TransformerEncoderLayer(d, 4, 4 * d, 0.1, batch_first=True, norm_first=True), layers) | |
| self.out = nn.Sequential(nn.LayerNorm(d), nn.Linear(d, 1)) | |
| def forward(self, tokens, token_mask, style): | |
| h = self.emb(tokens) + self.pos(torch.arange(tokens.shape[1], device=tokens.device))[None] + self.sty(style)[:, None] | |
| return self.out(self.enc(h, src_key_padding_mask=token_mask < 0.5))[..., 0] | |
| def _plan(char_ids, durations): | |
| """Per-character plan -> per-point conditioning.""" | |
| char, prev, next_, phase, progress, duration = [], [], [], [], [], [] | |
| for k, (c, d) in enumerate(zip(char_ids, durations)): | |
| d = max(1, int(d)) | |
| before = char_ids[k - 1] if k > 0 else 0 | |
| after = char_ids[k + 1] if k + 1 < len(char_ids) else 0 | |
| for q in range(d): | |
| char.append(c) | |
| prev.append(before) | |
| next_.append(after) | |
| phase.append(min(PHASES - 1, (q * PHASES) // d)) | |
| progress.append(q / d) | |
| duration.append(d) | |
| return char, prev, next_, phase, progress, duration | |
| class TextModel: | |
| def __init__(self): | |
| state = load_checkpoint(CKPT_TEXT) | |
| self.itos = state["itos"] | |
| self.stoi = {s: i for i, s in enumerate(self.itos)} | |
| self.null_token = len(self.itos) | |
| self.net = Denoiser(len(self.itos)).to(DEVICE) | |
| self.net.load_state_dict({k: v.float() for k, v in state["e"].items()}) | |
| self.net.eval() | |
| state = load_checkpoint(CKPT_PLANNER) | |
| self.planner = Planner(len(state["itos"])).to(DEVICE) | |
| self.planner.load_state_dict(state["m"]) | |
| self.planner.eval() | |
| def style(self, semb): | |
| """The writer's style vector -> the vector the denoiser takes.""" | |
| return self.net.semb_proj(semb) | |
| def durations(self, lines, semb): | |
| """lines: lists of characters; semb [B, 256]. -> points per character for each line.""" | |
| B = len(lines) | |
| U = min(Planner.MAX_TOKENS + 2, -(-(max(len(t) for t in lines) + 2) // 16) * 16) | |
| tokens = torch.zeros(B, U, dtype=torch.long) | |
| token_mask = torch.zeros(B, U) | |
| for b, line in enumerate(lines): | |
| ids = [1] + [self.stoi.get(c, 0) for c in line] + [2] # <bos> ... <eos> | |
| tokens[b, : len(ids)] = torch.tensor(ids) | |
| token_mask[b, : len(ids)] = 1 | |
| out = self.planner(tokens.to(DEVICE), token_mask.to(DEVICE), semb.to(DEVICE).float()) | |
| d = torch.expm1(out).clamp(min=1).round().long().cpu().numpy() | |
| return [list(d[b, 1 : 1 + len(t)]) for b, t in enumerate(lines)] | |
| def sample(self, lines, durations, style): | |
| """-> for each line an array [n, 3]: dx, dy, pen_up. Uses the global torch random generator.""" | |
| B = len(lines) | |
| plans = [_plan([self.stoi.get(c, 0) for c in t], d) for t, d in zip(lines, durations)] | |
| length = min(MAX_POINTS, max(len(p[0]) for p in plans)) | |
| lengths = [min(length, len(p[0])) for p in plans] | |
| cond = {k: torch.zeros(B, length, dtype=torch.long, device=DEVICE) for k in ("char", "prev", "next", "phase")} | |
| cond.update( | |
| progress=torch.zeros(B, length, device=DEVICE), | |
| duration=torch.zeros(B, length, device=DEVICE), | |
| mask=torch.zeros(B, length, device=DEVICE), | |
| ) | |
| for b, plan in enumerate(plans): | |
| n = lengths[b] | |
| for key, values in zip(("char", "prev", "next", "phase", "progress", "duration"), plan): | |
| cond[key][b, :n] = torch.tensor(values[:n], dtype=cond[key].dtype) | |
| cond["mask"][b, :n] = 1 | |
| style = style.expand(B, -1).clone() | |
| # classifier-free guidance: the same batch twice, the second time with the characters dropped | |
| dropped = { | |
| k: (torch.where(cond["mask"] > 0, torch.full_like(v, self.null_token), v) if k in ("char", "prev", "next") else v) | |
| for k, v in cond.items() | |
| } | |
| cond2 = {k: torch.cat([cond[k], dropped[k]], 0) for k in cond} | |
| style2 = torch.cat([style, style], 0) | |
| mask = cond["mask"][..., None] | |
| z = torch.randn(B, length, 3, device=DEVICE) * mask | |
| self_cond = torch.zeros_like(z) | |
| steps = np.linspace(NOISE_STEPS - 1, 0, SAMPLE_STEPS).round().astype(int) | |
| for k, t in enumerate(steps): | |
| tt = torch.full((B,), int(t), device=DEVICE, dtype=torch.long) | |
| with torch.autocast("cuda", dtype=torch.bfloat16, enabled=AMP): | |
| xy, pen = self.net(torch.cat([z, z], 0), torch.cat([tt, tt], 0), cond2, style2, torch.cat([self_cond, self_cond], 0)) | |
| xy = xy[B:] + GUIDANCE * (xy[:B] - xy[B:]) | |
| pen = pen[B:] + GUIDANCE * (pen[:B] - pen[B:]) | |
| x0 = torch.cat([xy.float(), torch.tanh(pen.float() / 2)[..., None]], -1) * mask | |
| self_cond = x0 | |
| if k == len(steps) - 1: | |
| break | |
| a, a_next = ALPHA_BAR[t], ALPHA_BAR[steps[k + 1]] | |
| eps = (z - a.sqrt() * x0) / (1 - a).sqrt() | |
| z = (a_next.sqrt() * x0 + (1 - a_next).sqrt() * eps) * mask # DDIM | |
| index = torch.arange(length, device=DEVICE, dtype=torch.float32)[None] | |
| points = torch.stack([x0[..., 0] * SCALE_X + ADVANCE * index, x0[..., 1] * SCALE_Y], -1).cpu().numpy() | |
| pen_up = (pen.float() > 0.0).float().cpu().numpy() | |
| result = [] | |
| for b in range(B): | |
| n = lengths[b] | |
| d = np.zeros((n, 3), np.float32) | |
| d[1:, :2] = np.diff(points[b, :n], axis=0) | |
| d[:, 2] = pen_up[b, :n] | |
| d[-1, 2] = 1 | |
| result.append(d) | |
| return result | |