English and Russian model: write_line.py --model bilingual (non-commercial research only, trained on BRUSH)
a34a22a verified Download scrawler/site_model.py from levossadtchi/Scrawler-open: direct link, hf CLI and curl.
- Browser
- Download file 16.7 kB
-
https://huggingface.co/levossadtchi/Scrawler-open/resolve/main/scrawler/site_model.py
- Command line
-
hf download hf://levossadtchi/Scrawler-open/scrawler/site_model.py
-
curl -L -o site_model.py https://huggingface.co/levossadtchi/Scrawler-open/resolve/main/scrawler/site_model.py
16.7 kB
| """The text model of synthink.dev and its recipe: one reference line and a text -> the text as a line in that hand. | |
| Three things differ from scrawler/text_model.py: | |
| - The denoiser also attends to the pen points of the reference line (two cross-attention layers), so it reads the line itself | |
| and not only its style vector. | |
| - The planner's point counts are multiplied by the writer's spread: the length of the reference in points over the length the | |
| planner gives for the reference text, clipped to [0.6, 1.4]. This needs the text of the reference line. | |
| - Every line is sampled four times. A recogniser keeps the attempts that read best (character error rate within 0.02 of the | |
| lowest), and of those a writer classifier picks the one closest to the reference. | |
| SiteModel("bilingual") is the English and Russian model: the site model trained further on English pen lines of BRUSH, with a | |
| style encoder that also tells English writers apart. The recipe is the same, with two changes measured on it: an English line is | |
| read by a recogniser that knows English handwriting, and a Russian line that starts with a capital letter gets for that letter | |
| the length the planner gives it inside a phrase (without this the first word often comes out with a stray stroke). | |
| Lines are arrays [n, 3] of (dx, dy, pen_up) with line height 1, as everywhere in this package. | |
| """ | |
| import numpy as np | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from .common import AMP, CKPT_CLASSIFIER, CKPT_PLANNER, CKPT_RECOGNISER, CKPT_SITE_TEXT, DEVICE, load_checkpoint | |
| from .common import CKPT_BILINGUAL_RECOGNISER, CKPT_BILINGUAL_STYLE, CKPT_BILINGUAL_TEXT, CKPT_STYLE | |
| from .style_encoder import SCALE, features, load_style_net, resample, style_vector | |
| from .text_model import ADVANCE, ALPHA_BAR, GUIDANCE, MAX_POINTS, NOISE_STEPS, SAMPLE_STEPS, SCALE_X, SCALE_Y | |
| from .text_model import Denoiser, Planner, _plan, time_embedding | |
| ATTEMPTS = 4 | |
| SPREAD_MIN, SPREAD_MAX = 0.6, 1.4 | |
| CER_SLACK = 0.02 | |
| REF_STEP = 0.05 | |
| MODELS = { | |
| "site": dict(text=CKPT_SITE_TEXT, style=CKPT_STYLE, recognisers=dict(any=CKPT_RECOGNISER)), | |
| "bilingual": dict(text=CKPT_BILINGUAL_TEXT, style=CKPT_BILINGUAL_STYLE, recognisers=dict(ru=CKPT_RECOGNISER, any=CKPT_BILINGUAL_RECOGNISER)), | |
| } | |
| FIRST_LETTER_PREFIX = "и " # bilingual model, Russian: the first letter is planned as if the line went on after this | |
| def is_russian(text): | |
| return any("а" <= c.lower() <= "я" or c in "ёЁ" for c in text) | |
| def to_strokes(line): | |
| """A line [n, 3] -> a list of strokes, each an array [k, 2] of absolute points.""" | |
| line = np.asarray(line, np.float32) | |
| points = np.cumsum(line[:, :2], 0) | |
| strokes, start = [], 0 | |
| for i in np.flatnonzero(line[:, 2] > 0.5): | |
| strokes.append(points[start : i + 1]) | |
| start = i + 1 | |
| if start < len(points): | |
| strokes.append(points[start:]) | |
| return strokes | |
| def to_reference(strokes): | |
| """Strokes in any units -> a reference line: height 1, a point every 0.05 along the pen path. The same as make_ref.py.""" | |
| strokes = [np.asarray(s, np.float64)[:, :2] for s in strokes if len(s)] | |
| everything = np.concatenate(strokes) | |
| origin = everything.min(0) | |
| height = float(everything[:, 1].max() - origin[1]) | |
| out = [] | |
| for p in strokes: | |
| p = (p - origin) / height | |
| if len(p) == 1: | |
| out.append(np.array([[p[0, 0], p[0, 1], 1.0]])) | |
| continue | |
| cum = np.r_[0, np.cumsum(np.hypot(*np.diff(p, axis=0).T))] | |
| u = np.r_[np.arange(0, cum[-1], REF_STEP), cum[-1]] if cum[-1] > 1e-6 else np.array([0.0, cum[-1]]) | |
| r = np.stack([np.interp(u, cum, p[:, 0]), np.interp(u, cum, p[:, 1]), np.zeros(len(u))], 1) | |
| r[-1, 2] = 1 | |
| out.append(r) | |
| absolute = np.concatenate(out) | |
| line = absolute.copy() | |
| line[1:, :2] = absolute[1:, :2] - absolute[:-1, :2] | |
| line[0, :2] = 0 | |
| return line.astype(np.float32) | |
| class ReferenceEncoder(nn.Module): | |
| """A reference line -> the keys the denoiser attends to: one learned "sink" key and one key for every 4 pen points.""" | |
| def __init__(self, d=192): | |
| super().__init__() | |
| self.stem = nn.Sequential( | |
| nn.Conv1d(4, 128, 5, padding=2), | |
| nn.GELU(), | |
| nn.Conv1d(128, 192, 5, stride=2, padding=2), | |
| nn.GELU(), | |
| nn.Conv1d(192, 256, 5, stride=2, padding=2), | |
| nn.GELU(), | |
| ) | |
| self.gru = nn.GRU(256, d, num_layers=2, batch_first=True, bidirectional=True, dropout=0.1) | |
| self.rkv = nn.Sequential(nn.Linear(2 * d, 256), nn.LayerNorm(256)) | |
| self.sink = nn.Parameter(torch.zeros(1, 256)) | |
| def forward(self, line): | |
| """line [n, 3] -> keys [1, 1 + ceil(n / 4), 256].""" | |
| y = torch.cumsum(line[:, 1], 0) | |
| middle = torch.sort(y).values[(len(y) - 1) // 2] # the lower median | |
| x = torch.cat([line[:, :2] * SCALE, line[:, 2:3], (y - middle)[:, None]], -1)[None] | |
| h = self.stem(x.transpose(1, 2)).transpose(1, 2) | |
| out, _ = self.gru(h) | |
| return torch.cat([self.sink[None], self.rkv(out)], 1) | |
| class SiteDenoiser(Denoiser): | |
| """The denoiser of text_model.py with cross-attention to the reference keys after the GRU and before the head.""" | |
| def __init__(self, vocab_size): | |
| super().__init__(vocab_size) | |
| D = self.head.in_features | |
| self.enc = ReferenceEncoder() | |
| self.src_emb = nn.Parameter(torch.zeros(3, 256)) # where the ink came from: 0 tablet, 1 traced scan, 2 unknown | |
| self.xln = nn.ModuleList([nn.LayerNorm(D) for _ in range(2)]) | |
| self.xatt = nn.ModuleList([nn.MultiheadAttention(D, 4, kdim=256, vdim=256, batch_first=True) for _ in range(2)]) | |
| def forward(self, z, t, cond, style, self_cond, keys): | |
| mask_ = cond["mask"] | |
| mask = mask_[..., None] | |
| length = z.size(1) | |
| pos = torch.arange(length, device=z.device, dtype=torch.float32)[None, :, None].expand(z.size(0), -1, -1) / 600.0 | |
| phase = cond["progress"][..., None] | |
| feats = torch.cat( | |
| [z, self_cond, phase, torch.sin(2 * np.pi * phase), torch.cos(2 * np.pi * phase), torch.log1p(cond["duration"])[..., None] / 3.0, pos], | |
| -1, | |
| ) | |
| x = (self.inp(feats) + self.ec(cond["char"]) + self.ep(cond["prev"]) + self.en(cond["next"]) + self.eph(cond["phase"]) + self.sin(style)[:, None]) * mask | |
| g = F.silu(self.tm(time_embedding(t)) + self.sp(style)) | |
| for block in self.pre: | |
| x = block(x, mask, g) | |
| h = self.gfilm(self.gln(x), g) | |
| lengths = mask_.sum(1).long().clamp(min=1) | |
| with torch.autocast("cuda", enabled=False): | |
| packed = nn.utils.rnn.pack_padded_sequence(h.float(), lengths.cpu(), batch_first=True, enforce_sorted=False) | |
| out, _ = self.gru(packed) | |
| out, _ = nn.utils.rnn.pad_packed_sequence(out, batch_first=True, total_length=length) | |
| x = (x + self.gout(out.to(x.dtype))) * mask | |
| keys = keys.to(x.dtype) | |
| x = (x + self.xatt[0](self.xln[0](x), keys, keys, need_weights=False)[0]) * mask | |
| for block in self.post: | |
| x = block(x, mask, g) | |
| x = (x + self.xatt[1](self.xln[1](x), keys, keys, need_weights=False)[0]) * mask | |
| out = self.head(self.oln(x)).float() | |
| return out[..., :2] * mask, out[..., 2] | |
| class Recogniser(nn.Module): | |
| """Pen points -> per-point character scores (CTC). Reads generated lines to tell how legible they are.""" | |
| def __init__(self, vocab_size, hidden=384): | |
| super().__init__() | |
| self.inp = nn.Sequential(nn.Linear(4, hidden), nn.ReLU(), nn.Linear(hidden, hidden)) | |
| self.rnn = nn.LSTM(hidden, hidden, num_layers=3, bidirectional=True, batch_first=True) | |
| self.out = nn.Linear(2 * hidden, vocab_size) | |
| def forward(self, x): | |
| out, _ = self.rnn(self.inp(x)) | |
| return self.out(out) | |
| class WriterEmbedder(nn.Module): | |
| """A writer classifier without its class layer: a line -> a vector that is close for lines of the same hand.""" | |
| def __init__(self, d=192): | |
| super().__init__() | |
| self.stem = ReferenceEncoder(d).stem | |
| self.gru = nn.GRU(256, d, num_layers=2, batch_first=True, bidirectional=True, dropout=0.1) | |
| self.head = nn.Sequential(nn.Linear(4 * d, 256), nn.GELU()) | |
| def forward(self, x): | |
| out, _ = self.gru(self.stem(x.transpose(1, 2)).transpose(1, 2)) | |
| return self.head(torch.cat([out.mean(1), out.max(1).values], -1)) | |
| def _edit_rate(target, hypothesis): | |
| """Edit distance between two id lists over the length of the target.""" | |
| if not target and not hypothesis: | |
| return 0.0 | |
| row = list(range(len(hypothesis) + 1)) | |
| for i, a in enumerate(target, 1): | |
| prev, row[0] = row[0], i | |
| for j, b in enumerate(hypothesis, 1): | |
| prev, row[j] = row[j], min(row[j] + 1, row[j - 1] + 1, prev + (a != b)) | |
| return row[len(hypothesis)] / max(1, len(target)) | |
| class SiteModel: | |
| def __init__(self, kind="site"): | |
| if kind not in MODELS: | |
| raise ValueError(f"unknown model {kind!r}: {' or '.join(MODELS)}") | |
| self.kind = kind | |
| paths = MODELS[kind] | |
| state = load_checkpoint(paths["text"]) | |
| self.itos = state["itos"] | |
| self.stoi = {s: i for i, s in enumerate(self.itos)} | |
| self.null_token = len(self.itos) | |
| self.net = SiteDenoiser(len(self.itos)).to(DEVICE) | |
| unused = ("enc.proj.", "enc.cls.") # the classifier head the reference encoder was started from | |
| self.net.load_state_dict({k: v.float() for k, v in state["e"].items() if not k.startswith(unused)}) | |
| self.net.eval() | |
| state = load_checkpoint(CKPT_PLANNER) | |
| self.planner = Planner(len(state["itos"])).to(DEVICE) | |
| self.planner.load_state_dict(state["m"]) | |
| self.planner.eval() | |
| self.style_net = load_style_net(paths["style"]) | |
| self.recognisers = {} | |
| for name, path in paths["recognisers"].items(): | |
| state = load_checkpoint(path) | |
| net = Recogniser(len(state["itos"])).to(DEVICE) | |
| net.load_state_dict({k: v.float() for k, v in state["m"].items()}) | |
| self.recognisers[name] = net.eval() | |
| state = load_checkpoint(CKPT_CLASSIFIER) | |
| self.embedder = WriterEmbedder().to(DEVICE) | |
| self.embedder.load_state_dict({k: v.float() for k, v in state["m"].items()}) | |
| self.embedder.eval() | |
| def clean(self, text): | |
| """Drops the characters the model does not know.""" | |
| return "".join(c for c in text if c in self.stoi) | |
| def _durations(self, text, semb): | |
| ids = [1] + [self.stoi[c] for c in text] + [2] # <bos> ... <eos> | |
| tokens = torch.tensor([ids], device=DEVICE) | |
| out = self.planner(tokens, torch.ones(1, len(ids), device=DEVICE), semb.float()) | |
| d = torch.expm1(out).clamp(min=1).round().long().cpu().numpy() | |
| return [int(v) for v in d[0, 1:-1]] | |
| def _recogniser(self, text): | |
| return self.recognisers["ru" if "ru" in self.recognisers and is_russian(text) else "any"] | |
| def _read(self, lines, recogniser=None): | |
| """Equal-length lines -> the recognised character ids of each.""" | |
| x = np.stack([np.stack([q[:, 0] * 20.0, q[:, 1] * 20.0, q[:, 2], np.cumsum(q[:, 1])], 1) for q in lines]).astype(np.float32) | |
| best = (recogniser or self.recognisers["any"])(torch.from_numpy(x).to(DEVICE)).argmax(-1).cpu().numpy() | |
| out = [] | |
| for row in best: | |
| ids, prev = [], 0 | |
| for c in row: | |
| if c != 0 and c != prev: | |
| ids.append(int(c)) | |
| prev = c | |
| out.append(ids) | |
| return out | |
| def read(self, line): | |
| """What the recogniser reads in a line. Use it to check or fill in the text of your reference.""" | |
| return "".join(s for s in (self.itos[i] for i in self._read([np.asarray(line, np.float32)])[0]) if len(s) == 1) | |
| def _embed(self, line): | |
| """The unit writer-classifier vector of a line, after bringing the line to the reference form.""" | |
| x = features(resample(to_reference(to_strokes(line)))) | |
| e = self.embedder(torch.from_numpy(x[None]).to(DEVICE)).float()[0] | |
| return F.normalize(e, dim=-1) | |
| def reference(self, line, text): | |
| """A reference line and what is written in it -> everything the recipe needs from the writer.""" | |
| line = np.asarray(line, np.float32)[:, :3] | |
| semb = style_vector(self.style_net, line) | |
| text = self.clean(text) | |
| spread = 1.0 | |
| if text: | |
| spread = float(np.clip(len(line) / max(1, sum(self._durations(text, semb))), SPREAD_MIN, SPREAD_MAX)) | |
| return { | |
| "semb": semb, | |
| "style": self.net.semb_proj(semb) + self.net.src_emb[0][None], # written as tablet ink | |
| "keys": self.net.enc(torch.from_numpy(line[:MAX_POINTS]).to(DEVICE)), | |
| "embedding": self._embed(line), | |
| "spread": spread, | |
| } | |
| def write(self, text, ref, attempts=ATTEMPTS, seed=0): | |
| """-> {"lines": the attempts, "pick": the index of the chosen one, "cer", "cos", "text": the text after cleaning}.""" | |
| text = self.clean(text) | |
| if not text: | |
| raise ValueError("no characters the model knows") | |
| ids = [self.stoi[c] for c in text] | |
| durations = [max(1, int(round(d * ref["spread"]))) for d in self._durations(text, ref["semb"])] | |
| if self.kind == "bilingual" and text[0].isupper() and is_russian(text[0]): | |
| inside = self._durations(FIRST_LETTER_PREFIX + text, ref["semb"])[len(FIRST_LETTER_PREFIX)] | |
| durations[0] = min(durations[0], max(1, int(round(max(1, inside) * ref["spread"])))) | |
| length = sum(durations) | |
| if length > MAX_POINTS: | |
| raise ValueError(f"the line is too long: {length} pen points, the model takes {MAX_POINTS}. Shorten the text.") | |
| B = attempts | |
| plan = _plan(ids, durations) | |
| cond = { | |
| k: torch.tensor(v, dtype=torch.float32 if k in ("progress", "duration") else torch.long, device=DEVICE)[None].expand(B, -1) | |
| for k, v in zip(("char", "prev", "next", "phase", "progress", "duration"), plan) | |
| } | |
| cond["mask"] = torch.ones(B, length, device=DEVICE) | |
| # classifier-free guidance: the same batch twice, the second time with the characters dropped; style and reference stay | |
| dropped = {k: (torch.full_like(v, self.null_token) if k in ("char", "prev", "next") else v) for k, v in cond.items()} | |
| cond2 = {k: torch.cat([cond[k], dropped[k]], 0) for k in cond} | |
| style2 = ref["style"].expand(2 * B, -1) | |
| keys2 = ref["keys"].expand(2 * B, -1, -1) | |
| generator = torch.Generator().manual_seed(seed) | |
| z = torch.randn(B, length, 3, generator=generator).to(DEVICE) | |
| self_cond = torch.zeros_like(z) | |
| steps = np.linspace(NOISE_STEPS - 1, 0, SAMPLE_STEPS).round().astype(int) | |
| for k, t in enumerate(steps): | |
| tt = torch.full((2 * B,), int(t), device=DEVICE, dtype=torch.long) | |
| with torch.autocast("cuda", dtype=torch.bfloat16, enabled=AMP): | |
| xy, pen = self.net(torch.cat([z, z], 0), tt, cond2, style2, torch.cat([self_cond, self_cond], 0), keys2) | |
| xy = xy[B:] + GUIDANCE * (xy[:B] - xy[B:]) | |
| pen = pen[B:] + GUIDANCE * (pen[:B] - pen[B:]) | |
| x0 = torch.cat([xy.float(), torch.tanh(pen.float() / 2)[..., None]], -1) | |
| self_cond = x0 | |
| if k == len(steps) - 1: | |
| break | |
| a, a_next = ALPHA_BAR[t], ALPHA_BAR[steps[k + 1]] | |
| eps = (z - a.sqrt() * x0) / (1 - a).sqrt() | |
| z = a_next.sqrt() * x0 + (1 - a_next).sqrt() * eps # DDIM | |
| index = torch.arange(length, device=DEVICE, dtype=torch.float32)[None] | |
| points = torch.stack([x0[..., 0] * SCALE_X + ADVANCE * index, x0[..., 1] * SCALE_Y], -1).cpu().numpy() | |
| pen_up = (pen.float() > 0.0).float().cpu().numpy() | |
| lines = [] | |
| for b in range(B): | |
| d = np.zeros((length, 3), np.float32) | |
| d[1:, :2] = np.diff(points[b], axis=0) | |
| d[:, 2] = pen_up[b] | |
| d[-1, 2] = 1 | |
| lines.append(d) | |
| cer = np.array([_edit_rate(ids, h) for h in self._read(lines, self._recogniser(text))], np.float32) | |
| cos = np.array([float(self._embed(q) @ ref["embedding"]) for q in lines], np.float32) | |
| legible = np.flatnonzero(cer <= cer.min() + CER_SLACK) | |
| pick = int(legible[np.argmax(cos[legible])]) | |
| return {"lines": lines, "pick": pick, "cer": cer, "cos": cos, "text": text, "durations": durations} | |