Scrawler-open / scrawler /site_model.py
levossadtchi's picture
English and Russian model: write_line.py --model bilingual (non-commercial research only, trained on BRUSH)
a34a22a verified
Raw History Blame Contribute Delete
16.7 kB
"""The text model of synthink.dev and its recipe: one reference line and a text -> the text as a line in that hand.
Three things differ from scrawler/text_model.py:
- The denoiser also attends to the pen points of the reference line (two cross-attention layers), so it reads the line itself
and not only its style vector.
- The planner's point counts are multiplied by the writer's spread: the length of the reference in points over the length the
planner gives for the reference text, clipped to [0.6, 1.4]. This needs the text of the reference line.
- Every line is sampled four times. A recogniser keeps the attempts that read best (character error rate within 0.02 of the
lowest), and of those a writer classifier picks the one closest to the reference.
SiteModel("bilingual") is the English and Russian model: the site model trained further on English pen lines of BRUSH, with a
style encoder that also tells English writers apart. The recipe is the same, with two changes measured on it: an English line is
read by a recogniser that knows English handwriting, and a Russian line that starts with a capital letter gets for that letter
the length the planner gives it inside a phrase (without this the first word often comes out with a stray stroke).
Lines are arrays [n, 3] of (dx, dy, pen_up) with line height 1, as everywhere in this package.
"""
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from .common import AMP, CKPT_CLASSIFIER, CKPT_PLANNER, CKPT_RECOGNISER, CKPT_SITE_TEXT, DEVICE, load_checkpoint
from .common import CKPT_BILINGUAL_RECOGNISER, CKPT_BILINGUAL_STYLE, CKPT_BILINGUAL_TEXT, CKPT_STYLE
from .style_encoder import SCALE, features, load_style_net, resample, style_vector
from .text_model import ADVANCE, ALPHA_BAR, GUIDANCE, MAX_POINTS, NOISE_STEPS, SAMPLE_STEPS, SCALE_X, SCALE_Y
from .text_model import Denoiser, Planner, _plan, time_embedding
ATTEMPTS = 4
SPREAD_MIN, SPREAD_MAX = 0.6, 1.4
CER_SLACK = 0.02
REF_STEP = 0.05
MODELS = {
"site": dict(text=CKPT_SITE_TEXT, style=CKPT_STYLE, recognisers=dict(any=CKPT_RECOGNISER)),
"bilingual": dict(text=CKPT_BILINGUAL_TEXT, style=CKPT_BILINGUAL_STYLE, recognisers=dict(ru=CKPT_RECOGNISER, any=CKPT_BILINGUAL_RECOGNISER)),
}
FIRST_LETTER_PREFIX = "и " # bilingual model, Russian: the first letter is planned as if the line went on after this
def is_russian(text):
return any("а" <= c.lower() <= "я" or c in "ёЁ" for c in text)
def to_strokes(line):
"""A line [n, 3] -> a list of strokes, each an array [k, 2] of absolute points."""
line = np.asarray(line, np.float32)
points = np.cumsum(line[:, :2], 0)
strokes, start = [], 0
for i in np.flatnonzero(line[:, 2] > 0.5):
strokes.append(points[start : i + 1])
start = i + 1
if start < len(points):
strokes.append(points[start:])
return strokes
def to_reference(strokes):
"""Strokes in any units -> a reference line: height 1, a point every 0.05 along the pen path. The same as make_ref.py."""
strokes = [np.asarray(s, np.float64)[:, :2] for s in strokes if len(s)]
everything = np.concatenate(strokes)
origin = everything.min(0)
height = float(everything[:, 1].max() - origin[1])
out = []
for p in strokes:
p = (p - origin) / height
if len(p) == 1:
out.append(np.array([[p[0, 0], p[0, 1], 1.0]]))
continue
cum = np.r_[0, np.cumsum(np.hypot(*np.diff(p, axis=0).T))]
u = np.r_[np.arange(0, cum[-1], REF_STEP), cum[-1]] if cum[-1] > 1e-6 else np.array([0.0, cum[-1]])
r = np.stack([np.interp(u, cum, p[:, 0]), np.interp(u, cum, p[:, 1]), np.zeros(len(u))], 1)
r[-1, 2] = 1
out.append(r)
absolute = np.concatenate(out)
line = absolute.copy()
line[1:, :2] = absolute[1:, :2] - absolute[:-1, :2]
line[0, :2] = 0
return line.astype(np.float32)
class ReferenceEncoder(nn.Module):
"""A reference line -> the keys the denoiser attends to: one learned "sink" key and one key for every 4 pen points."""
def __init__(self, d=192):
super().__init__()
self.stem = nn.Sequential(
nn.Conv1d(4, 128, 5, padding=2),
nn.GELU(),
nn.Conv1d(128, 192, 5, stride=2, padding=2),
nn.GELU(),
nn.Conv1d(192, 256, 5, stride=2, padding=2),
nn.GELU(),
)
self.gru = nn.GRU(256, d, num_layers=2, batch_first=True, bidirectional=True, dropout=0.1)
self.rkv = nn.Sequential(nn.Linear(2 * d, 256), nn.LayerNorm(256))
self.sink = nn.Parameter(torch.zeros(1, 256))
def forward(self, line):
"""line [n, 3] -> keys [1, 1 + ceil(n / 4), 256]."""
y = torch.cumsum(line[:, 1], 0)
middle = torch.sort(y).values[(len(y) - 1) // 2] # the lower median
x = torch.cat([line[:, :2] * SCALE, line[:, 2:3], (y - middle)[:, None]], -1)[None]
h = self.stem(x.transpose(1, 2)).transpose(1, 2)
out, _ = self.gru(h)
return torch.cat([self.sink[None], self.rkv(out)], 1)
class SiteDenoiser(Denoiser):
"""The denoiser of text_model.py with cross-attention to the reference keys after the GRU and before the head."""
def __init__(self, vocab_size):
super().__init__(vocab_size)
D = self.head.in_features
self.enc = ReferenceEncoder()
self.src_emb = nn.Parameter(torch.zeros(3, 256)) # where the ink came from: 0 tablet, 1 traced scan, 2 unknown
self.xln = nn.ModuleList([nn.LayerNorm(D) for _ in range(2)])
self.xatt = nn.ModuleList([nn.MultiheadAttention(D, 4, kdim=256, vdim=256, batch_first=True) for _ in range(2)])
def forward(self, z, t, cond, style, self_cond, keys):
mask_ = cond["mask"]
mask = mask_[..., None]
length = z.size(1)
pos = torch.arange(length, device=z.device, dtype=torch.float32)[None, :, None].expand(z.size(0), -1, -1) / 600.0
phase = cond["progress"][..., None]
feats = torch.cat(
[z, self_cond, phase, torch.sin(2 * np.pi * phase), torch.cos(2 * np.pi * phase), torch.log1p(cond["duration"])[..., None] / 3.0, pos],
-1,
)
x = (self.inp(feats) + self.ec(cond["char"]) + self.ep(cond["prev"]) + self.en(cond["next"]) + self.eph(cond["phase"]) + self.sin(style)[:, None]) * mask
g = F.silu(self.tm(time_embedding(t)) + self.sp(style))
for block in self.pre:
x = block(x, mask, g)
h = self.gfilm(self.gln(x), g)
lengths = mask_.sum(1).long().clamp(min=1)
with torch.autocast("cuda", enabled=False):
packed = nn.utils.rnn.pack_padded_sequence(h.float(), lengths.cpu(), batch_first=True, enforce_sorted=False)
out, _ = self.gru(packed)
out, _ = nn.utils.rnn.pad_packed_sequence(out, batch_first=True, total_length=length)
x = (x + self.gout(out.to(x.dtype))) * mask
keys = keys.to(x.dtype)
x = (x + self.xatt[0](self.xln[0](x), keys, keys, need_weights=False)[0]) * mask
for block in self.post:
x = block(x, mask, g)
x = (x + self.xatt[1](self.xln[1](x), keys, keys, need_weights=False)[0]) * mask
out = self.head(self.oln(x)).float()
return out[..., :2] * mask, out[..., 2]
class Recogniser(nn.Module):
"""Pen points -> per-point character scores (CTC). Reads generated lines to tell how legible they are."""
def __init__(self, vocab_size, hidden=384):
super().__init__()
self.inp = nn.Sequential(nn.Linear(4, hidden), nn.ReLU(), nn.Linear(hidden, hidden))
self.rnn = nn.LSTM(hidden, hidden, num_layers=3, bidirectional=True, batch_first=True)
self.out = nn.Linear(2 * hidden, vocab_size)
def forward(self, x):
out, _ = self.rnn(self.inp(x))
return self.out(out)
class WriterEmbedder(nn.Module):
"""A writer classifier without its class layer: a line -> a vector that is close for lines of the same hand."""
def __init__(self, d=192):
super().__init__()
self.stem = ReferenceEncoder(d).stem
self.gru = nn.GRU(256, d, num_layers=2, batch_first=True, bidirectional=True, dropout=0.1)
self.head = nn.Sequential(nn.Linear(4 * d, 256), nn.GELU())
def forward(self, x):
out, _ = self.gru(self.stem(x.transpose(1, 2)).transpose(1, 2))
return self.head(torch.cat([out.mean(1), out.max(1).values], -1))
def _edit_rate(target, hypothesis):
"""Edit distance between two id lists over the length of the target."""
if not target and not hypothesis:
return 0.0
row = list(range(len(hypothesis) + 1))
for i, a in enumerate(target, 1):
prev, row[0] = row[0], i
for j, b in enumerate(hypothesis, 1):
prev, row[j] = row[j], min(row[j] + 1, row[j - 1] + 1, prev + (a != b))
return row[len(hypothesis)] / max(1, len(target))
class SiteModel:
def __init__(self, kind="site"):
if kind not in MODELS:
raise ValueError(f"unknown model {kind!r}: {' or '.join(MODELS)}")
self.kind = kind
paths = MODELS[kind]
state = load_checkpoint(paths["text"])
self.itos = state["itos"]
self.stoi = {s: i for i, s in enumerate(self.itos)}
self.null_token = len(self.itos)
self.net = SiteDenoiser(len(self.itos)).to(DEVICE)
unused = ("enc.proj.", "enc.cls.") # the classifier head the reference encoder was started from
self.net.load_state_dict({k: v.float() for k, v in state["e"].items() if not k.startswith(unused)})
self.net.eval()
state = load_checkpoint(CKPT_PLANNER)
self.planner = Planner(len(state["itos"])).to(DEVICE)
self.planner.load_state_dict(state["m"])
self.planner.eval()
self.style_net = load_style_net(paths["style"])
self.recognisers = {}
for name, path in paths["recognisers"].items():
state = load_checkpoint(path)
net = Recogniser(len(state["itos"])).to(DEVICE)
net.load_state_dict({k: v.float() for k, v in state["m"].items()})
self.recognisers[name] = net.eval()
state = load_checkpoint(CKPT_CLASSIFIER)
self.embedder = WriterEmbedder().to(DEVICE)
self.embedder.load_state_dict({k: v.float() for k, v in state["m"].items()})
self.embedder.eval()
def clean(self, text):
"""Drops the characters the model does not know."""
return "".join(c for c in text if c in self.stoi)
@torch.no_grad()
def _durations(self, text, semb):
ids = [1] + [self.stoi[c] for c in text] + [2] # <bos> ... <eos>
tokens = torch.tensor([ids], device=DEVICE)
out = self.planner(tokens, torch.ones(1, len(ids), device=DEVICE), semb.float())
d = torch.expm1(out).clamp(min=1).round().long().cpu().numpy()
return [int(v) for v in d[0, 1:-1]]
def _recogniser(self, text):
return self.recognisers["ru" if "ru" in self.recognisers and is_russian(text) else "any"]
@torch.no_grad()
def _read(self, lines, recogniser=None):
"""Equal-length lines -> the recognised character ids of each."""
x = np.stack([np.stack([q[:, 0] * 20.0, q[:, 1] * 20.0, q[:, 2], np.cumsum(q[:, 1])], 1) for q in lines]).astype(np.float32)
best = (recogniser or self.recognisers["any"])(torch.from_numpy(x).to(DEVICE)).argmax(-1).cpu().numpy()
out = []
for row in best:
ids, prev = [], 0
for c in row:
if c != 0 and c != prev:
ids.append(int(c))
prev = c
out.append(ids)
return out
def read(self, line):
"""What the recogniser reads in a line. Use it to check or fill in the text of your reference."""
return "".join(s for s in (self.itos[i] for i in self._read([np.asarray(line, np.float32)])[0]) if len(s) == 1)
@torch.no_grad()
def _embed(self, line):
"""The unit writer-classifier vector of a line, after bringing the line to the reference form."""
x = features(resample(to_reference(to_strokes(line))))
e = self.embedder(torch.from_numpy(x[None]).to(DEVICE)).float()[0]
return F.normalize(e, dim=-1)
@torch.no_grad()
def reference(self, line, text):
"""A reference line and what is written in it -> everything the recipe needs from the writer."""
line = np.asarray(line, np.float32)[:, :3]
semb = style_vector(self.style_net, line)
text = self.clean(text)
spread = 1.0
if text:
spread = float(np.clip(len(line) / max(1, sum(self._durations(text, semb))), SPREAD_MIN, SPREAD_MAX))
return {
"semb": semb,
"style": self.net.semb_proj(semb) + self.net.src_emb[0][None], # written as tablet ink
"keys": self.net.enc(torch.from_numpy(line[:MAX_POINTS]).to(DEVICE)),
"embedding": self._embed(line),
"spread": spread,
}
@torch.no_grad()
def write(self, text, ref, attempts=ATTEMPTS, seed=0):
"""-> {"lines": the attempts, "pick": the index of the chosen one, "cer", "cos", "text": the text after cleaning}."""
text = self.clean(text)
if not text:
raise ValueError("no characters the model knows")
ids = [self.stoi[c] for c in text]
durations = [max(1, int(round(d * ref["spread"]))) for d in self._durations(text, ref["semb"])]
if self.kind == "bilingual" and text[0].isupper() and is_russian(text[0]):
inside = self._durations(FIRST_LETTER_PREFIX + text, ref["semb"])[len(FIRST_LETTER_PREFIX)]
durations[0] = min(durations[0], max(1, int(round(max(1, inside) * ref["spread"]))))
length = sum(durations)
if length > MAX_POINTS:
raise ValueError(f"the line is too long: {length} pen points, the model takes {MAX_POINTS}. Shorten the text.")
B = attempts
plan = _plan(ids, durations)
cond = {
k: torch.tensor(v, dtype=torch.float32 if k in ("progress", "duration") else torch.long, device=DEVICE)[None].expand(B, -1)
for k, v in zip(("char", "prev", "next", "phase", "progress", "duration"), plan)
}
cond["mask"] = torch.ones(B, length, device=DEVICE)
# classifier-free guidance: the same batch twice, the second time with the characters dropped; style and reference stay
dropped = {k: (torch.full_like(v, self.null_token) if k in ("char", "prev", "next") else v) for k, v in cond.items()}
cond2 = {k: torch.cat([cond[k], dropped[k]], 0) for k in cond}
style2 = ref["style"].expand(2 * B, -1)
keys2 = ref["keys"].expand(2 * B, -1, -1)
generator = torch.Generator().manual_seed(seed)
z = torch.randn(B, length, 3, generator=generator).to(DEVICE)
self_cond = torch.zeros_like(z)
steps = np.linspace(NOISE_STEPS - 1, 0, SAMPLE_STEPS).round().astype(int)
for k, t in enumerate(steps):
tt = torch.full((2 * B,), int(t), device=DEVICE, dtype=torch.long)
with torch.autocast("cuda", dtype=torch.bfloat16, enabled=AMP):
xy, pen = self.net(torch.cat([z, z], 0), tt, cond2, style2, torch.cat([self_cond, self_cond], 0), keys2)
xy = xy[B:] + GUIDANCE * (xy[:B] - xy[B:])
pen = pen[B:] + GUIDANCE * (pen[:B] - pen[B:])
x0 = torch.cat([xy.float(), torch.tanh(pen.float() / 2)[..., None]], -1)
self_cond = x0
if k == len(steps) - 1:
break
a, a_next = ALPHA_BAR[t], ALPHA_BAR[steps[k + 1]]
eps = (z - a.sqrt() * x0) / (1 - a).sqrt()
z = a_next.sqrt() * x0 + (1 - a_next).sqrt() * eps # DDIM
index = torch.arange(length, device=DEVICE, dtype=torch.float32)[None]
points = torch.stack([x0[..., 0] * SCALE_X + ADVANCE * index, x0[..., 1] * SCALE_Y], -1).cpu().numpy()
pen_up = (pen.float() > 0.0).float().cpu().numpy()
lines = []
for b in range(B):
d = np.zeros((length, 3), np.float32)
d[1:, :2] = np.diff(points[b], axis=0)
d[:, 2] = pen_up[b]
d[-1, 2] = 1
lines.append(d)
cer = np.array([_edit_rate(ids, h) for h in self._read(lines, self._recogniser(text))], np.float32)
cos = np.array([float(self._embed(q) @ ref["embedding"]) for q in lines], np.float32)
legible = np.flatnonzero(cer <= cer.min() + CER_SLACK)
pick = int(legible[np.argmax(cos[legible])])
return {"lines": lines, "pick": pick, "cer": cer, "cos": cos, "text": text, "durations": durations}