Download scrawler/glyph_model.py from levossadtchi/Scrawler-open: direct link, hf CLI and curl.
- Browser
- Download file 11.6 kB
-
https://huggingface.co/levossadtchi/Scrawler-open/resolve/main/scrawler/glyph_model.py
- Command line
-
hf download hf://levossadtchi/Scrawler-open/scrawler/glyph_model.py
-
curl -L -o glyph_model.py https://huggingface.co/levossadtchi/Scrawler-open/resolve/main/scrawler/glyph_model.py
11.6 kB
| """The glyph model and its picker. | |
| The glyph model is a diffusion denoiser that draws one formula symbol of a given class in the style of a given writer. Several | |
| variants are drawn for each symbol; a small image classifier keeps the most recognisable one. | |
| """ | |
| import json | |
| import math | |
| import numpy as np | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from PIL import Image, ImageDraw | |
| from .common import AMP, CKPT_GLYPH, CKPT_GLYPH_CLASSIFIER, DEVICE, STROKE_COUNTS, load_checkpoint | |
| from .text_model import ConvBlock, FiLM, alpha_bar, time_embedding | |
| STYLE_DIM = 256 | |
| HIDDEN = 256 | |
| GRU_HIDDEN = 256 | |
| COND_DIM = 192 | |
| BLOCKS_BEFORE = 4 | |
| BLOCKS_AFTER = 3 | |
| NOISE_STEPS = 100 | |
| SAMPLE_STEPS = 10 | |
| GUIDANCE = 1.2 | |
| SCALE = 0.15 | |
| PICK_MARGIN = 0.25 # the right class must beat the best other class by this much | |
| MIN_STROKE_SHARE = 0.4 # a variant's stroke count must be this common among real glyphs of the class | |
| ALPHA_BAR = torch.tensor(alpha_bar(NOISE_STEPS), device=DEVICE) | |
| class GlyphDenoiser(nn.Module): | |
| def __init__(self, classes): | |
| super().__init__() | |
| D, G = HIDDEN, COND_DIM | |
| self.ncls = classes | |
| self.ec = nn.Embedding(classes + 1, D) # the last row = "dropped" for guidance | |
| self.gc = nn.Embedding(classes + 1, G) | |
| self.ew = nn.Linear(STYLE_DIM + 1, D) # style = [embedding, null flag] | |
| self.gw = nn.Linear(STYLE_DIM + 1, G) | |
| self.inp = nn.Linear(4 + 4 + 5, D) | |
| self.tm = nn.Sequential(nn.Linear(128, G), nn.SiLU(), nn.Linear(G, G)) | |
| dilations = [1, 2, 4, 1, 2, 4, 8] | |
| self.pre = nn.ModuleList([ConvBlock(D, G, dilations[k]) for k in range(BLOCKS_BEFORE)]) | |
| self.gln = nn.LayerNorm(D) | |
| self.gfilm = FiLM(G, D) | |
| self.gru = nn.GRU(D, GRU_HIDDEN, num_layers=2, batch_first=True, bidirectional=True) | |
| self.gout = nn.Linear(2 * GRU_HIDDEN, D) | |
| self.post = nn.ModuleList([ConvBlock(D, G, dilations[k]) for k in range(BLOCKS_AFTER)]) | |
| self.oln = nn.LayerNorm(D) | |
| self.head = nn.Linear(D, 4) | |
| def forward(self, z, t, classes, style, mask_, self_cond): | |
| mask = mask_[..., None] | |
| B, length = mask_.shape | |
| lengths = mask_.sum(1).clamp(min=1) | |
| i = torch.arange(length, device=z.device, dtype=torch.float32)[None].expand(B, -1) | |
| phase = (i / lengths[:, None])[..., None] | |
| feats = torch.cat( | |
| [z, self_cond, phase, torch.sin(2 * math.pi * phase), torch.cos(2 * math.pi * phase), torch.log(lengths)[:, None, None].expand(-1, length, -1) / 4.0, i[..., None] / 100.0], | |
| -1, | |
| ) | |
| x = (self.inp(feats) + self.ec(classes)[:, None] + self.ew(style)[:, None]) * mask | |
| g = F.silu(self.tm(time_embedding(t)) + self.gc(classes) + self.gw(style)) | |
| for block in self.pre: | |
| x = block(x, mask, g) | |
| h = self.gfilm(self.gln(x), g) | |
| with torch.autocast(DEVICE if DEVICE != "mps" else "cpu", enabled=False): | |
| packed = nn.utils.rnn.pack_padded_sequence(h.float(), lengths.long().cpu(), batch_first=True, enforce_sorted=False) | |
| out, _ = self.gru(packed) | |
| out, _ = nn.utils.rnn.pad_packed_sequence(out, batch_first=True, total_length=length) | |
| x = (x + self.gout(out.to(x.dtype))) * mask | |
| for block in self.post: | |
| x = block(x, mask, g) | |
| out = self.head(self.oln(x)).float() | |
| return out[..., :3] * mask, out[..., 3] | |
| def _denoise(net, classes, style, lengths, generator): | |
| B = len(classes) | |
| length = int(max(lengths)) | |
| mask_ = torch.zeros(B, length, device=DEVICE) | |
| for b, n in enumerate(lengths): | |
| mask_[b, : int(n)] = 1 | |
| mask = mask_[..., None] | |
| classes = torch.tensor(classes, device=DEVICE, dtype=torch.long) | |
| style = torch.tensor(np.asarray(style), device=DEVICE, dtype=torch.float32) | |
| z = torch.randn(B, length, 4, generator=generator).to(DEVICE) * mask | |
| self_cond = torch.zeros_like(z) | |
| steps = np.linspace(NOISE_STEPS - 1, 0, SAMPLE_STEPS).round().astype(int) | |
| classes2 = torch.cat([classes, torch.full_like(classes, net.ncls)]) | |
| style2 = torch.cat([style, style]) | |
| mask2 = torch.cat([mask_, mask_]) | |
| for k, t in enumerate(steps): | |
| tt = torch.full((B,), int(t), device=DEVICE, dtype=torch.long) | |
| with torch.autocast("cuda", dtype=torch.bfloat16, enabled=AMP): | |
| xy, pen = net(torch.cat([z, z]), torch.cat([tt, tt]), classes2, style2, mask2, torch.cat([self_cond, self_cond])) | |
| xy = xy[B:] + GUIDANCE * (xy[:B] - xy[B:]) | |
| pen = pen[B:] + GUIDANCE * (pen[:B] - pen[B:]) | |
| x0 = torch.cat([xy[..., :2].float(), torch.tanh(pen.float() / 2)[..., None], xy[..., 2:3].float()], -1) * mask | |
| self_cond = x0 | |
| if k == len(steps) - 1: | |
| break | |
| a, a_next = ALPHA_BAR[t], ALPHA_BAR[steps[k + 1]] | |
| eps = (z - a.sqrt() * x0) / (1 - a).sqrt() | |
| z = (a_next.sqrt() * x0 + (1 - a_next).sqrt() * eps) * mask | |
| x0 = x0.cpu().numpy() | |
| pen_up = (pen.float() > 0.0).cpu().numpy() | |
| result = [] | |
| for b in range(B): | |
| n = int(lengths[b]) | |
| q = np.zeros((n, 4), np.float32) | |
| q[:, 0] = x0[b, :n, 0] * SCALE | |
| q[:, 1] = x0[b, :n, 1] * SCALE | |
| q[:, 2] = pen_up[b, :n] | |
| q[-1, 2] = 1 | |
| q[:, 3] = np.clip((x0[b, :n, 3] + 1) / 2, 0, 1) | |
| result.append(q) | |
| return result | |
| def _to_strokes(q): | |
| strokes, current = [], [] | |
| for k in range(len(q)): | |
| current.append([float(q[k, 0]), float(q[k, 1]), 0.0, float(q[k, 3])]) | |
| if q[k, 2] > 0.5 or k == len(q) - 1: | |
| strokes.append(current) | |
| current = [] | |
| return [s for s in strokes if s] | |
| class GlyphModel: | |
| def __init__(self, stats): | |
| state = load_checkpoint(CKPT_GLYPH) | |
| self.stats = stats | |
| self.classes = state["classes"] | |
| self.class_index = {c: i for i, c in enumerate(self.classes)} | |
| self.net = GlyphDenoiser(len(self.classes)).to(DEVICE) | |
| self.net.load_state_dict({k: v.float() for k, v in state["e"].items()}) | |
| self.net.eval() | |
| # how many pen points real glyphs of each class have: a variant's length is drawn from this | |
| self.lengths = {int(k.split("|")[1]): np.array(v, np.int32) for k, v in state["lens"].items() if k.startswith("-1|")} | |
| def _draw_length(self, class_id, rng): | |
| a = self.lengths.get(class_id) | |
| if a is None or len(a) == 0: | |
| return 30 | |
| return int(a[int(rng.integers(len(a)))]) | |
| def sample(self, classes, style, rng): | |
| """One batch: class names -> {"strokes": ...}; classes the net does not know come back with no strokes.""" | |
| labels = ["—" if c == "BAR" else "\u20d7" if c == "VEC" else c for c in classes] | |
| known = [i for i, label in enumerate(labels) if label in self.class_index] | |
| result = [dict(strokes=[]) for _ in labels] | |
| self.stats["glyph_net_net"] += len(known) | |
| self.stats["glyph_net_unknown"] += len(labels) - len(known) | |
| if not known: | |
| return result | |
| ids = [self.class_index[labels[i]] for i in known] | |
| lengths = [self._draw_length(c, rng) for c in ids] | |
| rows = np.zeros((len(known), STYLE_DIM + 1), np.float32) | |
| rows[:, :STYLE_DIM] = style | |
| generator = torch.Generator(device="cpu").manual_seed(int(rng.integers(1 << 31))) | |
| out = _denoise(self.net, ids, rows, lengths, generator) | |
| for j, i in enumerate(known): | |
| result[i] = dict(strokes=_to_strokes(out[j])) | |
| return result | |
| class ClassifierNet(nn.Module): | |
| def __init__(self, n): | |
| super().__init__() | |
| self.c = nn.Sequential( | |
| nn.Conv2d(1, 32, 3, padding=1), | |
| nn.ReLU(), | |
| nn.MaxPool2d(2), | |
| nn.Conv2d(32, 64, 3, padding=1), | |
| nn.ReLU(), | |
| nn.MaxPool2d(2), | |
| nn.Conv2d(64, 128, 3, padding=1), | |
| nn.ReLU(), | |
| nn.AdaptiveAvgPool2d(2), | |
| ) | |
| self.f = nn.Sequential(nn.Linear(128 * 4 + 2, 256), nn.ReLU(), nn.Dropout(0.3), nn.Linear(256, n)) | |
| def forward(self, image, extra): | |
| return self.f(torch.cat([self.c(image).flatten(1), extra], 1)) | |
| def _raster(strokes, size=32): | |
| points = np.array([p[:2] for s in strokes for p in s], np.float32) | |
| x0, y0 = points.min(0) | |
| x1, y1 = points.max(0) | |
| scale = (size - 4) / max(x1 - x0, y1 - y0, 1e-6) | |
| image = Image.new("L", (size, size), 0) | |
| draw = ImageDraw.Draw(image) | |
| ox = (size - (x1 - x0) * scale) / 2 | |
| oy = (size - (y1 - y0) * scale) / 2 | |
| for s in strokes: | |
| q = [((p[0] - x0) * scale + ox, (p[1] - y0) * scale + oy) for p in s] | |
| if len(q) > 1: | |
| draw.line(q, fill=255, width=2) | |
| elif q: | |
| draw.point(q[0], fill=255) | |
| return np.asarray(image, np.float32) / 255.0 | |
| def _shape_features(strokes, relative_height): | |
| points = np.array([p[:2] for s in strokes for p in s], np.float32) | |
| w = np.ptp(points[:, 0]) | |
| h = np.ptp(points[:, 1]) | |
| return np.array([relative_height, np.log((w + 1e-3) / (h + 1e-3))], np.float32) | |
| class GlyphPicker: | |
| """Chooses the most recognisable of several drawn variants of one symbol.""" | |
| def __init__(self, stats): | |
| self.stats = stats | |
| state = load_checkpoint(CKPT_GLYPH_CLASSIFIER) | |
| self.net = ClassifierNet(len(state["classes"])).to(DEVICE) | |
| self.net.load_state_dict(state["state"]) | |
| self.net.eval() | |
| self.index = {c: i for i, c in enumerate(state["classes"])} | |
| counts = json.load(open(STROKE_COUNTS, encoding="utf-8")) | |
| self.stroke_counts = {c: {int(k): v for k, v in d.items()} for c, d in counts.items()} | |
| def _stroke_share(self, cls, n): | |
| counts = self.stroke_counts.get(cls) | |
| total = sum(counts.values()) if counts else 0 | |
| return counts.get(n, 0) / total if total >= 20 else 1.0 | |
| def pick(self, variants, cls, relative_height, force=False): | |
| """Index of the best variant, or None if none is recognised as `cls`. force: the best of what there is.""" | |
| if cls not in self.index: | |
| self.stats["unscored"] += 1 | |
| return 0 | |
| drawn = lambda q: q["strokes"] and sum(len(t) for t in q["strokes"]) >= 2 | |
| ok = [k for k, q in enumerate(variants) if drawn(q) and self._stroke_share(cls, len(q["strokes"])) >= MIN_STROKE_SHARE] | |
| if cls in ("0", "O", "o"): # no slashed zeros | |
| ok = [k for k in ok if len(variants[k]["strokes"]) == 1] | |
| if not ok and force: | |
| ok = [k for k, q in enumerate(variants) if drawn(q)] | |
| if not ok: | |
| self.stats["fallback_nstrokes"] += 1 | |
| return None | |
| images = np.stack([_raster(variants[k]["strokes"]) for k in ok]) | |
| extras = np.stack([_shape_features(variants[k]["strokes"], relative_height) for k in ok]) | |
| with torch.no_grad(): | |
| logits = self.net(torch.from_numpy(images)[:, None].to(DEVICE), torch.from_numpy(extras).to(DEVICE)) | |
| probs = torch.softmax(logits, -1).cpu().numpy() | |
| own = probs[:, self.index[cls]] | |
| recognised = probs.argmax(1) == self.index[cls] | |
| others = probs.copy() | |
| others[:, self.index[cls]] = -1 | |
| own = own - others.max(1) | |
| recognised = recognised & (own >= PICK_MARGIN) | |
| if not recognised.any() and force: | |
| self.stats["forced"] += 1 | |
| return ok[int(np.argmax(own))] | |
| if not recognised.any(): | |
| self.stats["fallback"] += 1 | |
| return None | |
| self.stats["selected"] += 1 | |
| self.stats["first_ok"] += int(recognised[0]) | |
| return ok[int(np.argmax(own))] | |