"""Formulas: MathJax says where every symbol stands, the glyph model draws the symbols, and thin marks are drawn as pen lines. Coordinates inside this module: MathJax units x LAYOUT_SCALE, y down. """ import collections import json import math import os import subprocess import unicodedata import numpy as np import torch from .common import MATHJAX_DIR from .glyph_model import GlyphModel, GlyphPicker LAYOUT_SCALE = 0.1 X_HEIGHT = 450 * LAYOUT_SCALE X_HEIGHT_EM = 0.442 # x-height of a composed formula in the units compose() returns VARIANTS = 4 RETRIES = 16 POINT_STEP = 0.05 # fractions, radicals and arrows are pen lines over their layout box LINE_MARKS = {"BAR", "√", "→", "←", "↑", "↓", "⇒", "⇐", "⇔", "↔", "VEC"} # accents, slashes and the like, which the glyph model does not draw well OWN_MARKS = {"/", "↔", "↦", "ˆ", "^", "˜", "~", " ̄", "¯", " ̃", " ̂", "‾", "̄", "̂", "̃", "̅"} SUBSTITUTE_CODES = {"2A7E": "2265", "2A7D": "2264"} # slanted >= and <= are drawn as the plain ones # classes the glyph model has no row for, drawn as a similar shape DONORS = { "O": "0", "κ": "k", "Σ": "∑", "≃": "≈", "∙": "⋅", "*": "∗", "⋯": "…", "⇓": "↓", "⟸": "←", "⇉": "→", "⎧": "{", "⎩": "{", "⎛": "(", "∩": "u", "Θ": "θ", } # fmt: skip KEEP_SHAPE = set("=+−-<>≠±") # a sheared '=' reads as '>' VERTICAL_BARS = ("|", "∥", "‖", "∣") DOTS = ("⋅", "·", "…", "...", "⋯", ".", ":") DOT_ACCENTS = (" ̇", "˙", "̇") def class_of(code): if code == "BAR": return "BAR" ch = chr(int(code, 16)) if ch == "⃗": return "VEC" normal = unicodedata.normalize("NFKC", ch) return normal if normal.strip() else ch def layout(formulas): """LaTeX -> [{ok, glyphs: [{c, bbox, node}], ...}] with MathJax under node.""" lines = "\n".join(json.dumps(dict(id=i, tex=t), ensure_ascii=False) for i, t in enumerate(formulas)) + "\n" result = subprocess.run( ["node", "layout.mjs"], input=lines, capture_output=True, text=True, cwd=MATHJAX_DIR, env=dict(os.environ, LAYOUT_TEXT="1") ) out = {} for line in result.stdout.splitlines(): try: o = json.loads(line) out[o["id"]] = o except ValueError: pass return [out.get(i) for i in range(len(formulas))] def _merge_radicals(glyphs): """A radical sign and its overbar -> one box.""" out = [] i = 0 while i < len(glyphs): g = glyphs[i] if g["c"] == "221A" and i + 1 < len(glyphs) and glyphs[i + 1]["c"] == "BAR" and glyphs[i + 1].get("node") in ("msqrt", "mroot"): b = glyphs[i + 1]["bbox"] g = dict(c="221A", bbox=[min(g["bbox"][0], b[0]), min(g["bbox"][1], b[1]), max(g["bbox"][2], b[2]), max(g["bbox"][3], b[3])]) i += 2 else: i += 1 out.append(g) return out def _fix_limits(glyphs): """MathJax puts the lower limit of an integral under its slanted foot; handwriting puts it to the right.""" for i, g in enumerate(glyphs): if g["c"] not in ("222B", "222E"): continue x0, y0, x1, y1 = g["bbox"] middle = (y0 + y1) / 2 group = [h for h in glyphs[i + 1 : i + 4] if h["bbox"][0] < x1 and h["bbox"][1] > middle] if group: dx = x1 + 0.05 * (y1 - y0) - min(h["bbox"][0] for h in group) for h in group: h["bbox"] = [h["bbox"][0] + dx, h["bbox"][1], h["bbox"][2] + dx, h["bbox"][3]] return glyphs def _segment(a, b, n=8): return [[a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t] for t in np.linspace(0, 1, n)] def _polyline(points, rng, jitter, n=6): out = [] for i in range(len(points) - 1): q = _segment(points[i], points[i + 1], n) out += q if not out else q[1:] return [[x + rng.normal(0, jitter), y + rng.normal(0, jitter), 0.5] for x, y in out] def _line_mark(c, box, rng): """Pen-line strokes for a fraction bar, a radical or an arrow over a layout box.""" x0, y0, x1, y1 = box w = x1 - x0 h = max(y1 - y0, 1e-6) ym = (y0 + y1) / 2 xm = (x0 + x1) / 2 j = 0.02 * X_HEIGHT tilt = rng.normal(0, 0.01) * w if c == "BAR": return [_polyline([[x0 - 0.03 * w, ym - tilt], [x1 + 0.03 * w, ym + tilt]], rng, j, 12)] if c == "√": tw = min(0.5 * h, 0.4 * w) return [_polyline([[x0, y0 + 0.6 * h], [x0 + 0.18 * tw, y0 + 0.52 * h], [x0 + 0.55 * tw, y1], [x0 + tw, y0], [x1, y0 + tilt]], rng, j, 6)] head = min(0.35 * w, max(0.9 * h, 0.25 * X_HEIGHT)) if c not in ("↑", "↓") else min(0.35 * h, max(0.9 * w, 0.25 * X_HEIGHT)) if c == "VEC": return [_polyline([[x0, ym], [x1, ym]], rng, j, 8), _polyline([[x1 - head, ym - 0.5 * head], [x1, ym], [x1 - head, ym + 0.5 * head]], rng, j, 4)] if c in ("↑", "↓"): tip, sign = (y1, -1) if c == "↓" else (y0, 1) return [ _polyline([[xm, y0], [xm, y1]], rng, j, 10), _polyline([[xm - 0.5 * head, tip + sign * head], [xm, tip], [xm + 0.5 * head, tip + sign * head]], rng, j, 4), ] double = c in ("⇒", "⇐", "⇔") d = 0.22 * h if double else 0.0 strokes = [] for yy in [ym - d, ym + d] if double else [ym]: strokes.append(_polyline([[x0 + (0.15 * head if c in "⇐⇔" else 0), yy], [x1 - (0.15 * head if c in "⇒⇔" else 0), yy]], rng, j, 10)) if c in ("→", "⇒", "⇔", "↔"): strokes.append(_polyline([[x1 - head, ym - 0.55 * head - d], [x1, ym], [x1 - head, ym + 0.55 * head + d]], rng, j, 4)) if c in ("←", "⇐", "⇔", "↔"): strokes.append(_polyline([[x0 + head, ym - 0.55 * head - d], [x0, ym], [x0 + head, ym + 0.55 * head + d]], rng, j, 4)) return strokes def _transform_glyph(strokes, mx, my, size, rng): """Small random shear, bend, proportions and pen speed, so that two instances of a symbol are not the same shape.""" shear_x = rng.normal(0, 0.06) shear_y = rng.normal(0, 0.05) bend_v = rng.normal(0, 0.10) bend_h = rng.normal(0, 0.08) aspect = rng.normal(0, 0.04) ascender = rng.normal(0, 0.04) descender = rng.normal(0, 0.04) px, py = rng.normal(0, 0.02, 2) speed = 0.5 pressure = math.exp(rng.normal(0, 0.27)) all_points = np.concatenate([(p[:, :2] - [mx, my]) / size for p in strokes]) mean_x2 = float(np.mean(all_points[:, 0] ** 2)) mean_y2 = float(np.mean(all_points[:, 1] ** 2)) out = [] for p in strokes: q = p.copy() x = (p[:, 0] - mx) / size y = (p[:, 1] - my) / size if len(p) > 3: # uneven pen speed: the points move along the same path d = np.r_[0, np.cumsum(np.linalg.norm(np.diff(np.stack([x, y], 1), axis=0), axis=1))] if d[-1] > 0: u = np.linspace(0, 1, len(p)) phase = rng.uniform(0, 6.283) w = ( u + speed * np.sin(2 * np.pi * u + phase) / (2 * np.pi) - speed * np.sin(phase) / (2 * np.pi) * (1 - u) - speed * np.sin(2 * np.pi + phase) / (2 * np.pi) * u ) w = np.clip(np.maximum.accumulate(w), 0, 1) * d[-1] x = np.interp(w, d, x) y = np.interp(w, d, y) if q.shape[1] > 3: q[:, 3] = np.interp(w, d, p[:, 3]) x = x + shear_x * y y = y + shear_y * x y = y + bend_v * (x**2 - mean_x2) x = x + bend_h * (y**2 - mean_y2) x = x * math.exp(aspect / 2) y = y * math.exp(-aspect / 2) y = np.where(y < -0.25, -0.25 + (y + 0.25) * (1 + ascender), np.where(y > 0.25, 0.25 + (y - 0.25) * (1 + descender), y)) denominator = 1 + px * x + py * y x = x / denominator y = y / denominator q[:, 0] = x * size + mx q[:, 1] = y * size + my if q.shape[1] > 3: q[:, 3] = np.clip(q[:, 3] * pressure, 0, 1) out.append(q) return out def _bounds(strokes): a = np.array([p[:2] for t in strokes for p in t], np.float32) return float(a[:, 0].min()), float(a[:, 1].min()), float(a[:, 0].max()), float(a[:, 1].max()) class FormulaWriter: def __init__(self, text_model, reference, semb): self.text = text_model self.reference = np.asarray(reference, np.float32) self.semb = semb self.style_row = semb[0].detach().cpu().numpy().astype(np.float32).reshape(-1) self.stats = collections.Counter() self._glyphs = None self._picker = None self._base_heights = {} @property def glyphs(self): if self._glyphs is None: self._glyphs = GlyphModel(self.stats) return self._glyphs @property def picker(self): if self._picker is None: self._picker = GlyphPicker(self.stats) return self._picker def _variants(self, classes, rng, k=1): n = len(classes) // max(k, 1) out = [] for j in range(max(k, 1)): out += self.glyphs.sample(classes[j * n : (j + 1) * n], self.style_row, rng) return out def _write_words(self, words): """Words inside \\text{} -> {word: strokes}, written by the text model.""" words = [w for w in dict.fromkeys(words) if w.strip()] if not words: return {} lines = [list(w) for w in words] style = torch.cat([self.text.style(self.semb)] * len(words), 0) semb = torch.cat([self.semb] * len(words), 0) durations = self.text.durations(lines, semb) out = {} for word, q in zip(words, self.text.sample(lines, durations, style)): q = np.asarray(q, np.float32) xy = np.cumsum(q[:, :2], 0) * 100.0 strokes, current = [], [] for k in range(len(q)): current.append([float(xy[k, 0]), float(xy[k, 1]), 0.5]) if q[k, 2] > 0.5: strokes.append(current) current = [] if current: strokes.append(current) out[word] = strokes return out def _compose_text(self, boxes): """Places words of \\text{} in their layout boxes, fitted by x-height.""" s = LAYOUT_SCALE words, current = [], [] for g in sorted(boxes, key=lambda g: g["bbox"][0]): if not g.get("ch", "").strip(): if current: words.append(current) current = [] continue if current and g["bbox"][0] - current[-1]["bbox"][2] > 50: words.append(current) current = [] current.append(g) if current: words.append(current) written = self._write_words(["".join(g["ch"] for g in w) for w in words]) out = [] for word in words: strokes = written.get("".join(g["ch"] for g in word)) if not strokes: continue a = np.array([p[:2] for t in strokes for p in t], np.float32) bx0 = min(g["bbox"][0] for g in word) * s bx1 = max(g["bbox"][2] for g in word) * s baseline = float(np.median([g["bbox"][3] for g in word])) * s box_height = float(np.median([g["bbox"][3] - g["bbox"][1] for g in word])) * s word_height = max(np.percentile(a[:, 1], 85) - np.percentile(a[:, 1], 15), 1e-6) k = min(1.1 * (bx1 - bx0) / max(np.ptp(a[:, 0]), 1e-6), box_height / word_height) bottom = np.percentile(a[:, 1], 85) for t in strokes: out.append([[float((p[0] - a[:, 0].min()) * k + bx0), float((p[1] - bottom) * k + baseline), float(p[2])] for p in t]) return out def _own_marks(self, glyphs, rng): """Marks the glyph model does not draw -> pen lines. Returns the remaining glyphs and the strokes of the marks.""" self.glyphs # loaded here so that the order of random draws does not depend on the input s = LAYOUT_SCALE keep, out = [], [] for g in glyphs: c = class_of(g["c"]) if g["c"] not in ("BAR", "TEXT") else g["c"] if g["c"] in SUBSTITUTE_CODES: keep.append(dict(g, c=SUBSTITUTE_CODES[g["c"]])) continue if c not in OWN_MARKS: keep.append(g) continue x0, y0, x1, y1 = [v * s for v in g["bbox"]] w = max(x1 - x0, 1e-6) h = max(y1 - y0, 1e-6) j = 0.015 * X_HEIGHT ym = (y0 + y1) / 2 if c == "↔": out += _line_mark("↔", [x0, y0, x1, y1], rng) elif c == "↦": out += _line_mark("→", [x0, y0, x1, y1], rng) + [_polyline([[x0, ym - 0.4 * h], [x0, ym + 0.4 * h]], rng, j, 5)] elif c == "/": out.append(_polyline([[x0 + 0.15 * w, y1], [x1 - 0.15 * w, y0]], rng, j, 10)) elif c in ("ˆ", "^", " ̂", "̂"): out.append(_polyline([[x0 + 0.1 * w, y1], [(x0 + x1) / 2, y0], [x1 - 0.1 * w, y1]], rng, j, 5)) elif c in ("˜", "~", " ̃", "̃"): out.append([[float(x0 + t * w), float(ym - 0.35 * h * np.sin(2 * np.pi * t) + rng.normal(0, j)), 0.5] for t in np.linspace(0, 1, 14)]) else: # overline, macron out.append(_polyline([[x0, ym], [x1, ym]], rng, j, 10)) self.stats["own_mark_" + c] += 1 return keep, out def _load_base_heights(self, codes): new = [c for c in set(codes) if c not in self._base_heights and c not in ("BAR", "TEXT")] if new: for c, o in zip(new, layout([chr(int(c, 16)) for c in new])): self._base_heights[c] = (o["glyphs"][0]["bbox"][3] - o["glyphs"][0]["bbox"][1]) if o and o.get("ok") and o["glyphs"] else None def _script_factor(self, g): """Subscripts: MathJax shrinks them by 0.71, handwriting shrinks digits as much but letters less.""" if g["c"] in ("BAR", "TEXT"): return 1.0 height = self._base_heights.get(g["c"]) if not height or (g["bbox"][3] - g["bbox"][1]) >= 0.85 * height: return 1.0 ch = unicodedata.normalize("NFKC", chr(int(g["c"], 16))) if ch.isdigit(): return 0.71 if ch.isascii() and ch.isalpha(): return 1.0 return 0.92 def _push_right(self, strokes, placed): """Moves a glyph right if it collides with a neighbour on the same line.""" if not strokes or not placed: return a = np.concatenate([np.asarray(q)[:, :2] for q in strokes]) bx0, by0, bx1, by1 = a[:, 0].min(), a[:, 1].min(), a[:, 0].max(), a[:, 1].max() shift = 0.0 for px0, py0, px1, py1 in placed: overlap = min(by1, py1) - max(by0, py0) smaller = min(by1 - by0, py1 - py0) same_line = abs((py0 + py1) / 2 - (by0 + by1) / 2) < 0.3 * max(by1 - by0, py1 - py0) # a subscript is not a collision if (py1 - py0) > 1.8 * (by1 - by0) and py0 <= by0 and py1 >= by1: # a radical or a tall bracket encloses, it is no neighbour continue if same_line and smaller > 0.12 * X_HEIGHT and overlap > 0.5 * smaller and (px0 + px1) / 2 < (bx0 + bx1) / 2 and px1 + 0.35 * X_HEIGHT > bx0: shift = max(shift, px1 + 0.35 * X_HEIGHT - bx0) if 0 < shift < 2.0 * X_HEIGHT: for q in strokes: q[:, 0] += shift self.stats["pushed"] += 1 def _compose_layout(self, glyphs, rng): s = LAYOUT_SCALE xh = X_HEIGHT strokes, missing = [], [] merged = _merge_radicals(glyphs) texts = [g for g in merged if g["c"] == "TEXT"] glyphs = [] for g in merged: if g["c"] == "TEXT": continue if g["c"] == "226A": # << = two < b = g["bbox"] m = (b[0] + b[2]) / 2 w = b[2] - b[0] glyphs += [dict(g, c="3C", bbox=[b[0], b[1], m + 0.08 * w, b[3]]), dict(g, c="3C", bbox=[m - 0.08 * w, b[1], b[2], b[3]])] else: glyphs.append(g) self._load_base_heights([g["c"] for g in glyphs]) if texts: strokes += self._compose_text(texts) if not glyphs: return strokes, missing class_for_net = lambda c: DONORS.get(c, c) x_min = min(g["bbox"][0] for g in glyphs) n = len(glyphs) top = min(g["bbox"][1] for g in glyphs) height = max(max(g["bbox"][3] for g in glyphs) - top, 1e-6) variants = self._variants([class_for_net(class_of(g["c"])) for g in glyphs] * VARIANTS, rng, VARIANTS) shape_rng = np.random.default_rng(int(rng.integers(1 << 31))) gap = base = 0.0 placed = [] last = 0 for gi, g in enumerate(glyphs): if len(strokes) > last: placed.append(_bounds(strokes[last:])) last = len(strokes) upside_down = class_of(g["c"]) == "∩" # ∩ is a u turned over c = class_of(g["c"]) if c in DONORS: self.stats["donor_" + c] += 1 c = DONORS[c] if c in DOTS: # the net draws a dot as a ring b = [v * s for v in g["bbox"]] r = 0.05 * xh if c == ":": dots = [((b[0] + b[2]) / 2, b[1] + 0.15 * (b[3] - b[1])), ((b[0] + b[2]) / 2, b[3] - 0.15 * (b[3] - b[1]))] else: count = 1 if c in ("⋅", "·", ".") else 3 cy = (b[1] + b[3]) / 2 if c != "." else b[3] - r dots = [(b[0] + (k + 0.5) * (b[2] - b[0]) / count, cy) for k in range(count)] for cx, cy in dots: strokes.append([[float(cx - r), float(cy), 0.5], [float(cx), float(cy + r * 0.6), 0.5], [float(cx + r), float(cy), 0.5]]) self.stats["dot_mark"] += 1 continue if c in DOT_ACCENTS: b = [v * s for v in g["bbox"]] cx = (b[0] + b[2]) / 2 cy = (b[1] + b[3]) / 2 r = 0.05 * xh strokes.append([[float(cx - r), float(cy), 0.5], [float(cx), float(cy + r * 0.6), 0.5], [float(cx + r), float(cy), 0.5]]) self.stats["dot_mark"] += 1 continue if c in VERTICAL_BARS: # the net draws | as / ( ) ? 7 b = [v * s for v in g["bbox"]] xm = (b[0] + b[2]) / 2 offsets = [0.0] if c in ("|", "∣") else [-0.22 * (b[2] - b[0]), 0.22 * (b[2] - b[0])] bars = [np.asarray(_polyline([[xm + d, b[1]], [xm + d, b[3]]], rng, 0.015 * xh, 10), np.float64) for d in offsets] self._push_right(bars, placed) strokes += [t.tolist() for t in bars] self.stats["vbar_pen"] += 1 continue if c in LINE_MARKS: strokes += _line_mark(c, [v * s for v in g["bbox"]], rng) self.stats["line_mark"] += 1 continue relative_height = (g["bbox"][3] - g["bbox"][1]) / height q = None if c not in ("[", "]"): # square brackets always go through the retry below k = self.picker.pick([variants[k * n + gi] for k in range(VARIANTS)], c, relative_height) q = variants[k * n + gi] if k is not None else None if q is not None and not q["strokes"]: q = None if q is None: if not variants[gi]["strokes"]: # the net does not know this class missing.append(c) continue more = self._variants([class_for_net(class_of(g["c"]))] * RETRIES, rng, 1) k = self.picker.pick(more, c, relative_height) if k is not None: q = more[k] self.stats["retry_ok"] += 1 else: everything = more + [variants[j * n + gi] for j in range(VARIANTS)] k = self.picker.pick(everything, c, relative_height, force=True) q = everything[k if k is not None else 0] self.stats["retry_forced"] += 1 b = [v * s for v in g["bbox"]] bw = b[2] - b[0] bh = b[3] - b[1] points = [np.asarray(t, np.float32) for t in q["strokes"]] all_points = np.concatenate([p[:, :2] for p in points]) x0, y0 = all_points.min(0) x1, y1 = all_points.max(0) qw = max(x1 - x0, 1e-3) qh = max(y1 - y0, 1e-3) if bw > 2.5 * bh: kx = bw / qw ky = min(kx, bh / qh * 1.5) elif bh > 2.2 * bw and c in "()[]{}|∫∮∑∏": ky = bh / qh kx = min(ky, 1.6 * bw / qw) else: ky = bh / qh kx = ky f = 1.0 + rng.normal(0, 0.06) kx *= f ky *= f angle = math.radians(rng.normal(0, 2.5)) cos_a, sin_a = math.cos(angle), math.sin(angle) drift = 0.02 * xh * math.sin((b[0] - x_min * s) / (6 * xh) + rng.uniform(0, 6.28)) cx = (b[0] + b[2]) / 2 + rng.normal(0, 0.06 * xh) cy = (b[1] + b[3]) / 2 + rng.normal(0, 0.06 * xh) + drift if c not in ("(", ")", "[", "]", "{", "}", "|") and qw * kx > 1.3 * bw and bw > 0: # much wider than its box: crowds the neighbours f2 = 1.3 * bw / (qw * kx) kx *= f2 ky *= f2 self.stats["narrowed"] += 1 if class_of(g["c"]) == "Θ": # capital Theta from a lowercase one kx *= 1.35 fs = self._script_factor(g) if fs != 1.0: kx *= fs ky *= fs cx = b[0] + (cx - b[0]) * fs self.stats["script_scaled"] += 1 mx = (x0 + x1) / 2 my = (y0 + y1) / 2 transformed = _transform_glyph(points, mx, my, max(qw, qh), shape_rng) if c not in KEEP_SHAPE: # the random numbers are drawn either way points = transformed gap = 0.7 * gap + math.sqrt(1 - 0.49) * shape_rng.normal(0, 0.1) base = 0.8 * base + 0.6 * shape_rng.normal(0, 0.08) upright = int(g["c"], 16) < 128 and chr(int(g["c"], 16)).isalpha() # letters of cos, sin: no gaps inside the word cx += (0 if upright else gap) * min(bh, 1.5 * xh) cy += base * min(bh, 1.5 * xh) if upside_down: points = [np.concatenate([2 * mx - p[:, :1], 2 * my - p[:, 1:2], p[:, 2:]], 1) for p in points] placed_strokes = [] for p in points: u = (p[:, 0] - mx) * kx v = (p[:, 1] - my) * ky placed_strokes.append(np.stack([cx + u * cos_a - v * sin_a, cy + u * sin_a + v * cos_a, p[:, 3]], 1)) self._push_right(placed_strokes, placed) strokes += [t.tolist() for t in placed_strokes] return strokes, missing def compose(self, tex, rng): """LaTeX -> (strokes, things it could not write). Strokes are None if the formula failed.""" tex = tex.strip().strip("$").strip() try: o = layout([tex])[0] if not o or not o.get("ok"): return None, ["LAYOUT"] glyphs, extra = self._own_marks(_fix_limits(o["glyphs"]), rng) strokes, missing = self._compose_layout(glyphs, rng) strokes += extra except Exception as e: return None, ["ERROR " + repr(e)[:80]] if not strokes: return None, missing or ["EMPTY"] scale = LAYOUT_SCALE * 1000.0 return [[[p[0] / scale, p[1] / scale] for p in stroke] for stroke in strokes], missing def mixed_line(self, seq, tokens, durations, rng): """The text model wrote the whole line, `$...$` included. Its points of each formula are replaced by the composed formula, and the text after it moves right. -> (line [n, 3], things it could not write, formulas replaced); line None on failure.""" q = np.asarray(seq, np.float32) char_of_point = np.repeat(np.arange(len(tokens)), [max(1, int(x)) for x in durations]) if len(char_of_point) != len(q): return None, ["PLAN"], 0 a_ = np.cumsum(q[:, :2], 0) pen = q[:, 2].copy() spans = _dollar_spans(tokens) if not spans: return q, [], 0 in_formula = np.zeros(len(q), bool) for a, b in spans: in_formula |= (char_of_point >= a) & (char_of_point < b) is_text = ~in_formula & np.array([not tokens[c].isspace() for c in char_of_point]) if is_text.sum() < 10: return None, ["NOTEXT"], 0 y = a_[is_text, 1] baseline = float(np.percentile(y, 85)) xh = max(float(np.percentile(y, 85) - np.percentile(y, 15)), 1e-3) gap = 0.6 * xh k = xh / X_HEIGHT_EM pieces = [] shift = 0.0 missing = [] prev = 0 replaced = 0 for a, b in spans: ix = np.where((char_of_point >= a) & (char_of_point < b))[0] before = np.arange(prev, ix[0]) pieces.append((a_[before] + [shift, 0], pen[before])) x_left = (a_[before, 0].max() + shift) if len(before) else a_[ix, 0].min() + shift strokes, ms = self.compose("".join(tokens[a:b]).strip("$"), rng) missing += ms if not strokes: # keep the text model's own points of the span pieces.append((a_[ix] + [shift, 0], pen[ix])) prev = ix[-1] + 1 continue f = to_points(strokes, k) f[:, 0] += x_left + gap - f[:, 0].min() f[:, 1] += baseline if len(before): pieces[-1][1][-1] = 1.0 # lift the pen before the formula pieces.append((f[:, :2], f[:, 2])) replaced += 1 after = np.arange(ix[-1] + 1, len(q)) x_right = a_[after, 0].min() if len(after) else None if x_right is not None: shift = f[:, 0].max() + gap - x_right prev = ix[-1] + 1 rest = np.arange(prev, len(q)) pieces.append((a_[rest] + [shift, 0], pen[rest])) points = np.concatenate([p for p, _ in pieces if len(p)]) pens = np.concatenate([p for _, p in pieces if len(p)]) pens[-1] = 1.0 d = np.zeros((len(points), 3), np.float32) d[1:, :2] = points[1:] - points[:-1] d[0, :2] = 0 d[:, 2] = pens return d, missing, replaced def _dollar_spans(tokens): """Character spans of $...$ and $$...$$.""" out, start, k = [], None, 0 while k < len(tokens): if tokens[k] == "$": n = 2 if k + 1 < len(tokens) and tokens[k + 1] == "$" else 1 if start is None: start = k else: out.append((start, k + n)) start = None k += n continue k += 1 return out def to_points(strokes, k): """Composed strokes, scaled by k -> absolute points [n, 3]: x, y, pen_up, every POINT_STEP along the path.""" out = [] for stroke in strokes: p = np.asarray(stroke, np.float32)[:, :2] * k if len(p) == 1: out.append(np.array([[p[0, 0], p[0, 1], 1.0]], np.float32)) continue cum = np.r_[0, np.cumsum(np.hypot(*np.diff(p, axis=0).T))] s = np.r_[np.arange(0, cum[-1], POINT_STEP), cum[-1]] if cum[-1] > 1e-6 else np.array([0.0, cum[-1]]) r = np.stack([np.interp(s, cum, p[:, 0]), np.interp(s, cum, p[:, 1]), np.zeros(len(s))], 1).astype(np.float32) r[-1, 2] = 1 out.append(r) return np.concatenate(out)