| """Hand-weighted text-to-image model. |
| |
| Every weight in this file was written by hand or derived from geometry. |
| No dataset, no training, no gradients. |
| |
| python handweights.py "a sleepy orange cat with a red balloon on the grass" |
| python handweights.py --demo |
| python handweights.py --story "The bunny was happy. She found a yellow balloon. Then it started to rain." |
| |
| Architecture |
| 1. Embedding table word -> vector (shape / colour / position / size / expression / scene slots) |
| 2. Gated recurrent binder adjectives carry forward into the next object token, position phrases |
| ("on the left") bind back to the last object, and after a copula |
| ("the cat was sleepy") adjectives bind back too |
| 3. Slot decoder linear layers with default-colour / default-size matrices |
| 4. Per-pixel renderer layer 1: half-plane neurons (w = edge normal, b = edge offset) |
| layer 2: min-pool (AND) / OR / AND-NOT neurons that build shapes from half-planes |
| layer 3: shading neuron + alpha-compositing gate; outlines are the same |
| shape with every bias grown by ~2 pixels, drawn first in ink |
| """ |
| import re |
| import sys |
| from pathlib import Path |
|
|
| import numpy as np |
| from PIL import Image, ImageDraw, ImageFont |
|
|
| RES = 384 |
| OUT = Path(__file__).parent / "out" |
|
|
| |
| SHAPES = ["circle", "square", "triangle", "star", "moon", "heart", "house", "tree", "sun", "cloud", |
| "cat", "bunny", "bear", "kid", "balloon", "ball", "flower", "umbrella"] |
| CHARS = {"cat", "bunny", "bear", "kid"} |
| EXPRS = ["happy", "sad", "sleepy", "surprised", "angry"] |
| SKIES = ["day", "night", "sunset", "space", "rain"] |
| GROUNDS = ["sea", "grass", "desert", "snow"] |
|
|
| S0 = 0 |
| OBJ = S0 + len(SHAPES) |
| REF = OBJ + 1 |
| COL = REF + 1 |
| POS = COL + 4 |
| SIZ = POS + 4 |
| EXP = SIZ + 2 |
| PRED = EXP + len(EXPRS) |
| CLR = PRED + 1 |
| SKY = CLR + 1 |
| GND = SKY + len(SKIES) |
| DIM = GND + len(GROUNDS) |
|
|
| VOCAB = {} |
|
|
|
|
| def word(names, entries): |
| v = np.zeros(DIM) |
| for i, x in entries: |
| v[i] = x |
| for n in names.split(): |
| VOCAB[n] = v |
|
|
|
|
| for i, s in enumerate(SHAPES): |
| word(s, [(S0 + i, 1), (OBJ, 1)]) |
| for syn, s in [("dot planet", "circle"), ("box cube block", "square"), ("pyramid", "triangle"), |
| ("home cottage hut", "house"), ("love", "heart"), ("bush", "tree"), |
| ("kitty kitten", "cat"), ("rabbit bunnie", "bunny"), ("teddy", "bear"), |
| ("child children boy girl", "kid"), ("rose daisy", "flower")]: |
| word(syn, [(S0 + SHAPES.index(s), 1), (OBJ, 1)]) |
| word("he she it him her", [(OBJ, 1), (REF, 1)]) |
| word("they them", [(OBJ, 1), (REF, 2)]) |
|
|
| COLORS = { |
| "red": (0.86, 0.16, 0.16), "green": (0.2, 0.62, 0.25), "blue": (0.2, 0.38, 0.85), |
| "yellow": (0.98, 0.84, 0.2), "orange": (0.98, 0.55, 0.15), "purple": (0.55, 0.28, 0.75), |
| "pink": (0.97, 0.55, 0.7), "white": (0.96, 0.96, 0.96), "black": (0.1, 0.1, 0.12), |
| "brown": (0.5, 0.32, 0.16), "gray": (0.55, 0.55, 0.58), "grey": (0.55, 0.55, 0.58), |
| "cyan": (0.2, 0.8, 0.85), "gold": (0.9, 0.72, 0.2), "silver": (0.78, 0.8, 0.84), |
| } |
| for name, (r, g, b) in COLORS.items(): |
| word(name, [(COL, r), (COL + 1, g), (COL + 2, b), (COL + 3, 1)]) |
|
|
| for names, axis, val in [("left", 0, -0.5), ("right", 0, 0.5), ("top upper above", 1, -0.5), |
| ("bottom lower below", 1, 0.5)]: |
| word(names, [(POS + axis, val), (POS + 2 + axis, 1)]) |
| word("center centre middle", [(POS + 2, 1), (POS + 3, 1)]) |
|
|
| for names, s in [("tiny", 0.1), ("small little", 0.15), ("medium", 0.22), ("big large", 0.32), |
| ("huge giant enormous", 0.42)]: |
| word(names, [(SIZ, s), (SIZ + 1, 1)]) |
|
|
| for names, e in [("happy smiling glad joyful cheerful", "happy"), ("sad crying unhappy upset lonely", "sad"), |
| ("sleepy tired asleep sleeping", "sleepy"), ("surprised shocked amazed scared", "surprised"), |
| ("angry mad grumpy cross", "angry")]: |
| word(names, [(EXP + EXPRS.index(e), 1)]) |
| word("is was are were looks looked feels felt seems seemed became got", [(PRED, 1)]) |
| word("and with but then while so", [(CLR, 1)]) |
|
|
| word("day sunny morning", [(SKY + 0, 1)]) |
| word("sky", [(SKY + 0, 0.5)]) |
| word("night starry", [(SKY + 1, 1)]) |
| word("sunset dusk evening", [(SKY + 2, 1)]) |
| word("space", [(SKY + 3, 1)]) |
| word("rain rainy raining storm stormy", [(SKY + 4, 1)]) |
| word("sea ocean water lake beach", [(GND + 0, 1)]) |
| word("grass field meadow park garden", [(GND + 1, 1)]) |
| word("desert sand dunes", [(GND + 2, 1)]) |
| word("snow snowy winter", [(GND + 3, 1)]) |
|
|
| FWD = np.zeros(DIM); FWD[[COL, COL + 1, COL + 2, COL + 3, SIZ, SIZ + 1]] = 1; FWD[EXP:PRED] = 1 |
| BACK = np.zeros(DIM); BACK[POS:POS + 4] = 1 |
| SCN = np.zeros(DIM); SCN[SKY:] = 1 |
|
|
|
|
| def lookup(t): |
| if t in VOCAB: |
| return VOCAB[t] |
| if t.endswith("s") and t[:-1] in VOCAB: |
| return VOCAB[t[:-1]] |
| return None |
|
|
|
|
| |
| def encode(prompt): |
| pend = np.zeros(DIM) |
| pred = 0.0 |
| slots, scene, unknown = [], np.zeros(DIM), [] |
| for t in re.findall(r"[a-z]+", prompt.lower()): |
| e = lookup(t) |
| if e is None: |
| unknown.append(t) |
| continue |
| g = e[OBJ] |
| pred = max(pred * (1 - g) * (1 - e[CLR]), e[PRED]) |
| has_last = 1.0 if slots else 0.0 |
| to_back = pred * has_last |
| if g: |
| slots.append(pend + e) |
| elif has_last: |
| slots[-1] = slots[-1] + BACK * e + to_back * FWD * e |
| pend = (1 - g) * (pend + (1 - to_back) * FWD * e + (1 - has_last) * BACK * e) |
| scene += SCN * e |
| return slots, scene, unknown |
|
|
|
|
| |
| DEF_COLOR = np.array([ |
| (0.2, 0.38, 0.85), (0.86, 0.16, 0.16), (0.98, 0.55, 0.15), (0.98, 0.84, 0.2), |
| (0.96, 0.93, 0.75), (0.9, 0.2, 0.35), (0.93, 0.87, 0.75), (0.2, 0.58, 0.25), |
| (0.99, 0.8, 0.2), (0.97, 0.97, 0.98), |
| (0.98, 0.6, 0.25), (0.95, 0.94, 0.96), (0.62, 0.42, 0.25), (0.25, 0.55, 0.9), |
| (0.9, 0.2, 0.25), (0.95, 0.3, 0.25), (0.97, 0.5, 0.7), (0.3, 0.5, 0.9)]) |
| DEF_SIZE = np.array([0.22, 0.2, 0.22, 0.2, 0.16, 0.2, 0.26, 0.24, 0.16, 0.18, |
| 0.34, 0.34, 0.34, 0.34, 0.15, 0.1, 0.1, 0.2]) |
| |
| DEF_Y = np.array([0.0, 0.0, 0.0, -0.5, -0.55, 0.0, None, None, -0.55, -0.6, |
| "char", "char", "char", "char", -0.2, 0.45, 0.3, -0.1], dtype=object) |
|
|
| |
| PROPS = {"balloon": ("hand", (-1.4, 3.0), 0.45), "ball": ("foot", (0.0, 1.0), 0.25), |
| "flower": ("hand", (0.0, 1.6), 0.24), "umbrella": ("hand", (0.0, 2.4), 0.7)} |
| HAND = {"cat": (0.45, 0.35), "bunny": (0.45, 0.35), "bear": (0.5, 0.35), "kid": (0.56, 0.52)} |
| FOOT = (0.75, 1.0) |
|
|
|
|
| def decode(slot): |
| ref = slot[REF] |
| k = int(np.argmax(slot[S0:OBJ])) |
| onehot = slot[S0:OBJ] / max(slot[S0:OBJ].sum(), 1) |
| hc = min(slot[COL + 3], 1) |
| color = hc * slot[COL:COL + 3] / max(slot[COL + 3], 1) + (1 - hc) * onehot @ DEF_COLOR |
| hs = min(slot[SIZ + 1], 1) |
| size = hs * slot[SIZ] / max(slot[SIZ + 1], 1) + (1 - hs) * onehot @ DEF_SIZE |
| x = slot[POS] * 1.2 if slot[POS + 2] else None |
| y = slot[POS + 1] * 1.2 if slot[POS + 3] else None |
| ex = slot[EXP:PRED] |
| return dict(kind="ref" if ref else SHAPES[k], ref=int(ref), color=np.clip(color, 0, 1), size=size, x=x, y=y, |
| has_color=bool(hc), has_size=bool(hs), has_expr=bool(ex.any()), expr=EXPRS[int(np.argmax(ex))] if ex.any() else "happy") |
|
|
|
|
| def layout(objs): |
| for i, o in enumerate(objs): |
| if i and o["kind"] in PROPS and objs[i - 1]["kind"] in CHARS and objs[i - 1]["x"] is None and o["x"] is not None: |
| objs[i - 1]["x"], objs[i - 1]["y"], o["x"], o["y"] = o["x"], o["y"] if objs[i - 1]["y"] is None else objs[i - 1]["y"], None, None |
| held = [i > 0 and o["kind"] in PROPS and o["x"] is None and objs[i - 1]["kind"] in CHARS |
| for i, o in enumerate(objs)] |
| free = [o for o, h in zip(objs, held) if o["x"] is None and not h] |
| for o, x in zip(free, np.linspace(-0.55, 0.55, len(free)) if len(free) > 1 else [0.0] * len(free)): |
| o["x"] = x |
| for o, h in zip(objs, held): |
| if h or o["y"] is not None: |
| continue |
| d = DEF_Y[SHAPES.index(o["kind"])] |
| o["y"] = 0.55 - 0.9 * o["size"] if d is None else 0.62 - o["size"] if d == "char" else d |
| for i, (o, h) in enumerate(zip(objs, held)): |
| if h: |
| c = objs[i - 1] |
| anchor, grip, scale = PROPS[o["kind"]] |
| ax, ay = HAND[c["kind"]] if anchor == "hand" else FOOT |
| o["size"] = c["size"] * scale |
| o["x"] = c["x"] + ax * c["size"] - grip[0] * o["size"] |
| o["y"] = c["y"] + ay * c["size"] - grip[1] * o["size"] |
| else: |
| o["x"], o["y"] = float(np.clip(o["x"], -0.85, 0.85)), float(np.clip(o["y"], -0.85, 0.85)) |
| return objs |
|
|
|
|
| |
| |
| def circle(cx, cy, r, n=32): |
| a = np.linspace(0, 2 * np.pi, n, endpoint=False) |
| return ("and", np.stack([np.cos(a), np.sin(a), r + np.cos(a) * cx + np.sin(a) * cy], 1)) |
|
|
|
|
| def ellipse(cx, cy, rx, ry, rot=0.0, n=32): |
| t = np.linspace(0, 2 * np.pi, n, endpoint=False) |
| nx, ny = np.cos(t) / rx, np.sin(t) / ry |
| norm = np.hypot(nx, ny) |
| nx, ny = nx / norm, ny / norm |
| d = nx * rx * np.cos(t) + ny * ry * np.sin(t) |
| c, s = np.cos(rot), np.sin(rot) |
| rx_, ry_ = c * nx - s * ny, s * nx + c * ny |
| return ("and", np.stack([rx_, ry_, d + rx_ * cx + ry_ * cy], 1)) |
|
|
|
|
| def halfplane(nx, ny, d): |
| return ("and", np.array([[nx, ny, d]], float)) |
|
|
|
|
| def poly(verts): |
| v = np.array(verts, float) |
| c = v.mean(0) |
| rows = [] |
| for a, b in zip(v, np.roll(v, -1, 0)): |
| n = np.array([b[1] - a[1], a[0] - b[0]]) |
| n /= np.linalg.norm(n) |
| if n @ (c - a) > 0: |
| n = -n |
| rows.append((n[0], n[1], n @ a)) |
| return ("and", np.array(rows)) |
|
|
|
|
| def rect(cx, cy, w, h): |
| return poly([(cx - w / 2, cy - h / 2), (cx + w / 2, cy - h / 2), (cx + w / 2, cy + h / 2), (cx - w / 2, cy + h / 2)]) |
|
|
|
|
| def seg(x1, y1, x2, y2, w): |
| d = np.array([x2 - x1, y2 - y1], float) |
| p = np.array([-d[1], d[0]]) / np.linalg.norm(d) * w / 2 |
| a, b = np.array([x1, y1]), np.array([x2, y2]) |
| return poly([a - p, b - p, b + p, a + p]) |
|
|
|
|
| def arc(cx, cy, r, w, side, cut=0.0): |
| """Ring of width w, keeping only the part below (smile) or above (frown) the centre.""" |
| ring = ("minus", [circle(cx, cy, r + w / 2), circle(cx, cy, r - w / 2)]) |
| keep = halfplane(0, -1, -(cy + cut * r)) if side == "lower" else halfplane(0, 1, cy - cut * r) |
| return ("both", [ring, keep]) |
|
|
|
|
| def star5(): |
| out = [(np.cos(np.radians(-90 + 72 * k)), np.sin(np.radians(-90 + 72 * k))) for k in range(5)] |
| inn = [(0.4 * np.cos(np.radians(-54 + 72 * k)), 0.4 * np.sin(np.radians(-54 + 72 * k))) for k in range(5)] |
| points = [poly([out[k], inn[k], inn[k - 1]]) for k in range(5)] |
| return ("or", [poly(inn)] + points) |
|
|
|
|
| def sun_rays(): |
| rays = [] |
| for k in range(12): |
| a = np.radians(30 * k) |
| d, p = np.array([np.cos(a), np.sin(a)]), np.array([-np.sin(a), np.cos(a)]) |
| rays.append(poly([1.15 * d - 0.08 * p, 1.55 * d - 0.08 * p, 1.55 * d + 0.08 * p, 1.15 * d + 0.08 * p])) |
| return ("or", rays) |
|
|
|
|
| def P(geom, color="main", alpha=1.0, shaded=True, outline=False): |
| return (geom, color, alpha, shaded, outline) |
|
|
|
|
| ROOF, DOOR, WINDOW, TRUNK = (0.58, 0.2, 0.14), (0.38, 0.22, 0.12), (0.98, 0.92, 0.6), (0.45, 0.28, 0.12) |
| INK, WHITE, PINK, BLUSH = (0.16, 0.12, 0.12), (1.0, 1.0, 1.0), (0.98, 0.62, 0.68), (0.98, 0.5, 0.55) |
| SKIN, HAIR, SHOE, LEAF, TEAR = (0.98, 0.8, 0.65), (0.38, 0.23, 0.12), (0.3, 0.2, 0.18), (0.3, 0.62, 0.25), (0.45, 0.72, 1.0) |
| SIDES = (-1, 1) |
| GROUND_SHADOW = P(ellipse(0, 1.0, 0.62, 0.09), INK, 0.16, False) |
|
|
| |
| PARTS = { |
| "circle": [P(circle(0, 0, 1))], |
| "square": [P(rect(0, 0, 1.7, 1.7))], |
| "triangle": [P(poly([(0, -0.95), (1, 0.75), (-1, 0.75)]))], |
| "star": [P(star5())], |
| "moon": [P(("minus", [circle(0, 0, 1), circle(0.45, -0.3, 0.85)]))], |
| "heart": [P(("or", [circle(-0.42, -0.3, 0.5), circle(0.42, -0.3, 0.5), |
| poly([(-0.88, -0.15), (0.88, -0.15), (0, 1.0)])]))], |
| "house": [P(rect(0, 0.3, 1.4, 1.1)), |
| P(poly([(0, -0.95), (0.95, -0.22), (-0.95, -0.22)]), ROOF), |
| P(rect(0, 0.55, 0.32, 0.6), DOOR, shaded=False), |
| P(rect(-0.42, 0.15, 0.3, 0.3), WINDOW, shaded=False), |
| P(rect(0.42, 0.15, 0.3, 0.3), WINDOW, shaded=False)], |
| "tree": [P(rect(0, 0.55, 0.22, 0.9), TRUNK), |
| P(("or", [circle(0, -0.35, 0.55), circle(-0.38, 0, 0.45), circle(0.38, 0, 0.45)]))], |
| "sun": [P(circle(0, 0, 1.6), alpha=0.22, shaded=False), |
| P(sun_rays(), shaded=False), |
| P(circle(0, 0, 1))], |
| "cloud": [P(("or", [circle(-0.5, 0.1, 0.42), circle(0, -0.15, 0.58), circle(0.52, 0.12, 0.4), |
| rect(0, 0.3, 1.0, 0.35)]))], |
| |
| "cat": [GROUND_SHADOW, |
| P(ellipse(0.62, 0.42, 0.12, 0.42, rot=0.5), outline=True), |
| P(ellipse(0, 0.5, 0.52, 0.5), outline=True), |
| P(ellipse(0, 0.6, 0.3, 0.3), "light", shaded=False), |
| *[P(ellipse(s * 0.25, 0.95, 0.17, 0.1), outline=True) for s in SIDES], |
| *[P(poly([(s * 0.62, -0.45), (s * 0.18, -0.8), (s * 0.55, -1.12)]), outline=True) for s in SIDES], |
| *[P(poly([(s * 0.5, -0.58), (s * 0.3, -0.74), (s * 0.5, -0.95)]), PINK, shaded=False) for s in SIDES], |
| P(ellipse(0, -0.32, 0.62, 0.52), outline=True), |
| P(poly([(-0.07, -0.27), (0.07, -0.27), (0, -0.19)]), PINK, shaded=False), |
| *[P(seg(s * 0.3, -0.2, s * 0.72, -0.27, 0.025), INK, shaded=False) for s in SIDES], |
| *[P(seg(s * 0.3, -0.12, s * 0.72, -0.08, 0.025), INK, shaded=False) for s in SIDES]], |
| "bunny": [GROUND_SHADOW, |
| P(circle(0.5, 0.72, 0.15), "light", outline=True), |
| *[P(ellipse(s * 0.22, -0.98, 0.14, 0.42, rot=s * 0.12), outline=True) for s in SIDES], |
| *[P(ellipse(s * 0.22, -0.96, 0.065, 0.3, rot=s * 0.12), PINK, shaded=False) for s in SIDES], |
| P(ellipse(0, 0.5, 0.5, 0.5), outline=True), |
| P(ellipse(0, 0.6, 0.28, 0.3), "light", shaded=False), |
| *[P(ellipse(s * 0.25, 0.95, 0.2, 0.1), outline=True) for s in SIDES], |
| P(ellipse(0, -0.3, 0.55, 0.48), outline=True), |
| P(ellipse(0, -0.22, 0.06, 0.045), PINK, shaded=False)], |
| "bear": [GROUND_SHADOW, |
| *[P(circle(s * 0.45, -0.72, 0.2), outline=True) for s in SIDES], |
| *[P(circle(s * 0.45, -0.72, 0.1), "light", shaded=False) for s in SIDES], |
| P(ellipse(0, 0.5, 0.58, 0.52), outline=True), |
| P(ellipse(0, 0.58, 0.34, 0.34), "light", shaded=False), |
| *[P(ellipse(s * 0.28, 0.95, 0.2, 0.11), outline=True) for s in SIDES], |
| P(ellipse(0, -0.3, 0.62, 0.55), outline=True), |
| P(ellipse(0, -0.1, 0.27, 0.2), "light", shaded=False), |
| P(ellipse(0, -0.19, 0.09, 0.06), INK, shaded=False)], |
| "kid": [GROUND_SHADOW, |
| *[P(rect(s * 0.15, 0.8, 0.14, 0.4), SKIN, outline=True) for s in SIDES], |
| *[P(ellipse(s * 0.17, 0.98, 0.15, 0.07), SHOE, outline=True) for s in SIDES], |
| *[P(seg(s * 0.28, 0.15, s * 0.55, 0.5, 0.14), SKIN, outline=True) for s in SIDES], |
| *[P(circle(s * 0.56, 0.52, 0.08), SKIN, outline=True) for s in SIDES], |
| P(poly([(-0.3, 0.02), (0.3, 0.02), (0.45, 0.7), (-0.45, 0.7)]), outline=True), |
| P(ellipse(0, -0.3, 0.57, 0.57), HAIR, outline=True), |
| P(ellipse(0, -0.3, 0.5, 0.5), SKIN, outline=True), |
| P(("both", [ellipse(0, -0.34, 0.53, 0.52), halfplane(0, 1, -0.5)]), HAIR)], |
| |
| "balloon": [P(seg(0, 0.95, -1.4, 3.0, 0.04), INK, shaded=False), |
| P(poly([(-0.1, 1.06), (0.1, 1.06), (0, 0.9)]), outline=True), |
| P(ellipse(0, 0, 0.75, 0.95), outline=True), |
| P(ellipse(-0.28, -0.4, 0.12, 0.22, rot=0.5), WHITE, 0.7, False)], |
| "ball": [P(circle(0, 0, 1), outline=True), |
| P(("both", [circle(0, 0, 1), halfplane(0, 1, 0.22), halfplane(0, -1, 0.22)]), WHITE), |
| P(ellipse(-0.4, -0.45, 0.15, 0.1, rot=-0.6), WHITE, 0.7, False)], |
| "flower": [P(seg(0, 0.3, 0, 2.2, 0.12), LEAF, outline=True), |
| P(ellipse(0.32, 1.4, 0.3, 0.12, rot=-0.5), LEAF, outline=True), |
| *[P(circle(0.48 * np.cos(a), 0.48 * np.sin(a), 0.36), outline=True) |
| for a in np.radians(-90 + 72 * np.arange(5))], |
| P(circle(0, 0, 0.3), (0.98, 0.82, 0.25), outline=True)], |
| "umbrella": [P(seg(0, 0, 0, 2.6, 0.08), INK, shaded=False), |
| P(arc(0.2, 2.6, 0.2, 0.09, "lower"), INK, shaded=False), |
| P(("both", [ellipse(0, 0.05, 1.3, 1.0), halfplane(0, 1, 0.05)]), outline=True), |
| P(seg(0, -0.95, 0, -1.2, 0.08), INK, shaded=False)], |
| } |
| CASTS_SHADOW = {"circle", "square", "triangle", "heart", "house", "tree"} |
| EXTENT = {"balloon": 3.4, "umbrella": 3.0, "flower": 2.4, "bunny": 1.6, "sun": 1.7} |
|
|
| HEAD = {"cat": (0, -0.32, 0.55), "bunny": (0, -0.3, 0.5), "bear": (0, -0.3, 0.55), "kid": (0, -0.3, 0.45)} |
|
|
|
|
| def face(hx, hy, hr, expr): |
| ex, ey = 0.36 * hr, hy - 0.05 * hr |
| mx, my = hx, hy + 0.38 * hr |
| eyes = [hx - ex, hx + ex] |
| f = [] |
| if expr == "surprised": |
| for x in eyes: |
| f += [P(circle(x, ey, 0.19 * hr), INK, shaded=False), P(circle(x, ey, 0.15 * hr), WHITE, shaded=False), |
| P(circle(x, ey + 0.02 * hr, 0.07 * hr), INK, shaded=False)] |
| f.append(P(ellipse(mx, my + 0.03 * hr, 0.1 * hr, 0.13 * hr), INK, shaded=False)) |
| elif expr == "sleepy": |
| f += [P(arc(x, ey - 0.08 * hr, 0.11 * hr, 0.045 * hr, "lower", 0.1), INK, shaded=False) for x in eyes] |
| f.append(P(ellipse(mx, my, 0.05 * hr, 0.06 * hr), INK, shaded=False)) |
| zx, zy, s, w = hx + 0.9 * hr, hy - 0.8 * hr, 0.25 * hr, 0.06 * hr |
| f += [P(seg(zx, zy, zx + s, zy, w), INK, shaded=False), P(seg(zx + s, zy, zx, zy + s, w), INK, shaded=False), |
| P(seg(zx, zy + s, zx + s, zy + s, w), INK, shaded=False)] |
| else: |
| for x in eyes: |
| f += [P(ellipse(x, ey, 0.12 * hr, 0.16 * hr), INK, shaded=False), |
| P(circle(x + 0.04 * hr, ey - 0.06 * hr, 0.045 * hr), WHITE, shaded=False)] |
| if expr == "happy": |
| f.append(P(arc(mx, my - 0.25 * hr, 0.25 * hr, 0.06 * hr, "lower", 0.35), INK, shaded=False)) |
| f += [P(ellipse(hx + s * 0.58 * hr, hy + 0.22 * hr, 0.13 * hr, 0.08 * hr), BLUSH, 0.55, False) for s in SIDES] |
| else: |
| f.append(P(arc(mx, my + 0.2 * hr, 0.2 * hr, 0.06 * hr, "upper", 0.35), INK, shaded=False)) |
| for s, x in zip(SIDES, eyes): |
| inner, outer = (ey - 0.24 * hr, ey - 0.32 * hr) if expr == "angry" else (ey - 0.32 * hr, ey - 0.24 * hr) |
| f.append(P(seg(x - s * 0.13 * hr, inner, x + s * 0.13 * hr, outer, 0.05 * hr), INK, shaded=False)) |
| if expr == "sad": |
| f.append(P(ellipse(eyes[1] + 0.06 * hr, ey + 0.25 * hr, 0.05 * hr, 0.08 * hr), TEAR, shaded=False)) |
| return f |
|
|
|
|
| FACES = {(k, e): face(*HEAD[k], e) for k in CHARS for e in EXPRS} |
|
|
|
|
| def sig(z): |
| return 0.5 * (1 + np.tanh(0.5 * z)) |
|
|
|
|
| def mask(g, px, py, sharp, grow=0.0): |
| op, arg = g |
| if op == "and": |
| z = sharp * (arg[:, 2, None, None] + grow - arg[:, 0, None, None] * px - arg[:, 1, None, None] * py) |
| return sig(z.min(0)) |
| ms = [mask(c, px, py, sharp, -grow if op == "minus" and i else grow) for i, c in enumerate(arg)] |
| if op == "or": |
| return sig(12 * (sum(ms) - 0.5)) |
| if op == "both": |
| return sig(12 * (sum(ms) - (len(ms) - 0.5))) |
| return sig(12 * (ms[0] - ms[1] - 0.5)) |
|
|
|
|
| def part_color(spec, main): |
| if isinstance(spec, str): |
| return {"main": main, "light": main * 0.45 + 0.55, "dark": main * 0.65}[spec] |
| return np.array(spec) |
|
|
|
|
| |
| U, V = np.meshgrid(np.linspace(-1, 1, RES), np.linspace(-1, 1, RES)) |
| HORIZON = 0.3 |
|
|
|
|
| def lerp(a, b, t): |
| return np.array(a)[None, None] * (1 - t[..., None]) + np.array(b)[None, None] * t[..., None] |
|
|
|
|
| def hash2(i, j): |
| return np.abs(np.modf(np.sin(i * 12.9898 + j * 78.233) * 43758.5453)[0]) |
|
|
|
|
| def stars(threshold): |
| h = hash2(np.floor((U + 1) * RES / 3), np.floor((V + 1) * RES / 3)) |
| return ((h > threshold) * (h - threshold) / (1 - threshold) * 0.6 + (h > threshold) * 0.4)[..., None] |
|
|
|
|
| def rain(): |
| col = np.floor((U + 0.15 * V + 2) * RES / 9) |
| phase = np.modf((V + 1) * 1.5 + hash2(col, 1.0) * 5)[0] |
| frac = np.modf((U + 0.15 * V + 2) * RES / 9)[0] |
| return ((phase < 0.06) & (frac < 0.2)).astype(float)[..., None] * 0.7 |
|
|
|
|
| def background(sky, ground): |
| t = (V + 1) / 2 |
| if sky is None: |
| img = lerp((0.96, 0.95, 0.92), (0.88, 0.86, 0.82), t) |
| elif sky == "day": |
| img = lerp((0.4, 0.66, 0.95), (0.82, 0.91, 1.0), t) |
| elif sky == "night": |
| img = lerp((0.02, 0.03, 0.12), (0.12, 0.12, 0.32), t) |
| img = np.clip(img + stars(0.985), 0, 1) |
| elif sky == "sunset": |
| img = lerp((0.22, 0.18, 0.45), (0.95, 0.45, 0.35), np.clip(t / 0.6, 0, 1)) |
| img = img + lerp((0, 0, 0), (0.05, 0.35, 0.1), np.clip((t - 0.6) / 0.4, 0, 1)) |
| elif sky == "rain": |
| img = lerp((0.5, 0.55, 0.62), (0.74, 0.77, 0.82), t) |
| else: |
| img = np.clip(lerp((0.0, 0.0, 0.02), (0.04, 0.02, 0.08), t) + stars(0.97), 0, 1) |
| if ground is not None: |
| wiggle = {"sea": 0 * U, "grass": 0.025 * np.sin(4 * U), "desert": 0.06 * np.sin(3 * U + 1), |
| "snow": 0.035 * np.sin(2.5 * U + 2)}[ground] |
| g = np.clip((V - HORIZON) / (1 - HORIZON), 0, 1) |
| if ground == "sea": |
| col = lerp((0.25, 0.55, 0.75), (0.04, 0.2, 0.42), g) + 0.05 * np.sin(70 * V + 4 * np.sin(7 * U))[..., None] |
| elif ground == "grass": |
| col = lerp((0.5, 0.74, 0.32), (0.2, 0.45, 0.15), g) + 0.025 * np.sin(150 * U + 20 * V)[..., None] |
| elif ground == "desert": |
| col = lerp((0.96, 0.82, 0.52), (0.84, 0.6, 0.34), g) + 0.04 * np.sin(9 * U - 14 * V)[..., None] |
| else: |
| col = lerp((0.98, 0.99, 1.0), (0.82, 0.88, 0.97), g) |
| m = sig(RES * 0.6 * (V - HORIZON - wiggle))[..., None] |
| img = img * (1 - m) + col * m |
| return np.clip(img, 0, 1) |
|
|
|
|
| |
| class Story: |
| """Carries scene, character colours and the last cast across sentences, and resolves pronouns.""" |
|
|
| def __init__(self): |
| self.colors, self.cast, self.sky, self.ground = {}, [], "day", "grass" |
|
|
| def resolve(self, objs): |
| out = [] |
| for o in objs: |
| if o["kind"] == "ref": |
| for c in (self.cast[-1:] if o["ref"] == 1 else self.cast): |
| out.append(dict(c, expr=o["expr"], has_expr=o["has_expr"], x=o["x"], y=o["y"], size=o["size"] if o["has_size"] else c["size"])) |
| continue |
| if o["has_color"]: |
| self.colors[o["kind"]] = o["color"] |
| elif o["kind"] in self.colors: |
| o["color"] = self.colors[o["kind"]] |
| out.append(o) |
| first, dedup = {}, [] |
| for o in out: |
| if o["kind"] in CHARS and o["kind"] in first: |
| if o.get("has_expr"): |
| first[o["kind"]]["expr"] = o["expr"] |
| continue |
| first[o["kind"]] = o |
| dedup.append(o) |
| cast = [dict(kind=o["kind"], ref=0, color=o["color"], size=o["size"], has_color=True, has_size=False) |
| for o in dedup if o["kind"] in CHARS] |
| if cast: |
| self.cast = cast |
| return dedup |
|
|
|
|
| def split_story(text): |
| return [s.strip() for s in re.split(r"(?<=[.!?])\s+|\n+", text.strip()) if s.strip()] |
|
|
|
|
| |
| def render(prompt, story=None): |
| slots, scene, unknown = encode(prompt) |
| objs = [decode(s) for s in slots] |
| objs = story.resolve(objs) if story else [o for o in objs if o["kind"] != "ref"] |
| objs = layout(objs) |
| sky = SKIES[int(np.argmax(scene[SKY:GND]))] if scene[SKY:GND].any() else None |
| ground = GROUNDS[int(np.argmax(scene[GND:]))] if scene[GND:].any() else None |
| if story: |
| sky, ground = sky or story.sky, ground or story.ground |
| story.sky, story.ground = sky, ground |
| if ground and sky is None: |
| sky = "day" |
| img = background(sky, ground) |
| for o in objs: |
| r = o["size"] |
| ext = EXTENT.get(o["kind"], 1.8) * r |
| i0, i1 = max(0, int((o["y"] - ext + 1) / 2 * RES) - 3), min(RES, int((o["y"] + ext + 1) / 2 * RES) + 4) |
| j0, j1 = max(0, int((o["x"] - ext + 1) / 2 * RES) - 3), min(RES, int((o["x"] + ext + 1) / 2 * RES) + 4) |
| if i0 >= i1 or j0 >= j1: |
| continue |
| sub = img[i0:i1, j0:j1] |
| sharp = 4.5 * r * RES / 2 |
| ink = 2.4 * 2 / (RES * r) |
| px, py = (U[i0:i1, j0:j1] - o["x"]) / r, (V[i0:i1, j0:j1] - o["y"]) / r |
| parts = PARTS[o["kind"]] + FACES.get((o["kind"], o["expr"]), []) |
| if o["kind"] in CASTS_SHADOW: |
| sh = sum(mask(g, px - 0.12, py - 0.12, sharp) * a for g, _, a, _, _ in parts) |
| sub = sub * (1 - np.clip(sh, 0, 1)[..., None] * 0.22) |
| for geom, col, alpha, shaded, outline in parts: |
| if outline: |
| m = mask(geom, px, py, sharp, ink)[..., None] |
| sub = sub * (1 - m) + np.array(INK) * m |
| c = part_color(col, o["color"])[None, None] |
| if shaded: |
| c = c * (1 - 0.16 * (px + py))[..., None] |
| m = (mask(geom, px, py, sharp) * alpha)[..., None] |
| sub = sub * (1 - m) + np.clip(c, 0, 1) * m |
| img[i0:i1, j0:j1] = sub |
| if sky == "rain": |
| drops = rain() |
| img = img * (1 - drops) + np.array([0.85, 0.9, 1.0]) * drops |
| return Image.fromarray((np.clip(img, 0, 1) * 255).astype(np.uint8)), objs, sky, ground, unknown |
|
|
|
|
| def render_story(text): |
| story = Story() |
| return [(s, render(s, story)) for s in split_story(text)] |
|
|
|
|
| def count_params(): |
| def geo(g): |
| return g[1].size if g[0] == "and" else sum(geo(c) for c in g[1]) |
| parts = [p for ps in PARTS.values() for p in ps] + [p for ps in FACES.values() for p in ps] |
| return (len(VOCAB) * DIM + DEF_COLOR.size + DEF_SIZE.size + len(DEF_Y) |
| + sum(geo(g) + 3 for g, *_ in parts) + 3 * len(PROPS) + 2 * len(HAND)) |
|
|
|
|
| def slug(p): |
| return re.sub(r"[^a-z0-9]+", "_", p.lower()).strip("_")[:60] |
|
|
|
|
| def caption_font(size=20): |
| for f in ["C:/Windows/Fonts/comic.ttf", "C:/Windows/Fonts/segoeui.ttf", "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"]: |
| try: |
| return ImageFont.truetype(f, size) |
| except OSError: |
| pass |
| return ImageFont.load_default(size=size) |
|
|
|
|
| def book_sheet(pages, cols=3, cap=64): |
| font = caption_font() |
| sheet = Image.new("RGB", (cols * (RES + 16) + 16, ((len(pages) + cols - 1) // cols) * (RES + cap + 16) + 16), "white") |
| d = ImageDraw.Draw(sheet) |
| for i, (text, im) in enumerate(pages): |
| x, y = 16 + (i % cols) * (RES + 16), 16 + (i // cols) * (RES + cap + 16) |
| sheet.paste(im, (x, y)) |
| lines, line = [], "" |
| for w in text.split(): |
| if d.textlength(line + " " + w, font=font) > RES - 8 and line: |
| lines.append(line) |
| line = w |
| else: |
| line = (line + " " + w).strip() |
| d.multiline_text((x + 4, y + RES + 8), "\n".join(lines + [line]), fill="black", font=font, spacing=4) |
| return sheet |
|
|
|
|
| DEMO = [ |
| "a yellow sun over the sea", |
| "a red house and a green tree on grass", |
| "a white moon and a gold star in a night sky", |
| "an orange sun and a cloud over the desert at sunset", |
| "a happy orange cat with a red balloon on the grass", |
| "a sleepy bear and a surprised bunny at night", |
| "a kid with a blue umbrella in the rain", |
| "a sad white bunny with a pink flower in the snow", |
| ] |
|
|
| STORY = ("The orange cat was happy in the meadow. She found a red balloon. Then it started to rain and she was sad. " |
| "A kid with a yellow umbrella came by. The cat and the kid were happy together. " |
| "At night the cat was sleepy under the moon.") |
|
|
| if __name__ == "__main__": |
| OUT.mkdir(exist_ok=True) |
| print(f"hand-written parameters: {count_params():,}") |
| if sys.argv[1:2] == ["--story"]: |
| pages = render_story(" ".join(sys.argv[2:]) or STORY) |
| for s, (_, objs, sky, ground, unknown) in pages: |
| print(f"\n{s!r}\n sky={sky} ground={ground} " + ", ".join(f"{o['expr']} {o['kind']}" if o["kind"] in CHARS |
| else o["kind"] for o in objs)) |
| book_sheet([(s, r[0]) for s, r in pages]).save(OUT / "story_sheet.png") |
| print(f"\nbook -> {OUT / 'story_sheet.png'}") |
| sys.exit() |
| prompts = DEMO if sys.argv[1:] == ["--demo"] else [" ".join(sys.argv[1:]) or DEMO[0]] |
| tiles = [] |
| for p in prompts: |
| im, objs, sky, ground, unknown = render(p) |
| im.save(OUT / f"{slug(p)}.png") |
| tiles.append((p, im)) |
| print(f"\n{p!r}\n sky={sky} ground={ground} ignored={unknown}") |
| for o in objs: |
| print(f" {o['kind']:<8} {o['expr'] if o['kind'] in CHARS else '':<9} rgb={np.round(o['color'], 2)} " |
| f"size={o['size']:.2f} at ({o['x']:+.2f},{o['y']:+.2f})") |
| if len(tiles) > 1: |
| book_sheet(tiles, cols=4).save(OUT / "demo_sheet.png") |
| print(f"\nsheet -> {OUT / 'demo_sheet.png'}") |
|
|
