"""Hand-weighted text-to-image model. Every weight in this file was written by hand or derived from geometry. No dataset, no training, no gradients. python handweights.py "a sleepy orange cat with a red balloon on the grass" python handweights.py --demo python handweights.py --story "The bunny was happy. She found a yellow balloon. Then it started to rain." Architecture 1. Embedding table word -> vector (shape / colour / position / size / expression / scene slots) 2. Gated recurrent binder adjectives carry forward into the next object token, position phrases ("on the left") bind back to the last object, and after a copula ("the cat was sleepy") adjectives bind back too 3. Slot decoder linear layers with default-colour / default-size matrices 4. Per-pixel renderer layer 1: half-plane neurons (w = edge normal, b = edge offset) layer 2: min-pool (AND) / OR / AND-NOT neurons that build shapes from half-planes layer 3: shading neuron + alpha-compositing gate; outlines are the same shape with every bias grown by ~2 pixels, drawn first in ink """ import re import sys from pathlib import Path import numpy as np from PIL import Image, ImageDraw, ImageFont RES = 384 OUT = Path(__file__).parent / "out" # ------------------------------------------------------------------ embedding table SHAPES = ["circle", "square", "triangle", "star", "moon", "heart", "house", "tree", "sun", "cloud", "cat", "bunny", "bear", "kid", "balloon", "ball", "flower", "umbrella"] CHARS = {"cat", "bunny", "bear", "kid"} EXPRS = ["happy", "sad", "sleepy", "surprised", "angry"] SKIES = ["day", "night", "sunset", "space", "rain"] GROUNDS = ["sea", "grass", "desert", "snow"] S0 = 0 # shape one-hot OBJ = S0 + len(SHAPES) # gate: "this token is an object" REF = OBJ + 1 # pronoun: 1 = he/she/it, 2 = they COL = REF + 1 # r, g, b, has_color POS = COL + 4 # dx, dy, has_x, has_y SIZ = POS + 4 # size, has_size EXP = SIZ + 2 # expression one-hot PRED = EXP + len(EXPRS) # copula: following adjectives bind back CLR = PRED + 1 # conjunction: clears the copula register SKY = CLR + 1 # sky one-hot GND = SKY + len(SKIES) # ground one-hot DIM = GND + len(GROUNDS) VOCAB = {} def word(names, entries): v = np.zeros(DIM) for i, x in entries: v[i] = x for n in names.split(): VOCAB[n] = v for i, s in enumerate(SHAPES): word(s, [(S0 + i, 1), (OBJ, 1)]) for syn, s in [("dot planet", "circle"), ("box cube block", "square"), ("pyramid", "triangle"), ("home cottage hut", "house"), ("love", "heart"), ("bush", "tree"), ("kitty kitten", "cat"), ("rabbit bunnie", "bunny"), ("teddy", "bear"), ("child children boy girl", "kid"), ("rose daisy", "flower")]: word(syn, [(S0 + SHAPES.index(s), 1), (OBJ, 1)]) word("he she it him her", [(OBJ, 1), (REF, 1)]) word("they them", [(OBJ, 1), (REF, 2)]) COLORS = { "red": (0.86, 0.16, 0.16), "green": (0.2, 0.62, 0.25), "blue": (0.2, 0.38, 0.85), "yellow": (0.98, 0.84, 0.2), "orange": (0.98, 0.55, 0.15), "purple": (0.55, 0.28, 0.75), "pink": (0.97, 0.55, 0.7), "white": (0.96, 0.96, 0.96), "black": (0.1, 0.1, 0.12), "brown": (0.5, 0.32, 0.16), "gray": (0.55, 0.55, 0.58), "grey": (0.55, 0.55, 0.58), "cyan": (0.2, 0.8, 0.85), "gold": (0.9, 0.72, 0.2), "silver": (0.78, 0.8, 0.84), } for name, (r, g, b) in COLORS.items(): word(name, [(COL, r), (COL + 1, g), (COL + 2, b), (COL + 3, 1)]) for names, axis, val in [("left", 0, -0.5), ("right", 0, 0.5), ("top upper above", 1, -0.5), ("bottom lower below", 1, 0.5)]: word(names, [(POS + axis, val), (POS + 2 + axis, 1)]) word("center centre middle", [(POS + 2, 1), (POS + 3, 1)]) for names, s in [("tiny", 0.1), ("small little", 0.15), ("medium", 0.22), ("big large", 0.32), ("huge giant enormous", 0.42)]: word(names, [(SIZ, s), (SIZ + 1, 1)]) for names, e in [("happy smiling glad joyful cheerful", "happy"), ("sad crying unhappy upset lonely", "sad"), ("sleepy tired asleep sleeping", "sleepy"), ("surprised shocked amazed scared", "surprised"), ("angry mad grumpy cross", "angry")]: word(names, [(EXP + EXPRS.index(e), 1)]) word("is was are were looks looked feels felt seems seemed became got", [(PRED, 1)]) word("and with but then while so", [(CLR, 1)]) word("day sunny morning", [(SKY + 0, 1)]) word("sky", [(SKY + 0, 0.5)]) # "sky" leans day but loses to "night sky" word("night starry", [(SKY + 1, 1)]) word("sunset dusk evening", [(SKY + 2, 1)]) word("space", [(SKY + 3, 1)]) word("rain rainy raining storm stormy", [(SKY + 4, 1)]) word("sea ocean water lake beach", [(GND + 0, 1)]) word("grass field meadow park garden", [(GND + 1, 1)]) word("desert sand dunes", [(GND + 2, 1)]) word("snow snowy winter", [(GND + 3, 1)]) FWD = np.zeros(DIM); FWD[[COL, COL + 1, COL + 2, COL + 3, SIZ, SIZ + 1]] = 1; FWD[EXP:PRED] = 1 # to next object BACK = np.zeros(DIM); BACK[POS:POS + 4] = 1 # to last object SCN = np.zeros(DIM); SCN[SKY:] = 1 # pooled scene def lookup(t): if t in VOCAB: return VOCAB[t] if t.endswith("s") and t[:-1] in VOCAB: return VOCAB[t[:-1]] return None # ------------------------------------------------------------------ gated recurrent binder def encode(prompt): pend = np.zeros(DIM) # adjectives waiting for an object pred = 0.0 # copula register: "was" makes the next adjectives bind back slots, scene, unknown = [], np.zeros(DIM), [] for t in re.findall(r"[a-z]+", prompt.lower()): e = lookup(t) if e is None: unknown.append(t) continue g = e[OBJ] pred = max(pred * (1 - g) * (1 - e[CLR]), e[PRED]) has_last = 1.0 if slots else 0.0 to_back = pred * has_last if g: slots.append(pend + e) elif has_last: slots[-1] = slots[-1] + BACK * e + to_back * FWD * e pend = (1 - g) * (pend + (1 - to_back) * FWD * e + (1 - has_last) * BACK * e) scene += SCN * e return slots, scene, unknown # ------------------------------------------------------------------ slot decoder weights DEF_COLOR = np.array([ (0.2, 0.38, 0.85), (0.86, 0.16, 0.16), (0.98, 0.55, 0.15), (0.98, 0.84, 0.2), (0.96, 0.93, 0.75), (0.9, 0.2, 0.35), (0.93, 0.87, 0.75), (0.2, 0.58, 0.25), (0.99, 0.8, 0.2), (0.97, 0.97, 0.98), (0.98, 0.6, 0.25), (0.95, 0.94, 0.96), (0.62, 0.42, 0.25), (0.25, 0.55, 0.9), (0.9, 0.2, 0.25), (0.95, 0.3, 0.25), (0.97, 0.5, 0.7), (0.3, 0.5, 0.9)]) DEF_SIZE = np.array([0.22, 0.2, 0.22, 0.2, 0.16, 0.2, 0.26, 0.24, 0.16, 0.18, 0.34, 0.34, 0.34, 0.34, 0.15, 0.1, 0.1, 0.2]) # None: stands on the ground line like a house; "char": stands on it with feet at local y = 1 DEF_Y = np.array([0.0, 0.0, 0.0, -0.5, -0.55, 0.0, None, None, -0.55, -0.6, "char", "char", "char", "char", -0.2, 0.45, 0.3, -0.1], dtype=object) # props held by the previous character: (anchor on the character, grip point on the prop, scale) PROPS = {"balloon": ("hand", (-1.4, 3.0), 0.45), "ball": ("foot", (0.0, 1.0), 0.25), "flower": ("hand", (0.0, 1.6), 0.24), "umbrella": ("hand", (0.0, 2.4), 0.7)} HAND = {"cat": (0.45, 0.35), "bunny": (0.45, 0.35), "bear": (0.5, 0.35), "kid": (0.56, 0.52)} FOOT = (0.75, 1.0) def decode(slot): ref = slot[REF] k = int(np.argmax(slot[S0:OBJ])) onehot = slot[S0:OBJ] / max(slot[S0:OBJ].sum(), 1) hc = min(slot[COL + 3], 1) color = hc * slot[COL:COL + 3] / max(slot[COL + 3], 1) + (1 - hc) * onehot @ DEF_COLOR hs = min(slot[SIZ + 1], 1) size = hs * slot[SIZ] / max(slot[SIZ + 1], 1) + (1 - hs) * onehot @ DEF_SIZE x = slot[POS] * 1.2 if slot[POS + 2] else None y = slot[POS + 1] * 1.2 if slot[POS + 3] else None ex = slot[EXP:PRED] return dict(kind="ref" if ref else SHAPES[k], ref=int(ref), color=np.clip(color, 0, 1), size=size, x=x, y=y, has_color=bool(hc), has_size=bool(hs), has_expr=bool(ex.any()), expr=EXPRS[int(np.argmax(ex))] if ex.any() else "happy") def layout(objs): for i, o in enumerate(objs): # "a kid with a flower on the left": the place belongs to the kid if i and o["kind"] in PROPS and objs[i - 1]["kind"] in CHARS and objs[i - 1]["x"] is None and o["x"] is not None: objs[i - 1]["x"], objs[i - 1]["y"], o["x"], o["y"] = o["x"], o["y"] if objs[i - 1]["y"] is None else objs[i - 1]["y"], None, None held = [i > 0 and o["kind"] in PROPS and o["x"] is None and objs[i - 1]["kind"] in CHARS for i, o in enumerate(objs)] free = [o for o, h in zip(objs, held) if o["x"] is None and not h] for o, x in zip(free, np.linspace(-0.55, 0.55, len(free)) if len(free) > 1 else [0.0] * len(free)): o["x"] = x for o, h in zip(objs, held): if h or o["y"] is not None: continue d = DEF_Y[SHAPES.index(o["kind"])] o["y"] = 0.55 - 0.9 * o["size"] if d is None else 0.62 - o["size"] if d == "char" else d for i, (o, h) in enumerate(zip(objs, held)): if h: # put the prop's grip point on the character's hand / foot c = objs[i - 1] anchor, grip, scale = PROPS[o["kind"]] ax, ay = HAND[c["kind"]] if anchor == "hand" else FOOT o["size"] = c["size"] * scale o["x"] = c["x"] + ax * c["size"] - grip[0] * o["size"] o["y"] = c["y"] + ay * c["size"] - grip[1] * o["size"] else: o["x"], o["y"] = float(np.clip(o["x"], -0.85, 0.85)), float(np.clip(o["y"], -0.85, 0.85)) return objs # ------------------------------------------------------------------ geometry -> half-plane weights # A half-plane neuron fires where n.p <= d, i.e. w = -n, b = d. Each row is (nx, ny, d). def circle(cx, cy, r, n=32): a = np.linspace(0, 2 * np.pi, n, endpoint=False) return ("and", np.stack([np.cos(a), np.sin(a), r + np.cos(a) * cx + np.sin(a) * cy], 1)) def ellipse(cx, cy, rx, ry, rot=0.0, n=32): t = np.linspace(0, 2 * np.pi, n, endpoint=False) nx, ny = np.cos(t) / rx, np.sin(t) / ry norm = np.hypot(nx, ny) nx, ny = nx / norm, ny / norm d = nx * rx * np.cos(t) + ny * ry * np.sin(t) c, s = np.cos(rot), np.sin(rot) rx_, ry_ = c * nx - s * ny, s * nx + c * ny return ("and", np.stack([rx_, ry_, d + rx_ * cx + ry_ * cy], 1)) def halfplane(nx, ny, d): return ("and", np.array([[nx, ny, d]], float)) def poly(verts): v = np.array(verts, float) c = v.mean(0) rows = [] for a, b in zip(v, np.roll(v, -1, 0)): n = np.array([b[1] - a[1], a[0] - b[0]]) n /= np.linalg.norm(n) if n @ (c - a) > 0: n = -n rows.append((n[0], n[1], n @ a)) return ("and", np.array(rows)) def rect(cx, cy, w, h): return poly([(cx - w / 2, cy - h / 2), (cx + w / 2, cy - h / 2), (cx + w / 2, cy + h / 2), (cx - w / 2, cy + h / 2)]) def seg(x1, y1, x2, y2, w): d = np.array([x2 - x1, y2 - y1], float) p = np.array([-d[1], d[0]]) / np.linalg.norm(d) * w / 2 a, b = np.array([x1, y1]), np.array([x2, y2]) return poly([a - p, b - p, b + p, a + p]) def arc(cx, cy, r, w, side, cut=0.0): """Ring of width w, keeping only the part below (smile) or above (frown) the centre.""" ring = ("minus", [circle(cx, cy, r + w / 2), circle(cx, cy, r - w / 2)]) keep = halfplane(0, -1, -(cy + cut * r)) if side == "lower" else halfplane(0, 1, cy - cut * r) return ("both", [ring, keep]) def star5(): out = [(np.cos(np.radians(-90 + 72 * k)), np.sin(np.radians(-90 + 72 * k))) for k in range(5)] inn = [(0.4 * np.cos(np.radians(-54 + 72 * k)), 0.4 * np.sin(np.radians(-54 + 72 * k))) for k in range(5)] points = [poly([out[k], inn[k], inn[k - 1]]) for k in range(5)] return ("or", [poly(inn)] + points) def sun_rays(): rays = [] for k in range(12): a = np.radians(30 * k) d, p = np.array([np.cos(a), np.sin(a)]), np.array([-np.sin(a), np.cos(a)]) rays.append(poly([1.15 * d - 0.08 * p, 1.55 * d - 0.08 * p, 1.55 * d + 0.08 * p, 1.15 * d + 0.08 * p])) return ("or", rays) def P(geom, color="main", alpha=1.0, shaded=True, outline=False): return (geom, color, alpha, shaded, outline) ROOF, DOOR, WINDOW, TRUNK = (0.58, 0.2, 0.14), (0.38, 0.22, 0.12), (0.98, 0.92, 0.6), (0.45, 0.28, 0.12) INK, WHITE, PINK, BLUSH = (0.16, 0.12, 0.12), (1.0, 1.0, 1.0), (0.98, 0.62, 0.68), (0.98, 0.5, 0.55) SKIN, HAIR, SHOE, LEAF, TEAR = (0.98, 0.8, 0.65), (0.38, 0.23, 0.12), (0.3, 0.2, 0.18), (0.3, 0.62, 0.25), (0.45, 0.72, 1.0) SIDES = (-1, 1) GROUND_SHADOW = P(ellipse(0, 1.0, 0.62, 0.09), INK, 0.16, False) # Each shape is a stack of parts: (geometry, colour, alpha, shaded, outline) PARTS = { "circle": [P(circle(0, 0, 1))], "square": [P(rect(0, 0, 1.7, 1.7))], "triangle": [P(poly([(0, -0.95), (1, 0.75), (-1, 0.75)]))], "star": [P(star5())], "moon": [P(("minus", [circle(0, 0, 1), circle(0.45, -0.3, 0.85)]))], "heart": [P(("or", [circle(-0.42, -0.3, 0.5), circle(0.42, -0.3, 0.5), poly([(-0.88, -0.15), (0.88, -0.15), (0, 1.0)])]))], "house": [P(rect(0, 0.3, 1.4, 1.1)), P(poly([(0, -0.95), (0.95, -0.22), (-0.95, -0.22)]), ROOF), P(rect(0, 0.55, 0.32, 0.6), DOOR, shaded=False), P(rect(-0.42, 0.15, 0.3, 0.3), WINDOW, shaded=False), P(rect(0.42, 0.15, 0.3, 0.3), WINDOW, shaded=False)], "tree": [P(rect(0, 0.55, 0.22, 0.9), TRUNK), P(("or", [circle(0, -0.35, 0.55), circle(-0.38, 0, 0.45), circle(0.38, 0, 0.45)]))], "sun": [P(circle(0, 0, 1.6), alpha=0.22, shaded=False), P(sun_rays(), shaded=False), P(circle(0, 0, 1))], "cloud": [P(("or", [circle(-0.5, 0.1, 0.42), circle(0, -0.15, 0.58), circle(0.52, 0.12, 0.4), rect(0, 0.3, 1.0, 0.35)]))], # ---- characters: local units, feet on y = 1, ears up to about y = -1.15 "cat": [GROUND_SHADOW, P(ellipse(0.62, 0.42, 0.12, 0.42, rot=0.5), outline=True), P(ellipse(0, 0.5, 0.52, 0.5), outline=True), P(ellipse(0, 0.6, 0.3, 0.3), "light", shaded=False), *[P(ellipse(s * 0.25, 0.95, 0.17, 0.1), outline=True) for s in SIDES], *[P(poly([(s * 0.62, -0.45), (s * 0.18, -0.8), (s * 0.55, -1.12)]), outline=True) for s in SIDES], *[P(poly([(s * 0.5, -0.58), (s * 0.3, -0.74), (s * 0.5, -0.95)]), PINK, shaded=False) for s in SIDES], P(ellipse(0, -0.32, 0.62, 0.52), outline=True), P(poly([(-0.07, -0.27), (0.07, -0.27), (0, -0.19)]), PINK, shaded=False), *[P(seg(s * 0.3, -0.2, s * 0.72, -0.27, 0.025), INK, shaded=False) for s in SIDES], *[P(seg(s * 0.3, -0.12, s * 0.72, -0.08, 0.025), INK, shaded=False) for s in SIDES]], "bunny": [GROUND_SHADOW, P(circle(0.5, 0.72, 0.15), "light", outline=True), *[P(ellipse(s * 0.22, -0.98, 0.14, 0.42, rot=s * 0.12), outline=True) for s in SIDES], *[P(ellipse(s * 0.22, -0.96, 0.065, 0.3, rot=s * 0.12), PINK, shaded=False) for s in SIDES], P(ellipse(0, 0.5, 0.5, 0.5), outline=True), P(ellipse(0, 0.6, 0.28, 0.3), "light", shaded=False), *[P(ellipse(s * 0.25, 0.95, 0.2, 0.1), outline=True) for s in SIDES], P(ellipse(0, -0.3, 0.55, 0.48), outline=True), P(ellipse(0, -0.22, 0.06, 0.045), PINK, shaded=False)], "bear": [GROUND_SHADOW, *[P(circle(s * 0.45, -0.72, 0.2), outline=True) for s in SIDES], *[P(circle(s * 0.45, -0.72, 0.1), "light", shaded=False) for s in SIDES], P(ellipse(0, 0.5, 0.58, 0.52), outline=True), P(ellipse(0, 0.58, 0.34, 0.34), "light", shaded=False), *[P(ellipse(s * 0.28, 0.95, 0.2, 0.11), outline=True) for s in SIDES], P(ellipse(0, -0.3, 0.62, 0.55), outline=True), P(ellipse(0, -0.1, 0.27, 0.2), "light", shaded=False), P(ellipse(0, -0.19, 0.09, 0.06), INK, shaded=False)], "kid": [GROUND_SHADOW, *[P(rect(s * 0.15, 0.8, 0.14, 0.4), SKIN, outline=True) for s in SIDES], *[P(ellipse(s * 0.17, 0.98, 0.15, 0.07), SHOE, outline=True) for s in SIDES], *[P(seg(s * 0.28, 0.15, s * 0.55, 0.5, 0.14), SKIN, outline=True) for s in SIDES], *[P(circle(s * 0.56, 0.52, 0.08), SKIN, outline=True) for s in SIDES], P(poly([(-0.3, 0.02), (0.3, 0.02), (0.45, 0.7), (-0.45, 0.7)]), outline=True), P(ellipse(0, -0.3, 0.57, 0.57), HAIR, outline=True), P(ellipse(0, -0.3, 0.5, 0.5), SKIN, outline=True), P(("both", [ellipse(0, -0.34, 0.53, 0.52), halfplane(0, 1, -0.5)]), HAIR)], # ---- props "balloon": [P(seg(0, 0.95, -1.4, 3.0, 0.04), INK, shaded=False), P(poly([(-0.1, 1.06), (0.1, 1.06), (0, 0.9)]), outline=True), P(ellipse(0, 0, 0.75, 0.95), outline=True), P(ellipse(-0.28, -0.4, 0.12, 0.22, rot=0.5), WHITE, 0.7, False)], "ball": [P(circle(0, 0, 1), outline=True), P(("both", [circle(0, 0, 1), halfplane(0, 1, 0.22), halfplane(0, -1, 0.22)]), WHITE), P(ellipse(-0.4, -0.45, 0.15, 0.1, rot=-0.6), WHITE, 0.7, False)], "flower": [P(seg(0, 0.3, 0, 2.2, 0.12), LEAF, outline=True), P(ellipse(0.32, 1.4, 0.3, 0.12, rot=-0.5), LEAF, outline=True), *[P(circle(0.48 * np.cos(a), 0.48 * np.sin(a), 0.36), outline=True) for a in np.radians(-90 + 72 * np.arange(5))], P(circle(0, 0, 0.3), (0.98, 0.82, 0.25), outline=True)], "umbrella": [P(seg(0, 0, 0, 2.6, 0.08), INK, shaded=False), P(arc(0.2, 2.6, 0.2, 0.09, "lower"), INK, shaded=False), P(("both", [ellipse(0, 0.05, 1.3, 1.0), halfplane(0, 1, 0.05)]), outline=True), P(seg(0, -0.95, 0, -1.2, 0.08), INK, shaded=False)], } CASTS_SHADOW = {"circle", "square", "triangle", "heart", "house", "tree"} EXTENT = {"balloon": 3.4, "umbrella": 3.0, "flower": 2.4, "bunny": 1.6, "sun": 1.7} # bbox half-size, local units HEAD = {"cat": (0, -0.32, 0.55), "bunny": (0, -0.3, 0.5), "bear": (0, -0.3, 0.55), "kid": (0, -0.3, 0.45)} def face(hx, hy, hr, expr): ex, ey = 0.36 * hr, hy - 0.05 * hr mx, my = hx, hy + 0.38 * hr eyes = [hx - ex, hx + ex] f = [] if expr == "surprised": for x in eyes: f += [P(circle(x, ey, 0.19 * hr), INK, shaded=False), P(circle(x, ey, 0.15 * hr), WHITE, shaded=False), P(circle(x, ey + 0.02 * hr, 0.07 * hr), INK, shaded=False)] f.append(P(ellipse(mx, my + 0.03 * hr, 0.1 * hr, 0.13 * hr), INK, shaded=False)) elif expr == "sleepy": f += [P(arc(x, ey - 0.08 * hr, 0.11 * hr, 0.045 * hr, "lower", 0.1), INK, shaded=False) for x in eyes] f.append(P(ellipse(mx, my, 0.05 * hr, 0.06 * hr), INK, shaded=False)) zx, zy, s, w = hx + 0.9 * hr, hy - 0.8 * hr, 0.25 * hr, 0.06 * hr f += [P(seg(zx, zy, zx + s, zy, w), INK, shaded=False), P(seg(zx + s, zy, zx, zy + s, w), INK, shaded=False), P(seg(zx, zy + s, zx + s, zy + s, w), INK, shaded=False)] else: for x in eyes: f += [P(ellipse(x, ey, 0.12 * hr, 0.16 * hr), INK, shaded=False), P(circle(x + 0.04 * hr, ey - 0.06 * hr, 0.045 * hr), WHITE, shaded=False)] if expr == "happy": f.append(P(arc(mx, my - 0.25 * hr, 0.25 * hr, 0.06 * hr, "lower", 0.35), INK, shaded=False)) f += [P(ellipse(hx + s * 0.58 * hr, hy + 0.22 * hr, 0.13 * hr, 0.08 * hr), BLUSH, 0.55, False) for s in SIDES] else: f.append(P(arc(mx, my + 0.2 * hr, 0.2 * hr, 0.06 * hr, "upper", 0.35), INK, shaded=False)) for s, x in zip(SIDES, eyes): # sad: outer brow ends drop; angry: inner ends drop inner, outer = (ey - 0.24 * hr, ey - 0.32 * hr) if expr == "angry" else (ey - 0.32 * hr, ey - 0.24 * hr) f.append(P(seg(x - s * 0.13 * hr, inner, x + s * 0.13 * hr, outer, 0.05 * hr), INK, shaded=False)) if expr == "sad": f.append(P(ellipse(eyes[1] + 0.06 * hr, ey + 0.25 * hr, 0.05 * hr, 0.08 * hr), TEAR, shaded=False)) return f FACES = {(k, e): face(*HEAD[k], e) for k in CHARS for e in EXPRS} def sig(z): return 0.5 * (1 + np.tanh(0.5 * z)) def mask(g, px, py, sharp, grow=0.0): op, arg = g if op == "and": # layer 1 half-plane pre-activations -> layer 2 min-pool (exact AND at any size) z = sharp * (arg[:, 2, None, None] + grow - arg[:, 0, None, None] * px - arg[:, 1, None, None] * py) return sig(z.min(0)) ms = [mask(c, px, py, sharp, -grow if op == "minus" and i else grow) for i, c in enumerate(arg)] if op == "or": return sig(12 * (sum(ms) - 0.5)) if op == "both": return sig(12 * (sum(ms) - (len(ms) - 0.5))) return sig(12 * (ms[0] - ms[1] - 0.5)) # minus: AND-NOT neuron def part_color(spec, main): if isinstance(spec, str): return {"main": main, "light": main * 0.45 + 0.55, "dark": main * 0.65}[spec] return np.array(spec) # ------------------------------------------------------------------ backgrounds U, V = np.meshgrid(np.linspace(-1, 1, RES), np.linspace(-1, 1, RES)) HORIZON = 0.3 def lerp(a, b, t): return np.array(a)[None, None] * (1 - t[..., None]) + np.array(b)[None, None] * t[..., None] def hash2(i, j): return np.abs(np.modf(np.sin(i * 12.9898 + j * 78.233) * 43758.5453)[0]) def stars(threshold): h = hash2(np.floor((U + 1) * RES / 3), np.floor((V + 1) * RES / 3)) return ((h > threshold) * (h - threshold) / (1 - threshold) * 0.6 + (h > threshold) * 0.4)[..., None] def rain(): col = np.floor((U + 0.15 * V + 2) * RES / 9) phase = np.modf((V + 1) * 1.5 + hash2(col, 1.0) * 5)[0] frac = np.modf((U + 0.15 * V + 2) * RES / 9)[0] return ((phase < 0.06) & (frac < 0.2)).astype(float)[..., None] * 0.7 def background(sky, ground): t = (V + 1) / 2 if sky is None: img = lerp((0.96, 0.95, 0.92), (0.88, 0.86, 0.82), t) elif sky == "day": img = lerp((0.4, 0.66, 0.95), (0.82, 0.91, 1.0), t) elif sky == "night": img = lerp((0.02, 0.03, 0.12), (0.12, 0.12, 0.32), t) img = np.clip(img + stars(0.985), 0, 1) elif sky == "sunset": img = lerp((0.22, 0.18, 0.45), (0.95, 0.45, 0.35), np.clip(t / 0.6, 0, 1)) img = img + lerp((0, 0, 0), (0.05, 0.35, 0.1), np.clip((t - 0.6) / 0.4, 0, 1)) elif sky == "rain": img = lerp((0.5, 0.55, 0.62), (0.74, 0.77, 0.82), t) else: img = np.clip(lerp((0.0, 0.0, 0.02), (0.04, 0.02, 0.08), t) + stars(0.97), 0, 1) if ground is not None: wiggle = {"sea": 0 * U, "grass": 0.025 * np.sin(4 * U), "desert": 0.06 * np.sin(3 * U + 1), "snow": 0.035 * np.sin(2.5 * U + 2)}[ground] g = np.clip((V - HORIZON) / (1 - HORIZON), 0, 1) if ground == "sea": col = lerp((0.25, 0.55, 0.75), (0.04, 0.2, 0.42), g) + 0.05 * np.sin(70 * V + 4 * np.sin(7 * U))[..., None] elif ground == "grass": col = lerp((0.5, 0.74, 0.32), (0.2, 0.45, 0.15), g) + 0.025 * np.sin(150 * U + 20 * V)[..., None] elif ground == "desert": col = lerp((0.96, 0.82, 0.52), (0.84, 0.6, 0.34), g) + 0.04 * np.sin(9 * U - 14 * V)[..., None] else: col = lerp((0.98, 0.99, 1.0), (0.82, 0.88, 0.97), g) m = sig(RES * 0.6 * (V - HORIZON - wiggle))[..., None] img = img * (1 - m) + col * m return np.clip(img, 0, 1) # ------------------------------------------------------------------ story context (plain code, not weights) class Story: """Carries scene, character colours and the last cast across sentences, and resolves pronouns.""" def __init__(self): self.colors, self.cast, self.sky, self.ground = {}, [], "day", "grass" def resolve(self, objs): out = [] for o in objs: if o["kind"] == "ref": for c in (self.cast[-1:] if o["ref"] == 1 else self.cast): out.append(dict(c, expr=o["expr"], has_expr=o["has_expr"], x=o["x"], y=o["y"], size=o["size"] if o["has_size"] else c["size"])) continue if o["has_color"]: self.colors[o["kind"]] = o["color"] elif o["kind"] in self.colors: o["color"] = self.colors[o["kind"]] out.append(o) first, dedup = {}, [] for o in out: # "it ... she was sad": one cat, keep the stated expression if o["kind"] in CHARS and o["kind"] in first: if o.get("has_expr"): first[o["kind"]]["expr"] = o["expr"] continue first[o["kind"]] = o dedup.append(o) cast = [dict(kind=o["kind"], ref=0, color=o["color"], size=o["size"], has_color=True, has_size=False) for o in dedup if o["kind"] in CHARS] if cast: self.cast = cast return dedup def split_story(text): return [s.strip() for s in re.split(r"(?<=[.!?])\s+|\n+", text.strip()) if s.strip()] # ------------------------------------------------------------------ render def render(prompt, story=None): slots, scene, unknown = encode(prompt) objs = [decode(s) for s in slots] objs = story.resolve(objs) if story else [o for o in objs if o["kind"] != "ref"] objs = layout(objs) sky = SKIES[int(np.argmax(scene[SKY:GND]))] if scene[SKY:GND].any() else None ground = GROUNDS[int(np.argmax(scene[GND:]))] if scene[GND:].any() else None if story: sky, ground = sky or story.sky, ground or story.ground story.sky, story.ground = sky, ground if ground and sky is None: sky = "day" img = background(sky, ground) for o in objs: r = o["size"] ext = EXTENT.get(o["kind"], 1.8) * r i0, i1 = max(0, int((o["y"] - ext + 1) / 2 * RES) - 3), min(RES, int((o["y"] + ext + 1) / 2 * RES) + 4) j0, j1 = max(0, int((o["x"] - ext + 1) / 2 * RES) - 3), min(RES, int((o["x"] + ext + 1) / 2 * RES) + 4) if i0 >= i1 or j0 >= j1: continue sub = img[i0:i1, j0:j1] sharp = 4.5 * r * RES / 2 # ~1 px anti-aliased edge in local units ink = 2.4 * 2 / (RES * r) # outline width: ~2.4 px in local units px, py = (U[i0:i1, j0:j1] - o["x"]) / r, (V[i0:i1, j0:j1] - o["y"]) / r parts = PARTS[o["kind"]] + FACES.get((o["kind"], o["expr"]), []) if o["kind"] in CASTS_SHADOW: sh = sum(mask(g, px - 0.12, py - 0.12, sharp) * a for g, _, a, _, _ in parts) sub = sub * (1 - np.clip(sh, 0, 1)[..., None] * 0.22) for geom, col, alpha, shaded, outline in parts: if outline: m = mask(geom, px, py, sharp, ink)[..., None] sub = sub * (1 - m) + np.array(INK) * m c = part_color(col, o["color"])[None, None] if shaded: c = c * (1 - 0.16 * (px + py))[..., None] m = (mask(geom, px, py, sharp) * alpha)[..., None] sub = sub * (1 - m) + np.clip(c, 0, 1) * m img[i0:i1, j0:j1] = sub if sky == "rain": drops = rain() img = img * (1 - drops) + np.array([0.85, 0.9, 1.0]) * drops return Image.fromarray((np.clip(img, 0, 1) * 255).astype(np.uint8)), objs, sky, ground, unknown def render_story(text): story = Story() return [(s, render(s, story)) for s in split_story(text)] def count_params(): def geo(g): return g[1].size if g[0] == "and" else sum(geo(c) for c in g[1]) parts = [p for ps in PARTS.values() for p in ps] + [p for ps in FACES.values() for p in ps] return (len(VOCAB) * DIM + DEF_COLOR.size + DEF_SIZE.size + len(DEF_Y) + sum(geo(g) + 3 for g, *_ in parts) + 3 * len(PROPS) + 2 * len(HAND)) def slug(p): return re.sub(r"[^a-z0-9]+", "_", p.lower()).strip("_")[:60] def caption_font(size=20): for f in ["C:/Windows/Fonts/comic.ttf", "C:/Windows/Fonts/segoeui.ttf", "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"]: try: return ImageFont.truetype(f, size) except OSError: pass return ImageFont.load_default(size=size) def book_sheet(pages, cols=3, cap=64): font = caption_font() sheet = Image.new("RGB", (cols * (RES + 16) + 16, ((len(pages) + cols - 1) // cols) * (RES + cap + 16) + 16), "white") d = ImageDraw.Draw(sheet) for i, (text, im) in enumerate(pages): x, y = 16 + (i % cols) * (RES + 16), 16 + (i // cols) * (RES + cap + 16) sheet.paste(im, (x, y)) lines, line = [], "" for w in text.split(): if d.textlength(line + " " + w, font=font) > RES - 8 and line: lines.append(line) line = w else: line = (line + " " + w).strip() d.multiline_text((x + 4, y + RES + 8), "\n".join(lines + [line]), fill="black", font=font, spacing=4) return sheet DEMO = [ "a yellow sun over the sea", "a red house and a green tree on grass", "a white moon and a gold star in a night sky", "an orange sun and a cloud over the desert at sunset", "a happy orange cat with a red balloon on the grass", "a sleepy bear and a surprised bunny at night", "a kid with a blue umbrella in the rain", "a sad white bunny with a pink flower in the snow", ] STORY = ("The orange cat was happy in the meadow. She found a red balloon. Then it started to rain and she was sad. " "A kid with a yellow umbrella came by. The cat and the kid were happy together. " "At night the cat was sleepy under the moon.") if __name__ == "__main__": OUT.mkdir(exist_ok=True) print(f"hand-written parameters: {count_params():,}") if sys.argv[1:2] == ["--story"]: pages = render_story(" ".join(sys.argv[2:]) or STORY) for s, (_, objs, sky, ground, unknown) in pages: print(f"\n{s!r}\n sky={sky} ground={ground} " + ", ".join(f"{o['expr']} {o['kind']}" if o["kind"] in CHARS else o["kind"] for o in objs)) book_sheet([(s, r[0]) for s, r in pages]).save(OUT / "story_sheet.png") print(f"\nbook -> {OUT / 'story_sheet.png'}") sys.exit() prompts = DEMO if sys.argv[1:] == ["--demo"] else [" ".join(sys.argv[1:]) or DEMO[0]] tiles = [] for p in prompts: im, objs, sky, ground, unknown = render(p) im.save(OUT / f"{slug(p)}.png") tiles.append((p, im)) print(f"\n{p!r}\n sky={sky} ground={ground} ignored={unknown}") for o in objs: print(f" {o['kind']:<8} {o['expr'] if o['kind'] in CHARS else '':<9} rgb={np.round(o['color'], 2)} " f"size={o['size']:.2f} at ({o['x']:+.2f},{o['y']:+.2f})") if len(tiles) > 1: book_sheet(tiles, cols=4).save(OUT / "demo_sheet.png") print(f"\nsheet -> {OUT / 'demo_sheet.png'}")