eths / source /studio /render.py
Ethscriptions's picture
Update: ≤3s WeChat-compatible live photos, sets 3 & 4 (fill-in-the-blank), in-browser maker tool
0daf81d verified
Raw History Blame Contribute Delete
21.5 kB
"""Render one 'what's this in English?' Live Photo clip: cover.jpg + motion.mp4 (H.264 + AAC)."""
import json, math, os, subprocess, sys
import numpy as np
from PIL import Image, ImageDraw, ImageFilter, ImageFont
sys.path.insert(0, os.path.dirname(__file__))
import audio as A
W, H, FPS = 1080, 1440, 30
COVER_F = 6
FD = '/tmp/work/fonts'
F_SC = f'{FD}/NotoSansSC.ttf'
F_SS = f'{FD}/ss/SmileySans-Oblique.ttf'
F_POP = f'{FD}/Poppins-ExtraBold.ttf'
F_POP_SB = f'{FD}/Poppins-SemiBold.ttf'
F_NS = f'{FD}/NotoSans.ttf'
INK = (22, 22, 28)
ACCENT = (255, 59, 92)
YELLOW = (255, 214, 61)
GREEN = (34, 197, 94)
CONFETTI = [(255, 59, 92), (255, 214, 61), (61, 220, 151), (77, 150, 255), (255, 138, 61), (181, 124, 255)]
_fc = {}
def font(path, size, wght=None):
size = max(6, int(round(size)))
key = (path, size, wght)
if key not in _fc:
f = ImageFont.truetype(path, size)
if wght is not None:
axes = f.get_variation_axes()
vals = []
for ax in axes:
n = ax['name'].decode() if isinstance(ax['name'], bytes) else ax['name']
vals.append(wght if n == 'Weight' else ax['default'])
f.set_variation_by_axes(vals)
_fc[key] = f
return _fc[key]
# ---------- easing ----------
def clamp(x, a=0.0, b=1.0):
return max(a, min(b, x))
def prog(t, t0, d):
return clamp((t - t0) / d) if d > 0 else float(t >= t0)
def ease_out_cubic(x):
return 1 - (1 - x) ** 3
def ease_in_out(x):
return 4 * x ** 3 if x < .5 else 1 - (-2 * x + 2) ** 3 / 2
def ease_out_back(x, s=1.9):
return 1 + (s + 1) * (x - 1) ** 3 + s * (x - 1) ** 2
def lerp(a, b, x):
return a + (b - a) * x
# ---------- drawing helpers ----------
def rrect_mask(w, h, r, ss=3):
m = Image.new('L', (w * ss, h * ss), 0)
ImageDraw.Draw(m).rounded_rectangle((0, 0, w * ss - 1, h * ss - 1), r * ss, fill=255)
return m.resize((w, h), Image.LANCZOS)
def text_w(s, f):
return f.getlength(s)
class Clip:
def __init__(self, item, index, total, photo_path, out_dir):
self.it = item
self.idx = index
self.out = out_dir
os.makedirs(out_dir, exist_ok=True)
self.word = item['en']
self.CW = 920
self.cy = item.get('card_y', 720)
self.prepare_photo(photo_path)
self.prepare_audio()
self.prepare_layout()
rng = np.random.default_rng(int(item['id']) * 7 + 1)
self.grain = [rng.normal(0, 2.6, (H, W)).astype(np.float32) for _ in range(3)]
self.particles = self.make_particles(rng)
# ----- photo -----
def prepare_photo(self, path):
src = Image.open(path).convert('RGB')
sw, sh = src.size
target = W / H
if sw / sh > target: # crop width
nw = int(sh * target); x0 = (sw - nw) // 2; src = src.crop((x0, 0, x0 + nw, sh))
else:
nh = int(sw / target); y0 = (sh - nh) // 2; src = src.crop((0, y0, sw, y0 + nh))
S = 1.12
big = src.resize((int(W * S), int(H * S)), Image.LANCZOS)
if src.width < W * S:
big = big.filter(ImageFilter.UnsharpMask(radius=2.0, percent=60, threshold=2))
else:
big = big.filter(ImageFilter.UnsharpMask(radius=1.2, percent=35, threshold=3))
self.big = big
self.S = S
def camera(self, t):
z = 1.0 + 0.05 * ease_in_out(clamp(t / max(self.dur, 0.1)))
dx = 5.0 * math.sin(2 * math.pi * 0.31 * t) + 2.2 * math.sin(2 * math.pi * 0.87 * t + 1.1)
dy = 4.0 * math.sin(2 * math.pi * 0.23 * t + 0.4) - 4.0 * math.sin(0.4) + 1.8 * math.sin(2 * math.pi * 0.71 * t)
bw, bh = self.big.size
vw, vh = bw / (1.06 * z), bh / (1.06 * z)
cx, cy = bw / 2 + dx, bh / 2 + dy
a = vw / W
e = vh / H
frame = self.big.transform((W, H), Image.AFFINE, (a, 0, cx - vw / 2, 0, e, cy - vh / 2), resample=Image.BICUBIC)
return frame
# ----- audio & timeline -----
def typed_indices(self):
return [i for i, c in enumerate(self.word) if c != ' ']
def typing_dt(self, n):
return clamp(0.40 / max(n, 1), 0.035, 0.07)
MAX_DUR = 3.0 # WeChat Moments only accepts Live Photos of about 3 s (native iPhone length)
def prepare_audio(self):
say = self.it.get('tts', self.word)
en1 = A.tts(f"{say}.", 'en-US-AvaMultilingualNeural', rate='-5%')
self.typed = self.typed_indices()
n = len(self.typed)
T = {}
T['rev'] = 0.35
T['grow_d'] = 0.35
T['type0'] = 0.50
T['dt'] = self.typing_dt(n)
T['done'] = T['type0'] + (n - 1) * T['dt'] + 0.10
T['marker'] = T['done'] + 0.03
T['ipa'] = T['done'] + 0.08
T['en1'] = T['done'] + 0.10
d1 = len(en1) / A.SR
T['mean'] = T['en1'] + d1 * 0.5
T['sweep'] = T['done'] + 0.25
en2 = None
for rate in ('-22%', '-10%', '+0%', '+10%'):
cand = A.tts(f"{say}.", 'en-US-AndrewMultilingualNeural', rate=rate)
if T['en1'] + d1 + 0.10 + len(cand) / A.SR + 0.10 <= self.MAX_DUR:
en2 = cand
break
if en2 is not None:
T['en2'] = T['en1'] + d1 + 0.10
T['end'] = self.MAX_DUR
else:
T['en2'] = None
T['end'] = min(self.MAX_DUR, max(T['en1'] + d1 + 0.25, T['mean'] + 0.6))
nf = int(math.ceil(T['end'] * FPS))
self.nf = nf
self.dur = nf / FPS
self.T = T
mx = A.Mixer(self.dur)
mx.add(A.whoosh(), T['rev'] - 0.06, 0.9)
for k, li in enumerate(self.typed):
pan = (li / max(len(self.word) - 1, 1)) * 1.2 - 0.6
mx.add(A.tick(k), T['type0'] + k * T['dt'], 0.9, pan)
mx.add(A.chime(), T['done'], 0.85)
mx.add(A.shimmer(), T['done'] + 0.1, 1.0)
mx.add(A.pop(), T['ipa'], 0.7)
mx.add(en1, T['en1'], 1.0)
mx.add(A.pop(), T['mean'], 0.5)
if en2 is not None:
mx.add(en2, T['en2'], 1.0)
self.wav = os.path.join(self.out, 'audio.wav')
mx.render(self.dur, self.wav)
env = np.zeros(nf + 2)
for clip, st in ((en1, T['en1']), (en2, T['en2'])):
if clip is None:
continue
e = A.envelope(clip, FPS)
s = int(round(st * FPS))
n2 = min(len(e), len(env) - s)
env[s:s + n2] = np.maximum(env[s:s + n2], e[:n2])
self.env = np.convolve(env, [0.25, 0.5, 0.25], mode='same')
# ----- layout -----
CHQ, CHA = 252, 452
def prepare_layout(self):
CW = self.CW
word = self.word
size = 124
while text_w(word, font(F_POP, size)) > CW - 120:
size -= 2
self.wsize = size
fw = font(F_POP, size)
tw = text_w(word, fw)
x0 = (CW - tw) / 2
self.word_x0, self.word_w = x0, tw
self.final_cx = [x0 + text_w(word[:i], fw) + text_w(c, fw) / 2 for i, c in enumerate(word)]
hs = 84
while text_w(f"「{self.it['zh']}」用英语怎么说?", font(F_SS, hs)) > CW - 110 and hs > 40:
hs -= 2
self.hsize = hs
self._masks, self._shadows = {}, {}
self.shadow_pad = 90
bd = np.zeros((self.CHA * 2, 260), np.float32)
bd[:] = np.exp(-(np.linspace(-1, 1, 260) ** 2) * 4.5) * 120
self.band = Image.fromarray(bd.astype(np.uint8)).rotate(-18, resample=Image.BICUBIC, expand=True)
def mask_for(self, w, h, r):
k = (w, h, r)
if k not in self._masks:
self._masks[k] = rrect_mask(w, h, r)
return self._masks[k]
def shadow_for(self, w, h, r):
k = (w, h, r)
if k not in self._shadows:
pad = self.shadow_pad
sh = Image.new('L', (w + pad * 2, h + pad * 2), 0)
ImageDraw.Draw(sh).rounded_rectangle((pad, pad, pad + w, pad + h), r, fill=110)
self._shadows[k] = sh.filter(ImageFilter.GaussianBlur(34))
return self._shadows[k]
def card_h(self, t):
T = self.T
return lerp(self.CHQ, self.CHA, ease_in_out(prog(t, T['rev'], T['grow_d'])))
def make_particles(self, rng):
ps = []
for k in range(46):
ang = rng.uniform(-math.pi * 0.95, -math.pi * 0.05)
sp = rng.uniform(650, 1500)
ps.append(dict(x=rng.uniform(-260, 260), y=rng.uniform(-30, 30), vx=math.cos(ang) * sp * 0.9,
vy=math.sin(ang) * sp, rot=rng.uniform(0, 360), vr=rng.uniform(-720, 720),
kind=rng.integers(0, 3), col=CONFETTI[k % len(CONFETTI)], sz=rng.uniform(9, 17),
delay=rng.uniform(0, 0.08)))
return ps
# ----- card content -----
def draw_pill(self, layer, t):
T = self.T
p = ease_in_out(prog(t, T['rev'] + 0.05, 0.25))
f = font(F_SC, 28, 700)
y, h = 38, 54
CW = self.CW
if p < 1:
txt = getattr(self, "pill_q", "在线求助")
w = 48 + text_w(txt, f) + 26
x = (CW - w) / 2
pl = Image.new('RGBA', layer.size, (0, 0, 0, 0))
pd = ImageDraw.Draw(pl)
pd.rounded_rectangle((x, y, x + w, y + h), h / 2, fill=YELLOW + (255,))
pd.ellipse((x + 20, y + h / 2 - 7, x + 34, y + h / 2 + 7), fill=ACCENT + (255,))
if int(t * 2.2) % 2 == 0 or t >= T['rev']:
pd.ellipse((x + 16, y + h / 2 - 11, x + 38, y + h / 2 + 11), outline=ACCENT + (90,), width=3)
pd.text((x + 48, y + h / 2), txt, font=f, fill=INK + (255,), anchor='lm')
if p > 0:
pl.putalpha(pl.getchannel('A').point(lambda v: int(v * (1 - p))))
layer.alpha_composite(pl)
if p > 0:
txt = getattr(self, "pill_a", "答案揭晓")
w = 54 + text_w(txt, f) + 26
sc = 0.85 + 0.15 * ease_out_back(p)
pw, ph = int(w + 2), h + 2
pl = Image.new('RGBA', (pw, ph), (0, 0, 0, 0))
pd = ImageDraw.Draw(pl)
pd.rounded_rectangle((0, 0, w, h), h / 2, fill=GREEN + (255,))
cx, cy = 30, h / 2
pd.line([(cx - 9, cy + 1), (cx - 3, cy + 8), (cx + 10, cy - 8)], fill=(255, 255, 255, 255), width=5, joint='curve')
pd.text((54, h / 2), txt, font=f, fill=(255, 255, 255, 255), anchor='lm')
if sc != 1:
pl = pl.resize((max(1, int(pw * sc)), max(1, int(ph * sc))), Image.BICUBIC)
pl.putalpha(pl.getchannel('A').point(lambda v: int(v * p)))
layer.alpha_composite(pl, (int((CW - pl.width) / 2), int(y + (h - pl.height) / 2)))
def content(self, t, fidx, ch):
T = self.T
CW = self.CW
layer = Image.new('RGBA', (CW, ch), (0, 0, 0, 0))
d = ImageDraw.Draw(layer)
self.draw_pill(layer, t)
# question headline
hp = ease_in_out(prog(t, T['rev'], 0.22))
if hp < 1:
f = font(F_SS, self.hsize)
kw = f"「{self.it['zh']}」"
rest = "用英语怎么说?"
tw = text_w(kw + rest, f)
x = (CW - tw) / 2
y = 167 - 34 * hp
al = int(255 * (1 - hp))
d.text((x, y), kw, font=f, fill=ACCENT + (al,), anchor='lm')
d.text((x + text_w(kw, f), y), rest, font=f, fill=INK + (al,), anchor='lm')
# word: letters type in at their final positions
row_y = 186
mk = ease_out_cubic(prog(t, T['marker'], 0.32))
if mk > 0:
mx0 = self.word_x0 - 14
mw = (self.word_w + 28) * mk
yb = row_y + self.wsize * 0.22
d.rounded_rectangle((mx0, yb - self.wsize * 0.26, mx0 + mw, yb + self.wsize * 0.06), 12, fill=YELLOW + (215,))
for k, i in enumerate(self.typed):
c = self.word[i]
t_on = T['type0'] + k * T['dt']
appear = prog(t, t_on, 0.16)
if appear <= 0:
continue
pop = ease_out_back(appear, 2.6)
size = self.wsize * (0.45 + 0.55 * pop)
wave = math.sin(clamp((t - T['done'] - k * 0.03) / 0.4) * math.pi) * 14
base = row_y + self.wsize * 0.30 - wave - 26 * (1 - ease_out_cubic(appear))
d.text((self.final_cx[i], base), c, font=font(F_POP, size), fill=INK + (int(255 * min(1, appear * 1.8)),), anchor='ms')
# IPA row
ip = ease_out_cubic(prog(t, T['ipa'], 0.32))
if ip > 0:
fi = font(F_NS, 46, 500)
ipa = self.it['ipa']
iw = text_w(ipa, fi)
icon = 62
x0 = (CW - (icon + 20 + iw)) / 2
y = 306 + 26 * (1 - ip)
al = int(255 * ip)
env = float(self.env[min(fidx, len(self.env) - 1)])
r = icon / 2 * (1 + 0.10 * env)
icx = x0 + icon / 2
if env > 0.05:
rr = icon / 2 + 6 + 16 * env
d.ellipse((icx - rr, y - rr, icx + rr, y + rr), outline=ACCENT + (int(110 * env * ip),), width=3)
d.ellipse((icx - r, y - r, icx + r, y + r), fill=ACCENT + (al,))
bx = icx - 13
d.polygon([(bx, y - 6), (bx + 7, y - 6), (bx + 16, y - 14), (bx + 16, y + 14), (bx + 7, y + 6), (bx, y + 6)],
fill=(255, 255, 255, al))
for k2, rad in enumerate((9, 16)):
aa = int(al * (0.55 + 0.45 * clamp(env * 2.2 - k2 * 0.5)))
cxa = bx + 15
d.arc((cxa - rad, y - rad, cxa + rad, y + rad), -48, 48, fill=(255, 255, 255, aa), width=3)
d.text((x0 + icon + 20, y), ipa, font=fi, fill=(40, 40, 52, al), anchor='lm')
# meaning row
mpn = ease_out_cubic(prog(t, T['mean'], 0.3))
if mpn > 0:
al = int(255 * mpn)
fp_ = font(F_POP_SB, 28)
fz = font(F_SC, 38, 700)
pos, zh = self.it['pos'], self.it['zh']
tagw = text_w(pos, fp_) + 26
x0 = (CW - (tagw + 16 + text_w(zh, fz))) / 2
y = 394 + 20 * (1 - mpn)
d.rounded_rectangle((x0, y - 21, x0 + tagw, y + 21), 10, fill=(22, 22, 28, int(al * 0.9)))
d.text((x0 + tagw / 2, y), pos, font=fp_, fill=(255, 255, 255, al), anchor='mm')
d.text((x0 + tagw + 16, y), zh, font=fz, fill=(30, 30, 40, al), anchor='lm')
# glossy light sweep
swp = prog(t, T['sweep'], 0.55)
if 0 < swp < 1:
bx = int(lerp(-self.band.width, CW + 40, ease_in_out(swp)))
gl = Image.new('L', (CW, ch), 0)
gl.paste(self.band, (bx, -self.CHA // 2))
m = self.mask_for(CW, ch, 46)
gl = Image.fromarray((np.asarray(gl, np.float32) * np.asarray(m, np.float32) / 255).astype(np.uint8))
white = Image.new('RGBA', (CW, ch), (255, 255, 255, 0))
white.putalpha(gl)
layer = Image.alpha_composite(layer, white)
return layer
# ----- full frame -----
def frame(self, fidx):
t = fidx / FPS
T = self.T
img = self.camera(t).convert('RGBA')
bump = math.sin(math.pi * prog(t, T['rev'], 0.32)) * 0.02 + math.sin(math.pi * prog(t, T['done'], 0.3)) * 0.024
sc = self.it.get('card_scale', 1.0) * (1 + bump)
CW = self.CW
CH = int(round(self.card_h(t)))
cw, ch = int(round(CW * sc)), int(round(CH * sc))
x0 = int(round(W / 2 - cw / 2))
y0 = int(round(self.cy - ch / 2))
rad = int(46 * sc)
shd = self.shadow_for(cw, ch, rad)
pad = self.shadow_pad
dark = Image.new('RGBA', shd.size, (10, 10, 20, 0))
dark.putalpha(shd)
img.alpha_composite(dark, (x0 - pad, y0 - pad + 22))
m = 50
box = (max(0, x0 - m), max(0, y0 - m), min(W, x0 + cw + m), min(H, y0 + ch + m))
reg = img.crop(box).convert('RGB')
small = reg.resize((max(1, reg.width // 6), max(1, reg.height // 6)), Image.BILINEAR).filter(ImageFilter.GaussianBlur(getattr(self, 'BLUR', 4)))
glass = small.resize(reg.size, Image.BICUBIC)
g = getattr(self, 'GLASS', 0.70)
ga = np.asarray(glass, np.float32) * (1 - g) + 255 * g
glass = Image.fromarray(ga.clip(0, 255).astype(np.uint8)).crop((x0 - box[0], y0 - box[1], x0 - box[0] + cw, y0 - box[1] + ch))
glass = glass.convert('RGBA')
glass.putalpha(self.mask_for(cw, ch, rad))
img.alpha_composite(glass, (x0, y0))
rim = Image.new('RGBA', (cw, ch), (0, 0, 0, 0))
ImageDraw.Draw(rim).rounded_rectangle((1, 1, cw - 2, ch - 2), rad, outline=(255, 255, 255, 200), width=2)
img.alpha_composite(rim, (x0, y0))
layer = self.content(t, fidx, CH)
if sc != 1:
layer = layer.resize((cw, ch), Image.BICUBIC)
img.alpha_composite(layer, (x0, y0))
# confetti
ct = t - (T['done'] + 0.03)
if 0 < ct < 1.6:
ov = Image.new('RGBA', (W, H), (0, 0, 0, 0))
od = ImageDraw.Draw(ov)
ox, oy = W / 2, y0 + 186 * sc
for p in self.particles:
tt = ct - p['delay']
if tt <= 0:
continue
drag = (1 - math.exp(-2.6 * tt)) / 2.6
px = ox + p['x'] + p['vx'] * drag
py = oy + p['y'] + p['vy'] * drag + 0.5 * 1500 * tt * tt * 0.55
life = clamp(1 - (tt - 0.75) / 0.6)
if life <= 0:
continue
al = int(255 * life)
s = p['sz']
rot = math.radians(p['rot'] + p['vr'] * tt)
if p['kind'] == 0: # ribbon rect
pts = [(-s * 0.4, -s), (s * 0.4, -s), (s * 0.4, s), (-s * 0.4, s)]
flip = abs(math.cos(rot * 1.7)) * 0.8 + 0.2
pts = [(px + (x * flip) * math.cos(rot) - y * math.sin(rot), py + (x * flip) * math.sin(rot) + y * math.cos(rot)) for x, y in pts]
od.polygon(pts, fill=p['col'] + (al,))
elif p['kind'] == 1: # dot
od.ellipse((px - s * 0.5, py - s * 0.5, px + s * 0.5, py + s * 0.5), fill=p['col'] + (al,))
else: # 4-point sparkle
r1, r2 = s * 0.95, s * 0.28
pts = []
for k in range(8):
rr = r1 if k % 2 == 0 else r2
a = rot + k * math.pi / 4
pts.append((px + rr * math.cos(a), py + rr * math.sin(a)))
od.polygon(pts, fill=(255, 255, 255, al) if p['col'] == CONFETTI[1] else p['col'] + (al,))
img.alpha_composite(ov)
arr = np.asarray(img.convert('RGB'), np.float32)
arr += self.grain[fidx % 3][..., None]
return np.clip(arr, 0, 255).astype(np.uint8)
def render(self):
mp4 = os.path.join(self.out, 'motion.mp4')
cmd = ['ffmpeg', '-v', 'error', '-y', '-f', 'rawvideo', '-pix_fmt', 'rgb24', '-s', f'{W}x{H}', '-r', str(FPS), '-i', '-',
'-i', self.wav, '-map', '0:v', '-map', '1:a',
'-c:v', 'libx264', '-preset', 'medium', '-crf', '18', '-profile:v', 'high', '-pix_fmt', 'yuv420p',
'-color_primaries', 'bt709', '-color_trc', 'bt709', '-colorspace', 'bt709', '-tag:v', 'avc1',
'-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '1', '-shortest', '-movflags', '+faststart', mp4]
proc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
for i in range(self.nf):
fr = self.frame(i)
if i == COVER_F:
Image.fromarray(fr).save(os.path.join(self.out, 'cover.jpg'), quality=95, subsampling=0)
if i == self.nf - 1:
Image.fromarray(fr).save(os.path.join(self.out, 'final.jpg'), quality=90)
if i in getattr(self, 'snap', []):
Image.fromarray(fr).save(os.path.join(self.out, f'snap_{i:03d}.jpg'), quality=85)
proc.stdin.write(fr.tobytes())
proc.stdin.close()
if proc.wait() != 0:
raise RuntimeError('ffmpeg failed')
json.dump({'frames': self.nf, 'fps': FPS, 'duration': self.dur, 'coverFrame': COVER_F, 'coverTime': COVER_F / FPS,
'timeline': self.T}, open(os.path.join(self.out, 'render.json'), 'w'), indent=1)
return mp4
if __name__ == '__main__':
items_file = os.environ.get('ITEMS', 'items.json')
items = json.load(open(os.path.join(os.path.dirname(__file__), items_file)))
want = sys.argv[1].split(',') if len(sys.argv) > 1 else [i['id'] for i in items]
picks = json.load(open('/tmp/work/picks.json')) if os.path.exists('/tmp/work/picks.json') else {}
for k, it in enumerate(items, 1):
if it['id'] not in want:
continue
photo = picks.get(it['id'])
if not photo and os.path.exists(f"/tmp/work/real/{it['id']}_pick.jpg"):
photo = f"/tmp/work/real/{it['id']}_pick.jpg"
if not photo:
for d in ('/tmp/work/zi', '/tmp/work/hf', '/tmp/work/raw'):
c_ = sorted(f for f in os.listdir(d) if f.startswith(it['id'] + '_')) if os.path.isdir(d) else []
if c_:
photo = os.path.join(d, c_[0]); break
print('photo', photo)
c = Clip(it, k, len(items), photo, f"/tmp/work/out/{it['id']}")
c.snap = [int(x) for x in os.environ.get('SNAP', '').split(',') if x]
print(it['id'], it['en'], 'frames', c.nf, 'dur', round(c.dur, 2), flush=True)
c.render()