diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..23c0ac77bb49b5bf48c81e27b0e51fa802eccebd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,15 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +assets3d/claire.glb filter=lfs diff=lfs merge=lfs -text +assets3d/mark.glb filter=lfs diff=lfs merge=lfs -text +img/auto_visemes.png filter=lfs diff=lfs merge=lfs -text +img/flp/fail_handdrawn.png filter=lfs diff=lfs merge=lfs -text +img/flp_illust_ok.png filter=lfs diff=lfs merge=lfs -text +img/flp_painting_ok.png filter=lfs diff=lfs merge=lfs -text +img/pig_states.png filter=lfs diff=lfs merge=lfs -text +img/sprite_vs_flp.gif filter=lfs diff=lfs merge=lfs -text +img/stick_auto.png filter=lfs diff=lfs merge=lfs -text +img/studio_5.gif filter=lfs diff=lfs merge=lfs -text +img/studio_live.gif filter=lfs diff=lfs merge=lfs -text +img/studio_sync.gif filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md index 8d2aab6a5565e7af9893c75a39db87cb7eecd612..df98d865163bfe1b1186136432c81729ad9b15ed 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,15 @@ --- -title: Drawface Live -emoji: ๐Ÿš€ -colorFrom: purple -colorTo: yellow +title: DrawFace Live +emoji: ๐Ÿชž +colorFrom: gray +colorTo: indigo sdk: static +app_file: index.html pinned: false +short_description: ์›น์บ  ํ‘œ์ •์„ ์†๊ทธ๋ฆผยท3D ์บ๋ฆญํ„ฐ๊ฐ€ ์‹ค์‹œ๊ฐ„์œผ๋กœ ๋”ฐ๋ผ ํ•ฉ๋‹ˆ๋‹ค --- -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +์›น์บ  ์•ž ํ‘œ์ •์„ ๊ทธ๋ฆผ์ด ๊ทธ๋Œ€๋กœ ๋”ฐ๋ผ ํ•˜๋Š” ๋ฐ๋ชจ. ์–ผ๊ตด ์ธ์‹(MediaPipe)๋ถ€ํ„ฐ ๋ Œ๋”๊นŒ์ง€ +**์ „๋ถ€ ๋ธŒ๋ผ์šฐ์ € ์•ˆ์—์„œ** ์ฒ˜๋ฆฌ๋˜๊ณ  ์˜์ƒ์€ ์–ด๋””๋กœ๋„ ์ „์†ก๋˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. + +์†Œ์Šค์™€ ๊ธฐ์ˆ  ๋ฌธ์„œ: https://github.com/ingon1/drawface-live diff --git a/assets3d/LICENSE-claire.txt b/assets3d/LICENSE-claire.txt new file mode 100644 index 0000000000000000000000000000000000000000..340c6d9f9383c0cb3fe12bb9d134989cad8e6fcf --- /dev/null +++ b/assets3d/LICENSE-claire.txt @@ -0,0 +1,18 @@ +claire head (claire.glb) +Derived from claire_arkit_v2_topo1.ma in NVIDIA Maya-ACE (https://github.com/NVIDIA/Maya-ACE) +Blendshape geometry: nvidia/Audio2Face-3D-v2.3-Claire (HuggingFace). + +MIT License +Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND. diff --git a/assets3d/LICENSE-mark.txt b/assets3d/LICENSE-mark.txt new file mode 100644 index 0000000000000000000000000000000000000000..81739a3b02d9e37b0b102ad234f7c045f8687752 --- /dev/null +++ b/assets3d/LICENSE-mark.txt @@ -0,0 +1,17 @@ +mark head (mark.glb) +Derived from mark_arkit_v2_topo1.ma in NVIDIA Maya-ACE (https://github.com/NVIDIA/Maya-ACE) + +MIT License +Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND. diff --git a/assets3d/claire.glb b/assets3d/claire.glb new file mode 100644 index 0000000000000000000000000000000000000000..5ee82eb59158750bb2c17224a352f3b4666af40f --- /dev/null +++ b/assets3d/claire.glb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f2a96cd63604f8abe4a3f369afe7beb93341d5c53af9705d7a4c2900bdd687d +size 1928800 diff --git a/assets3d/mark.glb b/assets3d/mark.glb new file mode 100644 index 0000000000000000000000000000000000000000..0e2a0a430eb31c200f1c4bc4afb6c44a0463bd8a --- /dev/null +++ b/assets3d/mark.glb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efc715a341e24d8a166690d6525e6e1f83a4b3b7881fcd9dad0b2d9791eb07e5 +size 1839324 diff --git a/avatar_core.js b/avatar_core.js new file mode 100644 index 0000000000000000000000000000000000000000..38050ae5d96c1c7982b1dbe0dda426a2de49ab1e --- /dev/null +++ b/avatar_core.js @@ -0,0 +1,1256 @@ +/* avatar_core.js โ€” talking-drawing-avatar ๋ฆฌํฌ static/avatar_core.js ์˜ vendored ์‚ฌ๋ณธ. + * ์—ฌ๊ธฐ์„œ ์ˆ˜์ •ํ•˜์ง€ ๋ง ๊ฒƒ โ€” ์›๋ณธ์—์„œ ๊ณ ์นœ ๋’ค ์•„๋ž˜๋กœ ๊ฐฑ์‹ ํ•œ๋‹ค: + * ./scripts/sync_avatar_core.sh (์ด ํ—ค๋”๋Š” ๋ณด์กดํ•˜๊ณ  ๋ณธ๋ฌธ๋งŒ ๊ฐฑ์‹ ) + * ./scripts/sync_avatar_core.sh --check (๋’ค์ฒ˜์กŒ๋Š”์ง€ ํ™•์ธ โ€” CI ๊ฐ€ ๋งค๋ฒˆ ๋Œ๋ฆฐ๋‹ค) + * ์ด ๋ฆฌํฌ์—์„œ๋Š” index.html(๋ฏธ๋Ÿฌ๋ง ์ŠคํŠœ๋””์˜ค)์ด ์†Œ๋น„ํ•œ๋‹ค. + */ +window.AvatarCore = (() => { + + // ---------- ๋‚ด๋ถ€ ์œ ํ‹ธ ---------- + const norm = s => s.toLowerCase().replace(/[_\-\s]/g, ""); + const clamp01 = v => Math.min(1, Math.max(0, v)); // 0~1 ๋กœ ์ž๋ฅด๊ธฐ + const avgLR = (W, base) => (W(base + "left") + W(base + "right")) / 2; // ์ขŒ์šฐ ์ฑ„๋„ ํ‰๊ท  + const roundness = W => Math.max(W("mouthpucker"), W("mouthfunnel")); // ์˜ค๋ฏ€๋ฆผ ์„ธ๊ธฐ + const WARP_JAW_G = Math.exp(-(38 * 38) / (2 * 55 * 55)); // jaw ๋ณ€์œ„์žฅ์„ ์ž… ์•ต์ปค(38px ์œ„)์—์„œ ํ‰๊ฐ€ํ•œ ๊ฐ€์šฐ์‹œ์•ˆ (์‹œ๊ทธ๋งˆ 55 = ์›Œํ”„ ์…ฐ์ด๋”์™€ ๋™์ผ) + + // ํ…์ŠคํŠธ ๊ฐ์ • ์ถ”๋ก  (๋ฐœํ™” ์‹œ ์ž๋™ ํ”„๋ฆฌ์…‹ โ€” ์„ธ ํŽ˜์ด์ง€ ๋™์ผ ๊ทœ์น™) + // ๊ฐ์ • ์‚ฌ์ „: [์ •๊ทœ์‹, ๊ฐ€์ค‘์น˜]. ์–ด๊ฐ„ ๋ฌธ์žํด๋ž˜์Šค๋กœ ํ™œ์šฉํ˜•์„ ํ•จ๊ป˜ ์žก๋Š”๋‹ค(์Šฌํ”„/์Šฌํผ/์Šฌํސ/์Šฌํ”ˆโ€ฆ). + // ์ˆœ์„œ๊ฐ€ ์•„๋‹ˆ๋ผ ์ ์ˆ˜ ํ•ฉ์œผ๋กœ ๋ฝ‘์œผ๋ฏ€๋กœ "ใ…‹ใ…‹ ๋Œ€๋ฐ• ์›ƒ๊ฒจ"(joy 4.0 vs surprise 1.6)์ฒ˜๋Ÿผ ๊ฒน์ณ๋„ ์˜ณ๊ฒŒ ๊ฐˆ๋ฆฐ๋‹ค. + const EMO_RULES = { + joy: [ + [/ใ…‹{2,}|ใ…Ž{2,}/g, 2.2], [/ํ•˜ํ•˜|ํ˜ธํ˜ธ|ใ…ใ…Ž|์›ƒ[๊ฒจ๊ธด์Œ์—ˆ]/g, 1.8], + [/์‹ [๋‚˜๋‚œ๋‚ฌ]|ํ–‰๋ณต|๊ธฐ[์˜๋ป๋ปค์œ]|์ฆ๊ฑฐ|์ข‹[์•„๋‹ค์€๋„ค์•˜]|์ตœ๊ณ |์‚ฌ๋ž‘|๋ฐ˜[๊ฐ€๊ฐ‘]|์žฌ[๋ฐŒ๋ฏธ]|๊ณ ๋งˆ[์›Œ์› ์šด]|๊ฐ์‚ฌ|๐Ÿ˜Š|๐Ÿ˜„|๐ŸŽ‰|๐Ÿ‘|โค/g, 1.4], + [/์ถ•ํ•˜|์„ฑ๊ณต|ํ•ด๋ƒˆ|๊ตฟ|์งฑ/g, 1.2], [/!/g, 0.35], + ], + sad: [ + [/[ใ… ใ…œ]{2,}/g, 2.2], + [/์Šฌ[ํ”„ํผํސํ”ˆ]|์šฐ์šธ|๋ˆˆ๋ฌผ|์šธ[๊ณ ์—ˆ]|์™ธ๋กœ|์“ธ์“ธ|์†์ƒ|์„œ์šด|์•„์‰ฝ|์•ˆํƒ€๊น|๊ทธ๋ฆฝ|๐Ÿ˜ข|๐Ÿ˜ญ/g, 1.6], + [/ํž˜[๋“ค๋“ฆ๋“œ]|์•„[ํŒŒํ”„ํ”ˆ]|์ง€[์นœ์ณ์ณค]|๋ฏธ์•ˆ|์ฃ„์†ก|๋งํ–ˆ|์‹คํŒจ|ํฌ๊ธฐ/g, 1.2], [/\.{3,}/g, 0.4], + ], + angry: [ + [/ํ™”[๋‚˜๋‚œ๊ฐ€๋‚ฌ]|์งœ์ฆ|์—ด๋ฐ›|๋นก[์ณ์น˜์นœ]|๋ถ„๋…ธ|์–ต์šธ|์‹ซ[์–ด๋‹ค์€]|๊ทธ๋งŒํ•ด|๐Ÿ˜ |๐Ÿ˜ก/g, 1.8], + [/์ตœ์•…|์—‰๋ง|์–ด์ด์—†|๋ง๋„ ์•ˆ|์ฐธ[๋‚˜๋‚ด]/g, 1.4], + ], + surprise: [ + [/ํ—‰|๊นœ์ง|๋†€[๋ผ๋ž๋ž]|๋Œ€๋ฐ•|์„ธ์ƒ์—|์–ด๋จธ|์›ฌ์ผ|๋ฏฟ[์„๊ธฐ]\s*์ˆ˜\s*์—†|๐Ÿ˜ฒ|๐Ÿ˜ฎ/g, 1.6], + [/[?!]{2,}/g, 1.0], [/์ง„์งœ\?|์ •๋ง\?|\?/g, 0.3], + ], + fear: [ + [/๋ฌด[์„œ์„ญ์„ฐ]|๊ฒ[๋‚˜๋‚œ๋‚ฌ์ด]|์†Œ๋ฆ„|์„ฌ๋œฉ|์˜ค์‹น|๊ณตํฌ|๋”์ฐ|๐Ÿ˜ฑ|๐Ÿ˜จ/g, 1.8], + [/๋–จ[๋ ค๋ฆฐ๋ ธ]|๋ถˆ์•ˆ|์–ด๋–กํ•ด|์–ด์ฉŒ์ง€|์‚ด๋ ค/g, 1.2], + ], + shy: [ + [/๋ถ€๋„|์‘ฅ์Šค|๋ฏผ๋ง|์ˆ˜์ค|์ฐฝํ”ผ|๐Ÿ˜ณ|โ˜บ๏ธ/g, 1.8], + [/์–ด๋จธ๋‚˜|์•„์ด์ฐธ|ํ—คํ—ค|ํžˆํžˆ/g, 1.0], + ], + }; + // ๋ถ€์ • ํ‘œํ˜„ โ€” "์•ˆ ์ข‹์•„", "์ข‹์ง€ ์•Š์•„", "์žฌ๋ฏธ์—†์–ด"๋Š” ๊ธฐ์จ์ด ์•„๋‹ˆ๋‹ค. + const NEGATION = /์•ˆ\s*(์ข‹|๊ธฐ์˜|ํ–‰๋ณต|์ฆ๊ฑฐ|๋ฐ˜๊ฐ€)|(์ข‹|๊ธฐ์˜|ํ–‰๋ณต|์ฆ๊ฒ)์ง€\s*(์•Š|๋ชป)|์žฌ๋ฏธ\s*์—†|๋ณ„๋กœ|์‹ซ์ฆ/; + + /** ํ…์ŠคํŠธ โ†’ {emo, intensity} | null. ๋ชจ๋ธ ์—†์ด ์ ์ˆ˜ ์‚ฌ์ „์œผ๋กœ ์ถ”๋ก (์ •์  ๋ฐ๋ชจ์—์„œ๋„ ๋™์ž‘). */ + function inferEmotion(text) { + const score = {}; + for (const emo in EMO_RULES) score[emo] = 0; // ์‚ฌ์ „์— ๊ฐ์ • ์ถ”๊ฐ€ ์‹œ ์ž๋™ ๋ฐ˜์˜ + for (const emo in EMO_RULES) { + for (const [re, w] of EMO_RULES[emo]) { + const m = text.match(re); + if (m) score[emo] += w * Math.min(m.length, 3); // ๋ฐ˜๋ณต ๊ฐ•์กฐ๋Š” 3ํšŒ๊นŒ์ง€๋งŒ ๊ฐ€์‚ฐ + } + } + if (NEGATION.test(text)) { score.joy = 0; score.sad += 1.0; } + let best = null, top = 0; + for (const k in score) if (score[k] > top) { top = score[k]; best = k; } + if (!best || top < 1.0) return null; // ์‹ ํ˜ธ๊ฐ€ ์•ฝํ•˜๋ฉด ์ค‘๋ฆฝ ์œ ์ง€ + return { emo: best, intensity: Math.min(1, 0.45 + top * 0.16) }; // 0.45~1.0 + } + + // ---------- LLM ๊ฐ์ • ๋ถ„๋ฅ˜ (๊ทœ์น™ ๋งค์นญ์˜ ์ƒ์œ„ ๊ฒฝ๋กœ) ---------- + // inferEmotion ์ •๊ทœ์‹์€ ํ•œ๊ตญ์–ด 10๋ฌธ์žฅ ๋ฒค์น˜์—์„œ 1/10 ์ด์—ˆ๋‹ค โ€” ์ƒ์ฃผ LLM ์— ๋ฌผ์–ด๋ณธ๋‹ค. + // ์‹คํŒจยทํƒ€์ž„์•„์›ƒยท503 ์€ ์ „๋ถ€ null ๋กœ ์ ‘์–ด์„œ ํ˜ธ์ถœ์ธก์ด ๊ทœ์น™์œผ๋กœ ๋–จ์–ด์ง€๊ฒŒ ํ•œ๋‹ค. + async function classifyEmotion(text, { timeoutMs = 4000 } = {}) { + const ctl = new AbortController(); + const timer = setTimeout(() => ctl.abort(), timeoutMs); + try { + const res = await fetch("/api/emotion", { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text }), signal: ctl.signal, + }); + if (!res.ok) return null; + const segs = (await res.json()).segments; + return Array.isArray(segs) && segs.length ? segs : null; + } catch { + return null; // ์˜คํ”„๋ผ์ธยท์ค‘๋‹จยทํŒŒ์‹ฑ ์‹คํŒจ โ€” ๊ทœ์น™ ํด๋ฐฑ + } finally { + clearTimeout(timer); + } + } + + // ---------- ๊ฐ์ • โ†’ ๋ชฉ์†Œ๋ฆฌ ํ†ค (์–ผ๊ตด๋งŒ ์›ƒ๊ณ  ๋ชฉ์†Œ๋ฆฌ๋Š” ๋ฌดํ‘œ์ •ํ•œ ๊ดด๋ฆฌ ํ•ด์†Œ) ---------- + // ๋น„์œจ๊ฐ’: rate=์†๋„, pitch=์Œ๋†’์ด, volume=ํฌ๊ธฐ. edge-tts("+10%"/"+14Hz")์™€ ๋ธŒ๋ผ์šฐ์ € TTS(๋ฐฐ์ˆ˜) ์–‘์ชฝ์—์„œ ์‚ฌ์šฉ. + const VOICE_STYLE = { + joy: { rate: 0.10, pitch: 0.16 }, + sad: { rate: -0.12, pitch: -0.12 }, + angry: { rate: 0.06, pitch: -0.06, volume: 0.15 }, + surprise: { rate: 0.08, pitch: 0.22 }, + fear: { rate: 0.10, pitch: 0.10, volume: -0.10 }, // ๋น ๋ฅด๊ณ  ๋–จ๋ฆฌ๋Š” ์ž‘์€ ์†Œ๋ฆฌ + shy: { rate: -0.06, pitch: 0.06, volume: -0.15 }, // ์ž‘๊ณ  ์กฐ์‹ฌ์Šค๋Ÿฝ๊ฒŒ + neutral: {}, + }; + function voiceProsody(emo, intensity = 1) { + const s = VOICE_STYLE[emo] || VOICE_STYLE.neutral; + return { rate: (s.rate || 0) * intensity, pitch: (s.pitch || 0) * intensity, + volume: (s.volume || 0) * intensity }; + } + + // ์ง€์ˆ˜ ํ‰ํ™œ (๋ฌดํ• ๋‹น, in-place). target ์—๋งŒ ์žˆ๋Š” ์ƒˆ ํ‚ค๋Š” 0์—์„œ ์ถœ๋ฐœ. + function smoothStep(smooth, target) { + for (const k in target) if (!(k in smooth)) smooth[k] = 0; + for (const k in smooth) smooth[k] = 0.42 * smooth[k] + 0.58 * (target[k] || 0); + } + + // ์žฌ์ƒ ์ค‘ ๋ฐœํ™” ํ”„๋ ˆ์ž„ โ†’ ์ฑ„๋„ ๊ฐ€์ค‘์น˜ (puppetยทstudio3d; docs ๋Š” ๋ธŒ๋ผ์šฐ์ € TTS ํƒ€์ž„๋ผ์ธ์ด๋ผ ์ž์ฒด ๊ตฌํ˜„) + function weightsFromAnim(anim, audio) { + const w = {}; + if (anim && !audio.paused && !audio.ended) { + const f = anim.frames[Math.min(Math.floor(audio.currentTime * anim.fps), anim.frames.length - 1)]; + for (const [key, col] of anim.index) w[key] = f[col]; + } + return w; + } + + // ---------- ์—”์ง„ ์ถœ๋ ฅ ์…ฐ์ดํ•‘ (๋ฐœํ™”๋‹น 1ํšŒ) ---------- + // A2FยทNeuroSync ๋‘˜ ๋‹ค ํ„ฑ์„ ๊ฑฐ์˜ ์•ˆ ๋ฒŒ๋ฆฌ๊ณ (jawOpen ์ตœ๋Œ€ 0.31~0.40 ์‹ค์ธก) ๋ฐœํ™” ๋‚ด๋‚ด ์ผœ์ ธ ์žˆ๋Š” + // ํŽธํ–ฅ ์ฑ„๋„์ด ์žˆ๋‹ค(A2F jawRight ํ‰๊ท  0.17, NeuroSync browInnerUp 0.145). ์—”์ง„ ์ชฝ์—๋Š” ์„ธ๊ธฐ + // ์†์žก์ด๊ฐ€ ์—†์–ด์„œ(A2F model.json ์€ ๊ฒฝ๋กœ ์„ค์ •๋ฟ) ๋ฐ›์€ ํ”„๋ ˆ์ž„ ์ „์ฒด๋ฅผ ์—ฌ๊ธฐ์„œ ์ •๊ทœํ™”ํ•œ๋‹ค. + const SHAPE = { + a2f: { gain: { jawopen: 2.3 }, kill: ["jawright", "jawleft"] }, + // browInnerUpยทbrowOuterUpยทeyeWide ๋Š” NeuroSync๊ฐ€ ๋ฐœํ™” ๋‚ด๋‚ด ๋†€๋ž€ ํ‘œ์ •์„ ์ƒ์‹œ ๋„์šฐ๋Š” + // ์›์ธ(์ƒ์‹œ ์ผœ์ง์ด ์•„๋‹ˆ๋ผ ๋†’์€ ๊ฐ’ ์ฃผ๋ณ€์—์„œ ์ง„๋™) โ€” ๊ฒŒ์ธ<1๋กœ ๋ˆŒ๋Ÿฌ์•ผ ๋ชฉํ‘œ(ํ‰๊ท <=0.05)์— ๋‹ฟ๋Š”๋‹ค. + // 0.35/0.25 ๋Š” ์ฑ„๋„ ํ”„๋กœ๋ธŒ ์‹ค์ธก(๋ฌธ์žฅ1, ์ž์Œยท๋ฌด์Œ ์„ž์ธ ๋ฐœํ™”)์—์„œ ํ‰๊ท ์ด 0.047~0.060 ์œผ๋กœ + // ๋ชฉํ‘œ(10% ๋งˆ์ง„ 0.0455)์— ๊ฑธ์ณ ์žˆ์—ˆ๋‹ค โ€” ์žฌํ˜„ ์‹œ FAIL ์ด ๋‚˜์˜ฌ ์ˆ˜ ์žˆ๋Š” ์ˆ˜์ค€์ด๋ผ ๋” ๋ˆŒ๋ €๋‹ค. + // worst-case(brow 0.060, eyewide 0.050) ๊ธฐ์ค€ 0.035 ๋ชฉํ‘œ(๋งˆ์ง„์„ ๋ณด๋‹ค 23%โ†“, TTS ํ•ฉ์„ฑ ๋ณ€๋™ + // ์—ฌ์œ ๋ถ„)๋กœ ์—ญ์‚ฐ: brow 0.35*(0.035/0.060)=0.204โ†’0.20, eyewide 0.25*(0.035/0.050)=0.175โ†’0.18. + // ๊ฐ์ • ํ”„๋ฆฌ์…‹(EMOTIONS.sad/surprise/fear ์˜ brow*/eyewide* ๊ฐ’ 0.6~0.85)์€ applyMax ๋กœ + // ๋ณ„๋„ max-๊ฒฐํ•ฉ๋˜๋ฏ€๋กœ ์ด ๊ฒŒ์ธ๊ณผ ๋ฌด๊ด€ํ•˜๊ฒŒ ๊ทธ๋Œ€๋กœ ์œ ์ง€๋œ๋‹ค(makeEmotion.applyMax ์ฐธ๊ณ ). + neurosync: { gain: { jawopen: 2.8, mouthfunnel: 0.7, browinnerup: 0.20, browouterupleft: 0.20, + browouterupright: 0.20, eyewideleft: 0.18, eyewideright: 0.18 }, kill: [] }, + }; + const BASELINE_PCT = 10; // ์ฑ„๋„๋ณ„ ํ•˜์œ„ ๋ฐฑ๋ถ„์œ„ = "์ƒ์‹œ ์ผœ์ ธ ์žˆ๋Š”" ์„ฑ๋ถ„ + + function shapeAnim(anim, engine) { + const prof = SHAPE[engine]; + // ์ด๋ฒˆ ๋ฐœํ™”์— ์‹ค์ œ๋กœ ์ ์šฉํ•œ jawopen ๊ฒŒ์ธ์„ ๊ธฐ๋ก โ€” 3D ๋ชจํ”„์šฉ ์ฆํญ์ด๋ผ 2D(ํผํŽซ)์˜ ์ž… ํฌ๊ธฐยท + // ๋ชธ์ง“ ์ž„๊ณ„๊ฐ’์ฒ˜๋Ÿผ ์›๋ž˜(์ฆํญ ์ „) ์Šค์ผ€์ผ์— ๋งž์ถฐ์ง„ ์†Œ๋น„์ž๊ฐ€ ์ด๊ฑธ ๋‚˜๋ˆ  ๋˜๋Œ๋ฆฌ๋Š” ์šฉ๋„. + // ์—”์ง„ ๋ฏธ์ƒ/์ฆํญ ์—†์Œ์ด๋ฉด 1(๋ฌด๋ณด์ •). + if (anim) anim.jawGain = (prof && prof.gain.jawopen) || 1; + if (!prof || !anim || !anim.frames || !anim.frames.length || !anim.index) return anim; + const kill = new Set(prof.kill); + for (const [name, col] of anim.index) { + if (kill.has(name)) { + for (const f of anim.frames) f[col] = 0; + continue; + } + const vals = anim.frames.map(f => f[col]); // ์Šค๋ƒ…์ƒท โ€” ์•„๋ž˜์„œ f[col] ์„ ๋ฎ์–ด์จ๋„ ์›๋ณธ๊ฐ’ ๊ธฐ์ค€์œผ๋กœ ๊ณ„์‚ฐ + const base = vals.slice().sort((a, b) => a - b)[Math.floor((vals.length - 1) * BASELINE_PCT / 100)]; + const gain = prof.gain[name] || 1; + if (!base && gain === 1) continue; // ์†๋Œˆ ๊ฒƒ ์—†๋Š” ์ฑ„๋„์€ ๊ฑด๋„ˆ๋›ด๋‹ค + anim.frames.forEach((f, i) => { + f[col] = clamp01((vals[i] - base) * gain); + }); + } + return anim; + } + + // ---------- ๊ฐ์ • ํ”„๋ฆฌ์…‹ (studio3d ๋ฒ„์ „์ด superset ์ด๋ผ ๊ทธ๊ฒƒ์œผ๋กœ ํ†ตํ•ฉ) ---------- + const EMOTIONS = { + neutral: {}, + joy: { mouthsmileleft: 0.55, mouthsmileright: 0.55, cheeksquintleft: 0.45, cheeksquintright: 0.45, eyesquintleft: 0.25, eyesquintright: 0.25 }, + sad: { mouthfrownleft: 0.5, mouthfrownright: 0.5, browinnerup: 0.7, mouthshrugupper: 0.2 }, + angry: { browdownleft: 0.85, browdownright: 0.85, nosesneerleft: 0.4, nosesneerright: 0.4, mouthpressleft: 0.4, mouthpressright: 0.4, jawforward: 0.25 }, + surprise: { browinnerup: 0.6, browouterupleft: 0.75, browouterupright: 0.75, eyewideleft: 0.8, eyewideright: 0.8, jawopen: 0.3 }, + fear: { eyewideleft: 0.7, eyewideright: 0.7, browinnerup: 0.85, mouthstretchleft: 0.35, mouthstretchright: 0.35, jawopen: 0.12 }, + // shy ์˜ eyelookdown ์€ makeGaze ์ฑ„๋„ ๊ฒฐํ•ฉ์„ ํƒ€๊ณ  ๋ˆˆ๋™์ž๋„ ์‹ค์ œ๋กœ ๋‚ด๋ ค๊ฐ„๋‹ค. + shy: { mouthsmileleft: 0.3, mouthsmileright: 0.3, eyelookdownleft: 0.55, eyelookdownright: 0.55, mouthpressleft: 0.25, mouthpressright: 0.25 }, + }; + + const CROSSFADE_S = 0.25; // ๋ฌธ์žฅ ๊ฒฝ๊ณ„ ํ‘œ์ • ์ „ํ™˜ ์‹œ๊ฐ„ + + // ๊ฐ์ • ์ƒํƒœ + ๋ฒ„ํŠผ ๋ฐฐ์„ . buttons/activeColor ๋Š” ํŽ˜์ด์ง€๊ฐ€ ์ฃผ์ž…(2D #5b8cff / 3D #76b900). + function makeEmotion(buttons, activeColor) { + let emotion = EMOTIONS.neutral; + let sticky = true; // ์ˆ˜๋™ ๋ฒ„ํŠผ=์œ ์ง€, ์ž๋™(๋ฐœํ™” ๊ฐ์ •)=๋ฐœํ™” ๋๋‚˜๋ฉด ์ค‘๋ฆฝ ๋ณต๊ท€ + let hold = 1; // ๊ฐ์ • ์„ธ๊ธฐ ๊ฒŒ์ดํŠธ(0~1). ์ž๋™ ๊ฐ์ •์€ ์œ ํœด ์ค‘ ๊ฐ์‡ . + // intensity: ์ž๋™ ์ถ”๋ก  ์‹œ ๊ฐ์ • ์„ธ๊ธฐ(0~1)๋กœ ํ”„๋ฆฌ์…‹ ๊ฐ’์„ ์Šค์ผ€์ผ. ๋ฒ„ํŠผ ํด๋ฆญ์€ ํ•ญ์ƒ 1. + // ํ•ญ์ƒ ์ƒˆ ๊ฐ์ฒด๋กœ ์Šค์ผ€์ผ โ€” ๊ณต์œ  EMOTIONS ํ”„๋ฆฌ์…‹ ์•จ๋ฆฌ์–ด์‹ฑ ํšŒํ”ผ(v*1===v ๋ผ ๋ฌด์†์‹ค). + // isSticky=false(์ž๋™ ๋ฐœํ™” ๊ฐ์ •)๋ฉด ๋ฐœํ™”๊ฐ€ ๋๋‚œ ๋’ค ํ‘œ์ •์ด ์–ผ์–ด๋ถ™์ง€ ์•Š๊ณ  ์ฒœ์ฒœํžˆ ํ’€๋ฆฐ๋‹ค. + let curKey = "neutral", curInt = 1; // ๋ชธ์ง“ ์—ฐ๋™์šฉ ํ˜„์žฌ ๊ฐ์ • (current() ๋กœ ๋…ธ์ถœ) + let trackSeg = null, fadeFrom = {}; // ๋ฌธ์žฅ๋ณ„ ์ „ํ™˜์šฉ ํฌ๋กœ์ŠคํŽ˜์ด๋“œ ์ƒํƒœ + function setEmotion(key, intensity = 1, isSticky = true) { + const base = EMOTIONS[key] || EMOTIONS.neutral; + emotion = {}; + for (const k in base) emotion[k] = base[k] * intensity; + sticky = isSticky; hold = 1; + curKey = key; curInt = intensity; + buttons.forEach(x => x.style.background = x.dataset.emo === key ? activeColor : "#2a2a35"); + } + buttons.forEach(b => { b.onclick = () => setEmotion(b.dataset.emo); }); // ๋ฒ„ํŠผ์€ sticky ๊ธฐ๋ณธ + return { + setEmotion, + // ๋ฐœํ™” ์ค‘ ๋ฌธ์žฅ์ด ๋ฐ”๋€Œ๋ฉด ํ‘œ์ •๋„ ๋ฐ”๋€๋‹ค. track=[{start, emo, intensity}], tSec=audio.currentTime. + // setEmotion ์„ ์“ฐ์ง€ ์•Š๊ณ  ์ง์ ‘ ์„ž๋Š” ์ด์œ : setEmotion ์€ ์ƒˆ ํ”„๋ฆฌ์…‹์œผ๋กœ ํ†ต์งธ ๊ต์ฒด๋ผ + // ๊ฒฝ๊ณ„์—์„œ ์ด์ „ ํ‘œ์ •์ด ํ•œ ํ”„๋ ˆ์ž„์— ์‚ฌ๋ผ์ง„๋‹ค(๊ทธ๊ฒŒ ๋ฐ”๋กœ ์—†์• ๋ ค๋Š” ํŒ์ด๋‹ค). + // ๋ฌธ์žฅ์ด ๋ฐ”๋€Œ๋Š” ์ˆœ๊ฐ„์˜ ํ‘œ์ •์„ ๋ฐ•์ œํ•ด ๋‘๊ณ  CROSSFADE_S ๋™์•ˆ ์ƒˆ ํ”„๋ฆฌ์…‹๊ณผ ๊ฒน์ณ ๋„˜๊ธด๋‹ค. + followTrack(track, tSec) { + if (!track || !track.length || sticky) return; // ์ˆ˜๋™ ๋ฒ„ํŠผ์ด ๋ˆŒ๋ ธ์œผ๋ฉด ์‚ฌ์šฉ์ž ์˜๋„๊ฐ€ ์šฐ์„  + let seg = track[0]; + for (const t of track) if (tSec >= t.start) seg = t; + if (seg !== trackSeg) { fadeFrom = emotion; trackSeg = seg; } + // fadeAt=seg.start: ๊ฒฝ๊ณ„์—์„œ tSecโ‰ˆseg.start(์œ„ ๋ฃจํ”„๊ฐ€ tSec>=seg.start ์ธ ์„ธ๊ทธ๋ฅผ ๊ณ ๋ฅด๋ฏ€๋กœ)์ด๊ณ , + // ์ฒซ ํ˜ธ์ถœ์€ speakWithEmotion ์ด ์ด๋ฏธ ์„ธ๊ทธ 0 ํ”„๋ฆฌ์…‹์„ ์ ์šฉํ•ด๋‘” ๋’ค๋ผ k=1 ์ด ๊ณง๋ฐ”๋กœ ๋งž๋‹ค. + // ๋ณ„๋„ ๋ณ€์ˆ˜๋กœ ๊ด€์ธก ์‹œ๊ฐ์„ ๋ฐ•์ œํ•˜์ง€ ์•Š์œผ๋ฏ€๋กœ ์ผ์‹œ์ •์ง€ ํ›„์—๋„ ํŽ˜์ด๋“œ๊ฐ€ ์–ด๊ธ‹๋‚˜์ง€ ์•Š๋Š”๋‹ค. + const k = clamp01((tSec - seg.start) / CROSSFADE_S); + const base = EMOTIONS[seg.emo] || EMOTIONS.neutral; + const blended = {}; + for (const key in fadeFrom) blended[key] = fadeFrom[key] * (1 - k); + for (const key in base) blended[key] = (blended[key] || 0) + base[key] * seg.intensity * k; + emotion = blended; + hold = 1; // ๋ฐœํ™” ์ค‘ ์œ ์ง€, ๋๋‚˜๋ฉด ๊ธฐ์กด๋Œ€๋กœ ๊ฐ์‡  + curKey = seg.emo; curInt = seg.intensity; + }, + // ๊ฐ์ • ํ”„๋ฆฌ์…‹์„ ํ˜„์žฌ ํ‰ํ™œ๊ฐ’์— max-๊ฒฐํ•ฉ. speaking=๋ฐœํ™” ์ค‘์ด๋ฉด ์œ ์ง€, ์ž๋™ ๊ฐ์ •์€ ์œ ํœด ์‹œ ~1.5s ๊ฐ์‡ . + applyMax(smooth, speaking) { + hold = (sticky || speaking) ? 1 : hold * 0.98; + for (const k in emotion) smooth[k] = Math.max(smooth[k] || 0, emotion[k] * hold); + }, + // ํ˜„์žฌ ๊ฐ์ •๊ณผ ์„ธ๊ธฐ(ํ‘œ์ •๊ณผ ๊ฐ™์€ hold ๊ฐ์‡ ๋ฅผ ๊ณต์œ ) โ€” makeHeadWander ๋ชธ์ง“ ์—ฐ๋™์šฉ. + current() { return { key: curKey, level: curInt * hold }; }, + }; + } + + // ---------- ๊นœ๋นก์ž„ (๋ฒ„ํŠผ + ์ž๋™) ---------- + // autoBlink:()=>bool, intervalMs:()=>ms ๋Š” ๋งค ํ”„๋ ˆ์ž„ ๋ผ์ด๋ธŒ ์กฐํšŒ. duration/jitter ์ƒ์ˆ˜ ํŽ˜์ด์ง€๋ณ„ + // (puppet/docs 140ยท0.6ยท์Šฌ๋ผ์ด๋”, studio3d 150ยท0.8ยท3500). ์ƒํƒœ๋Š” ํด๋กœ์ €์— ์บก์Аํ™”. + function makeBlink({ autoBlink, intervalMs, duration, jitter }) { + let blinkUntil = 0, nextAutoBlink = performance.now() + 4000; + return { + trigger() { blinkUntil = performance.now() + duration; }, + value(now) { + if (autoBlink() && now > nextAutoBlink) { + blinkUntil = now + duration; + nextAutoBlink = now + intervalMs() * (0.7 + Math.random() * jitter); // ์ž์—ฐ์Šค๋Ÿฌ์šด ์ง€ํ„ฐ + } + return now < blinkUntil ? 1 : 0; + }, + }; + } + + // ---------- ์ปค์„œ ์‹œ์„  ์ถ”์  ---------- + // el ์˜ pointermove/leave โ†’ {gx, gy} (-1..1). ํ”„๋ ˆ์ž„๋ณ„ ํ‰ํ™œยท์—”์ง„์ฑ„๋„ ๊ฒฐํ•ฉ์€ makeGaze ๋˜๋Š” ํŽ˜์ด์ง€ ์ธ๋ผ์ธ. + function makeCursorTracker(el) { + const s = { gx: 0, gy: 0 }; + el.addEventListener("pointermove", e => { + const r = el.getBoundingClientRect(); + s.gx = ((e.clientX - r.left) / r.width - 0.5) * 2; + s.gy = ((e.clientY - r.top) / r.height - 0.5) * 2; + }); + el.addEventListener("pointerleave", () => { s.gx = 0; s.gy = 0; }); + return s; + } + + // ---------- ์‹œ์„  ๊ฒฐํ•ฉ (์Šฌ๋ผ์ด๋” > ์—”์ง„ ์ฑ„๋„ > ์ปค์„œ) + 0.15 ํ‰ํ™œ ---------- + // cursor: makeCursorTracker ๊ฒฐ๊ณผ, mulX/mulY: ์ปค์„œ ๋ฐฐ์œจ(puppetยทdocs 0.9/0.6, studio3d 0.8/0.5). + // ๋ฐ˜ํ™˜: (sliderVal, W) => [gx, gy]. docs ๋Š” ์Šฌ๋ผ์ด๋”๊ฐ€ ์—†์–ด sliderVal=0 ๊ณ ์ •์œผ๋กœ ํ˜ธ์ถœ. + function makeGaze(cursor, { mulX, mulY }) { + let gx = 0, gy = 0, sacX = 0, sacY = 0, sacNext = 0; + return (sliderVal, W) => { + const now = performance.now(); + const chX = (W("eyelookoutright") + W("eyelookinleft") - W("eyelookoutleft") - W("eyelookinright")) / 2; + const chY = (W("eyelookdownleft") + W("eyelookdownright") - W("eyelookupleft") - W("eyelookupright")) / 2; + const manual = Math.abs(sliderVal) > 0.01; + // ์Šฌ๋ผ์ด๋”ยท์—”์ง„์ฑ„๋„ยท์ปค์„œ ๋‹ค ์—†์œผ๋ฉด ์œ ํœด โ†’ ๋ˆˆ๋™์ž ๋ฏธ์„ธ saccade(์ฃฝ์€ ๋ˆˆ ๋ฐฉ์ง€) + const idle = !manual && !chX && !chY + && Math.abs(cursor.gx) < 0.02 && Math.abs(cursor.gy) < 0.02; + if (idle && now > sacNext) { + sacNext = now + 1200 + Math.random() * 2000; + const center = Math.random() < 0.35; // ๊ฐ€๋” ์ •๋ฉด ๋ณต๊ท€ + sacX = center ? 0 : (Math.random() - 0.5) * 0.5; + sacY = center ? 0 : (Math.random() - 0.5) * 0.3; + } + // ์šฐ์„ ์ˆœ์œ„: ์œ ํœด saccade, ์•„๋‹ˆ๋ฉด ์Šฌ๋ผ์ด๋” > ์—”์ง„์ฑ„๋„ > ์ปค์„œ + const tgtX = idle ? sacX : (manual ? sliderVal : (chX || cursor.gx * mulX)); + const tgtY = idle ? sacY : (chY || cursor.gy * mulY); + gx += (tgtX - gx) * 0.15; + gy += (tgtY - gy) * 0.15; + return [gx, gy]; + }; + } + + // ---------- ๋จธ๋ฆฌ ์›Œ๋ธ” (2D: ๋ฐœํ™” ๋„๋•์ž„ nod + ๋А๋ฆฐ ํ‘œ๋ฅ˜ wander + ์ž”์ž”ํ•œ ์‚ฌ์ธ) ---------- + // shakeEl ์— CSS ๋ณ€ํ™˜ ์ ์šฉ. sway ๋Š” ํŽ˜์ด์ง€๊ฐ€ ๋„˜๊น€(๋ฐœํ™” ์ค‘ 1, ์•„๋‹ˆ๋ฉด 0.5). studio3d ๋Š” 3D๋ผ ๋ฏธ์‚ฌ์šฉ. + // ๊ฐ์ • โ†’ ๋ชธ์ง“ ๊ณ„์ˆ˜ (speed ๋ฐฐ์†ยทamp ์ง„ํญยทdroop px ์•„๋ž˜๋กœยทbeat ๋„๋•์ž„ ๋ฐฐ์œจ). ์—†๋Š” ํ‚ค(neutral)๋Š” ์ „๋ถ€ 1/0. + const EMO_MOTION = { + joy: { speed: 1.25, amp: 1.5, droop: -2, beat: 1.3 }, // ๋“ค์ฉ์ž„ ์ปค์ง€๊ณ  ์‚ด์ง ๋“ค๋ฆผ + sad: { speed: 0.55, amp: 0.5, droop: 9, beat: 0.4 }, // ๊ณ ๊ฐœ ์ˆ™์ด๊ณ  ๋А๋ฆฌ๊ณ  ์ž‘๊ฒŒ + angry: { speed: 1.5, amp: 1.25, droop: 0, beat: 1.6 }, // ๋น ๋ฅด๊ณ  ์ ˆ๋„ ์žˆ๊ฒŒ + surprise: { speed: 1.2, amp: 1.3, droop: -4, beat: 1.0 }, // ๋ฒˆ์ฉ ๋“ค๋ฆผ + fear: { speed: 1.45, amp: 0.7, droop: 3, beat: 0.6 }, // ์›€์ธ ๋ฆฌ๊ณ  ์ž”๋–จ๋ฆผ + shy: { speed: 0.8, amp: 0.6, droop: 5, beat: 0.5 }, // ์ˆ˜์ค๊ฒŒ ์ˆ™์ž„ + }; + + // 2D ๋Š” ์Šคํ”„๋ผ์ดํŠธ๋ผ yawยทpitch ๋ฅผ ํšŒ์ „์‹œํ‚ฌ ์ˆ˜ ์—†๋‹ค โ€” ํ‰ํ–‰์ด๋™(%)์œผ๋กœ ๊ทผ์‚ฌํ•œ๋‹ค. + // ์ฝ”์–ด ํด๋žจํ”„(ยฑ0.5/ยฑ0.35rad)์—์„œ ์ตœ๋Œ€ ยฑ5%/ยฑ3.5% ์ด๋™ โ€” ๊ณผํ•˜๋ฉด ์–ผ๊ตด์ด ํ”„๋ ˆ์ž„์„ ๋ฒ—์–ด๋‚œ๋‹ค. + const HEAD_SHIFT = 10; + + function makeHeadWander() { + let nod = 0, wanderNext = 0, wanderR = 0, wanderY = 0, wanderGoalR = 0, wanderGoalY = 0; + let beat = 0, beatTilt = 0, lowSince = 0; // ๊ฐ•์กฐ ์ œ์Šค์ฒ˜: ๊ตฌ์ ˆ ์‹œ์ž‘๋งˆ๋‹ค ๋„๋•์ž„ ์ž„ํŽ„์Šค + let phR = 0, phY = 0, phB = 0, last = 0; // ์‚ฌ์ธ ์œ„์ƒ ๋ˆ„์  โ€” ๊ฐ์ • ๋ฐฐ์†์ด ๋ณ€ํ•ด๋„ ์œ„์ƒ ์—ฐ์†(์ ํ”„ ์—†์Œ) + // head: ์›น์บ  ๋จธ๋ฆฌ ์ž์„ธ [yaw, pitch, roll] (mirror.head(), ์„ ํƒ). 2D ๋ผ roll ์€ ์‹ค์ œ ๊ธฐ์šธ๊ธฐ, + // yawยทpitch ๋Š” ํšŒ์ „ ๋Œ€์‹  ํ‰ํ–‰์ด๋™์œผ๋กœ ๊ทผ์‚ฌํ•œ๋‹ค โ€” ์Šคํ”„๋ผ์ดํŠธ๋ฅผ 3D ๋กœ ๋Œ๋ฆด ์ˆ˜ ์—†์œผ๋ฏ€๋กœ. + return function tick(shakeEl, now, jawopen, sway, emoState, head) { + const m = emoState && EMO_MOTION[emoState.key], lv = m ? Math.min(emoState.level, 1) : 0; + const speed = 1 + ((m ? m.speed : 1) - 1) * lv; + const amp = 1 + ((m ? m.amp : 1) - 1) * lv; + const beatG = 1 + ((m ? m.beat : 1) - 1) * lv; + const droop = (m ? m.droop : 0) * lv; + const dt = last ? Math.min((now - last) / 1000, 0.1) : 0; + last = now; + phR += dt * 0.9 * speed; phY += dt * 1.7 * speed; phB += dt * 1.2 * speed; + nod = 0.85 * nod + 0.15 * jawopen; + // ์กฐ์šฉ(โ‰ฅ250ms)ํ•˜๋‹ค ์ž…์ด ์—ด๋ฆฌ๋Š” ์ˆœ๊ฐ„ = ๊ตฌ์ ˆ ์‹œ์ž‘ โ†’ ๋„๋•์ž„ ๋น„ํŠธ + ๊ณ ๊ฐœ ๊ธฐ์šธ์ž„ ๋ณ€์ฃผ. + // ๋งค ์Œ์ ˆ๋งˆ๋‹ค๊ฐ€ ์•„๋‹ˆ๋ผ pause ๋’ค ์˜จ์…‹์—๋งŒ ๊ฑธ๋ ค "๋ง์˜ ๋ฆฌ๋“ฌ"์ด ๋จ. + if (jawopen < 0.1) { + if (!lowSince) lowSince = now; + } else { + if (jawopen > 0.2 && lowSince && now - lowSince > 250) { + beat = 1; + beatTilt = (Math.random() - 0.5) * 0.02; + } + lowSince = 0; + } + beat *= 0.90; // ~0.5s ๊ฐ์‡  + if (now > wanderNext) { + wanderNext = now + 2200 + Math.random() * 2500; + wanderGoalR = (Math.random() - 0.5) * 0.03; + wanderGoalY = (Math.random() - 0.5) * 5; + } + wanderR += (wanderGoalR - wanderR) * 0.02; + wanderY += (wanderGoalY - wanderY) * 0.02; + const breath = Math.sin(phB) * 2 * amp; // ~5s ์ฃผ๊ธฐ ํ˜ธํก โ€” sway ๋ฌด๊ด€(์œ ํœด์—๋„ ์ˆจ ์‰ผ) + // wander ๋„ amp ๋กœ ์Šค์ผ€์ผ โ€” ์›€์ธ ๋ฆฐ ๊ฐ์ •(sad/fear)์€ ํ‘œ๋ฅ˜๊นŒ์ง€ ์ž‘์•„์ ธ์•ผ ์ผ๊ด€๋จ. + const rot = Math.sin(phR) * 0.008 * sway * amp + wanderR * amp + nod * 0.015 + beatTilt * beat * beatG; + const dy = Math.sin(phY) * 1.5 * sway * amp + wanderY * amp + nod * 3 + breath + beat * 7 * beatG + droop; + // ๋จธ๋ฆฌ ํ”๋“ค๋ฆผ์€ ๋‘ ์บ”๋ฒ„์Šค(WebGL base + 2D ์˜ค๋ฒ„๋ ˆ์ด)๋ฅผ ํ•จ๊ป˜ ๊ฐ์‹ผ ๋ž˜ํผ์— CSS ๋ณ€ํ™˜์œผ๋กœ ์ ์šฉ. + // transform-origin=center + translateY(% of height) ์กฐํ•ฉ์ด ๊ธฐ์กด ctx translate/rotate์™€ ์ˆ˜ํ•™์ ์œผ๋กœ ๋™์ผ. + // ์›น์บ  ๋จธ๋ฆฌ ์ž์„ธ๊ฐ€ ์žˆ์œผ๋ฉด ๊ฐ™์€ ๋ฌธ์ž์—ด์— ํ•ฉ์„ฑ โ€” ์œ ํœด ํ”๋“ค๋ฆผ ์œ„์— ์‹ค์ œ ๊ณ ๊ฐœ ๋ฐฉํ–ฅ์ด ์–นํžŒ๋‹ค. + const hR = head ? head[2] : 0; // ๊ฐธ์›ƒ = ์‹ค์ œ ํšŒ์ „ + const hX = head ? head[0] * HEAD_SHIFT : 0; // ์ขŒ์šฐ = ํ‰ํ–‰์ด๋™(%) + const hY = head ? head[1] * HEAD_SHIFT : 0; // ์ƒํ•˜ = ํ‰ํ–‰์ด๋™(%) + shakeEl.style.transform = + `rotate(${(rot + hR).toFixed(5)}rad) translate(${hX.toFixed(3)}%, ${(dy / 512 * 100 + hY).toFixed(4)}%)`; + }; + } + + // ---------- ์Šคํ”„๋ผ์ดํŠธ ์ž…๋ชจ์–‘ ์„ ํƒ๊ธฐ (๊ฐœ๋ฐฉ๋„ ์šฐ์„  + ํžˆ์Šคํ…Œ๋ฆฌ์‹œ์Šค) ---------- + // W ๋ฅผ ์ƒ์„ฑ ์‹œ ์ฃผ์ž…. targetMouth ๋Š” puppet superset โ€” docs ๋ฐœํ™”๊ฒฝ๋กœ์—์„œ mouthpressยทmouthstretch=0 ์ด๋ผ ์ •ํ™•ํžˆ ํ™˜์›. + // pick(now) โ†’ {cur, prev, fade}: ํฌ๋กœ์ŠคํŽ˜์ด๋“œ ์ƒํƒœ๋ฅผ ๊ณ„์‚ฐํ•ด ๋ฐ˜ํ™˜(๋‚ด๋ถ€ ์ƒํƒœ๋Š” ๋…ธ์ถœ ์•ˆ ํ•จ). + function makeMouthPicker(W) { + let curMouth = "closed", prevMouth = null, switchAt = 0, mouthCand = "closed", candSince = 0; + const FADE_MS = 90; + function targetMouth() { + const jaw = W("jawopen"); + const round = roundness(W); + const wide = Math.max(avgLR(W, "mouthsmile"), avgLR(W, "mouthstretch")); + const press = avgLR(W, "mouthpress"); + if (jaw < 0.06) return (press > 0.2 || W("mouthclose") > 0.25) ? "M" : "closed"; + if (round > wide + 0.08) return jaw > 0.28 ? "O" : "U"; + if (jaw > 0.42) return "A"; + if (wide > 0.22) return jaw < 0.16 ? "I" : "E"; + return jaw < 0.14 ? "closed" : "E"; + } + return { + pick(now) { + const t = targetMouth(); + if (t !== mouthCand) { mouthCand = t; candSince = now; } + if (mouthCand !== curMouth && now - candSince >= 70) { // 70ms ์œ ์ง€ ์‹œ์—๋งŒ ์ „ํ™˜ + prevMouth = curMouth; switchAt = now; curMouth = mouthCand; + } + return { cur: curMouth, prev: prevMouth, fade: Math.min(1, (now - switchAt) / FADE_MS) }; + }, + }; + } + + // ---------- ๋ฒกํ„ฐ ์ž… (๊ทผ์œก ์ฑ„๋„ โ†’ ์œค๊ณฝ ์ œ์–ด์  ์—ฐ์† ๋ณ€ํ˜•) ---------- + // puppet ์˜ superset ๊ณต์‹์œผ๋กœ ํ†ตํ•ฉ. ๋‹ซํž˜๊ณก์„  ์ œ์–ด์  ์••๋ ฅ = max(๊ทผ์œก press, mouthclose*0.5) ๋กœ + // puppet(press ์œ„์ฃผ)ยทdocs(mouthclose ์œ„์ฃผ) ์–‘์ชฝ ๊ธฐ์กด ํ”ฝ์…€์„ ํšŒ๊ท€ ์—†์ด ์žฌํ˜„. frown ๋ฐ˜์˜์€ puppet ํ•ญ. + function drawVectorMouth(ctx, W, manifest, jawDy) { + const st = manifest.mouthStyle || {}; + const [mcx, mcy0] = manifest.mouthCenter || [256, 340]; + const jaw = W("jawopen"); + const round = roundness(W); + const pressM = avgLR(W, "mouthpress"); // ๊ทผ์œก ์••๋ ฅ (openH ํํ•ฉ์šฉ) + const upperUp = avgLR(W, "mouthupperup"); + const lowerDown = avgLR(W, "mouthlowerdown"); + const smL = W("mouthsmileleft"), smR = W("mouthsmileright"); + const frL = W("mouthfrownleft"), frR = W("mouthfrownright"); + // ๋‹ซํž˜๊ณก์„  ์ œ์–ด์  ์••๋ ฅ: ๊ทผ์œก press ์™€ mouthclose ์œ ๋ž˜ ์••๋ ฅ ์ค‘ ๊ฐ•ํ•œ ์ชฝ (ํ•˜์ด๋ธŒ๋ฆฌ๋“œ โ€” puppetยทdocs ์–‘์ชฝ ํšŒ๊ท€ 0) + const pressCurve = Math.max(pressM, W("mouthclose") * 0.5); + + // ์˜ค๋ฏ€๋ฆผ(์˜ค/์šฐ): ์„ธ๋กœ ๊ฐœ๋ฐฉ์— ๋ฐ”๋‹ฅ๊ฐ’(+round*18)์„ ์ค˜ ๋‚ฎ์€ ํ„ฑ์—์„œ๋„ ๋‘ฅ๊ทผ ๊ตฌ๋ฉ์ด ์ƒ๊ธฐ๊ฒŒ ํ•œ๋‹ค. + const openH = Math.max(0, jaw * 58 + lowerDown * 10 + round * 18 - Math.max(pressM * 8, W("mouthclose") * 30)); + const wBase = st.width || 34; + const halfL = wBase * (1 + 0.45 * W("mouthstretchleft") + 0.3 * smL - 0.6 * round); // ์˜ค๋ฏ€๋ฆผ์ผ์ˆ˜๋ก ํญ ์ขํž˜ + const halfR = wBase * (1 + 0.45 * W("mouthstretchright") + 0.3 * smR - 0.6 * round); + const cy = mcy0 + jawDy; + const xL = mcx - halfL, xR = mcx + halfR; + const yCL = cy - 2 - smL * 12 + frL * 12; // ์ž…๊ผฌ๋ฆฌ ์ขŒ์šฐ ๋…๋ฆฝ (๋น„๋Œ€์นญ ํ‘œ์ •) + const yCR = cy - 2 - smR * 12 + frR * 12; + // ์˜ค๋ฏ€๋ฆผ ๊ฐ•ํ• ์ˆ˜๋ก ์œ„/์•„๋ž˜ ๊ณก์„ ์„ ๋Œ€์นญ(0.5/0.5)์œผ๋กœ โ†’ ๋‚ฉ์ž‘ํ•œ ๋ Œ์ฆˆ๊ฐ€ ์•„๋‹ˆ๋ผ ๋™๊ทธ๋ž€ O. + const topF = 0.38 + 0.12 * round, botF = 1 - topF; + const yU = cy - openH * topF - upperUp * 8; + const yD = cy + openH * botF; + + ctx.lineWidth = 5; ctx.lineCap = "round"; ctx.lineJoin = "round"; + ctx.strokeStyle = st.line || "#3a2e2a"; + + if (openH < 2.5) { // ๋‹ค๋ฌธ ์ž…: ์••๋ ฅยท๋ฏธ์†Œยท์ฐก๊ทธ๋ฆผ์ด ๊ณก์„  ํ•˜๋‚˜์— ๋ฐ˜์˜ + ctx.beginPath(); + ctx.moveTo(xL, yCL); + ctx.quadraticCurveTo(mcx, cy + 5 + pressCurve * 4, xR, yCR); + ctx.stroke(); + return; + } + const path = new Path2D(); + path.moveTo(xL, yCL); + path.quadraticCurveTo(mcx, 2 * yU - (yCL + yCR) / 2, xR, yCR); // ์œ—์ž…์ˆ  + path.quadraticCurveTo(mcx, 2 * yD - (yCL + yCR) / 2, xL, yCL); // ์•„๋žซ์ž…์ˆ  + path.closePath(); + ctx.fillStyle = st.fill || "#8a3535"; ctx.fill(path); + ctx.save(); ctx.clip(path); + // ์˜ค/์šฐ๋Š” ์ž…์ˆ ์ด ๋ชจ์—ฌ ์ดยทํ˜€๊ฐ€ ๊ฑฐ์˜ ์•ˆ ๋ณด์ž„ โ€” ์˜ค๋ฏ€๋ฆผ์— ๋น„๋ก€ํ•ด ์—ฐ์† ๊ฐ์‡ (๊ฒฝ๊ณ„ ํŒ ์—†์Œ). + const inner = Math.max(0, 1 - 1.3 * round); + if (openH > 7) { // ์œ—๋‹ˆ (inner=0 ์ด๋ฉด ๋†’์ด 0 = ์•ˆ ๊ทธ๋ ค์ง) + ctx.fillStyle = st.teeth || "#ffffff"; + ctx.fillRect(xL, yU - 2, xR - xL, Math.min(9, openH * 0.32) * inner); + } + if (openH > 18) { // ํ˜€ + ctx.fillStyle = st.tongue || "#d97b7b"; + ctx.beginPath(); + ctx.ellipse(mcx, yD, (xR - xL) * 0.3, openH * 0.28 * inner, 0, 0, Math.PI * 2); + ctx.fill(); + } + ctx.restore(); + ctx.stroke(path); + } + + // ---------- ์Šคํ”„๋ผ์ดํŠธ ์ž… ํฌ๋กœ์ŠคํŽ˜์ด๋“œ ---------- + // ๋ฐ˜๋“œ์‹œ ์Šคํ”„๋ผ์ดํŠธ ๋ชจ๋“œ ๋ธŒ๋žœ์น˜์—์„œ๋งŒ(ํ”„๋ ˆ์ž„๋‹น 1ํšŒ) ํ˜ธ์ถœ โ€” pick() ์ด ํžˆ์Šคํ…Œ๋ฆฌ์‹œ์Šค ์ƒํƒœ๋ฅผ ์ „์ง„์‹œํ‚ด. + // ์ „ํ™˜ ์ค‘(fade<1)์ด๊ณ  ์ด์ „ ์Šคํ”„๋ผ์ดํŠธ๊ฐ€ ์กด์žฌํ•˜๋ฉด ฮฑ๋กœ ๊ฒน์ณ ํŽ˜์ด๋“œ, ์•„๋‹ˆ๋ฉด ํ˜„์žฌ๋งŒ. jawDy ๋งŒํผ ์„ธ๋กœ ์ด๋™. + function drawSpriteMouth(ctx, parts, picker, now, jawDy) { + const { cur, prev, fade } = picker.pick(now); + const drawM = name => parts[name] && ctx.drawImage(parts[name], 0, jawDy); + if (fade < 1 && prev && parts["mouth_" + prev]) { + ctx.globalAlpha = 1 - fade; drawM("mouth_" + prev); + ctx.globalAlpha = fade; drawM("mouth_" + cur); + ctx.globalAlpha = 1; + } else { + drawM("mouth_" + cur); + } + } + + // ---------- WebGL ์–ผ๊ตด ์›Œํ•‘ (base ์ •์  ๋ณ€์œ„ ๊ทธ๋ฆฌ๋“œ) ---------- + // ์„ธ๋ถ„ ํ‰๋ฉด(512ร—512, 48์„ธ๊ทธ) + base ํ…์Šค์ฒ˜๋ฅผ ์ง๊ต์นด๋ฉ”๋ผ๋กœ ํ”ฝ์…€ ์ •ํ•ฉ ๋ Œ๋”. ์ •์ ์…ฐ์ด๋”๊ฐ€ ๊ทผ์œก ์ฑ„๋„๊ฐ’ + // (uniform)์œผ๋กœ ๊ฐ€์šฐ์‹œ์•ˆ ๋ณ€์œ„์žฅ์„ ์ ์šฉํ•ด ํ„ฑยท๋ณผ์„ ๋ฏธ์„ธ ๋ณ€ํ˜•. ์ƒ‰๊ณต๊ฐ„์€ NoColorSpace + ์ˆœ์ˆ˜ ์…ฐ์ด๋” + // ํŒจ์Šค์Šค๋ฃจ๋ผ ์›Œํ”„ 0์ผ ๋•Œ 2D drawImage ์™€ ํ”ฝ์…€ ๋™์ผ. + // threeUrl ํŽ˜์ด์ง€๋ณ„(/static/vendor/โ€ฆ ์ ˆ๋Œ€ vs ./vendor/โ€ฆ ์ƒ๋Œ€ โ€” ํด๋ž˜์‹ ์Šคํฌ๋ฆฝํŠธ๋ผ ๋ฌธ์„œ ๊ธฐ์ค€ ํ•ด์„). + // getParts/getManifest/W ๋Š” ํŽ˜์ด์ง€ ์ƒํƒœ ์ ‘๊ทผ์ž. + function makeWarp({ threeUrl, glCanvas, getParts, getManifest, W }) { + return { + ready: false, + T: null, renderer: null, scene: null, camera: null, material: null, texture: null, + // ์ž… ์˜ค๋ฒ„๋ ˆ์ด ์„ธ๋กœ ์ด๋™: ์›Œํ”„ ON์ด๋ฉด jaw ๋ณ€์œ„์žฅ์„ ์ž… ์•ต์ปค์—์„œ ํ‰๊ฐ€ํ•œ ๊ฐ’(โ‰ˆ11pxยทjaw)์œผ๋กœ ๋Œ€์ฒดํ•ด + // base ์›Œํ”„์™€ ์ •ํ™•ํžˆ ํ•จ๊ป˜ ์›€์ง์ด๊ฒŒ ํ•จ(์ด์ค‘ ์ด๋™ ๋ฐฉ์ง€). OFF๋ฉด ๊ธฐ์กด jawDrop ์‚ฌ์šฉ. (38/55๊ฐ€ ์›Œํ”„ ์‹œ๊ทธ๋งˆ ๊ฒฐํ•ฉ) + jawOverlayDy(jaw, warpOn, manifest) { + return warpOn ? 14 * jaw * WARP_JAW_G : jaw * (manifest.jawDrop || 8); + }, + vert: ` + uniform vec2 uJawC, uCornerL, uCornerR; + uniform float uJaw, uSmileL, uSmileR, uRound, uFrownL, uFrownR; + varying vec2 vUv; + float gk(vec2 p, vec2 c, float s){ vec2 d = p - c; return exp(-dot(d, d) / (2.0 * s * s)); } + void main() { + vUv = uv; + vec2 img = vec2(position.x + 256.0, 256.0 - position.y); // plane โ†’ ์ด๋ฏธ์ง€ ํ”ฝ์…€์ขŒํ‘œ(y down) + vec2 disp = vec2(0.0); + disp += vec2( 0.0, 14.0) * uJaw * gk(img, uJawC, 55.0); // ํ„ฑ ๋“œ๋กญ + disp += vec2(-7.0, -9.0) * uSmileL * gk(img, uCornerL, 32.0); // ์ขŒ ์ž…๊ผฌ๋ฆฌ (๋ณผ ๋‹น๊น€) + disp += vec2( 7.0, -9.0) * uSmileR * gk(img, uCornerR, 32.0); // ์šฐ ์ž…๊ผฌ๋ฆฌ + disp += vec2( 8.0, 0.0) * uRound * gk(img, uCornerL, 32.0); // ์˜ค๋ฏ€๋ฆผ (์•ˆ์ชฝ) + disp += vec2(-8.0, 0.0) * uRound * gk(img, uCornerR, 32.0); + disp += vec2(-3.0, 8.0) * uFrownL * gk(img, uCornerL, 32.0); // ์ฐก๊ทธ๋ฆผ (๋‚ด๋ฆผ) + disp += vec2( 3.0, 8.0) * uFrownR * gk(img, uCornerR, 32.0); + vec2 pos = position.xy; + pos.x += disp.x; pos.y -= disp.y; // ์ด๋ฏธ์ง€ y-down โ†’ plane y-up + gl_Position = projectionMatrix * modelViewMatrix * vec4(pos, 0.0, 1.0); + }`, + frag: ` + uniform sampler2D uTex; + varying vec2 vUv; + void main() { gl_FragColor = texture2D(uTex, vUv); }`, // colorspace include ์—†์Œ โ†’ sRGB ๋ฐ”์ดํŠธ ๊ทธ๋Œ€๋กœ + async init() { + try { + const T = await import(threeUrl); + this.T = T; + // preserveDrawingBuffer: ์›€์งค ๋…นํ™”๊ฐ€ drawImage ๋กœ gl ์บ”๋ฒ„์Šค๋ฅผ ์บก์ฒ˜ํ•  ๋•Œ ๋นˆ ํ”„๋ ˆ์ž„ ๋ฐฉ์ง€(์ •์„ ์˜ต์…˜). + this.renderer = new T.WebGLRenderer({ canvas: glCanvas, alpha: true, antialias: false, premultipliedAlpha: false, preserveDrawingBuffer: true }); + // ์…ฐ์ด๋” ๋งํฌ ์‹คํŒจ๋Š” three๊ฐ€ throwํ•˜์ง€ ์•Š๊ณ  ์ฝ˜์†” ๋กœ๊น…๋งŒ ํ•จ โ†’ ์ด ์ฝœ๋ฐฑ์œผ๋กœ ๊ฐ์ง€ํ•ด ํด๋ฐฑ ์ „ํ™˜. + this.renderer.debug.onShaderError = () => { this.ready = false; }; + this.renderer.setClearColor(0x000000, 0); + this.renderer.setSize(512, 512, false); + this.scene = new T.Scene(); + this.camera = new T.OrthographicCamera(-256, 256, 256, -256, -1, 1); + this.material = new T.ShaderMaterial({ + uniforms: { + uTex: { value: null }, + uJawC: { value: new T.Vector2(256, 378) }, + uCornerL: { value: new T.Vector2(220, 340) }, + uCornerR: { value: new T.Vector2(292, 340) }, + uJaw: { value: 0 }, uSmileL: { value: 0 }, uSmileR: { value: 0 }, + uRound: { value: 0 }, uFrownL: { value: 0 }, uFrownR: { value: 0 }, + }, + vertexShader: this.vert, fragmentShader: this.frag, transparent: true, side: T.DoubleSide, + }); + this.scene.add(new T.Mesh(new T.PlaneGeometry(512, 512, 48, 48), this.material)); + this.ready = true; + this.setCharacter(); // ๊ฒฝ์Ÿ ์ฒ˜๋ฆฌ: base๊ฐ€ ์ด๋ฏธ ๋กœ๋“œ๋์œผ๋ฉด ์—ฌ๊ธฐ์„œ ํ…์Šค์ฒ˜ ์„ค์ • + // init ์‹œ ๊ฐ•์ œ 1ํšŒ ๋ Œ๋” โ†’ ์…ฐ์ด๋” ์ปดํŒŒ์ผ/๋งํฌ๋ฅผ ์ง€๊ธˆ ์œ ๋ฐœ. ์‹คํŒจํ•˜๋ฉด throw(catch) ๋˜๋Š” + // onShaderError๊ฐ€ ready=false๋กœ ๋‚ด๋ ค ๋ Œ๋” ๋ฃจํ”„๊ฐ€ base๋ฅผ 2D๋กœ ๊ทธ๋ฆฌ๋Š” ํด๋ฐฑ์ด ํ™•์‹คํžˆ ์ž‘๋™. + this.renderer.render(this.scene, this.camera); + } catch (e) { + this.ready = false; // WebGL ๋ถˆ๊ฐ€/๋กœ๋“œ ์‹คํŒจ โ†’ ์กฐ์šฉํžˆ ํด๋ฐฑ + } + }, + setCharacter() { + const parts = getParts(), manifest = getManifest(); + if (!this.ready || !parts.base) return; + const T = this.T; + if (this.texture) this.texture.dispose(); + // ์ •์  ์บ๋ฆญํ„ฐ base๋Š” Image, ๋“œ๋ž˜๊ทธ์•ค๋“œ๋ž ์œ ์ € ์บ๋ฆญํ„ฐ base๋Š” canvas โ†’ CanvasTexture๋กœ ์ˆ˜์šฉ. + this.texture = (parts.base instanceof HTMLCanvasElement) + ? new T.CanvasTexture(parts.base) + : new T.Texture(parts.base); + this.texture.colorSpace = T.NoColorSpace; // sRGB ๋ฐ”์ดํŠธ ๊ทธ๋Œ€๋กœ ์—…๋กœ๋“œ (GPU ์„ ํ˜•ํ™” ์•ˆ ํ•จ) + this.texture.premultiplyAlpha = false; + this.texture.minFilter = T.LinearFilter; + this.texture.magFilter = T.LinearFilter; + this.texture.generateMipmaps = false; + this.texture.needsUpdate = true; + this.material.uniforms.uTex.value = this.texture; + const mc = manifest.mouthCenter || [256, 340]; + const mw = (manifest.mouthStyle && manifest.mouthStyle.width) || 30; + this.material.uniforms.uJawC.value.set(mc[0], mc[1] + 38); + this.material.uniforms.uCornerL.value.set(mc[0] - mw * 1.2, mc[1]); + this.material.uniforms.uCornerR.value.set(mc[0] + mw * 1.2, mc[1]); + }, + render() { + if (!this.ready || !this.texture) return; + const u = this.material.uniforms; + u.uJaw.value = W("jawopen"); + u.uSmileL.value = W("mouthsmileleft"); + u.uSmileR.value = W("mouthsmileright"); + u.uRound.value = roundness(W); + u.uFrownL.value = W("mouthfrownleft"); + u.uFrownR.value = W("mouthfrownright"); + this.renderer.render(this.scene, this.camera); + }, + }; + } + + // ---------- ๋ฐœํ™” ์š”์ฒญ โ†’ ๊ฒฐ๊ณผ (๋‚ด๋ถ€) ---------- + // ์„œ๋ฒ„๊ฐ€ ๋™๊ธฐ ์‘๋‹ต(0.6์ดˆ๊ธ‰ ์ž‘์—…) โ€” ์žก ํด๋ง ์ œ๊ฑฐ๋กœ ๋ฐœํ™”๋‹น ~0.2~0.35s ๋‹จ์ถ•. + async function speakRT({ text, voice, engine, prosody }) { + const res = await fetch("/api/speak_rt", { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text, voice, engine, ...(prosody || {}) }), + }); + if (!res.ok) throw new Error((await res.json()).detail || res.statusText); + return res.json(); + } + + // ---------- ๋ฐœํ™” ํ”Œ๋กœ์šฐ (puppetยทstudio3d ๊ณต์šฉ) ---------- + // speakRT โ†’ anim ์กฐ๋ฆฝ(head ํฌํ•จ, 2D๋Š” ๋ฏธ์‚ฌ์šฉ์ด๋ผ ๋ฌดํ•ด) โ†’ onAnim(์žฌ์ƒ ์ „ ๋ฐ˜์˜) โ†’ audio.src โ†’ play. + // onAnim ์€ play ๋กœ๋”ฉ ์ฐฝ ๋™์•ˆ ๋ Œ๋” ๋ฃจํ”„๊ฐ€ ์ตœ์‹  anim ์„ ๋ณด๊ฒŒ ํ•˜๋ ค๊ณ  src/play ์ด์ „์— ํ˜ธ์ถœ(์›๋ณธ ์ˆœ์„œ ์œ ์ง€). + // ํŽ˜์ด์ง€ ํ•ธ๋“ค๋Ÿฌ๋Š” autoEmoยท๋ฒ„ํŠผ disableยท์—๋Ÿฌ setStatus ๋งŒ ๋‹ด๋‹น. + async function speakFlow({ text, voice, engine, audioEl, onAnim, prosody }) { + const r = await speakRT({ text, voice, engine, prosody }); + const anim = { fps: r.fps, frames: r.frames, head: r.head, index: r.names.map((n, i) => [norm(n), i]) }; + shapeAnim(anim, engine); // ์—”์ง„ ์ถœ๋ ฅ ์ •๊ทœํ™” (์ž… ๋ฒŒ๋ฆผยทํŽธํ–ฅ ์ œ๊ฑฐ) + anim.sentences = r.sentences; // ๋ฌธ์žฅ ์‹œ์ž‘ ์‹œ๊ฐ (์—†์„ ์ˆ˜ ์žˆ์Œ) + if (onAnim) onAnim(anim); + audioEl.src = r.audio_url; + await audioEl.play(); + return anim; + } + + // ---------- ๊ฐ์ • ๊ฒฐ์ • + ๋ฐœํ™” (puppetยทstudio3d ๊ณต์šฉ) ---------- + // emotion ์ง€์ •(LLM ํŒ๋‹จ) ์žˆ์œผ๋ฉด ๊ทธ๋Œ€๋กœ, ์—†์œผ๋ฉด LLM ๋ถ„๋ฅ˜ โ†’ ์‹คํŒจ ์‹œ ๊ทœ์น™ ์ถ”๋ก . autoEmo(ํ˜ธ์ถœ ์‹œ์  boolean) ์ผœ์ ธ ์žˆ์œผ๋ฉด + // emo(makeEmotion ์ธ์Šคํ„ด์Šค) ํ”„๋ฆฌ์…‹ + ๋ชฉ์†Œ๋ฆฌ ํ†ค ์ ์šฉ ํ›„ speakFlow. voice/engine ์€ ํ˜ธ์ถœ ์‹œ์  ๊ฐ’. + async function speakWithEmotion({ text, emotion, autoEmo, emo, voice, engine, audioEl, onAnim }) { + let segs = null; + let r = emotion ? { emo: emotion, intensity: 0.9 } : null; + if (!r && autoEmo) { + // ์ง๋ ฌ ํ˜ธ์ถœ: ๋ชฉ์†Œ๋ฆฌ ํ†ค(prosody)์ด TTS ์š”์ฒญ ํŒŒ๋ผ๋ฏธํ„ฐ๋ผ ๊ฐ์ •์„ ๋จผ์ € ์•Œ์•„์•ผ ํ•œ๋‹ค. + segs = await classifyEmotion(text); + r = segs ? { emo: segs[0].emo, intensity: segs[0].intensity } : inferEmotion(text); + } else if (!r) { + r = inferEmotion(text); + } + let prosody = null; + if (r && autoEmo) { + emo.setEmotion(r.emo, r.intensity, false); // ์ž๋™ ๊ฐ์ • โ€” ๋ฐœํ™” ๋๋‚˜๋ฉด ์ค‘๋ฆฝ ๋ณต๊ท€ + prosody = voiceProsody(r.emo, r.intensity); + } + const anim = await speakFlow({ text, voice, engine, audioEl, onAnim, prosody }); + // ๋ฌธ์žฅ๋ณ„ ๊ฐ์ • ์ „ํ™˜์€ ๋‘ ๋ฐฐ์—ด์˜ ๊ธธ์ด๊ฐ€ ๋งž์„ ๋•Œ๋งŒ (๋ถ„ํ•  ๊ฒฐ๊ณผ๊ฐ€ ๊ฐ™๋‹ค๋Š” ์ „์ œ ํ™•์ธ) + if (segs && anim.sentences && anim.sentences.length === segs.length) { + anim.emotionTrack = anim.sentences.map((s, i) => ({ + start: s.start, emo: segs[i].emo, intensity: segs[i].intensity })); + } + return anim; + } + + // ---------- ์ž๋™ ์‡ผ์ผ€์ด์Šค (์ฒซ ๋ฐฉ๋ฌธ์ž ์œ ํœด ์‹œ ์ธ์‚ฌยท๊ฐ์ • ์‹œ์—ฐ) ---------- + // ์˜ค๋Š˜ ๋„ฃ์€ ํ‘œํ˜„๋ ฅ(๊ฐ์ • 7์ข…ยท๋ชธ์ง“ยท์ž… ์˜ค๋ฏ€๋ฆผ)์„ ๋ฐฉ๋ฌธ์ž๊ฐ€ ์•„๋ฌด๊ฒƒ๋„ ์•ˆ ๋ˆŒ๋Ÿฌ๋„ ๋ณด๊ฒŒ ํ•˜๋Š” ์œ ๋„. + const SHOWCASE_SCRIPT = [ + { text: "์•ˆ๋…•ํ•˜์„ธ์š”! ์ €๋Š” ๊ทธ๋ฆผ์—์„œ ํƒœ์–ด๋‚œ ์•„๋ฐ”ํƒ€์˜ˆ์š”.", emo: "joy" }, + { text: "์ด๋ ‡๊ฒŒ ํ™œ์ง ์›ƒ๊ธฐ๋„ ํ•˜๊ณ ,", emo: "joy" }, + { text: "์‹œ๋ฌด๋ฃฉํ•ด์ง€๊ธฐ๋„,", emo: "sad" }, + { text: "๊นœ์ง ๋†€๋ผ๊ธฐ๋„,", emo: "surprise" }, + { text: "๋ฌด์„œ์›Œํ•˜๊ธฐ๋„,", emo: "fear" }, + { text: "๋ถ€๋„๋Ÿฌ์›Œํ•˜๊ธฐ๋„ ํ•œ๋‹ต๋‹ˆ๋‹ค.", emo: "shy" }, + { text: "์œ„์— ๋ฌธ์žฅ์„ ์ž…๋ ฅํ•˜๋ฉด ์ œ๊ฐ€ ๋งํ•ด๋“œ๋ฆด๊ฒŒ์š”!", emo: "joy" }, + ]; + // playStep(step) โ†’ Promise(๋ฐœํ™” ์™„๋ฃŒ). ๋ฐœํ™” ๋ฐฉ์‹์€ ํŽ˜์ด์ง€๊ฐ€ ์ฃผ์ž…(docs=speechSynthesis). + // ์œ ํœด ๊ฐ์ง€ยท์ค‘๋‹จ ํŠธ๋ฆฌ๊ฑฐ๋Š” ์—˜๋ฆฌ๋จผํŠธ๊ฐ€ ํŽ˜์ด์ง€๋งˆ๋‹ค ๋‹ฌ๋ผ ํŽ˜์ด์ง€๊ฐ€ ์†Œ์œ  โ€” ์—ฌ๊ธด ์ˆœ์ฐจ ์žฌ์ƒ ์—”์ง„๋งŒ. + function makeShowcase(playStep, script = SHOWCASE_SCRIPT) { + let running = false, cancelled = false; + return { + get running() { return running; }, + async play() { + if (running) return; + running = true; cancelled = false; + for (const step of script) { + if (cancelled) break; + await playStep(step); + if (cancelled) break; + await new Promise(r => setTimeout(r, 220)); // ์Šคํ… ์‚ฌ์ด ์งง์€ ์ˆจ + } + running = false; + }, + stop() { cancelled = true; }, + }; + } + + // ---------- ์บ๋ฆญํ„ฐ ํด๋ฆญ ๋ฐ˜์‘ (์•„๋ฐ”ํƒ€๋ฅผ ๋ˆ„๋ฅด๋ฉด ๊ฐ์ • ์„ž์ธ ํ•œ๋งˆ๋””) ---------- + const REACTIONS = [ + { text: "์šฐ์™€, ๊นœ์ง์ด์•ผ!", emo: "surprise" }, + { text: "ํ—คํ—ค, ๊ฐ„์ง€๋Ÿฌ์›Œ์š”.", emo: "joy" }, + { text: "์•„์ด, ๋ถ€๋„๋Ÿฝ๊ฒŒ ์™œ ๊ทธ๋ž˜์š”.", emo: "shy" }, + { text: "์•ˆ๋…•ํ•˜์„ธ์š”! ๋ฐ˜๊ฐ€์›Œ์š”.", emo: "joy" }, + { text: "์–ด? ์™œ ๊ทธ๋Ÿฌ์„ธ์š”?", emo: "surprise" }, + { text: "์œผ, ์‚ด์‚ด ํ•ด์ฃผ์„ธ์š”.", emo: "fear" }, + ]; + let _lastReact = -1; + function pickReaction() { // ์ง์ „๊ณผ ๋‹ค๋ฅธ ๋ฐ˜์‘์„ ๋ฝ‘์•„ ์—ฐ์† ์ค‘๋ณต ๋ฐฉ์ง€ + let i; + do { i = Math.floor(Math.random() * REACTIONS.length); } while (i === _lastReact && REACTIONS.length > 1); + _lastReact = i; + return REACTIONS[i]; + } + + // ---------- ๋ถ„์„ ํŒจ๋„ (๋น„๊ต๊ตฐ): ๋‚ด ์–ผ๊ตด + 478์  ๋žœ๋“œ๋งˆํฌ + ๊ตฌ๋™ ์ฑ„๋„๊ฐ’ ---------- + // ์–ด๋А ํŽ˜์ด์ง€๋“  ์ปจํ…Œ์ด๋„ˆ ํ•˜๋‚˜ ์ฃผ๋ฉด ๋™์ผ ํŒจ๋„ โ€” ์Šคํƒ€์ผ ์ธ๋ผ์ธ์ด๋ผ ํŽ˜์ด์ง€ CSS ์˜์กด ์—†์Œ. + // ๋ฐ˜ํ™˜๋œ draw(W) ๋ฅผ ๋ Œ๋” ๋ฃจํ”„์—์„œ ๋งค ํ”„๋ ˆ์ž„ ํ˜ธ์ถœ (W = ์ฑ„๋„ ์ ‘๊ทผ์ž). ์ปจํ…Œ์ด๋„ˆ ์ˆจ๊น€์ด๋ฉด ์ฆ‰์‹œ ๋ฐ˜ํ™˜. + // ๋ฏธ๋ฆฌ๋ณด๊ธฐ: ์›น์บ  + ์–ผ๊ตด ๋ฉ”์‹œ(์–ด๋””๋ฅผ ์žก์•˜๋‚˜) + ์˜ค๊ฐํ˜•(์–ผ๋งˆ๋‚˜ ์›€์ง์˜€๋‚˜). + // ๊ฐ’ ๋ง‰๋Œ€๋ฅผ ์Œ“์ง€ ์•Š๋Š” ์ด์œ  โ€” ์ฑ„๋„ ๊ณ„์ˆ˜๋Š” ์ „์†ก ๋ฐ์ดํ„ฐ์ด์ง€ ์ฝ๋Š” ์ •๋ณด๊ฐ€ ์•„๋‹ˆ๊ณ (Live Link Faceยท + // VSeeFace ๋„ ๋ฉ”์‹œ๋งŒ ๊ฒน์ณ ๋ณด์—ฌ์ค€๋‹ค), ๋ชจ๋ธ ๋‚ด๋ถ€ ์ฑ„๋„๋ช…ยท0.00 ์ •๋ฐ€๋„๋ฅผ UI ์— ๋…ธ์ถœํ•˜๋ฉด ๊ฐ’์„ + // ๊ทธ๋Œ€๋กœ ๋ Œ๋”ํ•œ ํ‹ฐ๊ฐ€ ๋‚œ๋‹ค. ๋Œ€์‹  ๋‹ค์„ฏ ์ถ•์ด ํ•œ ๋ฉ์–ด๋ฆฌ๋กœ ์ผ๊ทธ๋Ÿฌ์ง€๋Š” ๋ชจ์–‘์„ ๋ณด์—ฌ์ค€๋‹ค: + // ๊ฐœ๋ณ„ ์ˆ˜์น˜๋Š” ๋ชป ์ฝ์–ด๋„ "์ง€๊ธˆ ์‚ด์•„์„œ ๋ฐ˜์‘ํ•œ๋‹ค"๊ฐ€ ํ•œ๋ˆˆ์— ๋“ค์–ด์˜จ๋‹ค. + // + // ์‹œ์„ ์€ ์—ฌ๊ธฐ ์—†๋‹ค. ํฌ๊ธฐ๊ฐ€ ์•„๋‹ˆ๋ผ ๋ฐฉํ–ฅ(์ขŒ์šฐยท์ƒํ•˜)์ด๋ผ ์ถ• ํ•˜๋‚˜์— ์˜ฌ๋ฆด ์ˆ˜ ์—†๊ณ , + // ๋ฉ”์‹œ ์œ„ ํ™์ฑ„ ์ ์ด ์ด๋ฏธ ๊ทธ ์ž์ฒด๋กœ ๋ณด์—ฌ์ค€๋‹ค. + const RADAR_CHS = [ + ["์ž…", W => W("jawopen")], + ["๋ฏธ์†Œ", W => (W("mouthsmileleft") + W("mouthsmileright")) / 2], + ["์˜ค๋ฏ€๋ฆผ", W => W("mouthpucker")], + ["๋ˆˆ", W => (W("eyeblinkleft") + W("eyeblinkright")) / 2], + ["๋ˆˆ์น", W => W("browinnerup")], + ]; + function makeMirrorPanel(mirror, mount) { + let lastFrame = -1; // ๋งˆ์ง€๋ง‰์œผ๋กœ ๋ฏธ๋ฆฌ๋ณด๊ธฐ์— ๊ทธ๋ฆฐ ์›น์บ  ํ”„๋ ˆ์ž„ ๋ฒˆํ˜ธ + mount.innerHTML = '
๋‚ด ์–ผ๊ตด โ€” ํŠธ๋ž˜ํ‚น ๋ฏธ๋ฆฌ๋ณด๊ธฐ
'; + const cv = document.createElement("canvas"); + cv.width = 320; cv.height = 240; + cv.style.cssText = "width:100%;border-radius:12px;border:1px solid #2a2a35;background:#0d0d12;display:block"; + mount.appendChild(cv); + const ctx = cv.getContext("2d"); + + const rv = document.createElement("canvas"); + rv.width = 320; rv.height = 218; + rv.style.cssText = "width:100%;margin-top:8px;display:block"; + mount.appendChild(rv); + const rc = rv.getContext("2d"); + // 12์‹œ๋ถ€ํ„ฐ ์‹œ๊ณ„๋ฐฉํ–ฅ. ์ถ• ์ˆœ์„œ๋ฅผ ๋ฐ”๊พธ๋ฉด ๊ฐ™์€ ํ‘œ์ •์ด ๋‹ค๋ฅธ ๋ชจ์–‘์ด ๋˜๋ฏ€๋กœ ๊ณ ์ •์ด๋‹ค. + const N = RADAR_CHS.length, CX = 160, CY = 118, R = 90; + const AX = i => -Math.PI / 2 + i * 2 * Math.PI / N; + const poly = (get, close) => { + rc.beginPath(); + for (let i = 0; i < N; i++) { + const r = get(i), a = AX(i); + const x = CX + Math.cos(a) * r, y = CY + Math.sin(a) * r; + i ? rc.lineTo(x, y) : rc.moveTo(x, y); + } + if (close !== false) rc.closePath(); + }; + + return { + canvas: cv, // ๋™์‹œ ๊ตฌ๋™ ํ™”๋ฉด์˜ ๋…นํ™” ํ•ฉ์„ฑ์šฉ + draw(W) { + if (!mount.offsetParent) return; // ์ˆจ๊น€ ์ƒํƒœ โ€” ์ผ ์•ˆ ํ•จ + const d = mirror.debug(); + // ์›น์บ  ํ”„๋ ˆ์ž„์ด ๊ทธ๋Œ€๋กœ๋ฉด ๊ฐ™์€ ๊ทธ๋ฆผ์„ ๋‹ค์‹œ ๊ทธ๋ฆฌ๊ฒŒ ๋œ๋‹ค โ€” ๋ Œ๋” ๋ฃจํ”„(60~144fps)๊ฐ€ + // ์ถ”๋ก (~30fps)๋ณด๋‹ค ๋นจ๋ผ 4~5๋ฐฐ ํ—›์ผ. ์˜ค๊ฐํ˜•์€ ๊ฐ’์ด ๊ณ„์† ๋ณ€ํ•˜๋ฏ€๋กœ ๋งค ํ”„๋ ˆ์ž„ ๊ฐฑ์‹ ํ•œ๋‹ค. + if (d.frame !== lastFrame || !mirror.on) { + lastFrame = d.frame; + const w = cv.width, h = cv.height; + ctx.fillStyle = "#0d0d12"; ctx.fillRect(0, 0, w, h); + if (!mirror.on || !d.video || d.video.readyState < 2) { + ctx.fillStyle = "#9a9ab0"; ctx.font = "13px sans-serif"; ctx.textAlign = "center"; + ctx.fillText("๋ฏธ๋Ÿฌ๋ง์„ ์‹œ์ž‘ํ•˜๋ฉด ํ‘œ์‹œ๋ฉ๋‹ˆ๋‹ค", w / 2, h / 2); + } else { + ctx.save(); ctx.scale(-1, 1); ctx.drawImage(d.video, -w, 0, w, h); ctx.restore(); // ๊ฑฐ์šธ ๋ฐ˜์ „ + if (d.lm) { + const X = i => (1 - d.lm[i].x) * w, Y = i => d.lm[i].y * h; + if (d.mesh) { + // MediaPipe ํ‘œ์ค€ ํ† ํด๋กœ์ง€(2556 ์„ธ๊ทธ๋จผํŠธ)๋ฅผ ํ•œ ๊ฒฝ๋กœ๋กœ ๋ชจ์•„ ํ•œ ๋ฒˆ์— stroke โ€” + // ๊ฐœ๋ณ„ stroke ๋ฉด ํ”„๋ ˆ์ž„์„ ๋จน๋Š”๋‹ค. + ctx.beginPath(); + for (const c of d.mesh) { + const a = c.start ?? c[0], b = c.end ?? c[1]; + ctx.moveTo(X(a), Y(a)); ctx.lineTo(X(b), Y(b)); + } + ctx.strokeStyle = "rgba(232,232,239,.22)"; ctx.lineWidth = 0.5; ctx.stroke(); + } else { + ctx.fillStyle = "rgba(232,232,239,.5)"; // ๊ตฌ๋ฒ„์ „ tasks-vision ํด๋ฐฑ + for (let i = 0; i < 468; i++) ctx.fillRect(X(i) - .5, Y(i) - .5, 1.5, 1.5); + } + ctx.fillStyle = "#ffb03a"; // ํ™์ฑ„ 10์  = ์‹œ์„  ๊ณ„์‚ฐ์˜ ์‹ค์ œ ์ž…๋ ฅ + for (let i = 468; i < d.lm.length; i++) ctx.fillRect(X(i) - 1.5, Y(i) - 1.5, 3, 3); + } + } + } + + rc.clearRect(0, 0, rv.width, rv.height); + rc.strokeStyle = "rgba(232,232,239,.13)"; rc.lineWidth = 1; + for (const f of [0.5, 1]) { poly(() => R * f); rc.stroke(); } + rc.beginPath(); + for (let i = 0; i < N; i++) { rc.moveTo(CX, CY); rc.lineTo(CX + Math.cos(AX(i)) * R, CY + Math.sin(AX(i)) * R); } + rc.stroke(); + + poly(i => Math.max(0, Math.min(1, RADAR_CHS[i][1](W))) * R); + rc.fillStyle = "rgba(255,176,58,.20)"; rc.fill(); + rc.strokeStyle = "#ffb03a"; rc.lineWidth = 1.5; rc.stroke(); + + rc.fillStyle = "#9a9ab0"; rc.font = "13px sans-serif"; rc.textAlign = "center"; rc.textBaseline = "middle"; + for (let i = 0; i < N; i++) { + const a = AX(i); + rc.fillText(RADAR_CHS[i][0], CX + Math.cos(a) * (R + 22), CY + Math.sin(a) * (R + 16)); + } + }, + }; + } + + // ---------- 2D ์บ๋ฆญํ„ฐ ๊ทธ๋ฆฌ๊ธฐ (์†Œ๋…„ยท์†Œ๋…€ ๊ณต์šฉ) ---------- + // ํŒŒ์ธ  ์Šคํ”„๋ผ์ดํŠธ(๋ˆˆ์นยท๋ˆˆยท๋™๊ณต) + ๋ฒกํ„ฐ ์ž…์„ ํ•œ ๋ฒˆ์—. ์›Œํ”„๋ฅผ ์“ฐ๋Š” ํŽ˜์ด์ง€๋Š” warp ๋ฅผ ๋„˜๊ธฐ๋ฉด + // ๋ฐฐ๊ฒฝ/ํ„ฑ ์˜คํ”„์…‹์„ ์›Œํ”„ ๊ธฐ์ค€์œผ๋กœ, ์•ˆ ๋„˜๊ธฐ๋ฉด ํฐ ๋ฐฐ๊ฒฝ + manifest.jawDrop ํด๋ฐฑ์œผ๋กœ ๊ทธ๋ฆฐ๋‹ค. + // + // ctx: 2D ์ปจํ…์ŠคํŠธ(512ยฒ), parts: ์ด๋ฏธ์ง€ ๋งต, manifest: ์บ๋ฆญํ„ฐ ํŠœ๋‹๊ฐ’, + // W: ์ฑ„๋„ ์ ‘๊ทผ์ž, blink: ๊นœ๋นก์ž„(์ž๋™+๋ฏธ๋Ÿฌ max ๊ฒฐํ•ฉ), gaze: [gx, gy], + // opts.warp: makeWarp ์ธ์Šคํ„ด์Šค(์„ ํƒ), opts.clearBg: ๋ฐฐ๊ฒฝ ์ฑ„์šฐ๊ธฐ ์—ฌ๋ถ€(ํด๋ฆฐ ๋ชจ๋“œ๋ฉด false) + function drawChar2D(ctx, { parts, manifest, W, blink, gaze, warp, clearBg = true, head }) { + if (!parts.base || !manifest) return false; + const warpOn = !!(warp && warp.ready); + ctx.clearRect(0, 0, 512, 512); + // head ๋ฅผ ์ฃผ๋ฉด ์บ”๋ฒ„์Šค ์ž์ฒด๋ฅผ ๋ณ€ํ˜•ํ•œ๋‹ค โ€” CSS ๋ณ€ํ™˜์ด ์•ˆ ๋จน๋Š” ๊ฒฝ๋กœ(์บ”๋ฒ„์Šค๋ฅผ drawImage ๋กœ + // ํ•ฉ์„ฑํ•˜๋Š” ๋…นํ™” ๋“ฑ)์šฉ. CSS ๋ž˜ํผ๋ฅผ ์“ฐ๋Š” ํŽ˜์ด์ง€๋Š” head ๋ฅผ ๋„˜๊ธฐ์ง€ ๋ง ๊ฒƒ(์ด์ค‘ ์ ์šฉ). + if (head) { + ctx.save(); + ctx.translate(256 + head[0] * HEAD_SHIFT * 5.12, 256 + head[1] * HEAD_SHIFT * 5.12); + ctx.rotate(head[2]); + ctx.translate(-256, -256); + } + const draw = (n, dy = 0) => parts[n] && ctx.drawImage(parts[n], 0, dy); + const drawXY = (n, dx, dy) => parts[n] && ctx.drawImage(parts[n], dx, dy); + // ์›Œํ”„๊ฐ€ ์ผœ์ ธ ์žˆ์œผ๋ฉด base ๋Š” WebGL ๋ ˆ์ด์–ด๊ฐ€ ๊ทธ๋ฆฌ๋ฏ€๋กœ ์—ฌ๊ธฐ์„  ์ƒ๋žต + if (!warpOn) { + if (clearBg) { ctx.fillStyle = "#fff"; ctx.fillRect(0, 0, 512, 512); } + draw("base"); + } + const bR = manifest.browRange || 0; + const browUp = -bR * Math.min(1, W("browinnerup") + (W("browouterupleft") + W("browouterupright")) / 2) + + bR * 0.8 * (W("browdownleft") + W("browdownright")) / 2; // ์ฐก๊ทธ๋ฆผ์€ ๋ฐ˜๋Œ€๋กœ ๋‚ด๋ฆผ + draw("brow_L", browUp); draw("brow_R", browUp); + if (blink > 0.5) { draw("eye_L_closed"); draw("eye_R_closed"); } + else { + draw("eye_L_open"); draw("eye_R_open"); + const pr = manifest.pupilRange || 0; + drawXY("pupil_L", gaze[0] * pr, gaze[1] * pr); drawXY("pupil_R", gaze[0] * pr, gaze[1] * pr); + } + const jawDy = warp ? warp.jawOverlayDy(W("jawopen"), warpOn, manifest) + : W("jawopen") * (manifest.jawDrop || 8); + drawVectorMouth(ctx, W, manifest, jawDy); + if (head) ctx.restore(); + return warpOn; + } + + // ---------- 3D ํ—ค๋“œ ๋งˆ์šดํŠธ (markยทclaireยทRPM ๊ณต์šฉ) ---------- + // ์”ฌ์— GLB ๋ฅผ ์–น๊ณ  โ‘  ๋ชจํ”„ ๊ตฌ๋™ ๋Œ€์ƒ ์ˆ˜์ง‘ โ‘ก ๋ˆˆ์•Œ ๊ตฌ์ฒด(ํ•„์š” ์‹œ) โ‘ข ์นด๋ฉ”๋ผ ํ”„๋ ˆ์ด๋ฐ๊นŒ์ง€ ํ•œ ๋ฒˆ์—. + // three.js ๋Š” ํŽ˜์ด์ง€๊ฐ€ importmap ์œผ๋กœ ๋กœ๋“œํ•˜๋ฏ€๋กœ THREE ๋ฅผ ์ฃผ์ž…๋ฐ›๋Š”๋‹ค(์ฝ”์–ด๋Š” ํด๋ž˜์‹ ์Šคํฌ๋ฆฝํŠธ). + // + // cfg: { eyeSpheres, headFrac } โ€” eyeSpheres ๋Š” ์•ˆ๊ตฌ ๋ฉ”์‹œ๊ฐ€ ์—†๋Š” ์„๊ณ  ํ—ค๋“œ์šฉ, + // headFrac ๋Š” ๋ชจ๋ธ ๋†’์ด ์ค‘ ๋จธ๋ฆฌ๊ฐ€ ์ฐจ์ง€ํ•˜๋Š” ๋น„์œจ(์„๊ณ =1 ๋จธ๋ฆฌ๋งŒ / RPMโ‰ˆ0.22 ๋ฐ˜์‹ ). + // ๋ฐ˜ํ™˜: { sceneNode, morphMeshes, eyes } โ€” morphMeshes ๋Š” {inf, map} ๋ฐฐ์—ด๋กœ, + // RPM ์ฒ˜๋Ÿผ ๋จธ๋ฆฌยท๋ˆˆยท์ด๋นจ์ด ๊ฐ๊ฐ ๋ชจํ”„๋ฅผ ๊ฐ€์ง„ ๋ชจ๋ธ๋„ ์ „๋ถ€ ๊ตฌ๋™๋œ๋‹ค(ํ•˜๋‚˜๋งŒ ์“ฐ๋ฉด ๋ˆˆยท์ด๋นจ์ด ๋”ฐ๋กœ ๋…ผ๋‹ค). + function mountHead3D(THREE, { group, camera, gltf, cfg, prev }) { + // ๊ฒ€์‚ฌ๋ฅผ ๋จผ์ € โ€” ์‹คํŒจ ์‹œ ์ด์ „ ํ—ค๋“œ๋ฅผ ๊ฑด๋“œ๋ฆฌ์ง€ ์•Š์•„์•ผ ํ˜ธ์ถœ์ธก์˜ "์‹คํŒจํ•˜๋ฉด ์ด์ „ ์ƒํƒœ ์œ ์ง€"๊ฐ€ ์„ฑ๋ฆฝํ•œ๋‹ค. + const sceneNode = gltf.scene; + const morphMeshes = []; + sceneNode.traverse(o => { + if (!o.isMesh || !o.morphTargetInfluences) return; + o.frustumCulled = false; // ๋ชจํ”„๋กœ ๋ณ€ํ˜•๋˜๋ฉด ์›๋ž˜ ๋ฐ”์šด๋”ฉ์„ ๋ฒ—์–ด๋‚˜ ์ปฌ๋ง๋  ์ˆ˜ ์žˆ๋‹ค + morphMeshes.push({ inf: o.morphTargetInfluences, + map: Object.entries(o.morphTargetDictionary || {}).map(([n, i]) => [norm(n), i]) }); + }); + if (!morphMeshes.length) return null; // ํ‘œ์ • ๋ชจํ”„๊ฐ€ ์—†๋Š” ๋ชจ๋ธ โ€” ํ˜ธ์ถœ์ธก์ด ์ƒํƒœ ํ‘œ์‹œ + + if (prev) { + if (prev.sceneNode) group.remove(prev.sceneNode); + // ๋ˆˆ์•Œ ๊ตฌ์ฒด๋Š” ํ—ค๋“œ๋งˆ๋‹ค ์ƒˆ๋กœ ๋งŒ๋“œ๋ฏ€๋กœ ๊ต์ฒด ์‹œ GPU ์ž์›์„ ํ•ด์ œํ•œ๋‹ค(๋ฐ˜๋ณต ์ „ํ™˜ ์‹œ ๋ˆ„์ˆ˜). + (prev.eyes || []).forEach(e => { + group.remove(e); + e.traverse(o => { if (o.isMesh) { o.geometry.dispose(); o.material.dispose(); } }); + }); + } + group.position.set(0, 0, 0); + group.updateMatrixWorld(true); // .position ๋ณ€๊ฒฝ์„ matrixWorld ์— ์ฆ‰์‹œ ๋ฐ˜์˜ โ€” + // ์•ˆ ํ•˜๋ฉด setFromObject ๊ฐ€ ์ด์ „ ํ—ค๋“œ ์˜คํ”„์…‹์„ ๋ฌผ์–ด bbox ๊ฐ€ ์–ด๊ธ‹๋‚œ๋‹ค + group.add(sceneNode); + + const box = new THREE.Box3().setFromObject(sceneNode); + const c = box.getCenter(new THREE.Vector3()), s = box.getSize(new THREE.Vector3()); + const eyes = cfg.eyeSpheres ? [makeEyeball(THREE, group, 1, box), makeEyeball(THREE, group, -1, box)] : []; + // ํ”„๋ ˆ์ด๋ฐ: ์„๊ณ ๋Š” ์ „์ฒด๊ฐ€ ๋จธ๋ฆฌ / ๋ฐ˜์‹  ๋ชจ๋ธ์€ ์ƒ๋‹จ headFrac ๋งŒํผ์„ ๋จธ๋ฆฌ๋กœ ๋ณด๊ณ  ๋งž์ถ˜๋‹ค + const headH = s.y * cfg.headFrac; + const focusY = cfg.headFrac === 1 ? c.y : box.max.y - headH * 0.42; + group.position.set(-c.x, -focusY, -c.z); // ํ—ค๋“œยท๋ˆˆ ํ•จ๊ป˜ recenter + camera.position.set(0, 0, headH * 2.0); camera.lookAt(0, 0, 0); + camera.near = headH / 100; camera.far = headH * 100; camera.updateProjectionMatrix(); + return { sceneNode, morphMeshes, eyes }; + } + + // ๋ˆˆ์•Œ ๊ตฌ์ฒด โ€” ์•ˆ๊ตฌ ๋ฉ”์‹œ๊ฐ€ ์—†๋Š” ์„๊ณ  ํ—ค๋“œ(markยทclaire)์˜ ์†Œ์ผ“์„ ์ฑ„์šด๋‹ค. + // ์œ„์น˜ยทํฌ๊ธฐ๋Š” mark ๋กœ ๋ณด์ •ํ•œ bbox ๋ถ„์œจ์ด๋ผ ํ—ค๋“œ ํฌ๊ธฐ๊ฐ€ ๋‹ฌ๋ผ๋„ ๋”ฐ๋ผ๊ฐ„๋‹ค. + function makeEyeball(THREE, group, sign, box) { + const c = box.getCenter(new THREE.Vector3()), s = box.getSize(new THREE.Vector3()); + const k = s.y / 41.5; // mark ๊ธฐ์ค€ ์Šค์ผ€์ผ + const eg = new THREE.Group(); + eg.position.set(c.x + sign * 0.111 * s.x, c.y + 0.151 * s.y, c.z + 0.284 * s.z); + const mat = (color, roughness) => new THREE.MeshStandardMaterial({ color, roughness }); + const sclera = new THREE.Mesh(new THREE.SphereGeometry(1.15 * k, 32, 16), mat(0xf2f0ee, 0.35)); + const iris = new THREE.Mesh(new THREE.SphereGeometry(0.46 * k, 24, 12), mat(0x44546e, 0.3)); + iris.position.z = 0.78 * k; + const pupil = new THREE.Mesh(new THREE.SphereGeometry(0.2 * k, 16, 8), mat(0x0c0c0c, 0.4)); + pupil.position.z = 1.0 * k; + eg.add(sclera, iris, pupil); + group.add(eg); + return eg; + } + + // ๋ชจํ”„ ๊ตฌ๋™ โ€” mountHead3D ๊ฐ€ ์ค€ morphMeshes ์— ์ฑ„๋„๊ฐ’์„ ์“ด๋‹ค. blink ๋Š” ๋ณ„๋„(์ž๋™ ๊นœ๋นก์ž„๊ณผ max ๊ฒฐํ•ฉ). + function applyMorphs(morphMeshes, smooth, blink) { + for (const { inf, map } of morphMeshes) { + for (const [k, idx] of map) { + inf[idx] = k === "eyeblinkleft" || k === "eyeblinkright" ? blink : (smooth[k] || 0); + } + } + } + + // ---------- ์•„์ด๋ฆฌ์Šค ์‹œ์„  (478์  ๋žœ๋“œ๋งˆํฌ์˜ ํ™์ฑ„ 10์  โ†’ [-1..1] ๊ทผ์‚ฌ) ---------- + // ํ™์ฑ„ ์ค‘์‹ฌ์ด ๋ˆˆ๊ผฌ๋ฆฌ(๊ฐ€๋กœ)ยท๋ˆˆ๊บผํ’€(์„ธ๋กœ) ๊ธฐ์ค€ ์–ด๋”” ์žˆ๋Š”์ง€์˜ ๋น„์œจ โ€” ๋จธ๋ฆฌ ํšŒ์ „์— 1์ฐจ ์ž์ฒด ๋ณด์ •. + // ๋ˆˆ์„ ๊ฑฐ์˜ ๊ฐ์œผ๋ฉด(๊ฐœ๋ฐฉ๋„ < 0.28) null โ€” ํ˜ธ์ถœ์ธก์ด ์ง์ „ ์‹œ์„ ์„ ์œ ์ง€ํ•˜๊ฒŒ ํ•œ๋‹ค(๊นœ๋นก์ž„ ๊ฐ„์„ญ ์ฐจ๋‹จ). + function irisGaze(lm) { + if (!lm || lm.length < 478) return null; + const eye = (iris0, c0, c1, top, bot) => { + let ix = 0, iy = 0; + for (let i = iris0; i < iris0 + 5; i++) { ix += lm[i].x / 5; iy += lm[i].y / 5; } + const halfW = Math.abs(lm[c1].x - lm[c0].x) / 2 || 1e-6; + const h = Math.abs(lm[bot].y - lm[top].y); + const cx = (lm[c0].x + lm[c1].x) / 2, cy = (lm[top].y + lm[bot].y) / 2; + return { gx: (ix - cx) / halfW, gy: (iy - cy) / Math.max(h, 1e-6), open: h / halfW }; + }; + const L = eye(468, 33, 133, 159, 145); // ์˜์ƒ ๊ธฐ์ค€ ์™ผ๋ˆˆ + const R = eye(473, 362, 263, 386, 374); // ์˜์ƒ ๊ธฐ์ค€ ์˜ค๋ฅธ๋ˆˆ + if ((L.open + R.open) / 2 < 0.28) return null; // ๊ฐ๊น€ โ€” ์„ธ๋กœ ์‹ ํ˜ธ ๋ฌด์˜๋ฏธ + return [(L.gx + R.gx) / 2, (L.gy + R.gy) / 2]; + } + + // ๋จธ๋ฆฌ ํšŒ์ „ ๋ถ€ํ˜ธ โ€” ๊ฑฐ์šธ ๋ฐฉํ–ฅ์€ ๋ Œ๋”๋Ÿฌ์™€ ๋ฌด๊ด€ํ•˜๊ฒŒ ๊ณตํ†ต์ด๋ผ ์ฝ”์–ด์—์„œ ํ•œ ๋ฒˆ๋งŒ ์ •ํ•œ๋‹ค. + // yawยทroll ์ด ์Œ์ˆ˜์ธ ์ด์œ : ์›น์บ  ์˜์ƒ์€ ๊ฑฐ์šธ ๋ฐ˜์ „ํ•ด ์“ฐ๋ฏ€๋กœ(๋ถ„์„ ํŒจ๋„ scale(-1,1)) ๊ฐ€๋กœ ์„ฑ๋ถ„์€ + // ๋’ค์ง‘์–ด์•ผ ์‚ฌ์šฉ์ž๊ฐ€ ์™ผ์ชฝ์„ ํ–ฅํ•  ๋•Œ ์บ๋ฆญํ„ฐ๋„ ํ™”๋ฉด ์™ผ์ชฝ์„ ํ–ฅํ•œ๋‹ค โ€” ์‹œ์„  gx ์™€ ๊ฐ™์€ ๊ทœ์•ฝ(์•„๋ž˜ gazeX ํ•ญ). + // pitch(์ƒํ•˜)๋Š” ๋ฐ˜์ „๊ณผ ๋ฌด๊ด€ํ•ด ๊ทธ๋Œ€๋กœ. ์–ด๊ธ‹๋‚˜ ๋ณด์ด๋ฉด ?hs=y,p,r ๋กœ ์‹คํ—˜ ํ›„ ์ด ๊ธฐ๋ณธ๊ฐ’์„ ๊ณ ์นœ๋‹ค. + const HEAD_SIGN = (() => { + const def = [-1, 1, -1]; + const v = (new URLSearchParams(location.search).get("hs") || "").split(",").map(Number); + return def.map((d, i) => (Number.isFinite(v[i]) && v[i] !== 0 ? v[i] : d)); + })(); + + // ---------- ์›น์บ  ํ‘œ์ • ๋ฏธ๋Ÿฌ๋ง (MediaPipe FaceLandmarker ๋ธ”๋ Œ๋“œ์…ฐ์ดํ”„ 52์ฑ„๋„) ---------- + // ๋ธŒ๋ผ์šฐ์ € ์ „์šฉ(์„œ๋ฒ„ยทGPU ์ถ”๋ก  ๋ถˆํ•„์š”, github.io OK). ์ฑ„๋„ ์ด๋ฆ„์ด ARKit ํ‘œ์ค€์ด๋ผ lowercase ๋กœ + // ๋ Œ๋”๋Ÿฌ W() ์ฑ„๋„๊ณผ 1:1. ์‹œ์ž‘ ์‹œ 30ํ”„๋ ˆ์ž„ ์ค‘๋ฆฝ ์บ˜๋ฆฌ๋ธŒ๋ ˆ์ด์…˜ ํ›„ ์ƒ๋Œ€๊ฐ’๋งŒ ์ „์ด(drawface ์ •๊ทœํ™”). + // ์‚ฌ์šฉ: const mirror = makeMirror({ onStatus }); ๋ Œ๋” ๋ฃจํ”„์—์„œ mirror.apply(smooth, now) ํ•œ ์ค„. + // ๋จธ๋ฆฌ ํšŒ์ „์€ ์‚ฐ๋งŒํ•ด์„œ ์ „์ดํ•˜์ง€ ์•Š๋Š”๋‹ค(ํ‘œ์ • ์ฑ„๋„๋งŒ). gain ๊ธฐ๋ณธ๊ฐ’์€ ๋งํ•˜๊ธฐ ์ˆ˜์ค€ ๋ฒŒ๋ฆผ ๋ณด์ • โ€” + // ํŽ˜์ด์ง€๋ณ„ ์˜ค๋ฒ„๋ผ์ด๋“œ ๊ฐ€๋Šฅํ•˜๋‚˜ 6ํŽ˜์ด์ง€ ์‹ค์ธก์—์„œ ๋™์ผ ๊ฐ’์ด ๋งž์•˜๋‹ค. + function makeMirror({ gain, onStatus } = {}) { + gain = gain || { jawopen: 1.6, mouthsmileleft: 1.4, mouthsmileright: 1.4 }; + const st = { on: false, w: null, neutral: null, samples: [], gsamples: [], gN: [0, 0], head: null, hN: null, frame: 0 }; + let lm = null, video = null, lastT = -1; + const say = (msg, err) => onStatus && onStatus(msg, err); + + async function start() { + if (!lm) { + say("๋ฏธ๋Ÿฌ๋ง ๋ชจ๋ธ ๋กœ๋“œ ์ค‘โ€ฆ"); + const mp = await import("https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.17"); + const vision = await mp.FilesetResolver.forVisionTasks( + "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.17/wasm"); + // ์–ผ๊ตด ๋ฉ”์‹œ ํ† ํด๋กœ์ง€๋Š” MediaPipe ๊ฐ€ ์ƒ์ˆ˜๋กœ ๋“ค๊ณ  ์žˆ๋‹ค โ€” ์šฐ๋ฆฌ๊ฐ€ ์‚ผ๊ฐ๋ถ„ํ• ํ•  ํ•„์š”๊ฐ€ ์—†๋‹ค. + // ๋ฏธ๋ฆฌ๋ณด๊ธฐ ์˜ค๋ฒ„๋ ˆ์ด(makeMirrorPanel)๊ฐ€ debug().mesh ๋กœ ๋ฐ›์•„ ์„ ์œผ๋กœ ๊ทธ๋ฆฐ๋‹ค. + st.mesh = mp.FaceLandmarker.FACE_LANDMARKS_TESSELATION || null; + lm = await mp.FaceLandmarker.createFromOptions(vision, { + baseOptions: { modelAssetPath: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task", delegate: "GPU" }, + runningMode: "VIDEO", numFaces: 1, outputFaceBlendshapes: true, + outputFacialTransformationMatrixes: true, // ๋จธ๋ฆฌ ํšŒ์ „(yawยทpitchยทroll) ์ถ”์ถœ์šฉ + }); + } + video = document.createElement("video"); + video.muted = true; + video.srcObject = await navigator.mediaDevices.getUserMedia({ video: { width: 640, height: 480 } }); + await video.play(); + Object.assign(st, { on: true, w: null, neutral: null, samples: [], gsamples: [], gN: [0, 0], head: null, hN: null, frame: 0 }); + say("์บ˜๋ฆฌ๋ธŒ๋ ˆ์ด์…˜ โ€” ์ •๋ฉดยท๋ฌดํ‘œ์ •์œผ๋กœ ์ž ์‹œ ๊ณ„์„ธ์š”"); + } + function stop() { + st.on = false; st.w = null; + video?.srcObject?.getTracks().forEach(t => t.stop()); + video = null; + say(""); + } + // ๋จธ๋ฆฌ ํšŒ์ „: MediaPipe ์–ผ๊ตด ๋ณ€ํ™˜ํ–‰๋ ฌ(์—ด ์šฐ์„  4x4) โ†’ yawยทpitchยทroll(rad). + // ์ค‘๋ฆฝ ์บ˜๋ฆฌ๋ธŒ๋ ˆ์ด์…˜ ๊ธฐ์ค€ ํŽธ์ฐจ๋งŒ ์“ฐ๊ณ , ๊ฐ•ํ•œ ํ‰ํ™œ(0.88)+ํด๋žจํ”„๋กœ ์‚ฐ๋งŒํ•จ์„ ์–ต์ œํ•œ๋‹ค. + function headFromMatrix(m) { + if (!m || st.samples) { st.head = null; return; } // ์บ˜๋ฆฌ๋ธŒ๋ ˆ์ด์…˜ ์ค‘์—” ๋จธ๋ฆฌ ๋ฏธ์‚ฐ์ถœ + // ์—ด ์šฐ์„ (col*4+row): m[8]=R02, m[9]=R12, m[10]=R22, m[1]=R10, m[5]=R11. + // ํ‘œ์ค€ Tait-Bryan(Y-X-Z) ์ถ”์ถœ โ€” ์ถ• ํ˜ผ์„  ์—†์ด yaw/pitch/roll ๋ถ„๋ฆฌ. + const pitch = Math.asin(Math.max(-1, Math.min(1, -m[9]))); + const yaw = Math.atan2(m[8], m[10]); + const roll = Math.atan2(m[1], m[5]); + const cur = [yaw, pitch, roll]; + if (!st.hN) st.hN = cur.slice(); // ์บ˜๋ฆฌ๋ธŒ ์งํ›„ ์ฒซ ํ”„๋ ˆ์ž„ = ์ค‘๋ฆฝ ์ž์„ธ + const rel = cur.map((v, i) => v - st.hN[i]); + const cl = (v, lim) => Math.max(-lim, Math.min(lim, v)); + const t = [cl(rel[0], 0.5), cl(rel[1], 0.35), cl(rel[2], 0.35)]; + st.head = st.head ? st.head.map((v, i) => 0.88 * v + 0.12 * t[i]) : t; + } + + const LOOK = ["eyelookoutright", "eyelookinleft", "eyelookoutleft", "eyelookinright", + "eyelookdownleft", "eyelookdownright", "eyelookupleft", "eyelookupright"]; + function tick(now) { + if (!st.on || !video || video.readyState < 2) return; + if (video.currentTime === lastT) return; // ์ƒˆ ๋น„๋””์˜ค ํ”„๋ ˆ์ž„์—์„œ๋งŒ ์ถ”๋ก  + lastT = video.currentTime; + st.frame++; // ํŒจ๋„์ด "์ƒˆ ํ”„๋ ˆ์ž„์ผ ๋•Œ๋งŒ" ๋‹ค์‹œ ๊ทธ๋ฆฌ๋„๋ก + const res = lm.detectForVideo(video, now); + const cats = res.faceBlendshapes?.[0]?.categories; + if (!cats) { st.w = null; st.lm = null; return; } // ์–ผ๊ตด ๋†“์นจ โ†’ ๊ฐœ์ž… ์ค‘๋‹จ(์ž์—ฐ ๋ณต๊ท€) + st.lm = res.faceLandmarks?.[0] || null; // ๋ถ„์„ ํŒจ๋„(๋น„๊ต๊ตฐ ์‹œ๊ฐํ™”)์šฉ ์›๋ณธ ๋žœ๋“œ๋งˆํฌ + headFromMatrix(res.facialTransformationMatrixes?.[0]?.data); + const raw = {}; + for (const c of cats) raw[c.categoryName.toLowerCase()] = c.score; + const g = irisGaze(res.faceLandmarks?.[0]); // ์•„์ด๋ฆฌ์Šค ์ •๋ฐ€ ์‹œ์„  (๊ฐ๊น€์ด๋ฉด null) + if (!st.neutral) { // 30ํ”„๋ ˆ์ž„ ํ‰๊ท  = ์ค‘๋ฆฝ + st.samples.push(raw); + if (g) st.gsamples.push(g); + // ์บ˜๋ฆฌ๋ธŒ๋ ˆ์ด์…˜ ์ค‘ ์›€์ง์ด๋ฉด ์ค‘๋ฆฝ์ด ์˜ค์—ผ๋˜๋ฏ€๋กœ ์ง„ํ–‰ ์ƒํ™ฉ์„ ์•ˆ๋‚ด (6ํ”„๋ ˆ์ž„๋งˆ๋‹ค ๊ฐฑ์‹ ) + if (st.samples.length % 6 === 1) say(`์ •๋ฉด์„ ๋ณด๊ณ  ๋ฌดํ‘œ์ •์„ ์œ ์ง€ํ•ด์ฃผ์„ธ์š”โ€ฆ ${st.samples.length}/30`); + if (st.samples.length < 30) return; + const n = {}; + for (const k in raw) n[k] = st.samples.reduce((a, s) => a + (s[k] || 0), 0) / st.samples.length; + // ์‹œ์„ ์€ ์•„์ด๋ฆฌ์Šค ํ•ฉ์„ฑ๊ฐ’์œผ๋กœ ๋Œ€์ฒด๋˜๋ฏ€๋กœ eyeLook ์ฑ„๋„์˜ ๋ธ”๋ Œ๋“œ์…ฐ์ดํ”„ ์ค‘๋ฆฝ์€ 0์œผ๋กœ โ€” + // ํ•ฉ์„ฑ๊ฐ’(์ด๋ฏธ ๊ฒŒ์ธยทํด๋žจํ”„ ์ ์šฉ)์ด ํ‘œ์ค€ ํŒŒ์ดํ”„๋ผ์ธ(์ค‘๋ฆฝ์ฐจ๊ฐยทEMA)์„ ๊ทธ๋Œ€๋กœ ํ†ต๊ณผํ•˜๊ฒŒ ํ•œ๋‹ค. + for (const k of LOOK) n[k] = 0; + st.gN = st.gsamples.length + ? [st.gsamples.reduce((a, v) => a + v[0], 0) / st.gsamples.length, + st.gsamples.reduce((a, v) => a + v[1], 0) / st.gsamples.length] + : [0, 0]; + st.neutral = n; + st.samples = null; st.gsamples = null; // ์บ˜๋ฆฌ๋ธŒ๋ ˆ์ด์…˜ ๋ โ€” ์ƒ˜ํ”Œ ๋ฒ„ํผ ํ•ด์ œ + say("๋ฏธ๋Ÿฌ๋ง ์ค‘ โ€” ์บ๋ฆญํ„ฐ๊ฐ€ ๋”ฐ๋ผํ•ฉ๋‹ˆ๋‹ค (๋ฒ„ํŠผ์œผ๋กœ ์ข…๋ฃŒ)"); + return; + } + // ์•„์ด๋ฆฌ์Šค ์‹œ์„  โ†’ eyeLook 8์ฑ„๋„ ํ•ฉ์„ฑ ๋ฎ์–ด์“ฐ๊ธฐ (๋ธ”๋ Œ๋“œ์…ฐ์ดํ”„ ์‹œ์„ ์น˜๋Š” ๊ฑฐ์น ์–ด์„œ ๋Œ€์ฒด). + // ๋ถ€ํ˜ธ: ๊ฑฐ์šธ ๋А๋‚Œ โ€” ์‚ฌ์šฉ์ž๊ฐ€ ํ™”๋ฉด ์™ผ์ชฝ์„ ๋ณด๋ฉด ์บ๋ฆญํ„ฐ ๋ˆˆ๋™์ž๋„ ํ™”๋ฉด ์™ผ์ชฝ์œผ๋กœ. + if (g) { + const cl = v => Math.max(-1, Math.min(1, v)); + const gx = cl(-(g[0] - st.gN[0]) * (gain.gazeX ?? 2.4)); + const gy = cl((g[1] - st.gN[1]) * (gain.gazeY ?? 1.4)); // ์„ธ๋กœ๋Š” ๋ˆˆ๊บผํ’€ ๊ฐ€๋ฆผ ํƒ“ ์‹ ํ˜ธ ์•ฝํ•จ โ†’ ๋‚ฎ์€ ๊ฒŒ์ธ + raw.eyelookoutright = raw.eyelookinleft = Math.max(0, gx); + raw.eyelookoutleft = raw.eyelookinright = Math.max(0, -gx); + raw.eyelookdownleft = raw.eyelookdownright = Math.max(0, gy); + raw.eyelookupleft = raw.eyelookupright = Math.max(0, -gy); + } else { + for (const k of LOOK) delete raw[k]; // ๊นœ๋นก์ž„ ๋“ฑ โ€” ์ง์ „ ์‹œ์„ (EMA) ์œ ์ง€ + } + const w = st.w || {}; + for (const k in raw) { + if (k === "_neutral") continue; + const n = st.neutral[k] || 0; + const cal = clamp01((raw[k] - n) / Math.max(0.2, 1 - n) * (gain[k] || 1)); + w[k] = 0.55 * (w[k] || 0) + 0.45 * cal; // EMA ํ‰ํ™œ + } + st.w = w; + } + return { + get on() { return st.on; }, + start, stop, + // ๋ Œ๋” ๋ฃจํ”„ ํ•œ ์ค„: ์ถ”๋ก  tick + smooth ์— max-๊ฒฐํ•ฉ + apply(smooth, now) { + tick(now); + if (st.w) for (const k in st.w) smooth[k] = Math.max(smooth[k] || 0, st.w[k]); + }, + // ๋จธ๋ฆฌ ํšŒ์ „ [yaw, pitch, roll] (rad, ์ค‘๋ฆฝ ๋Œ€๋น„ยทํ‰ํ™œยทํด๋žจํ”„ยท๋ถ€ํ˜ธ์ ์šฉ๋จ). ๋ฏธ๊ฒ€์ถœ/๋ฏธ์‹œ์ž‘์ด๋ฉด null. + // ํŽ˜์ด์ง€๊ฐ€ ์ผค์ง€ ๋ง์ง€๋งŒ ๊ฒฐ์ •ํ•œ๋‹ค โ€” ์ถ• ๋งคํ•‘์€ ํŽ˜์ด์ง€ ๋ชซ, ๋ถ€ํ˜ธ(๊ฑฐ์šธ ๋ฐฉํ–ฅ)๋Š” ์ „ ํŽ˜์ด์ง€ ๊ณตํ†ต์ด๋ผ ์—ฌ๊ธฐ์„œ. + head: () => st.head && st.head.map((v, i) => HEAD_SIGN[i] * v), + // ๋ถ„์„ ํŒจ๋„์šฉ: ์›๋ณธ ๋น„๋””์˜ค + 478์  ๋žœ๋“œ๋งˆํฌ + ์บ˜๋ฆฌ๋ธŒ๋ ˆ์ด์…˜๋œ ์ฑ„๋„๊ฐ’(์บ๋ฆญํ„ฐ ๊ตฌ๋™๊ฐ’๊ณผ ๋™์ผ) + debug: () => ({ video, lm: st.lm, w: st.w, frame: st.frame, mesh: st.mesh }), + }; + } + + // ---------- ์ƒํƒœ์ค„ setter ---------- + function bindStatus(el) { + return (msg, isError) => { el.textContent = msg; el.className = isError ? "error" : ""; }; + } + + // ---------- ๋งˆ์ดํฌ ์Œ์„ฑ์ธ์‹ (Web Speech API โ€” ๋ธŒ๋ผ์šฐ์ € ๋‚ด์žฅ, ๋ชจ๋ธ ๋ถˆํ•„์š”) ---------- + // ๋ฏธ์ง€์› ๋ธŒ๋ผ์šฐ์ €(Chrome ๊ณ„์—ด ์™ธ)๋ฉด null ๋ฐ˜ํ™˜ โ†’ ํŽ˜์ด์ง€๊ฐ€ ๋งˆ์ดํฌ ๋ฒ„ํŠผ์„ ์ˆจ๊ธด๋‹ค. + // onText(ํ…์ŠคํŠธ, isFinal): ์ธ์‹ ์ค‘๊ฐ„๊ฒฐ๊ณผ(false)์™€ ์ตœ์ข…๊ฒฐ๊ณผ(true)๋ฅผ ๋ชจ๋‘ ์ „๋‹ฌ. + function makeMic({ lang = "ko-KR", onText, onState }) { + const SR = window.SpeechRecognition || window.webkitSpeechRecognition; + if (!SR) return null; + const rec = new SR(); + rec.lang = lang; + rec.interimResults = true; // ๋งํ•˜๋Š” ๋„์ค‘์—๋„ ํ…์ŠคํŠธ๋ฅผ ๋ณด์—ฌ์คŒ + rec.continuous = false; // ํ•œ ๋ฌธ์žฅ ๋งํ•˜๋ฉด ์ž๋™ ์ข…๋ฃŒ (ํ‘ธ์‹œํˆฌํ† ํฌ ๋ฐฉ์‹) + let listening = false; + rec.onstart = () => { listening = true; onState && onState("listening"); }; + rec.onend = () => { listening = false; onState && onState("idle"); }; + rec.onerror = e => { listening = false; onState && onState("error", e.error); }; + rec.onresult = e => { + let fin = "", interim = ""; + for (let i = e.resultIndex; i < e.results.length; i++) { + const r = e.results[i]; + if (r.isFinal) fin += r[0].transcript; else interim += r[0].transcript; + } + if (fin.trim()) onText(fin.trim(), true); + else if (interim) onText(interim, false); + }; + return { + toggle() { if (listening) rec.stop(); else { try { rec.start(); } catch (_) {} } }, + start() { if (!listening) { try { rec.start(); } catch (_) {} } }, + stop() { if (listening) rec.stop(); }, + }; + } + + // ---------- ๋Œ€ํ™” (LLM ์‘๋‹ต โ€” ๋กœ์ปฌ ์„œ๋ฒ„ ์ „์šฉ) ---------- + // ๋ฐ˜ํ™˜: {reply, emotion}. emotion ์€ EMOTIONS ํ‚ค ์ค‘ ํ•˜๋‚˜(LLM ํŒ๋‹จ). ์‹คํŒจ ์‹œ throw. + async function chat(text, history, persona) { + const res = await fetch("/api/chat", { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text, history: history || [], persona: persona || null }), + }); + if (!res.ok) throw new Error((await res.json()).detail || res.statusText); + return res.json(); + } + + // ---------- ๋Œ€ํ™” ๋ชจ๋“œ ๊ธ€๋ฃจ (๋งˆ์ดํฌ/์ž…๋ ฅ โ†’ LLM โ†’ ์•„๋ฐ”ํƒ€ ์‘๋‹ต; puppetยทstudio3d ๊ณต์šฉ) ---------- + // history ์ƒํƒœ + ์ฑ„ํŒ…๋กœ๊ทธ(addTurn) + runChat + ๋ชจ๋“œ ํ† ๊ธ€ + ์ง€์šฐ๊ธฐ + ๋งˆ์ดํฌ ๋ฐฐ์„ ์„ ์†Œ์œ . + // ํŽ˜์ด์ง€ ์ฃผ์ž…: speak(text,emotion) ๋ฐœํ™” ํ•จ์ˆ˜, botName(๋ด‡ ํ„ด ํ™”์ž๋ช…), placeholderOn(๋Œ€ํ™” ๋ชจ๋“œ ์•ˆ๋‚ด๋ฌธ), + // logEl/chatModeEl/clearBtnEl/textEl/sendEl/micBtnEl/formEl ์—˜๋ฆฌ๋จผํŠธ, statusSet(bindStatus ๊ฒฐ๊ณผ), + // getPersona()(์„ ํƒ โ€” ํ˜„์žฌ ์บ๋ฆญํ„ฐ ์„ฑ๊ฒฉ; ์—†์œผ๋ฉด ๊ธฐ๋ณธ ์ •์ฒด์„ฑ). placeholderOff ๋Š” ์ž…๋ ฅ์ฐฝ ์ดˆ๊ธฐ placeholder ์žฌ์‚ฌ์šฉ. + // ๋ฐ˜ํ™˜ { runChat } โ€” onsubmit ์˜ chat|say ๋ถ„๊ธฐ๋Š” ํŽ˜์ด์ง€๊ฐ€ ์–‡๊ฒŒ ์†Œ์œ (runChat ์•Œ๋งน์ด๋งŒ ์ฝ”์–ด). + function makeChat({ speak, logEl, botName, placeholderOn, chatModeEl, clearBtnEl, textEl, sendEl, micBtnEl, formEl, statusSet, getPersona, audioEl }) { + const history = []; // [{role, content}] ์ตœ๊ทผ ํ„ด๋งŒ ์œ ์ง€ + let handsFree = false, busy = false; // ์—ฐ์† ๋Œ€ํ™”: ์‘๋‹ต ์žฌ์ƒ์ด ๋๋‚˜๋ฉด ์ž๋™ ์žฌ์ฒญ์ทจ + const placeholderOff = textEl.placeholder; + function addTurn(who, text, cls) { + const div = document.createElement("div"); + div.className = "turn"; + div.innerHTML = `${who}`; + div.lastChild.textContent = text; // ์‚ฌ์šฉ์ž ์ž…๋ ฅ์ด๋ฏ€๋กœ textContent๋กœ ์•ˆ์ „ํ•˜๊ฒŒ + logEl.appendChild(div); + logEl.scrollTop = logEl.scrollHeight; + } + chatModeEl.onchange = () => { + const on = chatModeEl.checked; + logEl.style.display = on ? "grid" : "none"; + clearBtnEl.style.display = on ? "" : "none"; + sendEl.textContent = on ? "๋ง ๊ฑธ๊ธฐ" : "๋งํ•˜๊ธฐ"; + textEl.placeholder = on ? placeholderOn : placeholderOff; + }; + clearBtnEl.onclick = () => { history.length = 0; logEl.innerHTML = ""; statusSet(""); }; + async function runChat(text) { + busy = true; // ์•„๋ฐ”ํƒ€๊ฐ€ ์ƒ๊ฐยท๋ฐœํ™”ํ•˜๋Š” ๋™์•ˆ ์žฌ์ฒญ์ทจ ๊ธˆ์ง€ (์ž๊ธฐ ๋ชฉ์†Œ๋ฆฌ ์ธ์‹ ๋ฐฉ์ง€) + try { + addTurn("๋‚˜", text, "me"); + statusSet("์ƒ๊ฐ ์ค‘โ€ฆ"); + const { reply, emotion } = await chat(text, history.slice(-6), getPersona && getPersona()); // ์ตœ๊ทผ 3ํ„ด + ์บ๋ฆญํ„ฐ ์„ฑ๊ฒฉ + history.push({ role: "user", content: text }, { role: "assistant", content: reply }); + addTurn(botName, reply, "bot"); + statusSet("๋งํ•˜๋Š” ์ค‘โ€ฆ"); + await speak(reply, emotion); + // speak ๋Š” ์žฌ์ƒ ์‹œ์ž‘ ์‹œ์ ์— ๋ฐ˜ํ™˜ โ€” ์—ฐ์† ๋Œ€ํ™”๋ฉด ์žฌ์ƒ์ด ์‹ค์ œ๋กœ ๋๋‚  ๋•Œ๊นŒ์ง€ ๋Œ€๊ธฐ ํ›„ ์žฌ์ฒญ์ทจ + if (handsFree && audioEl && !audioEl.paused && !audioEl.ended) + await new Promise(res => audioEl.addEventListener("ended", res, { once: true })); + } finally { + busy = false; + if (handsFree && mic) { statusSet(""); mic.start(); } + } + } + // ๋งˆ์ดํฌ (๋ธŒ๋ผ์šฐ์ € ์Œ์„ฑ์ธ์‹ โ€” ๋ฏธ์ง€์› ๋ธŒ๋ผ์šฐ์ €๋ฉด makeMic null โ†’ ๋ฒ„ํŠผ ์ˆจ๊น€ ์œ ์ง€) + const mic = makeMic({ + onText: (t, isFinal) => { + textEl.value = t; + if (isFinal) formEl.requestSubmit(); // ๋ง์ด ๋๋‚˜๋ฉด ์ž๋™ ์ „์†ก + }, + onState: (st, err) => { + micBtnEl.classList.toggle("on", st === "listening" || handsFree); + if (st === "listening") statusSet(handsFree ? "๋“ฃ๊ณ  ์žˆ์–ด์š”โ€ฆ (์—ฐ์† ๋Œ€ํ™” โ€” ๋งˆ์ดํฌ ๋ฒ„ํŠผ์œผ๋กœ ์ข…๋ฃŒ)" : "๋“ฃ๊ณ  ์žˆ์–ด์š”โ€ฆ ๋ง์”€ํ•˜์„ธ์š”"); + else if (st === "error") { + if (err === "not-allowed") { handsFree = false; statusSet("๋งˆ์ดํฌ ๊ถŒํ•œ์ด ํ•„์š”ํ•ฉ๋‹ˆ๋‹ค.", true); } + else statusSet(`์Œ์„ฑ ์ธ์‹ ์˜ค๋ฅ˜: ${err}`, true); + } else if (st === "idle" && handsFree && !busy) { + // ์นจ๋ฌต ํƒ€์ž„์•„์›ƒ์œผ๋กœ ๋Š๊ฒจ๋„ ์—ฐ์† ๋ชจ๋“œ๋ฉด ์ž ์‹œ ํ›„ ์žฌ์ฒญ์ทจ (๋ฐœํ™” ์ฒ˜๋ฆฌ ์ค‘์ด๋ฉด runChat ๋์—์„œ ์žฌ๊ฐœ) + setTimeout(() => { if (handsFree && !busy) mic.start(); }, 400); + } + }, + }); + if (mic) { + micBtnEl.style.display = ""; + // ๋งˆ์ดํฌ ๋ฒ„ํŠผ = ์—ฐ์† ๋Œ€ํ™” ํ† ๊ธ€: ์ผœ๋ฉด ๋“ฃ๊ธฐโ†’์‘๋‹ตโ†’์žฌ์ฒญ์ทจ ๋ฃจํ”„, ๋‹ค์‹œ ๋ˆ„๋ฅด๋ฉด ์ข…๋ฃŒ + micBtnEl.onclick = () => { + if (!chatModeEl.checked) chatModeEl.click(); + handsFree = !handsFree; + micBtnEl.classList.toggle("on", handsFree); + if (handsFree) mic.start(); + else { mic.stop(); statusSet(""); } + }; + } + return { runChat }; + } + + // ---------- ๋“œ๋ž˜๊ทธ์•ค๋“œ๋ž ์บ๋ฆญํ„ฐ ์ƒ์„ฑ: ์–ด๋…ธํ…Œ์ด์…˜ ์บก์ฒ˜ UI (4ํด๋ฆญ ์ƒํƒœ๋จธ์‹ ) ---------- + // โ‘  ์™ผ๋ˆˆ โ‘ก ์˜ค๋ฅธ๋ˆˆ โ‘ข ์ž…์ค‘์‹ฌ ํด๋ฆญ โ†’ โ‘ฃ ์ž… ๋“œ๋ž˜๊ทธ. cv/ctx: 2D ์˜ค๋ฒ„๋ ˆ์ด ์บ”๋ฒ„์Šคยท์ปจํ…์ŠคํŠธ. + // needDataUrl: puppet(์„œ๋ฒ„ POST์— b64 ํ•„์š”) true / docs false. onCreate(annot, name, done): + // ์™„๋ฃŒ ์ฝœ๋ฐฑ(ํŽ˜์ด์ง€๋ณ„ โ€” ์„œ๋ฒ„ POST vs ํด๋ผ์ด์–ธํŠธ ๋นŒ๋“œ). ์„ฑ๊ณต ์‹œ done() ํ˜ธ์ถœ๋กœ annot ํ•ด์ œ(์ •๋ฆฌ ์‹œ์  ํŽ˜์ด์ง€ ์ œ์–ด). + function makeAnnotator({ cv, ctx, setStatus, needDataUrl, onCreate }) { + let annot = null; // {img, iw, ih, s, ox, oy, step, eyeL, eyeR, mouthC, box, dragStart, dataUrl} + const STEPS = ["โ‘  ์™ผ์ชฝ ๋ˆˆ์„ ํด๋ฆญํ•˜์„ธ์š”", "โ‘ก ์˜ค๋ฅธ์ชฝ ๋ˆˆ์„ ํด๋ฆญํ•˜์„ธ์š”", + "โ‘ข ์ž… ์ค‘์‹ฌ์„ ํด๋ฆญํ•˜์„ธ์š”", "โ‘ฃ ์ž… ์ „์ฒด๋ฅผ ๋“œ๋ž˜๊ทธ๋กœ ๊ฐ์‹ธ์„ธ์š”"]; + + function startAnnot(img, dataUrl) { + const s = Math.min(512 / img.width, 512 / img.height); + annot = { img, iw: img.width, ih: img.height, s, + ox: (512 - img.width * s) / 2, oy: (512 - img.height * s) / 2, step: 0, dataUrl }; + setStatus(STEPS[0] + " (ESC๋กœ ์ทจ์†Œ)"); + } + function toOrig(e) { + const r = cv.getBoundingClientRect(); + return [((e.clientX - r.left) * 512 / r.width - annot.ox) / annot.s, + ((e.clientY - r.top) * 512 / r.height - annot.oy) / annot.s]; + } + function draw() { + ctx.fillStyle = "#fff"; ctx.fillRect(0, 0, 512, 512); + ctx.drawImage(annot.img, annot.ox, annot.oy, annot.iw * annot.s, annot.ih * annot.s); + const dot = (p, color) => { + if (!p) return; + ctx.beginPath(); + ctx.arc(annot.ox + p[0] * annot.s, annot.oy + p[1] * annot.s, 6, 0, Math.PI * 2); + ctx.strokeStyle = color; ctx.lineWidth = 3; ctx.stroke(); + }; + dot(annot.eyeL, "#00a2ff"); dot(annot.eyeR, "#00a2ff"); dot(annot.mouthC, "#ff5b5b"); + if (annot.box) { + const [x0, y0, x1, y1] = annot.box; + ctx.strokeStyle = "#ff5b5b"; ctx.lineWidth = 2; + ctx.strokeRect(annot.ox + x0 * annot.s, annot.oy + y0 * annot.s, + (x1 - x0) * annot.s, (y1 - y0) * annot.s); + } + } + cv.addEventListener("pointerdown", e => { + if (!annot) return; + const p = toOrig(e); + if (annot.step === 0) { annot.eyeL = p; annot.step = 1; } + else if (annot.step === 1) { annot.eyeR = p; annot.step = 2; } + else if (annot.step === 2) { annot.mouthC = p; annot.step = 3; } + else if (annot.step === 3) { annot.dragStart = p; } + setStatus(STEPS[Math.min(annot.step, 3)] + " (ESC๋กœ ์ทจ์†Œ)"); + }); + cv.addEventListener("pointermove", e => { + if (!annot || annot.step !== 3 || !annot.dragStart) return; + const p = toOrig(e), s = annot.dragStart; + annot.box = [Math.min(s[0], p[0]), Math.min(s[1], p[1]), Math.max(s[0], p[0]), Math.max(s[1], p[1])]; + }); + cv.addEventListener("pointerup", async () => { + if (!annot || annot.step !== 3 || !annot.box) return; + if (annot.box[2] - annot.box[0] < 5) { annot.dragStart = null; return; } + const name = prompt("์บ๋ฆญํ„ฐ ์ด๋ฆ„:", "๋‚ด ์บ๋ฆญํ„ฐ"); + if (name === null) { annot.dragStart = null; annot.box = null; return; } + await onCreate(annot, name, () => { annot = null; }); + }); + document.addEventListener("keydown", e => { + if (e.key === "Escape" && annot) { annot = null; setStatus(""); } + }); + function acceptFile(file) { + if (!file || !file.type.startsWith("image/")) return; + if (needDataUrl) { // b64 dataURL ๋กœ ์ฝ์–ด annot.dataUrl ์— ๋ณด์กด (์„œ๋ฒ„ POST์šฉ) + const reader = new FileReader(); + reader.onload = () => { + const img = new Image(); + img.onload = () => startAnnot(img, reader.result); + img.src = reader.result; + }; + reader.readAsDataURL(file); + } else { // objectURL ๋กœ ๋ฐ”๋กœ ๋กœ๋“œ (ํด๋ผ์ด์–ธํŠธ ๋นŒ๋“œ๋Š” dataURL ๋ถˆํ•„์š”) + const img = new Image(); + img.onload = () => startAnnot(img, null); + img.src = URL.createObjectURL(file); + } + } + return { active: () => !!annot, draw, acceptFile }; + } + + return { + norm, inferEmotion, classifyEmotion, voiceProsody, smoothStep, weightsFromAnim, shapeAnim, SHAPE, + EMOTIONS, makeEmotion, makeBlink, makeCursorTracker, makeGaze, makeHeadWander, + makeMouthPicker, drawVectorMouth, drawSpriteMouth, makeWarp, speakFlow, speakWithEmotion, + bindStatus, makeAnnotator, makeMic, chat, makeChat, makeShowcase, pickReaction, makeMirror, irisGaze, makeMirrorPanel, + mountHead3D, applyMorphs, drawChar2D, + }; +})(); diff --git a/boy.html b/boy.html new file mode 100644 index 0000000000000000000000000000000000000000..776e12967961c36f3e6e9f40302038c80804e488 --- /dev/null +++ b/boy.html @@ -0,0 +1,5 @@ + + + + +์†Œ๋…„ ๋ฏธ๋Ÿฌ๋ง์œผ๋กœ ์ด๋™โ€ฆ diff --git a/characters/boy/base.png b/characters/boy/base.png new file mode 100644 index 0000000000000000000000000000000000000000..ab48beb4ba7f21d7fae0c02f2d75dab83665684e Binary files /dev/null and b/characters/boy/base.png differ diff --git a/characters/boy/brow_L.png b/characters/boy/brow_L.png new file mode 100644 index 0000000000000000000000000000000000000000..ed55083638512868e60f9d9e6089db3c00fa085d Binary files /dev/null and b/characters/boy/brow_L.png differ diff --git a/characters/boy/brow_R.png b/characters/boy/brow_R.png new file mode 100644 index 0000000000000000000000000000000000000000..b1a6d7f66ad8c12ed17f3d662bca916ff48951fe Binary files /dev/null and b/characters/boy/brow_R.png differ diff --git a/characters/boy/eye_L_closed.png b/characters/boy/eye_L_closed.png new file mode 100644 index 0000000000000000000000000000000000000000..2bd2f4777a5a44ccc309690b5513ec75007fe33f Binary files /dev/null and b/characters/boy/eye_L_closed.png differ diff --git a/characters/boy/eye_L_open.png b/characters/boy/eye_L_open.png new file mode 100644 index 0000000000000000000000000000000000000000..9bca38d104c1c52eeec6a37cc5e487050e0121a2 Binary files /dev/null and b/characters/boy/eye_L_open.png differ diff --git a/characters/boy/eye_R_closed.png b/characters/boy/eye_R_closed.png new file mode 100644 index 0000000000000000000000000000000000000000..b0b13fe881035e87fa818ca80e9d0c61a464a20d Binary files /dev/null and b/characters/boy/eye_R_closed.png differ diff --git a/characters/boy/eye_R_open.png b/characters/boy/eye_R_open.png new file mode 100644 index 0000000000000000000000000000000000000000..9d8dee74a72070753eb7de7016a2c93ce453f118 Binary files /dev/null and b/characters/boy/eye_R_open.png differ diff --git a/characters/boy/manifest.json b/characters/boy/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..b7fbca912f9ef0af4136aa5973e377af2497e9ea --- /dev/null +++ b/characters/boy/manifest.json @@ -0,0 +1,18 @@ +{ + "name": "์†Œ๋…„", + "pupilRange": 8, + "browRange": 12, + "jawDrop": 9, + "mouthCenter": [ + 256, + 336 + ], + "proceduralMouth": true, + "mouthStyle": { + "line": "#4a3626", + "fill": "#a2453b", + "tongue": "#e08a8a", + "teeth": "#ffffff", + "width": 30 + } +} \ No newline at end of file diff --git a/characters/boy/pupil_L.png b/characters/boy/pupil_L.png new file mode 100644 index 0000000000000000000000000000000000000000..d85ab19434c1c889ec03aa6ef9e8d803c6b91087 Binary files /dev/null and b/characters/boy/pupil_L.png differ diff --git a/characters/boy/pupil_R.png b/characters/boy/pupil_R.png new file mode 100644 index 0000000000000000000000000000000000000000..c4b7871f47b932db357af2eeeac250fdf10ea954 Binary files /dev/null and b/characters/boy/pupil_R.png differ diff --git a/characters/girl/base.png b/characters/girl/base.png new file mode 100644 index 0000000000000000000000000000000000000000..0ba544814b373f0982aee79ea5997fe79be665e6 Binary files /dev/null and b/characters/girl/base.png differ diff --git a/characters/girl/brow_L.png b/characters/girl/brow_L.png new file mode 100644 index 0000000000000000000000000000000000000000..207fc58317272b47071162a887e4a9a7d0997d68 Binary files /dev/null and b/characters/girl/brow_L.png differ diff --git a/characters/girl/brow_R.png b/characters/girl/brow_R.png new file mode 100644 index 0000000000000000000000000000000000000000..61da2962cf297320ec8448f7f433920ba93071e6 Binary files /dev/null and b/characters/girl/brow_R.png differ diff --git a/characters/girl/eye_L_closed.png b/characters/girl/eye_L_closed.png new file mode 100644 index 0000000000000000000000000000000000000000..60b0b35e2d3ff40ea144b1ea4c9c320145d8c236 Binary files /dev/null and b/characters/girl/eye_L_closed.png differ diff --git a/characters/girl/eye_L_open.png b/characters/girl/eye_L_open.png new file mode 100644 index 0000000000000000000000000000000000000000..cb5ab5bf89b84289d4ccfef5be90dc9587559c0d Binary files /dev/null and b/characters/girl/eye_L_open.png differ diff --git a/characters/girl/eye_R_closed.png b/characters/girl/eye_R_closed.png new file mode 100644 index 0000000000000000000000000000000000000000..19c159e6d2ed80b0a5919e401f3ba73756e51ba2 Binary files /dev/null and b/characters/girl/eye_R_closed.png differ diff --git a/characters/girl/eye_R_open.png b/characters/girl/eye_R_open.png new file mode 100644 index 0000000000000000000000000000000000000000..472d3366e117a3c96f55b67b6e817700f1428d32 Binary files /dev/null and b/characters/girl/eye_R_open.png differ diff --git a/characters/girl/manifest.json b/characters/girl/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..ff25e6428ed31a3e74401314d16de6af0dd74824 --- /dev/null +++ b/characters/girl/manifest.json @@ -0,0 +1,19 @@ +{ + "name": "์†Œ๋…€", + "persona": "๋„ˆ๋Š” ์‚ฌ์šฉ์ž๊ฐ€ ๊ทธ๋ฆฐ ๊ทธ๋ฆผ์—์„œ ํƒœ์–ด๋‚œ ๋ฐ๊ณ  ๋‹ค์ •ํ•œ ์•„๋ฐ”ํƒ€์•ผ. ํ˜ธ๊ธฐ์‹ฌ ๋งŽ๊ณ  ์นœ๊ตฌ์ฒ˜๋Ÿผ ํŽธํ•˜๊ฒŒ ๋Œ€ํ™”ํ•œ๋‹ค.", + "pupilRange": 8, + "browRange": 12, + "jawDrop": 9, + "mouthCenter": [ + 256, + 336 + ], + "proceduralMouth": true, + "mouthStyle": { + "line": "#5a3a2e", + "fill": "#b5504a", + "tongue": "#e79aa0", + "teeth": "#ffffff", + "width": 30 + } +} \ No newline at end of file diff --git a/characters/girl/pupil_L.png b/characters/girl/pupil_L.png new file mode 100644 index 0000000000000000000000000000000000000000..aae8fd6f0d25f3428e8a3b8c799a7ed68f1fc5dc Binary files /dev/null and b/characters/girl/pupil_L.png differ diff --git a/characters/girl/pupil_R.png b/characters/girl/pupil_R.png new file mode 100644 index 0000000000000000000000000000000000000000..3f0353080b514c116c9ca2ceb1550ba4aa20a92f Binary files /dev/null and b/characters/girl/pupil_R.png differ diff --git a/combo.html b/combo.html new file mode 100644 index 0000000000000000000000000000000000000000..e0b077b465e01a3d26f69798909b78436ab2b0d9 --- /dev/null +++ b/combo.html @@ -0,0 +1,223 @@ + + + + + +DrawFace Live โ€” 5์ธ ๋™์‹œ ๋ฏธ๋Ÿฌ๋ง + + + + + +
+

5์ธ ๋™์‹œ ๋ฏธ๋Ÿฌ๋ง โ€” ๋‚ด ํ‘œ์ • โ†’ ์„๊ณ  ๋‚จ/์—ฌ + ์†Œ๋…„/์†Œ๋…€

+
+
๋‚˜ (MediaPipe)
+
์„๊ณ  ๋‚จ
+
์†Œ๋…„
+
์„๊ณ  ์—ฌ
+
์†Œ๋…€
+
+
+ + + + โ† ์ŠคํŠœ๋””์˜ค +
+
๋กœ๋”ฉ ์ค‘โ€ฆ
+
+ + + + + diff --git a/dev/authoring_test.html b/dev/authoring_test.html new file mode 100644 index 0000000000000000000000000000000000000000..de82b7f9814eb46258c659f9af49080dedfa7419 --- /dev/null +++ b/dev/authoring_test.html @@ -0,0 +1,94 @@ + + + + + authoring test + + + +

authoring pipeline test

+
runningโ€ฆ
+
+

+
+  
+
+
diff --git a/dev/runtime_test.html b/dev/runtime_test.html
new file mode 100644
index 0000000000000000000000000000000000000000..799c04fce8c57ada16fb34017a1a9ae1f8c29af3
--- /dev/null
+++ b/dev/runtime_test.html
@@ -0,0 +1,127 @@
+
+
+
+  
+  runtime tests
+
+
+  

DrawFace runtime unit tests

+
runningโ€ฆ
+ + + + + diff --git a/draw.html b/draw.html new file mode 100644 index 0000000000000000000000000000000000000000..0bb7aa7fbeeb1499c1c9792b7e2f795dffca461e --- /dev/null +++ b/draw.html @@ -0,0 +1,99 @@ + + + + + +DrawFace Live โ€” ๋‚ด ์†๊ทธ๋ฆผ + + + +
+

DrawFace Live โ€” ๋‚ด ์†๊ทธ๋ฆผ ๋ฏธ๋Ÿฌ๋ง ์ŠคํŠœ๋””์˜ค โ†’

+

์†๊ทธ๋ฆผ ํ•œ ์žฅ์ด ์›น์บ  ํ‘œ์ •์„ ์‹ค์‹œ๊ฐ„์œผ๋กœ ๋”ฐ๋ผ ํ•ฉ๋‹ˆ๋‹ค โ€” ๋ชจ๋“  ์ฒ˜๋ฆฌ๋Š” ๋ธŒ๋ผ์šฐ์ € ์•ˆ์—์„œ๋งŒ ์ด๋ค„์ง‘๋‹ˆ๋‹ค.

+
+ +
+ + +
+ + +
+
+ + +

์บ๋ฆญํ„ฐ ์˜จ๋ณด๋”ฉ

+ +
+ + +
+
+
+ + + +
+ +
+ + + + + + diff --git a/illust.html b/illust.html new file mode 100644 index 0000000000000000000000000000000000000000..053979a2b88a3fe4aee7c67567cf7e473755ce06 --- /dev/null +++ b/illust.html @@ -0,0 +1,114 @@ + + + + + +์ผ๋Ÿฌ์ŠคํŠธ ํŠธ๋ž™ โ€” DrawFace Live + + + + +
+

์ผ๋Ÿฌ์ŠคํŠธ ํŠธ๋ž™ โ€” ๊ทธ๋ฆผ์ด ๋‚ด ํ‘œ์ •์„ ๋”ฐ๋ผ ํ•œ๋‹ค

+

FasterLivePortrait ยท ๋กœ์ปฌ GPU ์ „์šฉ ยท ์•„๋ž˜๋Š” ๋ฏธ๋ฆฌ ๋ Œ๋”ํ•œ ๊ฒฐ๊ณผ์ž…๋‹ˆ๋‹ค

+ +

+ ์ด ์ €์žฅ์†Œ์—๋Š” ์–ผ๊ตด์„ ์›€์ง์ด๋Š” ๋ฐฉ๋ฒ•์ด ๋‘ ๊ฐ€์ง€ ์žˆ์Šต๋‹ˆ๋‹ค. ๋Œ€์ƒ์— ๋”ฐ๋ผ ๋˜๋Š” ์ชฝ์ด ๊ฐˆ๋ฆฝ๋‹ˆ๋‹ค โ€” + ๊ฐ™์€ ๋ชจ๋ธ์ธ๋ฐ ๊ทธ๋ฆผ ์ข…๋ฅ˜์— ๋”ฐ๋ผ ๊ฒฐ๊ณผ๊ฐ€ ์ •๋ฐ˜๋Œ€๋ผ, ์‹ค์ธก์œผ๋กœ ํ™•์ธํ•œ ๋’ค ํŠธ๋ž™์„ ๋‚˜๋ˆด์Šต๋‹ˆ๋‹ค. +

+ +

๋˜๋Š” ๊ฒƒ  ํ‘œ์ค€ ๋น„์œจ ์ผ๋Ÿฌ์ŠคํŠธ ยท ๋ช…ํ™”

+
+
+ +
+ ๋ฒ ๋ฅด๋ฉ”๋ฅด ใ€ˆ์ง„์ฃผ ๊ท€๊ฑธ์ด๋ฅผ ํ•œ ์†Œ๋…€ใ€‰(1665, ํผ๋ธ”๋ฆญ ๋„๋ฉ”์ธ). + ๋ถ“์งˆ๊ณผ ์บ”๋ฒ„์Šค ๊ท ์—ด์ด ๊ทธ๋Œ€๋กœ ๋‚จ์€ ์ฑ„ ๊ณ ๊ฐœ์™€ ํ‘œ์ •๋งŒ ์›€์ง์ž…๋‹ˆ๋‹ค. +
+
+
+ ๋ฒกํ„ฐ ์ผ๋Ÿฌ์ŠคํŠธ๋ฅผ ๊ตฌ๋™ํ•œ 6ํ”„๋ ˆ์ž„ โ€” ์„ ๊ณผ ์ฑ„์ƒ‰์ด ์œ ์ง€๋œ ์ฑ„ ํ‘œ์ •๋งŒ ๋ฐ”๋€๋‹ค +
+ ๋ฒกํ„ฐ ์ผ๋Ÿฌ์ŠคํŠธ 6ํ”„๋ ˆ์ž„. ์„ ๊ณผ ์ฑ„์ƒ‰์ด ์žฌ์ƒ์„ฑ๋˜์ง€ ์•Š์•„ ์›๋ž˜ ํ™”ํ’์ด ์œ ์ง€๋ฉ๋‹ˆ๋‹ค. +
+
+
+ +

์•ˆ ๋˜๋Š” ๊ฒƒ  ์†๊ทธ๋ฆผ ยท ๋‚™์„œ ยท ์น˜๋น„

+
+ ์†๊ทธ๋ฆผ์— LivePortrait ๋ฅผ ์ ์šฉํ•œ ๊ฒฐ๊ณผ โ€” ์–ผ๊ตด์ด ๊ฒ€์ถœ๋˜์ง€ ์•Š์•„ ์„ธ ํ”„๋ ˆ์ž„ ๋ชจ๋‘ ๋ณ€ํ™”๊ฐ€ ์—†๋‹ค +
+ ๊ฐ™์€ ํŒŒ์ดํ”„๋ผ์ธ, ๊ฐ™์€ ์›น์บ  ์ž…๋ ฅ. ์†๊ทธ๋ฆผ์€ ์–ผ๊ตด ๊ฒ€์ถœ ์ž์ฒด๊ฐ€ ์‹คํŒจํ•ด ์•„๋ฌด ์ผ๋„ ์ผ์–ด๋‚˜์ง€ ์•Š๊ณ , + ๊ฒ€์ถœ์„ ๊ฐ•์ œํ•˜๋ฉด ๋จธ๋ฆฌ ์ „์ฒด๊ฐ€ ๋ญ‰๊ฐœ์ง‘๋‹ˆ๋‹ค(์Šค๋ฏธ์–ด๋ง). ํฐ ์ข…์ด ์œ„ ์„ ํ™”๊ฐ€ ํ•™์Šต ๋ถ„ํฌ ๋ฐ–์ด๊ธฐ ๋•Œ๋ฌธ์ž…๋‹ˆ๋‹ค. +
+
+

+ ๊ทธ๋ž˜์„œ ์†๊ทธ๋ฆผ์€ ์ด ํŠธ๋ž™์ด ์•„๋‹ˆ๋ผ ARAP ์›Œํ”„ ๋ฏธ๋Ÿฌ๋ง์ด ๋‹ด๋‹นํ•ฉ๋‹ˆ๋‹ค. + ์›๋ณธ ํš์„ ์‚ผ๊ฐํ˜• ๊ทธ๋ฌผ๋กœ ๋ณ€ํ˜•ํ•˜๋Š” ๋ฐฉ์‹์ด๋ผ ์—†๋Š” ํ”ฝ์…€์„ ๋งŒ๋“ค์ง€ ์•Š๊ณ , ๋ธŒ๋ผ์šฐ์ €์—์„œ ์‹ค์‹œ๊ฐ„์œผ๋กœ ๋•๋‹ˆ๋‹ค. +

+ +

์™œ ์›น์—์„œ ์ง์ ‘ ๋ชป ๋Œ๋ฆฌ๋‚˜

+
+ + + + +
๋ฐฑ์—”๋“œํ”„๋ ˆ์ž„๋‹น์ฒด๊ฐ
TensorRT (RTX 4070 Ti)27โ€“37 ms์‹ค์‹œ๊ฐ„ (์•ฝ 30fps)
ONNX GPU (๊ฐ™์€ ์นด๋“œ)118โ€“124 ms์•ฝ 8fps โ€” ์‹ค์‹œ๊ฐ„ ์•„๋‹˜
+

+ ์‹ค์‹œ๊ฐ„์€ TensorRT ์ „์šฉ ์ˆซ์ž์ž…๋‹ˆ๋‹ค. ๊ทธ๋Ÿฐ๋ฐ TensorRT ์—”์ง„์€ ๋นŒ๋“œํ•œ GPU์™€ ๋ฒ„์ „์— ๋ฌถ์—ฌ + ๋‹ค๋ฅธ ์žฅ๋น„์—์„œ ๋กœ๋“œ๋˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ๋ฌด๋ฃŒ ํ˜ธ์ŠคํŒ…์€ CPU๋ผ ๋” ๋А๋ฆฌ๊ณ , ์›น์บ  ํ”„๋ ˆ์ž„์„ ์„œ๋ฒ„๊นŒ์ง€ ์™•๋ณต์‹œํ‚ค๋ฉด + "๋”ฐ๋ผ ํ•œ๋‹ค"๋Š” ์ฒด๊ฐ๋„ ๋ฌด๋„ˆ์ง‘๋‹ˆ๋‹ค. ๊ทธ๋ž˜์„œ ์ด ํŽ˜์ด์ง€๋Š” ๋ฏธ๋ฆฌ ๋ Œ๋”ํ•œ ๊ฒฐ๊ณผ๋งŒ ๋ณด์—ฌ์ค๋‹ˆ๋‹ค. +

+
+ +
+ ์‹ค์‹œ๊ฐ„ ๋ฏธ๋Ÿฌ๋ง ํ•ด๋ณด๊ธฐ + ์ง์ ‘ ๋Œ๋ ค๋ณด๋Š” ๋ฒ• +
+ + +
+ + diff --git a/img/auto_visemes.png b/img/auto_visemes.png new file mode 100644 index 0000000000000000000000000000000000000000..4bd3f73a32e420b0cff060782c755ddcfd7449d2 --- /dev/null +++ b/img/auto_visemes.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fd6d6cffc747078ae50bd35a7f39ead6344d3e203b1a11ff2d8e8bd9f661c10 +size 572270 diff --git a/img/demo.gif b/img/demo.gif new file mode 100644 index 0000000000000000000000000000000000000000..d01f15991862465f56442552a07b742c6066fa0f Binary files /dev/null and b/img/demo.gif differ diff --git a/img/flp/fail_handdrawn.png b/img/flp/fail_handdrawn.png new file mode 100644 index 0000000000000000000000000000000000000000..83f3caed364882de333c02f1f93cb41e9e3173e5 --- /dev/null +++ b/img/flp/fail_handdrawn.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab9818119517d7a8538511f3e873d9caeffcbe54ae620daca2348e996b156ed7 +size 124255 diff --git a/img/flp/vermeer.mp4 b/img/flp/vermeer.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0e485449944abf615706e4467579d7f609d94a0e Binary files /dev/null and b/img/flp/vermeer.mp4 differ diff --git a/img/flp_illust_ok.png b/img/flp_illust_ok.png new file mode 100644 index 0000000000000000000000000000000000000000..b35d89da9c594a1a46e82041a4e11822d01a9212 --- /dev/null +++ b/img/flp_illust_ok.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daaab47ca73cb26fa6ba24d59660899fcdcd93d985c62058cb0b0ca54175be78 +size 185746 diff --git a/img/flp_painting_ok.png b/img/flp_painting_ok.png new file mode 100644 index 0000000000000000000000000000000000000000..67b8b15c788511488d266c1a1c794c4a40c052fa --- /dev/null +++ b/img/flp_painting_ok.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd5e9593263bd898ad8a38101f88f79b4fcf6e4d65da749e4fc939fd454a9b94 +size 118015 diff --git a/img/pig_states.png b/img/pig_states.png new file mode 100644 index 0000000000000000000000000000000000000000..0c40d9d2a6d68cc5301d60dcc7df3ae8c0bcf858 --- /dev/null +++ b/img/pig_states.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df5594634dc3c0c65afa590dae5176c7bb25a21fd1bfd62b0306b3ce8fcd8d2e +size 345943 diff --git a/img/sprite_vs_flp.gif b/img/sprite_vs_flp.gif new file mode 100644 index 0000000000000000000000000000000000000000..d18f70e889ebed12b211d8ea4cf8524a26daf675 --- /dev/null +++ b/img/sprite_vs_flp.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4b1fd2d3b5958bc66bfa369b7846e64cf277b4e1296992351ac4cfac5e62ac1 +size 2394078 diff --git a/img/stick_auto.png b/img/stick_auto.png new file mode 100644 index 0000000000000000000000000000000000000000..8d2706ded06fbea850ff03be3ae6553ebad9f963 --- /dev/null +++ b/img/stick_auto.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:875e67fed5809f86d734c7a12b0a7f6d1d94f811fa60fe485ace2af52e85306a +size 376176 diff --git a/img/studio_5.gif b/img/studio_5.gif new file mode 100644 index 0000000000000000000000000000000000000000..75820a3857cb2f138242497fbd7c4f9ab1603c90 --- /dev/null +++ b/img/studio_5.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0ccbeeaf039b8c5028c3589ef855a198e234bb0b382d2c3ff912bb6df483d4b +size 2307205 diff --git a/img/studio_live.gif b/img/studio_live.gif new file mode 100644 index 0000000000000000000000000000000000000000..2d48d0865cbd8188737e1c149b058cca9ebe13d3 --- /dev/null +++ b/img/studio_live.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08a3a73e9b6d1bbb75a903eb5e4185d252aca7308d327c1f4b0426de79bafec2 +size 2222411 diff --git a/img/studio_sync.gif b/img/studio_sync.gif new file mode 100644 index 0000000000000000000000000000000000000000..1747dc7fdda575ec245d87e265cb3214e3622117 --- /dev/null +++ b/img/studio_sync.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:736af3abdd7abd7b3ced7fca58c0d3c808d27c20710a3b1aed0706c3d0e789a9 +size 3231321 diff --git a/index.html b/index.html index b0c4b3666032a737f3903db53e6a8a9272483e28..4f7d75f46a142d6327c6bc6d55ab0d968bdd38d3 100644 --- a/index.html +++ b/index.html @@ -1,19 +1,412 @@ - - - - - - My static Space - - - -
-

Welcome to your static Space!

-

You can modify this app directly by editing index.html in the Files and versions tab.

-

- Also don't forget to check the - Spaces documentation. -

-
- + + + + + +DrawFace Live โ€” ํ‘œ์ • ๋ฏธ๋Ÿฌ๋ง + + + + + +
+

DrawFace Live โ€” ์›น์บ  ํ‘œ์ • ๋ฏธ๋Ÿฌ๋ง

+ +
+
+ +
+
+ +
+ + + + + + 5์ธ ๋™์‹œ + ๋‚ด ์†๊ทธ๋ฆผ โ†’ +
+ +
๋ชจ๋ธ ๋กœ๋”ฉ ์ค‘โ€ฆ
+ + +
+ + + + diff --git a/js/compositor.js b/js/compositor.js new file mode 100644 index 0000000000000000000000000000000000000000..38210e67ecc8541e688e8d0cf65ab5e30c41035a --- /dev/null +++ b/js/compositor.js @@ -0,0 +1,109 @@ +// Canvas compositor โ€” the web counterpart of app/sprite_backend.py's +// SpriteCharacter.compose + apply_head_transform. Sprites are 512x512 RGBA +// overlays aligned to a flattened base; canvas alpha compositing replaces the +// premultiplied numpy math. Sprite keys 'L'/'R' are VIEWER-left/right. +import { CANVAS } from "./config.js?v=20260729.6"; +import { newCanvas } from "./imageops.js?v=20260729.6"; +import { deriveAll } from "./derive.js?v=20260729.6"; + +const clamp = (v, lo, hi) => Math.max(lo, Math.min(hi, v)); + +function req(canvases, file) { + const c = canvases[file]; + if (!c) throw new Error(`missing sprite: ${file} (artwork is never auto-generated)`); + return c; +} + +// Flatten a possibly-transparent base onto white paper, once, into a 512 canvas. +function flattenBase(base) { + const c = newCanvas(CANVAS, CANVAS); + const ctx = c.getContext("2d"); + ctx.fillStyle = "#ffffff"; + ctx.fillRect(0, 0, CANVAS, CANVAS); + ctx.drawImage(base, 0, 0); + return c; +} + +// From {name, manifest, canvases:{file:canvas}} (store.js loadCharacter shape) build +// the runtime character: flattened base, eye/mouth sprite lookup, memo cache. Absent +// optional sprites resolve once here (half->open, smile->closed), like sprite_backend. +export function prepareCharacter(loaded) { + const { canvases } = loaded; + // store.js persists only the core sprite set; derived ones (visemes, halfs, + // smile) are deterministic, so rebuild them here on load (~100 ms). + if (loaded.manifest?.proceduralMouth && !canvases["mouth_A.png"]) { + deriveAll(canvases, loaded.manifest); + } + const base = flattenBase(req(canvases, "base.png")); + + const eyes = {}; + for (const side of ["L", "R"]) { + const open = req(canvases, `eye_${side}_open.png`); + const closed = req(canvases, `eye_${side}_closed.png`); + const half = canvases[`eye_${side}_half.png`] ?? open; + eyes[side] = { open, half, closed }; + } + + const mouths = {}; + for (const k of ["closed", "A", "E", "I", "O", "U"]) mouths[k] = req(canvases, `mouth_${k}.png`); + mouths.smile = canvases["mouth_smile.png"] ?? mouths.closed; + + return { name: loaded.name, manifest: loaded.manifest, base, eyes, mouths, + memo: new Map() }; +} + +// Memoized per-state composite (base + eye L + eye R + mouth), <=72 combinations. +// Callers must NOT mutate the returned canvas โ€” it is the cached instance. +export function composeCharacter(char, eyeL, eyeR, mouth) { + const key = `${eyeL}|${eyeR}|${mouth}`; + const cached = char.memo.get(key); + if (cached) return cached; + + const c = newCanvas(CANVAS, CANVAS); + const ctx = c.getContext("2d"); + ctx.drawImage(char.base, 0, 0); + ctx.drawImage(char.eyes.L[eyeL], 0, 0); + ctx.drawImage(char.eyes.R[eyeR], 0, 0); + ctx.drawImage(char.mouths[mouth], 0, 0); + char.memo.set(key, c); + return c; +} + +// getImageData is a synchronous GPU->CPU readback โ€” cache per composed canvas +// (composites are memoized and never mutated, so the corner pixel is invariant). +const bgColorCache = new WeakMap(); +function topLeftColor(canvas) { + let color = bgColorCache.get(canvas); + if (!color) { + const px = canvas.getContext("2d").getImageData(0, 0, 1, 1).data; + color = `rgb(${px[0]},${px[1]},${px[2]})`; + bgColorCache.set(canvas, color); + } + return color; +} + +// Draw a composed character onto ctx (512x512) with a cheap 2.5D head transform: +// translate for yaw/pitch, rotate for roll, both about the canvas center โ€” matching +// app/sprite_backend.apply_head_transform. BORDER_REPLICATE is approximated by +// pre-filling with the composed top-left pixel color. +export function drawScene(ctx, composed, head, headCfg) { + // The target canvas may be a hi-res warp output (source-resolution rig) โ€” + // scale the px-tuned head gains so motion stays proportional. + const S = ctx.canvas.width; + const k = S / CANVAS; + const dx = clamp(head.yaw * headCfg.yawGainPx, -headCfg.maxShiftPx, headCfg.maxShiftPx) * k; + const dy = clamp(head.pitch * headCfg.pitchGainPx, -headCfg.maxShiftPx, headCfg.maxShiftPx) * k; + const angleDeg = clamp(head.roll * headCfg.rollGain, -headCfg.maxRollDeg, headCfg.maxRollDeg); + const cx = S / 2, cy = S / 2; + + ctx.save(); + ctx.fillStyle = topLeftColor(composed); + ctx.fillRect(0, 0, S, S); + // Transform = Translate(dx,dy) โˆ˜ RotateAboutCenter(angle); negative angle matches + // OpenCV getRotationMatrix2D's CCW-positive convention (canvas rotate is CW-positive). + ctx.translate(cx + dx, cy + dy); + ctx.rotate((-angleDeg * Math.PI) / 180); + ctx.translate(-cx, -cy); + ctx.drawImage(composed, 0, 0, S, S); + ctx.restore(); +} diff --git a/js/config.js b/js/config.js new file mode 100644 index 0000000000000000000000000000000000000000..f56d733c9a273d85484758c893c7d292bd176cc0 --- /dev/null +++ b/js/config.js @@ -0,0 +1,49 @@ +// Mirror of configs/app.yaml โ€” keep thresholds in sync when tuning. +export const CONFIG = { + camera: { width: 640, height: 480 }, + control: { mirror: true }, + // Live loop uses the One Euro filter; the fixed alphas remain for reference + // parity with offline python tools. + smoothing: { + blendAlpha: 0.45, blinkAlpha: 0.70, headAlpha: 0.30, + minCutoff: 1.2, beta: 2.0, headMinCutoff: 1.0, headBeta: 0.02, + }, + eyes: { + closeThreshold: 0.38, openThreshold: 0.25, + halfCloseThreshold: 0.20, halfOpenThreshold: 0.12, + }, + mouth: { + jawClosed: 0.06, jawMid: 0.16, jawLarge: 0.32, + puckerThreshold: 0.40, funnelThreshold: 0.35, smileThreshold: 0.30, + }, + head: { yawGainPx: -1.6, pitchGainPx: 1.2, rollGain: 1.0, maxShiftPx: 48, maxRollDeg: 20 }, + lostFace: { holdMs: 300, decayMs: 600 }, + calibration: { frames: 30 }, + // jawGain 2.2: MediaPipe jawOpen ์€ ๋งํ•˜๊ธฐ ์ˆ˜์ค€์—์„œ 0.1~0.25 ๋ผ 1.6 ์œผ๋ก  ์›Œํ”„ ๋žจํ”„๋ฅผ ๋ชป ๋„˜์–ด + // ํฌ๊ฒŒ ์ฉ ๋ฒŒ๋ ค์•ผ๋งŒ ์ž…์ด ์—ด๋ ธ๋‹ค โ€” ๋ณดํ†ต ๋ฒŒ๋ฆผ์—๋„ ๋ฐ˜์‘ํ•˜๊ฒŒ ์ƒํ–ฅ (warp.js JAW_RAMP ํ•˜ํ–ฅ๊ณผ ์„ธํŠธ). + warp: { blinkGain: 2.0, smileGain: 2.0, jawGain: 2.2, headParallax: 1.0 }, + idle: { breathPeriodS: 3.6, breathAmp: 0.05, blinkMinS: 4.0, blinkMaxS: 7.0, blinkMs: 260 }, + // Keep tracking responsive on sustained low-end CPU pressure by temporarily + // rendering the cached sprite path instead of the mesh. + performance: { frameBudgetMs: 28, degradeAfterMs: 1800, recoverAfterMs: 4500 }, +}; + +export const CANVAS = 512; +// Hi-res source cap: onboarding keeps the original drawing up to this size. +// r6 ๋ถ€ํ„ฐ warp rig ๋Š” ์‚ฌ์šฉํ•˜์ง€ ์•Š์Œ โ€” hi-res base ์ธํŽ˜์ธํŠธ ์žฌ์„ค๊ณ„ยท์‹ ๊ฒฝ๋ง ํŠธ๋ž™ ๋Œ€๋น„ ๋ณด์กด. +export const SOURCE_MAX = 1024; + +// Sprite derivation parameters (mirror of scripts/derive_sprites.py) +export const DERIVE = { + squash: 0.45, // half-eye: kept fraction of eye height + smileAmpFrac: 0.10, // smile corner lift as fraction of mouth width + fillDarken: 0.72, // viseme interior darkening + visemes: { + A: { drop: 0.62, wx: 0.85, teeth: 0.00, tongue: 0.35 }, + E: { drop: 0.30, wx: 1.10, teeth: 0.22, tongue: 0.00 }, + I: { drop: 0.15, wx: 1.05, teeth: 0.30, tongue: 0.00 }, + O: { drop: 0.45, wx: 0.70, teeth: 0.00, tongue: 0.30 }, + U: { drop: 0.26, wx: 0.55, teeth: 0.00, tongue: 0.00 }, + }, + defaultStyle: { line: "#2b2b2b", fill: "#8a3535", tongue: "#d97b7b", teeth: "#ffffff" }, +}; diff --git a/js/delaunator.js b/js/delaunator.js new file mode 100644 index 0000000000000000000000000000000000000000..fe03b9710258126ad2146031f3aa9666d2523cbe --- /dev/null +++ b/js/delaunator.js @@ -0,0 +1,497 @@ +// Delaunator 4.0.1 โ€” vendored unmodified from https://github.com/mapbox/delaunator (ISC). +// See THIRD_PARTY.md. Used by warp.js to triangulate the face mesh. + +const EPSILON = Math.pow(2, -52); +const EDGE_STACK = new Uint32Array(512); + +export default class Delaunator { + + static from(points, getX = defaultGetX, getY = defaultGetY) { + const n = points.length; + const coords = new Float64Array(n * 2); + + for (let i = 0; i < n; i++) { + const p = points[i]; + coords[2 * i] = getX(p); + coords[2 * i + 1] = getY(p); + } + + return new Delaunator(coords); + } + + constructor(coords) { + const n = coords.length >> 1; + if (n > 0 && typeof coords[0] !== 'number') throw new Error('Expected coords to contain numbers.'); + + this.coords = coords; + + // arrays that will store the triangulation graph + const maxTriangles = Math.max(2 * n - 5, 0); + this._triangles = new Uint32Array(maxTriangles * 3); + this._halfedges = new Int32Array(maxTriangles * 3); + + // temporary arrays for tracking the edges of the advancing convex hull + this._hashSize = Math.ceil(Math.sqrt(n)); + this._hullPrev = new Uint32Array(n); // edge to prev edge + this._hullNext = new Uint32Array(n); // edge to next edge + this._hullTri = new Uint32Array(n); // edge to adjacent triangle + this._hullHash = new Int32Array(this._hashSize).fill(-1); // angular edge hash + + // temporary arrays for sorting points + this._ids = new Uint32Array(n); + this._dists = new Float64Array(n); + + this.update(); + } + + update() { + const {coords, _hullPrev: hullPrev, _hullNext: hullNext, _hullTri: hullTri, _hullHash: hullHash} = this; + const n = coords.length >> 1; + + // populate an array of point indices; calculate input data bbox + let minX = Infinity; + let minY = Infinity; + let maxX = -Infinity; + let maxY = -Infinity; + + for (let i = 0; i < n; i++) { + const x = coords[2 * i]; + const y = coords[2 * i + 1]; + if (x < minX) minX = x; + if (y < minY) minY = y; + if (x > maxX) maxX = x; + if (y > maxY) maxY = y; + this._ids[i] = i; + } + const cx = (minX + maxX) / 2; + const cy = (minY + maxY) / 2; + + let minDist = Infinity; + let i0, i1, i2; + + // pick a seed point close to the center + for (let i = 0; i < n; i++) { + const d = dist(cx, cy, coords[2 * i], coords[2 * i + 1]); + if (d < minDist) { + i0 = i; + minDist = d; + } + } + const i0x = coords[2 * i0]; + const i0y = coords[2 * i0 + 1]; + + minDist = Infinity; + + // find the point closest to the seed + for (let i = 0; i < n; i++) { + if (i === i0) continue; + const d = dist(i0x, i0y, coords[2 * i], coords[2 * i + 1]); + if (d < minDist && d > 0) { + i1 = i; + minDist = d; + } + } + let i1x = coords[2 * i1]; + let i1y = coords[2 * i1 + 1]; + + let minRadius = Infinity; + + // find the third point which forms the smallest circumcircle with the first two + for (let i = 0; i < n; i++) { + if (i === i0 || i === i1) continue; + const r = circumradius(i0x, i0y, i1x, i1y, coords[2 * i], coords[2 * i + 1]); + if (r < minRadius) { + i2 = i; + minRadius = r; + } + } + let i2x = coords[2 * i2]; + let i2y = coords[2 * i2 + 1]; + + if (minRadius === Infinity) { + // order collinear points by dx (or dy if all x are identical) + // and return the list as a hull + for (let i = 0; i < n; i++) { + this._dists[i] = (coords[2 * i] - coords[0]) || (coords[2 * i + 1] - coords[1]); + } + quicksort(this._ids, this._dists, 0, n - 1); + const hull = new Uint32Array(n); + let j = 0; + for (let i = 0, d0 = -Infinity; i < n; i++) { + const id = this._ids[i]; + if (this._dists[id] > d0) { + hull[j++] = id; + d0 = this._dists[id]; + } + } + this.hull = hull.subarray(0, j); + this.triangles = new Uint32Array(0); + this.halfedges = new Uint32Array(0); + return; + } + + // swap the order of the seed points for counter-clockwise orientation + if (orient(i0x, i0y, i1x, i1y, i2x, i2y)) { + const i = i1; + const x = i1x; + const y = i1y; + i1 = i2; + i1x = i2x; + i1y = i2y; + i2 = i; + i2x = x; + i2y = y; + } + + const center = circumcenter(i0x, i0y, i1x, i1y, i2x, i2y); + this._cx = center.x; + this._cy = center.y; + + for (let i = 0; i < n; i++) { + this._dists[i] = dist(coords[2 * i], coords[2 * i + 1], center.x, center.y); + } + + // sort the points by distance from the seed triangle circumcenter + quicksort(this._ids, this._dists, 0, n - 1); + + // set up the seed triangle as the starting hull + this._hullStart = i0; + let hullSize = 3; + + hullNext[i0] = hullPrev[i2] = i1; + hullNext[i1] = hullPrev[i0] = i2; + hullNext[i2] = hullPrev[i1] = i0; + + hullTri[i0] = 0; + hullTri[i1] = 1; + hullTri[i2] = 2; + + hullHash.fill(-1); + hullHash[this._hashKey(i0x, i0y)] = i0; + hullHash[this._hashKey(i1x, i1y)] = i1; + hullHash[this._hashKey(i2x, i2y)] = i2; + + this.trianglesLen = 0; + this._addTriangle(i0, i1, i2, -1, -1, -1); + + for (let k = 0, xp, yp; k < this._ids.length; k++) { + const i = this._ids[k]; + const x = coords[2 * i]; + const y = coords[2 * i + 1]; + + // skip near-duplicate points + if (k > 0 && Math.abs(x - xp) <= EPSILON && Math.abs(y - yp) <= EPSILON) continue; + xp = x; + yp = y; + + // skip seed triangle points + if (i === i0 || i === i1 || i === i2) continue; + + // find a visible edge on the convex hull using edge hash + let start = 0; + for (let j = 0, key = this._hashKey(x, y); j < this._hashSize; j++) { + start = hullHash[(key + j) % this._hashSize]; + if (start !== -1 && start !== hullNext[start]) break; + } + + start = hullPrev[start]; + let e = start, q; + while (q = hullNext[e], !orient(x, y, coords[2 * e], coords[2 * e + 1], coords[2 * q], coords[2 * q + 1])) { + e = q; + if (e === start) { + e = -1; + break; + } + } + if (e === -1) continue; // likely a near-duplicate point; skip it + + // add the first triangle from the point + let t = this._addTriangle(e, i, hullNext[e], -1, -1, hullTri[e]); + + // recursively flip triangles from the point until they satisfy the Delaunay condition + hullTri[i] = this._legalize(t + 2); + hullTri[e] = t; // keep track of boundary triangles on the hull + hullSize++; + + // walk forward through the hull, adding more triangles and flipping recursively + let n = hullNext[e]; + while (q = hullNext[n], orient(x, y, coords[2 * n], coords[2 * n + 1], coords[2 * q], coords[2 * q + 1])) { + t = this._addTriangle(n, i, q, hullTri[i], -1, hullTri[n]); + hullTri[i] = this._legalize(t + 2); + hullNext[n] = n; // mark as removed + hullSize--; + n = q; + } + + // walk backward from the other side, adding more triangles and flipping + if (e === start) { + while (q = hullPrev[e], orient(x, y, coords[2 * q], coords[2 * q + 1], coords[2 * e], coords[2 * e + 1])) { + t = this._addTriangle(q, i, e, -1, hullTri[e], hullTri[q]); + this._legalize(t + 2); + hullTri[q] = t; + hullNext[e] = e; // mark as removed + hullSize--; + e = q; + } + } + + // update the hull indices + this._hullStart = hullPrev[i] = e; + hullNext[e] = hullPrev[n] = i; + hullNext[i] = n; + + // save the two new edges in the hash table + hullHash[this._hashKey(x, y)] = i; + hullHash[this._hashKey(coords[2 * e], coords[2 * e + 1])] = e; + } + + this.hull = new Uint32Array(hullSize); + for (let i = 0, e = this._hullStart; i < hullSize; i++) { + this.hull[i] = e; + e = hullNext[e]; + } + + // trim typed triangle mesh arrays + this.triangles = this._triangles.subarray(0, this.trianglesLen); + this.halfedges = this._halfedges.subarray(0, this.trianglesLen); + } + + _hashKey(x, y) { + return Math.floor(pseudoAngle(x - this._cx, y - this._cy) * this._hashSize) % this._hashSize; + } + + _legalize(a) { + const {_triangles: triangles, _halfedges: halfedges, coords} = this; + + let i = 0; + let ar = 0; + + // recursion eliminated with a fixed-size stack + while (true) { + const b = halfedges[a]; + + /* if the pair of triangles doesn't satisfy the Delaunay condition + * (p1 is inside the circumcircle of [p0, pl, pr]), flip them, + * then do the same check/flip recursively for the new pair of triangles + * + * pl pl + * /||\ / \ + * al/ || \bl al/ \a + * / || \ / \ + * / a||b \ flip /___ar___\ + * p0\ || /p1 => p0\---bl---/p1 + * \ || / \ / + * ar\ || /br b\ /br + * \||/ \ / + * pr pr + */ + const a0 = a - a % 3; + ar = a0 + (a + 2) % 3; + + if (b === -1) { // convex hull edge + if (i === 0) break; + a = EDGE_STACK[--i]; + continue; + } + + const b0 = b - b % 3; + const al = a0 + (a + 1) % 3; + const bl = b0 + (b + 2) % 3; + + const p0 = triangles[ar]; + const pr = triangles[a]; + const pl = triangles[al]; + const p1 = triangles[bl]; + + const illegal = inCircle( + coords[2 * p0], coords[2 * p0 + 1], + coords[2 * pr], coords[2 * pr + 1], + coords[2 * pl], coords[2 * pl + 1], + coords[2 * p1], coords[2 * p1 + 1]); + + if (illegal) { + triangles[a] = p1; + triangles[b] = p0; + + const hbl = halfedges[bl]; + + // edge swapped on the other side of the hull (rare); fix the halfedge reference + if (hbl === -1) { + let e = this._hullStart; + do { + if (this._hullTri[e] === bl) { + this._hullTri[e] = a; + break; + } + e = this._hullPrev[e]; + } while (e !== this._hullStart); + } + this._link(a, hbl); + this._link(b, halfedges[ar]); + this._link(ar, bl); + + const br = b0 + (b + 1) % 3; + + // don't worry about hitting the cap: it can only happen on extremely degenerate input + if (i < EDGE_STACK.length) { + EDGE_STACK[i++] = br; + } + } else { + if (i === 0) break; + a = EDGE_STACK[--i]; + } + } + + return ar; + } + + _link(a, b) { + this._halfedges[a] = b; + if (b !== -1) this._halfedges[b] = a; + } + + // add a new triangle given vertex indices and adjacent half-edge ids + _addTriangle(i0, i1, i2, a, b, c) { + const t = this.trianglesLen; + + this._triangles[t] = i0; + this._triangles[t + 1] = i1; + this._triangles[t + 2] = i2; + + this._link(t, a); + this._link(t + 1, b); + this._link(t + 2, c); + + this.trianglesLen += 3; + + return t; + } +} + +// monotonically increases with real angle, but doesn't need expensive trigonometry +function pseudoAngle(dx, dy) { + const p = dx / (Math.abs(dx) + Math.abs(dy)); + return (dy > 0 ? 3 - p : 1 + p) / 4; // [0..1] +} + +function dist(ax, ay, bx, by) { + const dx = ax - bx; + const dy = ay - by; + return dx * dx + dy * dy; +} + +// return 2d orientation sign if we're confident in it through J. Shewchuk's error bound check +function orientIfSure(px, py, rx, ry, qx, qy) { + const l = (ry - py) * (qx - px); + const r = (rx - px) * (qy - py); + return Math.abs(l - r) >= 3.3306690738754716e-16 * Math.abs(l + r) ? l - r : 0; +} + +// a more robust orientation test that's stable in a given triangle (to fix robustness issues) +function orient(rx, ry, qx, qy, px, py) { + const sign = orientIfSure(px, py, rx, ry, qx, qy) || + orientIfSure(rx, ry, qx, qy, px, py) || + orientIfSure(qx, qy, px, py, rx, ry); + return sign < 0; +} + +function inCircle(ax, ay, bx, by, cx, cy, px, py) { + const dx = ax - px; + const dy = ay - py; + const ex = bx - px; + const ey = by - py; + const fx = cx - px; + const fy = cy - py; + + const ap = dx * dx + dy * dy; + const bp = ex * ex + ey * ey; + const cp = fx * fx + fy * fy; + + return dx * (ey * cp - bp * fy) - + dy * (ex * cp - bp * fx) + + ap * (ex * fy - ey * fx) < 0; +} + +function circumradius(ax, ay, bx, by, cx, cy) { + const dx = bx - ax; + const dy = by - ay; + const ex = cx - ax; + const ey = cy - ay; + + const bl = dx * dx + dy * dy; + const cl = ex * ex + ey * ey; + const d = 0.5 / (dx * ey - dy * ex); + + const x = (ey * bl - dy * cl) * d; + const y = (dx * cl - ex * bl) * d; + + return x * x + y * y; +} + +function circumcenter(ax, ay, bx, by, cx, cy) { + const dx = bx - ax; + const dy = by - ay; + const ex = cx - ax; + const ey = cy - ay; + + const bl = dx * dx + dy * dy; + const cl = ex * ex + ey * ey; + const d = 0.5 / (dx * ey - dy * ex); + + const x = ax + (ey * bl - dy * cl) * d; + const y = ay + (dx * cl - ex * bl) * d; + + return {x, y}; +} + +function quicksort(ids, dists, left, right) { + if (right - left <= 20) { + for (let i = left + 1; i <= right; i++) { + const temp = ids[i]; + const tempDist = dists[temp]; + let j = i - 1; + while (j >= left && dists[ids[j]] > tempDist) ids[j + 1] = ids[j--]; + ids[j + 1] = temp; + } + } else { + const median = (left + right) >> 1; + let i = left + 1; + let j = right; + swap(ids, median, i); + if (dists[ids[left]] > dists[ids[right]]) swap(ids, left, right); + if (dists[ids[i]] > dists[ids[right]]) swap(ids, i, right); + if (dists[ids[left]] > dists[ids[i]]) swap(ids, left, i); + + const temp = ids[i]; + const tempDist = dists[temp]; + while (true) { + do i++; while (dists[ids[i]] < tempDist); + do j--; while (dists[ids[j]] > tempDist); + if (j < i) break; + swap(ids, i, j); + } + ids[left + 1] = ids[j]; + ids[j] = temp; + + if (right - i + 1 >= j - left) { + quicksort(ids, dists, i, right); + quicksort(ids, dists, left, j - 1); + } else { + quicksort(ids, dists, left, j - 1); + quicksort(ids, dists, i, right); + } + } +} + +function swap(arr, i, j) { + const tmp = arr[i]; + arr[i] = arr[j]; + arr[j] = tmp; +} + +function defaultGetX(p) { + return p[0]; +} +function defaultGetY(p) { + return p[1]; +} diff --git a/js/derive.js b/js/derive.js new file mode 100644 index 0000000000000000000000000000000000000000..a027e3a2d4b01a6a6ace33f6aae51605ebb3aa95 --- /dev/null +++ b/js/derive.js @@ -0,0 +1,267 @@ +// Port of scripts/derive_sprites.py: mechanical warps of existing sprites plus +// procedural visemes from a closed-mouth stroke. No new artwork. deriveAll never +// overwrites an existing canvas key (hand-made sprites win). Pixel work is on +// typed arrays; canvas API only for whole-image ops (arc/ellipse draw, resize). +import { CANVAS, DERIVE } from "./config.js?v=20260729.6"; +import { newCanvas, getData, canvasFromData, hexToRgb, median, rad, bankersRound, bboxAlpha } from "./imageops.js?v=20260729.6"; + +const N = CANVAS * CANVAS; + +/** Port of make_procedural_closed: lower-half ellipse arc from manifest params. */ +function makeProceduralClosed(manifest) { + const [cx, cy] = manifest.mouthCenter; + const halfW = Math.trunc(manifest.mouthStyle?.width ?? 20); + const line = manifest.mouthStyle?.line ?? "#2e2e2e"; + const c = newCanvas(CANVAS, CANVAS); + const ctx = c.getContext("2d"); + ctx.strokeStyle = line; + ctx.lineWidth = 4; + ctx.beginPath(); + ctx.ellipse(cx, cy, halfW, Math.trunc(halfW * 0.55), 0, rad(25), rad(155), false); + ctx.stroke(); + return c; +} + +/** Mark mask=1 where placed lip's raw alpha exceeds thr, clipped to canvas. */ +function stampAlpha(mask, lip, xLeft, top, thr) { + const { data, width: lw, height: lh } = lip; + for (let ly = 0; ly < lh; ly++) { + const dy = top + ly; + if (dy < 0 || dy >= CANVAS) continue; + for (let lx = 0; lx < lw; lx++) { + const dx = xLeft + lx; + if (dx < 0 || dx >= CANVAS) continue; + if (data[(ly * lw + lx) * 4 + 3] > thr) mask[dy * CANVAS + dx] = 1; + } + } +} + +/** Src-over lip onto out; blends all 4 channels like python _place. */ +function compositeLip(out, lip, xLeft, top) { + const { data, width: lw, height: lh } = lip; + for (let ly = 0; ly < lh; ly++) { + const dy = top + ly; + if (dy < 0 || dy >= CANVAS) continue; + for (let lx = 0; lx < lw; lx++) { + const dx = xLeft + lx; + if (dx < 0 || dx >= CANVAS) continue; + const k = (ly * lw + lx) * 4; + const a = data[k + 3] / 255; + if (a === 0) continue; + const i = (dy * CANVAS + dx) * 4; + for (let ch = 0; ch < 4; ch++) out[i + ch] = data[k + ch] * a + out[i + ch] * (1 - a); + } + } +} + +/** 3x3 max dilation, `iters` times. */ +function dilate(mask, iters) { + let cur = mask; + for (let it = 0; it < iters; it++) { + const next = new Uint8Array(N); + for (let y = 0; y < CANVAS; y++) { + for (let x = 0; x < CANVAS; x++) { + let v = 0; + for (let dy = -1; dy <= 1 && !v; dy++) { + const ny = y + dy; + if (ny < 0 || ny >= CANVAS) continue; + for (let dx = -1; dx <= 1; dx++) { + const nx = x + dx; + if (nx >= 0 && nx < CANVAS && cur[ny * CANVAS + nx]) { v = 1; break; } + } + } + next[y * CANVAS + x] = v; + } + } + cur = next; + } + return cur; +} + +function buildViseme(y0, mw, sh, cxf, strokeCanvas, colors, ink, thickness, p) { + const lipW = Math.max(2, Math.trunc(mw * p.wx)); + const lipCanvas = newCanvas(lipW, sh); + const lctx = lipCanvas.getContext("2d"); + lctx.imageSmoothingEnabled = true; + lctx.imageSmoothingQuality = "high"; + lctx.drawImage(strokeCanvas, 0, 0, mw, sh, 0, 0, lipW, sh); + const lip = lctx.getImageData(0, 0, lipW, sh); + const drop = Math.trunc(mw * p.drop); + const xoff = bankersRound(cxf - lipW / 2); + + const out = new Uint8ClampedArray(N * 4); + + // interior: per column, fill between the bottom of the top lip and the top of + // the bottom lip. Masks threshold raw lip alpha at >123 (python _place squares + // alpha, so its `>60` on the placed layer equals raw alpha > 123.69). + const topMask = new Uint8Array(N); + const botMask = new Uint8Array(N); + stampAlpha(topMask, lip, xoff, y0, 123); + stampAlpha(botMask, lip, xoff, y0 + drop, 123); + for (let x = 0; x < CANVAS; x++) { + let tcLast = -1; + for (let y = CANVAS - 1; y >= 0; y--) { if (topMask[y * CANVAS + x]) { tcLast = y; break; } } + if (tcLast < 0) continue; + let bcFirst = -1; + for (let y = 0; y < CANVAS; y++) { if (botMask[y * CANVAS + x]) { bcFirst = y; break; } } + if (bcFirst < 0 || bcFirst <= tcLast) continue; + for (let y = tcLast; y <= bcFirst; y++) { + const i = (y * CANVAS + x) * 4; + out[i] = colors.fill[0]; out[i + 1] = colors.fill[1]; out[i + 2] = colors.fill[2]; out[i + 3] = 255; + } + } + + const inner = new Uint8Array(N); + let anyInner = false, firstRow = -1, lastRow = -1; + for (let y = 0; y < CANVAS; y++) { + let rowHas = false; + for (let x = 0; x < CANVAS; x++) { + if (out[(y * CANVAS + x) * 4 + 3] > 0) { inner[y * CANVAS + x] = 1; rowHas = true; anyInner = true; } + } + if (rowHas) { if (firstRow < 0) firstRow = y; lastRow = y; } + } + + if (p.teeth > 0 && anyInner) { + const bandEnd = Math.min(CANVAS, firstRow + Math.max(2, Math.trunc(drop * p.teeth))); + for (let y = firstRow; y < bandEnd; y++) { + for (let x = 0; x < CANVAS; x++) { + if (inner[y * CANVAS + x]) { + const i = (y * CANVAS + x) * 4; + out[i] = colors.teeth[0]; out[i + 1] = colors.teeth[1]; out[i + 2] = colors.teeth[2]; out[i + 3] = 255; + } + } + } + } + + if (p.tongue > 0 && anyInner) { + // top half of an ellipse centered at the interior floor (cv2 angles 180..360). + const tc = newCanvas(CANVAS, CANVAS); + const tctx = tc.getContext("2d"); + tctx.fillStyle = `rgb(${colors.tongue[0]},${colors.tongue[1]},${colors.tongue[2]})`; + tctx.beginPath(); + tctx.ellipse(Math.trunc(cxf), lastRow, Math.trunc(mw * 0.28), Math.max(2, Math.trunc(drop * p.tongue)), + 0, Math.PI, 2 * Math.PI, false); + tctx.fill(); + const td = tctx.getImageData(0, 0, CANVAS, CANVAS).data; + for (let idx = 0; idx < N; idx++) { + if (td[idx * 4 + 3] > 127 && inner[idx]) { + out[idx * 4] = colors.tongue[0]; out[idx * 4 + 1] = colors.tongue[1]; + out[idx * 4 + 2] = colors.tongue[2]; out[idx * 4 + 3] = 255; + } + } + } + + compositeLip(out, lip, xoff, y0); + compositeLip(out, lip, xoff, y0 + drop); + + // ink outline: dilate the content mask and paint the new ring in ink under the + // content (contract's equivalent of cv2.drawContours). + const mask0 = new Uint8Array(N); + for (let idx = 0; idx < N; idx++) mask0[idx] = out[idx * 4 + 3] > 60 ? 1 : 0; + const dil = dilate(mask0, Math.round(thickness / 2) + 1); + for (let idx = 0; idx < N; idx++) { + if (dil[idx] && !mask0[idx]) { + const ca = out[idx * 4 + 3] / 255; + out[idx * 4] = out[idx * 4] * ca + ink[0] * (1 - ca); + out[idx * 4 + 1] = out[idx * 4 + 1] * ca + ink[1] * (1 - ca); + out[idx * 4 + 2] = out[idx * 4 + 2] * ca + ink[2] * (1 - ca); + out[idx * 4 + 3] = 255; + } + } + return canvasFromData(out, CANVAS, CANVAS); +} + +/** Port of derive_mouth_set: A/E/I/O/U from the closed-mouth stroke + manifest. */ +function deriveMouthSet(canvases, manifest, closed) { + const closedData = getData(closed); + const bb = bboxAlpha(closedData); + if (!bb) return; + const [x0, y0, x1, y1] = bb; + const mw = x1 - x0, sh = y1 - y0, cxf = (x0 + x1) / 2; + const strokeCanvas = newCanvas(mw, sh); + strokeCanvas.getContext("2d").putImageData(closed.getContext("2d").getImageData(x0, y0, mw, sh), 0, 0); + + const style = manifest.mouthStyle ?? {}; + const colors = { + fill: hexToRgb(style.fill ?? "#8a3535").map((v) => Math.floor(v * DERIVE.fillDarken)), + tongue: hexToRgb(style.tongue ?? "#d97b7b"), + teeth: hexToRgb(style.teeth ?? "#ffffff"), + }; + + // ink color = median of opaque stroke pixels; thickness = mean ink px/col * 0.9. + const cd = closedData.data; + const iR = [], iG = [], iB = []; + for (let idx = 0; idx < N; idx++) { + if (cd[idx * 4 + 3] > 200) { iR.push(cd[idx * 4]); iG.push(cd[idx * 4 + 1]); iB.push(cd[idx * 4 + 2]); } + } + const ink = iR.length ? [median(iR), median(iG), median(iB)] : [43, 43, 43]; + let colSum = 0; + for (let x = x0; x < x1; x++) { + let cnt = 0; + for (let y = 0; y < CANVAS; y++) if (cd[(y * CANVAS + x) * 4 + 3] > 60) cnt++; + colSum += cnt; + } + const thickness = Math.max(2, bankersRound((colSum / (x1 - x0)) * 0.9)); + + for (const [name, p] of Object.entries(DERIVE.visemes)) { + const key = `mouth_${name}.png`; + if (key in canvases) continue; + canvases[key] = buildViseme(y0, mw, sh, cxf, strokeCanvas, colors, ink, thickness, p); + } +} + +/** Port of derive_half_eye: vertical squash of the open eye, anchored at its bbox bottom. */ +function deriveHalfEye(open) { + const bb = bboxAlpha(getData(open)); + if (!bb) return canvasFromData(new Uint8ClampedArray(N * 4), CANVAS, CANVAS); + const [x0, y0, x1, y1] = bb; + const cw = x1 - x0, chh = y1 - y0; + const newH = Math.max(2, bankersRound(chh * DERIVE.squash)); + const src = newCanvas(cw, chh); + src.getContext("2d").putImageData(open.getContext("2d").getImageData(x0, y0, cw, chh), 0, 0); + const out = newCanvas(CANVAS, CANVAS); + const octx = out.getContext("2d"); + octx.imageSmoothingEnabled = true; + octx.imageSmoothingQuality = "high"; + octx.drawImage(src, 0, 0, cw, chh, x0, y1 - newH, cw, newH); + return out; +} + +/** Port of derive_smile: per-column integer lift (corners rise, center still). */ +function deriveSmile(closed) { + const d = getData(closed); + const data = d.data; + const bb = bboxAlpha(d); + if (!bb) return canvasFromData(new Uint8ClampedArray(N * 4), CANVAS, CANVAS); + const [x0, y0, x1, y1] = bb; + const cxf = (x0 + x1) / 2, halfW = Math.max(1, (x1 - x0) / 2); + const amp = (x1 - x0) * DERIVE.smileAmpFrac; + const out = new Uint8ClampedArray(N * 4); + for (let x = x0; x < x1; x++) { + const t = Math.min(1, Math.abs(x - cxf) / halfW); + const shift = bankersRound(amp * t * t); + for (let y = 0; y < CANVAS; y++) { + if (shift > 0 && y >= CANVAS - shift) continue; + const si = ((y + shift) * CANVAS + x) * 4, di = (y * CANVAS + x) * 4; + out[di] = data[si]; out[di + 1] = data[si + 1]; out[di + 2] = data[si + 2]; out[di + 3] = data[si + 3]; + } + } + return canvasFromData(out, CANVAS, CANVAS); +} + +/** Adds derived sprites to `canvases` in place; never overwrites an existing key. */ +export function deriveAll(canvases, manifest) { + if (!("mouth_closed.png" in canvases) && manifest.proceduralMouth) { + canvases["mouth_closed.png"] = makeProceduralClosed(manifest); + } + const closed = canvases["mouth_closed.png"]; + if (closed) { + deriveMouthSet(canvases, manifest, closed); + if (!("mouth_smile.png" in canvases)) canvases["mouth_smile.png"] = deriveSmile(closed); + } + for (const side of ["L", "R"]) { + const open = canvases[`eye_${side}_open.png`]; + const key = `eye_${side}_half.png`; + if (open && !(key in canvases)) canvases[key] = deriveHalfEye(open); + } +} diff --git a/js/effects.js b/js/effects.js new file mode 100644 index 0000000000000000000000000000000000000000..1ca7105b4f214d0336515b4deb92f789cc99853a --- /dev/null +++ b/js/effects.js @@ -0,0 +1,94 @@ +// Expression-triggered sticker effects โ€” canvas-drawn shapes (no emoji font +// dependency) that spawn on strong smiles / surprise / kiss and float up over +// the character. Drawn on the output canvas after the character, so it never +// touches the sprite pipeline. + +const TRIGGERS = [ + // kind -> test on smoothed blendshapes, threshold, spawn cooldown (ms) + { kind: "star", test: (b) => (b.mouthSmileLeft + b.mouthSmileRight) / 2, on: 0.45, gap: 220 }, + { kind: "burst", test: (b) => b.jawOpen, on: 0.55, gap: 600 }, + { kind: "heart", test: (b) => b.mouthPucker, on: 0.55, gap: 260 }, +]; + +function star(ctx, r) { + ctx.beginPath(); + for (let i = 0; i < 8; i++) { + const a = (Math.PI / 4) * i; + const rad = i % 2 ? r * 0.42 : r; + ctx[i ? "lineTo" : "moveTo"](Math.cos(a) * rad, Math.sin(a) * rad); + } + ctx.closePath(); + ctx.fillStyle = "#ffcf3f"; + ctx.fill(); +} + +function heart(ctx, r) { + ctx.beginPath(); + ctx.moveTo(0, r * 0.35); + ctx.bezierCurveTo(r, -r * 0.5, r * 0.5, -r, 0, -r * 0.35); + ctx.bezierCurveTo(-r * 0.5, -r, -r, -r * 0.5, 0, r * 0.35); + ctx.closePath(); + ctx.fillStyle = "#ff5c8a"; + ctx.fill(); +} + +function burst(ctx, r) { + ctx.strokeStyle = "#ff8a3d"; + ctx.lineWidth = Math.max(2, r * 0.16); + ctx.lineCap = "round"; + for (let i = 0; i < 6; i++) { + const a = (Math.PI / 3) * i; + ctx.beginPath(); + ctx.moveTo(Math.cos(a) * r * 0.4, Math.sin(a) * r * 0.4); + ctx.lineTo(Math.cos(a) * r, Math.sin(a) * r); + ctx.stroke(); + } +} + +const SHAPES = { star, heart, burst }; + +export class StickerFx { + constructor(canvasSize = 512) { + this.size = canvasSize; + this.particles = []; + this.cooldown = {}; + this.seq = 0; + } + + /** Spawn/age particles from the current smoothed blendshapes. */ + update(blend, now) { + for (const t of TRIGGERS) { + const v = t.test(blend); + if (v >= t.on && now >= (this.cooldown[t.kind] ?? 0)) { + this.cooldown[t.kind] = now + t.gap; + const strength = Math.min(1, (v - t.on) / (1 - t.on)); + const count = 1 + Math.round(strength * 2); + for (let i = 0; i < count; i++) { + this.seq++; + this.particles.push({ + x: this.size * (0.26 + 0.48 * ((this.seq * 0.618) % 1)), + y: this.size * 0.34, kind: t.kind, + vx: (((this.seq * 0.37) % 1) - 0.5) * 0.5, + spin: (((this.seq * 0.29) % 1) - 0.5) * 0.02, + born: now, life: 1100 + strength * 500, r: 13 + strength * 8, + }); + } + } + } + this.particles = this.particles.filter((p) => now - p.born < p.life); + } + + /** Draw active particles onto ctx (rising + fading + spinning). */ + draw(ctx, now) { + for (const p of this.particles) { + const age = now - p.born; + const t = age / p.life; // 0..1 + ctx.save(); + ctx.globalAlpha = Math.max(0, t < 0.15 ? t / 0.15 : 1 - (t - 0.15) / 0.85); + ctx.translate(p.x + p.vx * age * 0.05, p.y - t * this.size * 0.22); + ctx.rotate(p.spin * age); + SHAPES[p.kind](ctx, p.r * (0.7 + 0.3 * t)); + ctx.restore(); + } + } +} diff --git a/js/imageops.js b/js/imageops.js new file mode 100644 index 0000000000000000000000000000000000000000..78ead27602caab97805dbc678e534c4dc9e80323 --- /dev/null +++ b/js/imageops.js @@ -0,0 +1,202 @@ +// Port of the pixel primitives in app/onboard.py (fit_512, inpaint_region, +// _ink_color, snap_to_ink). Pixel work goes through ImageData typed arrays; +// canvas API is used only for whole-image ops (drawImage scaling), never in +// per-pixel loops. Shared low-level helpers are exported for onboard.js/derive.js. +import { CANVAS } from "./config.js?v=20260729.6"; + +export function newCanvas(w, h) { + const c = document.createElement("canvas"); + c.width = w; + c.height = h; + return c; +} + +export function getData(canvas) { + return canvas.getContext("2d").getImageData(0, 0, canvas.width, canvas.height); +} + +/** Wrap a Uint8ClampedArray RGBA buffer as a canvas. */ +/** [x0,y0,x1,y1] with x1,y1 exclusive (max+1), or null when fully transparent. */ +export function bboxAlpha(imgData) { + const { data, width: w, height: h } = imgData; + let x0 = w, y0 = h, x1 = -1, y1 = -1; + for (let y = 0; y < h; y++) { + for (let x = 0; x < w; x++) { + if (data[(y * w + x) * 4 + 3] > 0) { + if (x < x0) x0 = x; + if (x > x1) x1 = x; + if (y < y0) y0 = y; + if (y > y1) y1 = y; + } + } + } + return x1 < 0 ? null : [x0, y0, x1 + 1, y1 + 1]; +} + +export function canvasFromData(buf, w, h) { + const c = newCanvas(w, h); + c.getContext("2d").putImageData(new ImageData(buf, w, h), 0, 0); + return c; +} + +export function hexToRgb(hex) { + const h = hex.replace(/^#/, ""); + return [parseInt(h.slice(0, 2), 16), parseInt(h.slice(2, 4), 16), parseInt(h.slice(4, 6), 16)]; +} + +function rgbToHex(r, g, b) { + const h = (v) => v.toString(16).padStart(2, "0"); + return `#${h(r)}${h(g)}${h(b)}`; +} + +export const rad = (deg) => (deg * Math.PI) / 180; + +/** Round half-to-even, matching python's built-in round() (Math.round is half-up). */ +export function bankersRound(v) { + const r = Math.round(v); + return Math.abs(v - Math.trunc(v)) === 0.5 && r % 2 !== 0 ? r - 1 : r; +} + +/** Median floored to int โ€” matches python int(statistics.median(...)). */ +export function median(arr) { + const n = arr.length; + if (!n) return 0; + const a = [...arr].sort((p, q) => p - q); + const m = n >> 1; + const v = n % 2 ? a[m] : (a[m - 1] + a[m]) / 2; + return Math.floor(v); +} + +function edgeMedian(canvas) { + const { data, width: w, height: h } = getData(canvas); + const R = [], G = [], B = []; + const push = (x, y) => { const i = (y * w + x) * 4; R.push(data[i]); G.push(data[i + 1]); B.push(data[i + 2]); }; + for (let x = 0; x < w; x++) for (const y of [0, 1, h - 2, h - 1]) push(x, y); + for (let y = 0; y < h; y++) for (const x of [0, 1, w - 2, w - 1]) push(x, y); + return [median(R), median(G), median(B)]; +} + +/** Port of fit_512: fit + center onto a 512 canvas, padded with the source's border-median color. */ +export function fitTo(imgSource, size) { + const sw = imgSource.naturalWidth || imgSource.width; + const sh = imgSource.naturalHeight || imgSource.height; + const src = newCanvas(sw, sh); + src.getContext("2d").drawImage(imgSource, 0, 0); + const [br, bg, bb] = edgeMedian(src); + const s = Math.min(size / sw, size / sh); + const w = Math.floor(sw * s), h = Math.floor(sh * s); + const out = newCanvas(size, size); + const ctx = out.getContext("2d"); + ctx.fillStyle = `rgb(${br},${bg},${bb})`; + ctx.fillRect(0, 0, size, size); + ctx.imageSmoothingEnabled = true; + ctx.imageSmoothingQuality = "high"; + ctx.drawImage(src, 0, 0, sw, sh, Math.floor((size - w) / 2), Math.floor((size - h) / 2), w, h); + return out; +} + +export function fit512(imgSource) { + return fitTo(imgSource, CANVAS); +} + +/** Port of snap_to_ink: centroid of ink pixels (r+g+b < 300) within ยฑr box. */ +export function snapToInk(canvas, cx, cy, r) { + const { data, width: w, height: h } = getData(canvas); + const x0 = Math.max(0, cx - r), y0 = Math.max(0, cy - r); + const x1 = Math.min(w, cx + r), y1 = Math.min(h, cy + r); + let sx = 0, sy = 0, n = 0; + for (let y = y0; y < y1; y++) { + for (let x = x0; x < x1; x++) { + const i = (y * w + x) * 4; + if (data[i] + data[i + 1] + data[i + 2] < 300) { sx += x - x0; sy += y - y0; n++; } + } + } + if (!n) return [cx, cy]; + return [x0 + Math.floor(sx / n), y0 + Math.floor(sy / n)]; +} + +/** Port of inpaint_region: erase EXACTLY the box in-place with the surrounding + * skin tone (median of non-ink ring pixels). Fills only the box โ€” the open-eye + * sprite covers the same box, so neutral stays pixel-perfect. Mutates canvas. */ +export function inpaintRegion(canvas, box, ring = 6) { + const ctx = canvas.getContext("2d"); + const w = canvas.width, h = canvas.height; + const img = ctx.getImageData(0, 0, w, h); + const d = img.data; + const [x0, y0, x1, y1] = box.map((v) => Math.round(v)); + + // skin = median of non-ink ring pixels (median, not mean: robust to the few + // feature-edge pixels โ€” e.g. lip red โ€” that survive the ink cut) + const R = [], G = [], B = [], AR = [], AG = [], AB = []; + for (let y = Math.max(0, y0 - ring); y < Math.min(h, y1 + ring); y++) { + for (let x = Math.max(0, x0 - ring); x < Math.min(w, x1 + ring); x++) { + const i = (y * w + x) * 4; + AR.push(d[i]); AG.push(d[i + 1]); AB.push(d[i + 2]); + if (d[i] + d[i + 1] + d[i + 2] > 300) { R.push(d[i]); G.push(d[i + 1]); B.push(d[i + 2]); } + } + } + const skin = R.length ? [median(R), median(G), median(B)] : [median(AR), median(AG), median(AB)]; + + for (let y = Math.max(0, y0); y < Math.min(h, y1); y++) { + for (let x = Math.max(0, x0); x < Math.min(w, x1); x++) { + const i = (y * w + x) * 4; + d[i] = skin[0]; d[i + 1] = skin[1]; d[i + 2] = skin[2]; + } + } + ctx.putImageData(img, 0, 0); +} + +/** Port of _ink_color: median of dark (r+g+b < 300) pixels in box, else #1a1a1a. */ +export function inkColor(canvas, box) { + const { data, width: w, height: h } = getData(canvas); + const x0 = Math.max(0, box[0]), y0 = Math.max(0, box[1]); + const x1 = Math.min(w, box[2]), y1 = Math.min(h, box[3]); + const R = [], G = [], B = []; + for (let y = y0; y < y1; y++) { + for (let x = x0; x < x1; x++) { + const i = (y * w + x) * 4; + if (data[i] + data[i + 1] + data[i + 2] < 300) { R.push(data[i]); G.push(data[i + 1]); B.push(data[i + 2]); } + } + } + if (!R.length) return "#1a1a1a"; + return rgbToHex(median(R), median(G), median(B)); +} + +// Grow a mouth box until it covers the whole connected ink blob (deep open +// mouths drawn with an interior otherwise get erased only partially and the +// leftover reads as a "second mouth"). Growth is capped so a mouth stroke +// touching the face outline cannot swallow the face. +export function expandBoxToInk(canvas, box, pad = 4, capScale = 2.5, maxIter = 8) { + const { data, width, height } = getData(canvas); + const ink = (x, y) => { + const i = (y * width + x) * 4; + return data[i] + data[i + 1] + data[i + 2] < 300; + }; + let [x0, y0, x1, y1] = box.map(Math.round); + const cx = (x0 + x1) / 2, cy = (y0 + y1) / 2; + const maxW = Math.max(24, (x1 - x0) * capScale), maxH = Math.max(24, (y1 - y0) * capScale); + const bound = [cx - maxW / 2, cy - maxH / 2, cx + maxW / 2, cy + maxH / 2].map(Math.round); + + for (let it = 0; it < maxIter; it++) { + let nx0 = Infinity, ny0 = Infinity, nx1 = -Infinity, ny1 = -Infinity; + const sx = Math.max(0, Math.max(bound[0], x0 - pad)); + const sy = Math.max(0, Math.max(bound[1], y0 - pad)); + const ex = Math.min(width - 1, Math.min(bound[2], x1 + pad)); + const ey = Math.min(height - 1, Math.min(bound[3], y1 + pad)); + for (let y = sy; y <= ey; y++) { + for (let x = sx; x <= ex; x++) { + if (ink(x, y)) { + if (x < nx0) nx0 = x; + if (y < ny0) ny0 = y; + if (x > nx1) nx1 = x; + if (y > ny1) ny1 = y; + } + } + } + if (nx0 === Infinity) break; // no ink at all โ€” keep the clicked box + const grown = [Math.min(nx0, x0), Math.min(ny0, y0), Math.max(nx1, x1), Math.max(ny1, y1)]; + if (grown[0] === x0 && grown[1] === y0 && grown[2] === x1 && grown[3] === y1) break; + [x0, y0, x1, y1] = grown; + } + return [x0 - 2, y0 - 2, x1 + 2, y1 + 2]; +} diff --git a/js/main.js b/js/main.js new file mode 100644 index 0000000000000000000000000000000000000000..f3616973a3a3734a0b452df948d785285246734b --- /dev/null +++ b/js/main.js @@ -0,0 +1,756 @@ +// DrawFace Live web โ€” UI wiring and the render loop (browser twin of app/main.py). +import { CANVAS, CONFIG, SOURCE_MAX } from "./config.js?v=20260729.6"; +import { fit512, fitTo, expandBoxToInk, newCanvas } from "./imageops.js?v=20260729.6"; +import { buildCharacter } from "./onboard.js?v=20260729.6"; +import { deriveAll } from "./derive.js?v=20260729.6"; +import { listCharacters, saveCharacter, deleteCharacter, loadCharacter } from "./store.js?v=20260729.6"; +import { + SMOOTH_KEYS, OneEuro, IdleMotion, TriStateEye, Calibration, + pickMouth, eyeKeyForUserSide, +} from "./pipeline.js?v=20260729.6"; +import { createTracker, createWorkerTracker, detectOnImage } from "./tracker.js?v=20260729.6"; +import { prepareCharacter, composeCharacter, drawScene } from "./compositor.js?v=20260729.6"; +import { buildWarpRig, renderWarp } from "./warp.js?v=20260729.6"; +import { StickerFx } from "./effects.js?v=20260729.6"; +import { RenderPerformance } from "./performance.js?v=20260729.6"; + +const $ = (id) => document.getElementById(id); +const status = (msg) => { $("status").textContent = msg; }; + +// Surface every failure in the status line โ€” the user has no console open. +window.addEventListener("error", (e) => status(`์˜ค๋ฅ˜: ${e.message}`)); +window.addEventListener("unhandledrejection", (e) => status(`์˜ค๋ฅ˜: ${e.reason?.message ?? e.reason}`)); + +const VIZ_BARS = [ + ["eyeL", "eyeBlinkLeft"], ["eyeR", "eyeBlinkRight"], ["jaw", "jawOpen"], + ["smile", "mouthSmileLeft"], ["pucker", "mouthPucker"], +]; +const CLICK_STEPS = ["์™ผ๋ˆˆ ์ค‘์‹ฌ", "์˜ค๋ฅธ๋ˆˆ ์ค‘์‹ฌ", "์ž… ์ขŒ์ƒ๋‹จ", "์ž… ์šฐํ•˜๋‹จ"]; +const PERF_MODE_KEY = "drawface-live:performance-mode"; +const PERF_MODES = new Set(["auto", "full", "economy"]); + +// In warp mode the mesh rolls the face itself, so the canvas keeps only this +// share of the roll (body sways a little, face leads) โ€” full canvas roll on top +// of the mesh roll would rotate the face twice. +const CANVAS_ROLL_SHARE = 0.35; + +// ---------- camera list ---------- +// RealSense-class devices expose several video inputs (RGB/depth/IR) โ€” the +// browser's default pick can be the wrong one, so let the user choose. +async function refreshCameras() { + const sel = $("camSelect"); + const cur = sel.value; + sel.innerHTML = ""; + try { + const devs = (await navigator.mediaDevices.enumerateDevices()).filter((d) => d.kind === "videoinput"); + devs.forEach((d, i) => sel.add(new Option(d.label || `์นด๋ฉ”๋ผ ${i + 1}`, d.deviceId))); + if ([...sel.options].some((o) => o.value === cur)) sel.value = cur; + const rgb = [...sel.options].find((o) => /rgb|color/i.test(o.text)); + if (rgb && !cur) sel.value = rgb.value; + } catch { /* enumerate unavailable โ€” default device will be used */ } +} +navigator.mediaDevices?.addEventListener?.("devicechange", refreshCameras); + +const CAM_ERRORS = { + NotAllowedError: "์นด๋ฉ”๋ผ ๊ถŒํ•œ์ด ์ฐจ๋‹จ๋์Šต๋‹ˆ๋‹ค โ€” ์ฃผ์†Œ์ฐฝ์˜ ์ž๋ฌผ์‡ /์นด๋ฉ”๋ผ ์•„์ด์ฝ˜์—์„œ 'ํ—ˆ์šฉ'์œผ๋กœ ๋ฐ”๊พธ๊ณ  ์ƒˆ๋กœ๊ณ ์นจํ•˜์„ธ์š”", + NotFoundError: "์นด๋ฉ”๋ผ๋ฅผ ์ฐพ์„ ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค โ€” ์—ฐ๊ฒฐ ์ƒํƒœ๋ฅผ ํ™•์ธํ•˜์„ธ์š” (WSL์— attach๋œ ์นด๋ฉ”๋ผ๋Š” Windows์—์„œ ๋ณด์ด์ง€ ์•Š์Šต๋‹ˆ๋‹ค)", + NotReadableError: "๋‹ค๋ฅธ ํ”„๋กœ๊ทธ๋žจ์ด ์นด๋ฉ”๋ผ๋ฅผ ์‚ฌ์šฉ ์ค‘์ž…๋‹ˆ๋‹ค โ€” ํ•ด๋‹น ์•ฑ์„ ๋‹ซ๊ณ  ๋‹ค์‹œ ์‹œ๋„ํ•˜์„ธ์š”", + OverconstrainedError: "์„ ํƒํ•œ ์นด๋ฉ”๋ผ๊ฐ€ ์š”์ฒญ ํ•ด์ƒ๋„๋ฅผ ์ง€์›ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค โ€” ๋‹ค๋ฅธ ์นด๋ฉ”๋ผ๋ฅผ ์„ ํƒํ•ด ๋ณด์„ธ์š”", +}; + +function showTrackerRetry(message) { + status(`${message} โ€” ๋„คํŠธ์›Œํฌ๋ฅผ ํ™•์ธํ•œ ๋’ค ๋‹ค์‹œ ์‹œ๋„ํ•˜์„ธ์š”`); + $("trackerRetryBtn").hidden = false; +} + +function clearTrackerRetry() { + $("trackerRetryBtn").hidden = true; +} + +function restorePerformanceMode() { + try { + const saved = localStorage.getItem(PERF_MODE_KEY); + if (PERF_MODES.has(saved)) $("perfMode").value = saved; + } catch { /* storage may be disabled in private browsing */ } +} + +function savePerformanceMode(mode) { + try { localStorage.setItem(PERF_MODE_KEY, mode); } catch { /* preference remains session-only */ } +} + +// ---------- character list ---------- +function refreshList(selectName) { + const sel = $("charSelect"); + sel.innerHTML = ""; + for (const name of listCharacters()) sel.add(new Option(name, name)); + if (selectName) sel.value = selectName; + $("startBtn").disabled = sel.options.length === 0; +} + +// ---------- onboarding ---------- +const BOX_HANDLE_RADIUS = 12; +const BOX_MIN_SIZE = 12; +const ob = { img: null, points: [], draft: null, previewChar: null, drag: null, landmarks: null, src: null }; + +function obStatus() { + const n = ob.points.length; + $("onboardStatus").textContent = n < 4 + ? `ํด๋ฆญ ${n + 1}/4: ${CLICK_STEPS[n]}` + : "์ขŒํ‘œ ์™„๋ฃŒ โ€” ์ด๋ฆ„์„ ํ™•์ธํ•˜๊ณ  [๋ฏธ๋ฆฌ๋ณด๊ธฐ]"; + $("obGenerate").disabled = n < 4 || !$("charName").value.trim(); +} + +function obRedraw() { + const ctx = $("onboardCanvas").getContext("2d"); + ctx.drawImage(ob.img, 0, 0); + ctx.strokeStyle = "#e33"; + ctx.lineWidth = 2; + ob.points.forEach(([x, y]) => { ctx.beginPath(); ctx.arc(x, y, 4, 0, 7); ctx.stroke(); }); + if (ob.points.length === 4) { + const [, , [x0, y0], [x1, y1]] = ob.points; + const left = Math.min(x0, x1), top = Math.min(y0, y1); + const right = Math.max(x0, x1), bottom = Math.max(y0, y1); + ctx.strokeRect(left, top, right - left, bottom - top); + ctx.fillStyle = "#fff"; + for (const [x, y] of [[left, top], [right, top], [right, bottom], [left, bottom]]) { + ctx.fillRect(x - 4, y - 4, 8, 8); + ctx.strokeRect(x - 4, y - 4, 8, 8); + } + } +} + +function onboardPoint(e) { + const r = e.currentTarget.getBoundingClientRect(); + return { + x: Math.max(0, Math.min(CANVAS, Math.round((e.clientX - r.left) * CANVAS / r.width))), + y: Math.max(0, Math.min(CANVAS, Math.round((e.clientY - r.top) * CANVAS / r.height))), + }; +} + +function mouthBox() { + const [, , [ax, ay], [bx, by]] = ob.points; + return { left: Math.min(ax, bx), top: Math.min(ay, by), right: Math.max(ax, bx), bottom: Math.max(ay, by) }; +} + +function setMouthBox({ left, top, right, bottom }) { + ob.points[2] = [Math.round(left), Math.round(top)]; + ob.points[3] = [Math.round(right), Math.round(bottom)]; +} + +function boxHit(x, y) { + if (ob.points.length !== 4) return null; + const box = mouthBox(); + const corners = [ + ["nw", box.left, box.top], ["ne", box.right, box.top], + ["se", box.right, box.bottom], ["sw", box.left, box.bottom], + ]; + for (const [handle, cx, cy] of corners) { + if (Math.abs(x - cx) <= BOX_HANDLE_RADIUS && Math.abs(y - cy) <= BOX_HANDLE_RADIUS) return handle; + } + return x >= box.left && x <= box.right && y >= box.top && y <= box.bottom ? "move" : null; +} + +function cursorForBoxHit(hit) { + if (hit === "move") return "move"; + if (hit === "nw" || hit === "se") return "nwse-resize"; + if (hit === "ne" || hit === "sw") return "nesw-resize"; + return "crosshair"; +} + +function invalidateOnboardingPreview() { + ob.draft = null; + ob.previewChar = null; + $("onboardReview").hidden = true; +} + +async function openOnboarding(file) { + const bmp = await createImageBitmap(file); + ob.img = fit512(bmp); + // Keep a hi-res copy when the original beats 512 โ€” ์›๋ณธ ๋ณด์กด์šฉ ๋ฉ”ํƒ€๋ฐ์ดํ„ฐ + // (r6 ๋ถ€ํ„ฐ warp rig ๋Š” ์‚ฌ์šฉ ์•ˆ ํ•จ; hi-res ์žฌ์„ค๊ณ„ยท์‹ ๊ฒฝ๋ง ํŠธ๋ž™ ๋Œ€๋น„ ์ €์žฅ๋งŒ). + ob.src = (bmp.width > CANVAS || bmp.height > CANVAS) ? fitTo(bmp, SOURCE_MAX) : null; + ob.points = []; + ob.drag = null; + ob.draft = null; + ob.previewChar = null; + $("onboardReview").hidden = true; + $("charName").value = file.name.replace(/\.[^.]+$/, "").replace(/[^\w-]+/g, "-").toLowerCase(); + $("onboardDlg").showModal(); + obRedraw(); + $("onboardStatus").textContent = "์–ผ๊ตด ์ž๋™ ์ธ์‹ ์‹œ๋„ ์ค‘โ€ฆ"; + // Photo-trained models often miss hand drawings โ€” prefill when it works, + // fall back to manual clicks when it doesn't (see outputs/benchmark.md). + const auto = await detectOnImage(ob.img); + ob.landmarks = auto?.landmarks ?? null; // ๋ฉ”ํƒ€๋ฐ์ดํ„ฐ๋กœ ์ €์žฅ๋งŒ โ€” rig ๋Š” ์‚ฌ์šฉ ์•ˆ ํ•จ(r6) + if (auto && ob.points.length === 0) { + ob.points = [auto.eyes.L, auto.eyes.R, + [auto.mouthBox[0], auto.mouthBox[1]], [auto.mouthBox[2], auto.mouthBox[3]]]; + expandMouthPoints(); // landmark lip box misses deep open-mouth interiors + $("eyeHalf").value = auto.eyeHalf; + obRedraw(); + obStatus(); + $("onboardStatus").textContent = "์–ผ๊ตด ์ž๋™ ์ธ์‹๋จ โ€” ๋ฐ•์Šค ๋ชจ์„œ๋ฆฌ๋กœ ํฌ๊ธฐ ์กฐ์ ˆ, ๋‚ด๋ถ€ ๋“œ๋ž˜๊ทธ๋กœ ์ด๋™ ํ›„ [๋ฏธ๋ฆฌ๋ณด๊ธฐ]"; + } else if (ob.points.length === 0) { + obStatus(); // manual flow from step โ‘  + } +} + +// After the 4th click (or auto-detect), grow the mouth box to cover the whole +// drawn mouth โ€” deep open mouths otherwise leave leftovers ("second mouth"). +function expandMouthPoints() { + const [, , [mx0, my0], [mx1, my1]] = ob.points; + const box = expandBoxToInk(ob.img, + [Math.min(mx0, mx1), Math.min(my0, my1), Math.max(mx0, mx1), Math.max(my0, my1)]); + ob.points[2] = [box[0], box[1]]; + ob.points[3] = [box[2], box[3]]; +} + +$("onboardCanvas").addEventListener("click", (e) => { + if (ob.points.length >= 4) return; + const { x, y } = onboardPoint(e); + ob.points.push([x, y]); + if (ob.points.length === 4) { + expandMouthPoints(); + $("onboardStatus").textContent = "์ž… ์˜์—ญ์„ ์ž‰ํฌ์— ๋งž์ถฐ ์ž๋™ ํ™•์žฅํ–ˆ์Šต๋‹ˆ๋‹ค โ€” ๋ชจ์„œ๋ฆฌ๋กœ ํฌ๊ธฐ ์กฐ์ ˆ, ๋‚ด๋ถ€ ๋“œ๋ž˜๊ทธ๋กœ ์ด๋™ํ•  ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค"; + } + obRedraw(); + if (ob.points.length !== 4) obStatus(); + else $("obGenerate").disabled = !$("charName").value.trim(); +}); + +$("onboardCanvas").addEventListener("pointerdown", (e) => { + const { x, y } = onboardPoint(e); + const handle = boxHit(x, y); + if (!handle) return; + invalidateOnboardingPreview(); + ob.drag = { handle, startX: x, startY: y, box: mouthBox() }; + e.currentTarget.setPointerCapture(e.pointerId); + e.preventDefault(); +}); + +$("onboardCanvas").addEventListener("pointermove", (e) => { + const { x, y } = onboardPoint(e); + const canvas = e.currentTarget; + if (!ob.drag) { + canvas.style.cursor = cursorForBoxHit(boxHit(x, y)); + return; + } + + const { handle, startX, startY, box } = ob.drag; + let { left, top, right, bottom } = box; + if (handle === "move") { + const width = right - left, height = bottom - top; + left = Math.max(0, Math.min(CANVAS - width, left + x - startX)); + top = Math.max(0, Math.min(CANVAS - height, top + y - startY)); + right = left + width; + bottom = top + height; + } else if (handle === "nw") { + left = Math.max(0, Math.min(right - BOX_MIN_SIZE, x)); + top = Math.max(0, Math.min(bottom - BOX_MIN_SIZE, y)); + } else if (handle === "ne") { + right = Math.min(CANVAS, Math.max(left + BOX_MIN_SIZE, x)); + top = Math.max(0, Math.min(bottom - BOX_MIN_SIZE, y)); + } else if (handle === "se") { + right = Math.min(CANVAS, Math.max(left + BOX_MIN_SIZE, x)); + bottom = Math.min(CANVAS, Math.max(top + BOX_MIN_SIZE, y)); + } else if (handle === "sw") { + left = Math.max(0, Math.min(right - BOX_MIN_SIZE, x)); + bottom = Math.min(CANVAS, Math.max(top + BOX_MIN_SIZE, y)); + } + setMouthBox({ left, top, right, bottom }); + obRedraw(); + canvas.style.cursor = cursorForBoxHit(handle); + e.preventDefault(); +}); + +function endBoxDrag(e) { + if (!ob.drag) return; + ob.drag = null; + e.currentTarget.style.cursor = "crosshair"; + $("onboardStatus").textContent = "์ž… ์˜์—ญ์„ ์กฐ์ •ํ–ˆ์Šต๋‹ˆ๋‹ค โ€” [๋ฏธ๋ฆฌ๋ณด๊ธฐ]๋กœ ๊ฒฐ๊ณผ๋ฅผ ํ™•์ธํ•˜์„ธ์š”"; +} + +$("onboardCanvas").addEventListener("pointerup", endBoxDrag); +$("onboardCanvas").addEventListener("pointercancel", endBoxDrag); +$("charName").addEventListener("input", obStatus); +$("obReset").onclick = () => { + ob.points = []; + ob.drag = null; + invalidateOnboardingPreview(); + obRedraw(); + obStatus(); +}; +$("obCancel").onclick = () => $("onboardDlg").close(); +$("obGenerate").onclick = () => { + const name = $("charName").value.trim(); + const [L, R, [mx0, my0], [mx1, my1]] = ob.points; + const mouth = [Math.min(mx0, mx1), Math.min(my0, my1), Math.max(mx0, mx1), Math.max(my0, my1)]; + try { + const { manifest, canvases } = buildCharacter(ob.img, name, { L, R }, + Number($("eyeHalf").value) || 16, mouth); + if (ob.landmarks) manifest.landmarks = ob.landmarks; // ์ €์žฅ๋งŒ โ€” rig ๋Š” ๋ฐ•์Šค ๊ธฐํ•˜ ๊ณ ์ •(r6) + if (ob.src) canvases["source.png"] = ob.src; // ์›๋ณธ ๋ณด์กด์šฉ โ€” rig ๋Š” ์‚ฌ์šฉ ์•ˆ ํ•จ(r6) + deriveAll(canvases, manifest); + ob.draft = { name, manifest, canvases }; + ob.previewChar = prepareCharacter(ob.draft); + $("onboardReview").hidden = false; + renderOnboardingPreview(); + $("onboardStatus").textContent = "ํ‘œ์ •๋ณ„ ๊ฒฐ๊ณผ๋ฅผ ํ™•์ธํ•œ ๋’ค ์ €์žฅํ•˜๊ฑฐ๋‚˜ ์œ„์น˜๋ฅผ ์ˆ˜์ •ํ•˜์„ธ์š”"; + } catch (err) { + $("onboardStatus").textContent = `๋ฏธ๋ฆฌ๋ณด๊ธฐ ์ƒ์„ฑ ์‹คํŒจ: ${err.message}`; + } +}; + +function renderOnboardingPreview() { + if (!ob.previewChar) return; + const state = $("reviewState").value; + const expressions = { + neutral: ["open", "open", "closed"], + blink: ["closed", "closed", "closed"], + smile: ["open", "open", "smile"], + A: ["open", "open", "A"], + }; + const [eyeL, eyeR, mouth] = expressions[state] ?? expressions.neutral; + const ctx = $("reviewCanvas").getContext("2d"); + ctx.clearRect(0, 0, CANVAS, CANVAS); + ctx.drawImage(composeCharacter(ob.previewChar, eyeL, eyeR, mouth), 0, 0); +} + +$("reviewState").onchange = renderOnboardingPreview; +$("obEdit").onclick = () => { + invalidateOnboardingPreview(); + $("onboardStatus").textContent = "๋นจ๊ฐ„ ๋ฐ•์Šค ๋ชจ์„œ๋ฆฌ๋ฅผ ๋“œ๋ž˜๊ทธํ•ด ํฌ๊ธฐ๋ฅผ ์กฐ์ ˆํ•˜๊ฑฐ๋‚˜, ๋‚ด๋ถ€๋ฅผ ๋“œ๋ž˜๊ทธํ•ด ์ด๋™ํ•˜์„ธ์š”"; +}; +$("obSave").onclick = () => { + if (!ob.draft) return; + try { + saveCharacter(ob.draft.name, ob.draft.manifest, ob.draft.canvases); + refreshList(ob.draft.name); + $("onboardDlg").close(); + status(`์บ๋ฆญํ„ฐ '${ob.draft.name}' ์ €์žฅ ์™„๋ฃŒ โ€” ์‹œ์ž‘์„ ๋ˆ„๋ฅด์„ธ์š”`); + } catch (err) { + $("onboardStatus").textContent = `์ €์žฅ ์‹คํŒจ: ${err.message}`; + } +}; + +// ---------- drop zone ---------- +const dz = $("dropZone"); +dz.onclick = () => $("fileInput").click(); +$("fileInput").onchange = (e) => e.target.files[0] && openOnboarding(e.target.files[0]); +dz.ondragover = (e) => { e.preventDefault(); dz.classList.add("drag"); }; +dz.ondragleave = () => dz.classList.remove("drag"); +dz.ondrop = (e) => { + e.preventDefault(); + dz.classList.remove("drag"); + const f = [...e.dataTransfer.files].find((f) => f.type.startsWith("image/")); + if (f) openOnboarding(f); +}; + +// Clean flat avatar โ€” solid colours, eyes/mouth on a solid skin field so the +// erased regions vanish. This is the ideal input shape for the sprite pipeline +// (unlike a textured hand drawing, where the erased patch is hard to hide). +function exampleDrawing() { + const c = newCanvas(CANVAS, CANVAS); + const ctx = c.getContext("2d"); + const SKIN = "#f6c9a0", SKIN_SH = "#e8b488", HAIR = "#4a3328", SHIRT = "#4c8bc4", SHIRT_D = "#3b71a3"; + const ell = (x0, y0, x1, y1, a0 = 0, a1 = Math.PI * 2) => { + ctx.beginPath(); + ctx.ellipse((x0 + x1) / 2, (y0 + y1) / 2, (x1 - x0) / 2, (y1 - y0) / 2, 0, a0, a1); + ctx.fill(); + }; + const rad = (deg) => (deg * Math.PI) / 180; + + ctx.fillStyle = "#f2e9de"; ctx.fillRect(0, 0, CANVAS, CANVAS); // background + ctx.fillStyle = SHIRT; ctx.beginPath(); ctx.roundRect(150, 360, 212, 152, 40); ctx.fill(); + ctx.fillStyle = SHIRT_D; + ctx.beginPath(); ctx.moveTo(180, 360); ctx.lineTo(256, 430); ctx.lineTo(332, 360); ctx.closePath(); ctx.fill(); + ctx.fillStyle = SKIN_SH; ctx.fillRect(228, 320, 56, 52); // neck + ctx.fillStyle = SKIN; ell(150, 96, 362, 350); // head + ell(140, 200, 172, 250); ell(340, 200, 372, 250); // ears + ctx.fillStyle = HAIR; + ell(150, 70, 362, 300, rad(180), rad(360)); // hair top + ctx.fillRect(150, 150, 212, 22); + ell(150, 96, 200, 210); ell(312, 96, 362, 210); // side hair + ctx.beginPath(); ctx.roundRect(196, 205, 48, 9, 4); ctx.fill(); // brows + ctx.beginPath(); ctx.roundRect(268, 205, 48, 9, 4); ctx.fill(); + for (const ex of [220, 292]) { // eyes + ctx.fillStyle = "#2b2420"; ell(ex - 16, 232, ex + 16, 272); + ctx.fillStyle = "#ffffff"; ell(ex - 6, 240, ex + 6, 256); + } + ctx.strokeStyle = SKIN_SH; ctx.lineWidth = 4; // nose + ctx.beginPath(); ctx.moveTo(256, 272); ctx.lineTo(250, 292); ctx.stroke(); + ctx.strokeStyle = "#b04a54"; ctx.lineWidth = 7; // mouth + ctx.beginPath(); ctx.ellipse(256, 311, 32, 19, 0, rad(15), rad(165)); ctx.stroke(); + return c; +} + +// ์†Œ๋…„ ํ”„๋ฆฌ์…‹: talking-drawing-avatar ์˜ ๋ Œ๋” ๋ฐฉ๋ฒ•(๋ฒกํ„ฐ ์ž…ยทWebGL ์›Œํ•‘ยท์‹œ์„ )์„ ๊ทธ๋Œ€๋กœ ์“ฐ๋Š” +// ์ „์šฉ ํŽ˜์ด์ง€๋กœ ์—ฐ๊ฒฐ โ€” drawface ํŒŒ์ดํ”„๋ผ์ธ์— ๋Œ€์ž…ํ•˜๋ฉด ์–ด์ƒ‰ํ•ด์„œ ์›๋ณธ ๋ฐฉ์‹ ๊ทธ๋Œ€๋กœ ๊ตฌ๋™ํ•œ๋‹ค. +$("boyBtn").onclick = () => { window.location.href = "boy.html"; }; + +$("exampleBtn").onclick = () => { + try { + const name = "์˜ˆ์‹œ ์บ๋ฆญํ„ฐ"; + const { manifest, canvases } = buildCharacter(exampleDrawing(), name, + { L: [220, 252], R: [292, 252] }, 20, [222, 290, 290, 332]); + deriveAll(canvases, manifest); + saveCharacter(name, manifest, canvases); + refreshList(name); + status("์˜ˆ์‹œ ์บ๋ฆญํ„ฐ๋ฅผ ๋ถˆ๋Ÿฌ์™”์Šต๋‹ˆ๋‹ค โ€” ์‹œ์ž‘์„ ๋ˆŒ๋Ÿฌ ์›น์บ  ํ‘œ์ •์„ ๋”ฐ๋ผ ํ•ด๋ณด์„ธ์š”"); + } catch (err) { + status(`์˜ˆ์‹œ ์บ๋ฆญํ„ฐ ์ƒ์„ฑ ์‹คํŒจ: ${err.message}`); + } +}; + +$("deleteBtn").onclick = () => { + const name = $("charSelect").value; + if (name && confirm(`'${name}' ์บ๋ฆญํ„ฐ๋ฅผ ์‚ญ์ œํ• ๊นŒ์š”?`)) { deleteCharacter(name); refreshList(); } +}; + +// ---------- live loop ---------- +const run = { + on: false, stream: null, tracker: null, workerTracker: null, trackerLoading: null, + video: null, raf: 0, videoFrame: 0, recording: null, perf: null, +}; + +async function start() { + const name = $("charSelect").value; + if (!name) return; + $("startBtn").disabled = true; + clearTrackerRetry(); + let loadingTracker = true; + try { + status("์ถ”์  ๋ชจ๋ธ ๋กœ๋”ฉ ์ค‘โ€ฆ"); + run.workerTracker = await createWorkerTracker(); // null โ†’ silent sync fallback + if (run.workerTracker) console.log("[tracker] worker mode"); + else run.tracker ??= await createTracker(); + loadingTracker = false; + status("์›น์บ  ์—ฌ๋Š” ์ค‘โ€ฆ"); + const video_c = { width: CONFIG.camera.width, height: CONFIG.camera.height }; + if ($("camSelect").value) video_c.deviceId = { exact: $("camSelect").value }; + try { + run.stream = await navigator.mediaDevices.getUserMedia({ video: video_c }); + } catch (err) { + throw new Error(CAM_ERRORS[err.name] ?? `${err.name}: ${err.message}`); + } + refreshCameras(); // labels become visible after the first grant + const video = document.createElement("video"); + video.muted = true; + video.playsInline = true; + video.srcObject = run.stream; + await video.play(); + run.video = video; + + const char = prepareCharacter(await loadCharacter(name)); + try { + char.warp = buildWarpRig(char); + } catch (err) { + char.warp = null; // sprite path still works โ€” warp is an upgrade, not a gate + console.warn("warp rig unavailable:", err); + } + $("warpChk").disabled = !char.warp; + // rig ์ถœ๋ ฅ ํฌ๊ธฐ์— backing store ๋ฅผ ๋งž์ถค (r6 ๋ถ€ํ„ฐ ํ•ญ์ƒ 512 โ€” CSS ๊ฐ€ ํ‘œ์‹œ ํฌ๊ธฐ ์œ ์ง€). + const outSize = char.warp ? char.warp.out.width : CANVAS; + $("output").width = $("output").height = outSize; + const st = { + mirror: $("mirrorChk").checked, + calib: new Calibration(CONFIG.calibration.frames), + emas: Object.fromEntries(SMOOTH_KEYS.map((k) => [k, + new OneEuro(CONFIG.smoothing.minCutoff, CONFIG.smoothing.beta)])), + headEmas: Object.fromEntries(["yaw", "pitch", "roll"].map((k) => [k, + new OneEuro(CONFIG.smoothing.headMinCutoff, CONFIG.smoothing.headBeta)])), + eyes: { left: new TriStateEye(CONFIG.eyes), right: new TriStateEye(CONFIG.eyes) }, + smoothed: Object.fromEntries(SMOOTH_KEYS.map((k) => [k, 0])), + head: { yaw: 0, pitch: 0, roll: 0 }, + lastSeen: performance.now(), fps: 0, tPrev: performance.now(), workerTs: -1, + perf: new RenderPerformance(CONFIG.performance, $("perfMode").value), + outCtx: $("output").getContext("2d"), // hoisted out of the frame loop + prevCtx: $("preview").getContext("2d"), + fx: new StickerFx(outSize), + idle: new IdleMotion(CONFIG.idle), + }; + run.perf = st.perf; + run.on = true; + $("startBtn").textContent = "์ •์ง€"; + $("startBtn").disabled = false; + $("calibBtn").disabled = false; + $("recordBtn").disabled = !("MediaRecorder" in window && "captureStream" in $("output")); + $("calibBtn").onclick = () => st.calib.restart(); + $("mirrorChk").onchange = () => { st.mirror = $("mirrorChk").checked; }; + window.onkeydown = (e) => { + if (e.key === "c") st.calib.restart(); + if (e.key === "m") { $("mirrorChk").checked = !$("mirrorChk").checked; st.mirror = $("mirrorChk").checked; } + }; + loop(video, char, st); + } catch (err) { + stop(); + if (loadingTracker) showTrackerRetry(`์ถ”์  ๋ชจ๋ธ ๋กœ๋”ฉ ์‹คํŒจ: ${err.message}`); + else status(`์‹œ์ž‘ ์‹คํŒจ: ${err.message}`); + } +} + +function loop(video, char, st) { + const render = (now) => { + if (!run.on) return; + try { + const began = performance.now(); + loopBody(video, char, st, now); + st.perf.record(now, performance.now() - began); + } catch (err) { + status(`๋ฃจํ”„ ์˜ค๋ฅ˜: ${err.message}`); + console.error(err); + stop(); + return; + } + schedule(); + }; + const schedule = () => { + if (!run.on) return; + if (typeof video.requestVideoFrameCallback === "function") { + run.videoFrame = video.requestVideoFrameCallback((now) => render(now)); + } else { + const tick = (now) => { + if (!run.on) return; + if (video.currentTime !== st.lastVideoTime) { + st.lastVideoTime = video.currentTime; + render(now); + } else { + run.raf = requestAnimationFrame(tick); + } + }; + run.raf = requestAnimationFrame(tick); + } + }; + schedule(); +} + +// Worker-mode observation: send the current frame (dropped if the worker is +// busy) and integrate the newest result. Results repeat across render frames +// until the worker posts a fresher one, so each is integrated exactly once, +// keyed by its capture timestamp; smoothing uses that capture ts while +// lost-face decay is keyed on when the last non-null obs ARRIVED. +function workerObserve(video, st, now) { + if (run.workerTracker.failed()) { + // A worker-side MediaPipe fault must not leave the app with a permanently + // busy frame gate. Load the normal tracker lazily and keep rendering neutral + // frames while it initializes. + run.workerTracker.close(); + run.workerTracker = null; + if (!run.tracker && !run.trackerLoading) { + status("์ถ”์  ์›Œ์ปค๋ฅผ ๋ณต๊ตฌํ•˜๋Š” ์ค‘โ€ฆ"); + run.trackerLoading = createTracker() + .then((tracker) => { run.tracker = tracker; }) + .catch((err) => { + if (run.on) { + stop(); + showTrackerRetry(`์ถ”์  ๋ณต๊ตฌ ์‹คํŒจ: ${err.message}`); + } + }) + .finally(() => { run.trackerLoading = null; }); + } + return null; + } + run.workerTracker.sendFrame(video, now); + const res = run.workerTracker.latest(); + const obs = res?.obs ?? null; + const fresh = !!res && res.ts !== st.workerTs; + if (fresh) st.workerTs = res.ts; + + if (fresh && obs) { + st.lastSeen = now; + if (st.calib.active) { + st.calib.feed(obs.blend); + } else { + const values = st.calib.apply(obs.blend); + const tSec = (res.ts ?? now) / 1000; + for (const k of SMOOTH_KEYS) st.smoothed[k] = st.emas[k].update(values[k], tSec); + for (const k of ["yaw", "pitch", "roll"]) st.head[k] = st.headEmas[k].update(obs[k], tSec); + } + } else { + const lost = now - st.lastSeen; + if (lost > CONFIG.lostFace.holdMs) { + const decay = Math.min(1, (lost - CONFIG.lostFace.holdMs) / CONFIG.lostFace.decayMs); + for (const k of SMOOTH_KEYS) st.smoothed[k] *= (1 - decay); + for (const k of ["yaw", "pitch", "roll"]) st.head[k] *= (1 - decay); + } + } + return obs; +} + +function loopBody(video, char, st, now) { + let obs = null; + if (run.workerTracker) { + obs = workerObserve(video, st, now); + } else if (run.tracker) { + obs = run.tracker.detect(video, now); + + if (obs) { + st.lastSeen = now; + if (st.calib.active) { + st.calib.feed(obs.blend); + } else { + const values = st.calib.apply(obs.blend); + const tSec = now / 1000; + for (const k of SMOOTH_KEYS) st.smoothed[k] = st.emas[k].update(values[k], tSec); + for (const k of ["yaw", "pitch", "roll"]) st.head[k] = st.headEmas[k].update(obs[k], tSec); + } + } else { + const lost = now - st.lastSeen; + if (lost > CONFIG.lostFace.holdMs) { + const decay = Math.min(1, (lost - CONFIG.lostFace.holdMs) / CONFIG.lostFace.decayMs); + for (const k of SMOOTH_KEYS) st.smoothed[k] *= (1 - decay); + for (const k of ["yaw", "pitch", "roll"]) st.head[k] *= (1 - decay); + } + } + } + + const eyeStates = {}; + for (const side of ["left", "right"]) { + const key = side === "left" ? "eyeBlinkLeft" : "eyeBlinkRight"; + eyeStates[eyeKeyForUserSide(side, st.mirror)] = st.eyes[side].update(st.smoothed[key]); + } + const mouth = pickMouth(st.smoothed, CONFIG.mouth); + + const economy = st.perf.useEconomy(now); + const useWarp = char.warp && $("warpChk").checked && !economy; + if (useWarp) { + const g = CONFIG.warp; + const blink = {}; + for (const side of ["left", "right"]) { + const key = side === "left" ? "eyeBlinkLeft" : "eyeBlinkRight"; + blink[eyeKeyForUserSide(side, st.mirror)] = st.smoothed[key] * g.blinkGain; + } + const ch = { + blinkL: blink.L, blinkR: blink.R, + smile: ((st.smoothed.mouthSmileLeft + st.smoothed.mouthSmileRight) / 2) * g.smileGain, + jaw: st.smoothed.jawOpen * g.jawGain, + // Mesh parallax reuses the canvas-shift gains for direction/normalization. + yaw: (st.head.yaw * CONFIG.head.yawGainPx / CONFIG.head.maxShiftPx) * g.headParallax, + pitch: (st.head.pitch * CONFIG.head.pitchGainPx / CONFIG.head.maxShiftPx) * g.headParallax, + roll: (st.head.roll * CONFIG.head.rollGain / CONFIG.head.maxRollDeg) * g.headParallax, + }; + if (!st.calib.active) { + st.idle.apply(ch, now, Math.max(st.smoothed.eyeBlinkLeft, st.smoothed.eyeBlinkRight)); + } + const frame = renderWarp(char.warp, ch); + // The mesh already rolled the face โ€” the canvas keeps only a share. + drawScene(st.outCtx, frame, { ...st.head, roll: st.head.roll * CANVAS_ROLL_SHARE }, CONFIG.head); + } else { + drawScene(st.outCtx, composeCharacter(char, eyeStates.L, eyeStates.R, mouth), st.head, CONFIG.head); + } + if ($("fxChk").checked && !economy) { + if (!st.calib.active) st.fx.update(st.smoothed, now); + st.fx.draw(st.outCtx, now); + } + + drawPreview(video, obs, st); + + st.fps = 0.9 * st.fps + 0.1 * (1000 / Math.max(1, now - st.tPrev)); + st.tPrev = now; + // The recording cue must survive this per-frame status overwrite (spec ยง9). + const rec = run.recording ? " โ— REC" : ""; + status((st.calib.active + ? "์บ˜๋ฆฌ๋ธŒ๋ ˆ์ด์…˜ ์ค‘ โ€” ์ •๋ฉด์„ ๋ณด๊ณ  ๋ฌดํ‘œ์ •์„ ์œ ์ง€ํ•˜์„ธ์š”" + : `${st.fps.toFixed(0)} FPS ยท ${st.perf.label()}${run.workerTracker ? " ยท worker" : ""} ยท ${useWarp ? "warp" : "sprite"} ยท ${obs ? "face:OK" : "face:LOST"}` + + (useWarp ? "" : ` ยท L:${eyeStates.L} R:${eyeStates.R} mouth:${mouth}`)) + rec); +} + +function drawPreview(video, obs, st) { + const ctx = st.prevCtx; + const { width: w, height: h } = ctx.canvas; + ctx.save(); + ctx.scale(-1, 1); // mirror ONLY the user-facing preview + ctx.drawImage(video, -w, 0, w, h); + ctx.restore(); + if (!$("vizChk").checked) return; + if (obs?.landmarks) { + ctx.fillStyle = "#50ff78"; + for (const lm of obs.landmarks) ctx.fillRect((1 - lm.x) * w, lm.y * h, 2, 2); + } + ctx.font = "11px monospace"; + VIZ_BARS.forEach(([label, key], i) => { + const y = h - 14 * VIZ_BARS.length - 8 + 14 * i; + ctx.fillStyle = "#fff"; + ctx.fillText(label, 6, y + 9); + ctx.strokeStyle = "#666"; + ctx.strokeRect(58, y, 80, 10); + ctx.fillStyle = "#50ff78"; + ctx.fillRect(58, y, 80 * Math.min(1, st.smoothed[key] ?? 0), 10); + }); +} + +function stop() { + run.on = false; + cancelAnimationFrame(run.raf); + if (run.videoFrame && run.video?.cancelVideoFrameCallback) run.video.cancelVideoFrameCallback(run.videoFrame); + run.videoFrame = 0; + stopRecording(); + run.workerTracker?.close(); + run.workerTracker = null; + run.stream?.getTracks().forEach((t) => t.stop()); + run.stream = null; + run.video = null; + run.perf = null; + window.onkeydown = null; + $("startBtn").textContent = "์‹œ์ž‘"; + $("startBtn").disabled = $("charSelect").options.length === 0; + $("calibBtn").disabled = true; + $("recordBtn").disabled = true; + $("recordBtn").textContent = "๋…นํ™” ์‹œ์ž‘"; + status("์ •์ง€๋จ"); +} + +function recordMimeType() { + return ["video/webm;codecs=vp9", "video/webm;codecs=vp8", "video/webm"] + .find((type) => MediaRecorder.isTypeSupported(type)); +} + +function startRecording() { + if (!run.on || run.recording || !("MediaRecorder" in window) || !("captureStream" in $("output"))) return; + const mimeType = recordMimeType(); + if (!mimeType) { status("์ด ๋ธŒ๋ผ์šฐ์ €๋Š” WebM ๋…นํ™”๋ฅผ ์ง€์›ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค"); return; } + const stream = $("output").captureStream(30); + const recording = { stream, chunks: [], recorder: null }; + const recorder = recording.recorder = new MediaRecorder(stream, { mimeType }); + recorder.ondataavailable = (e) => { if (e.data.size) recording.chunks.push(e.data); }; + recorder.onerror = () => status("๋…นํ™” ์ค‘ ์˜ค๋ฅ˜๊ฐ€ ๋ฐœ์ƒํ–ˆ์Šต๋‹ˆ๋‹ค"); + recorder.onstop = () => { + stream.getTracks().forEach((track) => track.stop()); + if (recording.chunks.length) { + const blob = new Blob(recording.chunks, { type: mimeType }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = `drawface-live-${new Date().toISOString().replace(/[:.]/g, "-")}.webm`; + a.click(); + setTimeout(() => URL.revokeObjectURL(url), 1000); + status("๋…นํ™” ์™„๋ฃŒ โ€” WebM ๋‹ค์šด๋กœ๋“œ๋ฅผ ์‹œ์ž‘ํ–ˆ์Šต๋‹ˆ๋‹ค"); + } + if (run.recording === recording) run.recording = null; + $("recordBtn").textContent = "๋…นํ™” ์‹œ์ž‘"; + }; + run.recording = recording; + recorder.start(1000); + $("recordBtn").textContent = "๋…นํ™” ์ข…๋ฃŒ"; + status("๋…นํ™” ์ค‘ โ€” ๊ฒฐ๊ณผ ์บ”๋ฒ„์Šค๋งŒ WebM์œผ๋กœ ์ €์žฅํ•ฉ๋‹ˆ๋‹ค"); +} + +function stopRecording() { + const recording = run.recording; + if (!recording) return; + if (recording.recorder.state !== "inactive") recording.recorder.stop(); +} + +$("recordBtn").onclick = () => (run.recording ? stopRecording() : startRecording()); + +$("startBtn").onclick = () => (run.on ? stop() : start()); +$("trackerRetryBtn").onclick = () => start(); +$("perfMode").onchange = () => { + const mode = $("perfMode").value; + savePerformanceMode(mode); + run.perf?.setPreference(mode); +}; + +restorePerformanceMode(); +refreshList(); +refreshCameras(); diff --git a/js/onboard.js b/js/onboard.js new file mode 100644 index 0000000000000000000000000000000000000000..1eec442285bd04731d7c2331004763dd341a1a0f --- /dev/null +++ b/js/onboard.js @@ -0,0 +1,73 @@ +// Port of app/onboard.py build_character: turns a fitted 512 drawing + eye/mouth +// coordinates into base + eye sprites and a proceduralMouth manifest. Geometry +// matches the python exactly so a web-onboarded character equals a CLI one. +import { CANVAS, DERIVE } from "./config.js?v=20260729.6"; +import { newCanvas, snapToInk, inpaintRegion, inkColor, rad } from "./imageops.js?v=20260729.6"; + +/** Transparent 512 canvas with the closed-eye arc stroke (PIL arc 20..160, width 4). */ +function drawClosedEye(cx, cy, eyeHalf, colorHex) { + const c = newCanvas(CANVAS, CANVAS); + const ctx = c.getContext("2d"); + ctx.strokeStyle = colorHex; + ctx.lineWidth = 4; + ctx.beginPath(); + // PIL bbox (cx-h, cy-0.6h, cx+h, cy+h) -> center (cx, cy+0.2h), radii (h, 0.8h). + ctx.ellipse(cx, cy + 0.2 * eyeHalf, eyeHalf, 0.8 * eyeHalf, 0, rad(20), rad(160), false); + ctx.stroke(); + return c; +} + +/** + * eyes = {L:[cx,cy], R:[cx,cy]} in 512-canvas px (snapped to ink here). + * mouthBox = [x0,y0,x1,y1]. Returns {manifest, canvases} with canvases keyed by + * sprite file name. + */ +export function buildCharacter(canvas512, name, eyes, eyeHalf, mouthBox) { + const snapped = {}; + for (const side of Object.keys(eyes)) { + const [cx, cy] = eyes[side]; + snapped[side] = snapToInk(canvas512, cx, cy, eyeHalf); + } + + const base = newCanvas(CANVAS, CANVAS); + const bctx = base.getContext("2d"); + bctx.drawImage(canvas512, 0, 0); + + const canvases = {}; + for (const side of Object.keys(snapped)) { + const [cx, cy] = snapped[side]; + const box = [cx - eyeHalf, cy - eyeHalf, cx + eyeHalf, cy + eyeHalf]; + + // open sprite: transparent 512 with the box region copied in place (crop is + // [x0,x1) x [y0,y1), i.e. 2*eyeHalf square). + const open = newCanvas(CANVAS, CANVAS); + const patch = bctx.getImageData(box[0], box[1], box[2] - box[0], box[3] - box[1]); + open.getContext("2d").putImageData(patch, box[0], box[1]); + canvases[`eye_${side}_open.png`] = open; + + canvases[`eye_${side}_closed.png`] = drawClosedEye(cx, cy, eyeHalf, inkColor(base, box)); + + // seamless erase (skin-tone fill + feathered edge) โ€” the open sprite above + // already captured the original pixels, so this only affects the base. + inpaintRegion(base, box); + } + + const line = inkColor(base, mouthBox); + inpaintRegion(base, mouthBox); + canvases["base.png"] = base; + + const mcx = Math.floor((mouthBox[0] + mouthBox[2]) / 2); + const mcy = Math.floor((mouthBox[1] + mouthBox[3]) / 2); + const halfW = Math.max(12, Math.floor((mouthBox[2] - mouthBox[0]) / 2)); + const manifest = { + name, + pupilRange: 0, + browRange: 0, + jawDrop: 6, + mouthCenter: [mcx, mcy], + proceduralMouth: true, + mouthStyle: { ...DERIVE.defaultStyle, line, width: halfW }, + }; + + return { manifest, canvases }; +} diff --git a/js/performance.js b/js/performance.js new file mode 100644 index 0000000000000000000000000000000000000000..a4a73d9b9cb08472cb1c895d48468cca884294d1 --- /dev/null +++ b/js/performance.js @@ -0,0 +1,73 @@ +// Runtime performance monitor for the live canvas. It measures the time spent +// in our own frame work (not time while a tab is backgrounded) and uses +// hysteresis, so one GC pause never makes quality visibly flicker. +export class RenderPerformance { + constructor({ frameBudgetMs, degradeAfterMs, recoverAfterMs }, preference = "auto") { + this.frameBudgetMs = frameBudgetMs; + this.degradeAfterMs = degradeAfterMs; + this.recoverAfterMs = recoverAfterMs; + this.mode = "full"; + this.workMs = 0; + this.slowSince = null; + this.nextProbeAt = null; + this.probing = false; + this.setPreference(preference); + } + + setPreference(preference) { + this.preference = ["auto", "full", "economy"].includes(preference) ? preference : "auto"; + this.slowSince = null; + this.nextProbeAt = null; + this.probing = false; + // Returning to auto starts a fresh high-quality measurement window. Keeping + // an earlier forced economy mode here would leave it with no scheduled probe. + this.mode = this.preference === "economy" ? "economy" : "full"; + } + + // In economy mode, periodically allow exactly one full-quality frame. The + // result tells us whether CPU headroom really returned; simply measuring the + // cheap sprite path would otherwise make a slow device oscillate forever. + useEconomy(now) { + if (this.preference === "economy") return true; + if (this.preference === "full") return false; + if (this.mode !== "economy") return false; + if (this.nextProbeAt !== null && now >= this.nextProbeAt) { + this.probing = true; + return false; + } + return true; + } + + record(now, workMs) { + this.workMs = this.workMs ? this.workMs * 0.85 + workMs * 0.15 : workMs; + if (this.preference !== "auto") return; + // A probe must be judged on its own full-quality frame. The smoothed value + // includes cheap economy frames and would incorrectly promote a slow device. + const slow = (this.probing ? workMs : this.workMs) > this.frameBudgetMs; + if (this.mode === "full") { + this.slowSince = slow ? (this.slowSince ?? now) : null; + if (this.slowSince !== null && now - this.slowSince >= this.degradeAfterMs) { + this.mode = "economy"; + this.slowSince = null; + this.nextProbeAt = now + this.recoverAfterMs; + } + } else if (this.probing) { + this.probing = false; + if (!slow) { + this.mode = "full"; + this.nextProbeAt = null; + } else { + this.nextProbeAt = now + this.recoverAfterMs; + } + } + } + + get economical() { return this.mode === "economy"; } + + label() { + const policy = this.preference === "full" ? "๊ณ ํ’ˆ์งˆ ๊ณ ์ •" + : this.preference === "economy" ? "์ ˆ์ „ ๊ณ ์ •" + : this.economical ? "์ž๋™ ์ ˆ์ „" : "์ž๋™"; + return `${this.workMs.toFixed(0)}ms ยท ${policy}`; + } +} diff --git a/js/pipeline.js b/js/pipeline.js new file mode 100644 index 0000000000000000000000000000000000000000..57c30ff59fa9972f9694ad7cddf1dae2a110d9b8 --- /dev/null +++ b/js/pipeline.js @@ -0,0 +1,200 @@ +// Pure signal-processing pipeline ported from app/main.py + app/sprite_backend.py. +// No DOM, no network โ€” safe to unit-test in a browser or Node. Config keys are +// camelCase to match web/js/config.js (Python used snake_case). + +// The channels the web loop actually renders (eyes, visemes). The python twin +// additionally smooths 13 brow/gaze channels for brow/pupil sprite overlays โ€” +// web-onboarded characters always have browRange/pupilRange 0, so those +// channels are omitted here rather than smoothed for nothing every frame. +export const SMOOTH_KEYS = [ + "eyeBlinkLeft", "eyeBlinkRight", "jawOpen", + "mouthSmileLeft", "mouthSmileRight", "mouthPucker", "mouthFunnel", +]; + +function median(values) { + const s = [...values].sort((a, b) => a - b); + const n = s.length; + const mid = n >> 1; + return n % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2; +} + +/** Speed-adaptive lowpass (Casiez et al. 2012): smooth at rest, near-lagless on + * fast motion โ€” the cutoff rises with the signal's own speed. Twin of + * app/sprite_backend.OneEuro. t in seconds, strictly increasing. */ +export class OneEuro { + constructor(minCutoff, beta, dCutoff = 1.0) { + this.minCutoff = minCutoff; + this.beta = beta; + this.dCutoff = dCutoff; + this.x = null; + this.dx = 0; + this.t = null; + } + + static alpha(cutoff, dt) { + const tau = 1 / (2 * Math.PI * cutoff); + return 1 / (1 + tau / dt); + } + + update(x, t) { + if (this.t === null || t <= this.t) { + this.x = x; + this.dx = 0; + this.t = t; + return x; + } + const dt = t - this.t; + this.t = t; + const aD = OneEuro.alpha(this.dCutoff, dt); + this.dx = aD * ((x - this.x) / dt) + (1 - aD) * this.dx; + const a = OneEuro.alpha(this.minCutoff + this.beta * Math.abs(this.dx), dt); + this.x = a * x + (1 - a) * this.x; + return this.x; + } +} + +/** Keeps the character alive when the face is still or lost: a subtle breathing + * bob on the pitch channel plus an occasional scripted blink when no real blink + * has happened for a while. Twin of app/warp_live.IdleMotion. */ +export class IdleMotion { + constructor(cfg) { + this.cfg = cfg; + this.lastBlink = 0; + this.envStart = -1; + this.nextAt = 0; + } + + apply(ch, nowMs, realBlink) { + const c = this.cfg; + ch.pitch += c.breathAmp * Math.sin((2 * Math.PI * nowMs) / (c.breathPeriodS * 1000)); + if (realBlink > 0.3) { + this.lastBlink = nowMs; + this.envStart = -1; + return ch; + } + if (!this.nextAt) this.nextAt = nowMs + this.interval(); + if (this.envStart < 0 && nowMs >= this.nextAt && nowMs - this.lastBlink >= c.blinkMinS * 1000) { + this.envStart = nowMs; + this.lastBlink = nowMs; + this.nextAt = nowMs + this.interval(); + } + if (this.envStart >= 0) { + const t = (nowMs - this.envStart) / c.blinkMs; + if (t >= 1) { + this.envStart = -1; + } else { + const env = 1 - Math.abs(2 * t - 1); // close-open triangle + ch.blinkL = Math.max(ch.blinkL, env); + ch.blinkR = Math.max(ch.blinkR, env); + } + } + return ch; + } + + interval() { + const c = this.cfg; + return (c.blinkMinS + Math.random() * (c.blinkMaxS - c.blinkMinS)) * 1000; + } +} + +export class Ema { + constructor(alpha) { + this.alpha = alpha; + this.value = null; + } + update(x) { + this.value = this.value === null ? x : this.alpha * x + (1 - this.alpha) * this.value; + return this.value; + } +} + +// Two-threshold open/closed switch to prevent blink flicker (closeTh > openTh). +export class Hysteresis { + constructor(closeTh, openTh) { + if (!(closeTh > openTh)) throw new Error(`closeTh must exceed openTh (${closeTh} <= ${openTh})`); + this.closeTh = closeTh; + this.openTh = openTh; + this.closed = false; + } + update(value) { + if (!this.closed && value >= this.closeTh) this.closed = true; + else if (this.closed && value <= this.openTh) this.closed = false; + return this.closed; + } +} + +// open / half / closed with independent hysteresis bands (no flicker at either edge). +export class TriStateEye { + constructor(eyesCfg) { + this.full = new Hysteresis(eyesCfg.closeThreshold, eyesCfg.openThreshold); + this.half = new Hysteresis(eyesCfg.halfCloseThreshold, eyesCfg.halfOpenThreshold); + } + update(value) { + const fullClosed = this.full.update(value); + const halfClosed = this.half.update(value); + return fullClosed ? "closed" : (halfClosed ? "half" : "open"); + } +} + +// Neutral-pose baseline: calibrated = max(0, raw - neutral) / max(0.2, 1 - neutral). +export class Calibration { + constructor(nFrames) { + this.nFrames = nFrames; + this.samples = []; + this.neutral = {}; + } + get active() { + return this.samples.length < this.nFrames; + } + feed(blend) { + this.samples.push(blend); + if (!this.active) { + for (const k of SMOOTH_KEYS) { + this.neutral[k] = median(this.samples.map((s) => s[k] ?? 0.0)); + } + } + } + apply(blend) { + const out = {}; + for (const k of SMOOTH_KEYS) { + const n = this.neutral[k] ?? 0.0; + out[k] = Math.max(0.0, (blend[k] ?? 0.0) - n) / Math.max(0.2, 1.0 - n); + } + return out; + } + restart() { + this.samples = []; + this.neutral = {}; + } +} + +// Map the USER's anatomical eye ('left'/'right') to a sprite side ('L'/'R'). +// mirror=true (default): mirror-like control โ€” user's left eye drives viewer-left. +export function eyeKeyForUserSide(side, mirror) { + if (side !== "left" && side !== "right") { + throw new Error(`side must be 'left' or 'right', got ${JSON.stringify(side)}`); + } + const same = side === "left" ? "L" : "R"; + const swap = side === "left" ? "R" : "L"; + return mirror ? same : swap; +} + + +// Select a mouth sprite key from calibrated blendshape values. Ladder mirrors +// app/sprite_backend.pick_mouth; cfg keys are camelCase (config.js CONFIG.mouth). +export function pickMouth(blend, mouthCfg) { + const jaw = blend.jawOpen ?? 0.0; + const pucker = blend.mouthPucker ?? 0.0; + const funnel = blend.mouthFunnel ?? 0.0; + const smile = ((blend.mouthSmileLeft ?? 0.0) + (blend.mouthSmileRight ?? 0.0)) / 2; + + if (jaw < mouthCfg.jawClosed) { + if (smile >= mouthCfg.smileThreshold) return "smile"; // resolves to closed overlay if absent + return "closed"; + } + if (pucker >= mouthCfg.puckerThreshold) return jaw >= mouthCfg.jawMid ? "O" : "U"; + if (funnel >= mouthCfg.funnelThreshold) return "O"; + if (jaw >= mouthCfg.jawLarge) return "A"; + if (jaw >= mouthCfg.jawMid) return "E"; + return "I"; +} diff --git a/js/store.js b/js/store.js new file mode 100644 index 0000000000000000000000000000000000000000..50ccdd166c639a5e53cf1a30b83a8667edf8e874 --- /dev/null +++ b/js/store.js @@ -0,0 +1,66 @@ +// Character persistence in localStorage as {name, manifest, images:{file:dataURL}}. +// ponytail: localStorage caps around 5-10MB (~a handful of characters); move to +// IndexedDB if users hit the ceiling. +const KEY = "drawface.characters"; + +function readAll() { + try { return JSON.parse(localStorage.getItem(KEY)) ?? {}; } catch { return {}; } +} + +export function listCharacters() { + return Object.keys(readAll()).sort(); +} + +// Derived sprites (visemes, half-eyes, smile) are deterministic re-derivations +// of the core set โ€” compositor.js rebuilds them at load, so storing them would +// only multiply the localStorage footprint. +// source.png ๋Š” ํ˜„์žฌ ๋Ÿฐํƒ€์ž„ ์†Œ๋น„์ž๊ฐ€ ์—†๋Š” ์›๋ณธ ๋ณด์กด๋ถ„(JPEG) โ€” hi-res ์ธํŽ˜์ธํŠธ +// ์žฌ์„ค๊ณ„ยท์‹ ๊ฒฝ๋ง ํŠธ๋ž™์ด ์“ธ ์˜ˆ์ •์ด๋ผ ์œ ์ง€ํ•œ๋‹ค. +const CORE_FILES = new Set(["base.png", "eye_L_open.png", "eye_R_open.png", + "eye_L_closed.png", "eye_R_closed.png", "mouth_closed.png", "source.png"]); + +export function saveCharacter(name, manifest, canvases) { + const all = readAll(); + const images = {}; + for (const [file, canvas] of Object.entries(canvases)) { + if (manifest.proceduralMouth && !CORE_FILES.has(file)) continue; + // source.png ๋Š” fitTo ๊ฐ€ ๋ถˆํˆฌ๋ช… ๋ฐฐ๊ฒฝ์œผ๋กœ ์ฑ„์šด hi-res warp ํ…์Šค์ฒ˜(์•ŒํŒŒ ์—†์Œ, ์ตœ๋Œ€ 1024px)๋ผ + // PNG ๋กœ ์ €์žฅํ•˜๋ฉด ์ˆ˜ MB โ†’ localStorage quota ์ดˆ๊ณผ๋กœ ์ €์žฅ์ด ํ†ต์งธ๋กœ ์‹คํŒจํ•œ๋‹ค. ์•ŒํŒŒ๊ฐ€ ์—†์œผ๋‹ˆ + // JPEG ๋กœ ์••์ถ•(ํ•ด์ƒ๋„๋Š” ์œ ์ง€). ์•ŒํŒŒ๊ฐ€ ํ•„์š”ํ•œ ๋‚˜๋จธ์ง€ ์Šคํ”„๋ผ์ดํŠธ๋งŒ PNG. + images[file] = file === "source.png" + ? canvas.toDataURL("image/jpeg", 0.85) + : canvas.toDataURL("image/png"); + } + all[name] = { name, manifest, images }; + try { + localStorage.setItem(KEY, JSON.stringify(all)); + } catch (e) { + if (e.name === "QuotaExceededError") + throw new Error("๋ธŒ๋ผ์šฐ์ € ์ €์žฅ ๊ณต๊ฐ„์ด ๊ฐ€๋“ ์ฐผ์–ด์š”. ๊ธฐ์กด ์บ๋ฆญํ„ฐ๋ฅผ ์‚ญ์ œํ•˜๊ณ  ๋‹ค์‹œ ์‹œ๋„ํ•˜์„ธ์š”."); + throw e; + } +} + +export function deleteCharacter(name) { + const all = readAll(); + delete all[name]; + localStorage.setItem(KEY, JSON.stringify(all)); +} + +async function toCanvas(dataURL) { + const img = new Image(); + await new Promise((res, rej) => { img.onload = res; img.onerror = rej; img.src = dataURL; }); + const c = document.createElement("canvas"); + c.width = img.width; c.height = img.height; + c.getContext("2d").drawImage(img, 0, 0); + return c; +} + +/** Load a stored character into runtime form (canvases keyed by sprite file name). */ +export async function loadCharacter(name) { + const rec = readAll()[name]; + if (!rec) throw new Error(`character not found: ${name}`); + const canvases = {}; + for (const [file, url] of Object.entries(rec.images)) canvases[file] = await toCanvas(url); + return { name, manifest: rec.manifest, canvases }; +} diff --git a/js/trackconfig.js b/js/trackconfig.js new file mode 100644 index 0000000000000000000000000000000000000000..e4bbd6dd35aeae7fbabb0eca6f007068b75db8ae --- /dev/null +++ b/js/trackconfig.js @@ -0,0 +1,32 @@ +// Shared between tracker.js (main thread) and trackworker.js (worker) โ€” a +// worker cannot import tracker.js without pulling the whole module, so the +// CDN/model locations and head-pose math live here as the single source of truth. +export const CDN_URL = "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.17"; +export const WASM_BASE = `${CDN_URL}/wasm`; +export const MODEL_URL = + "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/latest/face_landmarker.task"; + +const clamp = (v, lo, hi) => Math.max(lo, Math.min(hi, v)); +const DEG = 180 / Math.PI; + +// Decompose a 4x4 face transform (flat 16) into yaw/pitch/roll degrees, matching +// app/face_tracker.py's atan2/asin on the 3x3 rotation block. +// +// tasks-vision docs are ambiguous on row- vs column-major, so detect the layout +// at runtime: the homogeneous translation column/row is the tell. Row-major stores +// translation at indices 3,7,11 (indices 12,13,14 == 0); column-major stores it at +// 12,13,14 (indices 3,7,11 == 0). The face's camera-space translation (esp. tz) is +// always substantially non-zero, so whichever triple has the larger magnitude marks +// the layout. Getting this wrong would transpose the rotation and flip every angle. +export function eulerFromMatrix(data) { + const colMajorT = Math.abs(data[12]) + Math.abs(data[13]) + Math.abs(data[14]); + const rowMajorT = Math.abs(data[3]) + Math.abs(data[7]) + Math.abs(data[11]); + const colMajor = colMajorT >= rowMajorT; + // r(row, col) of the rotation block, layout-correct either way. + const r = colMajor ? (row, col) => data[col * 4 + row] : (row, col) => data[row * 4 + col]; + return { + yaw: Math.atan2(r(0, 2), r(2, 2)) * DEG, + pitch: Math.asin(clamp(-r(1, 2), -1, 1)) * DEG, + roll: Math.atan2(r(1, 0), r(0, 0)) * DEG, + }; +} diff --git a/js/tracker.js b/js/tracker.js new file mode 100644 index 0000000000000000000000000000000000000000..9bf63def81a924d16dafb41753b73057507eafc8 --- /dev/null +++ b/js/tracker.js @@ -0,0 +1,216 @@ +// MediaPipe Face Landmarker wrapper (browser only) โ€” the web counterpart of +// app/face_tracker.py. Loads @mediapipe/tasks-vision from CDN; detect() returns +// blendshapes + head euler angles + normalized landmarks, or null when no face. +import { CDN_URL, WASM_BASE, MODEL_URL, eulerFromMatrix } from "./trackconfig.js?v=20260729.6"; + +// Load MediaPipe only when tracking or image auto-detection is requested. A +// static CDN import prevents the entire authoring UI from starting when the +// user is offline or the CDN is temporarily unavailable. +let tasksVisionPromise = null; +let tasksVisionAttempt = 0; +async function loadTasksVision() { + if (!tasksVisionPromise) { + // Browsers cache a rejected module import by URL. Give retries a distinct + // URL so the retry button really performs another CDN request instead of + // immediately replaying the earlier offline failure. + const url = tasksVisionAttempt ? `${CDN_URL}?retry=${tasksVisionAttempt}` : CDN_URL; + tasksVisionPromise = import(url).catch((err) => { + tasksVisionPromise = null; // a later retry may succeed after reconnecting + tasksVisionAttempt += 1; + throw new Error(`could not load MediaPipe tasks-vision from CDN (offline?): ${err.message}`); + }); + } + return tasksVisionPromise; +} + +export async function createTracker() { + let vision; + let FaceLandmarker, FilesetResolver; + try { + ({ FaceLandmarker, FilesetResolver } = await loadTasksVision()); + vision = await FilesetResolver.forVisionTasks(WASM_BASE); + } catch (e) { + throw new Error(e.message ?? `could not load MediaPipe tasks-vision wasm from CDN (offline?)`); + } + + const options = { + baseOptions: { modelAssetPath: MODEL_URL, delegate: "GPU" }, + runningMode: "VIDEO", + numFaces: 1, + outputFaceBlendshapes: true, + outputFacialTransformationMatrixes: true, + }; + + let landmarker; + try { + landmarker = await FaceLandmarker.createFromOptions(vision, options); + } catch (gpuErr) { + try { + options.baseOptions.delegate = "CPU"; // GPU delegate unavailable โ€” retry on CPU + landmarker = await FaceLandmarker.createFromOptions(vision, options); + } catch (cpuErr) { + throw new Error(`could not create FaceLandmarker (model download failed?): ${cpuErr.message}`); + } + } + + return { + /** Detect one face in a